Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "classfile/symbolTable.hpp"

 #include "classfile/systemDictionary.hpp"

 #include "code/codeCache.hpp"

 #include "gc_implementation/concurrentMarkSweep/cmsAdaptiveSizePolicy.hpp"

 #include "gc_implementation/concurrentMarkSweep/cmsCollectorPolicy.hpp"

 #include "gc_implementation/concurrentMarkSweep/cmsGCAdaptivePolicyCounters.hpp"

 #include "gc_implementation/concurrentMarkSweep/cmsOopClosures.inline.hpp"

 #include "gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp"

 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.inline.hpp"

 #include "gc_implementation/concurrentMarkSweep/concurrentMarkSweepThread.hpp"

 #include "gc_implementation/concurrentMarkSweep/vmCMSOperations.hpp"

 #include "gc_implementation/parNew/parNewGeneration.hpp"

 #include "gc_implementation/shared/collectorCounters.hpp"

 #include "gc_implementation/shared/isGCActiveMark.hpp"

 #include "gc_interface/collectedHeap.inline.hpp"

 #include "memory/cardTableRS.hpp"

 #include "memory/collectorPolicy.hpp"

 #include "memory/gcLocker.inline.hpp"

 #include "memory/genCollectedHeap.hpp"

 #include "memory/genMarkSweep.hpp"

 #include "memory/genOopClosures.inline.hpp"

 #include "memory/iterator.hpp"

 #include "memory/referencePolicy.hpp"

 #include "memory/resourceArea.hpp"

 #include "oops/oop.inline.hpp"

 #include "prims/jvmtiExport.hpp"

 #include "runtime/globals_extension.hpp"

 #include "runtime/handles.inline.hpp"

 #include "runtime/java.hpp"

 #include "runtime/vmThread.hpp"

 #include "services/memoryService.hpp"

 #include "services/runtimeService.hpp"

 // statics

 CMSCollector* ConcurrentMarkSweepGeneration::_collector = NULL;

 bool          CMSCollector::_full_gc_requested          = false;

 //////////////////////////////////////////////////////////////////

 // In support of CMS/VM thread synchronization

 //////////////////////////////////////////////////////////////////

 // We split use of the CGC_lock into 2 "levels".

 // The low-level locking is of the usual CGC_lock monitor. We introduce

 // a higher level "token" (hereafter "CMS token") built on top of the

 // low level monitor (hereafter "CGC lock").

 // The token-passing protocol gives priority to the VM thread. The

 // CMS-lock doesn't provide any fairness guarantees, but clients

 // should ensure that it is only held for very short, bounded

 // durations.

 //

 // When either of the CMS thread or the VM thread is involved in

 // collection operations during which it does not want the other

 // thread to interfere, it obtains the CMS token.

 //

 // If either thread tries to get the token while the other has

 // it, that thread waits. However, if the VM thread and CMS thread

 // both want the token, then the VM thread gets priority while the

 // CMS thread waits. This ensures, for instance, that the "concurrent"

 // phases of the CMS thread's work do not block out the VM thread

 // for long periods of time as the CMS thread continues to hog

 // the token. (See bug 4616232).

 //

 // The baton-passing functions are, however, controlled by the

 // flags _foregroundGCShouldWait and _foregroundGCIsActive,

 // and here the low-level CMS lock, not the high level token,

 // ensures mutual exclusion.

 //

 // Two important conditions that we have to satisfy:

 // 1. if a thread does a low-level wait on the CMS lock, then it

 //    relinquishes the CMS token if it were holding that token

 //    when it acquired the low-level CMS lock.

 // 2. any low-level notifications on the low-level lock

 //    should only be sent when a thread has relinquished the token.

 //

 // In the absence of either property, we'd have potential deadlock.

 //

 // We protect each of the CMS (concurrent and sequential) phases

 // with the CMS _token_, not the CMS _lock_.

 //

 // The only code protected by CMS lock is the token acquisition code

 // itself, see ConcurrentMarkSweepThread::[de]synchronize(), and the

 // baton-passing code.

 //

 // Unfortunately, i couldn't come up with a good abstraction to factor and

 // hide the naked CGC_lock manipulation in the baton-passing code

 // further below. That's something we should try to do. Also, the proof

 // of correctness of this 2-level locking scheme is far from obvious,

 // and potentially quite slippery. We have an uneasy supsicion, for instance,

 // that there may be a theoretical possibility of delay/starvation in the

 // low-level lock/wait/notify scheme used for the baton-passing because of

 // potential intereference with the priority scheme embodied in the

 // CMS-token-passing protocol. See related comments at a CGC_lock->wait()

 // invocation further below and marked with "XXX 20011219YSR".

 // Indeed, as we note elsewhere, this may become yet more slippery

 // in the presence of multiple CMS and/or multiple VM threads. XXX

 class CMSTokenSync: public StackObj {

  private:

   bool _is_cms_thread;

  public:

   CMSTokenSync(bool is_cms_thread):

     _is_cms_thread(is_cms_thread) {

     assert(is_cms_thread == Thread::current()->is_ConcurrentGC_thread(),

            "Incorrect argument to constructor");

     ConcurrentMarkSweepThread::synchronize(_is_cms_thread);

   }

   ~CMSTokenSync() {

     assert(_is_cms_thread ?

              ConcurrentMarkSweepThread::cms_thread_has_cms_token() :

              ConcurrentMarkSweepThread::vm_thread_has_cms_token(),

           "Incorrect state");

     ConcurrentMarkSweepThread::desynchronize(_is_cms_thread);

   }

 };

 // Convenience class that does a CMSTokenSync, and then acquires

 // upto three locks.

 class CMSTokenSyncWithLocks: public CMSTokenSync {

  private:

   // Note: locks are acquired in textual declaration order

   // and released in the opposite order

   MutexLockerEx _locker1, _locker2, _locker3;

  public:

   CMSTokenSyncWithLocks(bool is_cms_thread, Mutex* mutex1,

                         Mutex* mutex2 = NULL, Mutex* mutex3 = NULL):

     CMSTokenSync(is_cms_thread),

     _locker1(mutex1, Mutex::_no_safepoint_check_flag),

     _locker2(mutex2, Mutex::_no_safepoint_check_flag),

     _locker3(mutex3, Mutex::_no_safepoint_check_flag)

   { }

 };

 // Wrapper class to temporarily disable icms during a foreground cms collection.

 class ICMSDisabler: public StackObj {

  public:

   // The ctor disables icms and wakes up the thread so it notices the change;

   // the dtor re-enables icms.  Note that the CMSCollector methods will check

   // CMSIncrementalMode.

   ICMSDisabler()  { CMSCollector::disable_icms(); CMSCollector::start_icms(); }

   ~ICMSDisabler() { CMSCollector::enable_icms(); }

 };

 //////////////////////////////////////////////////////////////////

 //  Concurrent Mark-Sweep Generation /////////////////////////////

 //////////////////////////////////////////////////////////////////

 NOT_PRODUCT(CompactibleFreeListSpace* debug_cms_space;)

 // This struct contains per-thread things necessary to support parallel

 // young-gen collection.

 class CMSParGCThreadState: public CHeapObj {

  public:

   CFLS_LAB lab;

   PromotionInfo promo;

   // Constructor.

   CMSParGCThreadState(CompactibleFreeListSpace* cfls) : lab(cfls) {

     promo.setSpace(cfls);

   }

 };

 ConcurrentMarkSweepGeneration::ConcurrentMarkSweepGeneration(

      ReservedSpace rs, size_t initial_byte_size, int level,

      CardTableRS* ct, bool use_adaptive_freelists,

      FreeBlockDictionary::DictionaryChoice dictionaryChoice) :

   CardGeneration(rs, initial_byte_size, level, ct),

   _dilatation_factor(((double)MinChunkSize)/((double)(CollectedHeap::min_fill_size()))),

   _debug_collection_type(Concurrent_collection_type)

 {

   HeapWord* bottom = (HeapWord*) _virtual_space.low();

   HeapWord* end    = (HeapWord*) _virtual_space.high();

   _direct_allocated_words = 0;

   NOT_PRODUCT(

     _numObjectsPromoted = 0;

     _numWordsPromoted = 0;

     _numObjectsAllocated = 0;

     _numWordsAllocated = 0;

   )

   _cmsSpace = new CompactibleFreeListSpace(_bts, MemRegion(bottom, end),

                                            use_adaptive_freelists,

                                            dictionaryChoice);

   NOT_PRODUCT(debug_cms_space = _cmsSpace;)

   if (_cmsSpace == NULL) {

     vm_exit_during_initialization(

       "CompactibleFreeListSpace allocation failure");

   }

   _cmsSpace->_gen = this;

   _gc_stats = new CMSGCStats();

   // Verify the assumption that FreeChunk::_prev and OopDesc::_klass

   // offsets match. The ability to tell free chunks from objects

   // depends on this property.

   debug_only(

     FreeChunk* junk = NULL;

     assert(UseCompressedOops ||

            junk->prev_addr() == (void*)(oop(junk)->klass_addr()),

            "Offset of FreeChunk::_prev within FreeChunk must match"

            "  that of OopDesc::_klass within OopDesc");

   )

   if (CollectedHeap::use_parallel_gc_threads()) {

     typedef CMSParGCThreadState* CMSParGCThreadStatePtr;

     _par_gc_thread_states =

       NEW_C_HEAP_ARRAY(CMSParGCThreadStatePtr, ParallelGCThreads);

     if (_par_gc_thread_states == NULL) {

       vm_exit_during_initialization("Could not allocate par gc structs");

     }

     for (uint i = 0; i < ParallelGCThreads; i++) {

       _par_gc_thread_states[i] = new CMSParGCThreadState(cmsSpace());

       if (_par_gc_thread_states[i] == NULL) {

         vm_exit_during_initialization("Could not allocate par gc structs");

       }

     }

   } else {

     _par_gc_thread_states = NULL;

   }

   _incremental_collection_failed = false;

   // The "dilatation_factor" is the expansion that can occur on

   // account of the fact that the minimum object size in the CMS

   // generation may be larger than that in, say, a contiguous young

   //  generation.

   // Ideally, in the calculation below, we'd compute the dilatation

   // factor as: MinChunkSize/(promoting_gen's min object size)

   // Since we do not have such a general query interface for the

   // promoting generation, we'll instead just use the mimimum

   // object size (which today is a header's worth of space);

   // note that all arithmetic is in units of HeapWords.

   assert(MinChunkSize >= CollectedHeap::min_fill_size(), "just checking");

   assert(_dilatation_factor >= 1.0, "from previous assert");

 }

 // The field "_initiating_occupancy" represents the occupancy percentage

 // at which we trigger a new collection cycle.  Unless explicitly specified

 // via CMSInitiating[Perm]OccupancyFraction (argument "io" below), it

 // is calculated by:

 //

 //   Let "f" be MinHeapFreeRatio in

 //

 //    _intiating_occupancy = 100-f +

 //                           f * (CMSTrigger[Perm]Ratio/100)

 //   where CMSTrigger[Perm]Ratio is the argument "tr" below.

 //

 // That is, if we assume the heap is at its desired maximum occupancy at the

 // end of a collection, we let CMSTrigger[Perm]Ratio of the (purported) free

 // space be allocated before initiating a new collection cycle.

 //

 void ConcurrentMarkSweepGeneration::init_initiating_occupancy(intx io, intx tr) {

   assert(io <= 100 && tr >= 0 && tr <= 100, "Check the arguments");

   if (io >= 0) {

     _initiating_occupancy = (double)io / 100.0;

   } else {

     _initiating_occupancy = ((100 - MinHeapFreeRatio) +

                              (double)(tr * MinHeapFreeRatio) / 100.0)

                             / 100.0;

   }

 }

 void ConcurrentMarkSweepGeneration::ref_processor_init() {

   assert(collector() != NULL, "no collector");

   collector()->ref_processor_init();

 }

 void CMSCollector::ref_processor_init() {

   if (_ref_processor == NULL) {

     // Allocate and initialize a reference processor

     _ref_processor = ReferenceProcessor::create_ref_processor(

         _span,                               // span

         _cmsGen->refs_discovery_is_atomic(), // atomic_discovery

         _cmsGen->refs_discovery_is_mt(),     // mt_discovery

         &_is_alive_closure,

         ParallelGCThreads,

         ParallelRefProcEnabled);

     // Initialize the _ref_processor field of CMSGen

     _cmsGen->set_ref_processor(_ref_processor);

     // Allocate a dummy ref processor for perm gen.

     ReferenceProcessor* rp2 = new ReferenceProcessor();

     if (rp2 == NULL) {

       vm_exit_during_initialization("Could not allocate ReferenceProcessor object");

     }

     _permGen->set_ref_processor(rp2);

   }

 }

 CMSAdaptiveSizePolicy* CMSCollector::size_policy() {

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   assert(gch->kind() == CollectedHeap::GenCollectedHeap,

     "Wrong type of heap");

   CMSAdaptiveSizePolicy* sp = (CMSAdaptiveSizePolicy*)

     gch->gen_policy()->size_policy();

   assert(sp->is_gc_cms_adaptive_size_policy(),

     "Wrong type of size policy");

   return sp;

 }

 CMSGCAdaptivePolicyCounters* CMSCollector::gc_adaptive_policy_counters() {

   CMSGCAdaptivePolicyCounters* results =

     (CMSGCAdaptivePolicyCounters*) collector_policy()->counters();

   assert(

     results->kind() == GCPolicyCounters::CMSGCAdaptivePolicyCountersKind,

     "Wrong gc policy counter kind");

   return results;

 }

 void ConcurrentMarkSweepGeneration::initialize_performance_counters() {

   const char* gen_name = "old";

   // Generation Counters - generation 1, 1 subspace

   _gen_counters = new GenerationCounters(gen_name, 1, 1, &_virtual_space);

   _space_counters = new GSpaceCounters(gen_name, 0,

                                        _virtual_space.reserved_size(),

                                        this, _gen_counters);

 }

 CMSStats::CMSStats(ConcurrentMarkSweepGeneration* cms_gen, unsigned int alpha):

   _cms_gen(cms_gen)

 {

   assert(alpha <= 100, "bad value");

   _saved_alpha = alpha;

   // Initialize the alphas to the bootstrap value of 100.

   _gc0_alpha = _cms_alpha = 100;

   _cms_begin_time.update();

   _cms_end_time.update();

   _gc0_duration = 0.0;

   _gc0_period = 0.0;

   _gc0_promoted = 0;

   _cms_duration = 0.0;

   _cms_period = 0.0;

   _cms_allocated = 0;

   _cms_used_at_gc0_begin = 0;

   _cms_used_at_gc0_end = 0;

   _allow_duty_cycle_reduction = false;

   _valid_bits = 0;

   _icms_duty_cycle = CMSIncrementalDutyCycle;

 }

 double CMSStats::cms_free_adjustment_factor(size_t free) const {

   // TBD: CR 6909490

   return 1.0;

 }

 void CMSStats::adjust_cms_free_adjustment_factor(bool fail, size_t free) {

 }

 // If promotion failure handling is on use

 // the padded average size of the promotion for each

 // young generation collection.

 double CMSStats::time_until_cms_gen_full() const {

   size_t cms_free = _cms_gen->cmsSpace()->free();

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   size_t expected_promotion = MIN2(gch->get_gen(0)->capacity(),

                                    (size_t) _cms_gen->gc_stats()->avg_promoted()->padded_average());

   if (cms_free > expected_promotion) {

     // Start a cms collection if there isn't enough space to promote

     // for the next minor collection.  Use the padded average as

     // a safety factor.

     cms_free -= expected_promotion;

     // Adjust by the safety factor.

     double cms_free_dbl = (double)cms_free;

     double cms_adjustment = (100.0 - CMSIncrementalSafetyFactor)/100.0;

     // Apply a further correction factor which tries to adjust

     // for recent occurance of concurrent mode failures.

     cms_adjustment = cms_adjustment * cms_free_adjustment_factor(cms_free);

     cms_free_dbl = cms_free_dbl * cms_adjustment;

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print_cr("CMSStats::time_until_cms_gen_full: cms_free "

         SIZE_FORMAT " expected_promotion " SIZE_FORMAT,

         cms_free, expected_promotion);

       gclog_or_tty->print_cr("  cms_free_dbl %f cms_consumption_rate %f",

         cms_free_dbl, cms_consumption_rate() + 1.0);

     }

     // Add 1 in case the consumption rate goes to zero.

     return cms_free_dbl / (cms_consumption_rate() + 1.0);

   }

   return 0.0;

 }

 // Compare the duration of the cms collection to the

 // time remaining before the cms generation is empty.

 // Note that the time from the start of the cms collection

 // to the start of the cms sweep (less than the total

 // duration of the cms collection) can be used.  This

 // has been tried and some applications experienced

 // promotion failures early in execution.  This was

 // possibly because the averages were not accurate

 // enough at the beginning.

 double CMSStats::time_until_cms_start() const {

   // We add "gc0_period" to the "work" calculation

   // below because this query is done (mostly) at the

   // end of a scavenge, so we need to conservatively

   // account for that much possible delay

   // in the query so as to avoid concurrent mode failures

   // due to starting the collection just a wee bit too

   // late.

   double work = cms_duration() + gc0_period();

   double deadline = time_until_cms_gen_full();

   // If a concurrent mode failure occurred recently, we want to be

   // more conservative and halve our expected time_until_cms_gen_full()

   if (work > deadline) {

     if (Verbose && PrintGCDetails) {

       gclog_or_tty->print(

         " CMSCollector: collect because of anticipated promotion "

         "before full %3.7f + %3.7f > %3.7f ", cms_duration(),

         gc0_period(), time_until_cms_gen_full());

     }

     return 0.0;

   }

   return work - deadline;

 }

 // Return a duty cycle based on old_duty_cycle and new_duty_cycle, limiting the

 // amount of change to prevent wild oscillation.

 unsigned int CMSStats::icms_damped_duty_cycle(unsigned int old_duty_cycle,

                                               unsigned int new_duty_cycle) {

   assert(old_duty_cycle <= 100, "bad input value");

   assert(new_duty_cycle <= 100, "bad input value");

   // Note:  use subtraction with caution since it may underflow (values are

   // unsigned).  Addition is safe since we're in the range 0-100.

   unsigned int damped_duty_cycle = new_duty_cycle;

   if (new_duty_cycle < old_duty_cycle) {

     const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 5U);

     if (new_duty_cycle + largest_delta < old_duty_cycle) {

       damped_duty_cycle = old_duty_cycle - largest_delta;

     }

   } else if (new_duty_cycle > old_duty_cycle) {

     const unsigned int largest_delta = MAX2(old_duty_cycle / 4, 15U);

     if (new_duty_cycle > old_duty_cycle + largest_delta) {

       damped_duty_cycle = MIN2(old_duty_cycle + largest_delta, 100U);

     }

   }

   assert(damped_duty_cycle <= 100, "invalid duty cycle computed");

   if (CMSTraceIncrementalPacing) {

     gclog_or_tty->print(" [icms_damped_duty_cycle(%d,%d) = %d] ",

                            old_duty_cycle, new_duty_cycle, damped_duty_cycle);

   }

   return damped_duty_cycle;

 }

 unsigned int CMSStats::icms_update_duty_cycle_impl() {

   assert(CMSIncrementalPacing && valid(),

          "should be handled in icms_update_duty_cycle()");

   double cms_time_so_far = cms_timer().seconds();

   double scaled_duration = cms_duration_per_mb() * _cms_used_at_gc0_end / M;

   double scaled_duration_remaining = fabsd(scaled_duration - cms_time_so_far);

   // Avoid division by 0.

   double time_until_full = MAX2(time_until_cms_gen_full(), 0.01);

   double duty_cycle_dbl = 100.0 * scaled_duration_remaining / time_until_full;

   unsigned int new_duty_cycle = MIN2((unsigned int)duty_cycle_dbl, 100U);

   if (new_duty_cycle > _icms_duty_cycle) {

     // Avoid very small duty cycles (1 or 2); 0 is allowed.

     if (new_duty_cycle > 2) {

       _icms_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle,

                                                 new_duty_cycle);

     }

   } else if (_allow_duty_cycle_reduction) {

     // The duty cycle is reduced only once per cms cycle (see record_cms_end()).

     new_duty_cycle = icms_damped_duty_cycle(_icms_duty_cycle, new_duty_cycle);

     // Respect the minimum duty cycle.

     unsigned int min_duty_cycle = (unsigned int)CMSIncrementalDutyCycleMin;

     _icms_duty_cycle = MAX2(new_duty_cycle, min_duty_cycle);

   }

   if (PrintGCDetails || CMSTraceIncrementalPacing) {

     gclog_or_tty->print(" icms_dc=%d ", _icms_duty_cycle);

   }

   _allow_duty_cycle_reduction = false;

   return _icms_duty_cycle;

 }

 #ifndef PRODUCT

 void CMSStats::print_on(outputStream *st) const {

   st->print(" gc0_alpha=%d,cms_alpha=%d", _gc0_alpha, _cms_alpha);

   st->print(",gc0_dur=%g,gc0_per=%g,gc0_promo=" SIZE_FORMAT,

                gc0_duration(), gc0_period(), gc0_promoted());

   st->print(",cms_dur=%g,cms_dur_per_mb=%g,cms_per=%g,cms_alloc=" SIZE_FORMAT,

             cms_duration(), cms_duration_per_mb(),

             cms_period(), cms_allocated());

   st->print(",cms_since_beg=%g,cms_since_end=%g",

             cms_time_since_begin(), cms_time_since_end());

   st->print(",cms_used_beg=" SIZE_FORMAT ",cms_used_end=" SIZE_FORMAT,

             _cms_used_at_gc0_begin, _cms_used_at_gc0_end);

   if (CMSIncrementalMode) {

     st->print(",dc=%d", icms_duty_cycle());

   }

   if (valid()) {

     st->print(",promo_rate=%g,cms_alloc_rate=%g",

               promotion_rate(), cms_allocation_rate());

     st->print(",cms_consumption_rate=%g,time_until_full=%g",

               cms_consumption_rate(), time_until_cms_gen_full());

   }

   st->print(" ");

 }

 #endif // #ifndef PRODUCT

 CMSCollector::CollectorState CMSCollector::_collectorState =

                              CMSCollector::Idling;

 bool CMSCollector::_foregroundGCIsActive = false;

 bool CMSCollector::_foregroundGCShouldWait = false;

 CMSCollector::CMSCollector(ConcurrentMarkSweepGeneration* cmsGen,

                            ConcurrentMarkSweepGeneration* permGen,

                            CardTableRS*                   ct,

                            ConcurrentMarkSweepPolicy*     cp):

   _cmsGen(cmsGen),

   _permGen(permGen),

   _ct(ct),

   _ref_processor(NULL),    // will be set later

   _conc_workers(NULL),     // may be set later

   _abort_preclean(false),

   _start_sampling(false),

   _between_prologue_and_epilogue(false),

   _markBitMap(0, Mutex::leaf + 1, "CMS_markBitMap_lock"),

   _perm_gen_verify_bit_map(0, -1 /* no mutex */, "No_lock"),

   _modUnionTable((CardTableModRefBS::card_shift - LogHeapWordSize),

                  -1 /* lock-free */, "No_lock" /* dummy */),

   _modUnionClosure(&_modUnionTable),

   _modUnionClosurePar(&_modUnionTable),

   // Adjust my span to cover old (cms) gen and perm gen

   _span(cmsGen->reserved()._union(permGen->reserved())),

   // Construct the is_alive_closure with _span & markBitMap

   _is_alive_closure(_span, &_markBitMap),

   _restart_addr(NULL),

   _overflow_list(NULL),

   _stats(cmsGen),

   _eden_chunk_array(NULL),     // may be set in ctor body

   _eden_chunk_capacity(0),     // -- ditto --

   _eden_chunk_index(0),        // -- ditto --

   _survivor_plab_array(NULL),  // -- ditto --

   _survivor_chunk_array(NULL), // -- ditto --

   _survivor_chunk_capacity(0), // -- ditto --

   _survivor_chunk_index(0),    // -- ditto --

   _ser_pmc_preclean_ovflw(0),

   _ser_kac_preclean_ovflw(0),

   _ser_pmc_remark_ovflw(0),

   _par_pmc_remark_ovflw(0),

   _ser_kac_ovflw(0),

   _par_kac_ovflw(0),

 #ifndef PRODUCT

   _num_par_pushes(0),

 #endif

   _collection_count_start(0),

   _verifying(false),

   _icms_start_limit(NULL),

   _icms_stop_limit(NULL),

   _verification_mark_bm(0, Mutex::leaf + 1, "CMS_verification_mark_bm_lock"),

   _completed_initialization(false),

   _collector_policy(cp),

   _should_unload_classes(false),

   _concurrent_cycles_since_last_unload(0),

   _roots_scanning_options(0),

   _inter_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding),

   _intra_sweep_estimate(CMS_SweepWeight, CMS_SweepPadding)

 {

   if (ExplicitGCInvokesConcurrentAndUnloadsClasses) {

     ExplicitGCInvokesConcurrent = true;

   }

   // Now expand the span and allocate the collection support structures

   // (MUT, marking bit map etc.) to cover both generations subject to

   // collection.

   // First check that _permGen is adjacent to _cmsGen and above it.

   assert(   _cmsGen->reserved().word_size()  > 0

          && _permGen->reserved().word_size() > 0,

          "generations should not be of zero size");

   assert(_cmsGen->reserved().intersection(_permGen->reserved()).is_empty(),

          "_cmsGen and _permGen should not overlap");

   assert(_cmsGen->reserved().end() == _permGen->reserved().start(),

          "_cmsGen->end() different from _permGen->start()");

   // For use by dirty card to oop closures.

   _cmsGen->cmsSpace()->set_collector(this);

   _permGen->cmsSpace()->set_collector(this);

   // Allocate MUT and marking bit map

   {

     MutexLockerEx x(_markBitMap.lock(), Mutex::_no_safepoint_check_flag);

     if (!_markBitMap.allocate(_span)) {

       warning("Failed to allocate CMS Bit Map");

       return;

     }

     assert(_markBitMap.covers(_span), "_markBitMap inconsistency?");

   }

   {

     _modUnionTable.allocate(_span);

     assert(_modUnionTable.covers(_span), "_modUnionTable inconsistency?");

   }

   if (!_markStack.allocate(MarkStackSize)) {

     warning("Failed to allocate CMS Marking Stack");

     return;

   }

   if (!_revisitStack.allocate(CMSRevisitStackSize)) {

     warning("Failed to allocate CMS Revisit Stack");

     return;

   }

   // Support for multi-threaded concurrent phases

   if (CollectedHeap::use_parallel_gc_threads() && CMSConcurrentMTEnabled) {

     if (FLAG_IS_DEFAULT(ConcGCThreads)) {

       // just for now

       FLAG_SET_DEFAULT(ConcGCThreads, (ParallelGCThreads + 3)/4);

     }

     if (ConcGCThreads > 1) {

       _conc_workers = new YieldingFlexibleWorkGang("Parallel CMS Threads",

                                  ConcGCThreads, true);

       if (_conc_workers == NULL) {

         warning("GC/CMS: _conc_workers allocation failure: "

               "forcing -CMSConcurrentMTEnabled");

         CMSConcurrentMTEnabled = false;

       } else {

         _conc_workers->initialize_workers();

       }

     } else {

       CMSConcurrentMTEnabled = false;

     }

   }

   if (!CMSConcurrentMTEnabled) {

     ConcGCThreads = 0;

   } else {

     // Turn off CMSCleanOnEnter optimization temporarily for

     // the MT case where it's not fixed yet; see 6178663.

     CMSCleanOnEnter = false;

   }

   assert((_conc_workers != NULL) == (ConcGCThreads > 1),

          "Inconsistency");

   // Parallel task queues; these are shared for the

   // concurrent and stop-world phases of CMS, but

   // are not shared with parallel scavenge (ParNew).

   {

     uint i;

     uint num_queues = (uint) MAX2(ParallelGCThreads, ConcGCThreads);

     if ((CMSParallelRemarkEnabled || CMSConcurrentMTEnabled

          || ParallelRefProcEnabled)

         && num_queues > 0) {

       _task_queues = new OopTaskQueueSet(num_queues);

       if (_task_queues == NULL) {

         warning("task_queues allocation failure.");

         return;

       }

       _hash_seed = NEW_C_HEAP_ARRAY(int, num_queues);

       if (_hash_seed == NULL) {

         warning("_hash_seed array allocation failure");

         return;

       }

       typedef Padded<OopTaskQueue> PaddedOopTaskQueue;

       for (i = 0; i < num_queues; i++) {

         PaddedOopTaskQueue *q = new PaddedOopTaskQueue();

         if (q == NULL) {

           warning("work_queue allocation failure.");

           return;

         }

         _task_queues->register_queue(i, q);

       }

       for (i = 0; i < num_queues; i++) {

         _task_queues->queue(i)->initialize();

         _hash_seed[i] = 17;  // copied from ParNew

       }

     }

   }

   _cmsGen ->init_initiating_occupancy(CMSInitiatingOccupancyFraction, CMSTriggerRatio);

   _permGen->init_initiating_occupancy(CMSInitiatingPermOccupancyFraction, CMSTriggerPermRatio);

   // Clip CMSBootstrapOccupancy between 0 and 100.

   _bootstrap_occupancy = ((double)MIN2((uintx)100, MAX2((uintx)0, CMSBootstrapOccupancy)))

                          /(double)100;

   _full_gcs_since_conc_gc = 0;

   // Now tell CMS generations the identity of their collector

   ConcurrentMarkSweepGeneration::set_collector(this);

   // Create & start a CMS thread for this CMS collector

   _cmsThread = ConcurrentMarkSweepThread::start(this);

   assert(cmsThread() != NULL, "CMS Thread should have been created");

   assert(cmsThread()->collector() == this,

          "CMS Thread should refer to this gen");

   assert(CGC_lock != NULL, "Where's the CGC_lock?");

   // Support for parallelizing young gen rescan

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   _young_gen = gch->prev_gen(_cmsGen);

   if (gch->supports_inline_contig_alloc()) {

     _top_addr = gch->top_addr();

     _end_addr = gch->end_addr();

     assert(_young_gen != NULL, "no _young_gen");

     _eden_chunk_index = 0;

     _eden_chunk_capacity = (_young_gen->max_capacity()+CMSSamplingGrain)/CMSSamplingGrain;

     _eden_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, _eden_chunk_capacity);

     if (_eden_chunk_array == NULL) {

       _eden_chunk_capacity = 0;

       warning("GC/CMS: _eden_chunk_array allocation failure");

     }

   }

   assert(_eden_chunk_array != NULL || _eden_chunk_capacity == 0, "Error");

   // Support for parallelizing survivor space rescan

   if (CMSParallelRemarkEnabled && CMSParallelSurvivorRemarkEnabled) {

     const size_t max_plab_samples =

       ((DefNewGeneration*)_young_gen)->max_survivor_size()/MinTLABSize;

     _survivor_plab_array  = NEW_C_HEAP_ARRAY(ChunkArray, ParallelGCThreads);

     _survivor_chunk_array = NEW_C_HEAP_ARRAY(HeapWord*, 2*max_plab_samples);

     _cursor               = NEW_C_HEAP_ARRAY(size_t, ParallelGCThreads);

     if (_survivor_plab_array == NULL || _survivor_chunk_array == NULL

         || _cursor == NULL) {

       warning("Failed to allocate survivor plab/chunk array");

       if (_survivor_plab_array  != NULL) {

         FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);

         _survivor_plab_array = NULL;

       }

       if (_survivor_chunk_array != NULL) {

         FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);

         _survivor_chunk_array = NULL;

       }

       if (_cursor != NULL) {

         FREE_C_HEAP_ARRAY(size_t, _cursor);

         _cursor = NULL;

       }

     } else {

       _survivor_chunk_capacity = 2*max_plab_samples;

       for (uint i = 0; i < ParallelGCThreads; i++) {

         HeapWord** vec = NEW_C_HEAP_ARRAY(HeapWord*, max_plab_samples);

         if (vec == NULL) {

           warning("Failed to allocate survivor plab array");

           for (int j = i; j > 0; j--) {

             FREE_C_HEAP_ARRAY(HeapWord*, _survivor_plab_array[j-1].array());

           }

           FREE_C_HEAP_ARRAY(ChunkArray, _survivor_plab_array);

           FREE_C_HEAP_ARRAY(HeapWord*, _survivor_chunk_array);

           _survivor_plab_array = NULL;

           _survivor_chunk_array = NULL;

           _survivor_chunk_capacity = 0;

           break;

         } else {

           ChunkArray* cur =

             ::new (&_survivor_plab_array[i]) ChunkArray(vec,

                                                         max_plab_samples);

           assert(cur->end() == 0, "Should be 0");

           assert(cur->array() == vec, "Should be vec");

           assert(cur->capacity() == max_plab_samples, "Error");

         }

       }

     }

   }

   assert(   (   _survivor_plab_array  != NULL

              && _survivor_chunk_array != NULL)

          || (   _survivor_chunk_capacity == 0

              && _survivor_chunk_index == 0),

          "Error");

   // Choose what strong roots should be scanned depending on verification options

   // and perm gen collection mode.

   if (!CMSClassUnloadingEnabled) {

     // If class unloading is disabled we want to include all classes into the root set.

     add_root_scanning_option(SharedHeap::SO_AllClasses);

   } else {

     add_root_scanning_option(SharedHeap::SO_SystemClasses);

   }

   NOT_PRODUCT(_overflow_counter = CMSMarkStackOverflowInterval;)

   _gc_counters = new CollectorCounters("CMS", 1);

   _completed_initialization = true;

   _inter_sweep_timer.start();  // start of time

 #ifdef SPARC

   // Issue a stern warning, but allow use for experimentation and debugging.

   if (VM_Version::is_sun4v() && UseMemSetInBOT) {

     assert(!FLAG_IS_DEFAULT(UseMemSetInBOT), "Error");

     warning("Experimental flag -XX:+UseMemSetInBOT is known to cause instability"

             " on sun4v; please understand that you are using at your own risk!");

   }

 #endif

 }

 const char* ConcurrentMarkSweepGeneration::name() const {

   return "concurrent mark-sweep generation";

 }

 void ConcurrentMarkSweepGeneration::update_counters() {

   if (UsePerfData) {

     _space_counters->update_all();

     _gen_counters->update_all();

   }

 }

 // this is an optimized version of update_counters(). it takes the

 // used value as a parameter rather than computing it.

 //

 void ConcurrentMarkSweepGeneration::update_counters(size_t used) {

   if (UsePerfData) {

     _space_counters->update_used(used);

     _space_counters->update_capacity();

     _gen_counters->update_all();

   }

 }

 void ConcurrentMarkSweepGeneration::print() const {

   Generation::print();

   cmsSpace()->print();

 }

 #ifndef PRODUCT

 void ConcurrentMarkSweepGeneration::print_statistics() {

   cmsSpace()->printFLCensus(0);

 }

 #endif

 void ConcurrentMarkSweepGeneration::printOccupancy(const char *s) {

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   if (PrintGCDetails) {

     if (Verbose) {

       gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"("SIZE_FORMAT")]",

         level(), short_name(), s, used(), capacity());

     } else {

       gclog_or_tty->print(" [%d %s-%s: "SIZE_FORMAT"K("SIZE_FORMAT"K)]",

         level(), short_name(), s, used() / K, capacity() / K);

     }

   }

   if (Verbose) {

     gclog_or_tty->print(" "SIZE_FORMAT"("SIZE_FORMAT")",

               gch->used(), gch->capacity());

   } else {

     gclog_or_tty->print(" "SIZE_FORMAT"K("SIZE_FORMAT"K)",

               gch->used() / K, gch->capacity() / K);

   }

 }

 size_t

 ConcurrentMarkSweepGeneration::contiguous_available() const {

   // dld proposes an improvement in precision here. If the committed

   // part of the space ends in a free block we should add that to

   // uncommitted size in the calculation below. Will make this

   // change later, staying with the approximation below for the

   // time being. -- ysr.

   return MAX2(_virtual_space.uncommitted_size(), unsafe_max_alloc_nogc());

 }

 size_t

 ConcurrentMarkSweepGeneration::unsafe_max_alloc_nogc() const {

   return _cmsSpace->max_alloc_in_words() * HeapWordSize;

 }

 size_t ConcurrentMarkSweepGeneration::max_available() const {

   return free() + _virtual_space.uncommitted_size();

 }

 bool ConcurrentMarkSweepGeneration::promotion_attempt_is_safe(size_t max_promotion_in_bytes) const {

   size_t available = max_available();

   size_t av_promo  = (size_t)gc_stats()->avg_promoted()->padded_average();

   bool   res = (available >= av_promo) || (available >= max_promotion_in_bytes);

   if (PrintGC && Verbose) {

     gclog_or_tty->print_cr(

       "CMS: promo attempt is%s safe: available("SIZE_FORMAT") %s av_promo("SIZE_FORMAT"),"

       "max_promo("SIZE_FORMAT")",

       res? "":" not", available, res? ">=":"<",

       av_promo, max_promotion_in_bytes);

   }

   return res;

 }

 // At a promotion failure dump information on block layout in heap

 // (cms old generation).

 void ConcurrentMarkSweepGeneration::promotion_failure_occurred() {

   if (CMSDumpAtPromotionFailure) {

     cmsSpace()->dump_at_safepoint_with_locks(collector(), gclog_or_tty);

   }

 }

 CompactibleSpace*

 ConcurrentMarkSweepGeneration::first_compaction_space() const {

   return _cmsSpace;

 }

 void ConcurrentMarkSweepGeneration::reset_after_compaction() {

   // Clear the promotion information.  These pointers can be adjusted

   // along with all the other pointers into the heap but

   // compaction is expected to be a rare event with

   // a heap using cms so don't do it without seeing the need.

   if (CollectedHeap::use_parallel_gc_threads()) {

     for (uint i = 0; i < ParallelGCThreads; i++) {

       _par_gc_thread_states[i]->promo.reset();

     }

   }

 }

 void ConcurrentMarkSweepGeneration::space_iterate(SpaceClosure* blk, bool usedOnly) {

   blk->do_space(_cmsSpace);

 }

 void ConcurrentMarkSweepGeneration::compute_new_size() {

   assert_locked_or_safepoint(Heap_lock);

   // If incremental collection failed, we just want to expand

   // to the limit.

   if (incremental_collection_failed()) {

     clear_incremental_collection_failed();

     grow_to_reserved();

     return;

   }

   size_t expand_bytes = 0;

   double free_percentage = ((double) free()) / capacity();

   double desired_free_percentage = (double) MinHeapFreeRatio / 100;

   double maximum_free_percentage = (double) MaxHeapFreeRatio / 100;

   // compute expansion delta needed for reaching desired free percentage

   if (free_percentage < desired_free_percentage) {

     size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));

     assert(desired_capacity >= capacity(), "invalid expansion size");

     expand_bytes = MAX2(desired_capacity - capacity(), MinHeapDeltaBytes);

   }

   if (expand_bytes > 0) {

     if (PrintGCDetails && Verbose) {

       size_t desired_capacity = (size_t)(used() / ((double) 1 - desired_free_percentage));

       gclog_or_tty->print_cr("\nFrom compute_new_size: ");

       gclog_or_tty->print_cr("  Free fraction %f", free_percentage);

       gclog_or_tty->print_cr("  Desired free fraction %f",

         desired_free_percentage);

       gclog_or_tty->print_cr("  Maximum free fraction %f",

         maximum_free_percentage);

       gclog_or_tty->print_cr("  Capactiy "SIZE_FORMAT, capacity()/1000);

       gclog_or_tty->print_cr("  Desired capacity "SIZE_FORMAT,

         desired_capacity/1000);

       int prev_level = level() - 1;

       if (prev_level >= 0) {

         size_t prev_size = 0;

         GenCollectedHeap* gch = GenCollectedHeap::heap();

         Generation* prev_gen = gch->_gens[prev_level];

         prev_size = prev_gen->capacity();

           gclog_or_tty->print_cr("  Younger gen size "SIZE_FORMAT,

                                  prev_size/1000);

       }

       gclog_or_tty->print_cr("  unsafe_max_alloc_nogc "SIZE_FORMAT,

         unsafe_max_alloc_nogc()/1000);

       gclog_or_tty->print_cr("  contiguous available "SIZE_FORMAT,

         contiguous_available()/1000);

       gclog_or_tty->print_cr("  Expand by "SIZE_FORMAT" (bytes)",

         expand_bytes);

     }

     // safe if expansion fails

     expand(expand_bytes, 0, CMSExpansionCause::_satisfy_free_ratio);

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print_cr("  Expanded free fraction %f",

         ((double) free()) / capacity());

     }

   }

 }

 Mutex* ConcurrentMarkSweepGeneration::freelistLock() const {

   return cmsSpace()->freelistLock();

 }

 HeapWord* ConcurrentMarkSweepGeneration::allocate(size_t size,

                                                   bool   tlab) {

   CMSSynchronousYieldRequest yr;

   MutexLockerEx x(freelistLock(),

                   Mutex::_no_safepoint_check_flag);

   return have_lock_and_allocate(size, tlab);

 }

 HeapWord* ConcurrentMarkSweepGeneration::have_lock_and_allocate(size_t size,

                                                   bool   tlab /* ignored */) {

   assert_lock_strong(freelistLock());

   size_t adjustedSize = CompactibleFreeListSpace::adjustObjectSize(size);

   HeapWord* res = cmsSpace()->allocate(adjustedSize);

   // Allocate the object live (grey) if the background collector has

   // started marking. This is necessary because the marker may

   // have passed this address and consequently this object will

   // not otherwise be greyed and would be incorrectly swept up.

   // Note that if this object contains references, the writing

   // of those references will dirty the card containing this object

   // allowing the object to be blackened (and its references scanned)

   // either during a preclean phase or at the final checkpoint.

   if (res != NULL) {

     // We may block here with an uninitialized object with

     // its mark-bit or P-bits not yet set. Such objects need

     // to be safely navigable by block_start().

     assert(oop(res)->klass_or_null() == NULL, "Object should be uninitialized here.");

     assert(!((FreeChunk*)res)->isFree(), "Error, block will look free but show wrong size");

     collector()->direct_allocated(res, adjustedSize);

     _direct_allocated_words += adjustedSize;

     // allocation counters

     NOT_PRODUCT(

       _numObjectsAllocated++;

       _numWordsAllocated += (int)adjustedSize;

     )

   }

   return res;

 }

 // In the case of direct allocation by mutators in a generation that

 // is being concurrently collected, the object must be allocated

 // live (grey) if the background collector has started marking.

 // This is necessary because the marker may

 // have passed this address and consequently this object will

 // not otherwise be greyed and would be incorrectly swept up.

 // Note that if this object contains references, the writing

 // of those references will dirty the card containing this object

 // allowing the object to be blackened (and its references scanned)

 // either during a preclean phase or at the final checkpoint.

 void CMSCollector::direct_allocated(HeapWord* start, size_t size) {

   assert(_markBitMap.covers(start, size), "Out of bounds");

   if (_collectorState >= Marking) {

     MutexLockerEx y(_markBitMap.lock(),

                     Mutex::_no_safepoint_check_flag);

     // [see comments preceding SweepClosure::do_blk() below for details]

     // 1. need to mark the object as live so it isn't collected

     // 2. need to mark the 2nd bit to indicate the object may be uninitialized

     // 3. need to mark the end of the object so marking, precleaning or sweeping

     //    can skip over uninitialized or unparsable objects. An allocated

     //    object is considered uninitialized for our purposes as long as

     //    its klass word is NULL. (Unparsable objects are those which are

     //    initialized in the sense just described, but whose sizes can still

     //    not be correctly determined. Note that the class of unparsable objects

     //    can only occur in the perm gen. All old gen objects are parsable

     //    as soon as they are initialized.)

     _markBitMap.mark(start);          // object is live

     _markBitMap.mark(start + 1);      // object is potentially uninitialized?

     _markBitMap.mark(start + size - 1);

                                       // mark end of object

   }

   // check that oop looks uninitialized

   assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");

 }

 void CMSCollector::promoted(bool par, HeapWord* start,

                             bool is_obj_array, size_t obj_size) {

   assert(_markBitMap.covers(start), "Out of bounds");

   // See comment in direct_allocated() about when objects should

   // be allocated live.

   if (_collectorState >= Marking) {

     // we already hold the marking bit map lock, taken in

     // the prologue

     if (par) {

       _markBitMap.par_mark(start);

     } else {

       _markBitMap.mark(start);

     }

     // We don't need to mark the object as uninitialized (as

     // in direct_allocated above) because this is being done with the

     // world stopped and the object will be initialized by the

     // time the marking, precleaning or sweeping get to look at it.

     // But see the code for copying objects into the CMS generation,

     // where we need to ensure that concurrent readers of the

     // block offset table are able to safely navigate a block that

     // is in flux from being free to being allocated (and in

     // transition while being copied into) and subsequently

     // becoming a bona-fide object when the copy/promotion is complete.

     assert(SafepointSynchronize::is_at_safepoint(),

            "expect promotion only at safepoints");

     if (_collectorState < Sweeping) {

       // Mark the appropriate cards in the modUnionTable, so that

       // this object gets scanned before the sweep. If this is

       // not done, CMS generation references in the object might

       // not get marked.

       // For the case of arrays, which are otherwise precisely

       // marked, we need to dirty the entire array, not just its head.

       if (is_obj_array) {

         // The [par_]mark_range() method expects mr.end() below to

         // be aligned to the granularity of a bit's representation

         // in the heap. In the case of the MUT below, that's a

         // card size.

         MemRegion mr(start,

                      (HeapWord*)round_to((intptr_t)(start + obj_size),

                         CardTableModRefBS::card_size /* bytes */));

         if (par) {

           _modUnionTable.par_mark_range(mr);

         } else {

           _modUnionTable.mark_range(mr);

         }

       } else {  // not an obj array; we can just mark the head

         if (par) {

           _modUnionTable.par_mark(start);

         } else {

           _modUnionTable.mark(start);

         }

       }

     }

   }

 }

 static inline size_t percent_of_space(Space* space, HeapWord* addr)

 {

   size_t delta = pointer_delta(addr, space->bottom());

   return (size_t)(delta * 100.0 / (space->capacity() / HeapWordSize));

 }

 void CMSCollector::icms_update_allocation_limits()

 {

   Generation* gen0 = GenCollectedHeap::heap()->get_gen(0);

   EdenSpace* eden = gen0->as_DefNewGeneration()->eden();

   const unsigned int duty_cycle = stats().icms_update_duty_cycle();

   if (CMSTraceIncrementalPacing) {

     stats().print();

   }

   assert(duty_cycle <= 100, "invalid duty cycle");

   if (duty_cycle != 0) {

     // The duty_cycle is a percentage between 0 and 100; convert to words and

     // then compute the offset from the endpoints of the space.

     size_t free_words = eden->free() / HeapWordSize;

     double free_words_dbl = (double)free_words;

     size_t duty_cycle_words = (size_t)(free_words_dbl * duty_cycle / 100.0);

     size_t offset_words = (free_words - duty_cycle_words) / 2;

     _icms_start_limit = eden->top() + offset_words;

     _icms_stop_limit = eden->end() - offset_words;

     // The limits may be adjusted (shifted to the right) by

     // CMSIncrementalOffset, to allow the application more mutator time after a

     // young gen gc (when all mutators were stopped) and before CMS starts and

     // takes away one or more cpus.

     if (CMSIncrementalOffset != 0) {

       double adjustment_dbl = free_words_dbl * CMSIncrementalOffset / 100.0;

       size_t adjustment = (size_t)adjustment_dbl;

       HeapWord* tmp_stop = _icms_stop_limit + adjustment;

       if (tmp_stop > _icms_stop_limit && tmp_stop < eden->end()) {

         _icms_start_limit += adjustment;

         _icms_stop_limit = tmp_stop;

       }

     }

   }

   if (duty_cycle == 0 || (_icms_start_limit == _icms_stop_limit)) {

     _icms_start_limit = _icms_stop_limit = eden->end();

   }

   // Install the new start limit.

   eden->set_soft_end(_icms_start_limit);

   if (CMSTraceIncrementalMode) {

     gclog_or_tty->print(" icms alloc limits:  "

                            PTR_FORMAT "," PTR_FORMAT

                            " (" SIZE_FORMAT "%%," SIZE_FORMAT "%%) ",

                            _icms_start_limit, _icms_stop_limit,

                            percent_of_space(eden, _icms_start_limit),

                            percent_of_space(eden, _icms_stop_limit));

     if (Verbose) {

       gclog_or_tty->print("eden:  ");

       eden->print_on(gclog_or_tty);

     }

   }

 }

 // Any changes here should try to maintain the invariant

 // that if this method is called with _icms_start_limit

 // and _icms_stop_limit both NULL, then it should return NULL

 // and not notify the icms thread.

 HeapWord*

 CMSCollector::allocation_limit_reached(Space* space, HeapWord* top,

                                        size_t word_size)

 {

   // A start_limit equal to end() means the duty cycle is 0, so treat that as a

   // nop.

   if (CMSIncrementalMode && _icms_start_limit != space->end()) {

     if (top <= _icms_start_limit) {

       if (CMSTraceIncrementalMode) {

         space->print_on(gclog_or_tty);

         gclog_or_tty->stamp();

         gclog_or_tty->print_cr(" start limit top=" PTR_FORMAT

                                ", new limit=" PTR_FORMAT

                                " (" SIZE_FORMAT "%%)",

                                top, _icms_stop_limit,

                                percent_of_space(space, _icms_stop_limit));

       }

       ConcurrentMarkSweepThread::start_icms();

       assert(top < _icms_stop_limit, "Tautology");

       if (word_size < pointer_delta(_icms_stop_limit, top)) {

         return _icms_stop_limit;

       }

       // The allocation will cross both the _start and _stop limits, so do the

       // stop notification also and return end().

       if (CMSTraceIncrementalMode) {

         space->print_on(gclog_or_tty);

         gclog_or_tty->stamp();

         gclog_or_tty->print_cr(" +stop limit top=" PTR_FORMAT

                                ", new limit=" PTR_FORMAT

                                " (" SIZE_FORMAT "%%)",

                                top, space->end(),

                                percent_of_space(space, space->end()));

       }

       ConcurrentMarkSweepThread::stop_icms();

       return space->end();

     }

     if (top <= _icms_stop_limit) {

       if (CMSTraceIncrementalMode) {

         space->print_on(gclog_or_tty);

         gclog_or_tty->stamp();

         gclog_or_tty->print_cr(" stop limit top=" PTR_FORMAT

                                ", new limit=" PTR_FORMAT

                                " (" SIZE_FORMAT "%%)",

                                top, space->end(),

                                percent_of_space(space, space->end()));

       }

       ConcurrentMarkSweepThread::stop_icms();

       return space->end();

     }

     if (CMSTraceIncrementalMode) {

       space->print_on(gclog_or_tty);

       gclog_or_tty->stamp();

       gclog_or_tty->print_cr(" end limit top=" PTR_FORMAT

                              ", new limit=" PTR_FORMAT,

                              top, NULL);

     }

   }

   return NULL;

 }

 oop ConcurrentMarkSweepGeneration::promote(oop obj, size_t obj_size) {

   assert(obj_size == (size_t)obj->size(), "bad obj_size passed in");

   // allocate, copy and if necessary update promoinfo --

   // delegate to underlying space.

   assert_lock_strong(freelistLock());

 #ifndef PRODUCT

   if (Universe::heap()->promotion_should_fail()) {

     return NULL;

   }

 #endif  // #ifndef PRODUCT

   oop res = _cmsSpace->promote(obj, obj_size);

   if (res == NULL) {

     // expand and retry

     size_t s = _cmsSpace->expansionSpaceRequired(obj_size);  // HeapWords

     expand(s*HeapWordSize, MinHeapDeltaBytes,

       CMSExpansionCause::_satisfy_promotion);

     // Since there's currently no next generation, we don't try to promote

     // into a more senior generation.

     assert(next_gen() == NULL, "assumption, based upon which no attempt "

                                "is made to pass on a possibly failing "

                                "promotion to next generation");

     res = _cmsSpace->promote(obj, obj_size);

   }

   if (res != NULL) {

     // See comment in allocate() about when objects should

     // be allocated live.

     assert(obj->is_oop(), "Will dereference klass pointer below");

     collector()->promoted(false,           // Not parallel

                           (HeapWord*)res, obj->is_objArray(), obj_size);

     // promotion counters

     NOT_PRODUCT(

       _numObjectsPromoted++;

       _numWordsPromoted +=

         (int)(CompactibleFreeListSpace::adjustObjectSize(obj->size()));

     )

   }

   return res;

 }

 HeapWord*

 ConcurrentMarkSweepGeneration::allocation_limit_reached(Space* space,

                                              HeapWord* top,

                                              size_t word_sz)

 {

   return collector()->allocation_limit_reached(space, top, word_sz);

 }

 // IMPORTANT: Notes on object size recognition in CMS.

 // ---------------------------------------------------

 // A block of storage in the CMS generation is always in

 // one of three states. A free block (FREE), an allocated

 // object (OBJECT) whose size() method reports the correct size,

 // and an intermediate state (TRANSIENT) in which its size cannot

 // be accurately determined.

 // STATE IDENTIFICATION:   (32 bit and 64 bit w/o COOPS)

 // -----------------------------------------------------

 // FREE:      klass_word & 1 == 1; mark_word holds block size

 //

 // OBJECT:    klass_word installed; klass_word != 0 && klass_word & 1 == 0;

 //            obj->size() computes correct size

 //            [Perm Gen objects needs to be "parsable" before they can be navigated]

 //

 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT

 //

 // STATE IDENTIFICATION: (64 bit+COOPS)

 // ------------------------------------

 // FREE:      mark_word & CMS_FREE_BIT == 1; mark_word & ~CMS_FREE_BIT gives block_size

 //

 // OBJECT:    klass_word installed; klass_word != 0;

 //            obj->size() computes correct size

 //            [Perm Gen comment above continues to hold]

 //

 // TRANSIENT: klass_word == 0; size is indeterminate until we become an OBJECT

 //

 //

 // STATE TRANSITION DIAGRAM

 //

 //        mut / parnew                     mut  /  parnew

 // FREE --------------------> TRANSIENT ---------------------> OBJECT --|

 //  ^                                                                   |

 //  |------------------------ DEAD <------------------------------------|

 //         sweep                            mut

 //

 // While a block is in TRANSIENT state its size cannot be determined

 // so readers will either need to come back later or stall until

 // the size can be determined. Note that for the case of direct

 // allocation, P-bits, when available, may be used to determine the

 // size of an object that may not yet have been initialized.

 // Things to support parallel young-gen collection.

 oop

 ConcurrentMarkSweepGeneration::par_promote(int thread_num,

                                            oop old, markOop m,

                                            size_t word_sz) {

 #ifndef PRODUCT

   if (Universe::heap()->promotion_should_fail()) {

     return NULL;

   }

 #endif  // #ifndef PRODUCT

   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

   PromotionInfo* promoInfo = &ps->promo;

   // if we are tracking promotions, then first ensure space for

   // promotion (including spooling space for saving header if necessary).

   // then allocate and copy, then track promoted info if needed.

   // When tracking (see PromotionInfo::track()), the mark word may

   // be displaced and in this case restoration of the mark word

   // occurs in the (oop_since_save_marks_)iterate phase.

   if (promoInfo->tracking() && !promoInfo->ensure_spooling_space()) {

     // Out of space for allocating spooling buffers;

     // try expanding and allocating spooling buffers.

     if (!expand_and_ensure_spooling_space(promoInfo)) {

       return NULL;

     }

   }

   assert(promoInfo->has_spooling_space(), "Control point invariant");

   const size_t alloc_sz = CompactibleFreeListSpace::adjustObjectSize(word_sz);

   HeapWord* obj_ptr = ps->lab.alloc(alloc_sz);

   if (obj_ptr == NULL) {

      obj_ptr = expand_and_par_lab_allocate(ps, alloc_sz);

      if (obj_ptr == NULL) {

        return NULL;

      }

   }

   oop obj = oop(obj_ptr);

   OrderAccess::storestore();

   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

   assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");

   // IMPORTANT: See note on object initialization for CMS above.

   // Otherwise, copy the object.  Here we must be careful to insert the

   // klass pointer last, since this marks the block as an allocated object.

   // Except with compressed oops it's the mark word.

   HeapWord* old_ptr = (HeapWord*)old;

   // Restore the mark word copied above.

   obj->set_mark(m);

   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

   assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");

   OrderAccess::storestore();

   if (UseCompressedOops) {

     // Copy gap missed by (aligned) header size calculation below

     obj->set_klass_gap(old->klass_gap());

   }

   if (word_sz > (size_t)oopDesc::header_size()) {

     Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),

                                  obj_ptr + oopDesc::header_size(),

                                  word_sz - oopDesc::header_size());

   }

   // Now we can track the promoted object, if necessary.  We take care

   // to delay the transition from uninitialized to full object

   // (i.e., insertion of klass pointer) until after, so that it

   // atomically becomes a promoted object.

   if (promoInfo->tracking()) {

     promoInfo->track((PromotedObject*)obj, old->klass());

   }

   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");

   assert(!((FreeChunk*)obj_ptr)->isFree(), "Error, block will look free but show wrong size");

   assert(old->is_oop(), "Will use and dereference old klass ptr below");

   // Finally, install the klass pointer (this should be volatile).

   OrderAccess::storestore();

   obj->set_klass(old->klass());

   // We should now be able to calculate the right size for this object

   assert(obj->is_oop() && obj->size() == (int)word_sz, "Error, incorrect size computed for promoted object");

   collector()->promoted(true,          // parallel

                         obj_ptr, old->is_objArray(), word_sz);

   NOT_PRODUCT(

     Atomic::inc_ptr(&_numObjectsPromoted);

     Atomic::add_ptr(alloc_sz, &_numWordsPromoted);

   )

   return obj;

 }

 void

 ConcurrentMarkSweepGeneration::

 par_promote_alloc_undo(int thread_num,

                        HeapWord* obj, size_t word_sz) {

   // CMS does not support promotion undo.

   ShouldNotReachHere();

 }

 void

 ConcurrentMarkSweepGeneration::

 par_promote_alloc_done(int thread_num) {

   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

   ps->lab.retire(thread_num);

 }

 void

 ConcurrentMarkSweepGeneration::

 par_oop_since_save_marks_iterate_done(int thread_num) {

   CMSParGCThreadState* ps = _par_gc_thread_states[thread_num];

   ParScanWithoutBarrierClosure* dummy_cl = NULL;

   ps->promo.promoted_oops_iterate_nv(dummy_cl);

 }

 // XXXPERM

 bool ConcurrentMarkSweepGeneration::should_collect(bool   full,

                                                    size_t size,

                                                    bool   tlab)

 {

   // We allow a STW collection only if a full

   // collection was requested.

   return full || should_allocate(size, tlab); // FIX ME !!!

   // This and promotion failure handling are connected at the

   // hip and should be fixed by untying them.

 }

 bool CMSCollector::shouldConcurrentCollect() {

   if (_full_gc_requested) {

     if (Verbose && PrintGCDetails) {

       gclog_or_tty->print_cr("CMSCollector: collect because of explicit "

                              " gc request (or gc_locker)");

     }

     return true;

   }

   // For debugging purposes, change the type of collection.

   // If the rotation is not on the concurrent collection

   // type, don't start a concurrent collection.

   NOT_PRODUCT(

     if (RotateCMSCollectionTypes &&

         (_cmsGen->debug_collection_type() !=

           ConcurrentMarkSweepGeneration::Concurrent_collection_type)) {

       assert(_cmsGen->debug_collection_type() !=

         ConcurrentMarkSweepGeneration::Unknown_collection_type,

         "Bad cms collection type");

       return false;

     }

   )

   FreelistLocker x(this);

   // ------------------------------------------------------------------

   // Print out lots of information which affects the initiation of

   // a collection.

   if (PrintCMSInitiationStatistics && stats().valid()) {

     gclog_or_tty->print("CMSCollector shouldConcurrentCollect: ");

     gclog_or_tty->stamp();

     gclog_or_tty->print_cr("");

     stats().print_on(gclog_or_tty);

     gclog_or_tty->print_cr("time_until_cms_gen_full %3.7f",

       stats().time_until_cms_gen_full());

     gclog_or_tty->print_cr("free="SIZE_FORMAT, _cmsGen->free());

     gclog_or_tty->print_cr("contiguous_available="SIZE_FORMAT,

                            _cmsGen->contiguous_available());

     gclog_or_tty->print_cr("promotion_rate=%g", stats().promotion_rate());

     gclog_or_tty->print_cr("cms_allocation_rate=%g", stats().cms_allocation_rate());

     gclog_or_tty->print_cr("occupancy=%3.7f", _cmsGen->occupancy());

     gclog_or_tty->print_cr("initiatingOccupancy=%3.7f", _cmsGen->initiating_occupancy());

     gclog_or_tty->print_cr("initiatingPermOccupancy=%3.7f", _permGen->initiating_occupancy());

   }

   // ------------------------------------------------------------------

   // If the estimated time to complete a cms collection (cms_duration())

   // is less than the estimated time remaining until the cms generation

   // is full, start a collection.

   if (!UseCMSInitiatingOccupancyOnly) {

     if (stats().valid()) {

       if (stats().time_until_cms_start() == 0.0) {

         return true;

       }

     } else {

       // We want to conservatively collect somewhat early in order

       // to try and "bootstrap" our CMS/promotion statistics;

       // this branch will not fire after the first successful CMS

       // collection because the stats should then be valid.

       if (_cmsGen->occupancy() >= _bootstrap_occupancy) {

         if (Verbose && PrintGCDetails) {

           gclog_or_tty->print_cr(

             " CMSCollector: collect for bootstrapping statistics:"

             " occupancy = %f, boot occupancy = %f", _cmsGen->occupancy(),

             _bootstrap_occupancy);

         }

         return true;

       }

     }

   }

   // Otherwise, we start a collection cycle if either the perm gen or

   // old gen want a collection cycle started. Each may use

   // an appropriate criterion for making this decision.

   // XXX We need to make sure that the gen expansion

   // criterion dovetails well with this. XXX NEED TO FIX THIS

   if (_cmsGen->should_concurrent_collect()) {

     if (Verbose && PrintGCDetails) {

       gclog_or_tty->print_cr("CMS old gen initiated");

     }

     return true;

   }

   // We start a collection if we believe an incremental collection may fail;

   // this is not likely to be productive in practice because it's probably too

   // late anyway.

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   assert(gch->collector_policy()->is_two_generation_policy(),

          "You may want to check the correctness of the following");

   if (gch->incremental_collection_will_fail()) {

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print("CMSCollector: collect because incremental collection will fail ");

     }

     return true;

   }

   if (CMSClassUnloadingEnabled && _permGen->should_concurrent_collect()) {

     bool res = update_should_unload_classes();

     if (res) {

       if (Verbose && PrintGCDetails) {

         gclog_or_tty->print_cr("CMS perm gen initiated");

       }

       return true;

     }

   }

   return false;

 }

 // Clear _expansion_cause fields of constituent generations

 void CMSCollector::clear_expansion_cause() {

   _cmsGen->clear_expansion_cause();

   _permGen->clear_expansion_cause();

 }

 // We should be conservative in starting a collection cycle.  To

 // start too eagerly runs the risk of collecting too often in the

 // extreme.  To collect too rarely falls back on full collections,

 // which works, even if not optimum in terms of concurrent work.

 // As a work around for too eagerly collecting, use the flag

 // UseCMSInitiatingOccupancyOnly.  This also has the advantage of

 // giving the user an easily understandable way of controlling the

 // collections.

 // We want to start a new collection cycle if any of the following

 // conditions hold:

 // . our current occupancy exceeds the configured initiating occupancy

 //   for this generation, or

 // . we recently needed to expand this space and have not, since that

 //   expansion, done a collection of this generation, or

 // . the underlying space believes that it may be a good idea to initiate

 //   a concurrent collection (this may be based on criteria such as the

 //   following: the space uses linear allocation and linear allocation is

 //   going to fail, or there is believed to be excessive fragmentation in

 //   the generation, etc... or ...

 // [.(currently done by CMSCollector::shouldConcurrentCollect() only for

 //   the case of the old generation, not the perm generation; see CR 6543076):

 //   we may be approaching a point at which allocation requests may fail because

 //   we will be out of sufficient free space given allocation rate estimates.]

 bool ConcurrentMarkSweepGeneration::should_concurrent_collect() const {

   assert_lock_strong(freelistLock());

   if (occupancy() > initiating_occupancy()) {

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print(" %s: collect because of occupancy %f / %f  ",

         short_name(), occupancy(), initiating_occupancy());

     }

     return true;

   }

   if (UseCMSInitiatingOccupancyOnly) {

     return false;

   }

   if (expansion_cause() == CMSExpansionCause::_satisfy_allocation) {

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print(" %s: collect because expanded for allocation ",

         short_name());

     }

     return true;

   }

   if (_cmsSpace->should_concurrent_collect()) {

     if (PrintGCDetails && Verbose) {

       gclog_or_tty->print(" %s: collect because cmsSpace says so ",

         short_name());

     }

     return true;

   }

   return false;

 }

 void ConcurrentMarkSweepGeneration::collect(bool   full,

                                             bool   clear_all_soft_refs,

                                             size_t size,

                                             bool   tlab)

 {

   collector()->collect(full, clear_all_soft_refs, size, tlab);

 }

 void CMSCollector::collect(bool   full,

                            bool   clear_all_soft_refs,

                            size_t size,

                            bool   tlab)

 {

   if (!UseCMSCollectionPassing && _collectorState > Idling) {

     // For debugging purposes skip the collection if the state

     // is not currently idle

     if (TraceCMSState) {

       gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " skipped full:%d CMS state %d",

         Thread::current(), full, _collectorState);

     }

     return;

   }

   // The following "if" branch is present for defensive reasons.

   // In the current uses of this interface, it can be replaced with:

   // assert(!GC_locker.is_active(), "Can't be called otherwise");

   // But I am not placing that assert here to allow future

   // generality in invoking this interface.

   if (GC_locker::is_active()) {

     // A consistency test for GC_locker

     assert(GC_locker::needs_gc(), "Should have been set already");

     // Skip this foreground collection, instead

     // expanding the heap if necessary.

     // Need the free list locks for the call to free() in compute_new_size()

     compute_new_size();

     return;

   }

   acquire_control_and_collect(full, clear_all_soft_refs);

   _full_gcs_since_conc_gc++;

 }

 void CMSCollector::request_full_gc(unsigned int full_gc_count) {

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   unsigned int gc_count = gch->total_full_collections();

   if (gc_count == full_gc_count) {

     MutexLockerEx y(CGC_lock, Mutex::_no_safepoint_check_flag);

     _full_gc_requested = true;

     CGC_lock->notify();   // nudge CMS thread

   }

 }

 // The foreground and background collectors need to coordinate in order

 // to make sure that they do not mutually interfere with CMS collections.

 // When a background collection is active,

 // the foreground collector may need to take over (preempt) and

 // synchronously complete an ongoing collection. Depending on the

 // frequency of the background collections and the heap usage

 // of the application, this preemption can be seldom or frequent.

 // There are only certain

 // points in the background collection that the "collection-baton"

 // can be passed to the foreground collector.

 //

 // The foreground collector will wait for the baton before

 // starting any part of the collection.  The foreground collector

 // will only wait at one location.

 //

 // The background collector will yield the baton before starting a new

 // phase of the collection (e.g., before initial marking, marking from roots,

 // precleaning, final re-mark, sweep etc.)  This is normally done at the head

 // of the loop which switches the phases. The background collector does some

 // of the phases (initial mark, final re-mark) with the world stopped.

 // Because of locking involved in stopping the world,

 // the foreground collector should not block waiting for the background

 // collector when it is doing a stop-the-world phase.  The background

 // collector will yield the baton at an additional point just before

 // it enters a stop-the-world phase.  Once the world is stopped, the

 // background collector checks the phase of the collection.  If the

 // phase has not changed, it proceeds with the collection.  If the

 // phase has changed, it skips that phase of the collection.  See

 // the comments on the use of the Heap_lock in collect_in_background().

 //

 // Variable used in baton passing.

 //   _foregroundGCIsActive - Set to true by the foreground collector when

 //      it wants the baton.  The foreground clears it when it has finished

 //      the collection.

 //   _foregroundGCShouldWait - Set to true by the background collector

 //        when it is running.  The foreground collector waits while

 //      _foregroundGCShouldWait is true.

 //  CGC_lock - monitor used to protect access to the above variables

 //      and to notify the foreground and background collectors.

 //  _collectorState - current state of the CMS collection.

 //

 // The foreground collector

 //   acquires the CGC_lock

 //   sets _foregroundGCIsActive

 //   waits on the CGC_lock for _foregroundGCShouldWait to be false

 //     various locks acquired in preparation for the collection

 //     are released so as not to block the background collector

 //     that is in the midst of a collection

 //   proceeds with the collection

 //   clears _foregroundGCIsActive

 //   returns

 //

 // The background collector in a loop iterating on the phases of the

 //      collection

 //   acquires the CGC_lock

 //   sets _foregroundGCShouldWait

 //   if _foregroundGCIsActive is set

 //     clears _foregroundGCShouldWait, notifies _CGC_lock

 //     waits on _CGC_lock for _foregroundGCIsActive to become false

 //     and exits the loop.

 //   otherwise

 //     proceed with that phase of the collection

 //     if the phase is a stop-the-world phase,

 //       yield the baton once more just before enqueueing

 //       the stop-world CMS operation (executed by the VM thread).

 //   returns after all phases of the collection are done

 //

 void CMSCollector::acquire_control_and_collect(bool full,

         bool clear_all_soft_refs) {

   assert(SafepointSynchronize::is_at_safepoint(), "should be at safepoint");

   assert(!Thread::current()->is_ConcurrentGC_thread(),

          "shouldn't try to acquire control from self!");

   // Start the protocol for acquiring control of the

   // collection from the background collector (aka CMS thread).

   assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),

          "VM thread should have CMS token");

   // Remember the possibly interrupted state of an ongoing

   // concurrent collection

   CollectorState first_state = _collectorState;

   // Signal to a possibly ongoing concurrent collection that

   // we want to do a foreground collection.

   _foregroundGCIsActive = true;

   // Disable incremental mode during a foreground collection.

   ICMSDisabler icms_disabler;

   // release locks and wait for a notify from the background collector

   // releasing the locks in only necessary for phases which

   // do yields to improve the granularity of the collection.

   assert_lock_strong(bitMapLock());

   // We need to lock the Free list lock for the space that we are

   // currently collecting.

   assert(haveFreelistLocks(), "Must be holding free list locks");

   bitMapLock()->unlock();

   releaseFreelistLocks();

   {

     MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

     if (_foregroundGCShouldWait) {

       // We are going to be waiting for action for the CMS thread;

       // it had better not be gone (for instance at shutdown)!

       assert(ConcurrentMarkSweepThread::cmst() != NULL,

              "CMS thread must be running");

       // Wait here until the background collector gives us the go-ahead

       ConcurrentMarkSweepThread::clear_CMS_flag(

         ConcurrentMarkSweepThread::CMS_vm_has_token);  // release token

       // Get a possibly blocked CMS thread going:

       //   Note that we set _foregroundGCIsActive true above,

       //   without protection of the CGC_lock.

       CGC_lock->notify();

       assert(!ConcurrentMarkSweepThread::vm_thread_wants_cms_token(),

              "Possible deadlock");

       while (_foregroundGCShouldWait) {

         // wait for notification

         CGC_lock->wait(Mutex::_no_safepoint_check_flag);

         // Possibility of delay/starvation here, since CMS token does

         // not know to give priority to VM thread? Actually, i think

         // there wouldn't be any delay/starvation, but the proof of

         // that "fact" (?) appears non-trivial. XXX 20011219YSR

       }

       ConcurrentMarkSweepThread::set_CMS_flag(

         ConcurrentMarkSweepThread::CMS_vm_has_token);

     }

   }

   // The CMS_token is already held.  Get back the other locks.

   assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),

          "VM thread should have CMS token");

   getFreelistLocks();

   bitMapLock()->lock_without_safepoint_check();

   if (TraceCMSState) {

     gclog_or_tty->print_cr("CMS foreground collector has asked for control "

       INTPTR_FORMAT " with first state %d", Thread::current(), first_state);

     gclog_or_tty->print_cr("    gets control with state %d", _collectorState);

   }

   // Check if we need to do a compaction, or if not, whether

   // we need to start the mark-sweep from scratch.

   bool should_compact    = false;

   bool should_start_over = false;

   decide_foreground_collection_type(clear_all_soft_refs,

     &should_compact, &should_start_over);

 NOT_PRODUCT(

   if (RotateCMSCollectionTypes) {

     if (_cmsGen->debug_collection_type() ==

         ConcurrentMarkSweepGeneration::MSC_foreground_collection_type) {

       should_compact = true;

     } else if (_cmsGen->debug_collection_type() ==

                ConcurrentMarkSweepGeneration::MS_foreground_collection_type) {

       should_compact = false;

     }

   }

 )

   if (PrintGCDetails && first_state > Idling) {

     GCCause::Cause cause = GenCollectedHeap::heap()->gc_cause();

     if (GCCause::is_user_requested_gc(cause) ||

         GCCause::is_serviceability_requested_gc(cause)) {

       gclog_or_tty->print(" (concurrent mode interrupted)");

     } else {

       gclog_or_tty->print(" (concurrent mode failure)");

     }

   }

   if (should_compact) {

     // If the collection is being acquired from the background

     // collector, there may be references on the discovered

     // references lists that have NULL referents (being those

     // that were concurrently cleared by a mutator) or

     // that are no longer active (having been enqueued concurrently

     // by the mutator).

     // Scrub the list of those references because Mark-Sweep-Compact

     // code assumes referents are not NULL and that all discovered

     // Reference objects are active.

     ref_processor()->clean_up_discovered_references();

     do_compaction_work(clear_all_soft_refs);

     // Has the GC time limit been exceeded?

     DefNewGeneration* young_gen = _young_gen->as_DefNewGeneration();

     size_t max_eden_size = young_gen->max_capacity() -

                            young_gen->to()->capacity() -

                            young_gen->from()->capacity();

     GenCollectedHeap* gch = GenCollectedHeap::heap();

     GCCause::Cause gc_cause = gch->gc_cause();

     size_policy()->check_gc_overhead_limit(_young_gen->used(),

                                            young_gen->eden()->used(),

                                            _cmsGen->max_capacity(),

                                            max_eden_size,

                                            full,

                                            gc_cause,

                                            gch->collector_policy());

   } else {

     do_mark_sweep_work(clear_all_soft_refs, first_state,

       should_start_over);

   }

   // Reset the expansion cause, now that we just completed

   // a collection cycle.

   clear_expansion_cause();

   _foregroundGCIsActive = false;

   return;

 }

 // Resize the perm generation and the tenured generation

 // after obtaining the free list locks for the

 // two generations.

 void CMSCollector::compute_new_size() {

   assert_locked_or_safepoint(Heap_lock);

   FreelistLocker z(this);

   _permGen->compute_new_size();

   _cmsGen->compute_new_size();

 }

 // A work method used by foreground collection to determine

 // what type of collection (compacting or not, continuing or fresh)

 // it should do.

 // NOTE: the intent is to make UseCMSCompactAtFullCollection

 // and CMSCompactWhenClearAllSoftRefs the default in the future

 // and do away with the flags after a suitable period.

 void CMSCollector::decide_foreground_collection_type(

   bool clear_all_soft_refs, bool* should_compact,

   bool* should_start_over) {

   // Normally, we'll compact only if the UseCMSCompactAtFullCollection

   // flag is set, and we have either requested a System.gc() or

   // the number of full gc's since the last concurrent cycle

   // has exceeded the threshold set by CMSFullGCsBeforeCompaction,

   // or if an incremental collection has failed

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   assert(gch->collector_policy()->is_two_generation_policy(),

          "You may want to check the correctness of the following");

   // Inform cms gen if this was due to partial collection failing.

   // The CMS gen may use this fact to determine its expansion policy.

   if (gch->incremental_collection_will_fail()) {

     assert(!_cmsGen->incremental_collection_failed(),

            "Should have been noticed, reacted to and cleared");

     _cmsGen->set_incremental_collection_failed();

   }

   *should_compact =

     UseCMSCompactAtFullCollection &&

     ((_full_gcs_since_conc_gc >= CMSFullGCsBeforeCompaction) ||

      GCCause::is_user_requested_gc(gch->gc_cause()) ||

      gch->incremental_collection_will_fail());

   *should_start_over = false;

   if (clear_all_soft_refs && !*should_compact) {

     // We are about to do a last ditch collection attempt

     // so it would normally make sense to do a compaction

     // to reclaim as much space as possible.

     if (CMSCompactWhenClearAllSoftRefs) {

       // Default: The rationale is that in this case either

       // we are past the final marking phase, in which case

       // we'd have to start over, or so little has been done

       // that there's little point in saving that work. Compaction

       // appears to be the sensible choice in either case.

       *should_compact = true;

     } else {

       // We have been asked to clear all soft refs, but not to

       // compact. Make sure that we aren't past the final checkpoint

       // phase, for that is where we process soft refs. If we are already

       // past that phase, we'll need to redo the refs discovery phase and

       // if necessary clear soft refs that weren't previously

       // cleared. We do so by remembering the phase in which

       // we came in, and if we are past the refs processing

       // phase, we'll choose to just redo the mark-sweep

       // collection from scratch.

       if (_collectorState > FinalMarking) {

         // We are past the refs processing phase;

         // start over and do a fresh synchronous CMS cycle

         _collectorState = Resetting; // skip to reset to start new cycle

         reset(false /* == !asynch */);

         *should_start_over = true;

       } // else we can continue a possibly ongoing current cycle

     }

   }

 }

 // A work method used by the foreground collector to do

 // a mark-sweep-compact.

 void CMSCollector::do_compaction_work(bool clear_all_soft_refs) {

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   TraceTime t("CMS:MSC ", PrintGCDetails && Verbose, true, gclog_or_tty);

   if (PrintGC && Verbose && !(GCCause::is_user_requested_gc(gch->gc_cause()))) {

     gclog_or_tty->print_cr("Compact ConcurrentMarkSweepGeneration after %d "

       "collections passed to foreground collector", _full_gcs_since_conc_gc);

   }

   // Sample collection interval time and reset for collection pause.

   if (UseAdaptiveSizePolicy) {

     size_policy()->msc_collection_begin();

   }

   // Temporarily widen the span of the weak reference processing to

   // the entire heap.

   MemRegion new_span(GenCollectedHeap::heap()->reserved_region());

   ReferenceProcessorSpanMutator x(ref_processor(), new_span);

   // Temporarily, clear the "is_alive_non_header" field of the

   // reference processor.

   ReferenceProcessorIsAliveMutator y(ref_processor(), NULL);

   // Temporarily make reference _processing_ single threaded (non-MT).

   ReferenceProcessorMTProcMutator z(ref_processor(), false);

   // Temporarily make refs discovery atomic

   ReferenceProcessorAtomicMutator w(ref_processor(), true);

   ref_processor()->set_enqueuing_is_done(false);

   ref_processor()->enable_discovery();

   ref_processor()->setup_policy(clear_all_soft_refs);

   // If an asynchronous collection finishes, the _modUnionTable is

   // all clear.  If we are assuming the collection from an asynchronous

   // collection, clear the _modUnionTable.

   assert(_collectorState != Idling || _modUnionTable.isAllClear(),

     "_modUnionTable should be clear if the baton was not passed");

   _modUnionTable.clear_all();

   // We must adjust the allocation statistics being maintained

   // in the free list space. We do so by reading and clearing

   // the sweep timer and updating the block flux rate estimates below.

   assert(!_intra_sweep_timer.is_active(), "_intra_sweep_timer should be inactive");

   if (_inter_sweep_timer.is_active()) {

     _inter_sweep_timer.stop();

     // Note that we do not use this sample to update the _inter_sweep_estimate.

     _cmsGen->cmsSpace()->beginSweepFLCensus((float)(_inter_sweep_timer.seconds()),

                                             _inter_sweep_estimate.padded_average(),

                                             _intra_sweep_estimate.padded_average());

   }

   {

     TraceCMSMemoryManagerStats();

   }

   GenMarkSweep::invoke_at_safepoint(_cmsGen->level(),

     ref_processor(), clear_all_soft_refs);

   #ifdef ASSERT

     CompactibleFreeListSpace* cms_space = _cmsGen->cmsSpace();

     size_t free_size = cms_space->free();

     assert(free_size ==

            pointer_delta(cms_space->end(), cms_space->compaction_top())

            * HeapWordSize,

       "All the free space should be compacted into one chunk at top");

     assert(cms_space->dictionary()->totalChunkSize(

                                       debug_only(cms_space->freelistLock())) == 0 ||

            cms_space->totalSizeInIndexedFreeLists() == 0,

       "All the free space should be in a single chunk");

     size_t num = cms_space->totalCount();

     assert((free_size == 0 && num == 0) ||

            (free_size > 0  && (num == 1 || num == 2)),

          "There should be at most 2 free chunks after compaction");

   #endif // ASSERT

   _collectorState = Resetting;

   assert(_restart_addr == NULL,

          "Should have been NULL'd before baton was passed");

   reset(false /* == !asynch */);

   _cmsGen->reset_after_compaction();

   _concurrent_cycles_since_last_unload = 0;

   if (verifying() && !should_unload_classes()) {

     perm_gen_verify_bit_map()->clear_all();

   }

   // Clear any data recorded in the PLAB chunk arrays.

   if (_survivor_plab_array != NULL) {

     reset_survivor_plab_arrays();

   }

   // Adjust the per-size allocation stats for the next epoch.

   _cmsGen->cmsSpace()->endSweepFLCensus(sweep_count() /* fake */);

   // Restart the "inter sweep timer" for the next epoch.

   _inter_sweep_timer.reset();

   _inter_sweep_timer.start();

   // Sample collection pause time and reset for collection interval.

   if (UseAdaptiveSizePolicy) {

     size_policy()->msc_collection_end(gch->gc_cause());

   }

   // For a mark-sweep-compact, compute_new_size() will be called

   // in the heap's do_collection() method.

 }

 // A work method used by the foreground collector to do

 // a mark-sweep, after taking over from a possibly on-going

 // concurrent mark-sweep collection.

 void CMSCollector::do_mark_sweep_work(bool clear_all_soft_refs,

   CollectorState first_state, bool should_start_over) {

   if (PrintGC && Verbose) {

     gclog_or_tty->print_cr("Pass concurrent collection to foreground "

       "collector with count %d",

       _full_gcs_since_conc_gc);

   }

   switch (_collectorState) {

     case Idling:

       if (first_state == Idling || should_start_over) {

         // The background GC was not active, or should

         // restarted from scratch;  start the cycle.

         _collectorState = InitialMarking;

       }

       // If first_state was not Idling, then a background GC

       // was in progress and has now finished.  No need to do it

       // again.  Leave the state as Idling.

       break;

     case Precleaning:

       // In the foreground case don't do the precleaning since

       // it is not done concurrently and there is extra work

       // required.

       _collectorState = FinalMarking;

   }

   if (PrintGCDetails &&

       (_collectorState > Idling ||

        !GCCause::is_user_requested_gc(GenCollectedHeap::heap()->gc_cause()))) {

     gclog_or_tty->print(" (concurrent mode failure)");

   }

   collect_in_foreground(clear_all_soft_refs);

   // For a mark-sweep, compute_new_size() will be called

   // in the heap's do_collection() method.

 }

 void CMSCollector::getFreelistLocks() const {

   // Get locks for all free lists in all generations that this

   // collector is responsible for

   _cmsGen->freelistLock()->lock_without_safepoint_check();

   _permGen->freelistLock()->lock_without_safepoint_check();

 }

 void CMSCollector::releaseFreelistLocks() const {

   // Release locks for all free lists in all generations that this

   // collector is responsible for

   _cmsGen->freelistLock()->unlock();

   _permGen->freelistLock()->unlock();

 }

 bool CMSCollector::haveFreelistLocks() const {

   // Check locks for all free lists in all generations that this

   // collector is responsible for

   assert_lock_strong(_cmsGen->freelistLock());

   assert_lock_strong(_permGen->freelistLock());

   PRODUCT_ONLY(ShouldNotReachHere());

   return true;

 }

 // A utility class that is used by the CMS collector to

 // temporarily "release" the foreground collector from its

 // usual obligation to wait for the background collector to

 // complete an ongoing phase before proceeding.

 class ReleaseForegroundGC: public StackObj {

  private:

   CMSCollector* _c;

  public:

   ReleaseForegroundGC(CMSCollector* c) : _c(c) {

     assert(_c->_foregroundGCShouldWait, "Else should not need to call");

     MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

     // allow a potentially blocked foreground collector to proceed

     _c->_foregroundGCShouldWait = false;

     if (_c->_foregroundGCIsActive) {

       CGC_lock->notify();

     }

     assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),

            "Possible deadlock");

   }

   ~ReleaseForegroundGC() {

     assert(!_c->_foregroundGCShouldWait, "Usage protocol violation?");

     MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

     _c->_foregroundGCShouldWait = true;

   }

 };

 // There are separate collect_in_background and collect_in_foreground because of

 // the different locking requirements of the background collector and the

 // foreground collector.  There was originally an attempt to share

 // one "collect" method between the background collector and the foreground

 // collector but the if-then-else required made it cleaner to have

 // separate methods.

 void CMSCollector::collect_in_background(bool clear_all_soft_refs) {

   assert(Thread::current()->is_ConcurrentGC_thread(),

     "A CMS asynchronous collection is only allowed on a CMS thread.");

   GenCollectedHeap* gch = GenCollectedHeap::heap();

   {

     bool safepoint_check = Mutex::_no_safepoint_check_flag;

     MutexLockerEx hl(Heap_lock, safepoint_check);

     FreelistLocker fll(this);

     MutexLockerEx x(CGC_lock, safepoint_check);

     if (_foregroundGCIsActive || !UseAsyncConcMarkSweepGC) {

       // The foreground collector is active or we're

       // not using asynchronous collections.  Skip this

       // background collection.

       assert(!_foregroundGCShouldWait, "Should be clear");

       return;

     } else {

       assert(_collectorState == Idling, "Should be idling before start.");

       _collectorState = InitialMarking;

       // Reset the expansion cause, now that we are about to begin

       // a new cycle.

       clear_expansion_cause();

     }

     // Decide if we want to enable class unloading as part of the

     // ensuing concurrent GC cycle.

     update_should_unload_classes();

     _full_gc_requested = false;           // acks all outstanding full gc requests

     // Signal that we are about to start a collection

     gch->increment_total_full_collections();  // ... starting a collection cycle

     _collection_count_start = gch->total_full_collections();

   }

   // Used for PrintGC

   size_t prev_used;

   if (PrintGC && Verbose) {

     prev_used = _cmsGen->used(); // XXXPERM

   }

   // The change of the collection state is normally done at this level;

   // the exceptions are phases that are executed while the world is

   // stopped.  For those phases the change of state is done while the

   // world is stopped.  For baton passing purposes this allows the

   // background collector to finish the phase and change state atomically.

   // The foreground collector cannot wait on a phase that is done

   // while the world is stopped because the foreground collector already

   // has the world stopped and would deadlock.

   while (_collectorState != Idling) {

     if (TraceCMSState) {

       gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",

         Thread::current(), _collectorState);

     }

     // The foreground collector

     //   holds the Heap_lock throughout its collection.

     //   holds the CMS token (but not the lock)

     //     except while it is waiting for the background collector to yield.

     //

     // The foreground collector should be blocked (not for long)

     //   if the background collector is about to start a phase

     //   executed with world stopped.  If the background

     //   collector has already started such a phase, the

     //   foreground collector is blocked waiting for the

     //   Heap_lock.  The stop-world phases (InitialMarking and FinalMarking)

     //   are executed in the VM thread.

     //

     // The locking order is

     //   PendingListLock (PLL)  -- if applicable (FinalMarking)

     //   Heap_lock  (both this & PLL locked in VM_CMS_Operation::prologue())

     //   CMS token  (claimed in

     //                stop_world_and_do() -->

     //                  safepoint_synchronize() -->

     //                    CMSThread::synchronize())

     {

       // Check if the FG collector wants us to yield.

       CMSTokenSync x(true); // is cms thread

       if (waitForForegroundGC()) {

         // We yielded to a foreground GC, nothing more to be

         // done this round.

         assert(_foregroundGCShouldWait == false, "We set it to false in "

                "waitForForegroundGC()");

         if (TraceCMSState) {

           gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT

             " exiting collection CMS state %d",

             Thread::current(), _collectorState);

         }

         return;

       } else {

         // The background collector can run but check to see if the

         // foreground collector has done a collection while the

         // background collector was waiting to get the CGC_lock

         // above.  If yes, break so that _foregroundGCShouldWait

         // is cleared before returning.

         if (_collectorState == Idling) {

           break;

         }

       }

     }

     assert(_foregroundGCShouldWait, "Foreground collector, if active, "

       "should be waiting");

     switch (_collectorState) {

       case InitialMarking:

         {

           ReleaseForegroundGC x(this);

           stats().record_cms_begin();

           VM_CMS_Initial_Mark initial_mark_op(this);

           VMThread::execute(&initial_mark_op);

         }

         // The collector state may be any legal state at this point

         // since the background collector may have yielded to the

         // foreground collector.

         break;

       case Marking:

         // initial marking in checkpointRootsInitialWork has been completed

         if (markFromRoots(true)) { // we were successful

           assert(_collectorState == Precleaning, "Collector state should "

             "have changed");

         } else {

           assert(_foregroundGCIsActive, "Internal state inconsistency");

         }

         break;

       case Precleaning:

         if (UseAdaptiveSizePolicy) {

           size_policy()->concurrent_precleaning_begin();

         }

         // marking from roots in markFromRoots has been completed

         preclean();

         if (UseAdaptiveSizePolicy) {

           size_policy()->concurrent_precleaning_end();

         }

         assert(_collectorState == AbortablePreclean ||

                _collectorState == FinalMarking,

                "Collector state should have changed");

         break;

       case AbortablePreclean:

         if (UseAdaptiveSizePolicy) {

         size_policy()->concurrent_phases_resume();

         }

         abortable_preclean();

         if (UseAdaptiveSizePolicy) {

           size_policy()->concurrent_precleaning_end();

         }

         assert(_collectorState == FinalMarking, "Collector state should "

           "have changed");

         break;

       case FinalMarking:

         {

           ReleaseForegroundGC x(this);

           VM_CMS_Final_Remark final_remark_op(this);

           VMThread::execute(&final_remark_op);

         }

         assert(_foregroundGCShouldWait, "block post-condition");

         break;

       case Sweeping:

         if (UseAdaptiveSizePolicy) {

           size_policy()->concurrent_sweeping_begin();

         }

         // final marking in checkpointRootsFinal has been completed

         sweep(true);

         assert(_collectorState == Resizing, "Collector state change "

           "to Resizing must be done under the free_list_lock");

         _full_gcs_since_conc_gc = 0;

         // Stop the timers for adaptive size policy for the concurrent phases

         if (UseAdaptiveSizePolicy) {

           size_policy()->concurrent_sweeping_end();

           size_policy()->concurrent_phases_end(gch->gc_cause(),

                                              gch->prev_gen(_cmsGen)->capacity(),

                                              _cmsGen->free());

         }

       case Resizing: {

         // Sweeping has been completed...

         // At this point the background collection has completed.

         // Don't move the call to compute_new_size() down

         // into code that might be executed if the background

         // collection was preempted.

         {

           ReleaseForegroundGC x(this);   // unblock FG collection

           MutexLockerEx       y(Heap_lock, Mutex::_no_safepoint_check_flag);

           CMSTokenSync        z(true);   // not strictly needed.

           if (_collectorState == Resizing) {

             compute_new_size();

             _collectorState = Resetting;

           } else {

             assert(_collectorState == Idling, "The state should only change"

                    " because the foreground collector has finished the collection");

           }

         }

         break;

       }

       case Resetting:

         // CMS heap resizing has been completed

         reset(true);

         assert(_collectorState == Idling, "Collector state should "

           "have changed");

         stats().record_cms_end();

         // Don't move the concurrent_phases_end() and compute_new_size()

         // calls to here because a preempted background collection

         // has it's state set to "Resetting".

         break;

       case Idling:

       default:

         ShouldNotReachHere();

         break;

     }

     if (TraceCMSState) {

       gclog_or_tty->print_cr("  Thread " INTPTR_FORMAT " done - next CMS state %d",

         Thread::current(), _collectorState);

     }

     assert(_foregroundGCShouldWait, "block post-condition");

   }

   // Should this be in gc_epilogue?

   collector_policy()->counters()->update_counters();

   {

     // Clear _foregroundGCShouldWait and, in the event that the

     // foreground collector is waiting, notify it, before

     // returning.

     MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

     _foregroundGCShouldWait = false;

     if (_foregroundGCIsActive) {

       CGC_lock->notify();

     }

     assert(!ConcurrentMarkSweepThread::cms_thread_has_cms_token(),

            "Possible deadlock");

   }

   if (TraceCMSState) {

     gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT

       " exiting collection CMS state %d",

       Thread::current(), _collectorState);

   }

   if (PrintGC && Verbose) {

     _cmsGen->print_heap_change(prev_used);

   }

 }

 void CMSCollector::collect_in_foreground(bool clear_all_soft_refs) {

   assert(_foregroundGCIsActive && !_foregroundGCShouldWait,

          "Foreground collector should be waiting, not executing");

   assert(Thread::current()->is_VM_thread(), "A foreground collection"

     "may only be done by the VM Thread with the world stopped");

   assert(ConcurrentMarkSweepThread::vm_thread_has_cms_token(),

          "VM thread should have CMS token");

   NOT_PRODUCT(TraceTime t("CMS:MS (foreground) ", PrintGCDetails && Verbose,

     true, gclog_or_tty);)

   if (UseAdaptiveSizePolicy) {

     size_policy()->ms_collection_begin();

   }

   COMPILER2_PRESENT(DerivedPointerTableDeactivate dpt_deact);

   HandleMark hm;  // Discard invalid handles created during verification

   if (VerifyBeforeGC &&

       GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

     Universe::verify(true);

   }

   // Snapshot the soft reference policy to be used in this collection cycle.

   ref_processor()->setup_policy(clear_all_soft_refs);

   bool init_mark_was_synchronous = false; // until proven otherwise

   while (_collectorState != Idling) {

     if (TraceCMSState) {

       gclog_or_tty->print_cr("Thread " INTPTR_FORMAT " in CMS state %d",

         Thread::current(), _collectorState);

     }

     switch (_collectorState) {

       case InitialMarking:

         init_mark_was_synchronous = true;  // fact to be exploited in re-mark

         checkpointRootsInitial(false);

         assert(_collectorState == Marking, "Collector state should have changed"

           " within checkpointRootsInitial()");

         break;

       case Marking:

         // initial marking in checkpointRootsInitialWork has been completed

         if (VerifyDuringGC &&

             GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

           gclog_or_tty->print("Verify before initial mark: ");

           Universe::verify(true);

         }

         {

           bool res = markFromRoots(false);

           assert(res && _collectorState == FinalMarking, "Collector state should "

             "have changed");

           break;

         }

       case FinalMarking:

         if (VerifyDuringGC &&

             GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

           gclog_or_tty->print("Verify before re-mark: ");

           Universe::verify(true);

         }

         checkpointRootsFinal(false, clear_all_soft_refs,

                              init_mark_was_synchronous);

         assert(_collectorState == Sweeping, "Collector state should not "

           "have changed within checkpointRootsFinal()");

         break;

       case Sweeping:

         // final marking in checkpointRootsFinal has been completed

         if (VerifyDuringGC &&

             GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

           gclog_or_tty->print("Verify before sweep: ");

           Universe::verify(true);

         }

         sweep(false);

         assert(_collectorState == Resizing, "Incorrect state");

         break;

       case Resizing: {

         // Sweeping has been completed; the actual resize in this case

         // is done separately; nothing to be done in this state.

         _collectorState = Resetting;

         break;

       }

       case Resetting:

         // The heap has been resized.

         if (VerifyDuringGC &&

             GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

           gclog_or_tty->print("Verify before reset: ");

           Universe::verify(true);

         }

         reset(false);

         assert(_collectorState == Idling, "Collector state should "

           "have changed");

         break;

       case Precleaning:

       case AbortablePreclean:

         // Elide the preclean phase

         _collectorState = FinalMarking;

         break;

       default:

         ShouldNotReachHere();

     }

     if (TraceCMSState) {

       gclog_or_tty->print_cr("  Thread " INTPTR_FORMAT " done - next CMS state %d",

         Thread::current(), _collectorState);

     }

   }

   if (UseAdaptiveSizePolicy) {

     GenCollectedHeap* gch = GenCollectedHeap::heap();

     size_policy()->ms_collection_end(gch->gc_cause());

   }

   if (VerifyAfterGC &&

       GenCollectedHeap::heap()->total_collections() >= VerifyGCStartAt) {

     Universe::verify(true);

   }

   if (TraceCMSState) {

     gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT

       " exiting collection CMS state %d",

       Thread::current(), _collectorState);

   }

 }

 bool CMSCollector::waitForForegroundGC() {

   bool res = false;

   assert(ConcurrentMarkSweepThread::cms_thread_has_cms_token(),

          "CMS thread should have CMS token");

   // Block the foreground collector until the

   // background collectors decides whether to

   // yield.

   MutexLockerEx x(CGC_lock, Mutex::_no_safepoint_check_flag);

   _foregroundGCShouldWait = true;

   if (_foregroundGCIsActive) {

     // The background collector yields to the

     // foreground collector and returns a value

     // indicating that it has yielded.  The foreground

     // collector can proceed.

     res = true;

     _foregroundGCShouldWait = false;

     ConcurrentMarkSweepThread::clear_CMS_flag(

       ConcurrentMarkSweepThread::CMS_cms_has_token);

     ConcurrentMarkSweepThread::set_CMS_flag(

       ConcurrentMarkSweepThread::CMS_cms_wants_token);

     // Get a possibly blocked foreground thread going

     CGC_lock->notify();

     if (TraceCMSState) {

       gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " waiting at CMS state %d",

         Thread::current(), _collectorState);

     }

     while (_foregroundGCIsActive) {

       CGC_lock->wait(Mutex::_no_safepoint_check_flag);

     }

     ConcurrentMarkSweepThread::set_CMS_flag(

       ConcurrentMarkSweepThread::CMS_cms_has_token);

     ConcurrentMarkSweepThread::clear_CMS_flag(

       ConcurrentMarkSweepThread::CMS_cms_wants_token);

   }

   if (TraceCMSState) {

     gclog_or_tty->print_cr("CMS Thread " INTPTR_FORMAT " continuing at CMS state %d",

       Thread::current(), _collectorState);

   }

   return res;

 }

 // Because of the need to lock the free lists and other structures in

 // the collector, common to all the generations that the collector is

 // collecting, we need the gc_prologues of individual CMS generations

 // delegate to their collector. It may have been simpler had the

 // current infrastructure allowed one to call a prologue on a

 // collector. In the absence of that we have the generation's

 // prologue delegate to the collector, which delegates back

 // some "local" work to a worker method in the individual generations

 // that it's responsible for collecting, while itself doing any

 // work common to all generations it's responsible for. A similar

 // comment applies to the  gc_epilogue()'s.

 // The role of the varaible _between_prologue_and_epilogue is to

 // enforce the invocation protocol.

 void CMSCollector::gc_prologue(bool full) {

   // Call gc_prologue_work() for each CMSGen and PermGen that

   // we are responsible for.

   // The following locking discipline assumes that we are only called

   // when the world is stopped.

   assert(SafepointSynchronize::is_at_safepoint(), "world is stopped assumption");

   // The CMSCollector prologue must call the gc_prologues for the

   // "generations" (including PermGen if any) that it's responsible

   // for.

   assert(   Thread::current()->is_VM_thread()

          || (   CMSScavengeBeforeRemark

              && Thread::current()->is_ConcurrentGC_thread()),

          "Incorrect thread type for prologue execution");

   if (_between_prologue_and_epilogue) {

     // We have already been invoked; this is a gc_prologue delegation

     // from yet another CMS generation that we are responsible for, just

     // ignore it since all relevant work has already been done.

     return;

   }

   // set a bit saying prologue has been called; cleared in epilogue

   _between_prologue_and_epilogue = true;

   // Claim locks for common data structures, then call gc_prologue_work()

   // for each CMSGen and PermGen that we are responsible for.

   getFreelistLocks();   // gets free list locks on constituent spaces

   bitMapLock()->lock_without_safepoint_check();

   // Should call gc_prologue_work() for all cms gens we are responsible for

   bool registerClosure =    _collectorState >= Marking

                          && _collectorState < Sweeping;

   ModUnionClosure* muc = CollectedHeap::use_parallel_gc_threads() ?

                                                &_modUnionClosurePar

                                                : &_modUnionClosure;

   _cmsGen->gc_prologue_work(full, registerClosure, muc);

   _permGen->gc_prologue_work(full, registerClosure, muc);

   if (!full) {

     stats().record_gc0_begin();

   }

 }

 void ConcurrentMarkSweepGeneration::gc_prologue(bool full) {

   // Delegate to CMScollector which knows how to coordinate between

   // this and any other CMS generations that it is responsible for

   // collecting.

   collector()->gc_prologue(full);

 }

 // This is a "private" interface for use by this generation's CMSCollector.

 // Not to be called directly by any other entity (for instance,

 // GenCollectedHeap, which calls the "public" gc_prologue method above).

 void ConcurrentMarkSweepGeneration::gc_prologue_work(bool full,

   bool registerClosure, ModUnionClosure* modUnionClosure) {

   assert(!incremental_collection_failed(), "Shouldn't be set yet");