Mercurial > jdk8-mips64-public > hotspot / changeset

     1.1 --- a/src/share/vm/c1/c1_Runtime1.cpp	Tue Jul 01 09:03:55 2014 +0200
     1.2 +++ b/src/share/vm/c1/c1_Runtime1.cpp	Mon Jul 07 10:12:40 2014 +0200
     1.3 @@ -1018,6 +1018,7 @@
     1.4                n_copy->set_data((intx) (load_klass()));
     1.5              } else {
     1.6                assert(mirror() != NULL, "klass not set");
     1.7 +              // Don't need a G1 pre-barrier here since we assert above that data isn't an oop.
     1.8                n_copy->set_data(cast_from_oop<intx>(mirror()));
     1.9              }
    1.10  

     2.1 --- a/src/share/vm/ci/ciEnv.hpp	Tue Jul 01 09:03:55 2014 +0200
     2.2 +++ b/src/share/vm/ci/ciEnv.hpp	Mon Jul 07 10:12:40 2014 +0200
     2.3 @@ -184,6 +184,10 @@
     2.4      }
     2.5    }
     2.6  
     2.7 +  void ensure_metadata_alive(ciMetadata* m) {
     2.8 +    _factory->ensure_metadata_alive(m);
     2.9 +  }
    2.10 +
    2.11    ciInstance* get_instance(oop o) {
    2.12      if (o == NULL) return NULL;
    2.13      return get_object(o)->as_instance();

     3.1 --- a/src/share/vm/ci/ciKlass.hpp	Tue Jul 01 09:03:55 2014 +0200
     3.2 +++ b/src/share/vm/ci/ciKlass.hpp	Mon Jul 07 10:12:40 2014 +0200
     3.3 @@ -43,6 +43,7 @@
     3.4    friend class ciMethod;
     3.5    friend class ciMethodData;
     3.6    friend class ciObjArrayKlass;
     3.7 +  friend class ciReceiverTypeData;
     3.8  
     3.9  private:
    3.10    ciSymbol* _name;

     4.1 --- a/src/share/vm/ci/ciMethodData.cpp	Tue Jul 01 09:03:55 2014 +0200
     4.2 +++ b/src/share/vm/ci/ciMethodData.cpp	Mon Jul 07 10:12:40 2014 +0200
     4.3 @@ -170,6 +170,7 @@
     4.4      Klass* k = data->as_ReceiverTypeData()->receiver(row);
     4.5      if (k != NULL) {
     4.6        ciKlass* klass = CURRENT_ENV->get_klass(k);
     4.7 +      CURRENT_ENV->ensure_metadata_alive(klass);
     4.8        set_receiver(row, klass);
     4.9      }
    4.10    }
    4.11 @@ -191,6 +192,7 @@
    4.12  void ciSpeculativeTrapData::translate_from(const ProfileData* data) {
    4.13    Method* m = data->as_SpeculativeTrapData()->method();
    4.14    ciMethod* ci_m = CURRENT_ENV->get_method(m);
    4.15 +  CURRENT_ENV->ensure_metadata_alive(ci_m);
    4.16    set_method(ci_m);
    4.17  }
    4.18  

     5.1 --- a/src/share/vm/ci/ciMethodData.hpp	Tue Jul 01 09:03:55 2014 +0200
     5.2 +++ b/src/share/vm/ci/ciMethodData.hpp	Mon Jul 07 10:12:40 2014 +0200
     5.3 @@ -70,6 +70,7 @@
     5.4      Klass* v = TypeEntries::valid_klass(k);
     5.5      if (v != NULL) {
     5.6        ciKlass* klass = CURRENT_ENV->get_klass(v);
     5.7 +      CURRENT_ENV->ensure_metadata_alive(klass);
     5.8        return with_status(klass, k);
     5.9      }
    5.10      return with_status(NULL, k);

     6.1 --- a/src/share/vm/ci/ciObjectFactory.cpp	Tue Jul 01 09:03:55 2014 +0200
     6.2 +++ b/src/share/vm/ci/ciObjectFactory.cpp	Mon Jul 07 10:12:40 2014 +0200
     6.3 @@ -46,6 +46,9 @@
     6.4  #include "oops/oop.inline.hpp"
     6.5  #include "oops/oop.inline2.hpp"
     6.6  #include "runtime/fieldType.hpp"
     6.7 +#if INCLUDE_ALL_GCS
     6.8 +# include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
     6.9 +#endif
    6.10  
    6.11  // ciObjectFactory
    6.12  //
    6.13 @@ -374,6 +377,37 @@
    6.14    return NULL;
    6.15  }
    6.16  
    6.17 +// ------------------------------------------------------------------
    6.18 +// ciObjectFactory::ensure_metadata_alive
    6.19 +//
    6.20 +// Ensure that the metadata wrapped by the ciMetadata is kept alive by GC.
    6.21 +// This is primarily useful for metadata which is considered as weak roots
    6.22 +// by the GC but need to be strong roots if reachable from a current compilation.
    6.23 +//
    6.24 +void ciObjectFactory::ensure_metadata_alive(ciMetadata* m) {
    6.25 +  ASSERT_IN_VM; // We're handling raw oops here.
    6.26 +
    6.27 +#if INCLUDE_ALL_GCS
    6.28 +  if (!UseG1GC) {
    6.29 +    return;
    6.30 +  }
    6.31 +  Klass* metadata_owner_klass;
    6.32 +  if (m->is_klass()) {
    6.33 +    metadata_owner_klass = m->as_klass()->get_Klass();
    6.34 +  } else if (m->is_method()) {
    6.35 +    metadata_owner_klass = m->as_method()->get_Method()->constants()->pool_holder();
    6.36 +  } else {
    6.37 +    fatal("Not implemented for other types of metadata");
    6.38 +  }
    6.39 +
    6.40 +  oop metadata_holder = metadata_owner_klass->klass_holder();
    6.41 +  if (metadata_holder != NULL) {
    6.42 +    G1SATBCardTableModRefBS::enqueue(metadata_holder);
    6.43 +  }
    6.44 +
    6.45 +#endif
    6.46 +}
    6.47 +
    6.48  //------------------------------------------------------------------
    6.49  // ciObjectFactory::get_unloaded_method
    6.50  //

     7.1 --- a/src/share/vm/ci/ciObjectFactory.hpp	Tue Jul 01 09:03:55 2014 +0200
     7.2 +++ b/src/share/vm/ci/ciObjectFactory.hpp	Mon Jul 07 10:12:40 2014 +0200
     7.3 @@ -75,6 +75,8 @@
     7.4    ciObject* create_new_object(oop o);
     7.5    ciMetadata* create_new_object(Metadata* o);
     7.6  
     7.7 +  void ensure_metadata_alive(ciMetadata* m);
     7.8 +
     7.9    static bool is_equal(NonPermObject* p, oop key) {
    7.10      return p->object()->get_oop() == key;
    7.11    }

     8.1 --- a/src/share/vm/classfile/classLoaderData.cpp	Tue Jul 01 09:03:55 2014 +0200
     8.2 +++ b/src/share/vm/classfile/classLoaderData.cpp	Mon Jul 07 10:12:40 2014 +0200
     8.3 @@ -321,6 +321,27 @@
     8.4    }
     8.5  }
     8.6  
     8.7 +#ifdef ASSERT
     8.8 +class AllAliveClosure : public OopClosure {
     8.9 +  BoolObjectClosure* _is_alive_closure;
    8.10 +  bool _found_dead;
    8.11 + public:
    8.12 +  AllAliveClosure(BoolObjectClosure* is_alive_closure) : _is_alive_closure(is_alive_closure), _found_dead(false) {}
    8.13 +  template <typename T> void do_oop_work(T* p) {
    8.14 +    T heap_oop = oopDesc::load_heap_oop(p);
    8.15 +    if (!oopDesc::is_null(heap_oop)) {
    8.16 +      oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
    8.17 +      if (!_is_alive_closure->do_object_b(obj)) {
    8.18 +        _found_dead = true;
    8.19 +      }
    8.20 +    }
    8.21 +  }
    8.22 +  void do_oop(oop* p)       { do_oop_work<oop>(p); }
    8.23 +  void do_oop(narrowOop* p) { do_oop_work<narrowOop>(p); }
    8.24 +  bool found_dead()         { return _found_dead; }
    8.25 +};
    8.26 +#endif
    8.27 +
    8.28  oop ClassLoaderData::keep_alive_object() const {
    8.29    assert(!keep_alive(), "Don't use with CLDs that are artificially kept alive");
    8.30    return is_anonymous() ? _klasses->java_mirror() : class_loader();
    8.31 @@ -330,7 +351,15 @@
    8.32    bool alive = keep_alive() // null class loader and incomplete anonymous klasses.
    8.33        || is_alive_closure->do_object_b(keep_alive_object());
    8.34  
    8.35 -  assert(!alive || claimed(), "must be claimed");
    8.36 +#ifdef ASSERT
    8.37 +  if (alive) {
    8.38 +    AllAliveClosure all_alive_closure(is_alive_closure);
    8.39 +    KlassToOopClosure klass_closure(&all_alive_closure);
    8.40 +    const_cast<ClassLoaderData*>(this)->oops_do(&all_alive_closure, &klass_closure, false);
    8.41 +    assert(!all_alive_closure.found_dead(), err_msg("Found dead oop in alive cld: " PTR_FORMAT, p2i(this)));
    8.42 +  }
    8.43 +#endif
    8.44 +
    8.45    return alive;
    8.46  }
    8.47  
    8.48 @@ -609,9 +638,36 @@
    8.49  
    8.50  void ClassLoaderDataGraph::always_strong_oops_do(OopClosure* f, KlassClosure* klass_closure, bool must_claim) {
    8.51    if (ClassUnloading) {
    8.52 -    ClassLoaderDataGraph::keep_alive_oops_do(f, klass_closure, must_claim);
    8.53 +    keep_alive_oops_do(f, klass_closure, must_claim);
    8.54    } else {
    8.55 -    ClassLoaderDataGraph::oops_do(f, klass_closure, must_claim);
    8.56 +    oops_do(f, klass_closure, must_claim);
    8.57 +  }
    8.58 +}
    8.59 +
    8.60 +void ClassLoaderDataGraph::cld_do(CLDClosure* cl) {
    8.61 +  for (ClassLoaderData* cld = _head; cl != NULL && cld != NULL; cld = cld->next()) {
    8.62 +    cl->do_cld(cld);
    8.63 +  }
    8.64 +}
    8.65 +
    8.66 +void ClassLoaderDataGraph::roots_cld_do(CLDClosure* strong, CLDClosure* weak) {
    8.67 +  for (ClassLoaderData* cld = _head;  cld != NULL; cld = cld->_next) {
    8.68 +    CLDClosure* closure = cld->keep_alive() ? strong : weak;
    8.69 +    if (closure != NULL) {
    8.70 +      closure->do_cld(cld);
    8.71 +    }
    8.72 +  }
    8.73 +}
    8.74 +
    8.75 +void ClassLoaderDataGraph::keep_alive_cld_do(CLDClosure* cl) {
    8.76 +  roots_cld_do(cl, NULL);
    8.77 +}
    8.78 +
    8.79 +void ClassLoaderDataGraph::always_strong_cld_do(CLDClosure* cl) {
    8.80 +  if (ClassUnloading) {
    8.81 +    keep_alive_cld_do(cl);
    8.82 +  } else {
    8.83 +    cld_do(cl);
    8.84    }
    8.85  }
    8.86  
    8.87 @@ -666,6 +722,16 @@
    8.88    return array;
    8.89  }
    8.90  
    8.91 +bool ClassLoaderDataGraph::unload_list_contains(const void* x) {
    8.92 +  assert(SafepointSynchronize::is_at_safepoint(), "only safe to call at safepoint");
    8.93 +  for (ClassLoaderData* cld = _unloading; cld != NULL; cld = cld->next()) {
    8.94 +    if (cld->metaspace_or_null() != NULL && cld->metaspace_or_null()->contains(x)) {
    8.95 +      return true;
    8.96 +    }
    8.97 +  }
    8.98 +  return false;
    8.99 +}
   8.100 +
   8.101  #ifndef PRODUCT
   8.102  bool ClassLoaderDataGraph::contains_loader_data(ClassLoaderData* loader_data) {
   8.103    for (ClassLoaderData* data = _head; data != NULL; data = data->next()) {
   8.104 @@ -786,6 +852,60 @@
   8.105    return _rw_metaspace;
   8.106  }
   8.107  
   8.108 +ClassLoaderDataGraphKlassIteratorAtomic::ClassLoaderDataGraphKlassIteratorAtomic()
   8.109 +    : _next_klass(NULL) {
   8.110 +  ClassLoaderData* cld = ClassLoaderDataGraph::_head;
   8.111 +  Klass* klass = NULL;
   8.112 +
   8.113 +  // Find the first klass in the CLDG.
   8.114 +  while (cld != NULL) {
   8.115 +    klass = cld->_klasses;
   8.116 +    if (klass != NULL) {
   8.117 +      _next_klass = klass;
   8.118 +      return;
   8.119 +    }
   8.120 +    cld = cld->next();
   8.121 +  }
   8.122 +}
   8.123 +
   8.124 +Klass* ClassLoaderDataGraphKlassIteratorAtomic::next_klass_in_cldg(Klass* klass) {
   8.125 +  Klass* next = klass->next_link();
   8.126 +  if (next != NULL) {
   8.127 +    return next;
   8.128 +  }
   8.129 +
   8.130 +  // No more klasses in the current CLD. Time to find a new CLD.
   8.131 +  ClassLoaderData* cld = klass->class_loader_data();
   8.132 +  while (next == NULL) {
   8.133 +    cld = cld->next();
   8.134 +    if (cld == NULL) {
   8.135 +      break;
   8.136 +    }
   8.137 +    next = cld->_klasses;
   8.138 +  }
   8.139 +
   8.140 +  return next;
   8.141 +}
   8.142 +
   8.143 +Klass* ClassLoaderDataGraphKlassIteratorAtomic::next_klass() {
   8.144 +  Klass* head = (Klass*)_next_klass;
   8.145 +
   8.146 +  while (head != NULL) {
   8.147 +    Klass* next = next_klass_in_cldg(head);
   8.148 +
   8.149 +    Klass* old_head = (Klass*)Atomic::cmpxchg_ptr(next, &_next_klass, head);
   8.150 +
   8.151 +    if (old_head == head) {
   8.152 +      return head; // Won the CAS.
   8.153 +    }
   8.154 +
   8.155 +    head = old_head;
   8.156 +  }
   8.157 +
   8.158 +  // Nothing more for the iterator to hand out.
   8.159 +  assert(head == NULL, err_msg("head is " PTR_FORMAT ", expected not null:", p2i(head)));
   8.160 +  return NULL;
   8.161 +}
   8.162  
   8.163  ClassLoaderDataGraphMetaspaceIterator::ClassLoaderDataGraphMetaspaceIterator() {
   8.164    _data = ClassLoaderDataGraph::_head;

     9.1 --- a/src/share/vm/classfile/classLoaderData.hpp	Tue Jul 01 09:03:55 2014 +0200
     9.2 +++ b/src/share/vm/classfile/classLoaderData.hpp	Mon Jul 07 10:12:40 2014 +0200
     9.3 @@ -31,7 +31,6 @@
     9.4  #include "memory/metaspaceCounters.hpp"
     9.5  #include "runtime/mutex.hpp"
     9.6  #include "utilities/growableArray.hpp"
     9.7 -
     9.8  #if INCLUDE_TRACE
     9.9  # include "utilities/ticks.hpp"
    9.10  #endif
    9.11 @@ -59,6 +58,7 @@
    9.12  class ClassLoaderDataGraph : public AllStatic {
    9.13    friend class ClassLoaderData;
    9.14    friend class ClassLoaderDataGraphMetaspaceIterator;
    9.15 +  friend class ClassLoaderDataGraphKlassIteratorAtomic;
    9.16    friend class VMStructs;
    9.17   private:
    9.18    // All CLDs (except the null CLD) can be reached by walking _head->_next->...
    9.19 @@ -75,9 +75,16 @@
    9.20    static ClassLoaderData* find_or_create(Handle class_loader, TRAPS);
    9.21    static void purge();
    9.22    static void clear_claimed_marks();
    9.23 +  // oops do
    9.24    static void oops_do(OopClosure* f, KlassClosure* klass_closure, bool must_claim);
    9.25 +  static void keep_alive_oops_do(OopClosure* blk, KlassClosure* klass_closure, bool must_claim);
    9.26    static void always_strong_oops_do(OopClosure* blk, KlassClosure* klass_closure, bool must_claim);
    9.27 -  static void keep_alive_oops_do(OopClosure* blk, KlassClosure* klass_closure, bool must_claim);
    9.28 +  // cld do
    9.29 +  static void cld_do(CLDClosure* cl);
    9.30 +  static void roots_cld_do(CLDClosure* strong, CLDClosure* weak);
    9.31 +  static void keep_alive_cld_do(CLDClosure* cl);
    9.32 +  static void always_strong_cld_do(CLDClosure* cl);
    9.33 +  // klass do
    9.34    static void classes_do(KlassClosure* klass_closure);
    9.35    static void classes_do(void f(Klass* const));
    9.36    static void loaded_classes_do(KlassClosure* klass_closure);
    9.37 @@ -102,6 +109,7 @@
    9.38    static void dump() { dump_on(tty); }
    9.39    static void verify();
    9.40  
    9.41 +  static bool unload_list_contains(const void* x);
    9.42  #ifndef PRODUCT
    9.43    static bool contains_loader_data(ClassLoaderData* loader_data);
    9.44  #endif
    9.45 @@ -134,6 +142,7 @@
    9.46    };
    9.47  
    9.48    friend class ClassLoaderDataGraph;
    9.49 +  friend class ClassLoaderDataGraphKlassIteratorAtomic;
    9.50    friend class ClassLoaderDataGraphMetaspaceIterator;
    9.51    friend class MetaDataFactory;
    9.52    friend class Method;
    9.53 @@ -195,7 +204,6 @@
    9.54  
    9.55    void unload();
    9.56    bool keep_alive() const       { return _keep_alive; }
    9.57 -  bool is_alive(BoolObjectClosure* is_alive_closure) const;
    9.58    void classes_do(void f(Klass*));
    9.59    void loaded_classes_do(KlassClosure* klass_closure);
    9.60    void classes_do(void f(InstanceKlass*));
    9.61 @@ -207,6 +215,9 @@
    9.62    MetaWord* allocate(size_t size);
    9.63  
    9.64   public:
    9.65 +
    9.66 +  bool is_alive(BoolObjectClosure* is_alive_closure) const;
    9.67 +
    9.68    // Accessors
    9.69    Metaspace* metaspace_or_null() const     { return _metaspace; }
    9.70  
    9.71 @@ -290,6 +301,16 @@
    9.72    void initialize_shared_metaspaces();
    9.73  };
    9.74  
    9.75 +// An iterator that distributes Klasses to parallel worker threads.
    9.76 +class ClassLoaderDataGraphKlassIteratorAtomic : public StackObj {
    9.77 +  volatile Klass* _next_klass;
    9.78 + public:
    9.79 +  ClassLoaderDataGraphKlassIteratorAtomic();
    9.80 +  Klass* next_klass();
    9.81 + private:
    9.82 +  static Klass* next_klass_in_cldg(Klass* klass);
    9.83 +};
    9.84 +
    9.85  class ClassLoaderDataGraphMetaspaceIterator : public StackObj {
    9.86    ClassLoaderData* _data;
    9.87   public:

    10.1 --- a/src/share/vm/classfile/dictionary.cpp	Tue Jul 01 09:03:55 2014 +0200
    10.2 +++ b/src/share/vm/classfile/dictionary.cpp	Mon Jul 07 10:12:40 2014 +0200
    10.3 @@ -199,6 +199,26 @@
    10.4    return class_was_unloaded;
    10.5  }
    10.6  
    10.7 +void Dictionary::roots_oops_do(OopClosure* strong, OopClosure* weak) {
    10.8 +  // Skip the strong roots probe marking if the closures are the same.
    10.9 +  if (strong == weak) {
   10.10 +    oops_do(strong);
   10.11 +    return;
   10.12 +  }
   10.13 +
   10.14 +  for (int index = 0; index < table_size(); index++) {
   10.15 +    for (DictionaryEntry *probe = bucket(index);
   10.16 +                          probe != NULL;
   10.17 +                          probe = probe->next()) {
   10.18 +      Klass* e = probe->klass();
   10.19 +      ClassLoaderData* loader_data = probe->loader_data();
   10.20 +      if (is_strongly_reachable(loader_data, e)) {
   10.21 +        probe->set_strongly_reachable();
   10.22 +      }
   10.23 +    }
   10.24 +  }
   10.25 +  _pd_cache_table->roots_oops_do(strong, weak);
   10.26 +}
   10.27  
   10.28  void Dictionary::always_strong_oops_do(OopClosure* blk) {
   10.29    // Follow all system classes and temporary placeholders in dictionary; only
   10.30 @@ -490,6 +510,23 @@
   10.31    }
   10.32  }
   10.33  
   10.34 +void ProtectionDomainCacheTable::roots_oops_do(OopClosure* strong, OopClosure* weak) {
   10.35 +  for (int index = 0; index < table_size(); index++) {
   10.36 +    for (ProtectionDomainCacheEntry* probe = bucket(index);
   10.37 +                                     probe != NULL;
   10.38 +                                     probe = probe->next()) {
   10.39 +      if (probe->is_strongly_reachable()) {
   10.40 +        probe->reset_strongly_reachable();
   10.41 +        probe->oops_do(strong);
   10.42 +      } else {
   10.43 +        if (weak != NULL) {
   10.44 +          probe->oops_do(weak);
   10.45 +        }
   10.46 +      }
   10.47 +    }
   10.48 +  }
   10.49 +}
   10.50 +
   10.51  uint ProtectionDomainCacheTable::bucket_size() {
   10.52    return sizeof(ProtectionDomainCacheEntry);
   10.53  }

    11.1 --- a/src/share/vm/classfile/dictionary.hpp	Tue Jul 01 09:03:55 2014 +0200
    11.2 +++ b/src/share/vm/classfile/dictionary.hpp	Mon Jul 07 10:12:40 2014 +0200
    11.3 @@ -89,6 +89,7 @@
    11.4    // GC support
    11.5    void oops_do(OopClosure* f);
    11.6    void always_strong_oops_do(OopClosure* blk);
    11.7 +  void roots_oops_do(OopClosure* strong, OopClosure* weak);
    11.8  
    11.9    void always_strong_classes_do(KlassClosure* closure);
   11.10  
   11.11 @@ -218,6 +219,7 @@
   11.12    // GC support
   11.13    void oops_do(OopClosure* f);
   11.14    void always_strong_oops_do(OopClosure* f);
   11.15 +  void roots_oops_do(OopClosure* strong, OopClosure* weak);
   11.16  
   11.17    static uint bucket_size();
   11.18  

    12.1 --- a/src/share/vm/classfile/metadataOnStackMark.cpp	Tue Jul 01 09:03:55 2014 +0200
    12.2 +++ b/src/share/vm/classfile/metadataOnStackMark.cpp	Mon Jul 07 10:12:40 2014 +0200
    12.3 @@ -47,8 +47,11 @@
    12.4    if (_marked_objects == NULL) {
    12.5      _marked_objects = new (ResourceObj::C_HEAP, mtClass) GrowableArray<Metadata*>(1000, true);
    12.6    }
    12.7 +
    12.8    Threads::metadata_do(Metadata::mark_on_stack);
    12.9 -  CodeCache::alive_nmethods_do(nmethod::mark_on_stack);
   12.10 +  if (JvmtiExport::has_redefined_a_class()) {
   12.11 +    CodeCache::alive_nmethods_do(nmethod::mark_on_stack);
   12.12 +  }
   12.13    CompileBroker::mark_on_stack();
   12.14    JvmtiCurrentBreakpoints::metadata_do(Metadata::mark_on_stack);
   12.15    ThreadService::metadata_do(Metadata::mark_on_stack);

    13.1 --- a/src/share/vm/classfile/symbolTable.cpp	Tue Jul 01 09:03:55 2014 +0200
    13.2 +++ b/src/share/vm/classfile/symbolTable.cpp	Mon Jul 07 10:12:40 2014 +0200
    13.3 @@ -36,6 +36,7 @@
    13.4  #include "runtime/mutexLocker.hpp"
    13.5  #include "utilities/hashtable.inline.hpp"
    13.6  #if INCLUDE_ALL_GCS
    13.7 +#include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
    13.8  #include "gc_implementation/g1/g1StringDedup.hpp"
    13.9  #endif
   13.10  
   13.11 @@ -704,11 +705,26 @@
   13.12    return lookup(chars, length);
   13.13  }
   13.14  
   13.15 +// Tell the GC that this string was looked up in the StringTable.
   13.16 +static void ensure_string_alive(oop string) {
   13.17 +  // A lookup in the StringTable could return an object that was previously
   13.18 +  // considered dead. The SATB part of G1 needs to get notified about this
   13.19 +  // potential resurrection, otherwise the marking might not find the object.
   13.20 +#if INCLUDE_ALL_GCS
   13.21 +  if (UseG1GC && string != NULL) {
   13.22 +    G1SATBCardTableModRefBS::enqueue(string);
   13.23 +  }
   13.24 +#endif
   13.25 +}
   13.26  
   13.27  oop StringTable::lookup(jchar* name, int len) {
   13.28    unsigned int hash = hash_string(name, len);
   13.29    int index = the_table()->hash_to_index(hash);
   13.30 -  return the_table()->lookup(index, name, len, hash);
   13.31 +  oop string = the_table()->lookup(index, name, len, hash);
   13.32 +
   13.33 +  ensure_string_alive(string);
   13.34 +
   13.35 +  return string;
   13.36  }
   13.37  
   13.38  
   13.39 @@ -719,7 +735,10 @@
   13.40    oop found_string = the_table()->lookup(index, name, len, hashValue);
   13.41  
   13.42    // Found
   13.43 -  if (found_string != NULL) return found_string;
   13.44 +  if (found_string != NULL) {
   13.45 +    ensure_string_alive(found_string);
   13.46 +    return found_string;
   13.47 +  }
   13.48  
   13.49    debug_only(StableMemoryChecker smc(name, len * sizeof(name[0])));
   13.50    assert(!Universe::heap()->is_in_reserved(name),
   13.51 @@ -744,11 +763,17 @@
   13.52  
   13.53    // Grab the StringTable_lock before getting the_table() because it could
   13.54    // change at safepoint.
   13.55 -  MutexLocker ml(StringTable_lock, THREAD);
   13.56 +  oop added_or_found;
   13.57 +  {
   13.58 +    MutexLocker ml(StringTable_lock, THREAD);
   13.59 +    // Otherwise, add to symbol to table
   13.60 +    added_or_found = the_table()->basic_add(index, string, name, len,
   13.61 +                                  hashValue, CHECK_NULL);
   13.62 +  }
   13.63  
   13.64 -  // Otherwise, add to symbol to table
   13.65 -  return the_table()->basic_add(index, string, name, len,
   13.66 -                                hashValue, CHECK_NULL);
   13.67 +  ensure_string_alive(added_or_found);
   13.68 +
   13.69 +  return added_or_found;
   13.70  }
   13.71  
   13.72  oop StringTable::intern(Symbol* symbol, TRAPS) {

    14.1 --- a/src/share/vm/classfile/systemDictionary.cpp	Tue Jul 01 09:03:55 2014 +0200
    14.2 +++ b/src/share/vm/classfile/systemDictionary.cpp	Mon Jul 07 10:12:40 2014 +0200
    14.3 @@ -1613,13 +1613,7 @@
    14.4  // system dictionary and follows the remaining classes' contents.
    14.5  
    14.6  void SystemDictionary::always_strong_oops_do(OopClosure* blk) {
    14.7 -  blk->do_oop(&_java_system_loader);
    14.8 -  blk->do_oop(&_system_loader_lock_obj);
    14.9 -
   14.10 -  dictionary()->always_strong_oops_do(blk);
   14.11 -
   14.12 -  // Visit extra methods
   14.13 -  invoke_method_table()->oops_do(blk);
   14.14 +  roots_oops_do(blk, NULL);
   14.15  }
   14.16  
   14.17  void SystemDictionary::always_strong_classes_do(KlassClosure* closure) {
   14.18 @@ -1686,6 +1680,17 @@
   14.19    return unloading_occurred;
   14.20  }
   14.21  
   14.22 +void SystemDictionary::roots_oops_do(OopClosure* strong, OopClosure* weak) {
   14.23 +  strong->do_oop(&_java_system_loader);
   14.24 +  strong->do_oop(&_system_loader_lock_obj);
   14.25 +
   14.26 +  // Adjust dictionary
   14.27 +  dictionary()->roots_oops_do(strong, weak);
   14.28 +
   14.29 +  // Visit extra methods
   14.30 +  invoke_method_table()->oops_do(strong);
   14.31 +}
   14.32 +
   14.33  void SystemDictionary::oops_do(OopClosure* f) {
   14.34    f->do_oop(&_java_system_loader);
   14.35    f->do_oop(&_system_loader_lock_obj);

    15.1 --- a/src/share/vm/classfile/systemDictionary.hpp	Tue Jul 01 09:03:55 2014 +0200
    15.2 +++ b/src/share/vm/classfile/systemDictionary.hpp	Mon Jul 07 10:12:40 2014 +0200
    15.3 @@ -335,6 +335,7 @@
    15.4  
    15.5    // Applies "f->do_oop" to all root oops in the system dictionary.
    15.6    static void oops_do(OopClosure* f);
    15.7 +  static void roots_oops_do(OopClosure* strong, OopClosure* weak);
    15.8  
    15.9    // System loader lock
   15.10    static oop system_loader_lock()           { return _system_loader_lock_obj; }

    16.1 --- a/src/share/vm/code/codeCache.cpp	Tue Jul 01 09:03:55 2014 +0200
    16.2 +++ b/src/share/vm/code/codeCache.cpp	Mon Jul 07 10:12:40 2014 +0200
    16.3 @@ -337,6 +337,11 @@
    16.4  // Walk the list of methods which might contain non-perm oops.
    16.5  void CodeCache::scavenge_root_nmethods_do(CodeBlobClosure* f) {
    16.6    assert_locked_or_safepoint(CodeCache_lock);
    16.7 +
    16.8 +  if (UseG1GC) {
    16.9 +    return;
   16.10 +  }
   16.11 +
   16.12    debug_only(mark_scavenge_root_nmethods());
   16.13  
   16.14    for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
   16.15 @@ -362,6 +367,11 @@
   16.16  
   16.17  void CodeCache::add_scavenge_root_nmethod(nmethod* nm) {
   16.18    assert_locked_or_safepoint(CodeCache_lock);
   16.19 +
   16.20 +  if (UseG1GC) {
   16.21 +    return;
   16.22 +  }
   16.23 +
   16.24    nm->set_on_scavenge_root_list();
   16.25    nm->set_scavenge_root_link(_scavenge_root_nmethods);
   16.26    set_scavenge_root_nmethods(nm);
   16.27 @@ -370,6 +380,11 @@
   16.28  
   16.29  void CodeCache::drop_scavenge_root_nmethod(nmethod* nm) {
   16.30    assert_locked_or_safepoint(CodeCache_lock);
   16.31 +
   16.32 +  if (UseG1GC) {
   16.33 +    return;
   16.34 +  }
   16.35 +
   16.36    print_trace("drop_scavenge_root", nm);
   16.37    nmethod* last = NULL;
   16.38    nmethod* cur = scavenge_root_nmethods();
   16.39 @@ -391,6 +406,11 @@
   16.40  
   16.41  void CodeCache::prune_scavenge_root_nmethods() {
   16.42    assert_locked_or_safepoint(CodeCache_lock);
   16.43 +
   16.44 +  if (UseG1GC) {
   16.45 +    return;
   16.46 +  }
   16.47 +
   16.48    debug_only(mark_scavenge_root_nmethods());
   16.49  
   16.50    nmethod* last = NULL;
   16.51 @@ -423,6 +443,10 @@
   16.52  
   16.53  #ifndef PRODUCT
   16.54  void CodeCache::asserted_non_scavengable_nmethods_do(CodeBlobClosure* f) {
   16.55 +  if (UseG1GC) {
   16.56 +    return;
   16.57 +  }
   16.58 +
   16.59    // While we are here, verify the integrity of the list.
   16.60    mark_scavenge_root_nmethods();
   16.61    for (nmethod* cur = scavenge_root_nmethods(); cur != NULL; cur = cur->scavenge_root_link()) {
   16.62 @@ -463,9 +487,36 @@
   16.63  }
   16.64  #endif //PRODUCT
   16.65  
   16.66 +void CodeCache::verify_clean_inline_caches() {
   16.67 +#ifdef ASSERT
   16.68 +  FOR_ALL_ALIVE_BLOBS(cb) {
   16.69 +    if (cb->is_nmethod()) {
   16.70 +      nmethod* nm = (nmethod*)cb;
   16.71 +      assert(!nm->is_unloaded(), "Tautology");
   16.72 +      nm->verify_clean_inline_caches();
   16.73 +      nm->verify();
   16.74 +    }
   16.75 +  }
   16.76 +#endif
   16.77 +}
   16.78 +
   16.79 +void CodeCache::verify_icholder_relocations() {
   16.80 +#ifdef ASSERT
   16.81 +  // make sure that we aren't leaking icholders
   16.82 +  int count = 0;
   16.83 +  FOR_ALL_BLOBS(cb) {
   16.84 +    if (cb->is_nmethod()) {
   16.85 +      nmethod* nm = (nmethod*)cb;
   16.86 +      count += nm->verify_icholder_relocations();
   16.87 +    }
   16.88 +  }
   16.89 +
   16.90 +  assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
   16.91 +         CompiledICHolder::live_count(), "must agree");
   16.92 +#endif
   16.93 +}
   16.94  
   16.95  void CodeCache::gc_prologue() {
   16.96 -  assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_epilogue must be called");
   16.97  }
   16.98  
   16.99  void CodeCache::gc_epilogue() {
  16.100 @@ -478,41 +529,15 @@
  16.101          nm->cleanup_inline_caches();
  16.102        }
  16.103        DEBUG_ONLY(nm->verify());
  16.104 -      nm->fix_oop_relocations();
  16.105 +      DEBUG_ONLY(nm->verify_oop_relocations());
  16.106      }
  16.107    }
  16.108    set_needs_cache_clean(false);
  16.109    prune_scavenge_root_nmethods();
  16.110 -  assert(!nmethod::oops_do_marking_is_active(), "oops_do_marking_prologue must be called");
  16.111  
  16.112 -#ifdef ASSERT
  16.113 -  // make sure that we aren't leaking icholders
  16.114 -  int count = 0;
  16.115 -  FOR_ALL_BLOBS(cb) {
  16.116 -    if (cb->is_nmethod()) {
  16.117 -      RelocIterator iter((nmethod*)cb);
  16.118 -      while(iter.next()) {
  16.119 -        if (iter.type() == relocInfo::virtual_call_type) {
  16.120 -          if (CompiledIC::is_icholder_call_site(iter.virtual_call_reloc())) {
  16.121 -            CompiledIC *ic = CompiledIC_at(&iter);
  16.122 -            if (TraceCompiledIC) {
  16.123 -              tty->print("noticed icholder " INTPTR_FORMAT " ", p2i(ic->cached_icholder()));
  16.124 -              ic->print();
  16.125 -            }
  16.126 -            assert(ic->cached_icholder() != NULL, "must be non-NULL");
  16.127 -            count++;
  16.128 -          }
  16.129 -        }
  16.130 -      }
  16.131 -    }
  16.132 -  }
  16.133 -
  16.134 -  assert(count + InlineCacheBuffer::pending_icholder_count() + CompiledICHolder::live_not_claimed_count() ==
  16.135 -         CompiledICHolder::live_count(), "must agree");
  16.136 -#endif
  16.137 +  verify_icholder_relocations();
  16.138  }
  16.139  
  16.140 -
  16.141  void CodeCache::verify_oops() {
  16.142    MutexLockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
  16.143    VerifyOopClosure voc;

    17.1 --- a/src/share/vm/code/codeCache.hpp	Tue Jul 01 09:03:55 2014 +0200
    17.2 +++ b/src/share/vm/code/codeCache.hpp	Mon Jul 07 10:12:40 2014 +0200
    17.3 @@ -134,10 +134,6 @@
    17.4    // to) any unmarked codeBlobs in the cache.  Sets "marked_for_unloading"
    17.5    // to "true" iff some code got unloaded.
    17.6    static void do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred);
    17.7 -  static void oops_do(OopClosure* f) {
    17.8 -    CodeBlobToOopClosure oopc(f, /*do_marking=*/ false);
    17.9 -    blobs_do(&oopc);
   17.10 -  }
   17.11    static void asserted_non_scavengable_nmethods_do(CodeBlobClosure* f = NULL) PRODUCT_RETURN;
   17.12    static void scavenge_root_nmethods_do(CodeBlobClosure* f);
   17.13  
   17.14 @@ -172,6 +168,9 @@
   17.15    static void set_needs_cache_clean(bool v)      { _needs_cache_clean = v;    }
   17.16    static void clear_inline_caches();             // clear all inline caches
   17.17  
   17.18 +  static void verify_clean_inline_caches();
   17.19 +  static void verify_icholder_relocations();
   17.20 +
   17.21    // Deoptimization
   17.22    static int  mark_for_deoptimization(DepChange& changes);
   17.23  #ifdef HOTSWAP

    18.1 --- a/src/share/vm/code/compiledIC.cpp	Tue Jul 01 09:03:55 2014 +0200
    18.2 +++ b/src/share/vm/code/compiledIC.cpp	Mon Jul 07 10:12:40 2014 +0200
    18.3 @@ -99,13 +99,13 @@
    18.4    }
    18.5  
    18.6    {
    18.7 -  MutexLockerEx pl(Patching_lock, Mutex::_no_safepoint_check_flag);
    18.8 +    MutexLockerEx pl(SafepointSynchronize::is_at_safepoint() ? NULL : Patching_lock, Mutex::_no_safepoint_check_flag);
    18.9  #ifdef ASSERT
   18.10 -  CodeBlob* cb = CodeCache::find_blob_unsafe(_ic_call);
   18.11 -  assert(cb != NULL && cb->is_nmethod(), "must be nmethod");
   18.12 +    CodeBlob* cb = CodeCache::find_blob_unsafe(_ic_call);
   18.13 +    assert(cb != NULL && cb->is_nmethod(), "must be nmethod");
   18.14  #endif
   18.15 -  _ic_call->set_destination_mt_safe(entry_point);
   18.16 -}
   18.17 +     _ic_call->set_destination_mt_safe(entry_point);
   18.18 +  }
   18.19  
   18.20    if (is_optimized() || is_icstub) {
   18.21      // Optimized call sites don't have a cache value and ICStub call
   18.22 @@ -529,7 +529,7 @@
   18.23  void CompiledStaticCall::set_to_clean() {
   18.24    assert (CompiledIC_lock->is_locked() || SafepointSynchronize::is_at_safepoint(), "mt unsafe call");
   18.25    // Reset call site
   18.26 -  MutexLockerEx pl(Patching_lock, Mutex::_no_safepoint_check_flag);
   18.27 +  MutexLockerEx pl(SafepointSynchronize::is_at_safepoint() ? NULL : Patching_lock, Mutex::_no_safepoint_check_flag);
   18.28  #ifdef ASSERT
   18.29    CodeBlob* cb = CodeCache::find_blob_unsafe(this);
   18.30    assert(cb != NULL && cb->is_nmethod(), "must be nmethod");

    19.1 --- a/src/share/vm/code/nmethod.cpp	Tue Jul 01 09:03:55 2014 +0200
    19.2 +++ b/src/share/vm/code/nmethod.cpp	Mon Jul 07 10:12:40 2014 +0200
    19.3 @@ -49,6 +49,8 @@
    19.4  
    19.5  PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
    19.6  
    19.7 +unsigned char nmethod::_global_unloading_clock = 0;
    19.8 +
    19.9  #ifdef DTRACE_ENABLED
   19.10  
   19.11  // Only bother with this argument setup if dtrace is available
   19.12 @@ -466,6 +468,7 @@
   19.13  // Fill in default values for various flag fields
   19.14  void nmethod::init_defaults() {
   19.15    _state                      = in_use;
   19.16 +  _unloading_clock            = 0;
   19.17    _marked_for_reclamation     = 0;
   19.18    _has_flushed_dependencies   = 0;
   19.19    _has_unsafe_access          = 0;
   19.20 @@ -484,7 +487,11 @@
   19.21    _oops_do_mark_link       = NULL;
   19.22    _jmethod_id              = NULL;
   19.23    _osr_link                = NULL;
   19.24 -  _scavenge_root_link      = NULL;
   19.25 +  if (UseG1GC) {
   19.26 +    _unloading_next        = NULL;
   19.27 +  } else {
   19.28 +    _scavenge_root_link    = NULL;
   19.29 +  }
   19.30    _scavenge_root_state     = 0;
   19.31    _compiler                = NULL;
   19.32  #if INCLUDE_RTM_OPT
   19.33 @@ -1190,6 +1197,77 @@
   19.34    }
   19.35  }
   19.36  
   19.37 +void nmethod::verify_clean_inline_caches() {
   19.38 +  assert_locked_or_safepoint(CompiledIC_lock);
   19.39 +
   19.40 +  // If the method is not entrant or zombie then a JMP is plastered over the
   19.41 +  // first few bytes.  If an oop in the old code was there, that oop
   19.42 +  // should not get GC'd.  Skip the first few bytes of oops on
   19.43 +  // not-entrant methods.
   19.44 +  address low_boundary = verified_entry_point();
   19.45 +  if (!is_in_use()) {
   19.46 +    low_boundary += NativeJump::instruction_size;
   19.47 +    // %%% Note:  On SPARC we patch only a 4-byte trap, not a full NativeJump.
   19.48 +    // This means that the low_boundary is going to be a little too high.
   19.49 +    // This shouldn't matter, since oops of non-entrant methods are never used.
   19.50 +    // In fact, why are we bothering to look at oops in a non-entrant method??
   19.51 +  }
   19.52 +
   19.53 +  ResourceMark rm;
   19.54 +  RelocIterator iter(this, low_boundary);
   19.55 +  while(iter.next()) {
   19.56 +    switch(iter.type()) {
   19.57 +      case relocInfo::virtual_call_type:
   19.58 +      case relocInfo::opt_virtual_call_type: {
   19.59 +        CompiledIC *ic = CompiledIC_at(&iter);
   19.60 +        // Ok, to lookup references to zombies here
   19.61 +        CodeBlob *cb = CodeCache::find_blob_unsafe(ic->ic_destination());
   19.62 +        if( cb != NULL && cb->is_nmethod() ) {
   19.63 +          nmethod* nm = (nmethod*)cb;
   19.64 +          // Verify that inline caches pointing to both zombie and not_entrant methods are clean
   19.65 +          if (!nm->is_in_use() || (nm->method()->code() != nm)) {
   19.66 +            assert(ic->is_clean(), "IC should be clean");
   19.67 +          }
   19.68 +        }
   19.69 +        break;
   19.70 +      }
   19.71 +      case relocInfo::static_call_type: {
   19.72 +        CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
   19.73 +        CodeBlob *cb = CodeCache::find_blob_unsafe(csc->destination());
   19.74 +        if( cb != NULL && cb->is_nmethod() ) {
   19.75 +          nmethod* nm = (nmethod*)cb;
   19.76 +          // Verify that inline caches pointing to both zombie and not_entrant methods are clean
   19.77 +          if (!nm->is_in_use() || (nm->method()->code() != nm)) {
   19.78 +            assert(csc->is_clean(), "IC should be clean");
   19.79 +          }
   19.80 +        }
   19.81 +        break;
   19.82 +      }
   19.83 +    }
   19.84 +  }
   19.85 +}
   19.86 +
   19.87 +int nmethod::verify_icholder_relocations() {
   19.88 +  int count = 0;
   19.89 +
   19.90 +  RelocIterator iter(this);
   19.91 +  while(iter.next()) {
   19.92 +    if (iter.type() == relocInfo::virtual_call_type) {
   19.93 +      if (CompiledIC::is_icholder_call_site(iter.virtual_call_reloc())) {
   19.94 +        CompiledIC *ic = CompiledIC_at(&iter);
   19.95 +        if (TraceCompiledIC) {
   19.96 +          tty->print("noticed icholder " INTPTR_FORMAT " ", p2i(ic->cached_icholder()));
   19.97 +          ic->print();
   19.98 +        }
   19.99 +        assert(ic->cached_icholder() != NULL, "must be non-NULL");
  19.100 +        count++;
  19.101 +      }
  19.102 +    }
  19.103 +  }
  19.104 +
  19.105 +  return count;
  19.106 +}
  19.107 +
  19.108  // This is a private interface with the sweeper.
  19.109  void nmethod::mark_as_seen_on_stack() {
  19.110    assert(is_alive(), "Must be an alive method");
  19.111 @@ -1222,6 +1300,23 @@
  19.112    mdo->inc_decompile_count();
  19.113  }
  19.114  
  19.115 +void nmethod::increase_unloading_clock() {
  19.116 +  _global_unloading_clock++;
  19.117 +  if (_global_unloading_clock == 0) {
  19.118 +    // _nmethods are allocated with _unloading_clock == 0,
  19.119 +    // so 0 is never used as a clock value.
  19.120 +    _global_unloading_clock = 1;
  19.121 +  }
  19.122 +}
  19.123 +
  19.124 +void nmethod::set_unloading_clock(unsigned char unloading_clock) {
  19.125 +  OrderAccess::release_store((volatile jubyte*)&_unloading_clock, unloading_clock);
  19.126 +}
  19.127 +
  19.128 +unsigned char nmethod::unloading_clock() {
  19.129 +  return (unsigned char)OrderAccess::load_acquire((volatile jubyte*)&_unloading_clock);
  19.130 +}
  19.131 +
  19.132  void nmethod::make_unloaded(BoolObjectClosure* is_alive, oop cause) {
  19.133  
  19.134    post_compiled_method_unload();
  19.135 @@ -1267,6 +1362,10 @@
  19.136      // for later on.
  19.137      CodeCache::set_needs_cache_clean(true);
  19.138    }
  19.139 +
  19.140 +  // Unregister must be done before the state change
  19.141 +  Universe::heap()->unregister_nmethod(this);
  19.142 +
  19.143    _state = unloaded;
  19.144  
  19.145    // Log the unloading.
  19.146 @@ -1621,6 +1720,35 @@
  19.147    set_unload_reported();
  19.148  }
  19.149  
  19.150 +void static clean_ic_if_metadata_is_dead(CompiledIC *ic, BoolObjectClosure *is_alive) {
  19.151 +  if (ic->is_icholder_call()) {
  19.152 +    // The only exception is compiledICHolder oops which may
  19.153 +    // yet be marked below. (We check this further below).
  19.154 +    CompiledICHolder* cichk_oop = ic->cached_icholder();
  19.155 +    if (cichk_oop->holder_method()->method_holder()->is_loader_alive(is_alive) &&
  19.156 +        cichk_oop->holder_klass()->is_loader_alive(is_alive)) {
  19.157 +      return;
  19.158 +    }
  19.159 +  } else {
  19.160 +    Metadata* ic_oop = ic->cached_metadata();
  19.161 +    if (ic_oop != NULL) {
  19.162 +      if (ic_oop->is_klass()) {
  19.163 +        if (((Klass*)ic_oop)->is_loader_alive(is_alive)) {
  19.164 +          return;
  19.165 +        }
  19.166 +      } else if (ic_oop->is_method()) {
  19.167 +        if (((Method*)ic_oop)->method_holder()->is_loader_alive(is_alive)) {
  19.168 +          return;
  19.169 +        }
  19.170 +      } else {
  19.171 +        ShouldNotReachHere();
  19.172 +      }
  19.173 +    }
  19.174 +  }
  19.175 +
  19.176 +  ic->set_to_clean();
  19.177 +}
  19.178 +
  19.179  // This is called at the end of the strong tracing/marking phase of a
  19.180  // GC to unload an nmethod if it contains otherwise unreachable
  19.181  // oops.
  19.182 @@ -1664,31 +1792,7 @@
  19.183      while(iter.next()) {
  19.184        if (iter.type() == relocInfo::virtual_call_type) {
  19.185          CompiledIC *ic = CompiledIC_at(&iter);
  19.186 -        if (ic->is_icholder_call()) {
  19.187 -          // The only exception is compiledICHolder oops which may
  19.188 -          // yet be marked below. (We check this further below).
  19.189 -          CompiledICHolder* cichk_oop = ic->cached_icholder();
  19.190 -          if (cichk_oop->holder_method()->method_holder()->is_loader_alive(is_alive) &&
  19.191 -              cichk_oop->holder_klass()->is_loader_alive(is_alive)) {
  19.192 -            continue;
  19.193 -          }
  19.194 -        } else {
  19.195 -          Metadata* ic_oop = ic->cached_metadata();
  19.196 -          if (ic_oop != NULL) {
  19.197 -            if (ic_oop->is_klass()) {
  19.198 -              if (((Klass*)ic_oop)->is_loader_alive(is_alive)) {
  19.199 -                continue;
  19.200 -              }
  19.201 -            } else if (ic_oop->is_method()) {
  19.202 -              if (((Method*)ic_oop)->method_holder()->is_loader_alive(is_alive)) {
  19.203 -                continue;
  19.204 -              }
  19.205 -            } else {
  19.206 -              ShouldNotReachHere();
  19.207 -            }
  19.208 -          }
  19.209 -        }
  19.210 -        ic->set_to_clean();
  19.211 +        clean_ic_if_metadata_is_dead(ic, is_alive);
  19.212        }
  19.213      }
  19.214    }
  19.215 @@ -1726,6 +1830,175 @@
  19.216    verify_metadata_loaders(low_boundary, is_alive);
  19.217  }
  19.218  
  19.219 +template <class CompiledICorStaticCall>
  19.220 +static bool clean_if_nmethod_is_unloaded(CompiledICorStaticCall *ic, address addr, BoolObjectClosure *is_alive, nmethod* from) {
  19.221 +  // Ok, to lookup references to zombies here
  19.222 +  CodeBlob *cb = CodeCache::find_blob_unsafe(addr);
  19.223 +  if (cb != NULL && cb->is_nmethod()) {
  19.224 +    nmethod* nm = (nmethod*)cb;
  19.225 +
  19.226 +    if (nm->unloading_clock() != nmethod::global_unloading_clock()) {
  19.227 +      // The nmethod has not been processed yet.
  19.228 +      return true;
  19.229 +    }
  19.230 +
  19.231 +    // Clean inline caches pointing to both zombie and not_entrant methods
  19.232 +    if (!nm->is_in_use() || (nm->method()->code() != nm)) {
  19.233 +      ic->set_to_clean();
  19.234 +      assert(ic->is_clean(), err_msg("nmethod " PTR_FORMAT "not clean %s", from, from->method()->name_and_sig_as_C_string()));
  19.235 +    }
  19.236 +  }
  19.237 +
  19.238 +  return false;
  19.239 +}
  19.240 +
  19.241 +static bool clean_if_nmethod_is_unloaded(CompiledIC *ic, BoolObjectClosure *is_alive, nmethod* from) {
  19.242 +  return clean_if_nmethod_is_unloaded(ic, ic->ic_destination(), is_alive, from);
  19.243 +}
  19.244 +
  19.245 +static bool clean_if_nmethod_is_unloaded(CompiledStaticCall *csc, BoolObjectClosure *is_alive, nmethod* from) {
  19.246 +  return clean_if_nmethod_is_unloaded(csc, csc->destination(), is_alive, from);
  19.247 +}
  19.248 +
  19.249 +bool nmethod::do_unloading_parallel(BoolObjectClosure* is_alive, bool unloading_occurred) {
  19.250 +  ResourceMark rm;
  19.251 +
  19.252 +  // Make sure the oop's ready to receive visitors
  19.253 +  assert(!is_zombie() && !is_unloaded(),
  19.254 +         "should not call follow on zombie or unloaded nmethod");
  19.255 +
  19.256 +  // If the method is not entrant then a JMP is plastered over the
  19.257 +  // first few bytes.  If an oop in the old code was there, that oop
  19.258 +  // should not get GC'd.  Skip the first few bytes of oops on
  19.259 +  // not-entrant methods.
  19.260 +  address low_boundary = verified_entry_point();
  19.261 +  if (is_not_entrant()) {
  19.262 +    low_boundary += NativeJump::instruction_size;
  19.263 +    // %%% Note:  On SPARC we patch only a 4-byte trap, not a full NativeJump.
  19.264 +    // (See comment above.)
  19.265 +  }
  19.266 +
  19.267 +  // The RedefineClasses() API can cause the class unloading invariant
  19.268 +  // to no longer be true. See jvmtiExport.hpp for details.
  19.269 +  // Also, leave a debugging breadcrumb in local flag.
  19.270 +  bool a_class_was_redefined = JvmtiExport::has_redefined_a_class();
  19.271 +  if (a_class_was_redefined) {
  19.272 +    // This set of the unloading_occurred flag is done before the
  19.273 +    // call to post_compiled_method_unload() so that the unloading
  19.274 +    // of this nmethod is reported.
  19.275 +    unloading_occurred = true;
  19.276 +  }
  19.277 +
  19.278 +  // Exception cache
  19.279 +  clean_exception_cache(is_alive);
  19.280 +
  19.281 +  bool is_unloaded = false;
  19.282 +  bool postponed = false;
  19.283 +
  19.284 +  RelocIterator iter(this, low_boundary);
  19.285 +  while(iter.next()) {
  19.286 +
  19.287 +    switch (iter.type()) {
  19.288 +
  19.289 +    case relocInfo::virtual_call_type:
  19.290 +      if (unloading_occurred) {
  19.291 +        // If class unloading occurred we first iterate over all inline caches and
  19.292 +        // clear ICs where the cached oop is referring to an unloaded klass or method.
  19.293 +        clean_ic_if_metadata_is_dead(CompiledIC_at(&iter), is_alive);
  19.294 +      }
  19.295 +
  19.296 +      postponed |= clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), is_alive, this);
  19.297 +      break;
  19.298 +
  19.299 +    case relocInfo::opt_virtual_call_type:
  19.300 +      postponed |= clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), is_alive, this);
  19.301 +      break;
  19.302 +
  19.303 +    case relocInfo::static_call_type:
  19.304 +      postponed |= clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), is_alive, this);
  19.305 +      break;
  19.306 +
  19.307 +    case relocInfo::oop_type:
  19.308 +      if (!is_unloaded) {
  19.309 +        // Unload check
  19.310 +        oop_Relocation* r = iter.oop_reloc();
  19.311 +        // Traverse those oops directly embedded in the code.
  19.312 +        // Other oops (oop_index>0) are seen as part of scopes_oops.
  19.313 +        assert(1 == (r->oop_is_immediate()) +
  19.314 +                  (r->oop_addr() >= oops_begin() && r->oop_addr() < oops_end()),
  19.315 +              "oop must be found in exactly one place");
  19.316 +        if (r->oop_is_immediate() && r->oop_value() != NULL) {
  19.317 +          if (can_unload(is_alive, r->oop_addr(), unloading_occurred)) {
  19.318 +            is_unloaded = true;
  19.319 +          }
  19.320 +        }
  19.321 +      }
  19.322 +      break;
  19.323 +
  19.324 +    }
  19.325 +  }
  19.326 +
  19.327 +  if (is_unloaded) {
  19.328 +    return postponed;
  19.329 +  }
  19.330 +
  19.331 +  // Scopes
  19.332 +  for (oop* p = oops_begin(); p < oops_end(); p++) {
  19.333 +    if (*p == Universe::non_oop_word())  continue;  // skip non-oops
  19.334 +    if (can_unload(is_alive, p, unloading_occurred)) {
  19.335 +      is_unloaded = true;
  19.336 +      break;
  19.337 +    }
  19.338 +  }
  19.339 +
  19.340 +  if (is_unloaded) {
  19.341 +    return postponed;
  19.342 +  }
  19.343 +
  19.344 +  // Ensure that all metadata is still alive
  19.345 +  verify_metadata_loaders(low_boundary, is_alive);
  19.346 +
  19.347 +  return postponed;
  19.348 +}
  19.349 +
  19.350 +void nmethod::do_unloading_parallel_postponed(BoolObjectClosure* is_alive, bool unloading_occurred) {
  19.351 +  ResourceMark rm;
  19.352 +
  19.353 +  // Make sure the oop's ready to receive visitors
  19.354 +  assert(!is_zombie(),
  19.355 +         "should not call follow on zombie nmethod");
  19.356 +
  19.357 +  // If the method is not entrant then a JMP is plastered over the
  19.358 +  // first few bytes.  If an oop in the old code was there, that oop
  19.359 +  // should not get GC'd.  Skip the first few bytes of oops on
  19.360 +  // not-entrant methods.
  19.361 +  address low_boundary = verified_entry_point();
  19.362 +  if (is_not_entrant()) {
  19.363 +    low_boundary += NativeJump::instruction_size;
  19.364 +    // %%% Note:  On SPARC we patch only a 4-byte trap, not a full NativeJump.
  19.365 +    // (See comment above.)
  19.366 +  }
  19.367 +
  19.368 +  RelocIterator iter(this, low_boundary);
  19.369 +  while(iter.next()) {
  19.370 +
  19.371 +    switch (iter.type()) {
  19.372 +
  19.373 +    case relocInfo::virtual_call_type:
  19.374 +      clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), is_alive, this);
  19.375 +      break;
  19.376 +
  19.377 +    case relocInfo::opt_virtual_call_type:
  19.378 +      clean_if_nmethod_is_unloaded(CompiledIC_at(&iter), is_alive, this);
  19.379 +      break;
  19.380 +
  19.381 +    case relocInfo::static_call_type:
  19.382 +      clean_if_nmethod_is_unloaded(compiledStaticCall_at(iter.reloc()), is_alive, this);
  19.383 +      break;
  19.384 +    }
  19.385 +  }
  19.386 +}
  19.387 +
  19.388  #ifdef ASSERT
  19.389  
  19.390  class CheckClass : AllStatic {
  19.391 @@ -1942,7 +2215,7 @@
  19.392      assert(cur != NULL, "not NULL-terminated");
  19.393      nmethod* next = cur->_oops_do_mark_link;
  19.394      cur->_oops_do_mark_link = NULL;
  19.395 -    cur->fix_oop_relocations();
  19.396 +    cur->verify_oop_relocations();
  19.397      NOT_PRODUCT(if (TraceScavenge)  cur->print_on(tty, "oops_do, unmark"));
  19.398      cur = next;
  19.399    }
  19.400 @@ -2484,6 +2757,10 @@
  19.401  };
  19.402  
  19.403  void nmethod::verify_scavenge_root_oops() {
  19.404 +  if (UseG1GC) {
  19.405 +    return;
  19.406 +  }
  19.407 +
  19.408    if (!on_scavenge_root_list()) {
  19.409      // Actually look inside, to verify the claim that it's clean.
  19.410      DebugScavengeRoot debug_scavenge_root(this);

    20.1 --- a/src/share/vm/code/nmethod.hpp	Tue Jul 01 09:03:55 2014 +0200
    20.2 +++ b/src/share/vm/code/nmethod.hpp	Mon Jul 07 10:12:40 2014 +0200
    20.3 @@ -116,6 +116,11 @@
    20.4    friend class NMethodSweeper;
    20.5    friend class CodeCache;  // scavengable oops
    20.6   private:
    20.7 +
    20.8 +  // GC support to help figure out if an nmethod has been
    20.9 +  // cleaned/unloaded by the current GC.
   20.10 +  static unsigned char _global_unloading_clock;
   20.11 +
   20.12    // Shared fields for all nmethod's
   20.13    Method*   _method;
   20.14    int       _entry_bci;        // != InvocationEntryBci if this nmethod is an on-stack replacement method
   20.15 @@ -123,7 +128,13 @@
   20.16  
   20.17    // To support simple linked-list chaining of nmethods:
   20.18    nmethod*  _osr_link;         // from InstanceKlass::osr_nmethods_head
   20.19 -  nmethod*  _scavenge_root_link; // from CodeCache::scavenge_root_nmethods
   20.20 +
   20.21 +  union {
   20.22 +    // Used by G1 to chain nmethods.
   20.23 +    nmethod* _unloading_next;
   20.24 +    // Used by non-G1 GCs to chain nmethods.
   20.25 +    nmethod* _scavenge_root_link; // from CodeCache::scavenge_root_nmethods
   20.26 +  };
   20.27  
   20.28    static nmethod* volatile _oops_do_mark_nmethods;
   20.29    nmethod*        volatile _oops_do_mark_link;
   20.30 @@ -185,6 +196,8 @@
   20.31    // Protected by Patching_lock
   20.32    volatile unsigned char _state;             // {alive, not_entrant, zombie, unloaded}
   20.33  
   20.34 +  volatile unsigned char _unloading_clock;   // Incremented after GC unloaded/cleaned the nmethod
   20.35 +
   20.36  #ifdef ASSERT
   20.37    bool _oops_are_stale;  // indicates that it's no longer safe to access oops section
   20.38  #endif
   20.39 @@ -442,6 +455,15 @@
   20.40    bool  unload_reported()                         { return _unload_reported; }
   20.41    void  set_unload_reported()                     { _unload_reported = true; }
   20.42  
   20.43 +  void set_unloading_next(nmethod* next)          { _unloading_next = next; }
   20.44 +  nmethod* unloading_next()                       { return _unloading_next; }
   20.45 +
   20.46 +  static unsigned char global_unloading_clock()   { return _global_unloading_clock; }
   20.47 +  static void increase_unloading_clock();
   20.48 +
   20.49 +  void set_unloading_clock(unsigned char unloading_clock);
   20.50 +  unsigned char unloading_clock();
   20.51 +
   20.52    bool  is_marked_for_deoptimization() const      { return _marked_for_deoptimization; }
   20.53    void  mark_for_deoptimization()                 { _marked_for_deoptimization = true; }
   20.54  
   20.55 @@ -557,6 +579,10 @@
   20.56      return (addr >= code_begin() && addr < verified_entry_point());
   20.57    }
   20.58  
   20.59 +  // Verify calls to dead methods have been cleaned.
   20.60 +  void verify_clean_inline_caches();
   20.61 +  // Verify and count cached icholder relocations.
   20.62 +  int  verify_icholder_relocations();
   20.63    // Check that all metadata is still alive
   20.64    void verify_metadata_loaders(address low_boundary, BoolObjectClosure* is_alive);
   20.65  
   20.66 @@ -582,6 +608,10 @@
   20.67  
   20.68    // GC support
   20.69    void do_unloading(BoolObjectClosure* is_alive, bool unloading_occurred);
   20.70 +  //  The parallel versions are used by G1.
   20.71 +  bool do_unloading_parallel(BoolObjectClosure* is_alive, bool unloading_occurred);
   20.72 +  void do_unloading_parallel_postponed(BoolObjectClosure* is_alive, bool unloading_occurred);
   20.73 +  //  Unload a nmethod if the *root object is dead.
   20.74    bool can_unload(BoolObjectClosure* is_alive, oop* root, bool unloading_occurred);
   20.75  
   20.76    void preserve_callee_argument_oops(frame fr, const RegisterMap *reg_map,

    21.1 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Tue Jul 01 09:03:55 2014 +0200
    21.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Mon Jul 07 10:12:40 2014 +0200
    21.3 @@ -1570,11 +1570,11 @@
    21.4    }
    21.5  
    21.6    if (MetaspaceGC::should_concurrent_collect()) {
    21.7 -      if (Verbose && PrintGCDetails) {
    21.8 +    if (Verbose && PrintGCDetails) {
    21.9        gclog_or_tty->print("CMSCollector: collect for metadata allocation ");
   21.10 -      }
   21.11 -      return true;
   21.12 -    }
   21.13 +    }
   21.14 +    return true;
   21.15 +  }
   21.16  
   21.17    return false;
   21.18  }
   21.19 @@ -3028,20 +3028,21 @@
   21.20    HandleMark  hm;
   21.21    GenCollectedHeap* gch = GenCollectedHeap::heap();
   21.22  
   21.23 -  // Get a clear set of claim bits for the strong roots processing to work with.
   21.24 +  // Get a clear set of claim bits for the roots processing to work with.
   21.25    ClassLoaderDataGraph::clear_claimed_marks();
   21.26  
   21.27    // Mark from roots one level into CMS
   21.28    MarkRefsIntoClosure notOlder(_span, verification_mark_bm());
   21.29    gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
   21.30  
   21.31 -  gch->gen_process_strong_roots(_cmsGen->level(),
   21.32 -                                true,   // younger gens are roots
   21.33 -                                true,   // activate StrongRootsScope
   21.34 -                                SharedHeap::ScanningOption(roots_scanning_options()),
   21.35 -                                &notOlder,
   21.36 -                                NULL,
   21.37 -                                NULL); // SSS: Provide correct closure
   21.38 +  gch->gen_process_roots(_cmsGen->level(),
   21.39 +                         true,   // younger gens are roots
   21.40 +                         true,   // activate StrongRootsScope
   21.41 +                         SharedHeap::ScanningOption(roots_scanning_options()),
   21.42 +                         should_unload_classes(),
   21.43 +                         &notOlder,
   21.44 +                         NULL,
   21.45 +                         NULL);  // SSS: Provide correct closure
   21.46  
   21.47    // Now mark from the roots
   21.48    MarkFromRootsClosure markFromRootsClosure(this, _span,
   21.49 @@ -3092,22 +3093,24 @@
   21.50    HandleMark  hm;
   21.51    GenCollectedHeap* gch = GenCollectedHeap::heap();
   21.52  
   21.53 -  // Get a clear set of claim bits for the strong roots processing to work with.
   21.54 +  // Get a clear set of claim bits for the roots processing to work with.
   21.55    ClassLoaderDataGraph::clear_claimed_marks();
   21.56  
   21.57    // Mark from roots one level into CMS
   21.58    MarkRefsIntoVerifyClosure notOlder(_span, verification_mark_bm(),
   21.59                                       markBitMap());
   21.60 -  KlassToOopClosure klass_closure(&notOlder);
   21.61 +  CLDToOopClosure cld_closure(&notOlder, true);
   21.62  
   21.63    gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
   21.64 -  gch->gen_process_strong_roots(_cmsGen->level(),
   21.65 -                                true,   // younger gens are roots
   21.66 -                                true,   // activate StrongRootsScope
   21.67 -                                SharedHeap::ScanningOption(roots_scanning_options()),
   21.68 -                                &notOlder,
   21.69 -                                NULL,
   21.70 -                                &klass_closure);
   21.71 +
   21.72 +  gch->gen_process_roots(_cmsGen->level(),
   21.73 +                         true,   // younger gens are roots
   21.74 +                         true,   // activate StrongRootsScope
   21.75 +                         SharedHeap::ScanningOption(roots_scanning_options()),
   21.76 +                         should_unload_classes(),
   21.77 +                         &notOlder,
   21.78 +                         NULL,
   21.79 +                         &cld_closure);
   21.80  
   21.81    // Now mark from the roots
   21.82    MarkFromRootsVerifyClosure markFromRootsClosure(this, _span,
   21.83 @@ -3294,12 +3297,10 @@
   21.84  void CMSCollector::setup_cms_unloading_and_verification_state() {
   21.85    const  bool should_verify =   VerifyBeforeGC || VerifyAfterGC || VerifyDuringGC
   21.86                               || VerifyBeforeExit;
   21.87 -  const  int  rso           =   SharedHeap::SO_Strings | SharedHeap::SO_AllCodeCache;
   21.88 +  const  int  rso           =   SharedHeap::SO_AllCodeCache;
   21.89  
   21.90    // We set the proper root for this CMS cycle here.
   21.91    if (should_unload_classes()) {   // Should unload classes this cycle
   21.92 -    remove_root_scanning_option(SharedHeap::SO_AllClasses);
   21.93 -    add_root_scanning_option(SharedHeap::SO_SystemClasses);
   21.94      remove_root_scanning_option(rso);  // Shrink the root set appropriately
   21.95      set_verifying(should_verify);    // Set verification state for this cycle
   21.96      return;                            // Nothing else needs to be done at this time
   21.97 @@ -3307,8 +3308,6 @@
   21.98  
   21.99    // Not unloading classes this cycle
  21.100    assert(!should_unload_classes(), "Inconsitency!");
  21.101 -  remove_root_scanning_option(SharedHeap::SO_SystemClasses);
  21.102 -  add_root_scanning_option(SharedHeap::SO_AllClasses);
  21.103  
  21.104    if ((!verifying() || unloaded_classes_last_cycle()) && should_verify) {
  21.105      // Include symbols, strings and code cache elements to prevent their resurrection.
  21.106 @@ -3719,15 +3718,16 @@
  21.107        gch->set_par_threads(0);
  21.108      } else {
  21.109        // The serial version.
  21.110 -      KlassToOopClosure klass_closure(&notOlder);
  21.111 +      CLDToOopClosure cld_closure(&notOlder, true);
  21.112        gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
  21.113 -      gch->gen_process_strong_roots(_cmsGen->level(),
  21.114 -                                    true,   // younger gens are roots
  21.115 -                                    true,   // activate StrongRootsScope
  21.116 -                                    SharedHeap::ScanningOption(roots_scanning_options()),
  21.117 -                                    &notOlder,
  21.118 -                                    NULL,
  21.119 -                                    &klass_closure);
  21.120 +      gch->gen_process_roots(_cmsGen->level(),
  21.121 +                             true,   // younger gens are roots
  21.122 +                             true,   // activate StrongRootsScope
  21.123 +                             SharedHeap::ScanningOption(roots_scanning_options()),
  21.124 +                             should_unload_classes(),
  21.125 +                             &notOlder,
  21.126 +                             NULL,
  21.127 +                             &cld_closure);
  21.128      }
  21.129    }
  21.130  
  21.131 @@ -5203,7 +5203,6 @@
  21.132    _timer.start();
  21.133    GenCollectedHeap* gch = GenCollectedHeap::heap();
  21.134    Par_MarkRefsIntoClosure par_mri_cl(_collector->_span, &(_collector->_markBitMap));
  21.135 -  KlassToOopClosure klass_closure(&par_mri_cl);
  21.136  
  21.137    // ---------- young gen roots --------------
  21.138    {
  21.139 @@ -5219,13 +5218,17 @@
  21.140    // ---------- remaining roots --------------
  21.141    _timer.reset();
  21.142    _timer.start();
  21.143 -  gch->gen_process_strong_roots(_collector->_cmsGen->level(),
  21.144 -                                false,     // yg was scanned above
  21.145 -                                false,     // this is parallel code
  21.146 -                                SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
  21.147 -                                &par_mri_cl,
  21.148 -                                NULL,
  21.149 -                                &klass_closure);
  21.150 +
  21.151 +  CLDToOopClosure cld_closure(&par_mri_cl, true);
  21.152 +
  21.153 +  gch->gen_process_roots(_collector->_cmsGen->level(),
  21.154 +                         false,     // yg was scanned above
  21.155 +                         false,     // this is parallel code
  21.156 +                         SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
  21.157 +                         _collector->should_unload_classes(),
  21.158 +                         &par_mri_cl,
  21.159 +                         NULL,
  21.160 +                         &cld_closure);
  21.161    assert(_collector->should_unload_classes()
  21.162           || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_AllCodeCache),
  21.163           "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
  21.164 @@ -5354,13 +5357,15 @@
  21.165    // ---------- remaining roots --------------
  21.166    _timer.reset();
  21.167    _timer.start();
  21.168 -  gch->gen_process_strong_roots(_collector->_cmsGen->level(),
  21.169 -                                false,     // yg was scanned above
  21.170 -                                false,     // this is parallel code
  21.171 -                                SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
  21.172 -                                &par_mrias_cl,
  21.173 -                                NULL,
  21.174 -                                NULL);     // The dirty klasses will be handled below
  21.175 +  gch->gen_process_roots(_collector->_cmsGen->level(),
  21.176 +                         false,     // yg was scanned above
  21.177 +                         false,     // this is parallel code
  21.178 +                         SharedHeap::ScanningOption(_collector->CMSCollector::roots_scanning_options()),
  21.179 +                         _collector->should_unload_classes(),
  21.180 +                         &par_mrias_cl,
  21.181 +                         NULL,
  21.182 +                         NULL);     // The dirty klasses will be handled below
  21.183 +
  21.184    assert(_collector->should_unload_classes()
  21.185           || (_collector->CMSCollector::roots_scanning_options() & SharedHeap::SO_AllCodeCache),
  21.186           "if we didn't scan the code cache, we have to be ready to drop nmethods with expired weak oops");
  21.187 @@ -5415,7 +5420,7 @@
  21.188    // We might have added oops to ClassLoaderData::_handles during the
  21.189    // concurrent marking phase. These oops point to newly allocated objects
  21.190    // that are guaranteed to be kept alive. Either by the direct allocation
  21.191 -  // code, or when the young collector processes the strong roots. Hence,
  21.192 +  // code, or when the young collector processes the roots. Hence,
  21.193    // we don't have to revisit the _handles block during the remark phase.
  21.194  
  21.195    // ---------- rescan dirty cards ------------
  21.196 @@ -5837,7 +5842,7 @@
  21.197      cms_space,
  21.198      n_workers, workers, task_queues());
  21.199  
  21.200 -  // Set up for parallel process_strong_roots work.
  21.201 +  // Set up for parallel process_roots work.
  21.202    gch->set_par_threads(n_workers);
  21.203    // We won't be iterating over the cards in the card table updating
  21.204    // the younger_gen cards, so we shouldn't call the following else
  21.205 @@ -5846,7 +5851,7 @@
  21.206    // gch->rem_set()->prepare_for_younger_refs_iterate(true); // parallel
  21.207  
  21.208    // The young gen rescan work will not be done as part of
  21.209 -  // process_strong_roots (which currently doesn't knw how to
  21.210 +  // process_roots (which currently doesn't know how to
  21.211    // parallelize such a scan), but rather will be broken up into
  21.212    // a set of parallel tasks (via the sampling that the [abortable]
  21.213    // preclean phase did of EdenSpace, plus the [two] tasks of
  21.214 @@ -5943,13 +5948,15 @@
  21.215  
  21.216      gch->rem_set()->prepare_for_younger_refs_iterate(false); // Not parallel.
  21.217      GenCollectedHeap::StrongRootsScope srs(gch);
  21.218 -    gch->gen_process_strong_roots(_cmsGen->level(),
  21.219 -                                  true,  // younger gens as roots
  21.220 -                                  false, // use the local StrongRootsScope
  21.221 -                                  SharedHeap::ScanningOption(roots_scanning_options()),
  21.222 -                                  &mrias_cl,
  21.223 -                                  NULL,
  21.224 -                                  NULL);  // The dirty klasses will be handled below
  21.225 +
  21.226 +    gch->gen_process_roots(_cmsGen->level(),
  21.227 +                           true,  // younger gens as roots
  21.228 +                           false, // use the local StrongRootsScope
  21.229 +                           SharedHeap::ScanningOption(roots_scanning_options()),
  21.230 +                           should_unload_classes(),
  21.231 +                           &mrias_cl,
  21.232 +                           NULL,
  21.233 +                           NULL); // The dirty klasses will be handled below
  21.234  
  21.235      assert(should_unload_classes()
  21.236             || (roots_scanning_options() & SharedHeap::SO_AllCodeCache),
  21.237 @@ -5989,7 +5996,7 @@
  21.238    // We might have added oops to ClassLoaderData::_handles during the
  21.239    // concurrent marking phase. These oops point to newly allocated objects
  21.240    // that are guaranteed to be kept alive. Either by the direct allocation
  21.241 -  // code, or when the young collector processes the strong roots. Hence,
  21.242 +  // code, or when the young collector processes the roots. Hence,
  21.243    // we don't have to revisit the _handles block during the remark phase.
  21.244  
  21.245    verify_work_stacks_empty();
  21.246 @@ -6239,15 +6246,14 @@
  21.247        // Clean up unreferenced symbols in symbol table.
  21.248        SymbolTable::unlink();
  21.249      }
  21.250 -  }
  21.251 -
  21.252 -  // CMS doesn't use the StringTable as hard roots when class unloading is turned off.
  21.253 -  // Need to check if we really scanned the StringTable.
  21.254 -  if ((roots_scanning_options() & SharedHeap::SO_Strings) == 0) {
  21.255 -    GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
  21.256 -    // Delete entries for dead interned strings.
  21.257 -    StringTable::unlink(&_is_alive_closure);
  21.258 -  }
  21.259 +
  21.260 +    {
  21.261 +      GCTraceTime t("scrub string table", PrintGCDetails, false, _gc_timer_cm, _gc_tracer_cm->gc_id());
  21.262 +      // Delete entries for dead interned strings.
  21.263 +      StringTable::unlink(&_is_alive_closure);
  21.264 +    }
  21.265 +  }
  21.266 +
  21.267  
  21.268    // Restore any preserved marks as a result of mark stack or
  21.269    // work queue overflow

    22.1 --- a/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp	Tue Jul 01 09:03:55 2014 +0200
    22.2 +++ b/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.hpp	Mon Jul 07 10:12:40 2014 +0200
    22.3 @@ -32,6 +32,7 @@
    22.4  #include "gc_implementation/shared/generationCounters.hpp"
    22.5  #include "memory/freeBlockDictionary.hpp"
    22.6  #include "memory/generation.hpp"
    22.7 +#include "memory/iterator.hpp"
    22.8  #include "runtime/mutexLocker.hpp"
    22.9  #include "runtime/virtualspace.hpp"
   22.10  #include "services/memoryService.hpp"

    23.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Tue Jul 01 09:03:55 2014 +0200
    23.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Mon Jul 07 10:12:40 2014 +0200
    23.3 @@ -24,6 +24,7 @@
    23.4  
    23.5  #include "precompiled.hpp"
    23.6  #include "classfile/symbolTable.hpp"
    23.7 +#include "code/codeCache.hpp"
    23.8  #include "gc_implementation/g1/concurrentMark.inline.hpp"
    23.9  #include "gc_implementation/g1/concurrentMarkThread.inline.hpp"
   23.10  #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
   23.11 @@ -39,6 +40,7 @@
   23.12  #include "gc_implementation/shared/gcTimer.hpp"
   23.13  #include "gc_implementation/shared/gcTrace.hpp"
   23.14  #include "gc_implementation/shared/gcTraceTime.hpp"
   23.15 +#include "memory/allocation.hpp"
   23.16  #include "memory/genOopClosures.inline.hpp"
   23.17  #include "memory/referencePolicy.hpp"
   23.18  #include "memory/resourceArea.hpp"
   23.19 @@ -57,8 +59,8 @@
   23.20    _bmWordSize = 0;
   23.21  }
   23.22  
   23.23 -HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr,
   23.24 -                                               HeapWord* limit) const {
   23.25 +HeapWord* CMBitMapRO::getNextMarkedWordAddress(const HeapWord* addr,
   23.26 +                                               const HeapWord* limit) const {
   23.27    // First we must round addr *up* to a possible object boundary.
   23.28    addr = (HeapWord*)align_size_up((intptr_t)addr,
   23.29                                    HeapWordSize << _shifter);
   23.30 @@ -75,8 +77,8 @@
   23.31    return nextAddr;
   23.32  }
   23.33  
   23.34 -HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr,
   23.35 -                                                 HeapWord* limit) const {
   23.36 +HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr,
   23.37 +                                                 const HeapWord* limit) const {
   23.38    size_t addrOffset = heapWordToOffset(addr);
   23.39    if (limit == NULL) {
   23.40      limit = _bmStartWord + _bmWordSize;
   23.41 @@ -1222,6 +1224,9 @@
   23.42  };
   23.43  
   23.44  void ConcurrentMark::scanRootRegions() {
   23.45 +  // Start of concurrent marking.
   23.46 +  ClassLoaderDataGraph::clear_claimed_marks();
   23.47 +
   23.48    // scan_in_progress() will have been set to true only if there was
   23.49    // at least one root region to scan. So, if it's false, we
   23.50    // should not attempt to do any further work.
   23.51 @@ -1270,7 +1275,7 @@
   23.52    CMConcurrentMarkingTask markingTask(this, cmThread());
   23.53    if (use_parallel_marking_threads()) {
   23.54      _parallel_workers->set_active_workers((int)active_workers);
   23.55 -    // Don't set _n_par_threads because it affects MT in proceess_strong_roots()
   23.56 +    // Don't set _n_par_threads because it affects MT in process_roots()
   23.57      // and the decisions on that MT processing is made elsewhere.
   23.58      assert(_parallel_workers->active_workers() > 0, "Should have been set");
   23.59      _parallel_workers->run_task(&markingTask);
   23.60 @@ -2138,14 +2143,6 @@
   23.61    // Update the soft reference policy with the new heap occupancy.
   23.62    Universe::update_heap_info_at_gc();
   23.63  
   23.64 -  // We need to make this be a "collection" so any collection pause that
   23.65 -  // races with it goes around and waits for completeCleanup to finish.
   23.66 -  g1h->increment_total_collections();
   23.67 -
   23.68 -  // We reclaimed old regions so we should calculate the sizes to make
   23.69 -  // sure we update the old gen/space data.
   23.70 -  g1h->g1mm()->update_sizes();
   23.71 -
   23.72    if (VerifyDuringGC) {
   23.73      HandleMark hm;  // handle scope
   23.74      Universe::heap()->prepare_for_verify();
   23.75 @@ -2154,6 +2151,19 @@
   23.76    }
   23.77  
   23.78    g1h->verify_region_sets_optional();
   23.79 +
   23.80 +  // We need to make this be a "collection" so any collection pause that
   23.81 +  // races with it goes around and waits for completeCleanup to finish.
   23.82 +  g1h->increment_total_collections();
   23.83 +
   23.84 +  // Clean out dead classes and update Metaspace sizes.
   23.85 +  ClassLoaderDataGraph::purge();
   23.86 +  MetaspaceGC::compute_new_size();
   23.87 +
   23.88 +  // We reclaimed old regions so we should calculate the sizes to make
   23.89 +  // sure we update the old gen/space data.
   23.90 +  g1h->g1mm()->update_sizes();
   23.91 +
   23.92    g1h->trace_heap_after_concurrent_cycle();
   23.93  }
   23.94  
   23.95 @@ -2440,6 +2450,26 @@
   23.96    _g1h->set_par_threads(0);
   23.97  }
   23.98  
   23.99 +void ConcurrentMark::weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes) {
  23.100 +  G1CollectedHeap::heap()->parallel_cleaning(is_alive, true, true, purged_classes);
  23.101 +}
  23.102 +
  23.103 +// Helper class to get rid of some boilerplate code.
  23.104 +class G1RemarkGCTraceTime : public GCTraceTime {
  23.105 +  static bool doit_and_prepend(bool doit) {
  23.106 +    if (doit) {
  23.107 +      gclog_or_tty->put(' ');
  23.108 +    }
  23.109 +    return doit;
  23.110 +  }
  23.111 +
  23.112 + public:
  23.113 +  G1RemarkGCTraceTime(const char* title, bool doit)
  23.114 +    : GCTraceTime(title, doit_and_prepend(doit), false, G1CollectedHeap::heap()->gc_timer_cm(),
  23.115 +        G1CollectedHeap::heap()->concurrent_mark()->concurrent_gc_id()) {
  23.116 +  }
  23.117 +};
  23.118 +
  23.119  void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
  23.120    if (has_overflown()) {
  23.121      // Skip processing the discovered references if we have
  23.122 @@ -2552,9 +2582,28 @@
  23.123      return;
  23.124    }
  23.125  
  23.126 -  g1h->unlink_string_and_symbol_table(&g1_is_alive,
  23.127 -                                      /* process_strings */ false, // currently strings are always roots
  23.128 -                                      /* process_symbols */ true);
  23.129 +  assert(_markStack.isEmpty(), "Marking should have completed");
  23.130 +
  23.131 +  // Unload Klasses, String, Symbols, Code Cache, etc.
  23.132 +
  23.133 +  G1RemarkGCTraceTime trace("Unloading", G1Log::finer());
  23.134 +
  23.135 +  bool purged_classes;
  23.136 +
  23.137 +  {
  23.138 +    G1RemarkGCTraceTime trace("System Dictionary Unloading", G1Log::finest());
  23.139 +    purged_classes = SystemDictionary::do_unloading(&g1_is_alive);
  23.140 +  }
  23.141 +
  23.142 +  {
  23.143 +    G1RemarkGCTraceTime trace("Parallel Unloading", G1Log::finest());
  23.144 +    weakRefsWorkParallelPart(&g1_is_alive, purged_classes);
  23.145 +  }
  23.146 +
  23.147 +  if (G1StringDedup::is_enabled()) {
  23.148 +    G1RemarkGCTraceTime trace("String Deduplication Unlink", G1Log::finest());
  23.149 +    G1StringDedup::unlink(&g1_is_alive);
  23.150 +  }
  23.151  }
  23.152  
  23.153  void ConcurrentMark::swapMarkBitMaps() {
  23.154 @@ -2563,6 +2612,57 @@
  23.155    _nextMarkBitMap  = (CMBitMap*)  temp;
  23.156  }
  23.157  
  23.158 +class CMObjectClosure;
  23.159 +
  23.160 +// Closure for iterating over objects, currently only used for
  23.161 +// processing SATB buffers.
  23.162 +class CMObjectClosure : public ObjectClosure {
  23.163 +private:
  23.164 +  CMTask* _task;
  23.165 +
  23.166 +public:
  23.167 +  void do_object(oop obj) {
  23.168 +    _task->deal_with_reference(obj);
  23.169 +  }
  23.170 +
  23.171 +  CMObjectClosure(CMTask* task) : _task(task) { }
  23.172 +};
  23.173 +
  23.174 +class G1RemarkThreadsClosure : public ThreadClosure {
  23.175 +  CMObjectClosure _cm_obj;
  23.176 +  G1CMOopClosure _cm_cl;
  23.177 +  MarkingCodeBlobClosure _code_cl;
  23.178 +  int _thread_parity;
  23.179 +  bool _is_par;
  23.180 +
  23.181 + public:
  23.182 +  G1RemarkThreadsClosure(G1CollectedHeap* g1h, CMTask* task, bool is_par) :
  23.183 +    _cm_obj(task), _cm_cl(g1h, g1h->concurrent_mark(), task), _code_cl(&_cm_cl, !CodeBlobToOopClosure::FixRelocations),
  23.184 +    _thread_parity(SharedHeap::heap()->strong_roots_parity()), _is_par(is_par) {}
  23.185 +
  23.186 +  void do_thread(Thread* thread) {
  23.187 +    if (thread->is_Java_thread()) {
  23.188 +      if (thread->claim_oops_do(_is_par, _thread_parity)) {
  23.189 +        JavaThread* jt = (JavaThread*)thread;
  23.190 +
  23.191 +        // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
  23.192 +        // however the liveness of oops reachable from nmethods have very complex lifecycles:
  23.193 +        // * Alive if on the stack of an executing method
  23.194 +        // * Weakly reachable otherwise
  23.195 +        // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
  23.196 +        // live by the SATB invariant but other oops recorded in nmethods may behave differently.
  23.197 +        jt->nmethods_do(&_code_cl);
  23.198 +
  23.199 +        jt->satb_mark_queue().apply_closure_and_empty(&_cm_obj);
  23.200 +      }
  23.201 +    } else if (thread->is_VM_thread()) {
  23.202 +      if (thread->claim_oops_do(_is_par, _thread_parity)) {
  23.203 +        JavaThread::satb_mark_queue_set().shared_satb_queue()->apply_closure_and_empty(&_cm_obj);
  23.204 +      }
  23.205 +    }
  23.206 +  }
  23.207 +};
  23.208 +
  23.209  class CMRemarkTask: public AbstractGangTask {
  23.210  private:
  23.211    ConcurrentMark* _cm;
  23.212 @@ -2574,6 +2674,14 @@
  23.213      if (worker_id < _cm->active_tasks()) {
  23.214        CMTask* task = _cm->task(worker_id);
  23.215        task->record_start_time();
  23.216 +      {
  23.217 +        ResourceMark rm;
  23.218 +        HandleMark hm;
  23.219 +
  23.220 +        G1RemarkThreadsClosure threads_f(G1CollectedHeap::heap(), task, !_is_serial);
  23.221 +        Threads::threads_do(&threads_f);
  23.222 +      }
  23.223 +
  23.224        do {
  23.225          task->do_marking_step(1000000000.0 /* something very large */,
  23.226                                true         /* do_termination       */,
  23.227 @@ -2596,6 +2704,8 @@
  23.228    HandleMark   hm;
  23.229    G1CollectedHeap* g1h = G1CollectedHeap::heap();
  23.230  
  23.231 +  G1RemarkGCTraceTime trace("Finalize Marking", G1Log::finer());
  23.232 +
  23.233    g1h->ensure_parsability(false);
  23.234  
  23.235    if (G1CollectedHeap::use_parallel_gc_threads()) {
  23.236 @@ -3421,20 +3531,6 @@
  23.237    }
  23.238  };
  23.239  
  23.240 -// Closure for iterating over objects, currently only used for
  23.241 -// processing SATB buffers.
  23.242 -class CMObjectClosure : public ObjectClosure {
  23.243 -private:
  23.244 -  CMTask* _task;
  23.245 -
  23.246 -public:
  23.247 -  void do_object(oop obj) {
  23.248 -    _task->deal_with_reference(obj);
  23.249 -  }
  23.250 -
  23.251 -  CMObjectClosure(CMTask* task) : _task(task) { }
  23.252 -};
  23.253 -
  23.254  G1CMOopClosure::G1CMOopClosure(G1CollectedHeap* g1h,
  23.255                                 ConcurrentMark* cm,
  23.256                                 CMTask* task)
  23.257 @@ -3900,15 +3996,6 @@
  23.258      }
  23.259    }
  23.260  
  23.261 -  if (!concurrent() && !has_aborted()) {
  23.262 -    // We should only do this during remark.
  23.263 -    if (G1CollectedHeap::use_parallel_gc_threads()) {
  23.264 -      satb_mq_set.par_iterate_closure_all_threads(_worker_id);
  23.265 -    } else {
  23.266 -      satb_mq_set.iterate_closure_all_threads();
  23.267 -    }
  23.268 -  }
  23.269 -
  23.270    _draining_satb_buffers = false;
  23.271  
  23.272    assert(has_aborted() ||

    24.1 --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Tue Jul 01 09:03:55 2014 +0200
    24.2 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp	Mon Jul 07 10:12:40 2014 +0200
    24.3 @@ -25,6 +25,7 @@
    24.4  #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP
    24.5  #define SHARE_VM_GC_IMPLEMENTATION_G1_CONCURRENTMARK_HPP
    24.6  
    24.7 +#include "classfile/javaClasses.hpp"
    24.8  #include "gc_implementation/g1/heapRegionSet.hpp"
    24.9  #include "gc_implementation/shared/gcId.hpp"
   24.10  #include "utilities/taskqueue.hpp"
   24.11 @@ -86,19 +87,19 @@
   24.12    // Return the address corresponding to the next marked bit at or after
   24.13    // "addr", and before "limit", if "limit" is non-NULL.  If there is no
   24.14    // such bit, returns "limit" if that is non-NULL, or else "endWord()".
   24.15 -  HeapWord* getNextMarkedWordAddress(HeapWord* addr,
   24.16 -                                     HeapWord* limit = NULL) const;
   24.17 +  HeapWord* getNextMarkedWordAddress(const HeapWord* addr,
   24.18 +                                     const HeapWord* limit = NULL) const;
   24.19    // Return the address corresponding to the next unmarked bit at or after
   24.20    // "addr", and before "limit", if "limit" is non-NULL.  If there is no
   24.21    // such bit, returns "limit" if that is non-NULL, or else "endWord()".
   24.22 -  HeapWord* getNextUnmarkedWordAddress(HeapWord* addr,
   24.23 -                                       HeapWord* limit = NULL) const;
   24.24 +  HeapWord* getNextUnmarkedWordAddress(const HeapWord* addr,
   24.25 +                                       const HeapWord* limit = NULL) const;
   24.26  
   24.27    // conversion utilities
   24.28    HeapWord* offsetToHeapWord(size_t offset) const {
   24.29      return _bmStartWord + (offset << _shifter);
   24.30    }
   24.31 -  size_t heapWordToOffset(HeapWord* addr) const {
   24.32 +  size_t heapWordToOffset(const HeapWord* addr) const {
   24.33      return pointer_delta(addr, _bmStartWord) >> _shifter;
   24.34    }
   24.35    int heapWordDiffToOffsetDiff(size_t diff) const;
   24.36 @@ -476,6 +477,7 @@
   24.37    ForceOverflowSettings _force_overflow_conc;
   24.38    ForceOverflowSettings _force_overflow_stw;
   24.39  
   24.40 +  void weakRefsWorkParallelPart(BoolObjectClosure* is_alive, bool purged_classes);
   24.41    void weakRefsWork(bool clear_all_soft_refs);
   24.42  
   24.43    void swapMarkBitMaps();

    25.1 --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Tue Jul 01 09:03:55 2014 +0200
    25.2 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.cpp	Mon Jul 07 10:12:40 2014 +0200
    25.3 @@ -426,7 +426,7 @@
    25.4        q = n;
    25.5        oop obj = oop(q);
    25.6        if (obj->klass_or_null() == NULL) return q;
    25.7 -      n += obj->size();
    25.8 +      n += block_size(q);
    25.9      }
   25.10      assert(q <= next_boundary && n > next_boundary, "Consequence of loop");
   25.11      // [q, n) is the block that crosses the boundary.

    26.1 --- a/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Tue Jul 01 09:03:55 2014 +0200
    26.2 +++ b/src/share/vm/gc_implementation/g1/g1BlockOffsetTable.inline.hpp	Mon Jul 07 10:12:40 2014 +0200
    26.3 @@ -113,7 +113,7 @@
    26.4      q = n;
    26.5      oop obj = oop(q);
    26.6      if (obj->klass_or_null() == NULL) return q;
    26.7 -    n += obj->size();
    26.8 +    n += block_size(q);
    26.9    }
   26.10    assert(q <= n, "wrong order for q and addr");
   26.11    assert(addr < n, "wrong order for addr and n");

    27.1 --- a/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp	Tue Jul 01 09:03:55 2014 +0200
    27.2 +++ b/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.cpp	Mon Jul 07 10:12:40 2014 +0200
    27.3 @@ -30,23 +30,52 @@
    27.4  
    27.5  PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
    27.6  
    27.7 -G1CodeRootChunk::G1CodeRootChunk() : _top(NULL), _next(NULL), _prev(NULL) {
    27.8 +G1CodeRootChunk::G1CodeRootChunk() : _top(NULL), _next(NULL), _prev(NULL), _free(NULL) {
    27.9    _top = bottom();
   27.10  }
   27.11  
   27.12  void G1CodeRootChunk::reset() {
   27.13    _next = _prev = NULL;
   27.14 +  _free = NULL;
   27.15    _top = bottom();
   27.16  }
   27.17  
   27.18  void G1CodeRootChunk::nmethods_do(CodeBlobClosure* cl) {
   27.19 -  nmethod** cur = bottom();
   27.20 +  NmethodOrLink* cur = bottom();
   27.21    while (cur != _top) {
   27.22 -    cl->do_code_blob(*cur);
   27.23 +    if (is_nmethod(cur)) {
   27.24 +      cl->do_code_blob(cur->_nmethod);
   27.25 +    }
   27.26      cur++;
   27.27    }
   27.28  }
   27.29  
   27.30 +bool G1CodeRootChunk::remove_lock_free(nmethod* method) {
   27.31 +  NmethodOrLink* cur = bottom();
   27.32 +
   27.33 +  for (NmethodOrLink* cur = bottom(); cur != _top; cur++) {
   27.34 +    if (cur->_nmethod == method) {
   27.35 +      bool result = Atomic::cmpxchg_ptr(NULL, &cur->_nmethod, method) == method;
   27.36 +
   27.37 +      if (!result) {
   27.38 +        // Someone else cleared out this entry.
   27.39 +        return false;
   27.40 +      }
   27.41 +
   27.42 +      // The method was cleared. Time to link it into the free list.
   27.43 +      NmethodOrLink* prev_free;
   27.44 +      do {
   27.45 +        prev_free = (NmethodOrLink*)_free;
   27.46 +        cur->_link = prev_free;
   27.47 +      } while (Atomic::cmpxchg_ptr(cur, &_free, prev_free) != prev_free);
   27.48 +
   27.49 +      return true;
   27.50 +    }
   27.51 +  }
   27.52 +
   27.53 +  return false;
   27.54 +}
   27.55 +
   27.56  G1CodeRootChunkManager::G1CodeRootChunkManager() : _free_list(), _num_chunks_handed_out(0) {
   27.57    _free_list.initialize();
   27.58    _free_list.set_size(G1CodeRootChunk::word_size());
   27.59 @@ -140,34 +169,43 @@
   27.60  
   27.61  void G1CodeRootSet::add(nmethod* method) {
   27.62    if (!contains(method)) {
   27.63 -    // Try to add the nmethod. If there is not enough space, get a new chunk.
   27.64 -    if (_list.head() == NULL || _list.head()->is_full()) {
   27.65 -      G1CodeRootChunk* cur = new_chunk();
   27.66 +    // Find the first chunk thatisn't full.
   27.67 +    G1CodeRootChunk* cur = _list.head();
   27.68 +    while (cur != NULL) {
   27.69 +      if (!cur->is_full()) {
   27.70 +        break;
   27.71 +      }
   27.72 +      cur = cur->next();
   27.73 +    }
   27.74 +
   27.75 +    // All chunks are full, get a new chunk.
   27.76 +    if (cur == NULL) {
   27.77 +      cur = new_chunk();
   27.78        _list.return_chunk_at_head(cur);
   27.79      }
   27.80 -    bool result = _list.head()->add(method);
   27.81 +
   27.82 +    // Add the nmethod.
   27.83 +    bool result = cur->add(method);
   27.84 +
   27.85      guarantee(result, err_msg("Not able to add nmethod "PTR_FORMAT" to newly allocated chunk.", method));
   27.86 +
   27.87      _length++;
   27.88    }
   27.89  }
   27.90  
   27.91 -void G1CodeRootSet::remove(nmethod* method) {
   27.92 +void G1CodeRootSet::remove_lock_free(nmethod* method) {
   27.93    G1CodeRootChunk* found = find(method);
   27.94    if (found != NULL) {
   27.95 -    bool result = found->remove(method);
   27.96 -    guarantee(result, err_msg("could not find nmethod "PTR_FORMAT" during removal although we previously found it", method));
   27.97 -    // eventually free completely emptied chunk
   27.98 -    if (found->is_empty()) {
   27.99 -      _list.remove_chunk(found);
  27.100 -      free(found);
  27.101 +    bool result = found->remove_lock_free(method);
  27.102 +    if (result) {
  27.103 +      Atomic::dec_ptr((volatile intptr_t*)&_length);
  27.104      }
  27.105 -    _length--;
  27.106    }
  27.107    assert(!contains(method), err_msg(PTR_FORMAT" still contains nmethod "PTR_FORMAT, this, method));
  27.108  }
  27.109  
  27.110  nmethod* G1CodeRootSet::pop() {
  27.111 -  do {
  27.112 +  while (true) {
  27.113      G1CodeRootChunk* cur = _list.head();
  27.114      if (cur == NULL) {
  27.115        assert(_length == 0, "when there are no chunks, there should be no elements");
  27.116 @@ -180,7 +218,7 @@
  27.117      } else {
  27.118        free(_list.get_chunk_at_head());
  27.119      }
  27.120 -  } while (true);
  27.121 +  }
  27.122  }
  27.123  
  27.124  G1CodeRootChunk* G1CodeRootSet::find(nmethod* method) {

    28.1 --- a/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp	Tue Jul 01 09:03:55 2014 +0200
    28.2 +++ b/src/share/vm/gc_implementation/g1/g1CodeCacheRemSet.hpp	Mon Jul 07 10:12:40 2014 +0200
    28.3 @@ -31,6 +31,14 @@
    28.4  
    28.5  class CodeBlobClosure;
    28.6  
    28.7 +// The elements of the G1CodeRootChunk is either:
    28.8 +//  1) nmethod pointers
    28.9 +//  2) nodes in an internally chained free list
   28.10 +typedef union {
   28.11 +  nmethod* _nmethod;
   28.12 +  void*    _link;
   28.13 +} NmethodOrLink;
   28.14 +
   28.15  class G1CodeRootChunk : public CHeapObj<mtGC> {
   28.16   private:
   28.17    static const int NUM_ENTRIES = 32;
   28.18 @@ -38,16 +46,28 @@
   28.19    G1CodeRootChunk*     _next;
   28.20    G1CodeRootChunk*     _prev;
   28.21  
   28.22 -  nmethod** _top;
   28.23 +  NmethodOrLink*          _top;
   28.24 +  // First free position within the chunk.
   28.25 +  volatile NmethodOrLink* _free;
   28.26  
   28.27 -  nmethod* _data[NUM_ENTRIES];
   28.28 +  NmethodOrLink _data[NUM_ENTRIES];
   28.29  
   28.30 -  nmethod** bottom() const {
   28.31 -    return (nmethod**) &(_data[0]);
   28.32 +  NmethodOrLink* bottom() const {
   28.33 +    return (NmethodOrLink*) &(_data[0]);
   28.34    }
   28.35  
   28.36 -  nmethod** end() const {
   28.37 -    return (nmethod**) &(_data[NUM_ENTRIES]);
   28.38 +  NmethodOrLink* end() const {
   28.39 +    return (NmethodOrLink*) &(_data[NUM_ENTRIES]);
   28.40 +  }
   28.41 +
   28.42 +  bool is_link(NmethodOrLink* nmethod_or_link) {
   28.43 +    return nmethod_or_link->_link == NULL ||
   28.44 +        (bottom() <= nmethod_or_link->_link
   28.45 +        && nmethod_or_link->_link < end());
   28.46 +  }
   28.47 +
   28.48 +  bool is_nmethod(NmethodOrLink* nmethod_or_link) {
   28.49 +    return !is_link(nmethod_or_link);
   28.50    }
   28.51  
   28.52   public:
   28.53 @@ -85,46 +105,55 @@
   28.54    }
   28.55  
   28.56    bool is_full() const {
   28.57 -    return _top == (nmethod**)end();
   28.58 +    return _top == end() && _free == NULL;
   28.59    }
   28.60  
   28.61    bool contains(nmethod* method) {
   28.62 -    nmethod** cur = bottom();
   28.63 +    NmethodOrLink* cur = bottom();
   28.64      while (cur != _top) {
   28.65 -      if (*cur == method) return true;
   28.66 +      if (cur->_nmethod == method) return true;
   28.67        cur++;
   28.68      }
   28.69      return false;
   28.70    }
   28.71  
   28.72    bool add(nmethod* method) {
   28.73 -    if (is_full()) return false;
   28.74 -    *_top = method;
   28.75 -    _top++;
   28.76 +    if (is_full()) {
   28.77 +      return false;
   28.78 +    }
   28.79 +
   28.80 +    if (_free != NULL) {
   28.81 +      // Take from internally chained free list
   28.82 +      NmethodOrLink* first_free = (NmethodOrLink*)_free;
   28.83 +      _free = (NmethodOrLink*)_free->_link;
   28.84 +      first_free->_nmethod = method;
   28.85 +    } else {
   28.86 +      // Take from top.
   28.87 +      _top->_nmethod = method;
   28.88 +      _top++;
   28.89 +    }
   28.90 +
   28.91      return true;
   28.92    }
   28.93  
   28.94 -  bool remove(nmethod* method) {
   28.95 -    nmethod** cur = bottom();
   28.96 -    while (cur != _top) {
   28.97 -      if (*cur == method) {
   28.98 -        memmove(cur, cur + 1, (_top - (cur + 1)) * sizeof(nmethod**));
   28.99 -        _top--;
  28.100 -        return true;
  28.101 -      }
  28.102 -      cur++;
  28.103 -    }
  28.104 -    return false;
  28.105 -  }
  28.106 +  bool remove_lock_free(nmethod* method);
  28.107  
  28.108    void nmethods_do(CodeBlobClosure* blk);
  28.109  
  28.110    nmethod* pop() {
  28.111 -    if (is_empty()) {
  28.112 -      return NULL;
  28.113 +    if (_free != NULL) {
  28.114 +      // Kill the free list.
  28.115 +      _free = NULL;
  28.116      }
  28.117 -    _top--;
  28.118 -    return *_top;
  28.119 +
  28.120 +    while (!is_empty()) {
  28.121 +      _top--;
  28.122 +      if (is_nmethod(_top)) {
  28.123 +        return _top->_nmethod;
  28.124 +      }
  28.125 +    }
  28.126 +
  28.127 +    return NULL;
  28.128    }
  28.129  };
  28.130  
  28.131 @@ -193,7 +222,7 @@
  28.132    // method is likely to be repeatedly called with the same nmethod.
  28.133    void add(nmethod* method);
  28.134  
  28.135 -  void remove(nmethod* method);
  28.136 +  void remove_lock_free(nmethod* method);
  28.137    nmethod* pop();
  28.138  
  28.139    bool contains(nmethod* method);

    29.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Tue Jul 01 09:03:55 2014 +0200
    29.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp	Mon Jul 07 10:12:40 2014 +0200
    29.3 @@ -55,6 +55,7 @@
    29.4  #include "gc_implementation/shared/gcTrace.hpp"
    29.5  #include "gc_implementation/shared/gcTraceTime.hpp"
    29.6  #include "gc_implementation/shared/isGCActiveMark.hpp"
    29.7 +#include "memory/allocation.hpp"
    29.8  #include "memory/gcLocker.inline.hpp"
    29.9  #include "memory/generationSpec.hpp"
   29.10  #include "memory/iterator.hpp"
   29.11 @@ -87,10 +88,10 @@
   29.12  // G1ParVerifyTask uses heap_region_par_iterate_chunked() for parallelism.
   29.13  // The number of GC workers is passed to heap_region_par_iterate_chunked().
   29.14  // It does use run_task() which sets _n_workers in the task.
   29.15 -// G1ParTask executes g1_process_strong_roots() ->
   29.16 -// SharedHeap::process_strong_roots() which calls eventually to
   29.17 +// G1ParTask executes g1_process_roots() ->
   29.18 +// SharedHeap::process_roots() which calls eventually to
   29.19  // CardTableModRefBS::par_non_clean_card_iterate_work() which uses
   29.20 -// SequentialSubTasksDone.  SharedHeap::process_strong_roots() also
   29.21 +// SequentialSubTasksDone.  SharedHeap::process_roots() also
   29.22  // directly uses SubTasksDone (_process_strong_tasks field in SharedHeap).
   29.23  //
   29.24  
   29.25 @@ -3391,25 +3392,19 @@
   29.26      if (!silent) { gclog_or_tty->print("Roots "); }
   29.27      VerifyRootsClosure rootsCl(vo);
   29.28      VerifyKlassClosure klassCl(this, &rootsCl);
   29.29 +    CLDToKlassAndOopClosure cldCl(&klassCl, &rootsCl, false);
   29.30  
   29.31      // We apply the relevant closures to all the oops in the
   29.32 -    // system dictionary, class loader data graph and the string table.
   29.33 -    // Don't verify the code cache here, since it's verified below.
   29.34 -    const int so = SO_AllClasses | SO_Strings;
   29.35 -
   29.36 -    // Need cleared claim bits for the strong roots processing
   29.37 -    ClassLoaderDataGraph::clear_claimed_marks();
   29.38 -
   29.39 -    process_strong_roots(true,      // activate StrongRootsScope
   29.40 -                         ScanningOption(so),  // roots scanning options
   29.41 -                         &rootsCl,
   29.42 -                         &klassCl
   29.43 -                         );
   29.44 -
   29.45 -    // Verify the nmethods in the code cache.
   29.46 +    // system dictionary, class loader data graph, the string table
   29.47 +    // and the nmethods in the code cache.
   29.48      G1VerifyCodeRootOopClosure codeRootsCl(this, &rootsCl, vo);
   29.49      G1VerifyCodeRootBlobClosure blobsCl(&codeRootsCl);
   29.50 -    CodeCache::blobs_do(&blobsCl);
   29.51 +
   29.52 +    process_all_roots(true,            // activate StrongRootsScope
   29.53 +                      SO_AllCodeCache, // roots scanning options
   29.54 +                      &rootsCl,
   29.55 +                      &cldCl,
   29.56 +                      &blobsCl);
   29.57  
   29.58      bool failures = rootsCl.failures() || codeRootsCl.failures();
   29.59  
   29.60 @@ -4339,11 +4334,7 @@
   29.61    assert(_mutator_alloc_region.get() == NULL, "post-condition");
   29.62  }
   29.63  
   29.64 -void G1CollectedHeap::init_gc_alloc_regions(EvacuationInfo& evacuation_info) {
   29.65 -  assert_at_safepoint(true /* should_be_vm_thread */);
   29.66 -
   29.67 -  _survivor_gc_alloc_region.init();
   29.68 -  _old_gc_alloc_region.init();
   29.69 +void G1CollectedHeap::use_retained_old_gc_alloc_region(EvacuationInfo& evacuation_info) {
   29.70    HeapRegion* retained_region = _retained_old_gc_alloc_region;
   29.71    _retained_old_gc_alloc_region = NULL;
   29.72  
   29.73 @@ -4375,6 +4366,15 @@
   29.74    }
   29.75  }
   29.76  
   29.77 +void G1CollectedHeap::init_gc_alloc_regions(EvacuationInfo& evacuation_info) {
   29.78 +  assert_at_safepoint(true /* should_be_vm_thread */);
   29.79 +
   29.80 +  _survivor_gc_alloc_region.init();
   29.81 +  _old_gc_alloc_region.init();
   29.82 +
   29.83 +  use_retained_old_gc_alloc_region(evacuation_info);
   29.84 +}
   29.85 +
   29.86  void G1CollectedHeap::release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info) {
   29.87    evacuation_info.set_allocation_regions(_survivor_gc_alloc_region.count() +
   29.88                                           _old_gc_alloc_region.count());
   29.89 @@ -4608,7 +4608,7 @@
   29.90    }
   29.91  }
   29.92  
   29.93 -template <G1Barrier barrier, bool do_mark_object>
   29.94 +template <G1Barrier barrier, G1Mark do_mark_object>
   29.95  template <class T>
   29.96  void G1ParCopyClosure<barrier, do_mark_object>::do_oop_work(T* p) {
   29.97    T heap_oop = oopDesc::load_heap_oop(p);
   29.98 @@ -4630,7 +4630,7 @@
   29.99      }
  29.100      assert(forwardee != NULL, "forwardee should not be NULL");
  29.101      oopDesc::encode_store_heap_oop(p, forwardee);
  29.102 -    if (do_mark_object && forwardee != obj) {
  29.103 +    if (do_mark_object != G1MarkNone && forwardee != obj) {
  29.104        // If the object is self-forwarded we don't need to explicitly
  29.105        // mark it, the evacuation failure protocol will do so.
  29.106        mark_forwarded_object(obj, forwardee);
  29.107 @@ -4641,9 +4641,8 @@
  29.108      }
  29.109    } else {
  29.110      // The object is not in collection set. If we're a root scanning
  29.111 -    // closure during an initial mark pause (i.e. do_mark_object will
  29.112 -    // be true) then attempt to mark the object.
  29.113 -    if (do_mark_object) {
  29.114 +    // closure during an initial mark pause then attempt to mark the object.
  29.115 +    if (do_mark_object == G1MarkFromRoot) {
  29.116        mark_object(obj);
  29.117      }
  29.118    }
  29.119 @@ -4653,8 +4652,8 @@
  29.120    }
  29.121  }
  29.122  
  29.123 -template void G1ParCopyClosure<G1BarrierEvac, false>::do_oop_work(oop* p);
  29.124 -template void G1ParCopyClosure<G1BarrierEvac, false>::do_oop_work(narrowOop* p);
  29.125 +template void G1ParCopyClosure<G1BarrierEvac, G1MarkNone>::do_oop_work(oop* p);
  29.126 +template void G1ParCopyClosure<G1BarrierEvac, G1MarkNone>::do_oop_work(narrowOop* p);
  29.127  
  29.128  class G1ParEvacuateFollowersClosure : public VoidClosure {
  29.129  protected:
  29.130 @@ -4767,6 +4766,51 @@
  29.131      _n_workers = active_workers;
  29.132    }
  29.133  
  29.134 +  // Helps out with CLD processing.
  29.135 +  //
  29.136 +  // During InitialMark we need to:
  29.137 +  // 1) Scavenge all CLDs for the young GC.
  29.138 +  // 2) Mark all objects directly reachable from strong CLDs.
  29.139 +  template <G1Mark do_mark_object>
  29.140 +  class G1CLDClosure : public CLDClosure {
  29.141 +    G1ParCopyClosure<G1BarrierNone,  do_mark_object>* _oop_closure;
  29.142 +    G1ParCopyClosure<G1BarrierKlass, do_mark_object>  _oop_in_klass_closure;
  29.143 +    G1KlassScanClosure                                _klass_in_cld_closure;
  29.144 +    bool                                              _claim;
  29.145 +
  29.146 +   public:
  29.147 +    G1CLDClosure(G1ParCopyClosure<G1BarrierNone, do_mark_object>* oop_closure,
  29.148 +                 bool only_young, bool claim)
  29.149 +        : _oop_closure(oop_closure),
  29.150 +          _oop_in_klass_closure(oop_closure->g1(),
  29.151 +                                oop_closure->pss(),
  29.152 +                                oop_closure->rp()),
  29.153 +          _klass_in_cld_closure(&_oop_in_klass_closure, only_young),
  29.154 +          _claim(claim) {
  29.155 +
  29.156 +    }
  29.157 +
  29.158 +    void do_cld(ClassLoaderData* cld) {
  29.159 +      cld->oops_do(_oop_closure, &_klass_in_cld_closure, _claim);
  29.160 +    }
  29.161 +  };
  29.162 +
  29.163 +  class G1CodeBlobClosure: public CodeBlobClosure {
  29.164 +    OopClosure* _f;
  29.165 +
  29.166 +   public:
  29.167 +    G1CodeBlobClosure(OopClosure* f) : _f(f) {}
  29.168 +    void do_code_blob(CodeBlob* blob) {
  29.169 +      nmethod* that = blob->as_nmethod_or_null();
  29.170 +      if (that != NULL) {
  29.171 +        if (!that->test_set_oops_do_mark()) {
  29.172 +          that->oops_do(_f);
  29.173 +          that->fix_oop_relocations();
  29.174 +        }
  29.175 +      }
  29.176 +    }
  29.177 +  };
  29.178 +
  29.179    void work(uint worker_id) {
  29.180      if (worker_id >= _n_workers) return;  // no work needed this round
  29.181  
  29.182 @@ -4784,40 +4828,62 @@
  29.183  
  29.184        pss.set_evac_failure_closure(&evac_failure_cl);
  29.185  
  29.186 -      G1ParScanExtRootClosure        only_scan_root_cl(_g1h, &pss, rp);
  29.187 -      G1ParScanMetadataClosure       only_scan_metadata_cl(_g1h, &pss, rp);
  29.188 -
  29.189 -      G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss, rp);
  29.190 -      G1ParScanAndMarkMetadataClosure scan_mark_metadata_cl(_g1h, &pss, rp);
  29.191 -
  29.192 -      bool only_young                 = _g1h->g1_policy()->gcs_are_young();
  29.193 -      G1KlassScanClosure              scan_mark_klasses_cl_s(&scan_mark_metadata_cl, false);
  29.194 -      G1KlassScanClosure              only_scan_klasses_cl_s(&only_scan_metadata_cl, only_young);
  29.195 -
  29.196 -      OopClosure*                    scan_root_cl = &only_scan_root_cl;
  29.197 -      G1KlassScanClosure*            scan_klasses_cl = &only_scan_klasses_cl_s;
  29.198 +      bool only_young = _g1h->g1_policy()->gcs_are_young();
  29.199 +
  29.200 +      // Non-IM young GC.
  29.201 +      G1ParCopyClosure<G1BarrierNone, G1MarkNone>             scan_only_root_cl(_g1h, &pss, rp);
  29.202 +      G1CLDClosure<G1MarkNone>                                scan_only_cld_cl(&scan_only_root_cl,
  29.203 +                                                                               only_young, // Only process dirty klasses.
  29.204 +                                                                               false);     // No need to claim CLDs.
  29.205 +      // IM young GC.
  29.206 +      //    Strong roots closures.
  29.207 +      G1ParCopyClosure<G1BarrierNone, G1MarkFromRoot>         scan_mark_root_cl(_g1h, &pss, rp);
  29.208 +      G1CLDClosure<G1MarkFromRoot>                            scan_mark_cld_cl(&scan_mark_root_cl,
  29.209 +                                                                               false, // Process all klasses.
  29.210 +                                                                               true); // Need to claim CLDs.
  29.211 +      //    Weak roots closures.
  29.212 +      G1ParCopyClosure<G1BarrierNone, G1MarkPromotedFromRoot> scan_mark_weak_root_cl(_g1h, &pss, rp);
  29.213 +      G1CLDClosure<G1MarkPromotedFromRoot>                    scan_mark_weak_cld_cl(&scan_mark_weak_root_cl,
  29.214 +                                                                                    false, // Process all klasses.
  29.215 +                                                                                    true); // Need to claim CLDs.
  29.216 +
  29.217 +      G1CodeBlobClosure scan_only_code_cl(&scan_only_root_cl);
  29.218 +      G1CodeBlobClosure scan_mark_code_cl(&scan_mark_root_cl);
  29.219 +      // IM Weak code roots are handled later.
  29.220 +
  29.221 +      OopClosure* strong_root_cl;
  29.222 +      OopClosure* weak_root_cl;
  29.223 +      CLDClosure* strong_cld_cl;
  29.224 +      CLDClosure* weak_cld_cl;
  29.225 +      CodeBlobClosure* strong_code_cl;
  29.226  
  29.227        if (_g1h->g1_policy()->during_initial_mark_pause()) {
  29.228          // We also need to mark copied objects.
  29.229 -        scan_root_cl = &scan_mark_root_cl;
  29.230 -        scan_klasses_cl = &scan_mark_klasses_cl_s;
  29.231 +        strong_root_cl = &scan_mark_root_cl;
  29.232 +        weak_root_cl   = &scan_mark_weak_root_cl;
  29.233 +        strong_cld_cl  = &scan_mark_cld_cl;
  29.234 +        weak_cld_cl    = &scan_mark_weak_cld_cl;
  29.235 +        strong_code_cl = &scan_mark_code_cl;
  29.236 +      } else {
  29.237 +        strong_root_cl = &scan_only_root_cl;
  29.238 +        weak_root_cl   = &scan_only_root_cl;
  29.239 +        strong_cld_cl  = &scan_only_cld_cl;
  29.240 +        weak_cld_cl    = &scan_only_cld_cl;
  29.241 +        strong_code_cl = &scan_only_code_cl;
  29.242        }
  29.243  
  29.244 -      G1ParPushHeapRSClosure          push_heap_rs_cl(_g1h, &pss);
  29.245 -
  29.246 -      // Don't scan the scavengable methods in the code cache as part
  29.247 -      // of strong root scanning. The code roots that point into a
  29.248 -      // region in the collection set are scanned when we scan the
  29.249 -      // region's RSet.
  29.250 -      int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings;
  29.251 +
  29.252 +      G1ParPushHeapRSClosure  push_heap_rs_cl(_g1h, &pss);
  29.253  
  29.254        pss.start_strong_roots();
  29.255 -      _g1h->g1_process_strong_roots(/* is scavenging */ true,
  29.256 -                                    SharedHeap::ScanningOption(so),
  29.257 -                                    scan_root_cl,
  29.258 -                                    &push_heap_rs_cl,
  29.259 -                                    scan_klasses_cl,
  29.260 -                                    worker_id);
  29.261 +      _g1h->g1_process_roots(strong_root_cl,
  29.262 +                             weak_root_cl,
  29.263 +                             &push_heap_rs_cl,
  29.264 +                             strong_cld_cl,
  29.265 +                             weak_cld_cl,
  29.266 +                             strong_code_cl,
  29.267 +                             worker_id);
  29.268 +
  29.269        pss.end_strong_roots();
  29.270  
  29.271        {
  29.272 @@ -4855,24 +4921,31 @@
  29.273  
  29.274  void
  29.275  G1CollectedHeap::
  29.276 -g1_process_strong_roots(bool is_scavenging,
  29.277 -                        ScanningOption so,
  29.278 -                        OopClosure* scan_non_heap_roots,
  29.279 -                        OopsInHeapRegionClosure* scan_rs,
  29.280 -                        G1KlassScanClosure* scan_klasses,
  29.281 -                        uint worker_i) {
  29.282 -
  29.283 -  // First scan the strong roots
  29.284 +g1_process_roots(OopClosure* scan_non_heap_roots,
  29.285 +                 OopClosure* scan_non_heap_weak_roots,
  29.286 +                 OopsInHeapRegionClosure* scan_rs,
  29.287 +                 CLDClosure* scan_strong_clds,
  29.288 +                 CLDClosure* scan_weak_clds,
  29.289 +                 CodeBlobClosure* scan_strong_code,
  29.290 +                 uint worker_i) {
  29.291 +
  29.292 +  // First scan the shared roots.
  29.293    double ext_roots_start = os::elapsedTime();
  29.294    double closure_app_time_sec = 0.0;
  29.295  
  29.296 +  bool during_im = _g1h->g1_policy()->during_initial_mark_pause();
  29.297 +
  29.298    BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
  29.299 -
  29.300 -  process_strong_roots(false, // no scoping; this is parallel code
  29.301 -                       so,
  29.302 -                       &buf_scan_non_heap_roots,
  29.303 -                       scan_klasses
  29.304 -                       );
  29.305 +  BufferingOopClosure buf_scan_non_heap_weak_roots(scan_non_heap_weak_roots);
  29.306 +
  29.307 +  process_roots(false, // no scoping; this is parallel code
  29.308 +                SharedHeap::SO_None,
  29.309 +                &buf_scan_non_heap_roots,
  29.310 +                &buf_scan_non_heap_weak_roots,
  29.311 +                scan_strong_clds,
  29.312 +                // Initial Mark handles the weak CLDs separately.
  29.313 +                (during_im ? NULL : scan_weak_clds),
  29.314 +                scan_strong_code);
  29.315  
  29.316    // Now the CM ref_processor roots.
  29.317    if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) {
  29.318 @@ -4883,10 +4956,21 @@
  29.319      ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots);
  29.320    }
  29.321  
  29.322 +  if (during_im) {
  29.323 +    // Barrier to make sure all workers passed
  29.324 +    // the strong CLD and strong nmethods phases.
  29.325 +    active_strong_roots_scope()->wait_until_all_workers_done_with_threads(n_par_threads());
  29.326 +
  29.327 +    // Now take the complement of the strong CLDs.
  29.328 +    ClassLoaderDataGraph::roots_cld_do(NULL, scan_weak_clds);
  29.329 +  }
  29.330 +
  29.331    // Finish up any enqueued closure apps (attributed as object copy time).
  29.332    buf_scan_non_heap_roots.done();
  29.333 -
  29.334 -  double obj_copy_time_sec = buf_scan_non_heap_roots.closure_app_seconds();
  29.335 +  buf_scan_non_heap_weak_roots.done();
  29.336 +
  29.337 +  double obj_copy_time_sec = buf_scan_non_heap_roots.closure_app_seconds()
  29.338 +      + buf_scan_non_heap_weak_roots.closure_app_seconds();
  29.339  
  29.340    g1_policy()->phase_times()->record_obj_copy_time(worker_i, obj_copy_time_sec * 1000.0);
  29.341  
  29.342 @@ -4910,22 +4994,10 @@
  29.343    }
  29.344    g1_policy()->phase_times()->record_satb_filtering_time(worker_i, satb_filtering_ms);
  29.345  
  29.346 -  // If this is an initial mark pause, and we're not scanning
  29.347 -  // the entire code cache, we need to mark the oops in the
  29.348 -  // strong code root lists for the regions that are not in
  29.349 -  // the collection set.
  29.350 -  // Note all threads participate in this set of root tasks.
  29.351 -  double mark_strong_code_roots_ms = 0.0;
  29.352 -  if (g1_policy()->during_initial_mark_pause() && !(so & SO_AllCodeCache)) {
  29.353 -    double mark_strong_roots_start = os::elapsedTime();
  29.354 -    mark_strong_code_roots(worker_i);
  29.355 -    mark_strong_code_roots_ms = (os::elapsedTime() - mark_strong_roots_start) * 1000.0;
  29.356 -  }
  29.357 -  g1_policy()->phase_times()->record_strong_code_root_mark_time(worker_i, mark_strong_code_roots_ms);
  29.358 -
  29.359    // Now scan the complement of the collection set.
  29.360 -  CodeBlobToOopClosure eager_scan_code_roots(scan_non_heap_roots, true /* do_marking */);
  29.361 -  g1_rem_set()->oops_into_collection_set_do(scan_rs, &eager_scan_code_roots, worker_i);
  29.362 +  MarkingCodeBlobClosure scavenge_cs_nmethods(scan_non_heap_weak_roots, CodeBlobToOopClosure::FixRelocations);
  29.363 +
  29.364 +  g1_rem_set()->oops_into_collection_set_do(scan_rs, &scavenge_cs_nmethods, worker_i);
  29.365  
  29.366    _process_strong_tasks->all_tasks_completed();
  29.367  }
  29.368 @@ -4947,7 +5019,8 @@
  29.369    bool _do_in_parallel;
  29.370  public:
  29.371    G1StringSymbolTableUnlinkTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols) :
  29.372 -    AbstractGangTask("Par String/Symbol table unlink"), _is_alive(is_alive),
  29.373 +    AbstractGangTask("String/Symbol Unlinking"),
  29.374 +    _is_alive(is_alive),
  29.375      _do_in_parallel(G1CollectedHeap::use_parallel_gc_threads()),
  29.376      _process_strings(process_strings), _strings_processed(0), _strings_removed(0),
  29.377      _process_symbols(process_symbols), _symbols_processed(0), _symbols_removed(0) {
  29.378 @@ -4969,6 +5042,14 @@
  29.379      guarantee(!_process_symbols || !_do_in_parallel || SymbolTable::parallel_claimed_index() >= _initial_symbol_table_size,
  29.380                err_msg("claim value "INT32_FORMAT" after unlink less than initial symbol table size "INT32_FORMAT,
  29.381                        SymbolTable::parallel_claimed_index(), _initial_symbol_table_size));
  29.382 +
  29.383 +    if (G1TraceStringSymbolTableScrubbing) {
  29.384 +      gclog_or_tty->print_cr("Cleaned string and symbol table, "
  29.385 +                             "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, "
  29.386 +                             "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed",
  29.387 +                             strings_processed(), strings_removed(),
  29.388 +                             symbols_processed(), symbols_removed());
  29.389 +    }
  29.390    }
  29.391  
  29.392    void work(uint worker_id) {
  29.393 @@ -5004,12 +5085,279 @@
  29.394    size_t symbols_removed()   const { return (size_t)_symbols_removed; }
  29.395  };
  29.396  
  29.397 -void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive,
  29.398 -                                                     bool process_strings, bool process_symbols) {
  29.399 +class G1CodeCacheUnloadingTask VALUE_OBJ_CLASS_SPEC {
  29.400 +private:
  29.401 +  static Monitor* _lock;
  29.402 +
  29.403 +  BoolObjectClosure* const _is_alive;
  29.404 +  const bool               _unloading_occurred;
  29.405 +  const uint               _num_workers;
  29.406 +
  29.407 +  // Variables used to claim nmethods.
  29.408 +  nmethod* _first_nmethod;
  29.409 +  volatile nmethod* _claimed_nmethod;
  29.410 +
  29.411 +  // The list of nmethods that need to be processed by the second pass.
  29.412 +  volatile nmethod* _postponed_list;
  29.413 +  volatile uint     _num_entered_barrier;
  29.414 +
  29.415 + public:
  29.416 +  G1CodeCacheUnloadingTask(uint num_workers, BoolObjectClosure* is_alive, bool unloading_occurred) :
  29.417 +      _is_alive(is_alive),
  29.418 +      _unloading_occurred(unloading_occurred),
  29.419 +      _num_workers(num_workers),
  29.420 +      _first_nmethod(NULL),
  29.421 +      _claimed_nmethod(NULL),
  29.422 +      _postponed_list(NULL),
  29.423 +      _num_entered_barrier(0)
  29.424 +  {
  29.425 +    nmethod::increase_unloading_clock();
  29.426 +    _first_nmethod = CodeCache::alive_nmethod(CodeCache::first());
  29.427 +    _claimed_nmethod = (volatile nmethod*)_first_nmethod;
  29.428 +  }
  29.429 +
  29.430 +  ~G1CodeCacheUnloadingTask() {
  29.431 +    CodeCache::verify_clean_inline_caches();
  29.432 +
  29.433 +    CodeCache::set_needs_cache_clean(false);
  29.434 +    guarantee(CodeCache::scavenge_root_nmethods() == NULL, "Must be");
  29.435 +
  29.436 +    CodeCache::verify_icholder_relocations();
  29.437 +  }
  29.438 +
  29.439 + private:
  29.440 +  void add_to_postponed_list(nmethod* nm) {
  29.441 +      nmethod* old;
  29.442 +      do {
  29.443 +        old = (nmethod*)_postponed_list;
  29.444 +        nm->set_unloading_next(old);
  29.445 +      } while ((nmethod*)Atomic::cmpxchg_ptr(nm, &_postponed_list, old) != old);
  29.446 +  }
  29.447 +
  29.448 +  void clean_nmethod(nmethod* nm) {
  29.449 +    bool postponed = nm->do_unloading_parallel(_is_alive, _unloading_occurred);
  29.450 +
  29.451 +    if (postponed) {
  29.452 +      // This nmethod referred to an nmethod that has not been cleaned/unloaded yet.
  29.453 +      add_to_postponed_list(nm);
  29.454 +    }
  29.455 +
  29.456 +    // Mark that this thread has been cleaned/unloaded.
  29.457 +    // After this call, it will be safe to ask if this nmethod was unloaded or not.
  29.458 +    nm->set_unloading_clock(nmethod::global_unloading_clock());
  29.459 +  }
  29.460 +
  29.461 +  void clean_nmethod_postponed(nmethod* nm) {
  29.462 +    nm->do_unloading_parallel_postponed(_is_alive, _unloading_occurred);
  29.463 +  }
  29.464 +
  29.465 +  static const int MaxClaimNmethods = 16;
  29.466 +
  29.467 +  void claim_nmethods(nmethod** claimed_nmethods, int *num_claimed_nmethods) {
  29.468 +    nmethod* first;
  29.469 +    nmethod* last;
  29.470 +
  29.471 +    do {
  29.472 +      *num_claimed_nmethods = 0;
  29.473 +
  29.474 +      first = last = (nmethod*)_claimed_nmethod;
  29.475 +
  29.476 +      if (first != NULL) {
  29.477 +        for (int i = 0; i < MaxClaimNmethods; i++) {
  29.478 +          last = CodeCache::alive_nmethod(CodeCache::next(last));
  29.479 +
  29.480 +          if (last == NULL) {
  29.481 +            break;
  29.482 +          }
  29.483 +
  29.484 +          claimed_nmethods[i] = last;
  29.485 +          (*num_claimed_nmethods)++;
  29.486 +        }
  29.487 +      }
  29.488 +
  29.489 +    } while ((nmethod*)Atomic::cmpxchg_ptr(last, &_claimed_nmethod, first) != first);
  29.490 +  }
  29.491 +
  29.492 +  nmethod* claim_postponed_nmethod() {
  29.493 +    nmethod* claim;
  29.494 +    nmethod* next;
  29.495 +
  29.496 +    do {
  29.497 +      claim = (nmethod*)_postponed_list;
  29.498 +      if (claim == NULL) {
  29.499 +        return NULL;
  29.500 +      }
  29.501 +
  29.502 +      next = claim->unloading_next();
  29.503 +
  29.504 +    } while ((nmethod*)Atomic::cmpxchg_ptr(next, &_postponed_list, claim) != claim);
  29.505 +
  29.506 +    return claim;
  29.507 +  }
  29.508 +
  29.509 + public:
  29.510 +  // Mark that we're done with the first pass of nmethod cleaning.
  29.511 +  void barrier_mark(uint worker_id) {
  29.512 +    MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
  29.513 +    _num_entered_barrier++;
  29.514 +    if (_num_entered_barrier == _num_workers) {
  29.515 +      ml.notify_all();
  29.516 +    }
  29.517 +  }
  29.518 +
  29.519 +  // See if we have to wait for the other workers to
  29.520 +  // finish their first-pass nmethod cleaning work.
  29.521 +  void barrier_wait(uint worker_id) {
  29.522 +    if (_num_entered_barrier < _num_workers) {
  29.523 +      MonitorLockerEx ml(_lock, Mutex::_no_safepoint_check_flag);
  29.524 +      while (_num_entered_barrier < _num_workers) {
  29.525 +          ml.wait(Mutex::_no_safepoint_check_flag, 0, false);
  29.526 +      }
  29.527 +    }
  29.528 +  }
  29.529 +
  29.530 +  // Cleaning and unloading of nmethods. Some work has to be postponed
  29.531 +  // to the second pass, when we know which nmethods survive.
  29.532 +  void work_first_pass(uint worker_id) {
  29.533 +    // The first nmethods is claimed by the first worker.
  29.534 +    if (worker_id == 0 && _first_nmethod != NULL) {
  29.535 +      clean_nmethod(_first_nmethod);
  29.536 +      _first_nmethod = NULL;
  29.537 +    }
  29.538 +
  29.539 +    int num_claimed_nmethods;
  29.540 +    nmethod* claimed_nmethods[MaxClaimNmethods];
  29.541 +
  29.542 +    while (true) {
  29.543 +      claim_nmethods(claimed_nmethods, &num_claimed_nmethods);
  29.544 +
  29.545 +      if (num_claimed_nmethods == 0) {
  29.546 +        break;
  29.547 +      }
  29.548 +
  29.549 +      for (int i = 0; i < num_claimed_nmethods; i++) {
  29.550 +        clean_nmethod(claimed_nmethods[i]);
  29.551 +      }
  29.552 +    }
  29.553 +  }
  29.554 +
  29.555 +  void work_second_pass(uint worker_id) {
  29.556 +    nmethod* nm;
  29.557 +    // Take care of postponed nmethods.
  29.558 +    while ((nm = claim_postponed_nmethod()) != NULL) {
  29.559 +      clean_nmethod_postponed(nm);
  29.560 +    }
  29.561 +  }
  29.562 +};
  29.563 +
  29.564 +Monitor* G1CodeCacheUnloadingTask::_lock = new Monitor(Mutex::leaf, "Code Cache Unload lock");
  29.565 +
  29.566 +class G1KlassCleaningTask : public StackObj {
  29.567 +  BoolObjectClosure*                      _is_alive;
  29.568 +  volatile jint                           _clean_klass_tree_claimed;
  29.569 +  ClassLoaderDataGraphKlassIteratorAtomic _klass_iterator;
  29.570 +
  29.571 + public:
  29.572 +  G1KlassCleaningTask(BoolObjectClosure* is_alive) :
  29.573 +      _is_alive(is_alive),
  29.574 +      _clean_klass_tree_claimed(0),
  29.575 +      _klass_iterator() {
  29.576 +  }
  29.577 +
  29.578 + private:
  29.579 +  bool claim_clean_klass_tree_task() {
  29.580 +    if (_clean_klass_tree_claimed) {
  29.581 +      return false;
  29.582 +    }
  29.583 +
  29.584 +    return Atomic::cmpxchg(1, (jint*)&_clean_klass_tree_claimed, 0) == 0;
  29.585 +  }
  29.586 +
  29.587 +  InstanceKlass* claim_next_klass() {
  29.588 +    Klass* klass;
  29.589 +    do {
  29.590 +      klass =_klass_iterator.next_klass();
  29.591 +    } while (klass != NULL && !klass->oop_is_instance());
  29.592 +
  29.593 +    return (InstanceKlass*)klass;
  29.594 +  }
  29.595 +
  29.596 +public:
  29.597 +
  29.598 +  void clean_klass(InstanceKlass* ik) {
  29.599 +    ik->clean_implementors_list(_is_alive);
  29.600 +    ik->clean_method_data(_is_alive);
  29.601 +
  29.602 +    // G1 specific cleanup work that has
  29.603 +    // been moved here to be done in parallel.
  29.604 +    ik->clean_dependent_nmethods();
  29.605 +  }
  29.606 +
  29.607 +  void work() {
  29.608 +    ResourceMark rm;
  29.609 +
  29.610 +    // One worker will clean the subklass/sibling klass tree.
  29.611 +    if (claim_clean_klass_tree_task()) {
  29.612 +      Klass::clean_subklass_tree(_is_alive);
  29.613 +    }
  29.614 +
  29.615 +    // All workers will help cleaning the classes,
  29.616 +    InstanceKlass* klass;
  29.617 +    while ((klass = claim_next_klass()) != NULL) {
  29.618 +      clean_klass(klass);
  29.619 +    }
  29.620 +  }
  29.621 +};
  29.622 +
  29.623 +// To minimize the remark pause times, the tasks below are done in parallel.
  29.624 +class G1ParallelCleaningTask : public AbstractGangTask {
  29.625 +private:
  29.626 +  G1StringSymbolTableUnlinkTask _string_symbol_task;
  29.627 +  G1CodeCacheUnloadingTask      _code_cache_task;
  29.628 +  G1KlassCleaningTask           _klass_cleaning_task;
  29.629 +
  29.630 +public:
  29.631 +  // The constructor is run in the VMThread.
  29.632 +  G1ParallelCleaningTask(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, uint num_workers, bool unloading_occurred) :
  29.633 +      AbstractGangTask("Parallel Cleaning"),
  29.634 +      _string_symbol_task(is_alive, process_strings, process_symbols),
  29.635 +      _code_cache_task(num_workers, is_alive, unloading_occurred),
  29.636 +      _klass_cleaning_task(is_alive) {
  29.637 +  }
  29.638 +
  29.639 +  // The parallel work done by all worker threads.
  29.640 +  void work(uint worker_id) {
  29.641 +    // Do first pass of code cache cleaning.
  29.642 +    _code_cache_task.work_first_pass(worker_id);
  29.643 +
  29.644 +    // Let the threads, mark that the first pass is done.
  29.645 +    _code_cache_task.barrier_mark(worker_id);
  29.646 +
  29.647 +    // Clean the Strings and Symbols.
  29.648 +    _string_symbol_task.work(worker_id);
  29.649 +
  29.650 +    // Wait for all workers to finish the first code cache cleaning pass.
  29.651 +    _code_cache_task.barrier_wait(worker_id);
  29.652 +
  29.653 +    // Do the second code cache cleaning work, which realize on
  29.654 +    // the liveness information gathered during the first pass.
  29.655 +    _code_cache_task.work_second_pass(worker_id);
  29.656 +
  29.657 +    // Clean all klasses that were not unloaded.
  29.658 +    _klass_cleaning_task.work();
  29.659 +  }
  29.660 +};
  29.661 +
  29.662 +
  29.663 +void G1CollectedHeap::parallel_cleaning(BoolObjectClosure* is_alive,
  29.664 +                                        bool process_strings,
  29.665 +                                        bool process_symbols,
  29.666 +                                        bool class_unloading_occurred) {
  29.667    uint n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
  29.668 -                   _g1h->workers()->active_workers() : 1);
  29.669 -
  29.670 -  G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols);
  29.671 +                    workers()->active_workers() : 1);
  29.672 +
  29.673 +  G1ParallelCleaningTask g1_unlink_task(is_alive, process_strings, process_symbols,
  29.674 +                                        n_workers, class_unloading_occurred);
  29.675    if (G1CollectedHeap::use_parallel_gc_threads()) {
  29.676      set_par_threads(n_workers);
  29.677      workers()->run_task(&g1_unlink_task);
  29.678 @@ -5017,12 +5365,21 @@
  29.679    } else {
  29.680      g1_unlink_task.work(0);
  29.681    }
  29.682 -  if (G1TraceStringSymbolTableScrubbing) {
  29.683 -    gclog_or_tty->print_cr("Cleaned string and symbol table, "
  29.684 -                           "strings: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed, "
  29.685 -                           "symbols: "SIZE_FORMAT" processed, "SIZE_FORMAT" removed",
  29.686 -                           g1_unlink_task.strings_processed(), g1_unlink_task.strings_removed(),
  29.687 -                           g1_unlink_task.symbols_processed(), g1_unlink_task.symbols_removed());
  29.688 +}
  29.689 +
  29.690 +void G1CollectedHeap::unlink_string_and_symbol_table(BoolObjectClosure* is_alive,
  29.691 +                                                     bool process_strings, bool process_symbols) {
  29.692 +  {
  29.693 +    uint n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
  29.694 +                     _g1h->workers()->active_workers() : 1);
  29.695 +    G1StringSymbolTableUnlinkTask g1_unlink_task(is_alive, process_strings, process_symbols);
  29.696 +    if (G1CollectedHeap::use_parallel_gc_threads()) {
  29.697 +      set_par_threads(n_workers);
  29.698 +      workers()->run_task(&g1_unlink_task);
  29.699 +      set_par_threads(0);
  29.700 +    } else {
  29.701 +      g1_unlink_task.work(0);
  29.702 +    }
  29.703    }
  29.704  
  29.705    if (G1StringDedup::is_enabled()) {
  29.706 @@ -5615,6 +5972,10 @@
  29.707  
  29.708    {
  29.709      StrongRootsScope srs(this);
  29.710 +    // InitialMark needs claim bits to keep track of the marked-through CLDs.
  29.711 +    if (g1_policy()->during_initial_mark_pause()) {
  29.712 +      ClassLoaderDataGraph::clear_claimed_marks();
  29.713 +    }
  29.714  
  29.715      if (G1CollectedHeap::use_parallel_gc_threads()) {
  29.716        // The individual threads will set their evac-failure closures.
  29.717 @@ -6566,106 +6927,6 @@
  29.718    g1_policy()->phase_times()->record_strong_code_root_purge_time(purge_time_ms);
  29.719  }
  29.720  
  29.721 -// Mark all the code roots that point into regions *not* in the
  29.722 -// collection set.
  29.723 -//
  29.724 -// Note we do not want to use a "marking" CodeBlobToOopClosure while
  29.725 -// walking the the code roots lists of regions not in the collection
  29.726 -// set. Suppose we have an nmethod (M) that points to objects in two
  29.727 -// separate regions - one in the collection set (R1) and one not (R2).
  29.728 -// Using a "marking" CodeBlobToOopClosure here would result in "marking"
  29.729 -// nmethod M when walking the code roots for R1. When we come to scan
  29.730 -// the code roots for R2, we would see that M is already marked and it
  29.731 -// would be skipped and the objects in R2 that are referenced from M
  29.732 -// would not be evacuated.
  29.733 -
  29.734 -class MarkStrongCodeRootCodeBlobClosure: public CodeBlobClosure {
  29.735 -
  29.736 -  class MarkStrongCodeRootOopClosure: public OopClosure {
  29.737 -    ConcurrentMark* _cm;
  29.738 -    HeapRegion* _hr;
  29.739 -    uint _worker_id;
  29.740 -
  29.741 -    template <class T> void do_oop_work(T* p) {
  29.742 -      T heap_oop = oopDesc::load_heap_oop(p);
  29.743 -      if (!oopDesc::is_null(heap_oop)) {
  29.744 -        oop obj = oopDesc::decode_heap_oop_not_null(heap_oop);
  29.745 -        // Only mark objects in the region (which is assumed
  29.746 -        // to be not in the collection set).
  29.747 -        if (_hr->is_in(obj)) {
  29.748 -          _cm->grayRoot(obj, (size_t) obj->size(), _worker_id);
  29.749 -        }
  29.750 -      }
  29.751 -    }
  29.752 -
  29.753 -  public:
  29.754 -    MarkStrongCodeRootOopClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id) :
  29.755 -      _cm(cm), _hr(hr), _worker_id(worker_id) {
  29.756 -      assert(!_hr->in_collection_set(), "sanity");
  29.757 -    }
  29.758 -
  29.759 -    void do_oop(narrowOop* p) { do_oop_work(p); }
  29.760 -    void do_oop(oop* p)       { do_oop_work(p); }
  29.761 -  };
  29.762 -
  29.763 -  MarkStrongCodeRootOopClosure _oop_cl;
  29.764 -
  29.765 -public:
  29.766 -  MarkStrongCodeRootCodeBlobClosure(ConcurrentMark* cm, HeapRegion* hr, uint worker_id):
  29.767 -    _oop_cl(cm, hr, worker_id) {}
  29.768 -
  29.769 -  void do_code_blob(CodeBlob* cb) {
  29.770 -    nmethod* nm = (cb == NULL) ? NULL : cb->as_nmethod_or_null();
  29.771 -    if (nm != NULL) {
  29.772 -      nm->oops_do(&_oop_cl);
  29.773 -    }
  29.774 -  }
  29.775 -};
  29.776 -
  29.777 -class MarkStrongCodeRootsHRClosure: public HeapRegionClosure {
  29.778 -  G1CollectedHeap* _g1h;
  29.779 -  uint _worker_id;
  29.780 -
  29.781 -public:
  29.782 -  MarkStrongCodeRootsHRClosure(G1CollectedHeap* g1h, uint worker_id) :
  29.783 -    _g1h(g1h), _worker_id(worker_id) {}
  29.784 -
  29.785 -  bool doHeapRegion(HeapRegion *hr) {
  29.786 -    HeapRegionRemSet* hrrs = hr->rem_set();
  29.787 -    if (hr->continuesHumongous()) {
  29.788 -      // Code roots should never be attached to a continuation of a humongous region
  29.789 -      assert(hrrs->strong_code_roots_list_length() == 0,
  29.790 -             err_msg("code roots should never be attached to continuations of humongous region "HR_FORMAT
  29.791 -                     " starting at "HR_FORMAT", but has "SIZE_FORMAT,
  29.792 -                     HR_FORMAT_PARAMS(hr), HR_FORMAT_PARAMS(hr->humongous_start_region()),
  29.793 -                     hrrs->strong_code_roots_list_length()));
  29.794 -      return false;
  29.795 -    }
  29.796 -
  29.797 -    if (hr->in_collection_set()) {
  29.798 -      // Don't mark code roots into regions in the collection set here.
  29.799 -      // They will be marked when we scan them.
  29.800 -      return false;
  29.801 -    }
  29.802 -
  29.803 -    MarkStrongCodeRootCodeBlobClosure cb_cl(_g1h->concurrent_mark(), hr, _worker_id);
  29.804 -    hr->strong_code_roots_do(&cb_cl);
  29.805 -    return false;
  29.806 -  }
  29.807 -};
  29.808 -
  29.809 -void G1CollectedHeap::mark_strong_code_roots(uint worker_id) {
  29.810 -  MarkStrongCodeRootsHRClosure cl(this, worker_id);
  29.811 -  if (G1CollectedHeap::use_parallel_gc_threads()) {
  29.812 -    heap_region_par_iterate_chunked(&cl,
  29.813 -                                    worker_id,
  29.814 -                                    workers()->active_workers(),
  29.815 -                                    HeapRegion::ParMarkRootClaimValue);
  29.816 -  } else {
  29.817 -    heap_region_iterate(&cl);
  29.818 -  }
  29.819 -}
  29.820 -
  29.821  class RebuildStrongCodeRootClosure: public CodeBlobClosure {
  29.822    G1CollectedHeap* _g1h;
  29.823  

    30.1 --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Tue Jul 01 09:03:55 2014 +0200
    30.2 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp	Mon Jul 07 10:12:40 2014 +0200
    30.3 @@ -210,6 +210,7 @@
    30.4  class RefineCardTableEntryClosure;
    30.5  
    30.6  class G1CollectedHeap : public SharedHeap {
    30.7 +  friend class VM_CollectForMetadataAllocation;
    30.8    friend class VM_G1CollectForAllocation;
    30.9    friend class VM_G1CollectFull;
   30.10    friend class VM_G1IncCollectionPause;
   30.11 @@ -219,7 +220,7 @@
   30.12    friend class OldGCAllocRegion;
   30.13  
   30.14    // Closures used in implementation.
   30.15 -  template <G1Barrier barrier, bool do_mark_object>
   30.16 +  template <G1Barrier barrier, G1Mark do_mark_object>
   30.17    friend class G1ParCopyClosure;
   30.18    friend class G1IsAliveClosure;
   30.19    friend class G1EvacuateFollowersClosure;
   30.20 @@ -346,6 +347,9 @@
   30.21    // It initializes the GC alloc regions at the start of a GC.
   30.22    void init_gc_alloc_regions(EvacuationInfo& evacuation_info);
   30.23  
   30.24 +  // Setup the retained old gc alloc region as the currrent old gc alloc region.
   30.25 +  void use_retained_old_gc_alloc_region(EvacuationInfo& evacuation_info);
   30.26 +
   30.27    // It releases the GC alloc regions at the end of a GC.
   30.28    void release_gc_alloc_regions(uint no_of_gc_workers, EvacuationInfo& evacuation_info);
   30.29  
   30.30 @@ -827,12 +831,13 @@
   30.31    // param is for use with parallel roots processing, and should be
   30.32    // the "i" of the calling parallel worker thread's work(i) function.
   30.33    // In the sequential case this param will be ignored.
   30.34 -  void g1_process_strong_roots(bool is_scavenging,
   30.35 -                               ScanningOption so,
   30.36 -                               OopClosure* scan_non_heap_roots,
   30.37 -                               OopsInHeapRegionClosure* scan_rs,
   30.38 -                               G1KlassScanClosure* scan_klasses,
   30.39 -                               uint worker_i);
   30.40 +  void g1_process_roots(OopClosure* scan_non_heap_roots,
   30.41 +                        OopClosure* scan_non_heap_weak_roots,
   30.42 +                        OopsInHeapRegionClosure* scan_rs,
   30.43 +                        CLDClosure* scan_strong_clds,
   30.44 +                        CLDClosure* scan_weak_clds,
   30.45 +                        CodeBlobClosure* scan_strong_code,
   30.46 +                        uint worker_i);
   30.47  
   30.48    // Notifies all the necessary spaces that the committed space has
   30.49    // been updated (either expanded or shrunk). It should be called
   30.50 @@ -1025,7 +1030,7 @@
   30.51    // of G1CollectedHeap::_gc_time_stamp.
   30.52    unsigned int* _worker_cset_start_region_time_stamp;
   30.53  
   30.54 -  enum G1H_process_strong_roots_tasks {
   30.55 +  enum G1H_process_roots_tasks {
   30.56      G1H_PS_filter_satb_buffers,
   30.57      G1H_PS_refProcessor_oops_do,
   30.58      // Leave this one last.
   30.59 @@ -1596,10 +1601,6 @@
   30.60    // Free up superfluous code root memory.
   30.61    void purge_code_root_memory();
   30.62  
   30.63 -  // During an initial mark pause, mark all the code roots that
   30.64 -  // point into regions *not* in the collection set.
   30.65 -  void mark_strong_code_roots(uint worker_id);
   30.66 -
   30.67    // Rebuild the stong code root lists for each region
   30.68    // after a full GC
   30.69    void rebuild_strong_code_roots();
   30.70 @@ -1608,6 +1609,9 @@
   30.71    // in symbol table, possibly in parallel.
   30.72    void unlink_string_and_symbol_table(BoolObjectClosure* is_alive, bool unlink_strings = true, bool unlink_symbols = true);
   30.73  
   30.74 +  // Parallel phase of unloading/cleaning after G1 concurrent mark.
   30.75 +  void parallel_cleaning(BoolObjectClosure* is_alive, bool process_strings, bool process_symbols, bool class_unloading_occurred);
   30.76 +
   30.77    // Redirty logged cards in the refinement queue.
   30.78    void redirty_logged_cards();
   30.79    // Verification

    31.1 --- a/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp	Tue Jul 01 09:03:55 2014 +0200
    31.2 +++ b/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp	Mon Jul 07 10:12:40 2014 +0200
    31.3 @@ -71,6 +71,9 @@
    31.4    bool _during_initial_mark;
    31.5    bool _during_conc_mark;
    31.6    uint _worker_id;
    31.7 +  HeapWord* _end_of_last_gap;
    31.8 +  HeapWord* _last_gap_threshold;
    31.9 +  HeapWord* _last_obj_threshold;
   31.10  
   31.11  public:
   31.12    RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm,
   31.13 @@ -83,7 +86,10 @@
   31.14      _update_rset_cl(update_rset_cl),
   31.15      _during_initial_mark(during_initial_mark),
   31.16      _during_conc_mark(during_conc_mark),
   31.17 -    _worker_id(worker_id) { }
   31.18 +    _worker_id(worker_id),
   31.19 +    _end_of_last_gap(hr->bottom()),
   31.20 +    _last_gap_threshold(hr->bottom()),
   31.21 +    _last_obj_threshold(hr->bottom()) { }
   31.22  
   31.23    size_t marked_bytes() { return _marked_bytes; }
   31.24  
   31.25 @@ -107,7 +113,12 @@
   31.26      HeapWord* obj_addr = (HeapWord*) obj;
   31.27      assert(_hr->is_in(obj_addr), "sanity");
   31.28      size_t obj_size = obj->size();
   31.29 -    _hr->update_bot_for_object(obj_addr, obj_size);
   31.30 +    HeapWord* obj_end = obj_addr + obj_size;
   31.31 +
   31.32 +    if (_end_of_last_gap != obj_addr) {
   31.33 +      // there was a gap before obj_addr
   31.34 +      _last_gap_threshold = _hr->cross_threshold(_end_of_last_gap, obj_addr);
   31.35 +    }
   31.36  
   31.37      if (obj->is_forwarded() && obj->forwardee() == obj) {
   31.38        // The object failed to move.
   31.39 @@ -115,7 +126,9 @@
   31.40        // We consider all objects that we find self-forwarded to be
   31.41        // live. What we'll do is that we'll update the prev marking
   31.42        // info so that they are all under PTAMS and explicitly marked.
   31.43 -      _cm->markPrev(obj);
   31.44 +      if (!_cm->isPrevMarked(obj)) {
   31.45 +        _cm->markPrev(obj);
   31.46 +      }
   31.47        if (_during_initial_mark) {
   31.48          // For the next marking info we'll only mark the
   31.49          // self-forwarded objects explicitly if we are during
   31.50 @@ -145,13 +158,18 @@
   31.51        // remembered set entries missing given that we skipped cards on
   31.52        // the collection set. So, we'll recreate such entries now.
   31.53        obj->oop_iterate(_update_rset_cl);
   31.54 -      assert(_cm->isPrevMarked(obj), "Should be marked!");
   31.55      } else {
   31.56 +
   31.57        // The object has been either evacuated or is dead. Fill it with a
   31.58        // dummy object.
   31.59 -      MemRegion mr((HeapWord*) obj, obj_size);
   31.60 +      MemRegion mr(obj_addr, obj_size);
   31.61        CollectedHeap::fill_with_object(mr);
   31.62 +
   31.63 +      // must nuke all dead objects which we skipped when iterating over the region
   31.64 +      _cm->clearRangePrevBitmap(MemRegion(_end_of_last_gap, obj_end));
   31.65      }
   31.66 +    _end_of_last_gap = obj_end;
   31.67 +    _last_obj_threshold = _hr->cross_threshold(obj_addr, obj_end);
   31.68    }
   31.69  };
   31.70  
   31.71 @@ -182,13 +200,6 @@
   31.72                                              during_conc_mark,
   31.73                                              _worker_id);
   31.74  
   31.75 -        MemRegion mr(hr->bottom(), hr->end());
   31.76 -        // We'll recreate the prev marking info so we'll first clear
   31.77 -        // the prev bitmap range for this region. We never mark any
   31.78 -        // CSet objects explicitly so the next bitmap range should be
   31.79 -        // cleared anyway.
   31.80 -        _cm->clearRangePrevBitmap(mr);
   31.81 -
   31.82          hr->note_self_forwarding_removal_start(during_initial_mark,
   31.83                                                 during_conc_mark);
   31.84  

    32.1 --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp	Tue Jul 01 09:03:55 2014 +0200
    32.2 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.cpp	Mon Jul 07 10:12:40 2014 +0200
    32.3 @@ -166,7 +166,6 @@
    32.4    _last_update_rs_processed_buffers(_max_gc_threads, "%d"),
    32.5    _last_scan_rs_times_ms(_max_gc_threads, "%.1lf"),
    32.6    _last_strong_code_root_scan_times_ms(_max_gc_threads, "%.1lf"),
    32.7 -  _last_strong_code_root_mark_times_ms(_max_gc_threads, "%.1lf"),
    32.8    _last_obj_copy_times_ms(_max_gc_threads, "%.1lf"),
    32.9    _last_termination_times_ms(_max_gc_threads, "%.1lf"),
   32.10    _last_termination_attempts(_max_gc_threads, SIZE_FORMAT),
   32.11 @@ -193,7 +192,6 @@
   32.12    _last_update_rs_processed_buffers.reset();
   32.13    _last_scan_rs_times_ms.reset();
   32.14    _last_strong_code_root_scan_times_ms.reset();
   32.15 -  _last_strong_code_root_mark_times_ms.reset();
   32.16    _last_obj_copy_times_ms.reset();
   32.17    _last_termination_times_ms.reset();
   32.18    _last_termination_attempts.reset();
   32.19 @@ -214,7 +212,6 @@
   32.20    _last_update_rs_processed_buffers.verify();
   32.21    _last_scan_rs_times_ms.verify();
   32.22    _last_strong_code_root_scan_times_ms.verify();
   32.23 -  _last_strong_code_root_mark_times_ms.verify();
   32.24    _last_obj_copy_times_ms.verify();
   32.25    _last_termination_times_ms.verify();
   32.26    _last_termination_attempts.verify();
   32.27 @@ -229,7 +226,6 @@
   32.28                                 _last_update_rs_times_ms.get(i) +
   32.29                                 _last_scan_rs_times_ms.get(i) +
   32.30                                 _last_strong_code_root_scan_times_ms.get(i) +
   32.31 -                               _last_strong_code_root_mark_times_ms.get(i) +
   32.32                                 _last_obj_copy_times_ms.get(i) +
   32.33                                 _last_termination_times_ms.get(i);
   32.34  
   32.35 @@ -301,9 +297,6 @@
   32.36      if (_last_satb_filtering_times_ms.sum() > 0.0) {
   32.37        _last_satb_filtering_times_ms.print(2, "SATB Filtering (ms)");
   32.38      }
   32.39 -    if (_last_strong_code_root_mark_times_ms.sum() > 0.0) {
   32.40 -     _last_strong_code_root_mark_times_ms.print(2, "Code Root Marking (ms)");
   32.41 -    }
   32.42      _last_update_rs_times_ms.print(2, "Update RS (ms)");
   32.43        _last_update_rs_processed_buffers.print(3, "Processed Buffers");
   32.44      _last_scan_rs_times_ms.print(2, "Scan RS (ms)");
   32.45 @@ -321,9 +314,6 @@
   32.46      if (_last_satb_filtering_times_ms.sum() > 0.0) {
   32.47        _last_satb_filtering_times_ms.print(1, "SATB Filtering (ms)");
   32.48      }
   32.49 -    if (_last_strong_code_root_mark_times_ms.sum() > 0.0) {
   32.50 -      _last_strong_code_root_mark_times_ms.print(1, "Code Root Marking (ms)");
   32.51 -    }
   32.52      _last_update_rs_times_ms.print(1, "Update RS (ms)");
   32.53        _last_update_rs_processed_buffers.print(2, "Processed Buffers");
   32.54      _last_scan_rs_times_ms.print(1, "Scan RS (ms)");

    33.1 --- a/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp	Tue Jul 01 09:03:55 2014 +0200
    33.2 +++ b/src/share/vm/gc_implementation/g1/g1GCPhaseTimes.hpp	Mon Jul 07 10:12:40 2014 +0200
    33.3 @@ -120,7 +120,6 @@
    33.4    WorkerDataArray<int>    _last_update_rs_processed_buffers;
    33.5    WorkerDataArray<double> _last_scan_rs_times_ms;
    33.6    WorkerDataArray<double> _last_strong_code_root_scan_times_ms;
    33.7 -  WorkerDataArray<double> _last_strong_code_root_mark_times_ms;
    33.8    WorkerDataArray<double> _last_obj_copy_times_ms;
    33.9    WorkerDataArray<double> _last_termination_times_ms;
   33.10    WorkerDataArray<size_t> _last_termination_attempts;
   33.11 @@ -199,10 +198,6 @@
   33.12      _last_strong_code_root_scan_times_ms.set(worker_i, ms);
   33.13    }
   33.14  
   33.15 -  void record_strong_code_root_mark_time(uint worker_i, double ms) {
   33.16 -    _last_strong_code_root_mark_times_ms.set(worker_i, ms);
   33.17 -  }
   33.18 -
   33.19    void record_obj_copy_time(uint worker_i, double ms) {
   33.20      _last_obj_copy_times_ms.set(worker_i, ms);
   33.21    }
   33.22 @@ -369,10 +364,6 @@
   33.23      return _last_strong_code_root_scan_times_ms.average();
   33.24    }
   33.25  
   33.26 -  double average_last_strong_code_root_mark_time(){
   33.27 -    return _last_strong_code_root_mark_times_ms.average();
   33.28 -  }
   33.29 -
   33.30    double average_last_obj_copy_time() {
   33.31      return _last_obj_copy_times_ms.average();
   33.32    }

    34.1 --- a/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp	Tue Jul 01 09:03:55 2014 +0200
    34.2 +++ b/src/share/vm/gc_implementation/g1/g1MarkSweep.cpp	Mon Jul 07 10:12:40 2014 +0200
    34.3 @@ -128,13 +128,15 @@
    34.4  
    34.5    SharedHeap* sh = SharedHeap::heap();
    34.6  
    34.7 -  // Need cleared claim bits for the strong roots processing
    34.8 +  // Need cleared claim bits for the roots processing
    34.9    ClassLoaderDataGraph::clear_claimed_marks();
   34.10  
   34.11 -  sh->process_strong_roots(true,  // activate StrongRootsScope
   34.12 -                           SharedHeap::SO_SystemClasses,
   34.13 +  MarkingCodeBlobClosure follow_code_closure(&GenMarkSweep::follow_root_closure, !CodeBlobToOopClosure::FixRelocations);
   34.14 +  sh->process_strong_roots(true,   // activate StrongRootsScope
   34.15 +                           SharedHeap::SO_None,
   34.16                             &GenMarkSweep::follow_root_closure,
   34.17 -                           &GenMarkSweep::follow_klass_closure);
   34.18 +                           &GenMarkSweep::follow_cld_closure,
   34.19 +                           &follow_code_closure);
   34.20  
   34.21    // Process reference objects found during marking
   34.22    ReferenceProcessor* rp = GenMarkSweep::ref_processor();
   34.23 @@ -303,13 +305,15 @@
   34.24  
   34.25    SharedHeap* sh = SharedHeap::heap();
   34.26  
   34.27 -  // Need cleared claim bits for the strong roots processing
   34.28 +  // Need cleared claim bits for the roots processing
   34.29    ClassLoaderDataGraph::clear_claimed_marks();
   34.30  
   34.31 -  sh->process_strong_roots(true,  // activate StrongRootsScope
   34.32 -                           SharedHeap::SO_AllClasses | SharedHeap::SO_Strings | SharedHeap::SO_AllCodeCache,
   34.33 -                           &GenMarkSweep::adjust_pointer_closure,
   34.34 -                           &GenMarkSweep::adjust_klass_closure);
   34.35 +  CodeBlobToOopClosure adjust_code_closure(&GenMarkSweep::adjust_pointer_closure, CodeBlobToOopClosure::FixRelocations);
   34.36 +  sh->process_all_roots(true,  // activate StrongRootsScope
   34.37 +                        SharedHeap::SO_AllCodeCache,
   34.38 +                        &GenMarkSweep::adjust_pointer_closure,
   34.39 +                        &GenMarkSweep::adjust_cld_closure,
   34.40 +                        &adjust_code_closure);
   34.41  
   34.42    assert(GenMarkSweep::ref_processor() == g1h->ref_processor_stw(), "Sanity");
   34.43    g1h->ref_processor_stw()->weak_oops_do(&GenMarkSweep::adjust_pointer_closure);

    35.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Tue Jul 01 09:03:55 2014 +0200
    35.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.hpp	Mon Jul 07 10:12:40 2014 +0200
    35.3 @@ -25,6 +25,8 @@
    35.4  #ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_HPP
    35.5  #define SHARE_VM_GC_IMPLEMENTATION_G1_G1OOPCLOSURES_HPP
    35.6  
    35.7 +#include "memory/iterator.hpp"
    35.8 +
    35.9  class HeapRegion;
   35.10  class G1CollectedHeap;
   35.11  class G1RemSet;
   35.12 @@ -106,7 +108,7 @@
   35.13    template <class T> void do_klass_barrier(T* p, oop new_obj);
   35.14  };
   35.15  
   35.16 -template <G1Barrier barrier, bool do_mark_object>
   35.17 +template <G1Barrier barrier, G1Mark do_mark_object>
   35.18  class G1ParCopyClosure : public G1ParCopyHelper {
   35.19  private:
   35.20    template <class T> void do_oop_work(T* p);
   35.21 @@ -121,19 +123,19 @@
   35.22    template <class T> void do_oop_nv(T* p) { do_oop_work(p); }
   35.23    virtual void do_oop(oop* p)       { do_oop_nv(p); }
   35.24    virtual void do_oop(narrowOop* p) { do_oop_nv(p); }
   35.25 +
   35.26 +  G1CollectedHeap*      g1()  { return _g1; };
   35.27 +  G1ParScanThreadState* pss() { return _par_scan_state; }
   35.28 +  ReferenceProcessor*   rp()  { return _ref_processor; };
   35.29  };
   35.30  
   35.31 -typedef G1ParCopyClosure<G1BarrierNone, false> G1ParScanExtRootClosure;
   35.32 -typedef G1ParCopyClosure<G1BarrierKlass, false> G1ParScanMetadataClosure;
   35.33 -
   35.34 -
   35.35 -typedef G1ParCopyClosure<G1BarrierNone, true> G1ParScanAndMarkExtRootClosure;
   35.36 -typedef G1ParCopyClosure<G1BarrierKlass, true> G1ParScanAndMarkMetadataClosure;
   35.37 -
   35.38 +typedef G1ParCopyClosure<G1BarrierNone,  G1MarkNone>             G1ParScanExtRootClosure;
   35.39 +typedef G1ParCopyClosure<G1BarrierNone,  G1MarkFromRoot>         G1ParScanAndMarkExtRootClosure;
   35.40 +typedef G1ParCopyClosure<G1BarrierNone,  G1MarkPromotedFromRoot> G1ParScanAndMarkWeakExtRootClosure;
   35.41  // We use a separate closure to handle references during evacuation
   35.42  // failure processing.
   35.43  
   35.44 -typedef G1ParCopyClosure<G1BarrierEvac, false> G1ParScanHeapEvacFailureClosure;
   35.45 +typedef G1ParCopyClosure<G1BarrierEvac, G1MarkNone> G1ParScanHeapEvacFailureClosure;
   35.46  
   35.47  class FilterIntoCSClosure: public ExtendedOopClosure {
   35.48    G1CollectedHeap* _g1;
   35.49 @@ -164,10 +166,11 @@
   35.50  };
   35.51  
   35.52  // Closure for iterating over object fields during concurrent marking
   35.53 -class G1CMOopClosure : public ExtendedOopClosure {
   35.54 +class G1CMOopClosure : public MetadataAwareOopClosure {
   35.55 +protected:
   35.56 +  ConcurrentMark*    _cm;
   35.57  private:
   35.58    G1CollectedHeap*   _g1h;
   35.59 -  ConcurrentMark*    _cm;
   35.60    CMTask*            _task;
   35.61  public:
   35.62    G1CMOopClosure(G1CollectedHeap* g1h, ConcurrentMark* cm, CMTask* task);
   35.63 @@ -177,7 +180,7 @@
   35.64  };
   35.65  
   35.66  // Closure to scan the root regions during concurrent marking
   35.67 -class G1RootRegionScanClosure : public ExtendedOopClosure {
   35.68 +class G1RootRegionScanClosure : public MetadataAwareOopClosure {
   35.69  private:
   35.70    G1CollectedHeap* _g1h;
   35.71    ConcurrentMark*  _cm;

    36.1 --- a/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Tue Jul 01 09:03:55 2014 +0200
    36.2 +++ b/src/share/vm/gc_implementation/g1/g1OopClosures.inline.hpp	Mon Jul 07 10:12:40 2014 +0200
    36.3 @@ -32,6 +32,7 @@
    36.4  #include "gc_implementation/g1/g1RemSet.hpp"
    36.5  #include "gc_implementation/g1/g1RemSet.inline.hpp"
    36.6  #include "gc_implementation/g1/heapRegionRemSet.hpp"
    36.7 +#include "memory/iterator.inline.hpp"
    36.8  #include "runtime/prefetch.inline.hpp"
    36.9  
   36.10  /*
   36.11 @@ -108,10 +109,6 @@
   36.12  
   36.13  template <class T>
   36.14  inline void G1CMOopClosure::do_oop_nv(T* p) {
   36.15 -  assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
   36.16 -  assert(!_g1h->is_on_master_free_list(
   36.17 -                    _g1h->heap_region_containing((HeapWord*) p)), "invariant");
   36.18 -
   36.19    oop obj = oopDesc::load_decode_heap_oop(p);
   36.20    if (_cm->verbose_high()) {
   36.21      gclog_or_tty->print_cr("[%u] we're looking at location "

    37.1 --- a/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Tue Jul 01 09:03:55 2014 +0200
    37.2 +++ b/src/share/vm/gc_implementation/g1/g1_specialized_oop_closures.hpp	Mon Jul 07 10:12:40 2014 +0200
    37.3 @@ -30,14 +30,21 @@
    37.4  // non-virtually, using a mechanism defined in this file.  Extend these
    37.5  // macros in the obvious way to add specializations for new closures.
    37.6  
    37.7 -// Forward declarations.
    37.8  enum G1Barrier {
    37.9    G1BarrierNone,
   37.10    G1BarrierEvac,
   37.11    G1BarrierKlass
   37.12  };
   37.13  
   37.14 -template<G1Barrier barrier, bool do_mark_object>
   37.15 +enum G1Mark {
   37.16 +  G1MarkNone,
   37.17 +  G1MarkFromRoot,
   37.18 +  G1MarkPromotedFromRoot
   37.19 +};
   37.20 +
   37.21 +// Forward declarations.
   37.22 +
   37.23 +template<G1Barrier barrier, G1Mark do_mark_object>
   37.24  class G1ParCopyClosure;
   37.25  
   37.26  class G1ParScanClosure;

    38.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp	Tue Jul 01 09:03:55 2014 +0200
    38.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp	Mon Jul 07 10:12:40 2014 +0200
    38.3 @@ -399,7 +399,6 @@
    38.4    // We always recreate the prev marking info and we'll explicitly
    38.5    // mark all objects we find to be self-forwarded on the prev
    38.6    // bitmap. So all objects need to be below PTAMS.
    38.7 -  _prev_top_at_mark_start = top();
    38.8    _prev_marked_bytes = 0;
    38.9  
   38.10    if (during_initial_mark) {
   38.11 @@ -423,6 +422,7 @@
   38.12    assert(0 <= marked_bytes && marked_bytes <= used(),
   38.13           err_msg("marked: "SIZE_FORMAT" used: "SIZE_FORMAT,
   38.14                   marked_bytes, used()));
   38.15 +  _prev_top_at_mark_start = top();
   38.16    _prev_marked_bytes = marked_bytes;
   38.17  }
   38.18  
   38.19 @@ -907,7 +907,8 @@
   38.20      size_t obj_size = block_size(p);
   38.21      object_num += 1;
   38.22  
   38.23 -    if (is_humongous != g1->isHumongous(obj_size)) {
   38.24 +    if (is_humongous != g1->isHumongous(obj_size) &&
   38.25 +        !g1->is_obj_dead(obj, this)) { // Dead objects may have bigger block_size since they span several objects.
   38.26        gclog_or_tty->print_cr("obj "PTR_FORMAT" is of %shumongous size ("
   38.27                               SIZE_FORMAT" words) in a %shumongous region",
   38.28                               p, g1->isHumongous(obj_size) ? "" : "non-",
   38.29 @@ -918,7 +919,9 @@
   38.30  
   38.31      // If it returns false, verify_for_object() will output the
   38.32      // appropriate messasge.
   38.33 -    if (do_bot_verify && !_offsets.verify_for_object(p, obj_size)) {
   38.34 +    if (do_bot_verify &&
   38.35 +        !g1->is_obj_dead(obj, this) &&
   38.36 +        !_offsets.verify_for_object(p, obj_size)) {
   38.37        *failures = true;
   38.38        return;
   38.39      }
   38.40 @@ -926,7 +929,10 @@
   38.41      if (!g1->is_obj_dead_cond(obj, this, vo)) {
   38.42        if (obj->is_oop()) {
   38.43          Klass* klass = obj->klass();
   38.44 -        if (!klass->is_metaspace_object()) {
   38.45 +        bool is_metaspace_object = Metaspace::contains(klass) ||
   38.46 +                                   (vo == VerifyOption_G1UsePrevMarking &&
   38.47 +                                   ClassLoaderDataGraph::unload_list_contains(klass));
   38.48 +        if (!is_metaspace_object) {
   38.49            gclog_or_tty->print_cr("klass "PTR_FORMAT" of object "PTR_FORMAT" "
   38.50                                   "not metadata", klass, (void *)obj);
   38.51            *failures = true;

    39.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp	Tue Jul 01 09:03:55 2014 +0200
    39.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp	Mon Jul 07 10:12:40 2014 +0200
    39.3 @@ -247,11 +247,9 @@
    39.4    bool _evacuation_failed;
    39.5  
    39.6    // A heap region may be a member one of a number of special subsets, each
    39.7 -  // represented as linked lists through the field below.  Currently, these
    39.8 -  // sets include:
    39.9 +  // represented as linked lists through the field below.  Currently, there
   39.10 +  // is only one set:
   39.11    //   The collection set.
   39.12 -  //   The set of allocation regions used in a collection pause.
   39.13 -  //   Spaces that may contain gray objects.
   39.14    HeapRegion* _next_in_special_set;
   39.15  
   39.16    // next region in the young "generation" region set

    40.1 --- a/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp	Tue Jul 01 09:03:55 2014 +0200
    40.2 +++ b/src/share/vm/gc_implementation/g1/heapRegion.inline.hpp	Mon Jul 07 10:12:40 2014 +0200
    40.3 @@ -93,18 +93,27 @@
    40.4  
    40.5  inline bool
    40.6  HeapRegion::block_is_obj(const HeapWord* p) const {
    40.7 -  return p < top();
    40.8 +  G1CollectedHeap* g1h = G1CollectedHeap::heap();
    40.9 +  return !g1h->is_obj_dead(oop(p), this);
   40.10  }
   40.11  
   40.12  inline size_t
   40.13  HeapRegion::block_size(const HeapWord *addr) const {
   40.14 -  const HeapWord* current_top = top();
   40.15 -  if (addr < current_top) {
   40.16 -    return oop(addr)->size();
   40.17 -  } else {
   40.18 -    assert(addr == current_top, "just checking");
   40.19 +  // Old regions' dead objects may have dead classes
   40.20 +  // We need to find the next live object in some other
   40.21 +  // manner than getting the oop size
   40.22 +  G1CollectedHeap* g1h = G1CollectedHeap::heap();
   40.23 +  if (g1h->is_obj_dead(oop(addr), this)) {
   40.24 +    HeapWord* next = g1h->concurrent_mark()->prevMarkBitMap()->
   40.25 +        getNextMarkedWordAddress(addr, prev_top_at_mark_start());
   40.26 +
   40.27 +    assert(next > addr, "must get the next live object");
   40.28 +
   40.29 +    return pointer_delta(next, addr);
   40.30 +  } else if (addr == top()) {
   40.31      return pointer_delta(end(), addr);
   40.32    }
   40.33 +  return oop(addr)->size();
   40.34  }
   40.35  
   40.36  inline HeapWord* HeapRegion::par_allocate_no_bot_updates(size_t word_size) {

    41.1 --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Tue Jul 01 09:03:55 2014 +0200
    41.2 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp	Mon Jul 07 10:12:40 2014 +0200
    41.3 @@ -929,7 +929,10 @@
    41.4  
    41.5  void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) {
    41.6    assert(nm != NULL, "sanity");
    41.7 -  _code_roots.remove(nm);
    41.8 +  assert_locked_or_safepoint(CodeCache_lock);
    41.9 +
   41.10 +  _code_roots.remove_lock_free(nm);
   41.11 +
   41.12    // Check that there were no duplicates
   41.13    guarantee(!_code_roots.contains(nm), "duplicate entry found");
   41.14  }

    42.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.cpp	Tue Jul 01 09:03:55 2014 +0200
    42.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp	Mon Jul 07 10:12:40 2014 +0200
    42.3 @@ -285,37 +285,6 @@
    42.4    _par_closures[i] = par_closure;
    42.5  }
    42.6  
    42.7 -void SATBMarkQueueSet::iterate_closure_all_threads() {
    42.8 -  for(JavaThread* t = Threads::first(); t; t = t->next()) {
    42.9 -    t->satb_mark_queue().apply_closure_and_empty(_closure);
   42.10 -  }
   42.11 -  shared_satb_queue()->apply_closure_and_empty(_closure);
   42.12 -}
   42.13 -
   42.14 -void SATBMarkQueueSet::par_iterate_closure_all_threads(uint worker) {
   42.15 -  SharedHeap* sh = SharedHeap::heap();
   42.16 -  int parity = sh->strong_roots_parity();
   42.17 -
   42.18 -  for(JavaThread* t = Threads::first(); t; t = t->next()) {
   42.19 -    if (t->claim_oops_do(true, parity)) {
   42.20 -      t->satb_mark_queue().apply_closure_and_empty(_par_closures[worker]);
   42.21 -    }
   42.22 -  }
   42.23 -
   42.24 -  // We also need to claim the VMThread so that its parity is updated
   42.25 -  // otherwise the next call to Thread::possibly_parallel_oops_do inside
   42.26 -  // a StrongRootsScope might skip the VMThread because it has a stale
   42.27 -  // parity that matches the parity set by the StrongRootsScope
   42.28 -  //
   42.29 -  // Whichever worker succeeds in claiming the VMThread gets to do
   42.30 -  // the shared queue.
   42.31 -
   42.32 -  VMThread* vmt = VMThread::vm_thread();
   42.33 -  if (vmt->claim_oops_do(true, parity)) {
   42.34 -    shared_satb_queue()->apply_closure_and_empty(_par_closures[worker]);
   42.35 -  }
   42.36 -}
   42.37 -
   42.38  bool SATBMarkQueueSet::apply_closure_to_completed_buffer_work(bool par,
   42.39                                                                uint worker) {
   42.40    BufferNode* nd = NULL;

    43.1 --- a/src/share/vm/gc_implementation/g1/satbQueue.hpp	Tue Jul 01 09:03:55 2014 +0200
    43.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp	Mon Jul 07 10:12:40 2014 +0200
    43.3 @@ -33,7 +33,9 @@
    43.4  
    43.5  // A ptrQueue whose elements are "oops", pointers to object heads.
    43.6  class ObjPtrQueue: public PtrQueue {
    43.7 +  friend class Threads;
    43.8    friend class SATBMarkQueueSet;
    43.9 +  friend class G1RemarkThreadsClosure;
   43.10  
   43.11  private:
   43.12    // Filter out unwanted entries from the buffer.
   43.13 @@ -119,13 +121,6 @@
   43.14    // closures, one for each parallel GC thread.
   43.15    void set_par_closure(int i, ObjectClosure* closure);
   43.16  
   43.17 -  // Apply the registered closure to all entries on each
   43.18 -  // currently-active buffer and then empty the buffer. It should only
   43.19 -  // be called serially and at a safepoint.
   43.20 -  void iterate_closure_all_threads();
   43.21 -  // Parallel version of the above.
   43.22 -  void par_iterate_closure_all_threads(uint worker);
   43.23 -
   43.24    // If there exists some completed buffer, pop it, then apply the
   43.25    // registered closure to all its elements, and return true.  If no
   43.26    // completed buffers exist, return false.

    44.1 --- a/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp	Tue Jul 01 09:03:55 2014 +0200
    44.2 +++ b/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp	Mon Jul 07 10:12:40 2014 +0200
    44.3 @@ -613,18 +613,21 @@
    44.4  
    44.5    KlassScanClosure klass_scan_closure(&par_scan_state.to_space_root_closure(),
    44.6                                        gch->rem_set()->klass_rem_set());
    44.7 -
    44.8 -  int so = SharedHeap::SO_AllClasses | SharedHeap::SO_Strings | SharedHeap::SO_ScavengeCodeCache;
    44.9 +  CLDToKlassAndOopClosure cld_scan_closure(&klass_scan_closure,
   44.10 +                                           &par_scan_state.to_space_root_closure(),
   44.11 +                                           false);
   44.12  
   44.13    par_scan_state.start_strong_roots();
   44.14 -  gch->gen_process_strong_roots(_gen->level(),
   44.15 -                                true,  // Process younger gens, if any,
   44.16 -                                       // as strong roots.
   44.17 -                                false, // no scope; this is parallel code
   44.18 -                                SharedHeap::ScanningOption(so),
   44.19 -                                &par_scan_state.to_space_root_closure(),
   44.20 -                                &par_scan_state.older_gen_closure(),
   44.21 -                                &klass_scan_closure);
   44.22 +  gch->gen_process_roots(_gen->level(),
   44.23 +                         true,  // Process younger gens, if any,
   44.24 +                                // as strong roots.
   44.25 +                         false, // no scope; this is parallel code
   44.26 +                         SharedHeap::SO_ScavengeCodeCache,
   44.27 +                         GenCollectedHeap::StrongAndWeakRoots,
   44.28 +                         &par_scan_state.to_space_root_closure(),
   44.29 +                         &par_scan_state.older_gen_closure(),
   44.30 +                         &cld_scan_closure);
   44.31 +
   44.32    par_scan_state.end_strong_roots();
   44.33  
   44.34    // "evacuate followers".