134 return true; |
134 return true; |
135 } |
135 } |
136 int calls() { return _calls; } |
136 int calls() { return _calls; } |
137 }; |
137 }; |
138 |
138 |
|
139 class RedirtyLoggedCardTableEntryFastClosure : public CardTableEntryClosure { |
|
140 public: |
|
141 bool do_card_ptr(jbyte* card_ptr, int worker_i) { |
|
142 *card_ptr = CardTableModRefBS::dirty_card_val(); |
|
143 return true; |
|
144 } |
|
145 }; |
|
146 |
139 YoungList::YoungList(G1CollectedHeap* g1h) |
147 YoungList::YoungList(G1CollectedHeap* g1h) |
140 : _g1h(g1h), _head(NULL), |
148 : _g1h(g1h), _head(NULL), |
141 _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL), |
149 _scan_only_head(NULL), _scan_only_tail(NULL), _curr_scan_only(NULL), |
142 _length(0), _scan_only_length(0), |
150 _length(0), _scan_only_length(0), |
143 _last_sampled_rs_lengths(0), |
151 _last_sampled_rs_lengths(0), |
959 // dirty-card logging system, some cards may be dirty by weak-ref |
967 // dirty-card logging system, some cards may be dirty by weak-ref |
960 // processing, and may be enqueued. But the whole card table is |
968 // processing, and may be enqueued. But the whole card table is |
961 // dirtied, so this should abandon those logs, and set "do_traversal" |
969 // dirtied, so this should abandon those logs, and set "do_traversal" |
962 // to true. |
970 // to true. |
963 concurrent_g1_refine()->set_pya_restart(); |
971 concurrent_g1_refine()->set_pya_restart(); |
964 |
972 assert(!G1DeferredRSUpdate |
|
973 || (G1DeferredRSUpdate && (dirty_card_queue_set().completed_buffers_num() == 0)), "Should not be any"); |
965 assert(regions_accounted_for(), "Region leakage!"); |
974 assert(regions_accounted_for(), "Region leakage!"); |
966 } |
975 } |
967 |
976 |
968 if (g1_policy()->in_young_gc_mode()) { |
977 if (g1_policy()->in_young_gc_mode()) { |
969 _young_list->reset_sampled_info(); |
978 _young_list->reset_sampled_info(); |
1464 JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, |
1473 JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, |
1465 DirtyCardQ_FL_lock, |
1474 DirtyCardQ_FL_lock, |
1466 G1DirtyCardQueueMax, |
1475 G1DirtyCardQueueMax, |
1467 Shared_DirtyCardQ_lock); |
1476 Shared_DirtyCardQ_lock); |
1468 } |
1477 } |
|
1478 if (G1DeferredRSUpdate) { |
|
1479 dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, |
|
1480 DirtyCardQ_FL_lock, |
|
1481 0, |
|
1482 Shared_DirtyCardQ_lock, |
|
1483 &JavaThread::dirty_card_queue_set()); |
|
1484 } |
1469 // In case we're keeping closure specialization stats, initialize those |
1485 // In case we're keeping closure specialization stats, initialize those |
1470 // counts and that mechanism. |
1486 // counts and that mechanism. |
1471 SpecializationStats::clear(); |
1487 SpecializationStats::clear(); |
1472 |
1488 |
1473 _gc_alloc_region_list = NULL; |
1489 _gc_alloc_region_list = NULL; |
2916 #endif // G1_DEBUG |
2932 #endif // G1_DEBUG |
2917 } |
2933 } |
2918 } |
2934 } |
2919 }; |
2935 }; |
2920 |
2936 |
2921 class RecreateRSetEntriesClosure: public OopClosure { |
2937 class UpdateRSetImmediate : public OopsInHeapRegionClosure { |
2922 private: |
2938 private: |
2923 G1CollectedHeap* _g1; |
2939 G1CollectedHeap* _g1; |
2924 G1RemSet* _g1_rem_set; |
2940 G1RemSet* _g1_rem_set; |
2925 HeapRegion* _from; |
|
2926 public: |
2941 public: |
2927 RecreateRSetEntriesClosure(G1CollectedHeap* g1, HeapRegion* from) : |
2942 UpdateRSetImmediate(G1CollectedHeap* g1) : |
2928 _g1(g1), _g1_rem_set(g1->g1_rem_set()), _from(from) |
2943 _g1(g1), _g1_rem_set(g1->g1_rem_set()) {} |
2929 {} |
|
2930 |
2944 |
2931 void do_oop(narrowOop* p) { |
2945 void do_oop(narrowOop* p) { |
2932 guarantee(false, "NYI"); |
2946 guarantee(false, "NYI"); |
2933 } |
2947 } |
2934 void do_oop(oop* p) { |
2948 void do_oop(oop* p) { |
2935 assert(_from->is_in_reserved(p), "paranoia"); |
2949 assert(_from->is_in_reserved(p), "paranoia"); |
2936 if (*p != NULL) { |
2950 if (*p != NULL && !_from->is_survivor()) { |
2937 _g1_rem_set->write_ref(_from, p); |
2951 _g1_rem_set->par_write_ref(_from, p, 0); |
2938 } |
2952 } |
2939 } |
2953 } |
2940 }; |
2954 }; |
|
2955 |
|
2956 class UpdateRSetDeferred : public OopsInHeapRegionClosure { |
|
2957 private: |
|
2958 G1CollectedHeap* _g1; |
|
2959 DirtyCardQueue *_dcq; |
|
2960 CardTableModRefBS* _ct_bs; |
|
2961 |
|
2962 public: |
|
2963 UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : |
|
2964 _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {} |
|
2965 |
|
2966 void do_oop(narrowOop* p) { |
|
2967 guarantee(false, "NYI"); |
|
2968 } |
|
2969 void do_oop(oop* p) { |
|
2970 assert(_from->is_in_reserved(p), "paranoia"); |
|
2971 if (!_from->is_in_reserved(*p) && !_from->is_survivor()) { |
|
2972 size_t card_index = _ct_bs->index_for(p); |
|
2973 if (_ct_bs->mark_card_deferred(card_index)) { |
|
2974 _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); |
|
2975 } |
|
2976 } |
|
2977 } |
|
2978 }; |
|
2979 |
|
2980 |
2941 |
2981 |
2942 class RemoveSelfPointerClosure: public ObjectClosure { |
2982 class RemoveSelfPointerClosure: public ObjectClosure { |
2943 private: |
2983 private: |
2944 G1CollectedHeap* _g1; |
2984 G1CollectedHeap* _g1; |
2945 ConcurrentMark* _cm; |
2985 ConcurrentMark* _cm; |
2946 HeapRegion* _hr; |
2986 HeapRegion* _hr; |
2947 size_t _prev_marked_bytes; |
2987 size_t _prev_marked_bytes; |
2948 size_t _next_marked_bytes; |
2988 size_t _next_marked_bytes; |
|
2989 OopsInHeapRegionClosure *_cl; |
2949 public: |
2990 public: |
2950 RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr) : |
2991 RemoveSelfPointerClosure(G1CollectedHeap* g1, OopsInHeapRegionClosure* cl) : |
2951 _g1(g1), _cm(_g1->concurrent_mark()), _hr(hr), |
2992 _g1(g1), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), |
2952 _prev_marked_bytes(0), _next_marked_bytes(0) |
2993 _next_marked_bytes(0), _cl(cl) {} |
2953 {} |
|
2954 |
2994 |
2955 size_t prev_marked_bytes() { return _prev_marked_bytes; } |
2995 size_t prev_marked_bytes() { return _prev_marked_bytes; } |
2956 size_t next_marked_bytes() { return _next_marked_bytes; } |
2996 size_t next_marked_bytes() { return _next_marked_bytes; } |
2957 |
2997 |
2958 // The original idea here was to coalesce evacuated and dead objects. |
2998 // The original idea here was to coalesce evacuated and dead objects. |
2986 // card in the collection set and coming across an array that |
3026 // card in the collection set and coming across an array that |
2987 // was being chunked and looking malformed. The problem is |
3027 // was being chunked and looking malformed. The problem is |
2988 // that, if evacuation fails, we might have remembered set |
3028 // that, if evacuation fails, we might have remembered set |
2989 // entries missing given that we skipped cards on the |
3029 // entries missing given that we skipped cards on the |
2990 // collection set. So, we'll recreate such entries now. |
3030 // collection set. So, we'll recreate such entries now. |
2991 RecreateRSetEntriesClosure cl(_g1, _hr); |
3031 obj->oop_iterate(_cl); |
2992 obj->oop_iterate(&cl); |
|
2993 assert(_cm->isPrevMarked(obj), "Should be marked!"); |
3032 assert(_cm->isPrevMarked(obj), "Should be marked!"); |
2994 } else { |
3033 } else { |
2995 // The object has been either evacuated or is dead. Fill it with a |
3034 // The object has been either evacuated or is dead. Fill it with a |
2996 // dummy object. |
3035 // dummy object. |
2997 MemRegion mr((HeapWord*)obj, obj->size()); |
3036 MemRegion mr((HeapWord*)obj, obj->size()); |
3000 } |
3039 } |
3001 } |
3040 } |
3002 }; |
3041 }; |
3003 |
3042 |
3004 void G1CollectedHeap::remove_self_forwarding_pointers() { |
3043 void G1CollectedHeap::remove_self_forwarding_pointers() { |
|
3044 UpdateRSetImmediate immediate_update(_g1h); |
|
3045 DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); |
|
3046 UpdateRSetDeferred deferred_update(_g1h, &dcq); |
|
3047 OopsInHeapRegionClosure *cl; |
|
3048 if (G1DeferredRSUpdate) { |
|
3049 cl = &deferred_update; |
|
3050 } else { |
|
3051 cl = &immediate_update; |
|
3052 } |
3005 HeapRegion* cur = g1_policy()->collection_set(); |
3053 HeapRegion* cur = g1_policy()->collection_set(); |
3006 |
|
3007 while (cur != NULL) { |
3054 while (cur != NULL) { |
3008 assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); |
3055 assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); |
3009 |
3056 |
|
3057 RemoveSelfPointerClosure rspc(_g1h, cl); |
3010 if (cur->evacuation_failed()) { |
3058 if (cur->evacuation_failed()) { |
3011 RemoveSelfPointerClosure rspc(_g1h, cur); |
|
3012 assert(cur->in_collection_set(), "bad CS"); |
3059 assert(cur->in_collection_set(), "bad CS"); |
|
3060 cl->set_region(cur); |
3013 cur->object_iterate(&rspc); |
3061 cur->object_iterate(&rspc); |
3014 |
3062 |
3015 // A number of manipulations to make the TAMS be the current top, |
3063 // A number of manipulations to make the TAMS be the current top, |
3016 // and the marked bytes be the ones observed in the iteration. |
3064 // and the marked bytes be the ones observed in the iteration. |
3017 if (_g1h->concurrent_mark()->at_least_one_mark_complete()) { |
3065 if (_g1h->concurrent_mark()->at_least_one_mark_complete()) { |
3557 |
3608 |
3558 void add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; } |
3609 void add_to_alloc_buffer_waste(size_t waste) { _alloc_buffer_waste += waste; } |
3559 |
3610 |
3560 void add_to_undo_waste(size_t waste) { _undo_waste += waste; } |
3611 void add_to_undo_waste(size_t waste) { _undo_waste += waste; } |
3561 |
3612 |
|
3613 DirtyCardQueue& dirty_card_queue() { return _dcq; } |
|
3614 CardTableModRefBS* ctbs() { return _ct_bs; } |
|
3615 |
|
3616 void immediate_rs_update(HeapRegion* from, oop* p, int tid) { |
|
3617 _g1_rem->par_write_ref(from, p, tid); |
|
3618 } |
|
3619 |
|
3620 void deferred_rs_update(HeapRegion* from, oop* p, int tid) { |
|
3621 // If the new value of the field points to the same region or |
|
3622 // is the to-space, we don't need to include it in the Rset updates. |
|
3623 if (!from->is_in_reserved(*p) && !from->is_survivor()) { |
|
3624 size_t card_index = ctbs()->index_for(p); |
|
3625 // If the card hasn't been added to the buffer, do it. |
|
3626 if (ctbs()->mark_card_deferred(card_index)) { |
|
3627 dirty_card_queue().enqueue((jbyte*)ctbs()->byte_for_index(card_index)); |
|
3628 } |
|
3629 } |
|
3630 } |
|
3631 |
3562 public: |
3632 public: |
3563 G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num) |
3633 G1ParScanThreadState(G1CollectedHeap* g1h, int queue_num) |
3564 : _g1h(g1h), |
3634 : _g1h(g1h), |
3565 _refs(g1h->task_queue(queue_num)), |
3635 _refs(g1h->task_queue(queue_num)), |
|
3636 _dcq(&g1h->dirty_card_queue_set()), |
|
3637 _ct_bs((CardTableModRefBS*)_g1h->barrier_set()), |
|
3638 _g1_rem(g1h->g1_rem_set()), |
3566 _hash_seed(17), _queue_num(queue_num), |
3639 _hash_seed(17), _queue_num(queue_num), |
3567 _term_attempts(0), |
3640 _term_attempts(0), |
3568 _age_table(false), |
3641 _age_table(false), |
3569 #if G1_DETAILED_STATS |
3642 #if G1_DETAILED_STATS |
3570 _pushes(0), _pops(0), _steals(0), |
3643 _pushes(0), _pops(0), _steals(0), |
3638 } |
3711 } |
3639 |
3712 |
3640 int refs_to_scan() { return refs()->size(); } |
3713 int refs_to_scan() { return refs()->size(); } |
3641 int overflowed_refs_to_scan() { return overflowed_refs()->length(); } |
3714 int overflowed_refs_to_scan() { return overflowed_refs()->length(); } |
3642 |
3715 |
|
3716 void update_rs(HeapRegion* from, oop* p, int tid) { |
|
3717 if (G1DeferredRSUpdate) { |
|
3718 deferred_rs_update(from, p, tid); |
|
3719 } else { |
|
3720 immediate_rs_update(from, p, tid); |
|
3721 } |
|
3722 } |
|
3723 |
3643 HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) { |
3724 HeapWord* allocate_slow(GCAllocPurpose purpose, size_t word_sz) { |
3644 |
3725 |
3645 HeapWord* obj = NULL; |
3726 HeapWord* obj = NULL; |
3646 if (word_sz * 100 < |
3727 if (word_sz * 100 < |
3647 (size_t)(ParallelGCG1AllocBufferSize / HeapWordSize) * |
3728 (size_t)(ParallelGCG1AllocBufferSize / HeapWordSize) * |
3832 // problems before we go into push_on_queue to know where the |
3912 // problems before we go into push_on_queue to know where the |
3833 // problem is coming from |
3913 // problem is coming from |
3834 assert(obj == *p, "the value of *p should not have changed"); |
3914 assert(obj == *p, "the value of *p should not have changed"); |
3835 _par_scan_state->push_on_queue(p); |
3915 _par_scan_state->push_on_queue(p); |
3836 } else { |
3916 } else { |
3837 _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); |
3917 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); |
3838 } |
3918 } |
3839 } |
3919 } |
3840 } |
3920 } |
3841 |
3921 |
3842 void G1ParCopyHelper::mark_forwardee(oop* p) { |
3922 void G1ParCopyHelper::mark_forwardee(oop* p) { |
3970 } else { |
4050 } else { |
3971 *p = copy_to_survivor_space(obj); |
4051 *p = copy_to_survivor_space(obj); |
3972 } |
4052 } |
3973 // When scanning the RS, we only care about objs in CS. |
4053 // When scanning the RS, we only care about objs in CS. |
3974 if (barrier == G1BarrierRS) { |
4054 if (barrier == G1BarrierRS) { |
3975 _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); |
4055 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); |
3976 } |
4056 } |
3977 } |
4057 } |
3978 |
4058 |
3979 // When scanning moved objs, must look at all oops. |
4059 // When scanning moved objs, must look at all oops. |
3980 if (barrier == G1BarrierEvac && obj != NULL) { |
4060 if (barrier == G1BarrierEvac && obj != NULL) { |
3981 _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num()); |
4061 _par_scan_state->update_rs(_from, p, _par_scan_state->queue_num()); |
3982 } |
4062 } |
3983 |
4063 |
3984 if (do_gen_barrier && obj != NULL) { |
4064 if (do_gen_barrier && obj != NULL) { |
3985 par_do_barrier(p); |
4065 par_do_barrier(p); |
3986 } |
4066 } |
4125 pss.set_partial_scan_closure(&partial_scan_cl); |
4205 pss.set_partial_scan_closure(&partial_scan_cl); |
4126 |
4206 |
4127 G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss); |
4207 G1ParScanExtRootClosure only_scan_root_cl(_g1h, &pss); |
4128 G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss); |
4208 G1ParScanPermClosure only_scan_perm_cl(_g1h, &pss); |
4129 G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss); |
4209 G1ParScanHeapRSClosure only_scan_heap_rs_cl(_g1h, &pss); |
|
4210 |
4130 G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss); |
4211 G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss); |
4131 G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss); |
4212 G1ParScanAndMarkPermClosure scan_mark_perm_cl(_g1h, &pss); |
4132 G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss); |
4213 G1ParScanAndMarkHeapRSClosure scan_mark_heap_rs_cl(_g1h, &pss); |
4133 |
4214 |
4134 OopsInHeapRegionClosure *scan_root_cl; |
4215 OopsInHeapRegionClosure *scan_root_cl; |
4380 set_evacuation_failed(false); |
4461 set_evacuation_failed(false); |
4381 |
4462 |
4382 g1_rem_set()->prepare_for_oops_into_collection_set_do(); |
4463 g1_rem_set()->prepare_for_oops_into_collection_set_do(); |
4383 concurrent_g1_refine()->set_use_cache(false); |
4464 concurrent_g1_refine()->set_use_cache(false); |
4384 int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1); |
4465 int n_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1); |
4385 |
|
4386 set_par_threads(n_workers); |
4466 set_par_threads(n_workers); |
4387 G1ParTask g1_par_task(this, n_workers, _task_queues); |
4467 G1ParTask g1_par_task(this, n_workers, _task_queues); |
4388 |
4468 |
4389 init_for_evac_failure(NULL); |
4469 init_for_evac_failure(NULL); |
4390 |
4470 |
4391 change_strong_roots_parity(); // In preparation for parallel strong roots. |
4471 change_strong_roots_parity(); // In preparation for parallel strong roots. |
4392 rem_set()->prepare_for_younger_refs_iterate(true); |
4472 rem_set()->prepare_for_younger_refs_iterate(true); |
|
4473 |
|
4474 assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty"); |
4393 double start_par = os::elapsedTime(); |
4475 double start_par = os::elapsedTime(); |
4394 |
|
4395 if (ParallelGCThreads > 0) { |
4476 if (ParallelGCThreads > 0) { |
4396 // The individual threads will set their evac-failure closures. |
4477 // The individual threads will set their evac-failure closures. |
4397 workers()->run_task(&g1_par_task); |
4478 workers()->run_task(&g1_par_task); |
4398 } else { |
4479 } else { |
4399 g1_par_task.work(0); |
4480 g1_par_task.work(0); |
4409 { |
4490 { |
4410 G1IsAliveClosure is_alive(this); |
4491 G1IsAliveClosure is_alive(this); |
4411 G1KeepAliveClosure keep_alive(this); |
4492 G1KeepAliveClosure keep_alive(this); |
4412 JNIHandles::weak_oops_do(&is_alive, &keep_alive); |
4493 JNIHandles::weak_oops_do(&is_alive, &keep_alive); |
4413 } |
4494 } |
4414 |
|
4415 g1_rem_set()->cleanup_after_oops_into_collection_set_do(); |
4495 g1_rem_set()->cleanup_after_oops_into_collection_set_do(); |
|
4496 |
4416 concurrent_g1_refine()->set_use_cache(true); |
4497 concurrent_g1_refine()->set_use_cache(true); |
4417 |
4498 |
4418 finalize_for_evac_failure(); |
4499 finalize_for_evac_failure(); |
4419 |
4500 |
4420 // Must do this before removing self-forwarding pointers, which clears |
4501 // Must do this before removing self-forwarding pointers, which clears |
4421 // the per-region evac-failure flags. |
4502 // the per-region evac-failure flags. |
4422 concurrent_mark()->complete_marking_in_collection_set(); |
4503 concurrent_mark()->complete_marking_in_collection_set(); |
4423 |
4504 |
4424 if (evacuation_failed()) { |
4505 if (evacuation_failed()) { |
4425 remove_self_forwarding_pointers(); |
4506 remove_self_forwarding_pointers(); |
4426 |
|
4427 if (PrintGCDetails) { |
4507 if (PrintGCDetails) { |
4428 gclog_or_tty->print(" (evacuation failed)"); |
4508 gclog_or_tty->print(" (evacuation failed)"); |
4429 } else if (PrintGC) { |
4509 } else if (PrintGC) { |
4430 gclog_or_tty->print("--"); |
4510 gclog_or_tty->print("--"); |
4431 } |
4511 } |
|
4512 } |
|
4513 |
|
4514 if (G1DeferredRSUpdate) { |
|
4515 RedirtyLoggedCardTableEntryFastClosure redirty; |
|
4516 dirty_card_queue_set().set_closure(&redirty); |
|
4517 dirty_card_queue_set().apply_closure_to_all_completed_buffers(); |
|
4518 JavaThread::dirty_card_queue_set().merge_bufferlists(&dirty_card_queue_set()); |
|
4519 assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); |
4432 } |
4520 } |
4433 |
4521 |
4434 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); |
4522 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); |
4435 } |
4523 } |
4436 |
4524 |