1926 _mark_in_progress(false), |
1925 _mark_in_progress(false), |
1927 _cg1r(NULL), _summary_bytes_used(0), |
1926 _cg1r(NULL), _summary_bytes_used(0), |
1928 _g1mm(NULL), |
1927 _g1mm(NULL), |
1929 _refine_cte_cl(NULL), |
1928 _refine_cte_cl(NULL), |
1930 _full_collection(false), |
1929 _full_collection(false), |
1931 _free_list("Master Free List"), |
1930 _free_list("Master Free List", new MasterFreeRegionListMtSafeChecker()), |
1932 _secondary_free_list("Secondary Free List"), |
1931 _secondary_free_list("Secondary Free List", new SecondaryFreeRegionListMtSafeChecker()), |
1933 _old_set("Old Set"), |
1932 _old_set("Old Set", false /* humongous */, new OldRegionSetMtSafeChecker()), |
1934 _humongous_set("Master Humongous Set"), |
1933 _humongous_set("Master Humongous Set", true /* humongous */, new HumongousRegionSetMtSafeChecker()), |
1935 _free_regions_coming(false), |
1934 _free_regions_coming(false), |
1936 _young_list(new YoungList(this)), |
1935 _young_list(new YoungList(this)), |
1937 _gc_time_stamp(0), |
1936 _gc_time_stamp(0), |
1938 _retained_old_gc_alloc_region(NULL), |
1937 _retained_old_gc_alloc_region(NULL), |
1939 _survivor_plab_stats(YoungPLABSize, PLABWeight), |
1938 _survivor_plab_stats(YoungPLABSize, PLABWeight), |
5965 assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); |
5963 assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); |
5966 } |
5964 } |
5967 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); |
5965 COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); |
5968 } |
5966 } |
5969 |
5967 |
5970 void G1CollectedHeap::free_region_if_empty(HeapRegion* hr, |
|
5971 size_t* pre_used, |
|
5972 FreeRegionList* free_list, |
|
5973 OldRegionSet* old_proxy_set, |
|
5974 HumongousRegionSet* humongous_proxy_set, |
|
5975 HRRSCleanupTask* hrrs_cleanup_task, |
|
5976 bool par) { |
|
5977 if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { |
|
5978 if (hr->isHumongous()) { |
|
5979 assert(hr->startsHumongous(), "we should only see starts humongous"); |
|
5980 free_humongous_region(hr, pre_used, free_list, humongous_proxy_set, par); |
|
5981 } else { |
|
5982 _old_set.remove_with_proxy(hr, old_proxy_set); |
|
5983 free_region(hr, pre_used, free_list, par); |
|
5984 } |
|
5985 } else { |
|
5986 hr->rem_set()->do_cleanup_work(hrrs_cleanup_task); |
|
5987 } |
|
5988 } |
|
5989 |
|
5990 void G1CollectedHeap::free_region(HeapRegion* hr, |
5968 void G1CollectedHeap::free_region(HeapRegion* hr, |
5991 size_t* pre_used, |
|
5992 FreeRegionList* free_list, |
5969 FreeRegionList* free_list, |
5993 bool par) { |
5970 bool par) { |
5994 assert(!hr->isHumongous(), "this is only for non-humongous regions"); |
5971 assert(!hr->isHumongous(), "this is only for non-humongous regions"); |
5995 assert(!hr->is_empty(), "the region should not be empty"); |
5972 assert(!hr->is_empty(), "the region should not be empty"); |
5996 assert(free_list != NULL, "pre-condition"); |
5973 assert(free_list != NULL, "pre-condition"); |
5999 // Note: we only need to do this if the region is not young |
5976 // Note: we only need to do this if the region is not young |
6000 // (since we don't refine cards in young regions). |
5977 // (since we don't refine cards in young regions). |
6001 if (!hr->is_young()) { |
5978 if (!hr->is_young()) { |
6002 _cg1r->hot_card_cache()->reset_card_counts(hr); |
5979 _cg1r->hot_card_cache()->reset_card_counts(hr); |
6003 } |
5980 } |
6004 *pre_used += hr->used(); |
|
6005 hr->hr_clear(par, true /* clear_space */); |
5981 hr->hr_clear(par, true /* clear_space */); |
6006 free_list->add_as_head(hr); |
5982 free_list->add_as_head(hr); |
6007 } |
5983 } |
6008 |
5984 |
6009 void G1CollectedHeap::free_humongous_region(HeapRegion* hr, |
5985 void G1CollectedHeap::free_humongous_region(HeapRegion* hr, |
6010 size_t* pre_used, |
|
6011 FreeRegionList* free_list, |
5986 FreeRegionList* free_list, |
6012 HumongousRegionSet* humongous_proxy_set, |
|
6013 bool par) { |
5987 bool par) { |
6014 assert(hr->startsHumongous(), "this is only for starts humongous regions"); |
5988 assert(hr->startsHumongous(), "this is only for starts humongous regions"); |
6015 assert(free_list != NULL, "pre-condition"); |
5989 assert(free_list != NULL, "pre-condition"); |
6016 assert(humongous_proxy_set != NULL, "pre-condition"); |
5990 |
6017 |
|
6018 size_t hr_used = hr->used(); |
|
6019 size_t hr_capacity = hr->capacity(); |
5991 size_t hr_capacity = hr->capacity(); |
6020 size_t hr_pre_used = 0; |
|
6021 _humongous_set.remove_with_proxy(hr, humongous_proxy_set); |
|
6022 // We need to read this before we make the region non-humongous, |
5992 // We need to read this before we make the region non-humongous, |
6023 // otherwise the information will be gone. |
5993 // otherwise the information will be gone. |
6024 uint last_index = hr->last_hc_index(); |
5994 uint last_index = hr->last_hc_index(); |
6025 hr->set_notHumongous(); |
5995 hr->set_notHumongous(); |
6026 free_region(hr, &hr_pre_used, free_list, par); |
5996 free_region(hr, free_list, par); |
6027 |
5997 |
6028 uint i = hr->hrs_index() + 1; |
5998 uint i = hr->hrs_index() + 1; |
6029 while (i < last_index) { |
5999 while (i < last_index) { |
6030 HeapRegion* curr_hr = region_at(i); |
6000 HeapRegion* curr_hr = region_at(i); |
6031 assert(curr_hr->continuesHumongous(), "invariant"); |
6001 assert(curr_hr->continuesHumongous(), "invariant"); |
6032 curr_hr->set_notHumongous(); |
6002 curr_hr->set_notHumongous(); |
6033 free_region(curr_hr, &hr_pre_used, free_list, par); |
6003 free_region(curr_hr, free_list, par); |
6034 i += 1; |
6004 i += 1; |
6035 } |
6005 } |
6036 assert(hr_pre_used == hr_used, |
6006 } |
6037 err_msg("hr_pre_used: "SIZE_FORMAT" and hr_used: "SIZE_FORMAT" " |
6007 |
6038 "should be the same", hr_pre_used, hr_used)); |
6008 void G1CollectedHeap::remove_from_old_sets(const HeapRegionSetCount& old_regions_removed, |
6039 *pre_used += hr_pre_used; |
6009 const HeapRegionSetCount& humongous_regions_removed) { |
6040 } |
6010 if (old_regions_removed.length() > 0 || humongous_regions_removed.length() > 0) { |
6041 |
6011 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); |
6042 void G1CollectedHeap::update_sets_after_freeing_regions(size_t pre_used, |
6012 _old_set.bulk_remove(old_regions_removed); |
6043 FreeRegionList* free_list, |
6013 _humongous_set.bulk_remove(humongous_regions_removed); |
6044 OldRegionSet* old_proxy_set, |
6014 } |
6045 HumongousRegionSet* humongous_proxy_set, |
6015 |
6046 bool par) { |
6016 } |
6047 if (pre_used > 0) { |
6017 |
6048 Mutex* lock = (par) ? ParGCRareEvent_lock : NULL; |
6018 void G1CollectedHeap::prepend_to_freelist(FreeRegionList* list) { |
6049 MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); |
6019 assert(list != NULL, "list can't be null"); |
6050 assert(_summary_bytes_used >= pre_used, |
6020 if (!list->is_empty()) { |
6051 err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" " |
|
6052 "should be >= pre_used: "SIZE_FORMAT, |
|
6053 _summary_bytes_used, pre_used)); |
|
6054 _summary_bytes_used -= pre_used; |
|
6055 } |
|
6056 if (free_list != NULL && !free_list->is_empty()) { |
|
6057 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); |
6021 MutexLockerEx x(FreeList_lock, Mutex::_no_safepoint_check_flag); |
6058 _free_list.add_as_head(free_list); |
6022 _free_list.add_as_head(list); |
6059 } |
6023 } |
6060 if (old_proxy_set != NULL && !old_proxy_set->is_empty()) { |
6024 } |
6061 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); |
6025 |
6062 _old_set.update_from_proxy(old_proxy_set); |
6026 void G1CollectedHeap::decrement_summary_bytes(size_t bytes) { |
6063 } |
6027 assert(_summary_bytes_used >= bytes, |
6064 if (humongous_proxy_set != NULL && !humongous_proxy_set->is_empty()) { |
6028 err_msg("invariant: _summary_bytes_used: "SIZE_FORMAT" should be >= bytes: "SIZE_FORMAT, |
6065 MutexLockerEx x(OldSets_lock, Mutex::_no_safepoint_check_flag); |
6029 _summary_bytes_used, bytes)); |
6066 _humongous_set.update_from_proxy(humongous_proxy_set); |
6030 _summary_bytes_used -= bytes; |
6067 } |
|
6068 } |
6031 } |
6069 |
6032 |
6070 class G1ParCleanupCTTask : public AbstractGangTask { |
6033 class G1ParCleanupCTTask : public AbstractGangTask { |
6071 G1SATBCardTableModRefBS* _ct_bs; |
6034 G1SATBCardTableModRefBS* _ct_bs; |
6072 G1CollectedHeap* _g1h; |
6035 G1CollectedHeap* _g1h; |
6440 } |
6402 } |
6441 |
6403 |
6442 class RebuildRegionSetsClosure : public HeapRegionClosure { |
6404 class RebuildRegionSetsClosure : public HeapRegionClosure { |
6443 private: |
6405 private: |
6444 bool _free_list_only; |
6406 bool _free_list_only; |
6445 OldRegionSet* _old_set; |
6407 HeapRegionSet* _old_set; |
6446 FreeRegionList* _free_list; |
6408 FreeRegionList* _free_list; |
6447 size_t _total_used; |
6409 size_t _total_used; |
6448 |
6410 |
6449 public: |
6411 public: |
6450 RebuildRegionSetsClosure(bool free_list_only, |
6412 RebuildRegionSetsClosure(bool free_list_only, |
6451 OldRegionSet* old_set, FreeRegionList* free_list) : |
6413 HeapRegionSet* old_set, FreeRegionList* free_list) : |
6452 _free_list_only(free_list_only), |
6414 _free_list_only(free_list_only), |
6453 _old_set(old_set), _free_list(free_list), _total_used(0) { |
6415 _old_set(old_set), _free_list(free_list), _total_used(0) { |
6454 assert(_free_list->is_empty(), "pre-condition"); |
6416 assert(_free_list->is_empty(), "pre-condition"); |
6455 if (!free_list_only) { |
6417 if (!free_list_only) { |
6456 assert(_old_set->is_empty(), "pre-condition"); |
6418 assert(_old_set->is_empty(), "pre-condition"); |
6643 } |
6605 } |
6644 // Heap region set verification |
6606 // Heap region set verification |
6645 |
6607 |
6646 class VerifyRegionListsClosure : public HeapRegionClosure { |
6608 class VerifyRegionListsClosure : public HeapRegionClosure { |
6647 private: |
6609 private: |
6648 FreeRegionList* _free_list; |
6610 HeapRegionSet* _old_set; |
6649 OldRegionSet* _old_set; |
6611 HeapRegionSet* _humongous_set; |
6650 HumongousRegionSet* _humongous_set; |
6612 FreeRegionList* _free_list; |
6651 uint _region_count; |
|
6652 |
6613 |
6653 public: |
6614 public: |
6654 VerifyRegionListsClosure(OldRegionSet* old_set, |
6615 HeapRegionSetCount _old_count; |
6655 HumongousRegionSet* humongous_set, |
6616 HeapRegionSetCount _humongous_count; |
|
6617 HeapRegionSetCount _free_count; |
|
6618 |
|
6619 VerifyRegionListsClosure(HeapRegionSet* old_set, |
|
6620 HeapRegionSet* humongous_set, |
6656 FreeRegionList* free_list) : |
6621 FreeRegionList* free_list) : |
6657 _old_set(old_set), _humongous_set(humongous_set), |
6622 _old_set(old_set), _humongous_set(humongous_set), _free_list(free_list), |
6658 _free_list(free_list), _region_count(0) { } |
6623 _old_count(), _humongous_count(), _free_count(){ } |
6659 |
|
6660 uint region_count() { return _region_count; } |
|
6661 |
6624 |
6662 bool doHeapRegion(HeapRegion* hr) { |
6625 bool doHeapRegion(HeapRegion* hr) { |
6663 _region_count += 1; |
|
6664 |
|
6665 if (hr->continuesHumongous()) { |
6626 if (hr->continuesHumongous()) { |
6666 return false; |
6627 return false; |
6667 } |
6628 } |
6668 |
6629 |
6669 if (hr->is_young()) { |
6630 if (hr->is_young()) { |
6670 // TODO |
6631 // TODO |
6671 } else if (hr->startsHumongous()) { |
6632 } else if (hr->startsHumongous()) { |
6672 _humongous_set->verify_next_region(hr); |
6633 assert(hr->containing_set() == _humongous_set, err_msg("Heap region %u is starts humongous but not in humongous set.", hr->region_num())); |
|
6634 _humongous_count.increment(1u, hr->capacity()); |
6673 } else if (hr->is_empty()) { |
6635 } else if (hr->is_empty()) { |
6674 _free_list->verify_next_region(hr); |
6636 assert(hr->containing_set() == _free_list, err_msg("Heap region %u is empty but not on the free list.", hr->region_num())); |
|
6637 _free_count.increment(1u, hr->capacity()); |
6675 } else { |
6638 } else { |
6676 _old_set->verify_next_region(hr); |
6639 assert(hr->containing_set() == _old_set, err_msg("Heap region %u is old but not in the old set.", hr->region_num())); |
|
6640 _old_count.increment(1u, hr->capacity()); |
6677 } |
6641 } |
6678 return false; |
6642 return false; |
|
6643 } |
|
6644 |
|
6645 void verify_counts(HeapRegionSet* old_set, HeapRegionSet* humongous_set, FreeRegionList* free_list) { |
|
6646 guarantee(old_set->length() == _old_count.length(), err_msg("Old set count mismatch. Expected %u, actual %u.", old_set->length(), _old_count.length())); |
|
6647 guarantee(old_set->total_capacity_bytes() == _old_count.capacity(), err_msg("Old set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, |
|
6648 old_set->total_capacity_bytes(), _old_count.capacity())); |
|
6649 |
|
6650 guarantee(humongous_set->length() == _humongous_count.length(), err_msg("Hum set count mismatch. Expected %u, actual %u.", humongous_set->length(), _humongous_count.length())); |
|
6651 guarantee(humongous_set->total_capacity_bytes() == _humongous_count.capacity(), err_msg("Hum set capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, |
|
6652 humongous_set->total_capacity_bytes(), _humongous_count.capacity())); |
|
6653 |
|
6654 guarantee(free_list->length() == _free_count.length(), err_msg("Free list count mismatch. Expected %u, actual %u.", free_list->length(), _free_count.length())); |
|
6655 guarantee(free_list->total_capacity_bytes() == _free_count.capacity(), err_msg("Free list capacity mismatch. Expected " SIZE_FORMAT ", actual " SIZE_FORMAT, |
|
6656 free_list->total_capacity_bytes(), _free_count.capacity())); |
6679 } |
6657 } |
6680 }; |
6658 }; |
6681 |
6659 |
6682 HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, |
6660 HeapRegion* G1CollectedHeap::new_heap_region(uint hrs_index, |
6683 HeapWord* bottom) { |
6661 HeapWord* bottom) { |
6690 |
6668 |
6691 void G1CollectedHeap::verify_region_sets() { |
6669 void G1CollectedHeap::verify_region_sets() { |
6692 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); |
6670 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */); |
6693 |
6671 |
6694 // First, check the explicit lists. |
6672 // First, check the explicit lists. |
6695 _free_list.verify(); |
6673 _free_list.verify_list(); |
6696 { |
6674 { |
6697 // Given that a concurrent operation might be adding regions to |
6675 // Given that a concurrent operation might be adding regions to |
6698 // the secondary free list we have to take the lock before |
6676 // the secondary free list we have to take the lock before |
6699 // verifying it. |
6677 // verifying it. |
6700 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); |
6678 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); |
6701 _secondary_free_list.verify(); |
6679 _secondary_free_list.verify_list(); |
6702 } |
6680 } |
6703 _old_set.verify(); |
|
6704 _humongous_set.verify(); |
|
6705 |
6681 |
6706 // If a concurrent region freeing operation is in progress it will |
6682 // If a concurrent region freeing operation is in progress it will |
6707 // be difficult to correctly attributed any free regions we come |
6683 // be difficult to correctly attributed any free regions we come |
6708 // across to the correct free list given that they might belong to |
6684 // across to the correct free list given that they might belong to |
6709 // one of several (free_list, secondary_free_list, any local lists, |
6685 // one of several (free_list, secondary_free_list, any local lists, |
6722 // attributed to the free_list. |
6698 // attributed to the free_list. |
6723 append_secondary_free_list_if_not_empty_with_lock(); |
6699 append_secondary_free_list_if_not_empty_with_lock(); |
6724 |
6700 |
6725 // Finally, make sure that the region accounting in the lists is |
6701 // Finally, make sure that the region accounting in the lists is |
6726 // consistent with what we see in the heap. |
6702 // consistent with what we see in the heap. |
6727 _old_set.verify_start(); |
|
6728 _humongous_set.verify_start(); |
|
6729 _free_list.verify_start(); |
|
6730 |
6703 |
6731 VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list); |
6704 VerifyRegionListsClosure cl(&_old_set, &_humongous_set, &_free_list); |
6732 heap_region_iterate(&cl); |
6705 heap_region_iterate(&cl); |
6733 |
6706 cl.verify_counts(&_old_set, &_humongous_set, &_free_list); |
6734 _old_set.verify_end(); |
|
6735 _humongous_set.verify_end(); |
|
6736 _free_list.verify_end(); |
|
6737 } |
6707 } |
6738 |
6708 |
6739 // Optimized nmethod scanning |
6709 // Optimized nmethod scanning |
6740 |
6710 |
6741 class RegisterNMethodOopClosure: public OopClosure { |
6711 class RegisterNMethodOopClosure: public OopClosure { |