424 // structures (local and global) have been re-initialised. When they |
424 // structures (local and global) have been re-initialised. When they |
425 // exit it, they are free to start working again. |
425 // exit it, they are free to start working again. |
426 WorkGangBarrierSync _first_overflow_barrier_sync; |
426 WorkGangBarrierSync _first_overflow_barrier_sync; |
427 WorkGangBarrierSync _second_overflow_barrier_sync; |
427 WorkGangBarrierSync _second_overflow_barrier_sync; |
428 |
428 |
429 |
|
430 // this is set by any task, when an overflow on the global data |
429 // this is set by any task, when an overflow on the global data |
431 // structures is detected. |
430 // structures is detected. |
432 volatile bool _has_overflown; |
431 volatile bool _has_overflown; |
433 // true: marking is concurrent, false: we're in remark |
432 // true: marking is concurrent, false: we're in remark |
434 volatile bool _concurrent; |
433 volatile bool _concurrent; |
576 } else { |
575 } else { |
577 return force_overflow_stw(); |
576 return force_overflow_stw(); |
578 } |
577 } |
579 } |
578 } |
580 |
579 |
|
580 // Live Data Counting data structures... |
|
581 // These data structures are initialized at the start of |
|
582 // marking. They are written to while marking is active. |
|
583 // They are aggregated during remark; the aggregated values |
|
584 // are then used to populate the _region_bm, _card_bm, and |
|
585 // the total live bytes, which are then subsequently updated |
|
586 // during cleanup. |
|
587 |
|
588 // An array of bitmaps (one bit map per task). Each bitmap |
|
589 // is used to record the cards spanned by the live objects |
|
590 // marked by that task/worker. |
|
591 BitMap* _count_card_bitmaps; |
|
592 |
|
593 // Used to record the number of marked live bytes |
|
594 // (for each region, by worker thread). |
|
595 size_t** _count_marked_bytes; |
|
596 |
|
597 // Card index of the bottom of the G1 heap. Used for biasing indices into |
|
598 // the card bitmaps. |
|
599 intptr_t _heap_bottom_card_num; |
|
600 |
581 public: |
601 public: |
582 // Manipulation of the global mark stack. |
602 // Manipulation of the global mark stack. |
583 // Notice that the first mark_stack_push is CAS-based, whereas the |
603 // Notice that the first mark_stack_push is CAS-based, whereas the |
584 // two below are Mutex-based. This is OK since the first one is only |
604 // two below are Mutex-based. This is OK since the first one is only |
585 // called during evacuation pauses and doesn't compete with the |
605 // called during evacuation pauses and doesn't compete with the |
719 // The following three are interaction between CM and |
740 // The following three are interaction between CM and |
720 // G1CollectedHeap |
741 // G1CollectedHeap |
721 |
742 |
722 // This notifies CM that a root during initial-mark needs to be |
743 // This notifies CM that a root during initial-mark needs to be |
723 // grayed. It is MT-safe. |
744 // grayed. It is MT-safe. |
724 inline void grayRoot(oop obj, size_t word_size); |
745 inline void grayRoot(oop obj, size_t word_size, uint worker_id); |
725 |
746 |
726 // It's used during evacuation pauses to gray a region, if |
747 // It's used during evacuation pauses to gray a region, if |
727 // necessary, and it's MT-safe. It assumes that the caller has |
748 // necessary, and it's MT-safe. It assumes that the caller has |
728 // marked any objects on that region. If _should_gray_objects is |
749 // marked any objects on that region. If _should_gray_objects is |
729 // true and we're still doing concurrent marking, the region is |
750 // true and we're still doing concurrent marking, the region is |
779 // beginning of an evacuation pause. |
800 // beginning of an evacuation pause. |
780 void drainAllSATBBuffers(); |
801 void drainAllSATBBuffers(); |
781 |
802 |
782 void checkpointRootsFinal(bool clear_all_soft_refs); |
803 void checkpointRootsFinal(bool clear_all_soft_refs); |
783 void checkpointRootsFinalWork(); |
804 void checkpointRootsFinalWork(); |
784 void calcDesiredRegions(); |
|
785 void cleanup(); |
805 void cleanup(); |
786 void completeCleanup(); |
806 void completeCleanup(); |
787 |
807 |
788 // Mark in the previous bitmap. NB: this is usually read-only, so use |
808 // Mark in the previous bitmap. NB: this is usually read-only, so use |
789 // this carefully! |
809 // this carefully! |
790 inline void markPrev(oop p); |
810 inline void markPrev(oop p); |
791 inline void markNext(oop p); |
811 |
792 void clear(oop p); |
|
793 // Clears marks for all objects in the given range, for the prev, |
812 // Clears marks for all objects in the given range, for the prev, |
794 // next, or both bitmaps. NB: the previous bitmap is usually |
813 // next, or both bitmaps. NB: the previous bitmap is usually |
795 // read-only, so use this carefully! |
814 // read-only, so use this carefully! |
796 void clearRangePrevBitmap(MemRegion mr); |
815 void clearRangePrevBitmap(MemRegion mr); |
797 void clearRangeNextBitmap(MemRegion mr); |
816 void clearRangeNextBitmap(MemRegion mr); |
911 return _MARKING_VERBOSE_ && _verbose_level >= medium_verbose; |
930 return _MARKING_VERBOSE_ && _verbose_level >= medium_verbose; |
912 } |
931 } |
913 bool verbose_high() { |
932 bool verbose_high() { |
914 return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; |
933 return _MARKING_VERBOSE_ && _verbose_level >= high_verbose; |
915 } |
934 } |
|
935 |
|
936 // Counting data structure accessors |
|
937 |
|
938 // Returns the card number of the bottom of the G1 heap. |
|
939 // Used in biasing indices into accounting card bitmaps. |
|
940 intptr_t heap_bottom_card_num() const { |
|
941 return _heap_bottom_card_num; |
|
942 } |
|
943 |
|
944 // Returns the card bitmap for a given task or worker id. |
|
945 BitMap* count_card_bitmap_for(uint worker_id) { |
|
946 assert(0 <= worker_id && worker_id < _max_task_num, "oob"); |
|
947 assert(_count_card_bitmaps != NULL, "uninitialized"); |
|
948 BitMap* task_card_bm = &_count_card_bitmaps[worker_id]; |
|
949 assert(task_card_bm->size() == _card_bm.size(), "size mismatch"); |
|
950 return task_card_bm; |
|
951 } |
|
952 |
|
953 // Returns the array containing the marked bytes for each region, |
|
954 // for the given worker or task id. |
|
955 size_t* count_marked_bytes_array_for(uint worker_id) { |
|
956 assert(0 <= worker_id && worker_id < _max_task_num, "oob"); |
|
957 assert(_count_marked_bytes != NULL, "uninitialized"); |
|
958 size_t* marked_bytes_array = _count_marked_bytes[worker_id]; |
|
959 assert(marked_bytes_array != NULL, "uninitialized"); |
|
960 return marked_bytes_array; |
|
961 } |
|
962 |
|
963 // Returns the index in the liveness accounting card table bitmap |
|
964 // for the given address |
|
965 inline BitMap::idx_t card_bitmap_index_for(HeapWord* addr); |
|
966 |
|
967 // Counts the size of the given memory region in the the given |
|
968 // marked_bytes array slot for the given HeapRegion. |
|
969 // Sets the bits in the given card bitmap that are associated with the |
|
970 // cards that are spanned by the memory region. |
|
971 inline void count_region(MemRegion mr, HeapRegion* hr, |
|
972 size_t* marked_bytes_array, |
|
973 BitMap* task_card_bm); |
|
974 |
|
975 // Counts the given memory region in the task/worker counting |
|
976 // data structures for the given worker id. |
|
977 inline void count_region(MemRegion mr, uint worker_id); |
|
978 |
|
979 // Counts the given object in the given task/worker counting |
|
980 // data structures. |
|
981 inline void count_object(oop obj, HeapRegion* hr, |
|
982 size_t* marked_bytes_array, |
|
983 BitMap* task_card_bm); |
|
984 |
|
985 // Counts the given object in the task/worker counting data |
|
986 // structures for the given worker id. |
|
987 inline void count_object(oop obj, HeapRegion* hr, uint worker_id); |
|
988 |
|
989 // Attempts to mark the given object and, if successful, counts |
|
990 // the object in the given task/worker counting structures. |
|
991 inline bool par_mark_and_count(oop obj, HeapRegion* hr, |
|
992 size_t* marked_bytes_array, |
|
993 BitMap* task_card_bm); |
|
994 |
|
995 // Attempts to mark the given object and, if successful, counts |
|
996 // the object in the task/worker counting structures for the |
|
997 // given worker id. |
|
998 inline bool par_mark_and_count(oop obj, HeapRegion* hr, uint worker_id); |
|
999 |
|
1000 // Similar to the above routine but we don't know the heap region that |
|
1001 // contains the object to be marked/counted, which this routine looks up. |
|
1002 inline bool par_mark_and_count(oop obj, uint worker_id); |
|
1003 |
|
1004 // Similar to the above routine but there are times when we cannot |
|
1005 // safely calculate the size of obj due to races and we, therefore, |
|
1006 // pass the size in as a parameter. It is the caller's reponsibility |
|
1007 // to ensure that the size passed in for obj is valid. |
|
1008 inline bool par_mark_and_count(oop obj, size_t word_size, uint worker_id); |
|
1009 |
|
1010 // Unconditionally mark the given object, and unconditinally count |
|
1011 // the object in the counting structures for worker id 0. |
|
1012 // Should *not* be called from parallel code. |
|
1013 inline bool mark_and_count(oop obj, HeapRegion* hr); |
|
1014 |
|
1015 // Similar to the above routine but we don't know the heap region that |
|
1016 // contains the object to be marked/counted, which this routine looks up. |
|
1017 // Should *not* be called from parallel code. |
|
1018 inline bool mark_and_count(oop obj); |
|
1019 |
|
1020 protected: |
|
1021 // Clear all the per-task bitmaps and arrays used to store the |
|
1022 // counting data. |
|
1023 void clear_all_count_data(); |
|
1024 |
|
1025 // Aggregates the counting data for each worker/task |
|
1026 // that was constructed while marking. Also sets |
|
1027 // the amount of marked bytes for each region and |
|
1028 // the top at concurrent mark count. |
|
1029 void aggregate_count_data(); |
|
1030 |
|
1031 // Verification routine |
|
1032 void verify_count_data(); |
916 }; |
1033 }; |
917 |
1034 |
918 // A class representing a marking task. |
1035 // A class representing a marking task. |
919 class CMTask : public TerminatorTerminator { |
1036 class CMTask : public TerminatorTerminator { |
920 private: |
1037 private: |