3262 |
3262 |
3263 HeapWord* |
3263 HeapWord* |
3264 ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size, |
3264 ConcurrentMarkSweepGeneration::expand_and_allocate(size_t word_size, |
3265 bool tlab, |
3265 bool tlab, |
3266 bool parallel) { |
3266 bool parallel) { |
|
3267 CMSSynchronousYieldRequest yr; |
3267 assert(!tlab, "Can't deal with TLAB allocation"); |
3268 assert(!tlab, "Can't deal with TLAB allocation"); |
3268 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); |
3269 MutexLockerEx x(freelistLock(), Mutex::_no_safepoint_check_flag); |
3269 expand(word_size*HeapWordSize, MinHeapDeltaBytes, |
3270 expand(word_size*HeapWordSize, MinHeapDeltaBytes, |
3270 CMSExpansionCause::_satisfy_allocation); |
3271 CMSExpansionCause::_satisfy_allocation); |
3271 if (GCExpandToAllocateDelayMillis > 0) { |
3272 if (GCExpandToAllocateDelayMillis > 0) { |
3708 class CMSConcMarkingTask; |
3709 class CMSConcMarkingTask; |
3709 |
3710 |
3710 class CMSConcMarkingTerminator: public ParallelTaskTerminator { |
3711 class CMSConcMarkingTerminator: public ParallelTaskTerminator { |
3711 CMSCollector* _collector; |
3712 CMSCollector* _collector; |
3712 CMSConcMarkingTask* _task; |
3713 CMSConcMarkingTask* _task; |
3713 bool _yield; |
3714 public: |
3714 protected: |
|
3715 virtual void yield(); |
3715 virtual void yield(); |
3716 public: |
3716 |
3717 // "n_threads" is the number of threads to be terminated. |
3717 // "n_threads" is the number of threads to be terminated. |
3718 // "queue_set" is a set of work queues of other threads. |
3718 // "queue_set" is a set of work queues of other threads. |
3719 // "collector" is the CMS collector associated with this task terminator. |
3719 // "collector" is the CMS collector associated with this task terminator. |
3720 // "yield" indicates whether we need the gang as a whole to yield. |
3720 // "yield" indicates whether we need the gang as a whole to yield. |
3721 CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, |
3721 CMSConcMarkingTerminator(int n_threads, TaskQueueSetSuper* queue_set, CMSCollector* collector) : |
3722 CMSCollector* collector, bool yield) : |
|
3723 ParallelTaskTerminator(n_threads, queue_set), |
3722 ParallelTaskTerminator(n_threads, queue_set), |
3724 _collector(collector), |
3723 _collector(collector) { } |
3725 _yield(yield) { } |
3724 |
3726 |
3725 void set_task(CMSConcMarkingTask* task) { |
|
3726 _task = task; |
|
3727 } |
|
3728 }; |
|
3729 |
|
3730 class CMSConcMarkingTerminatorTerminator: public TerminatorTerminator { |
|
3731 CMSConcMarkingTask* _task; |
|
3732 public: |
|
3733 bool should_exit_termination(); |
3727 void set_task(CMSConcMarkingTask* task) { |
3734 void set_task(CMSConcMarkingTask* task) { |
3728 _task = task; |
3735 _task = task; |
3729 } |
3736 } |
3730 }; |
3737 }; |
3731 |
3738 |
3735 int _n_workers; // requested/desired # workers |
3742 int _n_workers; // requested/desired # workers |
3736 bool _asynch; |
3743 bool _asynch; |
3737 bool _result; |
3744 bool _result; |
3738 CompactibleFreeListSpace* _cms_space; |
3745 CompactibleFreeListSpace* _cms_space; |
3739 CompactibleFreeListSpace* _perm_space; |
3746 CompactibleFreeListSpace* _perm_space; |
3740 HeapWord* _global_finger; |
3747 char _pad_front[64]; // padding to ... |
|
3748 HeapWord* _global_finger; // ... avoid sharing cache line |
|
3749 char _pad_back[64]; |
3741 HeapWord* _restart_addr; |
3750 HeapWord* _restart_addr; |
3742 |
3751 |
3743 // Exposed here for yielding support |
3752 // Exposed here for yielding support |
3744 Mutex* const _bit_map_lock; |
3753 Mutex* const _bit_map_lock; |
3745 |
3754 |
3746 // The per thread work queues, available here for stealing |
3755 // The per thread work queues, available here for stealing |
3747 OopTaskQueueSet* _task_queues; |
3756 OopTaskQueueSet* _task_queues; |
|
3757 |
|
3758 // Termination (and yielding) support |
3748 CMSConcMarkingTerminator _term; |
3759 CMSConcMarkingTerminator _term; |
|
3760 CMSConcMarkingTerminatorTerminator _term_term; |
3749 |
3761 |
3750 public: |
3762 public: |
3751 CMSConcMarkingTask(CMSCollector* collector, |
3763 CMSConcMarkingTask(CMSCollector* collector, |
3752 CompactibleFreeListSpace* cms_space, |
3764 CompactibleFreeListSpace* cms_space, |
3753 CompactibleFreeListSpace* perm_space, |
3765 CompactibleFreeListSpace* perm_space, |
3758 _collector(collector), |
3770 _collector(collector), |
3759 _cms_space(cms_space), |
3771 _cms_space(cms_space), |
3760 _perm_space(perm_space), |
3772 _perm_space(perm_space), |
3761 _asynch(asynch), _n_workers(0), _result(true), |
3773 _asynch(asynch), _n_workers(0), _result(true), |
3762 _task_queues(task_queues), |
3774 _task_queues(task_queues), |
3763 _term(_n_workers, task_queues, _collector, asynch), |
3775 _term(_n_workers, task_queues, _collector), |
3764 _bit_map_lock(collector->bitMapLock()) |
3776 _bit_map_lock(collector->bitMapLock()) |
3765 { |
3777 { |
3766 _requested_size = _n_workers; |
3778 _requested_size = _n_workers; |
3767 _term.set_task(this); |
3779 _term.set_task(this); |
|
3780 _term_term.set_task(this); |
3768 assert(_cms_space->bottom() < _perm_space->bottom(), |
3781 assert(_cms_space->bottom() < _perm_space->bottom(), |
3769 "Finger incorrectly initialized below"); |
3782 "Finger incorrectly initialized below"); |
3770 _restart_addr = _global_finger = _cms_space->bottom(); |
3783 _restart_addr = _global_finger = _cms_space->bottom(); |
3771 } |
3784 } |
3772 |
3785 |
3782 virtual void set_for_termination(int active_workers) { |
3795 virtual void set_for_termination(int active_workers) { |
3783 terminator()->reset_for_reuse(active_workers); |
3796 terminator()->reset_for_reuse(active_workers); |
3784 } |
3797 } |
3785 |
3798 |
3786 void work(int i); |
3799 void work(int i); |
|
3800 bool should_yield() { |
|
3801 return ConcurrentMarkSweepThread::should_yield() |
|
3802 && !_collector->foregroundGCIsActive() |
|
3803 && _asynch; |
|
3804 } |
3787 |
3805 |
3788 virtual void coordinator_yield(); // stuff done by coordinator |
3806 virtual void coordinator_yield(); // stuff done by coordinator |
3789 bool result() { return _result; } |
3807 bool result() { return _result; } |
3790 |
3808 |
3791 void reset(HeapWord* ra) { |
3809 void reset(HeapWord* ra) { |
3803 void do_scan_and_mark(int i, CompactibleFreeListSpace* sp); |
3821 void do_scan_and_mark(int i, CompactibleFreeListSpace* sp); |
3804 void do_work_steal(int i); |
3822 void do_work_steal(int i); |
3805 void bump_global_finger(HeapWord* f); |
3823 void bump_global_finger(HeapWord* f); |
3806 }; |
3824 }; |
3807 |
3825 |
|
3826 bool CMSConcMarkingTerminatorTerminator::should_exit_termination() { |
|
3827 assert(_task != NULL, "Error"); |
|
3828 return _task->yielding(); |
|
3829 // Note that we do not need the disjunct || _task->should_yield() above |
|
3830 // because we want terminating threads to yield only if the task |
|
3831 // is already in the midst of yielding, which happens only after at least one |
|
3832 // thread has yielded. |
|
3833 } |
|
3834 |
3808 void CMSConcMarkingTerminator::yield() { |
3835 void CMSConcMarkingTerminator::yield() { |
3809 if (ConcurrentMarkSweepThread::should_yield() && |
3836 if (_task->should_yield()) { |
3810 !_collector->foregroundGCIsActive() && |
|
3811 _yield) { |
|
3812 _task->yield(); |
3837 _task->yield(); |
3813 } else { |
3838 } else { |
3814 ParallelTaskTerminator::yield(); |
3839 ParallelTaskTerminator::yield(); |
3815 } |
3840 } |
3816 } |
3841 } |
4031 pst->all_tasks_completed(); |
4056 pst->all_tasks_completed(); |
4032 } |
4057 } |
4033 |
4058 |
4034 class Par_ConcMarkingClosure: public Par_KlassRememberingOopClosure { |
4059 class Par_ConcMarkingClosure: public Par_KlassRememberingOopClosure { |
4035 private: |
4060 private: |
|
4061 CMSConcMarkingTask* _task; |
4036 MemRegion _span; |
4062 MemRegion _span; |
4037 CMSBitMap* _bit_map; |
4063 CMSBitMap* _bit_map; |
4038 CMSMarkStack* _overflow_stack; |
4064 CMSMarkStack* _overflow_stack; |
4039 OopTaskQueue* _work_queue; |
4065 OopTaskQueue* _work_queue; |
4040 protected: |
4066 protected: |
4041 DO_OOP_WORK_DEFN |
4067 DO_OOP_WORK_DEFN |
4042 public: |
4068 public: |
4043 Par_ConcMarkingClosure(CMSCollector* collector, OopTaskQueue* work_queue, |
4069 Par_ConcMarkingClosure(CMSCollector* collector, CMSConcMarkingTask* task, OopTaskQueue* work_queue, |
4044 CMSBitMap* bit_map, CMSMarkStack* overflow_stack, |
4070 CMSBitMap* bit_map, CMSMarkStack* overflow_stack, |
4045 CMSMarkStack* revisit_stack): |
4071 CMSMarkStack* revisit_stack): |
4046 Par_KlassRememberingOopClosure(collector, NULL, revisit_stack), |
4072 Par_KlassRememberingOopClosure(collector, NULL, revisit_stack), |
4047 _span(_collector->_span), |
4073 _task(task), |
|
4074 _span(collector->_span), |
4048 _work_queue(work_queue), |
4075 _work_queue(work_queue), |
4049 _bit_map(bit_map), |
4076 _bit_map(bit_map), |
4050 _overflow_stack(overflow_stack) |
4077 _overflow_stack(overflow_stack) |
4051 { } |
4078 { } |
4052 virtual void do_oop(oop* p); |
4079 virtual void do_oop(oop* p); |
4053 virtual void do_oop(narrowOop* p); |
4080 virtual void do_oop(narrowOop* p); |
4054 void trim_queue(size_t max); |
4081 void trim_queue(size_t max); |
4055 void handle_stack_overflow(HeapWord* lost); |
4082 void handle_stack_overflow(HeapWord* lost); |
|
4083 void do_yield_check() { |
|
4084 if (_task->should_yield()) { |
|
4085 _task->yield(); |
|
4086 } |
|
4087 } |
4056 }; |
4088 }; |
4057 |
4089 |
4058 // Grey object scanning during work stealing phase -- |
4090 // Grey object scanning during work stealing phase -- |
4059 // the salient assumption here is that any references |
4091 // the salient assumption here is that any references |
4060 // that are in these stolen objects being scanned must |
4092 // that are in these stolen objects being scanned must |
4094 _work_queue->size() == _work_queue->max_elems(), |
4126 _work_queue->size() == _work_queue->max_elems(), |
4095 "Else push should have succeeded"); |
4127 "Else push should have succeeded"); |
4096 handle_stack_overflow(addr); |
4128 handle_stack_overflow(addr); |
4097 } |
4129 } |
4098 } // Else, some other thread got there first |
4130 } // Else, some other thread got there first |
|
4131 do_yield_check(); |
4099 } |
4132 } |
4100 } |
4133 } |
4101 |
4134 |
4102 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); } |
4135 void Par_ConcMarkingClosure::do_oop(oop* p) { Par_ConcMarkingClosure::do_oop_work(p); } |
4103 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); } |
4136 void Par_ConcMarkingClosure::do_oop(narrowOop* p) { Par_ConcMarkingClosure::do_oop_work(p); } |
4109 assert(new_oop->is_oop(), "Should be an oop"); |
4142 assert(new_oop->is_oop(), "Should be an oop"); |
4110 assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object"); |
4143 assert(_bit_map->isMarked((HeapWord*)new_oop), "Grey object"); |
4111 assert(_span.contains((HeapWord*)new_oop), "Not in span"); |
4144 assert(_span.contains((HeapWord*)new_oop), "Not in span"); |
4112 assert(new_oop->is_parsable(), "Should be parsable"); |
4145 assert(new_oop->is_parsable(), "Should be parsable"); |
4113 new_oop->oop_iterate(this); // do_oop() above |
4146 new_oop->oop_iterate(this); // do_oop() above |
|
4147 do_yield_check(); |
4114 } |
4148 } |
4115 } |
4149 } |
4116 } |
4150 } |
4117 |
4151 |
4118 // Upon stack overflow, we discard (part of) the stack, |
4152 // Upon stack overflow, we discard (part of) the stack, |
4136 oop obj_to_scan; |
4170 oop obj_to_scan; |
4137 CMSBitMap* bm = &(_collector->_markBitMap); |
4171 CMSBitMap* bm = &(_collector->_markBitMap); |
4138 CMSMarkStack* ovflw = &(_collector->_markStack); |
4172 CMSMarkStack* ovflw = &(_collector->_markStack); |
4139 CMSMarkStack* revisit = &(_collector->_revisitStack); |
4173 CMSMarkStack* revisit = &(_collector->_revisitStack); |
4140 int* seed = _collector->hash_seed(i); |
4174 int* seed = _collector->hash_seed(i); |
4141 Par_ConcMarkingClosure cl(_collector, work_q, bm, ovflw, revisit); |
4175 Par_ConcMarkingClosure cl(_collector, this, work_q, bm, ovflw, revisit); |
4142 while (true) { |
4176 while (true) { |
4143 cl.trim_queue(0); |
4177 cl.trim_queue(0); |
4144 assert(work_q->size() == 0, "Should have been emptied above"); |
4178 assert(work_q->size() == 0, "Should have been emptied above"); |
4145 if (get_work_from_overflow_stack(ovflw, work_q)) { |
4179 if (get_work_from_overflow_stack(ovflw, work_q)) { |
4146 // Can't assert below because the work obtained from the |
4180 // Can't assert below because the work obtained from the |
4149 continue; |
4183 continue; |
4150 } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { |
4184 } else if (task_queues()->steal(i, seed, /* reference */ obj_to_scan)) { |
4151 assert(obj_to_scan->is_oop(), "Should be an oop"); |
4185 assert(obj_to_scan->is_oop(), "Should be an oop"); |
4152 assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object"); |
4186 assert(bm->isMarked((HeapWord*)obj_to_scan), "Grey object"); |
4153 obj_to_scan->oop_iterate(&cl); |
4187 obj_to_scan->oop_iterate(&cl); |
4154 } else if (terminator()->offer_termination()) { |
4188 } else if (terminator()->offer_termination(&_term_term)) { |
4155 assert(work_q->size() == 0, "Impossible!"); |
4189 assert(work_q->size() == 0, "Impossible!"); |
4156 break; |
4190 break; |
|
4191 } else if (yielding() || should_yield()) { |
|
4192 yield(); |
4157 } |
4193 } |
4158 } |
4194 } |
4159 } |
4195 } |
4160 |
4196 |
4161 // This is run by the CMS (coordinator) thread. |
4197 // This is run by the CMS (coordinator) thread. |