1.1 --- a/src/share/vm/gc_implementation/g1/ptrQueue.cpp Fri Dec 11 09:30:48 2009 -0800
1.2 +++ b/src/share/vm/gc_implementation/g1/ptrQueue.cpp Wed Dec 16 15:12:51 2009 -0800
1.3 @@ -64,8 +64,8 @@
1.4 while (_index == 0) {
1.5 handle_zero_index();
1.6 }
1.7 +
1.8 assert(_index > 0, "postcondition");
1.9 -
1.10 _index -= oopSize;
1.11 _buf[byte_index_to_index((int)_index)] = ptr;
1.12 assert(0 <= _index && _index <= _sz, "Invariant.");
1.13 @@ -99,95 +99,110 @@
1.14 assert(_sz > 0, "Didn't set a buffer size.");
1.15 MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
1.16 if (_fl_owner->_buf_free_list != NULL) {
1.17 - void** res = _fl_owner->_buf_free_list;
1.18 - _fl_owner->_buf_free_list = (void**)_fl_owner->_buf_free_list[0];
1.19 + void** res = BufferNode::make_buffer_from_node(_fl_owner->_buf_free_list);
1.20 + _fl_owner->_buf_free_list = _fl_owner->_buf_free_list->next();
1.21 _fl_owner->_buf_free_list_sz--;
1.22 - // Just override the next pointer with NULL, just in case we scan this part
1.23 - // of the buffer.
1.24 - res[0] = NULL;
1.25 return res;
1.26 } else {
1.27 - return (void**) NEW_C_HEAP_ARRAY(char, _sz);
1.28 + // Allocate space for the BufferNode in front of the buffer.
1.29 + char *b = NEW_C_HEAP_ARRAY(char, _sz + BufferNode::aligned_size());
1.30 + return BufferNode::make_buffer_from_block(b);
1.31 }
1.32 }
1.33
1.34 void PtrQueueSet::deallocate_buffer(void** buf) {
1.35 assert(_sz > 0, "Didn't set a buffer size.");
1.36 MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag);
1.37 - buf[0] = (void*)_fl_owner->_buf_free_list;
1.38 - _fl_owner->_buf_free_list = buf;
1.39 + BufferNode *node = BufferNode::make_node_from_buffer(buf);
1.40 + node->set_next(_fl_owner->_buf_free_list);
1.41 + _fl_owner->_buf_free_list = node;
1.42 _fl_owner->_buf_free_list_sz++;
1.43 }
1.44
1.45 void PtrQueueSet::reduce_free_list() {
1.46 + assert(_fl_owner == this, "Free list reduction is allowed only for the owner");
1.47 // For now we'll adopt the strategy of deleting half.
1.48 MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
1.49 size_t n = _buf_free_list_sz / 2;
1.50 while (n > 0) {
1.51 assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong.");
1.52 - void** head = _buf_free_list;
1.53 - _buf_free_list = (void**)_buf_free_list[0];
1.54 - FREE_C_HEAP_ARRAY(char, head);
1.55 + void* b = BufferNode::make_block_from_node(_buf_free_list);
1.56 + _buf_free_list = _buf_free_list->next();
1.57 + FREE_C_HEAP_ARRAY(char, b);
1.58 _buf_free_list_sz --;
1.59 n--;
1.60 }
1.61 }
1.62
1.63 -void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index, bool ignore_max_completed) {
1.64 - // I use explicit locking here because there's a bailout in the middle.
1.65 - _cbl_mon->lock_without_safepoint_check();
1.66 +void PtrQueue::handle_zero_index() {
1.67 + assert(0 == _index, "Precondition.");
1.68 + // This thread records the full buffer and allocates a new one (while
1.69 + // holding the lock if there is one).
1.70 + if (_buf != NULL) {
1.71 + if (_lock) {
1.72 + locking_enqueue_completed_buffer(_buf);
1.73 + } else {
1.74 + if (qset()->process_or_enqueue_complete_buffer(_buf)) {
1.75 + // Recycle the buffer. No allocation.
1.76 + _sz = qset()->buffer_size();
1.77 + _index = _sz;
1.78 + return;
1.79 + }
1.80 + }
1.81 + }
1.82 + // Reallocate the buffer
1.83 + _buf = qset()->allocate_buffer();
1.84 + _sz = qset()->buffer_size();
1.85 + _index = _sz;
1.86 + assert(0 <= _index && _index <= _sz, "Invariant.");
1.87 +}
1.88
1.89 - Thread* thread = Thread::current();
1.90 - assert( ignore_max_completed ||
1.91 - thread->is_Java_thread() ||
1.92 - SafepointSynchronize::is_at_safepoint(),
1.93 - "invariant" );
1.94 - ignore_max_completed = ignore_max_completed || !thread->is_Java_thread();
1.95 +bool PtrQueueSet::process_or_enqueue_complete_buffer(void** buf) {
1.96 + if (Thread::current()->is_Java_thread()) {
1.97 + // We don't lock. It is fine to be epsilon-precise here.
1.98 + if (_max_completed_queue == 0 || _max_completed_queue > 0 &&
1.99 + _n_completed_buffers >= _max_completed_queue + _completed_queue_padding) {
1.100 + bool b = mut_process_buffer(buf);
1.101 + if (b) {
1.102 + // True here means that the buffer hasn't been deallocated and the caller may reuse it.
1.103 + return true;
1.104 + }
1.105 + }
1.106 + }
1.107 + // The buffer will be enqueued. The caller will have to get a new one.
1.108 + enqueue_complete_buffer(buf);
1.109 + return false;
1.110 +}
1.111
1.112 - if (!ignore_max_completed && _max_completed_queue > 0 &&
1.113 - _n_completed_buffers >= (size_t) _max_completed_queue) {
1.114 - _cbl_mon->unlock();
1.115 - bool b = mut_process_buffer(buf);
1.116 - if (b) {
1.117 - deallocate_buffer(buf);
1.118 - return;
1.119 - }
1.120 -
1.121 - // Otherwise, go ahead and enqueue the buffer. Must reaquire the lock.
1.122 - _cbl_mon->lock_without_safepoint_check();
1.123 - }
1.124 -
1.125 - // Here we still hold the _cbl_mon.
1.126 - CompletedBufferNode* cbn = new CompletedBufferNode;
1.127 - cbn->buf = buf;
1.128 - cbn->next = NULL;
1.129 - cbn->index = index;
1.130 +void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index) {
1.131 + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
1.132 + BufferNode* cbn = BufferNode::new_from_buffer(buf);
1.133 + cbn->set_index(index);
1.134 if (_completed_buffers_tail == NULL) {
1.135 assert(_completed_buffers_head == NULL, "Well-formedness");
1.136 _completed_buffers_head = cbn;
1.137 _completed_buffers_tail = cbn;
1.138 } else {
1.139 - _completed_buffers_tail->next = cbn;
1.140 + _completed_buffers_tail->set_next(cbn);
1.141 _completed_buffers_tail = cbn;
1.142 }
1.143 _n_completed_buffers++;
1.144
1.145 - if (!_process_completed &&
1.146 + if (!_process_completed && _process_completed_threshold >= 0 &&
1.147 _n_completed_buffers >= _process_completed_threshold) {
1.148 _process_completed = true;
1.149 if (_notify_when_complete)
1.150 - _cbl_mon->notify_all();
1.151 + _cbl_mon->notify();
1.152 }
1.153 debug_only(assert_completed_buffer_list_len_correct_locked());
1.154 - _cbl_mon->unlock();
1.155 }
1.156
1.157 int PtrQueueSet::completed_buffers_list_length() {
1.158 int n = 0;
1.159 - CompletedBufferNode* cbn = _completed_buffers_head;
1.160 + BufferNode* cbn = _completed_buffers_head;
1.161 while (cbn != NULL) {
1.162 n++;
1.163 - cbn = cbn->next;
1.164 + cbn = cbn->next();
1.165 }
1.166 return n;
1.167 }
1.168 @@ -198,7 +213,7 @@
1.169 }
1.170
1.171 void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() {
1.172 - guarantee((size_t)completed_buffers_list_length() == _n_completed_buffers,
1.173 + guarantee(completed_buffers_list_length() == _n_completed_buffers,
1.174 "Completed buffer length is wrong.");
1.175 }
1.176
1.177 @@ -207,12 +222,8 @@
1.178 _sz = sz * oopSize;
1.179 }
1.180
1.181 -void PtrQueueSet::set_process_completed_threshold(size_t sz) {
1.182 - _process_completed_threshold = sz;
1.183 -}
1.184 -
1.185 -// Merge lists of buffers. Notify waiting threads if the length of the list
1.186 -// exceeds threshold. The source queue is emptied as a result. The queues
1.187 +// Merge lists of buffers. Notify the processing threads.
1.188 +// The source queue is emptied as a result. The queues
1.189 // must share the monitor.
1.190 void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) {
1.191 assert(_cbl_mon == src->_cbl_mon, "Should share the same lock");
1.192 @@ -224,7 +235,7 @@
1.193 } else {
1.194 assert(_completed_buffers_head != NULL, "Well formedness");
1.195 if (src->_completed_buffers_head != NULL) {
1.196 - _completed_buffers_tail->next = src->_completed_buffers_head;
1.197 + _completed_buffers_tail->set_next(src->_completed_buffers_head);
1.198 _completed_buffers_tail = src->_completed_buffers_tail;
1.199 }
1.200 }
1.201 @@ -237,31 +248,13 @@
1.202 assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL ||
1.203 _completed_buffers_head != NULL && _completed_buffers_tail != NULL,
1.204 "Sanity");
1.205 +}
1.206
1.207 - if (!_process_completed &&
1.208 - _n_completed_buffers >= _process_completed_threshold) {
1.209 +void PtrQueueSet::notify_if_necessary() {
1.210 + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
1.211 + if (_n_completed_buffers >= _process_completed_threshold || _max_completed_queue == 0) {
1.212 _process_completed = true;
1.213 if (_notify_when_complete)
1.214 - _cbl_mon->notify_all();
1.215 + _cbl_mon->notify();
1.216 }
1.217 }
1.218 -
1.219 -// Merge free lists of the two queues. The free list of the source
1.220 -// queue is emptied as a result. The queues must share the same
1.221 -// mutex that guards free lists.
1.222 -void PtrQueueSet::merge_freelists(PtrQueueSet* src) {
1.223 - assert(_fl_lock == src->_fl_lock, "Should share the same lock");
1.224 - MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag);
1.225 - if (_buf_free_list != NULL) {
1.226 - void **p = _buf_free_list;
1.227 - while (*p != NULL) {
1.228 - p = (void**)*p;
1.229 - }
1.230 - *p = src->_buf_free_list;
1.231 - } else {
1.232 - _buf_free_list = src->_buf_free_list;
1.233 - }
1.234 - _buf_free_list_sz += src->_buf_free_list_sz;
1.235 - src->_buf_free_list = NULL;
1.236 - src->_buf_free_list_sz = 0;
1.237 -}
