Fri, 18 Feb 2011 10:07:34 -0800
7020042: G1: Partially remove fix for 6994628
Summary: Disable reference discovery and processing during concurrent marking by disabling fix for 6994628.
Reviewed-by: tonyp, ysr
ysr@777 | 1 | /* |
tonyp@2469 | 2 | * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. |
ysr@777 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
ysr@777 | 4 | * |
ysr@777 | 5 | * This code is free software; you can redistribute it and/or modify it |
ysr@777 | 6 | * under the terms of the GNU General Public License version 2 only, as |
ysr@777 | 7 | * published by the Free Software Foundation. |
ysr@777 | 8 | * |
ysr@777 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
ysr@777 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
ysr@777 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
ysr@777 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
ysr@777 | 13 | * accompanied this code). |
ysr@777 | 14 | * |
ysr@777 | 15 | * You should have received a copy of the GNU General Public License version |
ysr@777 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
ysr@777 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
ysr@777 | 18 | * |
trims@1907 | 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
trims@1907 | 20 | * or visit www.oracle.com if you need additional information or have any |
trims@1907 | 21 | * questions. |
ysr@777 | 22 | * |
ysr@777 | 23 | */ |
ysr@777 | 24 | |
stefank@2314 | 25 | #include "precompiled.hpp" |
stefank@2314 | 26 | #include "gc_implementation/g1/ptrQueue.hpp" |
stefank@2314 | 27 | #include "memory/allocation.hpp" |
stefank@2314 | 28 | #include "memory/allocation.inline.hpp" |
stefank@2314 | 29 | #include "runtime/mutex.hpp" |
stefank@2314 | 30 | #include "runtime/mutexLocker.hpp" |
stefank@2314 | 31 | #ifdef TARGET_OS_FAMILY_linux |
stefank@2314 | 32 | # include "thread_linux.inline.hpp" |
stefank@2314 | 33 | #endif |
stefank@2314 | 34 | #ifdef TARGET_OS_FAMILY_solaris |
stefank@2314 | 35 | # include "thread_solaris.inline.hpp" |
stefank@2314 | 36 | #endif |
stefank@2314 | 37 | #ifdef TARGET_OS_FAMILY_windows |
stefank@2314 | 38 | # include "thread_windows.inline.hpp" |
stefank@2314 | 39 | #endif |
ysr@777 | 40 | |
tonyp@2469 | 41 | PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) : |
tonyp@2469 | 42 | _qset(qset), _buf(NULL), _index(0), _active(active), |
ysr@777 | 43 | _perm(perm), _lock(NULL) |
ysr@777 | 44 | {} |
ysr@777 | 45 | |
iveresov@876 | 46 | void PtrQueue::flush() { |
ysr@777 | 47 | if (!_perm && _buf != NULL) { |
ysr@777 | 48 | if (_index == _sz) { |
ysr@777 | 49 | // No work to do. |
ysr@777 | 50 | qset()->deallocate_buffer(_buf); |
ysr@777 | 51 | } else { |
ysr@777 | 52 | // We must NULL out the unused entries, then enqueue. |
ysr@777 | 53 | for (size_t i = 0; i < _index; i += oopSize) { |
ysr@777 | 54 | _buf[byte_index_to_index((int)i)] = NULL; |
ysr@777 | 55 | } |
ysr@777 | 56 | qset()->enqueue_complete_buffer(_buf); |
ysr@777 | 57 | } |
iveresov@876 | 58 | _buf = NULL; |
iveresov@876 | 59 | _index = 0; |
ysr@777 | 60 | } |
ysr@777 | 61 | } |
ysr@777 | 62 | |
ysr@777 | 63 | |
ysr@777 | 64 | static int byte_index_to_index(int ind) { |
ysr@777 | 65 | assert((ind % oopSize) == 0, "Invariant."); |
ysr@777 | 66 | return ind / oopSize; |
ysr@777 | 67 | } |
ysr@777 | 68 | |
ysr@777 | 69 | static int index_to_byte_index(int byte_ind) { |
ysr@777 | 70 | return byte_ind * oopSize; |
ysr@777 | 71 | } |
ysr@777 | 72 | |
ysr@777 | 73 | void PtrQueue::enqueue_known_active(void* ptr) { |
ysr@777 | 74 | assert(0 <= _index && _index <= _sz, "Invariant."); |
ysr@777 | 75 | assert(_index == 0 || _buf != NULL, "invariant"); |
ysr@777 | 76 | |
ysr@777 | 77 | while (_index == 0) { |
ysr@777 | 78 | handle_zero_index(); |
ysr@777 | 79 | } |
iveresov@1546 | 80 | |
ysr@777 | 81 | assert(_index > 0, "postcondition"); |
ysr@777 | 82 | _index -= oopSize; |
ysr@777 | 83 | _buf[byte_index_to_index((int)_index)] = ptr; |
ysr@777 | 84 | assert(0 <= _index && _index <= _sz, "Invariant."); |
ysr@777 | 85 | } |
ysr@777 | 86 | |
ysr@777 | 87 | void PtrQueue::locking_enqueue_completed_buffer(void** buf) { |
ysr@777 | 88 | assert(_lock->owned_by_self(), "Required."); |
johnc@1604 | 89 | |
johnc@1604 | 90 | // We have to unlock _lock (which may be Shared_DirtyCardQ_lock) before |
johnc@1604 | 91 | // we acquire DirtyCardQ_CBL_mon inside enqeue_complete_buffer as they |
johnc@1604 | 92 | // have the same rank and we may get the "possible deadlock" message |
ysr@777 | 93 | _lock->unlock(); |
johnc@1604 | 94 | |
ysr@777 | 95 | qset()->enqueue_complete_buffer(buf); |
ysr@777 | 96 | // We must relock only because the caller will unlock, for the normal |
ysr@777 | 97 | // case. |
ysr@777 | 98 | _lock->lock_without_safepoint_check(); |
ysr@777 | 99 | } |
ysr@777 | 100 | |
ysr@777 | 101 | |
ysr@777 | 102 | PtrQueueSet::PtrQueueSet(bool notify_when_complete) : |
ysr@777 | 103 | _max_completed_queue(0), |
ysr@777 | 104 | _cbl_mon(NULL), _fl_lock(NULL), |
ysr@777 | 105 | _notify_when_complete(notify_when_complete), |
ysr@777 | 106 | _sz(0), |
ysr@777 | 107 | _completed_buffers_head(NULL), |
ysr@777 | 108 | _completed_buffers_tail(NULL), |
ysr@777 | 109 | _n_completed_buffers(0), |
ysr@777 | 110 | _process_completed_threshold(0), _process_completed(false), |
ysr@777 | 111 | _buf_free_list(NULL), _buf_free_list_sz(0) |
iveresov@1051 | 112 | { |
iveresov@1051 | 113 | _fl_owner = this; |
iveresov@1051 | 114 | } |
ysr@777 | 115 | |
ysr@777 | 116 | void** PtrQueueSet::allocate_buffer() { |
ysr@777 | 117 | assert(_sz > 0, "Didn't set a buffer size."); |
iveresov@1051 | 118 | MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); |
iveresov@1051 | 119 | if (_fl_owner->_buf_free_list != NULL) { |
iveresov@1546 | 120 | void** res = BufferNode::make_buffer_from_node(_fl_owner->_buf_free_list); |
iveresov@1546 | 121 | _fl_owner->_buf_free_list = _fl_owner->_buf_free_list->next(); |
iveresov@1051 | 122 | _fl_owner->_buf_free_list_sz--; |
ysr@777 | 123 | return res; |
ysr@777 | 124 | } else { |
iveresov@1546 | 125 | // Allocate space for the BufferNode in front of the buffer. |
iveresov@1546 | 126 | char *b = NEW_C_HEAP_ARRAY(char, _sz + BufferNode::aligned_size()); |
iveresov@1546 | 127 | return BufferNode::make_buffer_from_block(b); |
ysr@777 | 128 | } |
ysr@777 | 129 | } |
ysr@777 | 130 | |
ysr@777 | 131 | void PtrQueueSet::deallocate_buffer(void** buf) { |
ysr@777 | 132 | assert(_sz > 0, "Didn't set a buffer size."); |
iveresov@1051 | 133 | MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); |
iveresov@1546 | 134 | BufferNode *node = BufferNode::make_node_from_buffer(buf); |
iveresov@1546 | 135 | node->set_next(_fl_owner->_buf_free_list); |
iveresov@1546 | 136 | _fl_owner->_buf_free_list = node; |
iveresov@1051 | 137 | _fl_owner->_buf_free_list_sz++; |
ysr@777 | 138 | } |
ysr@777 | 139 | |
ysr@777 | 140 | void PtrQueueSet::reduce_free_list() { |
iveresov@1546 | 141 | assert(_fl_owner == this, "Free list reduction is allowed only for the owner"); |
ysr@777 | 142 | // For now we'll adopt the strategy of deleting half. |
ysr@777 | 143 | MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); |
ysr@777 | 144 | size_t n = _buf_free_list_sz / 2; |
ysr@777 | 145 | while (n > 0) { |
ysr@777 | 146 | assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong."); |
iveresov@1546 | 147 | void* b = BufferNode::make_block_from_node(_buf_free_list); |
iveresov@1546 | 148 | _buf_free_list = _buf_free_list->next(); |
iveresov@1546 | 149 | FREE_C_HEAP_ARRAY(char, b); |
johnc@1519 | 150 | _buf_free_list_sz --; |
ysr@777 | 151 | n--; |
ysr@777 | 152 | } |
ysr@777 | 153 | } |
ysr@777 | 154 | |
iveresov@1546 | 155 | void PtrQueue::handle_zero_index() { |
tonyp@2469 | 156 | assert(_index == 0, "Precondition."); |
tonyp@2469 | 157 | |
iveresov@1546 | 158 | // This thread records the full buffer and allocates a new one (while |
iveresov@1546 | 159 | // holding the lock if there is one). |
iveresov@1546 | 160 | if (_buf != NULL) { |
tonyp@2469 | 161 | if (!should_enqueue_buffer()) { |
tonyp@2469 | 162 | assert(_index > 0, "the buffer can only be re-used if it's not full"); |
tonyp@2469 | 163 | return; |
tonyp@2469 | 164 | } |
tonyp@2469 | 165 | |
iveresov@1546 | 166 | if (_lock) { |
johnc@1604 | 167 | assert(_lock->owned_by_self(), "Required."); |
johnc@1604 | 168 | |
johnc@1604 | 169 | // The current PtrQ may be the shared dirty card queue and |
johnc@1604 | 170 | // may be being manipulated by more than one worker thread |
johnc@1604 | 171 | // during a pause. Since the enqueuing of the completed |
johnc@1604 | 172 | // buffer unlocks the Shared_DirtyCardQ_lock more than one |
johnc@1604 | 173 | // worker thread can 'race' on reading the shared queue attributes |
johnc@1604 | 174 | // (_buf and _index) and multiple threads can call into this |
johnc@1604 | 175 | // routine for the same buffer. This will cause the completed |
johnc@1604 | 176 | // buffer to be added to the CBL multiple times. |
johnc@1604 | 177 | |
johnc@1604 | 178 | // We "claim" the current buffer by caching value of _buf in |
johnc@1604 | 179 | // a local and clearing the field while holding _lock. When |
johnc@1604 | 180 | // _lock is released (while enqueueing the completed buffer) |
johnc@1604 | 181 | // the thread that acquires _lock will skip this code, |
johnc@1604 | 182 | // preventing the subsequent the multiple enqueue, and |
johnc@1604 | 183 | // install a newly allocated buffer below. |
johnc@1604 | 184 | |
johnc@1604 | 185 | void** buf = _buf; // local pointer to completed buffer |
johnc@1604 | 186 | _buf = NULL; // clear shared _buf field |
johnc@1604 | 187 | |
johnc@1604 | 188 | locking_enqueue_completed_buffer(buf); // enqueue completed buffer |
johnc@1604 | 189 | |
johnc@1604 | 190 | // While the current thread was enqueuing the buffer another thread |
johnc@1604 | 191 | // may have a allocated a new buffer and inserted it into this pointer |
johnc@1604 | 192 | // queue. If that happens then we just return so that the current |
johnc@1604 | 193 | // thread doesn't overwrite the buffer allocated by the other thread |
johnc@1604 | 194 | // and potentially losing some dirtied cards. |
johnc@1604 | 195 | |
johnc@1604 | 196 | if (_buf != NULL) return; |
iveresov@1546 | 197 | } else { |
iveresov@1546 | 198 | if (qset()->process_or_enqueue_complete_buffer(_buf)) { |
iveresov@1546 | 199 | // Recycle the buffer. No allocation. |
iveresov@1546 | 200 | _sz = qset()->buffer_size(); |
iveresov@1546 | 201 | _index = _sz; |
iveresov@1546 | 202 | return; |
iveresov@1546 | 203 | } |
iveresov@1546 | 204 | } |
iveresov@1546 | 205 | } |
iveresov@1546 | 206 | // Reallocate the buffer |
iveresov@1546 | 207 | _buf = qset()->allocate_buffer(); |
iveresov@1546 | 208 | _sz = qset()->buffer_size(); |
iveresov@1546 | 209 | _index = _sz; |
iveresov@1546 | 210 | assert(0 <= _index && _index <= _sz, "Invariant."); |
iveresov@1546 | 211 | } |
ysr@777 | 212 | |
iveresov@1546 | 213 | bool PtrQueueSet::process_or_enqueue_complete_buffer(void** buf) { |
iveresov@1546 | 214 | if (Thread::current()->is_Java_thread()) { |
iveresov@1546 | 215 | // We don't lock. It is fine to be epsilon-precise here. |
iveresov@1546 | 216 | if (_max_completed_queue == 0 || _max_completed_queue > 0 && |
iveresov@1546 | 217 | _n_completed_buffers >= _max_completed_queue + _completed_queue_padding) { |
iveresov@1546 | 218 | bool b = mut_process_buffer(buf); |
iveresov@1546 | 219 | if (b) { |
iveresov@1546 | 220 | // True here means that the buffer hasn't been deallocated and the caller may reuse it. |
iveresov@1546 | 221 | return true; |
iveresov@1546 | 222 | } |
iveresov@1546 | 223 | } |
iveresov@1546 | 224 | } |
iveresov@1546 | 225 | // The buffer will be enqueued. The caller will have to get a new one. |
iveresov@1546 | 226 | enqueue_complete_buffer(buf); |
iveresov@1546 | 227 | return false; |
iveresov@1546 | 228 | } |
ysr@777 | 229 | |
iveresov@1546 | 230 | void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index) { |
iveresov@1546 | 231 | MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
iveresov@1546 | 232 | BufferNode* cbn = BufferNode::new_from_buffer(buf); |
iveresov@1546 | 233 | cbn->set_index(index); |
ysr@777 | 234 | if (_completed_buffers_tail == NULL) { |
ysr@777 | 235 | assert(_completed_buffers_head == NULL, "Well-formedness"); |
ysr@777 | 236 | _completed_buffers_head = cbn; |
ysr@777 | 237 | _completed_buffers_tail = cbn; |
ysr@777 | 238 | } else { |
iveresov@1546 | 239 | _completed_buffers_tail->set_next(cbn); |
ysr@777 | 240 | _completed_buffers_tail = cbn; |
ysr@777 | 241 | } |
ysr@777 | 242 | _n_completed_buffers++; |
ysr@777 | 243 | |
iveresov@1546 | 244 | if (!_process_completed && _process_completed_threshold >= 0 && |
iveresov@1229 | 245 | _n_completed_buffers >= _process_completed_threshold) { |
ysr@777 | 246 | _process_completed = true; |
ysr@777 | 247 | if (_notify_when_complete) |
iveresov@1546 | 248 | _cbl_mon->notify(); |
ysr@777 | 249 | } |
ysr@777 | 250 | debug_only(assert_completed_buffer_list_len_correct_locked()); |
ysr@777 | 251 | } |
ysr@777 | 252 | |
ysr@777 | 253 | int PtrQueueSet::completed_buffers_list_length() { |
ysr@777 | 254 | int n = 0; |
iveresov@1546 | 255 | BufferNode* cbn = _completed_buffers_head; |
ysr@777 | 256 | while (cbn != NULL) { |
ysr@777 | 257 | n++; |
iveresov@1546 | 258 | cbn = cbn->next(); |
ysr@777 | 259 | } |
ysr@777 | 260 | return n; |
ysr@777 | 261 | } |
ysr@777 | 262 | |
ysr@777 | 263 | void PtrQueueSet::assert_completed_buffer_list_len_correct() { |
ysr@777 | 264 | MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
ysr@777 | 265 | assert_completed_buffer_list_len_correct_locked(); |
ysr@777 | 266 | } |
ysr@777 | 267 | |
ysr@777 | 268 | void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() { |
iveresov@1546 | 269 | guarantee(completed_buffers_list_length() == _n_completed_buffers, |
ysr@777 | 270 | "Completed buffer length is wrong."); |
ysr@777 | 271 | } |
ysr@777 | 272 | |
ysr@777 | 273 | void PtrQueueSet::set_buffer_size(size_t sz) { |
ysr@777 | 274 | assert(_sz == 0 && sz > 0, "Should be called only once."); |
ysr@777 | 275 | _sz = sz * oopSize; |
ysr@777 | 276 | } |
ysr@777 | 277 | |
iveresov@1546 | 278 | // Merge lists of buffers. Notify the processing threads. |
iveresov@1546 | 279 | // The source queue is emptied as a result. The queues |
iveresov@1051 | 280 | // must share the monitor. |
iveresov@1051 | 281 | void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) { |
iveresov@1051 | 282 | assert(_cbl_mon == src->_cbl_mon, "Should share the same lock"); |
iveresov@1051 | 283 | MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
iveresov@1051 | 284 | if (_completed_buffers_tail == NULL) { |
iveresov@1051 | 285 | assert(_completed_buffers_head == NULL, "Well-formedness"); |
iveresov@1051 | 286 | _completed_buffers_head = src->_completed_buffers_head; |
iveresov@1051 | 287 | _completed_buffers_tail = src->_completed_buffers_tail; |
iveresov@1051 | 288 | } else { |
iveresov@1051 | 289 | assert(_completed_buffers_head != NULL, "Well formedness"); |
iveresov@1051 | 290 | if (src->_completed_buffers_head != NULL) { |
iveresov@1546 | 291 | _completed_buffers_tail->set_next(src->_completed_buffers_head); |
iveresov@1051 | 292 | _completed_buffers_tail = src->_completed_buffers_tail; |
iveresov@1051 | 293 | } |
iveresov@1051 | 294 | } |
iveresov@1051 | 295 | _n_completed_buffers += src->_n_completed_buffers; |
iveresov@1051 | 296 | |
iveresov@1051 | 297 | src->_n_completed_buffers = 0; |
iveresov@1051 | 298 | src->_completed_buffers_head = NULL; |
iveresov@1051 | 299 | src->_completed_buffers_tail = NULL; |
iveresov@1051 | 300 | |
iveresov@1051 | 301 | assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || |
iveresov@1051 | 302 | _completed_buffers_head != NULL && _completed_buffers_tail != NULL, |
iveresov@1051 | 303 | "Sanity"); |
iveresov@1546 | 304 | } |
iveresov@1051 | 305 | |
iveresov@1546 | 306 | void PtrQueueSet::notify_if_necessary() { |
iveresov@1546 | 307 | MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
iveresov@1546 | 308 | if (_n_completed_buffers >= _process_completed_threshold || _max_completed_queue == 0) { |
iveresov@1051 | 309 | _process_completed = true; |
iveresov@1051 | 310 | if (_notify_when_complete) |
iveresov@1546 | 311 | _cbl_mon->notify(); |
iveresov@1051 | 312 | } |
iveresov@1051 | 313 | } |