1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp Wed Apr 27 01:25:04 2016 +0800
1.3 @@ -0,0 +1,423 @@
1.4 +/*
1.5 + * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation.
1.11 + *
1.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.15 + * version 2 for more details (a copy is included in the LICENSE file that
1.16 + * accompanied this code).
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License version
1.19 + * 2 along with this work; if not, write to the Free Software Foundation,
1.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.21 + *
1.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.23 + * or visit www.oracle.com if you need additional information or have any
1.24 + * questions.
1.25 + *
1.26 + */
1.27 +
1.28 +#include "precompiled.hpp"
1.29 +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
1.30 +#include "gc_implementation/g1/satbQueue.hpp"
1.31 +#include "memory/allocation.inline.hpp"
1.32 +#include "memory/sharedHeap.hpp"
1.33 +#include "oops/oop.inline.hpp"
1.34 +#include "runtime/mutexLocker.hpp"
1.35 +#include "runtime/thread.hpp"
1.36 +#include "runtime/vmThread.hpp"
1.37 +
1.38 +PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
1.39 +
1.40 +void ObjPtrQueue::flush() {
1.41 + // The buffer might contain refs into the CSet. We have to filter it
1.42 + // first before we flush it, otherwise we might end up with an
1.43 + // enqueued buffer with refs into the CSet which breaks our invariants.
1.44 + filter();
1.45 + PtrQueue::flush();
1.46 +}
1.47 +
1.48 +// This method removes entries from an SATB buffer that will not be
1.49 +// useful to the concurrent marking threads. An entry is removed if it
1.50 +// satisfies one of the following conditions:
1.51 +//
1.52 +// * it points to an object outside the G1 heap (G1's concurrent
1.53 +// marking only visits objects inside the G1 heap),
1.54 +// * it points to an object that has been allocated since marking
1.55 +// started (according to SATB those objects do not need to be
1.56 +// visited during marking), or
1.57 +// * it points to an object that has already been marked (no need to
1.58 +// process it again).
1.59 +//
1.60 +// The rest of the entries will be retained and are compacted towards
1.61 +// the top of the buffer. Note that, because we do not allow old
1.62 +// regions in the CSet during marking, all objects on the CSet regions
1.63 +// are young (eden or survivors) and therefore implicitly live. So any
1.64 +// references into the CSet will be removed during filtering.
1.65 +
1.66 +void ObjPtrQueue::filter() {
1.67 + G1CollectedHeap* g1h = G1CollectedHeap::heap();
1.68 + void** buf = _buf;
1.69 + size_t sz = _sz;
1.70 +
1.71 + if (buf == NULL) {
1.72 + // nothing to do
1.73 + return;
1.74 + }
1.75 +
1.76 + // Used for sanity checking at the end of the loop.
1.77 + debug_only(size_t entries = 0; size_t retained = 0;)
1.78 +
1.79 + size_t i = sz;
1.80 + size_t new_index = sz;
1.81 +
1.82 + while (i > _index) {
1.83 + assert(i > 0, "we should have at least one more entry to process");
1.84 + i -= oopSize;
1.85 + debug_only(entries += 1;)
1.86 + oop* p = (oop*) &buf[byte_index_to_index((int) i)];
1.87 + oop obj = *p;
1.88 + // NULL the entry so that unused parts of the buffer contain NULLs
1.89 + // at the end. If we are going to retain it we will copy it to its
1.90 + // final place. If we have retained all entries we have visited so
1.91 + // far, we'll just end up copying it to the same place.
1.92 + *p = NULL;
1.93 +
1.94 + bool retain = g1h->is_obj_ill(obj);
1.95 + if (retain) {
1.96 + assert(new_index > 0, "we should not have already filled up the buffer");
1.97 + new_index -= oopSize;
1.98 + assert(new_index >= i,
1.99 + "new_index should never be below i, as we alwaysr compact 'up'");
1.100 + oop* new_p = (oop*) &buf[byte_index_to_index((int) new_index)];
1.101 + assert(new_p >= p, "the destination location should never be below "
1.102 + "the source as we always compact 'up'");
1.103 + assert(*new_p == NULL,
1.104 + "we should have already cleared the destination location");
1.105 + *new_p = obj;
1.106 + debug_only(retained += 1;)
1.107 + }
1.108 + }
1.109 +
1.110 +#ifdef ASSERT
1.111 + size_t entries_calc = (sz - _index) / oopSize;
1.112 + assert(entries == entries_calc, "the number of entries we counted "
1.113 + "should match the number of entries we calculated");
1.114 + size_t retained_calc = (sz - new_index) / oopSize;
1.115 + assert(retained == retained_calc, "the number of retained entries we counted "
1.116 + "should match the number of retained entries we calculated");
1.117 +#endif // ASSERT
1.118 +
1.119 + _index = new_index;
1.120 +}
1.121 +
1.122 +// This method will first apply the above filtering to the buffer. If
1.123 +// post-filtering a large enough chunk of the buffer has been cleared
1.124 +// we can re-use the buffer (instead of enqueueing it) and we can just
1.125 +// allow the mutator to carry on executing using the same buffer
1.126 +// instead of replacing it.
1.127 +
1.128 +bool ObjPtrQueue::should_enqueue_buffer() {
1.129 + assert(_lock == NULL || _lock->owned_by_self(),
1.130 + "we should have taken the lock before calling this");
1.131 +
1.132 + // Even if G1SATBBufferEnqueueingThresholdPercent == 0 we have to
1.133 + // filter the buffer given that this will remove any references into
1.134 + // the CSet as we currently assume that no such refs will appear in
1.135 + // enqueued buffers.
1.136 +
1.137 + // This method should only be called if there is a non-NULL buffer
1.138 + // that is full.
1.139 + assert(_index == 0, "pre-condition");
1.140 + assert(_buf != NULL, "pre-condition");
1.141 +
1.142 + filter();
1.143 +
1.144 + size_t sz = _sz;
1.145 + size_t all_entries = sz / oopSize;
1.146 + size_t retained_entries = (sz - _index) / oopSize;
1.147 + size_t perc = retained_entries * 100 / all_entries;
1.148 + bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent;
1.149 + return should_enqueue;
1.150 +}
1.151 +
1.152 +void ObjPtrQueue::apply_closure(ObjectClosure* cl) {
1.153 + if (_buf != NULL) {
1.154 + apply_closure_to_buffer(cl, _buf, _index, _sz);
1.155 + }
1.156 +}
1.157 +
1.158 +void ObjPtrQueue::apply_closure_and_empty(ObjectClosure* cl) {
1.159 + if (_buf != NULL) {
1.160 + apply_closure_to_buffer(cl, _buf, _index, _sz);
1.161 + _index = _sz;
1.162 + }
1.163 +}
1.164 +
1.165 +void ObjPtrQueue::apply_closure_to_buffer(ObjectClosure* cl,
1.166 + void** buf, size_t index, size_t sz) {
1.167 + if (cl == NULL) return;
1.168 + for (size_t i = index; i < sz; i += oopSize) {
1.169 + oop obj = (oop)buf[byte_index_to_index((int)i)];
1.170 + // There can be NULL entries because of destructors.
1.171 + if (obj != NULL) {
1.172 + cl->do_object(obj);
1.173 + }
1.174 + }
1.175 +}
1.176 +
1.177 +#ifndef PRODUCT
1.178 +// Helpful for debugging
1.179 +
1.180 +void ObjPtrQueue::print(const char* name) {
1.181 + print(name, _buf, _index, _sz);
1.182 +}
1.183 +
1.184 +void ObjPtrQueue::print(const char* name,
1.185 + void** buf, size_t index, size_t sz) {
1.186 + gclog_or_tty->print_cr(" SATB BUFFER [%s] buf: "PTR_FORMAT" "
1.187 + "index: "SIZE_FORMAT" sz: "SIZE_FORMAT,
1.188 + name, buf, index, sz);
1.189 +}
1.190 +#endif // PRODUCT
1.191 +
1.192 +#ifdef ASSERT
1.193 +void ObjPtrQueue::verify_oops_in_buffer() {
1.194 + if (_buf == NULL) return;
1.195 + for (size_t i = _index; i < _sz; i += oopSize) {
1.196 + oop obj = (oop)_buf[byte_index_to_index((int)i)];
1.197 + assert(obj != NULL && obj->is_oop(true /* ignore mark word */),
1.198 + "Not an oop");
1.199 + }
1.200 +}
1.201 +#endif
1.202 +
1.203 +#ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away
1.204 +#pragma warning( disable:4355 ) // 'this' : used in base member initializer list
1.205 +#endif // _MSC_VER
1.206 +
1.207 +SATBMarkQueueSet::SATBMarkQueueSet() :
1.208 + PtrQueueSet(), _closure(NULL), _par_closures(NULL),
1.209 + _shared_satb_queue(this, true /*perm*/) { }
1.210 +
1.211 +void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock,
1.212 + int process_completed_threshold,
1.213 + Mutex* lock) {
1.214 + PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, -1);
1.215 + _shared_satb_queue.set_lock(lock);
1.216 + if (ParallelGCThreads > 0) {
1.217 + _par_closures = NEW_C_HEAP_ARRAY(ObjectClosure*, ParallelGCThreads, mtGC);
1.218 + }
1.219 +}
1.220 +
1.221 +void SATBMarkQueueSet::handle_zero_index_for_thread(JavaThread* t) {
1.222 + DEBUG_ONLY(t->satb_mark_queue().verify_oops_in_buffer();)
1.223 + t->satb_mark_queue().handle_zero_index();
1.224 +}
1.225 +
1.226 +#ifdef ASSERT
1.227 +void SATBMarkQueueSet::dump_active_states(bool expected_active) {
1.228 + gclog_or_tty->print_cr("Expected SATB active state: %s",
1.229 + expected_active ? "ACTIVE" : "INACTIVE");
1.230 + gclog_or_tty->print_cr("Actual SATB active states:");
1.231 + gclog_or_tty->print_cr(" Queue set: %s", is_active() ? "ACTIVE" : "INACTIVE");
1.232 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.233 + gclog_or_tty->print_cr(" Thread \"%s\" queue: %s", t->name(),
1.234 + t->satb_mark_queue().is_active() ? "ACTIVE" : "INACTIVE");
1.235 + }
1.236 + gclog_or_tty->print_cr(" Shared queue: %s",
1.237 + shared_satb_queue()->is_active() ? "ACTIVE" : "INACTIVE");
1.238 +}
1.239 +
1.240 +void SATBMarkQueueSet::verify_active_states(bool expected_active) {
1.241 + // Verify queue set state
1.242 + if (is_active() != expected_active) {
1.243 + dump_active_states(expected_active);
1.244 + guarantee(false, "SATB queue set has an unexpected active state");
1.245 + }
1.246 +
1.247 + // Verify thread queue states
1.248 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.249 + if (t->satb_mark_queue().is_active() != expected_active) {
1.250 + dump_active_states(expected_active);
1.251 + guarantee(false, "Thread SATB queue has an unexpected active state");
1.252 + }
1.253 + }
1.254 +
1.255 + // Verify shared queue state
1.256 + if (shared_satb_queue()->is_active() != expected_active) {
1.257 + dump_active_states(expected_active);
1.258 + guarantee(false, "Shared SATB queue has an unexpected active state");
1.259 + }
1.260 +}
1.261 +#endif // ASSERT
1.262 +
1.263 +void SATBMarkQueueSet::set_active_all_threads(bool active, bool expected_active) {
1.264 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
1.265 +#ifdef ASSERT
1.266 + verify_active_states(expected_active);
1.267 +#endif // ASSERT
1.268 + _all_active = active;
1.269 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.270 + t->satb_mark_queue().set_active(active);
1.271 + }
1.272 + shared_satb_queue()->set_active(active);
1.273 +}
1.274 +
1.275 +void SATBMarkQueueSet::filter_thread_buffers() {
1.276 + for(JavaThread* t = Threads::first(); t; t = t->next()) {
1.277 + t->satb_mark_queue().filter();
1.278 + }
1.279 + shared_satb_queue()->filter();
1.280 +}
1.281 +
1.282 +void SATBMarkQueueSet::set_closure(ObjectClosure* closure) {
1.283 + _closure = closure;
1.284 +}
1.285 +
1.286 +void SATBMarkQueueSet::set_par_closure(int i, ObjectClosure* par_closure) {
1.287 + assert(ParallelGCThreads > 0 && _par_closures != NULL, "Precondition");
1.288 + _par_closures[i] = par_closure;
1.289 +}
1.290 +
1.291 +void SATBMarkQueueSet::iterate_closure_all_threads() {
1.292 + for(JavaThread* t = Threads::first(); t; t = t->next()) {
1.293 + t->satb_mark_queue().apply_closure_and_empty(_closure);
1.294 + }
1.295 + shared_satb_queue()->apply_closure_and_empty(_closure);
1.296 +}
1.297 +
1.298 +void SATBMarkQueueSet::par_iterate_closure_all_threads(uint worker) {
1.299 + SharedHeap* sh = SharedHeap::heap();
1.300 + int parity = sh->strong_roots_parity();
1.301 +
1.302 + for(JavaThread* t = Threads::first(); t; t = t->next()) {
1.303 + if (t->claim_oops_do(true, parity)) {
1.304 + t->satb_mark_queue().apply_closure_and_empty(_par_closures[worker]);
1.305 + }
1.306 + }
1.307 +
1.308 + // We also need to claim the VMThread so that its parity is updated
1.309 + // otherwise the next call to Thread::possibly_parallel_oops_do inside
1.310 + // a StrongRootsScope might skip the VMThread because it has a stale
1.311 + // parity that matches the parity set by the StrongRootsScope
1.312 + //
1.313 + // Whichever worker succeeds in claiming the VMThread gets to do
1.314 + // the shared queue.
1.315 +
1.316 + VMThread* vmt = VMThread::vm_thread();
1.317 + if (vmt->claim_oops_do(true, parity)) {
1.318 + shared_satb_queue()->apply_closure_and_empty(_par_closures[worker]);
1.319 + }
1.320 +}
1.321 +
1.322 +bool SATBMarkQueueSet::apply_closure_to_completed_buffer_work(bool par,
1.323 + uint worker) {
1.324 + BufferNode* nd = NULL;
1.325 + {
1.326 + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
1.327 + if (_completed_buffers_head != NULL) {
1.328 + nd = _completed_buffers_head;
1.329 + _completed_buffers_head = nd->next();
1.330 + if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL;
1.331 + _n_completed_buffers--;
1.332 + if (_n_completed_buffers == 0) _process_completed = false;
1.333 + }
1.334 + }
1.335 + ObjectClosure* cl = (par ? _par_closures[worker] : _closure);
1.336 + if (nd != NULL) {
1.337 + void **buf = BufferNode::make_buffer_from_node(nd);
1.338 + ObjPtrQueue::apply_closure_to_buffer(cl, buf, 0, _sz);
1.339 + deallocate_buffer(buf);
1.340 + return true;
1.341 + } else {
1.342 + return false;
1.343 + }
1.344 +}
1.345 +
1.346 +void SATBMarkQueueSet::iterate_completed_buffers_read_only(ObjectClosure* cl) {
1.347 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
1.348 + assert(cl != NULL, "pre-condition");
1.349 +
1.350 + BufferNode* nd = _completed_buffers_head;
1.351 + while (nd != NULL) {
1.352 + void** buf = BufferNode::make_buffer_from_node(nd);
1.353 + ObjPtrQueue::apply_closure_to_buffer(cl, buf, 0, _sz);
1.354 + nd = nd->next();
1.355 + }
1.356 +}
1.357 +
1.358 +void SATBMarkQueueSet::iterate_thread_buffers_read_only(ObjectClosure* cl) {
1.359 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
1.360 + assert(cl != NULL, "pre-condition");
1.361 +
1.362 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.363 + t->satb_mark_queue().apply_closure(cl);
1.364 + }
1.365 + shared_satb_queue()->apply_closure(cl);
1.366 +}
1.367 +
1.368 +#ifndef PRODUCT
1.369 +// Helpful for debugging
1.370 +
1.371 +#define SATB_PRINTER_BUFFER_SIZE 256
1.372 +
1.373 +void SATBMarkQueueSet::print_all(const char* msg) {
1.374 + char buffer[SATB_PRINTER_BUFFER_SIZE];
1.375 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
1.376 +
1.377 + gclog_or_tty->cr();
1.378 + gclog_or_tty->print_cr("SATB BUFFERS [%s]", msg);
1.379 +
1.380 + BufferNode* nd = _completed_buffers_head;
1.381 + int i = 0;
1.382 + while (nd != NULL) {
1.383 + void** buf = BufferNode::make_buffer_from_node(nd);
1.384 + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Enqueued: %d", i);
1.385 + ObjPtrQueue::print(buffer, buf, 0, _sz);
1.386 + nd = nd->next();
1.387 + i += 1;
1.388 + }
1.389 +
1.390 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.391 + jio_snprintf(buffer, SATB_PRINTER_BUFFER_SIZE, "Thread: %s", t->name());
1.392 + t->satb_mark_queue().print(buffer);
1.393 + }
1.394 +
1.395 + shared_satb_queue()->print("Shared");
1.396 +
1.397 + gclog_or_tty->cr();
1.398 +}
1.399 +#endif // PRODUCT
1.400 +
1.401 +void SATBMarkQueueSet::abandon_partial_marking() {
1.402 + BufferNode* buffers_to_delete = NULL;
1.403 + {
1.404 + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);
1.405 + while (_completed_buffers_head != NULL) {
1.406 + BufferNode* nd = _completed_buffers_head;
1.407 + _completed_buffers_head = nd->next();
1.408 + nd->set_next(buffers_to_delete);
1.409 + buffers_to_delete = nd;
1.410 + }
1.411 + _completed_buffers_tail = NULL;
1.412 + _n_completed_buffers = 0;
1.413 + DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked());
1.414 + }
1.415 + while (buffers_to_delete != NULL) {
1.416 + BufferNode* nd = buffers_to_delete;
1.417 + buffers_to_delete = nd->next();
1.418 + deallocate_buffer(BufferNode::make_buffer_from_node(nd));
1.419 + }
1.420 + assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");
1.421 + // So we can safely manipulate these queues.
1.422 + for (JavaThread* t = Threads::first(); t; t = t->next()) {
1.423 + t->satb_mark_queue().reset();
1.424 + }
1.425 + shared_satb_queue()->reset();
1.426 +}
