src/share/vm/utilities/taskqueue.hpp

Mon, 09 Aug 2010 17:51:56 -0700

author
never
date
Mon, 09 Aug 2010 17:51:56 -0700
changeset 2044
f4f596978298
parent 2020
a93a9eda13f7
child 2064
5f429ee79634
permissions
-rw-r--r--

Merge

duke@435 1 /*
jcoomes@1993 2 * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
duke@435 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435 4 *
duke@435 5 * This code is free software; you can redistribute it and/or modify it
duke@435 6 * under the terms of the GNU General Public License version 2 only, as
duke@435 7 * published by the Free Software Foundation.
duke@435 8 *
duke@435 9 * This code is distributed in the hope that it will be useful, but WITHOUT
duke@435 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435 12 * version 2 for more details (a copy is included in the LICENSE file that
duke@435 13 * accompanied this code).
duke@435 14 *
duke@435 15 * You should have received a copy of the GNU General Public License version
duke@435 16 * 2 along with this work; if not, write to the Free Software Foundation,
duke@435 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435 18 *
trims@1907 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@1907 20 * or visit www.oracle.com if you need additional information or have any
trims@1907 21 * questions.
duke@435 22 *
duke@435 23 */
duke@435 24
jcoomes@2020 25 // Simple TaskQueue stats that are collected by default in debug builds.
jcoomes@2020 26
jcoomes@2020 27 #if !defined(TASKQUEUE_STATS) && defined(ASSERT)
jcoomes@2020 28 #define TASKQUEUE_STATS 1
jcoomes@2020 29 #elif !defined(TASKQUEUE_STATS)
jcoomes@2020 30 #define TASKQUEUE_STATS 0
jcoomes@2020 31 #endif
jcoomes@2020 32
jcoomes@2020 33 #if TASKQUEUE_STATS
jcoomes@2020 34 #define TASKQUEUE_STATS_ONLY(code) code
jcoomes@2020 35 #else
jcoomes@2020 36 #define TASKQUEUE_STATS_ONLY(code)
jcoomes@2020 37 #endif // TASKQUEUE_STATS
jcoomes@2020 38
jcoomes@2020 39 #if TASKQUEUE_STATS
jcoomes@2020 40 class TaskQueueStats {
jcoomes@2020 41 public:
jcoomes@2020 42 enum StatId {
jcoomes@2020 43 push, // number of taskqueue pushes
jcoomes@2020 44 pop, // number of taskqueue pops
jcoomes@2020 45 pop_slow, // subset of taskqueue pops that were done slow-path
jcoomes@2020 46 steal_attempt, // number of taskqueue steal attempts
jcoomes@2020 47 steal, // number of taskqueue steals
jcoomes@2020 48 overflow, // number of overflow pushes
jcoomes@2020 49 overflow_max_len, // max length of overflow stack
jcoomes@2020 50 last_stat_id
jcoomes@2020 51 };
jcoomes@2020 52
jcoomes@2020 53 public:
jcoomes@2020 54 inline TaskQueueStats() { reset(); }
jcoomes@2020 55
jcoomes@2020 56 inline void record_push() { ++_stats[push]; }
jcoomes@2020 57 inline void record_pop() { ++_stats[pop]; }
jcoomes@2020 58 inline void record_pop_slow() { record_pop(); ++_stats[pop_slow]; }
jcoomes@2020 59 inline void record_steal(bool success);
jcoomes@2020 60 inline void record_overflow(size_t new_length);
jcoomes@2020 61
jcoomes@2020 62 inline size_t get(StatId id) const { return _stats[id]; }
jcoomes@2020 63 inline const size_t* get() const { return _stats; }
jcoomes@2020 64
jcoomes@2020 65 inline void reset();
jcoomes@2020 66
jcoomes@2020 67 static void print_header(unsigned int line, outputStream* const stream = tty,
jcoomes@2020 68 unsigned int width = 10);
jcoomes@2020 69 void print(outputStream* const stream = tty, unsigned int width = 10) const;
jcoomes@2020 70
jcoomes@2020 71 private:
jcoomes@2020 72 size_t _stats[last_stat_id];
jcoomes@2020 73 static const char * const _names[last_stat_id];
jcoomes@2020 74 };
jcoomes@2020 75
jcoomes@2020 76 void TaskQueueStats::record_steal(bool success) {
jcoomes@2020 77 ++_stats[steal_attempt];
jcoomes@2020 78 if (success) ++_stats[steal];
jcoomes@2020 79 }
jcoomes@2020 80
jcoomes@2020 81 void TaskQueueStats::record_overflow(size_t new_len) {
jcoomes@2020 82 ++_stats[overflow];
jcoomes@2020 83 if (new_len > _stats[overflow_max_len]) _stats[overflow_max_len] = new_len;
jcoomes@2020 84 }
jcoomes@2020 85
jcoomes@2020 86 void TaskQueueStats::reset() {
jcoomes@2020 87 memset(_stats, 0, sizeof(_stats));
jcoomes@2020 88 }
jcoomes@2020 89 #endif // TASKQUEUE_STATS
jcoomes@2020 90
jcoomes@1746 91 template <unsigned int N>
duke@435 92 class TaskQueueSuper: public CHeapObj {
duke@435 93 protected:
jcoomes@1342 94 // Internal type for indexing the queue; also used for the tag.
jcoomes@1342 95 typedef NOT_LP64(uint16_t) LP64_ONLY(uint32_t) idx_t;
jcoomes@1342 96
jcoomes@1342 97 // The first free element after the last one pushed (mod N).
ysr@976 98 volatile uint _bottom;
duke@435 99
jcoomes@1746 100 enum { MOD_N_MASK = N - 1 };
duke@435 101
jcoomes@1342 102 class Age {
jcoomes@1342 103 public:
jcoomes@1342 104 Age(size_t data = 0) { _data = data; }
jcoomes@1342 105 Age(const Age& age) { _data = age._data; }
jcoomes@1342 106 Age(idx_t top, idx_t tag) { _fields._top = top; _fields._tag = tag; }
duke@435 107
jcoomes@1342 108 Age get() const volatile { return _data; }
jcoomes@1342 109 void set(Age age) volatile { _data = age._data; }
duke@435 110
jcoomes@1342 111 idx_t top() const volatile { return _fields._top; }
jcoomes@1342 112 idx_t tag() const volatile { return _fields._tag; }
duke@435 113
jcoomes@1342 114 // Increment top; if it wraps, increment tag also.
jcoomes@1342 115 void increment() {
jcoomes@1342 116 _fields._top = increment_index(_fields._top);
jcoomes@1342 117 if (_fields._top == 0) ++_fields._tag;
jcoomes@1342 118 }
duke@435 119
jcoomes@1342 120 Age cmpxchg(const Age new_age, const Age old_age) volatile {
jcoomes@1342 121 return (size_t) Atomic::cmpxchg_ptr((intptr_t)new_age._data,
jcoomes@1342 122 (volatile intptr_t *)&_data,
jcoomes@1342 123 (intptr_t)old_age._data);
duke@435 124 }
jcoomes@1342 125
jcoomes@1342 126 bool operator ==(const Age& other) const { return _data == other._data; }
jcoomes@1342 127
jcoomes@1342 128 private:
jcoomes@1342 129 struct fields {
jcoomes@1342 130 idx_t _top;
jcoomes@1342 131 idx_t _tag;
jcoomes@1342 132 };
jcoomes@1342 133 union {
jcoomes@1342 134 size_t _data;
jcoomes@1342 135 fields _fields;
jcoomes@1342 136 };
duke@435 137 };
jcoomes@1342 138
jcoomes@1342 139 volatile Age _age;
jcoomes@1342 140
jcoomes@1342 141 // These both operate mod N.
jcoomes@1342 142 static uint increment_index(uint ind) {
jcoomes@1342 143 return (ind + 1) & MOD_N_MASK;
duke@435 144 }
jcoomes@1342 145 static uint decrement_index(uint ind) {
jcoomes@1342 146 return (ind - 1) & MOD_N_MASK;
duke@435 147 }
duke@435 148
jcoomes@1342 149 // Returns a number in the range [0..N). If the result is "N-1", it should be
jcoomes@1342 150 // interpreted as 0.
jcoomes@1746 151 uint dirty_size(uint bot, uint top) const {
jcoomes@1342 152 return (bot - top) & MOD_N_MASK;
duke@435 153 }
duke@435 154
duke@435 155 // Returns the size corresponding to the given "bot" and "top".
jcoomes@1746 156 uint size(uint bot, uint top) const {
ysr@976 157 uint sz = dirty_size(bot, top);
jcoomes@1342 158 // Has the queue "wrapped", so that bottom is less than top? There's a
jcoomes@1342 159 // complicated special case here. A pair of threads could perform pop_local
jcoomes@1342 160 // and pop_global operations concurrently, starting from a state in which
jcoomes@1342 161 // _bottom == _top+1. The pop_local could succeed in decrementing _bottom,
jcoomes@1342 162 // and the pop_global in incrementing _top (in which case the pop_global
jcoomes@1342 163 // will be awarded the contested queue element.) The resulting state must
jcoomes@1342 164 // be interpreted as an empty queue. (We only need to worry about one such
jcoomes@1342 165 // event: only the queue owner performs pop_local's, and several concurrent
jcoomes@1342 166 // threads attempting to perform the pop_global will all perform the same
jcoomes@1342 167 // CAS, and only one can succeed.) Any stealing thread that reads after
jcoomes@1342 168 // either the increment or decrement will see an empty queue, and will not
jcoomes@1342 169 // join the competitors. The "sz == -1 || sz == N-1" state will not be
jcoomes@1342 170 // modified by concurrent queues, so the owner thread can reset the state to
jcoomes@1342 171 // _bottom == top so subsequent pushes will be performed normally.
jcoomes@1342 172 return (sz == N - 1) ? 0 : sz;
duke@435 173 }
duke@435 174
duke@435 175 public:
duke@435 176 TaskQueueSuper() : _bottom(0), _age() {}
duke@435 177
jcoomes@1993 178 // Return true if the TaskQueue contains/does not contain any tasks.
jcoomes@1993 179 bool peek() const { return _bottom != _age.top(); }
jcoomes@1993 180 bool is_empty() const { return size() == 0; }
duke@435 181
duke@435 182 // Return an estimate of the number of elements in the queue.
duke@435 183 // The "careful" version admits the possibility of pop_local/pop_global
duke@435 184 // races.
jcoomes@1746 185 uint size() const {
jcoomes@1342 186 return size(_bottom, _age.top());
duke@435 187 }
duke@435 188
jcoomes@1746 189 uint dirty_size() const {
jcoomes@1342 190 return dirty_size(_bottom, _age.top());
duke@435 191 }
duke@435 192
ysr@777 193 void set_empty() {
ysr@777 194 _bottom = 0;
jcoomes@1342 195 _age.set(0);
ysr@777 196 }
ysr@777 197
duke@435 198 // Maximum number of elements allowed in the queue. This is two less
duke@435 199 // than the actual queue size, for somewhat complicated reasons.
jcoomes@1746 200 uint max_elems() const { return N - 2; }
jmasa@1719 201
jmasa@1719 202 // Total size of queue.
jmasa@1719 203 static const uint total_size() { return N; }
jcoomes@2020 204
jcoomes@2020 205 TASKQUEUE_STATS_ONLY(TaskQueueStats stats;)
duke@435 206 };
duke@435 207
jcoomes@1746 208 template<class E, unsigned int N = TASKQUEUE_SIZE>
jcoomes@1746 209 class GenericTaskQueue: public TaskQueueSuper<N> {
jcoomes@1746 210 protected:
jcoomes@1746 211 typedef typename TaskQueueSuper<N>::Age Age;
jcoomes@1746 212 typedef typename TaskQueueSuper<N>::idx_t idx_t;
jcoomes@1746 213
jcoomes@1746 214 using TaskQueueSuper<N>::_bottom;
jcoomes@1746 215 using TaskQueueSuper<N>::_age;
jcoomes@1746 216 using TaskQueueSuper<N>::increment_index;
jcoomes@1746 217 using TaskQueueSuper<N>::decrement_index;
jcoomes@1746 218 using TaskQueueSuper<N>::dirty_size;
jcoomes@1746 219
jcoomes@1746 220 public:
jcoomes@1746 221 using TaskQueueSuper<N>::max_elems;
jcoomes@1746 222 using TaskQueueSuper<N>::size;
jcoomes@2020 223 TASKQUEUE_STATS_ONLY(using TaskQueueSuper<N>::stats;)
jcoomes@1746 224
duke@435 225 private:
duke@435 226 // Slow paths for push, pop_local. (pop_global has no fast path.)
ysr@976 227 bool push_slow(E t, uint dirty_n_elems);
ysr@976 228 bool pop_local_slow(uint localBot, Age oldAge);
duke@435 229
duke@435 230 public:
jcoomes@1746 231 typedef E element_type;
jcoomes@1746 232
duke@435 233 // Initializes the queue to empty.
duke@435 234 GenericTaskQueue();
duke@435 235
duke@435 236 void initialize();
duke@435 237
jcoomes@1993 238 // Push the task "t" on the queue. Returns "false" iff the queue is full.
duke@435 239 inline bool push(E t);
duke@435 240
jcoomes@1993 241 // Attempts to claim a task from the "local" end of the queue (the most
jcoomes@1993 242 // recently pushed). If successful, returns true and sets t to the task;
jcoomes@1993 243 // otherwise, returns false (the queue is empty).
duke@435 244 inline bool pop_local(E& t);
duke@435 245
jcoomes@1993 246 // Like pop_local(), but uses the "global" end of the queue (the least
jcoomes@1993 247 // recently pushed).
duke@435 248 bool pop_global(E& t);
duke@435 249
duke@435 250 // Delete any resource associated with the queue.
duke@435 251 ~GenericTaskQueue();
duke@435 252
ysr@777 253 // apply the closure to all elements in the task queue
ysr@777 254 void oops_do(OopClosure* f);
ysr@777 255
duke@435 256 private:
duke@435 257 // Element array.
duke@435 258 volatile E* _elems;
duke@435 259 };
duke@435 260
jcoomes@1746 261 template<class E, unsigned int N>
jcoomes@1746 262 GenericTaskQueue<E, N>::GenericTaskQueue() {
jcoomes@1342 263 assert(sizeof(Age) == sizeof(size_t), "Depends on this.");
duke@435 264 }
duke@435 265
jcoomes@1746 266 template<class E, unsigned int N>
jcoomes@1746 267 void GenericTaskQueue<E, N>::initialize() {
jcoomes@1342 268 _elems = NEW_C_HEAP_ARRAY(E, N);
duke@435 269 }
duke@435 270
jcoomes@1746 271 template<class E, unsigned int N>
jcoomes@1746 272 void GenericTaskQueue<E, N>::oops_do(OopClosure* f) {
ysr@777 273 // tty->print_cr("START OopTaskQueue::oops_do");
ysr@976 274 uint iters = size();
ysr@976 275 uint index = _bottom;
ysr@976 276 for (uint i = 0; i < iters; ++i) {
ysr@777 277 index = decrement_index(index);
ysr@777 278 // tty->print_cr(" doing entry %d," INTPTR_T " -> " INTPTR_T,
ysr@777 279 // index, &_elems[index], _elems[index]);
ysr@777 280 E* t = (E*)&_elems[index]; // cast away volatility
ysr@777 281 oop* p = (oop*)t;
ysr@777 282 assert((*t)->is_oop_or_null(), "Not an oop or null");
ysr@777 283 f->do_oop(p);
ysr@777 284 }
ysr@777 285 // tty->print_cr("END OopTaskQueue::oops_do");
ysr@777 286 }
ysr@777 287
jcoomes@1746 288 template<class E, unsigned int N>
jcoomes@1746 289 bool GenericTaskQueue<E, N>::push_slow(E t, uint dirty_n_elems) {
jcoomes@1342 290 if (dirty_n_elems == N - 1) {
duke@435 291 // Actually means 0, so do the push.
ysr@976 292 uint localBot = _bottom;
jcoomes@1746 293 // g++ complains if the volatile result of the assignment is unused.
jcoomes@1746 294 const_cast<E&>(_elems[localBot] = t);
bobv@1459 295 OrderAccess::release_store(&_bottom, increment_index(localBot));
jcoomes@2020 296 TASKQUEUE_STATS_ONLY(stats.record_push());
duke@435 297 return true;
jcoomes@1342 298 }
jcoomes@1342 299 return false;
duke@435 300 }
duke@435 301
jcoomes@1746 302 template<class E, unsigned int N>
jcoomes@2020 303 bool GenericTaskQueue<E, N>::pop_local_slow(uint localBot, Age oldAge) {
duke@435 304 // This queue was observed to contain exactly one element; either this
duke@435 305 // thread will claim it, or a competing "pop_global". In either case,
duke@435 306 // the queue will be logically empty afterwards. Create a new Age value
duke@435 307 // that represents the empty queue for the given value of "_bottom". (We
duke@435 308 // must also increment "tag" because of the case where "bottom == 1",
duke@435 309 // "top == 0". A pop_global could read the queue element in that case,
duke@435 310 // then have the owner thread do a pop followed by another push. Without
duke@435 311 // the incrementing of "tag", the pop_global's CAS could succeed,
duke@435 312 // allowing it to believe it has claimed the stale element.)
jcoomes@1342 313 Age newAge((idx_t)localBot, oldAge.tag() + 1);
duke@435 314 // Perhaps a competing pop_global has already incremented "top", in which
duke@435 315 // case it wins the element.
duke@435 316 if (localBot == oldAge.top()) {
duke@435 317 // No competing pop_global has yet incremented "top"; we'll try to
duke@435 318 // install new_age, thus claiming the element.
jcoomes@1342 319 Age tempAge = _age.cmpxchg(newAge, oldAge);
duke@435 320 if (tempAge == oldAge) {
duke@435 321 // We win.
jcoomes@1342 322 assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
jcoomes@2020 323 TASKQUEUE_STATS_ONLY(stats.record_pop_slow());
duke@435 324 return true;
duke@435 325 }
duke@435 326 }
jcoomes@1342 327 // We lose; a completing pop_global gets the element. But the queue is empty
jcoomes@1342 328 // and top is greater than bottom. Fix this representation of the empty queue
jcoomes@1342 329 // to become the canonical one.
jcoomes@1342 330 _age.set(newAge);
jcoomes@1342 331 assert(dirty_size(localBot, _age.top()) != N - 1, "sanity");
duke@435 332 return false;
duke@435 333 }
duke@435 334
jcoomes@1746 335 template<class E, unsigned int N>
jcoomes@1746 336 bool GenericTaskQueue<E, N>::pop_global(E& t) {
jcoomes@1342 337 Age oldAge = _age.get();
ysr@976 338 uint localBot = _bottom;
ysr@976 339 uint n_elems = size(localBot, oldAge.top());
duke@435 340 if (n_elems == 0) {
duke@435 341 return false;
duke@435 342 }
jcoomes@1342 343
jcoomes@1746 344 const_cast<E&>(t = _elems[oldAge.top()]);
jcoomes@1342 345 Age newAge(oldAge);
jcoomes@1342 346 newAge.increment();
jcoomes@1342 347 Age resAge = _age.cmpxchg(newAge, oldAge);
jcoomes@1342 348
duke@435 349 // Note that using "_bottom" here might fail, since a pop_local might
duke@435 350 // have decremented it.
jcoomes@1342 351 assert(dirty_size(localBot, newAge.top()) != N - 1, "sanity");
jcoomes@1342 352 return resAge == oldAge;
duke@435 353 }
duke@435 354
jcoomes@1746 355 template<class E, unsigned int N>
jcoomes@1746 356 GenericTaskQueue<E, N>::~GenericTaskQueue() {
duke@435 357 FREE_C_HEAP_ARRAY(E, _elems);
duke@435 358 }
duke@435 359
jcoomes@1993 360 // OverflowTaskQueue is a TaskQueue that also includes an overflow stack for
jcoomes@1993 361 // elements that do not fit in the TaskQueue.
jcoomes@1993 362 //
jcoomes@1993 363 // Three methods from super classes are overridden:
jcoomes@1993 364 //
jcoomes@1993 365 // initialize() - initialize the super classes and create the overflow stack
jcoomes@1993 366 // push() - push onto the task queue or, if that fails, onto the overflow stack
jcoomes@1993 367 // is_empty() - return true if both the TaskQueue and overflow stack are empty
jcoomes@1993 368 //
jcoomes@1993 369 // Note that size() is not overridden--it returns the number of elements in the
jcoomes@1993 370 // TaskQueue, and does not include the size of the overflow stack. This
jcoomes@1993 371 // simplifies replacement of GenericTaskQueues with OverflowTaskQueues.
jcoomes@1993 372 template<class E, unsigned int N = TASKQUEUE_SIZE>
jcoomes@1993 373 class OverflowTaskQueue: public GenericTaskQueue<E, N>
jcoomes@1993 374 {
jcoomes@1993 375 public:
jcoomes@1993 376 typedef GrowableArray<E> overflow_t;
jcoomes@1993 377 typedef GenericTaskQueue<E, N> taskqueue_t;
jcoomes@1993 378
jcoomes@2020 379 TASKQUEUE_STATS_ONLY(using taskqueue_t::stats;)
jcoomes@2020 380
jcoomes@1993 381 OverflowTaskQueue();
jcoomes@1993 382 ~OverflowTaskQueue();
jcoomes@1993 383 void initialize();
jcoomes@1993 384
jcoomes@1993 385 inline overflow_t* overflow_stack() const { return _overflow_stack; }
jcoomes@1993 386
jcoomes@1993 387 // Push task t onto the queue or onto the overflow stack. Return true.
jcoomes@1993 388 inline bool push(E t);
jcoomes@1993 389
jcoomes@1993 390 // Attempt to pop from the overflow stack; return true if anything was popped.
jcoomes@1993 391 inline bool pop_overflow(E& t);
jcoomes@1993 392
jcoomes@1993 393 inline bool taskqueue_empty() const { return taskqueue_t::is_empty(); }
jcoomes@1993 394 inline bool overflow_empty() const { return overflow_stack()->is_empty(); }
jcoomes@1993 395 inline bool is_empty() const {
jcoomes@1993 396 return taskqueue_empty() && overflow_empty();
jcoomes@1993 397 }
jcoomes@1993 398
jcoomes@1993 399 private:
jcoomes@1993 400 overflow_t* _overflow_stack;
jcoomes@1993 401 };
jcoomes@1993 402
jcoomes@1993 403 template <class E, unsigned int N>
jcoomes@1993 404 OverflowTaskQueue<E, N>::OverflowTaskQueue()
jcoomes@1993 405 {
jcoomes@1993 406 _overflow_stack = NULL;
jcoomes@1993 407 }
jcoomes@1993 408
jcoomes@1993 409 template <class E, unsigned int N>
jcoomes@1993 410 OverflowTaskQueue<E, N>::~OverflowTaskQueue()
jcoomes@1993 411 {
jcoomes@1993 412 if (_overflow_stack != NULL) {
jcoomes@1993 413 delete _overflow_stack;
jcoomes@1993 414 _overflow_stack = NULL;
jcoomes@1993 415 }
jcoomes@1993 416 }
jcoomes@1993 417
jcoomes@1993 418 template <class E, unsigned int N>
jcoomes@1993 419 void OverflowTaskQueue<E, N>::initialize()
jcoomes@1993 420 {
jcoomes@1993 421 taskqueue_t::initialize();
jcoomes@1993 422 assert(_overflow_stack == NULL, "memory leak");
jcoomes@1993 423 _overflow_stack = new (ResourceObj::C_HEAP) GrowableArray<E>(10, true);
jcoomes@1993 424 }
jcoomes@1993 425
jcoomes@1993 426 template <class E, unsigned int N>
jcoomes@1993 427 bool OverflowTaskQueue<E, N>::push(E t)
jcoomes@1993 428 {
jcoomes@1993 429 if (!taskqueue_t::push(t)) {
jcoomes@1993 430 overflow_stack()->push(t);
jcoomes@2020 431 TASKQUEUE_STATS_ONLY(stats.record_overflow(overflow_stack()->length()));
jcoomes@1993 432 }
jcoomes@1993 433 return true;
jcoomes@1993 434 }
jcoomes@1993 435
jcoomes@1993 436 template <class E, unsigned int N>
jcoomes@1993 437 bool OverflowTaskQueue<E, N>::pop_overflow(E& t)
jcoomes@1993 438 {
jcoomes@1993 439 if (overflow_empty()) return false;
jcoomes@1993 440 t = overflow_stack()->pop();
jcoomes@1993 441 return true;
jcoomes@1993 442 }
jcoomes@1993 443
duke@435 444 class TaskQueueSetSuper: public CHeapObj {
duke@435 445 protected:
duke@435 446 static int randomParkAndMiller(int* seed0);
duke@435 447 public:
duke@435 448 // Returns "true" if some TaskQueue in the set contains a task.
duke@435 449 virtual bool peek() = 0;
duke@435 450 };
duke@435 451
jcoomes@1746 452 template<class T>
jcoomes@1746 453 class GenericTaskQueueSet: public TaskQueueSetSuper {
duke@435 454 private:
ysr@976 455 uint _n;
jcoomes@1746 456 T** _queues;
duke@435 457
duke@435 458 public:
jcoomes@1746 459 typedef typename T::element_type E;
jcoomes@1746 460
duke@435 461 GenericTaskQueueSet(int n) : _n(n) {
jcoomes@1746 462 typedef T* GenericTaskQueuePtr;
duke@435 463 _queues = NEW_C_HEAP_ARRAY(GenericTaskQueuePtr, n);
duke@435 464 for (int i = 0; i < n; i++) {
duke@435 465 _queues[i] = NULL;
duke@435 466 }
duke@435 467 }
duke@435 468
ysr@976 469 bool steal_1_random(uint queue_num, int* seed, E& t);
ysr@976 470 bool steal_best_of_2(uint queue_num, int* seed, E& t);
ysr@976 471 bool steal_best_of_all(uint queue_num, int* seed, E& t);
duke@435 472
jcoomes@1746 473 void register_queue(uint i, T* q);
duke@435 474
jcoomes@1746 475 T* queue(uint n);
duke@435 476
jcoomes@1993 477 // The thread with queue number "queue_num" (and whose random number seed is
jcoomes@1993 478 // at "seed") is trying to steal a task from some other queue. (It may try
jcoomes@1993 479 // several queues, according to some configuration parameter.) If some steal
jcoomes@1993 480 // succeeds, returns "true" and sets "t" to the stolen task, otherwise returns
jcoomes@1993 481 // false.
ysr@976 482 bool steal(uint queue_num, int* seed, E& t);
duke@435 483
duke@435 484 bool peek();
duke@435 485 };
duke@435 486
jcoomes@1746 487 template<class T> void
jcoomes@1746 488 GenericTaskQueueSet<T>::register_queue(uint i, T* q) {
ysr@976 489 assert(i < _n, "index out of range.");
duke@435 490 _queues[i] = q;
duke@435 491 }
duke@435 492
jcoomes@1746 493 template<class T> T*
jcoomes@1746 494 GenericTaskQueueSet<T>::queue(uint i) {
duke@435 495 return _queues[i];
duke@435 496 }
duke@435 497
jcoomes@1746 498 template<class T> bool
jcoomes@1746 499 GenericTaskQueueSet<T>::steal(uint queue_num, int* seed, E& t) {
jcoomes@2020 500 for (uint i = 0; i < 2 * _n; i++) {
jcoomes@2020 501 if (steal_best_of_2(queue_num, seed, t)) {
jcoomes@2020 502 TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(true));
duke@435 503 return true;
jcoomes@2020 504 }
jcoomes@2020 505 }
jcoomes@2020 506 TASKQUEUE_STATS_ONLY(queue(queue_num)->stats.record_steal(false));
duke@435 507 return false;
duke@435 508 }
duke@435 509
jcoomes@1746 510 template<class T> bool
jcoomes@1746 511 GenericTaskQueueSet<T>::steal_best_of_all(uint queue_num, int* seed, E& t) {
duke@435 512 if (_n > 2) {
duke@435 513 int best_k;
ysr@976 514 uint best_sz = 0;
ysr@976 515 for (uint k = 0; k < _n; k++) {
duke@435 516 if (k == queue_num) continue;
ysr@976 517 uint sz = _queues[k]->size();
duke@435 518 if (sz > best_sz) {
duke@435 519 best_sz = sz;
duke@435 520 best_k = k;
duke@435 521 }
duke@435 522 }
duke@435 523 return best_sz > 0 && _queues[best_k]->pop_global(t);
duke@435 524 } else if (_n == 2) {
duke@435 525 // Just try the other one.
duke@435 526 int k = (queue_num + 1) % 2;
duke@435 527 return _queues[k]->pop_global(t);
duke@435 528 } else {
duke@435 529 assert(_n == 1, "can't be zero.");
duke@435 530 return false;
duke@435 531 }
duke@435 532 }
duke@435 533
jcoomes@1746 534 template<class T> bool
jcoomes@1746 535 GenericTaskQueueSet<T>::steal_1_random(uint queue_num, int* seed, E& t) {
duke@435 536 if (_n > 2) {
ysr@976 537 uint k = queue_num;
duke@435 538 while (k == queue_num) k = randomParkAndMiller(seed) % _n;
duke@435 539 return _queues[2]->pop_global(t);
duke@435 540 } else if (_n == 2) {
duke@435 541 // Just try the other one.
duke@435 542 int k = (queue_num + 1) % 2;
duke@435 543 return _queues[k]->pop_global(t);
duke@435 544 } else {
duke@435 545 assert(_n == 1, "can't be zero.");
duke@435 546 return false;
duke@435 547 }
duke@435 548 }
duke@435 549
jcoomes@1746 550 template<class T> bool
jcoomes@1746 551 GenericTaskQueueSet<T>::steal_best_of_2(uint queue_num, int* seed, E& t) {
duke@435 552 if (_n > 2) {
ysr@976 553 uint k1 = queue_num;
duke@435 554 while (k1 == queue_num) k1 = randomParkAndMiller(seed) % _n;
ysr@976 555 uint k2 = queue_num;
duke@435 556 while (k2 == queue_num || k2 == k1) k2 = randomParkAndMiller(seed) % _n;
duke@435 557 // Sample both and try the larger.
ysr@976 558 uint sz1 = _queues[k1]->size();
ysr@976 559 uint sz2 = _queues[k2]->size();
duke@435 560 if (sz2 > sz1) return _queues[k2]->pop_global(t);
duke@435 561 else return _queues[k1]->pop_global(t);
duke@435 562 } else if (_n == 2) {
duke@435 563 // Just try the other one.
ysr@976 564 uint k = (queue_num + 1) % 2;
duke@435 565 return _queues[k]->pop_global(t);
duke@435 566 } else {
duke@435 567 assert(_n == 1, "can't be zero.");
duke@435 568 return false;
duke@435 569 }
duke@435 570 }
duke@435 571
jcoomes@1746 572 template<class T>
jcoomes@1746 573 bool GenericTaskQueueSet<T>::peek() {
duke@435 574 // Try all the queues.
ysr@976 575 for (uint j = 0; j < _n; j++) {
duke@435 576 if (_queues[j]->peek())
duke@435 577 return true;
duke@435 578 }
duke@435 579 return false;
duke@435 580 }
duke@435 581
ysr@777 582 // When to terminate from the termination protocol.
ysr@777 583 class TerminatorTerminator: public CHeapObj {
ysr@777 584 public:
ysr@777 585 virtual bool should_exit_termination() = 0;
ysr@777 586 };
ysr@777 587
duke@435 588 // A class to aid in the termination of a set of parallel tasks using
duke@435 589 // TaskQueueSet's for work stealing.
duke@435 590
jmasa@981 591 #undef TRACESPINNING
jmasa@981 592
duke@435 593 class ParallelTaskTerminator: public StackObj {
duke@435 594 private:
duke@435 595 int _n_threads;
duke@435 596 TaskQueueSetSuper* _queue_set;
ysr@976 597 int _offered_termination;
duke@435 598
jmasa@981 599 #ifdef TRACESPINNING
jmasa@981 600 static uint _total_yields;
jmasa@981 601 static uint _total_spins;
jmasa@981 602 static uint _total_peeks;
jmasa@981 603 #endif
jmasa@981 604
duke@435 605 bool peek_in_queue_set();
duke@435 606 protected:
duke@435 607 virtual void yield();
duke@435 608 void sleep(uint millis);
duke@435 609
duke@435 610 public:
duke@435 611
duke@435 612 // "n_threads" is the number of threads to be terminated. "queue_set" is a
duke@435 613 // queue sets of work queues of other threads.
duke@435 614 ParallelTaskTerminator(int n_threads, TaskQueueSetSuper* queue_set);
duke@435 615
duke@435 616 // The current thread has no work, and is ready to terminate if everyone
duke@435 617 // else is. If returns "true", all threads are terminated. If returns
duke@435 618 // "false", available work has been observed in one of the task queues,
duke@435 619 // so the global task is not complete.
ysr@777 620 bool offer_termination() {
ysr@777 621 return offer_termination(NULL);
ysr@777 622 }
ysr@777 623
jcoomes@1342 624 // As above, but it also terminates if the should_exit_termination()
ysr@777 625 // method of the terminator parameter returns true. If terminator is
ysr@777 626 // NULL, then it is ignored.
ysr@777 627 bool offer_termination(TerminatorTerminator* terminator);
duke@435 628
duke@435 629 // Reset the terminator, so that it may be reused again.
duke@435 630 // The caller is responsible for ensuring that this is done
duke@435 631 // in an MT-safe manner, once the previous round of use of
duke@435 632 // the terminator is finished.
duke@435 633 void reset_for_reuse();
duke@435 634
jmasa@981 635 #ifdef TRACESPINNING
jmasa@981 636 static uint total_yields() { return _total_yields; }
jmasa@981 637 static uint total_spins() { return _total_spins; }
jmasa@981 638 static uint total_peeks() { return _total_peeks; }
jmasa@981 639 static void print_termination_counts();
jmasa@981 640 #endif
duke@435 641 };
duke@435 642
jcoomes@1746 643 template<class E, unsigned int N> inline bool
jcoomes@1746 644 GenericTaskQueue<E, N>::push(E t) {
ysr@976 645 uint localBot = _bottom;
jcoomes@1342 646 assert((localBot >= 0) && (localBot < N), "_bottom out of range.");
jcoomes@1342 647 idx_t top = _age.top();
ysr@976 648 uint dirty_n_elems = dirty_size(localBot, top);
jcoomes@1746 649 assert(dirty_n_elems < N, "n_elems out of range.");
duke@435 650 if (dirty_n_elems < max_elems()) {
jcoomes@1746 651 // g++ complains if the volatile result of the assignment is unused.
jcoomes@1746 652 const_cast<E&>(_elems[localBot] = t);
bobv@1459 653 OrderAccess::release_store(&_bottom, increment_index(localBot));
jcoomes@2020 654 TASKQUEUE_STATS_ONLY(stats.record_push());
duke@435 655 return true;
duke@435 656 } else {
duke@435 657 return push_slow(t, dirty_n_elems);
duke@435 658 }
duke@435 659 }
duke@435 660
jcoomes@1746 661 template<class E, unsigned int N> inline bool
jcoomes@1746 662 GenericTaskQueue<E, N>::pop_local(E& t) {
ysr@976 663 uint localBot = _bottom;
jcoomes@1342 664 // This value cannot be N-1. That can only occur as a result of
duke@435 665 // the assignment to bottom in this method. If it does, this method
jcoomes@1993 666 // resets the size to 0 before the next call (which is sequential,
duke@435 667 // since this is pop_local.)
jcoomes@1342 668 uint dirty_n_elems = dirty_size(localBot, _age.top());
jcoomes@1342 669 assert(dirty_n_elems != N - 1, "Shouldn't be possible...");
duke@435 670 if (dirty_n_elems == 0) return false;
duke@435 671 localBot = decrement_index(localBot);
duke@435 672 _bottom = localBot;
duke@435 673 // This is necessary to prevent any read below from being reordered
duke@435 674 // before the store just above.
duke@435 675 OrderAccess::fence();
jcoomes@1746 676 const_cast<E&>(t = _elems[localBot]);
duke@435 677 // This is a second read of "age"; the "size()" above is the first.
duke@435 678 // If there's still at least one element in the queue, based on the
duke@435 679 // "_bottom" and "age" we've read, then there can be no interference with
duke@435 680 // a "pop_global" operation, and we're done.
jcoomes@1342 681 idx_t tp = _age.top(); // XXX
duke@435 682 if (size(localBot, tp) > 0) {
jcoomes@1342 683 assert(dirty_size(localBot, tp) != N - 1, "sanity");
jcoomes@2020 684 TASKQUEUE_STATS_ONLY(stats.record_pop());
duke@435 685 return true;
duke@435 686 } else {
duke@435 687 // Otherwise, the queue contained exactly one element; we take the slow
duke@435 688 // path.
jcoomes@1342 689 return pop_local_slow(localBot, _age.get());
duke@435 690 }
duke@435 691 }
duke@435 692
jcoomes@1993 693 typedef GenericTaskQueue<oop> OopTaskQueue;
jcoomes@1746 694 typedef GenericTaskQueueSet<OopTaskQueue> OopTaskQueueSet;
duke@435 695
jcoomes@1746 696 #ifdef _MSC_VER
jcoomes@1746 697 #pragma warning(push)
jcoomes@1746 698 // warning C4522: multiple assignment operators specified
jcoomes@1746 699 #pragma warning(disable:4522)
jcoomes@1746 700 #endif
coleenp@548 701
coleenp@548 702 // This is a container class for either an oop* or a narrowOop*.
coleenp@548 703 // Both are pushed onto a task queue and the consumer will test is_narrow()
coleenp@548 704 // to determine which should be processed.
coleenp@548 705 class StarTask {
coleenp@548 706 void* _holder; // either union oop* or narrowOop*
jcoomes@1746 707
jcoomes@1746 708 enum { COMPRESSED_OOP_MASK = 1 };
jcoomes@1746 709
coleenp@548 710 public:
ysr@1280 711 StarTask(narrowOop* p) {
ysr@1280 712 assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
ysr@1280 713 _holder = (void *)((uintptr_t)p | COMPRESSED_OOP_MASK);
ysr@1280 714 }
ysr@1280 715 StarTask(oop* p) {
ysr@1280 716 assert(((uintptr_t)p & COMPRESSED_OOP_MASK) == 0, "Information loss!");
ysr@1280 717 _holder = (void*)p;
ysr@1280 718 }
coleenp@548 719 StarTask() { _holder = NULL; }
coleenp@548 720 operator oop*() { return (oop*)_holder; }
coleenp@548 721 operator narrowOop*() {
coleenp@548 722 return (narrowOop*)((uintptr_t)_holder & ~COMPRESSED_OOP_MASK);
coleenp@548 723 }
coleenp@548 724
jcoomes@1746 725 StarTask& operator=(const StarTask& t) {
jcoomes@1746 726 _holder = t._holder;
jcoomes@1746 727 return *this;
jcoomes@1746 728 }
jcoomes@1746 729 volatile StarTask& operator=(const volatile StarTask& t) volatile {
jcoomes@1746 730 _holder = t._holder;
jcoomes@1746 731 return *this;
jcoomes@1746 732 }
coleenp@548 733
coleenp@548 734 bool is_narrow() const {
coleenp@548 735 return (((uintptr_t)_holder & COMPRESSED_OOP_MASK) != 0);
coleenp@548 736 }
coleenp@548 737 };
coleenp@548 738
jcoomes@1746 739 class ObjArrayTask
jcoomes@1746 740 {
jcoomes@1746 741 public:
jcoomes@1746 742 ObjArrayTask(oop o = NULL, int idx = 0): _obj(o), _index(idx) { }
jcoomes@1746 743 ObjArrayTask(oop o, size_t idx): _obj(o), _index(int(idx)) {
jcoomes@1746 744 assert(idx <= size_t(max_jint), "too big");
jcoomes@1746 745 }
jcoomes@1746 746 ObjArrayTask(const ObjArrayTask& t): _obj(t._obj), _index(t._index) { }
jcoomes@1746 747
jcoomes@1746 748 ObjArrayTask& operator =(const ObjArrayTask& t) {
jcoomes@1746 749 _obj = t._obj;
jcoomes@1746 750 _index = t._index;
jcoomes@1746 751 return *this;
jcoomes@1746 752 }
jcoomes@1746 753 volatile ObjArrayTask&
jcoomes@1746 754 operator =(const volatile ObjArrayTask& t) volatile {
jcoomes@1746 755 _obj = t._obj;
jcoomes@1746 756 _index = t._index;
jcoomes@1746 757 return *this;
jcoomes@1746 758 }
jcoomes@1746 759
jcoomes@1746 760 inline oop obj() const { return _obj; }
jcoomes@1746 761 inline int index() const { return _index; }
jcoomes@1746 762
jcoomes@1746 763 DEBUG_ONLY(bool is_valid() const); // Tasks to be pushed/popped must be valid.
jcoomes@1746 764
jcoomes@1746 765 private:
jcoomes@1746 766 oop _obj;
jcoomes@1746 767 int _index;
jcoomes@1746 768 };
jcoomes@1746 769
jcoomes@1746 770 #ifdef _MSC_VER
jcoomes@1746 771 #pragma warning(pop)
jcoomes@1746 772 #endif
jcoomes@1746 773
jcoomes@1993 774 typedef OverflowTaskQueue<StarTask> OopStarTaskQueue;
jcoomes@1746 775 typedef GenericTaskQueueSet<OopStarTaskQueue> OopStarTaskQueueSet;
duke@435 776
jcoomes@1993 777 typedef OverflowTaskQueue<size_t> RegionTaskQueue;
jcoomes@1993 778 typedef GenericTaskQueueSet<RegionTaskQueue> RegionTaskQueueSet;

mercurial