94 bool mt_processing, |
94 bool mt_processing, |
95 uint mt_processing_degree, |
95 uint mt_processing_degree, |
96 bool mt_discovery, |
96 bool mt_discovery, |
97 uint mt_discovery_degree, |
97 uint mt_discovery_degree, |
98 bool atomic_discovery, |
98 bool atomic_discovery, |
99 BoolObjectClosure* is_alive_non_header, |
99 BoolObjectClosure* is_alive_non_header) : |
100 bool discovered_list_needs_post_barrier) : |
|
101 _discovering_refs(false), |
100 _discovering_refs(false), |
102 _enqueuing_is_done(false), |
101 _enqueuing_is_done(false), |
103 _is_alive_non_header(is_alive_non_header), |
102 _is_alive_non_header(is_alive_non_header), |
104 _discovered_list_needs_post_barrier(discovered_list_needs_post_barrier), |
|
105 _processing_is_mt(mt_processing), |
103 _processing_is_mt(mt_processing), |
106 _next_id(0) |
104 _next_id(0) |
107 { |
105 { |
108 _span = span; |
106 _span = span; |
109 _discovery_is_atomic = atomic_discovery; |
107 _discovery_is_atomic = atomic_discovery; |
338 HeapWord* pending_list_addr) { |
336 HeapWord* pending_list_addr) { |
339 // Given a list of refs linked through the "discovered" field |
337 // Given a list of refs linked through the "discovered" field |
340 // (java.lang.ref.Reference.discovered), self-loop their "next" field |
338 // (java.lang.ref.Reference.discovered), self-loop their "next" field |
341 // thus distinguishing them from active References, then |
339 // thus distinguishing them from active References, then |
342 // prepend them to the pending list. |
340 // prepend them to the pending list. |
|
341 // |
|
342 // The Java threads will see the Reference objects linked together through |
|
343 // the discovered field. Instead of trying to do the write barrier updates |
|
344 // in all places in the reference processor where we manipulate the discovered |
|
345 // field we make sure to do the barrier here where we anyway iterate through |
|
346 // all linked Reference objects. Note that it is important to not dirty any |
|
347 // cards during reference processing since this will cause card table |
|
348 // verification to fail for G1. |
|
349 // |
343 // BKWRD COMPATIBILITY NOTE: For older JDKs (prior to the fix for 4956777), |
350 // BKWRD COMPATIBILITY NOTE: For older JDKs (prior to the fix for 4956777), |
344 // the "next" field is used to chain the pending list, not the discovered |
351 // the "next" field is used to chain the pending list, not the discovered |
345 // field. |
352 // field. |
346 |
|
347 if (TraceReferenceGC && PrintGCDetails) { |
353 if (TraceReferenceGC && PrintGCDetails) { |
348 gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " |
354 gclog_or_tty->print_cr("ReferenceProcessor::enqueue_discovered_reflist list " |
349 INTPTR_FORMAT, (address)refs_list.head()); |
355 INTPTR_FORMAT, (address)refs_list.head()); |
350 } |
356 } |
351 |
357 |
352 oop obj = NULL; |
358 oop obj = NULL; |
353 oop next_d = refs_list.head(); |
359 oop next_d = refs_list.head(); |
354 if (pending_list_uses_discovered_field()) { // New behaviour |
360 if (pending_list_uses_discovered_field()) { // New behavior |
355 // Walk down the list, self-looping the next field |
361 // Walk down the list, self-looping the next field |
356 // so that the References are not considered active. |
362 // so that the References are not considered active. |
357 while (obj != next_d) { |
363 while (obj != next_d) { |
358 obj = next_d; |
364 obj = next_d; |
359 assert(obj->is_instanceRef(), "should be reference object"); |
365 assert(obj->is_instanceRef(), "should be reference object"); |
363 (void *)obj, (void *)next_d); |
369 (void *)obj, (void *)next_d); |
364 } |
370 } |
365 assert(java_lang_ref_Reference::next(obj) == NULL, |
371 assert(java_lang_ref_Reference::next(obj) == NULL, |
366 "Reference not active; should not be discovered"); |
372 "Reference not active; should not be discovered"); |
367 // Self-loop next, so as to make Ref not active. |
373 // Self-loop next, so as to make Ref not active. |
368 // Post-barrier not needed when looping to self. |
|
369 java_lang_ref_Reference::set_next_raw(obj, obj); |
374 java_lang_ref_Reference::set_next_raw(obj, obj); |
370 if (next_d == obj) { // obj is last |
375 if (next_d != obj) { |
371 // Swap refs_list into pendling_list_addr and |
376 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), next_d); |
|
377 } else { |
|
378 // This is the last object. |
|
379 // Swap refs_list into pending_list_addr and |
372 // set obj's discovered to what we read from pending_list_addr. |
380 // set obj's discovered to what we read from pending_list_addr. |
373 oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); |
381 oop old = oopDesc::atomic_exchange_oop(refs_list.head(), pending_list_addr); |
374 // Need post-barrier on pending_list_addr above; |
382 // Need post-barrier on pending_list_addr. See enqueue_discovered_ref_helper() above. |
375 // see special post-barrier code at the end of |
|
376 // enqueue_discovered_reflists() further below. |
|
377 java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL |
383 java_lang_ref_Reference::set_discovered_raw(obj, old); // old may be NULL |
378 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); |
384 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(obj), old); |
379 } |
385 } |
380 } |
386 } |
381 } else { // Old behaviour |
387 } else { // Old behaviour |
494 } |
500 } |
495 // Remove Reference object from discovered list. Note that G1 does not need a |
501 // Remove Reference object from discovered list. Note that G1 does not need a |
496 // pre-barrier here because we know the Reference has already been found/marked, |
502 // pre-barrier here because we know the Reference has already been found/marked, |
497 // that's how it ended up in the discovered list in the first place. |
503 // that's how it ended up in the discovered list in the first place. |
498 oop_store_raw(_prev_next, new_next); |
504 oop_store_raw(_prev_next, new_next); |
499 if (_discovered_list_needs_post_barrier && _prev_next != _refs_list.adr_head()) { |
|
500 // Needs post-barrier and this is not the list head (which is not on the heap) |
|
501 oopDesc::bs()->write_ref_field(_prev_next, new_next); |
|
502 } |
|
503 NOT_PRODUCT(_removed++); |
505 NOT_PRODUCT(_removed++); |
504 _refs_list.dec_length(1); |
506 _refs_list.dec_length(1); |
505 } |
507 } |
506 |
508 |
507 // Make the Reference object active again. |
509 // Make the Reference object active again. |
508 void DiscoveredListIterator::make_active() { |
510 void DiscoveredListIterator::make_active() { |
509 // For G1 we don't want to use set_next - it |
511 // The pre barrier for G1 is probably just needed for the old |
510 // will dirty the card for the next field of |
512 // reference processing behavior. Should we guard this with |
511 // the reference object and will fail |
513 // ReferenceProcessor::pending_list_uses_discovered_field() ? |
512 // CT verification. |
|
513 if (UseG1GC) { |
514 if (UseG1GC) { |
514 HeapWord* next_addr = java_lang_ref_Reference::next_addr(_ref); |
515 HeapWord* next_addr = java_lang_ref_Reference::next_addr(_ref); |
515 if (UseCompressedOops) { |
516 if (UseCompressedOops) { |
516 oopDesc::bs()->write_ref_field_pre((narrowOop*)next_addr, NULL); |
517 oopDesc::bs()->write_ref_field_pre((narrowOop*)next_addr, NULL); |
517 } else { |
518 } else { |
518 oopDesc::bs()->write_ref_field_pre((oop*)next_addr, NULL); |
519 oopDesc::bs()->write_ref_field_pre((oop*)next_addr, NULL); |
519 } |
520 } |
520 java_lang_ref_Reference::set_next_raw(_ref, NULL); |
521 } |
521 } else { |
522 java_lang_ref_Reference::set_next_raw(_ref, NULL); |
522 java_lang_ref_Reference::set_next(_ref, NULL); |
|
523 } |
|
524 } |
523 } |
525 |
524 |
526 void DiscoveredListIterator::clear_referent() { |
525 void DiscoveredListIterator::clear_referent() { |
527 oop_store_raw(_referent_addr, NULL); |
526 oop_store_raw(_referent_addr, NULL); |
528 } |
527 } |
544 ReferencePolicy* policy, |
543 ReferencePolicy* policy, |
545 BoolObjectClosure* is_alive, |
544 BoolObjectClosure* is_alive, |
546 OopClosure* keep_alive, |
545 OopClosure* keep_alive, |
547 VoidClosure* complete_gc) { |
546 VoidClosure* complete_gc) { |
548 assert(policy != NULL, "Must have a non-NULL policy"); |
547 assert(policy != NULL, "Must have a non-NULL policy"); |
549 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); |
548 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
550 // Decide which softly reachable refs should be kept alive. |
549 // Decide which softly reachable refs should be kept alive. |
551 while (iter.has_next()) { |
550 while (iter.has_next()) { |
552 iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); |
551 iter.load_ptrs(DEBUG_ONLY(!discovery_is_atomic() /* allow_null_referent */)); |
553 bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); |
552 bool referent_is_dead = (iter.referent() != NULL) && !iter.is_referent_alive(); |
554 if (referent_is_dead && |
553 if (referent_is_dead && |
584 void |
583 void |
585 ReferenceProcessor::pp2_work(DiscoveredList& refs_list, |
584 ReferenceProcessor::pp2_work(DiscoveredList& refs_list, |
586 BoolObjectClosure* is_alive, |
585 BoolObjectClosure* is_alive, |
587 OopClosure* keep_alive) { |
586 OopClosure* keep_alive) { |
588 assert(discovery_is_atomic(), "Error"); |
587 assert(discovery_is_atomic(), "Error"); |
589 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); |
588 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
590 while (iter.has_next()) { |
589 while (iter.has_next()) { |
591 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
590 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
592 DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) |
591 DEBUG_ONLY(oop next = java_lang_ref_Reference::next(iter.obj());) |
593 assert(next == NULL, "Should not discover inactive Reference"); |
592 assert(next == NULL, "Should not discover inactive Reference"); |
594 if (iter.is_referent_alive()) { |
593 if (iter.is_referent_alive()) { |
621 ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, |
620 ReferenceProcessor::pp2_work_concurrent_discovery(DiscoveredList& refs_list, |
622 BoolObjectClosure* is_alive, |
621 BoolObjectClosure* is_alive, |
623 OopClosure* keep_alive, |
622 OopClosure* keep_alive, |
624 VoidClosure* complete_gc) { |
623 VoidClosure* complete_gc) { |
625 assert(!discovery_is_atomic(), "Error"); |
624 assert(!discovery_is_atomic(), "Error"); |
626 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); |
625 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
627 while (iter.has_next()) { |
626 while (iter.has_next()) { |
628 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
627 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
629 HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); |
628 HeapWord* next_addr = java_lang_ref_Reference::next_addr(iter.obj()); |
630 oop next = java_lang_ref_Reference::next(iter.obj()); |
629 oop next = java_lang_ref_Reference::next(iter.obj()); |
631 if ((iter.referent() == NULL || iter.is_referent_alive() || |
630 if ((iter.referent() == NULL || iter.is_referent_alive() || |
664 bool clear_referent, |
663 bool clear_referent, |
665 BoolObjectClosure* is_alive, |
664 BoolObjectClosure* is_alive, |
666 OopClosure* keep_alive, |
665 OopClosure* keep_alive, |
667 VoidClosure* complete_gc) { |
666 VoidClosure* complete_gc) { |
668 ResourceMark rm; |
667 ResourceMark rm; |
669 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); |
668 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
670 while (iter.has_next()) { |
669 while (iter.has_next()) { |
671 iter.update_discovered(); |
670 iter.update_discovered(); |
672 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
671 iter.load_ptrs(DEBUG_ONLY(false /* allow_null_referent */)); |
673 if (clear_referent) { |
672 if (clear_referent) { |
674 // NULL out referent pointer |
673 // NULL out referent pointer |
780 } |
779 } |
781 private: |
780 private: |
782 bool _clear_referent; |
781 bool _clear_referent; |
783 }; |
782 }; |
784 |
783 |
785 void ReferenceProcessor::set_discovered(oop ref, oop value) { |
|
786 java_lang_ref_Reference::set_discovered_raw(ref, value); |
|
787 if (_discovered_list_needs_post_barrier) { |
|
788 oopDesc::bs()->write_ref_field(java_lang_ref_Reference::discovered_addr(ref), value); |
|
789 } |
|
790 } |
|
791 |
|
792 // Balances reference queues. |
784 // Balances reference queues. |
793 // Move entries from all queues[0, 1, ..., _max_num_q-1] to |
785 // Move entries from all queues[0, 1, ..., _max_num_q-1] to |
794 // queues[0, 1, ..., _num_q-1] because only the first _num_q |
786 // queues[0, 1, ..., _num_q-1] because only the first _num_q |
795 // corresponding to the active workers will be processed. |
787 // corresponding to the active workers will be processed. |
796 void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) |
788 void ReferenceProcessor::balance_queues(DiscoveredList ref_lists[]) |
844 } |
836 } |
845 |
837 |
846 // Add the chain to the to list. |
838 // Add the chain to the to list. |
847 if (ref_lists[to_idx].head() == NULL) { |
839 if (ref_lists[to_idx].head() == NULL) { |
848 // to list is empty. Make a loop at the end. |
840 // to list is empty. Make a loop at the end. |
849 set_discovered(move_tail, move_tail); |
841 java_lang_ref_Reference::set_discovered_raw(move_tail, move_tail); |
850 } else { |
842 } else { |
851 set_discovered(move_tail, ref_lists[to_idx].head()); |
843 java_lang_ref_Reference::set_discovered_raw(move_tail, ref_lists[to_idx].head()); |
852 } |
844 } |
853 ref_lists[to_idx].set_head(move_head); |
845 ref_lists[to_idx].set_head(move_head); |
854 ref_lists[to_idx].inc_length(refs_to_move); |
846 ref_lists[to_idx].inc_length(refs_to_move); |
855 |
847 |
856 // Remove the chain from the from list. |
848 // Remove the chain from the from list. |
980 } |
972 } |
981 } |
973 } |
982 |
974 |
983 void ReferenceProcessor::clean_up_discovered_reflist(DiscoveredList& refs_list) { |
975 void ReferenceProcessor::clean_up_discovered_reflist(DiscoveredList& refs_list) { |
984 assert(!discovery_is_atomic(), "Else why call this method?"); |
976 assert(!discovery_is_atomic(), "Else why call this method?"); |
985 DiscoveredListIterator iter(refs_list, NULL, NULL, _discovered_list_needs_post_barrier); |
977 DiscoveredListIterator iter(refs_list, NULL, NULL); |
986 while (iter.has_next()) { |
978 while (iter.has_next()) { |
987 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
979 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
988 oop next = java_lang_ref_Reference::next(iter.obj()); |
980 oop next = java_lang_ref_Reference::next(iter.obj()); |
989 assert(next->is_oop_or_null(), "bad next field"); |
981 assert(next->is_oop_or_null(), "bad next field"); |
990 // If referent has been cleared or Reference is not active, |
982 // If referent has been cleared or Reference is not active, |
1069 // discovered_addr. |
1061 // discovered_addr. |
1070 oop current_head = refs_list.head(); |
1062 oop current_head = refs_list.head(); |
1071 // The last ref must have its discovered field pointing to itself. |
1063 // The last ref must have its discovered field pointing to itself. |
1072 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1064 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1073 |
1065 |
1074 // Note: In the case of G1, this specific pre-barrier is strictly |
|
1075 // not necessary because the only case we are interested in |
|
1076 // here is when *discovered_addr is NULL (see the CAS further below), |
|
1077 // so this will expand to nothing. As a result, we have manually |
|
1078 // elided this out for G1, but left in the test for some future |
|
1079 // collector that might have need for a pre-barrier here, e.g.:- |
|
1080 // oopDesc::bs()->write_ref_field_pre((oop* or narrowOop*)discovered_addr, next_discovered); |
|
1081 assert(!_discovered_list_needs_post_barrier || UseG1GC, |
|
1082 "Need to check non-G1 collector: " |
|
1083 "may need a pre-write-barrier for CAS from NULL below"); |
|
1084 oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, |
1066 oop retest = oopDesc::atomic_compare_exchange_oop(next_discovered, discovered_addr, |
1085 NULL); |
1067 NULL); |
1086 if (retest == NULL) { |
1068 if (retest == NULL) { |
1087 // This thread just won the right to enqueue the object. |
1069 // This thread just won the right to enqueue the object. |
1088 // We have separate lists for enqueueing, so no synchronization |
1070 // We have separate lists for enqueueing, so no synchronization |
1089 // is necessary. |
1071 // is necessary. |
1090 refs_list.set_head(obj); |
1072 refs_list.set_head(obj); |
1091 refs_list.inc_length(1); |
1073 refs_list.inc_length(1); |
1092 if (_discovered_list_needs_post_barrier) { |
|
1093 oopDesc::bs()->write_ref_field((void*)discovered_addr, next_discovered); |
|
1094 } |
|
1095 |
1074 |
1096 if (TraceReferenceGC) { |
1075 if (TraceReferenceGC) { |
1097 gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", |
1076 gclog_or_tty->print_cr("Discovered reference (mt) (" INTPTR_FORMAT ": %s)", |
1098 (void *)obj, obj->klass()->internal_name()); |
1077 (void *)obj, obj->klass()->internal_name()); |
1099 } |
1078 } |
1240 } |
1219 } |
1241 |
1220 |
1242 if (_discovery_is_mt) { |
1221 if (_discovery_is_mt) { |
1243 add_to_discovered_list_mt(*list, obj, discovered_addr); |
1222 add_to_discovered_list_mt(*list, obj, discovered_addr); |
1244 } else { |
1223 } else { |
1245 // If "_discovered_list_needs_post_barrier", we do write barriers when |
1224 // We do a raw store here: the field will be visited later when processing |
1246 // updating the discovered reference list. Otherwise, we do a raw store |
1225 // the discovered references. |
1247 // here: the field will be visited later when processing the discovered |
|
1248 // references. |
|
1249 oop current_head = list->head(); |
1226 oop current_head = list->head(); |
1250 // The last ref must have its discovered field pointing to itself. |
1227 // The last ref must have its discovered field pointing to itself. |
1251 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1228 oop next_discovered = (current_head != NULL) ? current_head : obj; |
1252 |
1229 |
1253 // As in the case further above, since we are over-writing a NULL |
|
1254 // pre-value, we can safely elide the pre-barrier here for the case of G1. |
|
1255 // e.g.:- oopDesc::bs()->write_ref_field_pre((oop* or narrowOop*)discovered_addr, next_discovered); |
|
1256 assert(discovered == NULL, "control point invariant"); |
1230 assert(discovered == NULL, "control point invariant"); |
1257 assert(!_discovered_list_needs_post_barrier || UseG1GC, |
|
1258 "For non-G1 collector, may need a pre-write-barrier for CAS from NULL below"); |
|
1259 oop_store_raw(discovered_addr, next_discovered); |
1231 oop_store_raw(discovered_addr, next_discovered); |
1260 if (_discovered_list_needs_post_barrier) { |
|
1261 oopDesc::bs()->write_ref_field((void*)discovered_addr, next_discovered); |
|
1262 } |
|
1263 list->set_head(obj); |
1232 list->set_head(obj); |
1264 list->inc_length(1); |
1233 list->inc_length(1); |
1265 |
1234 |
1266 if (TraceReferenceGC) { |
1235 if (TraceReferenceGC) { |
1267 gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", |
1236 gclog_or_tty->print_cr("Discovered reference (" INTPTR_FORMAT ": %s)", |
1351 ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, |
1320 ReferenceProcessor::preclean_discovered_reflist(DiscoveredList& refs_list, |
1352 BoolObjectClosure* is_alive, |
1321 BoolObjectClosure* is_alive, |
1353 OopClosure* keep_alive, |
1322 OopClosure* keep_alive, |
1354 VoidClosure* complete_gc, |
1323 VoidClosure* complete_gc, |
1355 YieldClosure* yield) { |
1324 YieldClosure* yield) { |
1356 DiscoveredListIterator iter(refs_list, keep_alive, is_alive, _discovered_list_needs_post_barrier); |
1325 DiscoveredListIterator iter(refs_list, keep_alive, is_alive); |
1357 while (iter.has_next()) { |
1326 while (iter.has_next()) { |
1358 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
1327 iter.load_ptrs(DEBUG_ONLY(true /* allow_null_referent */)); |
1359 oop obj = iter.obj(); |
1328 oop obj = iter.obj(); |
1360 oop next = java_lang_ref_Reference::next(obj); |
1329 oop next = java_lang_ref_Reference::next(obj); |
1361 if (iter.referent() == NULL || iter.is_referent_alive() || |
1330 if (iter.referent() == NULL || iter.is_referent_alive() || |