• file
• revisions
• annotate
• diff
• comparison
• raw

src/share/vm/memory/threadLocalAllocBuffer.cpp@5134fa1cfe63 (annotated)

src/share/vm/memory/threadLocalAllocBuffer.cpp

Thu, 24 Mar 2011 15:47:01 -0700

author

ysr

date

Thu, 24 Mar 2011 15:47:01 -0700

changeset 2710

5134fa1cfe63

parent 2423

b1a2afa37ec4

child 3156

f08d439fab8c

permissions

-rw-r--r--

7029036: Card-table verification hangs with all framework collectors, except G1, even before the first GC
Summary: When verifying clean card ranges, use memory-range-bounded iteration over oops of objects overlapping that range, thus avoiding the otherwise quadratic worst-case cost of scanning large object arrays.
Reviewed-by: jmasa, jwilhelm, tonyp

duke@435	1	/*
phh@2423	2	* Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
trims@1907	19	* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@1907	20	* or visit www.oracle.com if you need additional information or have any
trims@1907	21	* questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
stefank@2314	25	#include "precompiled.hpp"
stefank@2314	26	#include "memory/genCollectedHeap.hpp"
stefank@2314	27	#include "memory/resourceArea.hpp"
stefank@2314	28	#include "memory/threadLocalAllocBuffer.inline.hpp"
stefank@2314	29	#include "memory/universe.inline.hpp"
stefank@2314	30	#include "oops/oop.inline.hpp"
stefank@2314	31	#include "utilities/copy.hpp"
stefank@2314	32	#ifdef TARGET_OS_FAMILY_linux
stefank@2314	33	# include "thread_linux.inline.hpp"
stefank@2314	34	#endif
stefank@2314	35	#ifdef TARGET_OS_FAMILY_solaris
stefank@2314	36	# include "thread_solaris.inline.hpp"
stefank@2314	37	#endif
stefank@2314	38	#ifdef TARGET_OS_FAMILY_windows
stefank@2314	39	# include "thread_windows.inline.hpp"
stefank@2314	40	#endif
stefank@2314	41
duke@435	42	// Thread-Local Edens support
duke@435	43
duke@435	44	// static member initialization
duke@435	45	unsigned ThreadLocalAllocBuffer::_target_refills = 0;
duke@435	46	GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL;
duke@435	47
duke@435	48	void ThreadLocalAllocBuffer::clear_before_allocation() {
duke@435	49	_slow_refill_waste += (unsigned)remaining();
duke@435	50	make_parsable(true); // also retire the TLAB
duke@435	51	}
duke@435	52
duke@435	53	void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() {
duke@435	54	global_stats()->initialize();
duke@435	55
duke@435	56	for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
duke@435	57	thread->tlab().accumulate_statistics();
duke@435	58	thread->tlab().initialize_statistics();
duke@435	59	}
duke@435	60
duke@435	61	// Publish new stats if some allocation occurred.
duke@435	62	if (global_stats()->allocation() != 0) {
duke@435	63	global_stats()->publish();
duke@435	64	if (PrintTLAB) {
duke@435	65	global_stats()->print();
duke@435	66	}
duke@435	67	}
duke@435	68	}
duke@435	69
duke@435	70	void ThreadLocalAllocBuffer::accumulate_statistics() {
duke@435	71	size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize;
duke@435	72	size_t unused = Universe::heap()->unsafe_max_tlab_alloc(myThread()) / HeapWordSize;
duke@435	73	size_t used = capacity - unused;
duke@435	74
duke@435	75	// Update allocation history if a reasonable amount of eden was allocated.
duke@435	76	bool update_allocation_history = used > 0.5 * capacity;
duke@435	77
duke@435	78	_gc_waste += (unsigned)remaining();
duke@435	79
duke@435	80	if (PrintTLAB && (_number_of_refills > 0 || Verbose)) {
duke@435	81	print_stats("gc");
duke@435	82	}
duke@435	83
duke@435	84	if (_number_of_refills > 0) {
duke@435	85
duke@435	86	if (update_allocation_history) {
duke@435	87	// Average the fraction of eden allocated in a tlab by this
duke@435	88	// thread for use in the next resize operation.
duke@435	89	// _gc_waste is not subtracted because it's included in
duke@435	90	// "used".
duke@435	91	size_t allocation = _number_of_refills * desired_size();
duke@435	92	double alloc_frac = allocation / (double) used;
duke@435	93	_allocation_fraction.sample(alloc_frac);
duke@435	94	}
duke@435	95	global_stats()->update_allocating_threads();
duke@435	96	global_stats()->update_number_of_refills(_number_of_refills);
duke@435	97	global_stats()->update_allocation(_number_of_refills * desired_size());
duke@435	98	global_stats()->update_gc_waste(_gc_waste);
duke@435	99	global_stats()->update_slow_refill_waste(_slow_refill_waste);
duke@435	100	global_stats()->update_fast_refill_waste(_fast_refill_waste);
duke@435	101
duke@435	102	} else {
duke@435	103	assert(_number_of_refills == 0 && _fast_refill_waste == 0 &&
duke@435	104	_slow_refill_waste == 0 && _gc_waste == 0,
duke@435	105	"tlab stats == 0");
duke@435	106	}
duke@435	107	global_stats()->update_slow_allocations(_slow_allocations);
duke@435	108	}
duke@435	109
duke@435	110	// Fills the current tlab with a dummy filler array to create
duke@435	111	// an illusion of a contiguous Eden and optionally retires the tlab.
duke@435	112	// Waste accounting should be done in caller as appropriate; see,
duke@435	113	// for example, clear_before_allocation().
duke@435	114	void ThreadLocalAllocBuffer::make_parsable(bool retire) {
duke@435	115	if (end() != NULL) {
duke@435	116	invariants();
phh@2423	117
phh@2423	118	if (retire) {
phh@2423	119	myThread()->incr_allocated_bytes(used_bytes());
phh@2423	120	}
phh@2423	121
johnc@1600	122	CollectedHeap::fill_with_object(top(), hard_end(), retire);
duke@435	123
duke@435	124	if (retire || ZeroTLAB) { // "Reset" the TLAB
duke@435	125	set_start(NULL);
duke@435	126	set_top(NULL);
duke@435	127	set_pf_top(NULL);
duke@435	128	set_end(NULL);
duke@435	129	}
duke@435	130	}
duke@435	131	assert(!(retire || ZeroTLAB) ||
duke@435	132	(start() == NULL && end() == NULL && top() == NULL),
duke@435	133	"TLAB must be reset");
duke@435	134	}
duke@435	135
duke@435	136	void ThreadLocalAllocBuffer::resize_all_tlabs() {
duke@435	137	for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) {
duke@435	138	thread->tlab().resize();
duke@435	139	}
duke@435	140	}
duke@435	141
duke@435	142	void ThreadLocalAllocBuffer::resize() {
duke@435	143
duke@435	144	if (ResizeTLAB) {
duke@435	145	// Compute the next tlab size using expected allocation amount
duke@435	146	size_t alloc = (size_t)(_allocation_fraction.average() *
duke@435	147	(Universe::heap()->tlab_capacity(myThread()) / HeapWordSize));
duke@435	148	size_t new_size = alloc / _target_refills;
duke@435	149
duke@435	150	new_size = MIN2(MAX2(new_size, min_size()), max_size());
duke@435	151
duke@435	152	size_t aligned_new_size = align_object_size(new_size);
duke@435	153
duke@435	154	if (PrintTLAB && Verbose) {
duke@435	155	gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]"
duke@435	156	" refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n",
duke@435	157	myThread(), myThread()->osthread()->thread_id(),
duke@435	158	_target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size);
duke@435	159	}
duke@435	160	set_desired_size(aligned_new_size);
duke@435	161
duke@435	162	set_refill_waste_limit(initial_refill_waste_limit());
duke@435	163	}
duke@435	164	}
duke@435	165
duke@435	166	void ThreadLocalAllocBuffer::initialize_statistics() {
duke@435	167	_number_of_refills = 0;
duke@435	168	_fast_refill_waste = 0;
duke@435	169	_slow_refill_waste = 0;
duke@435	170	_gc_waste = 0;
duke@435	171	_slow_allocations = 0;
duke@435	172	}
duke@435	173
duke@435	174	void ThreadLocalAllocBuffer::fill(HeapWord* start,
duke@435	175	HeapWord* top,
duke@435	176	size_t new_size) {
duke@435	177	_number_of_refills++;
duke@435	178	if (PrintTLAB && Verbose) {
duke@435	179	print_stats("fill");
duke@435	180	}
duke@435	181	assert(top <= start + new_size - alignment_reserve(), "size too small");
duke@435	182	initialize(start, top, start + new_size - alignment_reserve());
duke@435	183
duke@435	184	// Reset amount of internal fragmentation
duke@435	185	set_refill_waste_limit(initial_refill_waste_limit());
duke@435	186	}
duke@435	187
duke@435	188	void ThreadLocalAllocBuffer::initialize(HeapWord* start,
duke@435	189	HeapWord* top,
duke@435	190	HeapWord* end) {
duke@435	191	set_start(start);
duke@435	192	set_top(top);
duke@435	193	set_pf_top(top);
duke@435	194	set_end(end);
duke@435	195	invariants();
duke@435	196	}
duke@435	197
duke@435	198	void ThreadLocalAllocBuffer::initialize() {
duke@435	199	initialize(NULL, // start
duke@435	200	NULL, // top
duke@435	201	NULL); // end
duke@435	202
duke@435	203	set_desired_size(initial_desired_size());
duke@435	204
duke@435	205	// Following check is needed because at startup the main (primordial)
duke@435	206	// thread is initialized before the heap is. The initialization for
duke@435	207	// this thread is redone in startup_initialization below.
duke@435	208	if (Universe::heap() != NULL) {
duke@435	209	size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize;
duke@435	210	double alloc_frac = desired_size() * target_refills() / (double) capacity;
duke@435	211	_allocation_fraction.sample(alloc_frac);
duke@435	212	}
duke@435	213
duke@435	214	set_refill_waste_limit(initial_refill_waste_limit());
duke@435	215
duke@435	216	initialize_statistics();
duke@435	217	}
duke@435	218
duke@435	219	void ThreadLocalAllocBuffer::startup_initialization() {
duke@435	220
duke@435	221	// Assuming each thread's active tlab is, on average,
duke@435	222	// 1/2 full at a GC
duke@435	223	_target_refills = 100 / (2 * TLABWasteTargetPercent);
duke@435	224	_target_refills = MAX2(_target_refills, (unsigned)1U);
duke@435	225
duke@435	226	_global_stats = new GlobalTLABStats();
duke@435	227
duke@435	228	// During jvm startup, the main (primordial) thread is initialized
duke@435	229	// before the heap is initialized. So reinitialize it now.
duke@435	230	guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread");
duke@435	231	Thread::current()->tlab().initialize();
duke@435	232
duke@435	233	if (PrintTLAB && Verbose) {
duke@435	234	gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n",
duke@435	235	min_size(), Thread::current()->tlab().initial_desired_size(), max_size());
duke@435	236	}
duke@435	237	}
duke@435	238
duke@435	239	size_t ThreadLocalAllocBuffer::initial_desired_size() {
duke@435	240	size_t init_sz;
duke@435	241
duke@435	242	if (TLABSize > 0) {
duke@435	243	init_sz = MIN2(TLABSize / HeapWordSize, max_size());
duke@435	244	} else if (global_stats() == NULL) {
duke@435	245	// Startup issue - main thread initialized before heap initialized.
duke@435	246	init_sz = min_size();
duke@435	247	} else {
duke@435	248	// Initial size is a function of the average number of allocating threads.
duke@435	249	unsigned nof_threads = global_stats()->allocating_threads_avg();
duke@435	250
duke@435	251	init_sz = (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize) /
duke@435	252	(nof_threads * target_refills());
duke@435	253	init_sz = align_object_size(init_sz);
duke@435	254	init_sz = MIN2(MAX2(init_sz, min_size()), max_size());
duke@435	255	}
duke@435	256	return init_sz;
duke@435	257	}
duke@435	258
duke@435	259	const size_t ThreadLocalAllocBuffer::max_size() {
duke@435	260
duke@435	261	// TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE].
duke@435	262	// This restriction could be removed by enabling filling with multiple arrays.
duke@435	263	// If we compute that the reasonable way as
duke@435	264	// header_size + ((sizeof(jint) * max_jint) / HeapWordSize)
duke@435	265	// we'll overflow on the multiply, so we do the divide first.
duke@435	266	// We actually lose a little by dividing first,
duke@435	267	// but that just makes the TLAB somewhat smaller than the biggest array,
duke@435	268	// which is fine, since we'll be able to fill that.
duke@435	269
duke@435	270	size_t unaligned_max_size = typeArrayOopDesc::header_size(T_INT) +
duke@435	271	sizeof(jint) *
duke@435	272	((juint) max_jint / (size_t) HeapWordSize);
duke@435	273	return align_size_down(unaligned_max_size, MinObjAlignment);
duke@435	274	}
duke@435	275
duke@435	276	void ThreadLocalAllocBuffer::print_stats(const char* tag) {
duke@435	277	Thread* thrd = myThread();
duke@435	278	size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste;
duke@435	279	size_t alloc = _number_of_refills * _desired_size;
duke@435	280	double waste_percent = alloc == 0 ? 0.0 :
duke@435	281	100.0 * waste / alloc;
duke@435	282	size_t tlab_used = Universe::heap()->tlab_capacity(thrd) -
duke@435	283	Universe::heap()->unsafe_max_tlab_alloc(thrd);
duke@435	284	gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]"
duke@435	285	" desired_size: " SIZE_FORMAT "KB"
duke@435	286	" slow allocs: %d refill waste: " SIZE_FORMAT "B"
duke@435	287	" alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB"
duke@435	288	" slow: %dB fast: %dB\n",
duke@435	289	tag, thrd, thrd->osthread()->thread_id(),
duke@435	290	_desired_size / (K / HeapWordSize),
duke@435	291	_slow_allocations, _refill_waste_limit * HeapWordSize,
duke@435	292	_allocation_fraction.average(),
duke@435	293	_allocation_fraction.average() * tlab_used / K,
duke@435	294	_number_of_refills, waste_percent,
duke@435	295	_gc_waste * HeapWordSize,
duke@435	296	_slow_refill_waste * HeapWordSize,
duke@435	297	_fast_refill_waste * HeapWordSize);
duke@435	298	}
duke@435	299
duke@435	300	void ThreadLocalAllocBuffer::verify() {
duke@435	301	HeapWord* p = start();
duke@435	302	HeapWord* t = top();
duke@435	303	HeapWord* prev_p = NULL;
duke@435	304	while (p < t) {
duke@435	305	oop(p)->verify();
duke@435	306	prev_p = p;
duke@435	307	p += oop(p)->size();
duke@435	308	}
duke@435	309	guarantee(p == top(), "end of last object must match end of space");
duke@435	310	}
duke@435	311
duke@435	312	Thread* ThreadLocalAllocBuffer::myThread() {
duke@435	313	return (Thread*)(((char *)this) +
duke@435	314	in_bytes(start_offset()) -
duke@435	315	in_bytes(Thread::tlab_start_offset()));
duke@435	316	}
duke@435	317
duke@435	318
duke@435	319	GlobalTLABStats::GlobalTLABStats() :
duke@435	320	_allocating_threads_avg(TLABAllocationWeight) {
duke@435	321
duke@435	322	initialize();
duke@435	323
duke@435	324	_allocating_threads_avg.sample(1); // One allocating thread at startup
duke@435	325
duke@435	326	if (UsePerfData) {
duke@435	327
duke@435	328	EXCEPTION_MARK;
duke@435	329	ResourceMark rm;
duke@435	330
duke@435	331	char* cname = PerfDataManager::counter_name("tlab", "allocThreads");
duke@435	332	_perf_allocating_threads =
duke@435	333	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
duke@435	334
duke@435	335	cname = PerfDataManager::counter_name("tlab", "fills");
duke@435	336	_perf_total_refills =
duke@435	337	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
duke@435	338
duke@435	339	cname = PerfDataManager::counter_name("tlab", "maxFills");
duke@435	340	_perf_max_refills =
duke@435	341	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
duke@435	342
duke@435	343	cname = PerfDataManager::counter_name("tlab", "alloc");
duke@435	344	_perf_allocation =
duke@435	345	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	346
duke@435	347	cname = PerfDataManager::counter_name("tlab", "gcWaste");
duke@435	348	_perf_gc_waste =
duke@435	349	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	350
duke@435	351	cname = PerfDataManager::counter_name("tlab", "maxGcWaste");
duke@435	352	_perf_max_gc_waste =
duke@435	353	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	354
duke@435	355	cname = PerfDataManager::counter_name("tlab", "slowWaste");
duke@435	356	_perf_slow_refill_waste =
duke@435	357	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	358
duke@435	359	cname = PerfDataManager::counter_name("tlab", "maxSlowWaste");
duke@435	360	_perf_max_slow_refill_waste =
duke@435	361	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	362
duke@435	363	cname = PerfDataManager::counter_name("tlab", "fastWaste");
duke@435	364	_perf_fast_refill_waste =
duke@435	365	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	366
duke@435	367	cname = PerfDataManager::counter_name("tlab", "maxFastWaste");
duke@435	368	_perf_max_fast_refill_waste =
duke@435	369	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK);
duke@435	370
duke@435	371	cname = PerfDataManager::counter_name("tlab", "slowAlloc");
duke@435	372	_perf_slow_allocations =
duke@435	373	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
duke@435	374
duke@435	375	cname = PerfDataManager::counter_name("tlab", "maxSlowAlloc");
duke@435	376	_perf_max_slow_allocations =
duke@435	377	PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK);
duke@435	378	}
duke@435	379	}
duke@435	380
duke@435	381	void GlobalTLABStats::initialize() {
duke@435	382	// Clear counters summarizing info from all threads
duke@435	383	_allocating_threads = 0;
duke@435	384	_total_refills = 0;
duke@435	385	_max_refills = 0;
duke@435	386	_total_allocation = 0;
duke@435	387	_total_gc_waste = 0;
duke@435	388	_max_gc_waste = 0;
duke@435	389	_total_slow_refill_waste = 0;
duke@435	390	_max_slow_refill_waste = 0;
duke@435	391	_total_fast_refill_waste = 0;
duke@435	392	_max_fast_refill_waste = 0;
duke@435	393	_total_slow_allocations = 0;
duke@435	394	_max_slow_allocations = 0;
duke@435	395	}
duke@435	396
duke@435	397	void GlobalTLABStats::publish() {
duke@435	398	_allocating_threads_avg.sample(_allocating_threads);
duke@435	399	if (UsePerfData) {
duke@435	400	_perf_allocating_threads ->set_value(_allocating_threads);
duke@435	401	_perf_total_refills ->set_value(_total_refills);
duke@435	402	_perf_max_refills ->set_value(_max_refills);
duke@435	403	_perf_allocation ->set_value(_total_allocation);
duke@435	404	_perf_gc_waste ->set_value(_total_gc_waste);
duke@435	405	_perf_max_gc_waste ->set_value(_max_gc_waste);
duke@435	406	_perf_slow_refill_waste ->set_value(_total_slow_refill_waste);
duke@435	407	_perf_max_slow_refill_waste->set_value(_max_slow_refill_waste);
duke@435	408	_perf_fast_refill_waste ->set_value(_total_fast_refill_waste);
duke@435	409	_perf_max_fast_refill_waste->set_value(_max_fast_refill_waste);
duke@435	410	_perf_slow_allocations ->set_value(_total_slow_allocations);
duke@435	411	_perf_max_slow_allocations ->set_value(_max_slow_allocations);
duke@435	412	}
duke@435	413	}
duke@435	414
duke@435	415	void GlobalTLABStats::print() {
duke@435	416	size_t waste = _total_gc_waste + _total_slow_refill_waste + _total_fast_refill_waste;
duke@435	417	double waste_percent = _total_allocation == 0 ? 0.0 :
duke@435	418	100.0 * waste / _total_allocation;
duke@435	419	gclog_or_tty->print("TLAB totals: thrds: %d refills: %d max: %d"
duke@435	420	" slow allocs: %d max %d waste: %4.1f%%"
duke@435	421	" gc: " SIZE_FORMAT "B max: " SIZE_FORMAT "B"
duke@435	422	" slow: " SIZE_FORMAT "B max: " SIZE_FORMAT "B"
duke@435	423	" fast: " SIZE_FORMAT "B max: " SIZE_FORMAT "B\n",
duke@435	424	_allocating_threads,
duke@435	425	_total_refills, _max_refills,
duke@435	426	_total_slow_allocations, _max_slow_allocations,
duke@435	427	waste_percent,
duke@435	428	_total_gc_waste * HeapWordSize,
duke@435	429	_max_gc_waste * HeapWordSize,
duke@435	430	_total_slow_refill_waste * HeapWordSize,
duke@435	431	_max_slow_refill_waste * HeapWordSize,
duke@435	432	_total_fast_refill_waste * HeapWordSize,
duke@435	433	_max_fast_refill_waste * HeapWordSize);
duke@435	434	}