• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/stubGenerator_x86_64.cpp@f3de1255b035 (annotated)

src/cpu/x86/vm/stubGenerator_x86_64.cpp

Wed, 07 May 2008 08:06:46 -0700

author

rasbold

date

Wed, 07 May 2008 08:06:46 -0700

changeset 580

f3de1255b035

parent 559

b130b98db9cf

child 631

d1605aabd0a1

child 777

37f87013dfd8

permissions

-rw-r--r--

6603011: RFE: Optimize long division
Summary: Transform long division by constant into multiply
Reviewed-by: never, kvn

duke@435	1	/*
duke@435	2	* Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_stubGenerator_x86_64.cpp.incl"
duke@435	27
duke@435	28	// Declaration and definition of StubGenerator (no .hpp file).
duke@435	29	// For a more detailed description of the stub routine structure
duke@435	30	// see the comment in stubRoutines.hpp
duke@435	31
duke@435	32	#define __ _masm->
coleenp@548	33	#define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
duke@435	34
duke@435	35	#ifdef PRODUCT
duke@435	36	#define BLOCK_COMMENT(str) /* nothing */
duke@435	37	#else
duke@435	38	#define BLOCK_COMMENT(str) __ block_comment(str)
duke@435	39	#endif
duke@435	40
duke@435	41	#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
duke@435	42	const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
duke@435	43
duke@435	44	// Stub Code definitions
duke@435	45
duke@435	46	static address handle_unsafe_access() {
duke@435	47	JavaThread* thread = JavaThread::current();
duke@435	48	address pc = thread->saved_exception_pc();
duke@435	49	// pc is the instruction which we must emulate
duke@435	50	// doing a no-op is fine: return garbage from the load
duke@435	51	// therefore, compute npc
duke@435	52	address npc = Assembler::locate_next_instruction(pc);
duke@435	53
duke@435	54	// request an async exception
duke@435	55	thread->set_pending_unsafe_access_error();
duke@435	56
duke@435	57	// return address of next instruction to execute
duke@435	58	return npc;
duke@435	59	}
duke@435	60
duke@435	61	class StubGenerator: public StubCodeGenerator {
duke@435	62	private:
duke@435	63
duke@435	64	#ifdef PRODUCT
duke@435	65	#define inc_counter_np(counter) (0)
duke@435	66	#else
duke@435	67	void inc_counter_np_(int& counter) {
duke@435	68	__ incrementl(ExternalAddress((address)&counter));
duke@435	69	}
duke@435	70	#define inc_counter_np(counter) \
duke@435	71	BLOCK_COMMENT("inc_counter " #counter); \
duke@435	72	inc_counter_np_(counter);
duke@435	73	#endif
duke@435	74
duke@435	75	// Call stubs are used to call Java from C
duke@435	76	//
duke@435	77	// Linux Arguments:
duke@435	78	// c_rarg0: call wrapper address address
duke@435	79	// c_rarg1: result address
duke@435	80	// c_rarg2: result type BasicType
duke@435	81	// c_rarg3: method methodOop
duke@435	82	// c_rarg4: (interpreter) entry point address
duke@435	83	// c_rarg5: parameters intptr_t*
duke@435	84	// 16(rbp): parameter size (in words) int
duke@435	85	// 24(rbp): thread Thread*
duke@435	86	//
duke@435	87	// [ return_from_Java ] <--- rsp
duke@435	88	// [ argument word n ]
duke@435	89	// ...
duke@435	90	// -12 [ argument word 1 ]
duke@435	91	// -11 [ saved r15 ] <--- rsp_after_call
duke@435	92	// -10 [ saved r14 ]
duke@435	93	// -9 [ saved r13 ]
duke@435	94	// -8 [ saved r12 ]
duke@435	95	// -7 [ saved rbx ]
duke@435	96	// -6 [ call wrapper ]
duke@435	97	// -5 [ result ]
duke@435	98	// -4 [ result type ]
duke@435	99	// -3 [ method ]
duke@435	100	// -2 [ entry point ]
duke@435	101	// -1 [ parameters ]
duke@435	102	// 0 [ saved rbp ] <--- rbp
duke@435	103	// 1 [ return address ]
duke@435	104	// 2 [ parameter size ]
duke@435	105	// 3 [ thread ]
duke@435	106	//
duke@435	107	// Windows Arguments:
duke@435	108	// c_rarg0: call wrapper address address
duke@435	109	// c_rarg1: result address
duke@435	110	// c_rarg2: result type BasicType
duke@435	111	// c_rarg3: method methodOop
duke@435	112	// 48(rbp): (interpreter) entry point address
duke@435	113	// 56(rbp): parameters intptr_t*
duke@435	114	// 64(rbp): parameter size (in words) int
duke@435	115	// 72(rbp): thread Thread*
duke@435	116	//
duke@435	117	// [ return_from_Java ] <--- rsp
duke@435	118	// [ argument word n ]
duke@435	119	// ...
duke@435	120	// -8 [ argument word 1 ]
duke@435	121	// -7 [ saved r15 ] <--- rsp_after_call
duke@435	122	// -6 [ saved r14 ]
duke@435	123	// -5 [ saved r13 ]
duke@435	124	// -4 [ saved r12 ]
duke@435	125	// -3 [ saved rdi ]
duke@435	126	// -2 [ saved rsi ]
duke@435	127	// -1 [ saved rbx ]
duke@435	128	// 0 [ saved rbp ] <--- rbp
duke@435	129	// 1 [ return address ]
duke@435	130	// 2 [ call wrapper ]
duke@435	131	// 3 [ result ]
duke@435	132	// 4 [ result type ]
duke@435	133	// 5 [ method ]
duke@435	134	// 6 [ entry point ]
duke@435	135	// 7 [ parameters ]
duke@435	136	// 8 [ parameter size ]
duke@435	137	// 9 [ thread ]
duke@435	138	//
duke@435	139	// Windows reserves the callers stack space for arguments 1-4.
duke@435	140	// We spill c_rarg0-c_rarg3 to this space.
duke@435	141
duke@435	142	// Call stub stack layout word offsets from rbp
duke@435	143	enum call_stub_layout {
duke@435	144	#ifdef _WIN64
duke@435	145	rsp_after_call_off = -7,
duke@435	146	r15_off = rsp_after_call_off,
duke@435	147	r14_off = -6,
duke@435	148	r13_off = -5,
duke@435	149	r12_off = -4,
duke@435	150	rdi_off = -3,
duke@435	151	rsi_off = -2,
duke@435	152	rbx_off = -1,
duke@435	153	rbp_off = 0,
duke@435	154	retaddr_off = 1,
duke@435	155	call_wrapper_off = 2,
duke@435	156	result_off = 3,
duke@435	157	result_type_off = 4,
duke@435	158	method_off = 5,
duke@435	159	entry_point_off = 6,
duke@435	160	parameters_off = 7,
duke@435	161	parameter_size_off = 8,
duke@435	162	thread_off = 9
duke@435	163	#else
duke@435	164	rsp_after_call_off = -12,
duke@435	165	mxcsr_off = rsp_after_call_off,
duke@435	166	r15_off = -11,
duke@435	167	r14_off = -10,
duke@435	168	r13_off = -9,
duke@435	169	r12_off = -8,
duke@435	170	rbx_off = -7,
duke@435	171	call_wrapper_off = -6,
duke@435	172	result_off = -5,
duke@435	173	result_type_off = -4,
duke@435	174	method_off = -3,
duke@435	175	entry_point_off = -2,
duke@435	176	parameters_off = -1,
duke@435	177	rbp_off = 0,
duke@435	178	retaddr_off = 1,
duke@435	179	parameter_size_off = 2,
duke@435	180	thread_off = 3
duke@435	181	#endif
duke@435	182	};
duke@435	183
duke@435	184	address generate_call_stub(address& return_address) {
duke@435	185	assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
duke@435	186	(int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
duke@435	187	"adjust this code");
duke@435	188	StubCodeMark mark(this, "StubRoutines", "call_stub");
duke@435	189	address start = __ pc();
duke@435	190
duke@435	191	// same as in generate_catch_exception()!
duke@435	192	const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
duke@435	193
duke@435	194	const Address call_wrapper (rbp, call_wrapper_off * wordSize);
duke@435	195	const Address result (rbp, result_off * wordSize);
duke@435	196	const Address result_type (rbp, result_type_off * wordSize);
duke@435	197	const Address method (rbp, method_off * wordSize);
duke@435	198	const Address entry_point (rbp, entry_point_off * wordSize);
duke@435	199	const Address parameters (rbp, parameters_off * wordSize);
duke@435	200	const Address parameter_size(rbp, parameter_size_off * wordSize);
duke@435	201
duke@435	202	// same as in generate_catch_exception()!
duke@435	203	const Address thread (rbp, thread_off * wordSize);
duke@435	204
duke@435	205	const Address r15_save(rbp, r15_off * wordSize);
duke@435	206	const Address r14_save(rbp, r14_off * wordSize);
duke@435	207	const Address r13_save(rbp, r13_off * wordSize);
duke@435	208	const Address r12_save(rbp, r12_off * wordSize);
duke@435	209	const Address rbx_save(rbp, rbx_off * wordSize);
duke@435	210
duke@435	211	// stub code
duke@435	212	__ enter();
duke@435	213	__ subq(rsp, -rsp_after_call_off * wordSize);
duke@435	214
duke@435	215	// save register parameters
duke@435	216	#ifndef _WIN64
duke@435	217	__ movq(parameters, c_rarg5); // parameters
duke@435	218	__ movq(entry_point, c_rarg4); // entry_point
duke@435	219	#endif
duke@435	220
duke@435	221	__ movq(method, c_rarg3); // method
duke@435	222	__ movl(result_type, c_rarg2); // result type
duke@435	223	__ movq(result, c_rarg1); // result
duke@435	224	__ movq(call_wrapper, c_rarg0); // call wrapper
duke@435	225
duke@435	226	// save regs belonging to calling function
duke@435	227	__ movq(rbx_save, rbx);
duke@435	228	__ movq(r12_save, r12);
duke@435	229	__ movq(r13_save, r13);
duke@435	230	__ movq(r14_save, r14);
duke@435	231	__ movq(r15_save, r15);
duke@435	232
duke@435	233	#ifdef _WIN64
duke@435	234	const Address rdi_save(rbp, rdi_off * wordSize);
duke@435	235	const Address rsi_save(rbp, rsi_off * wordSize);
duke@435	236
duke@435	237	__ movq(rsi_save, rsi);
duke@435	238	__ movq(rdi_save, rdi);
duke@435	239	#else
duke@435	240	const Address mxcsr_save(rbp, mxcsr_off * wordSize);
duke@435	241	{
duke@435	242	Label skip_ldmx;
duke@435	243	__ stmxcsr(mxcsr_save);
duke@435	244	__ movl(rax, mxcsr_save);
duke@435	245	__ andl(rax, MXCSR_MASK); // Only check control and mask bits
duke@435	246	ExternalAddress mxcsr_std(StubRoutines::amd64::mxcsr_std());
duke@435	247	__ cmp32(rax, mxcsr_std);
duke@435	248	__ jcc(Assembler::equal, skip_ldmx);
duke@435	249	__ ldmxcsr(mxcsr_std);
duke@435	250	__ bind(skip_ldmx);
duke@435	251	}
duke@435	252	#endif
duke@435	253
duke@435	254	// Load up thread register
duke@435	255	__ movq(r15_thread, thread);
coleenp@548	256	__ reinit_heapbase();
duke@435	257
duke@435	258	#ifdef ASSERT
duke@435	259	// make sure we have no pending exceptions
duke@435	260	{
duke@435	261	Label L;
duke@435	262	__ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD);
duke@435	263	__ jcc(Assembler::equal, L);
duke@435	264	__ stop("StubRoutines::call_stub: entered with pending exception");
duke@435	265	__ bind(L);
duke@435	266	}
duke@435	267	#endif
duke@435	268
duke@435	269	// pass parameters if any
duke@435	270	BLOCK_COMMENT("pass parameters if any");
duke@435	271	Label parameters_done;
duke@435	272	__ movl(c_rarg3, parameter_size);
duke@435	273	__ testl(c_rarg3, c_rarg3);
duke@435	274	__ jcc(Assembler::zero, parameters_done);
duke@435	275
duke@435	276	Label loop;
duke@435	277	__ movq(c_rarg2, parameters); // parameter pointer
duke@435	278	__ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1
duke@435	279	__ BIND(loop);
duke@435	280	if (TaggedStackInterpreter) {
duke@435	281	__ movq(rax, Address(c_rarg2, 0)); // get tag
duke@435	282	__ addq(c_rarg2, wordSize); // advance to next tag
duke@435	283	__ pushq(rax); // pass tag
duke@435	284	}
duke@435	285	__ movq(rax, Address(c_rarg2, 0)); // get parameter
duke@435	286	__ addq(c_rarg2, wordSize); // advance to next parameter
duke@435	287	__ decrementl(c_rarg1); // decrement counter
duke@435	288	__ pushq(rax); // pass parameter
duke@435	289	__ jcc(Assembler::notZero, loop);
duke@435	290
duke@435	291	// call Java function
duke@435	292	__ BIND(parameters_done);
duke@435	293	__ movq(rbx, method); // get methodOop
duke@435	294	__ movq(c_rarg1, entry_point); // get entry_point
duke@435	295	__ movq(r13, rsp); // set sender sp
duke@435	296	BLOCK_COMMENT("call Java function");
duke@435	297	__ call(c_rarg1);
duke@435	298
duke@435	299	BLOCK_COMMENT("call_stub_return_address:");
duke@435	300	return_address = __ pc();
duke@435	301
duke@435	302	// store result depending on type (everything that is not
duke@435	303	// T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
duke@435	304	__ movq(c_rarg0, result);
duke@435	305	Label is_long, is_float, is_double, exit;
duke@435	306	__ movl(c_rarg1, result_type);
duke@435	307	__ cmpl(c_rarg1, T_OBJECT);
duke@435	308	__ jcc(Assembler::equal, is_long);
duke@435	309	__ cmpl(c_rarg1, T_LONG);
duke@435	310	__ jcc(Assembler::equal, is_long);
duke@435	311	__ cmpl(c_rarg1, T_FLOAT);
duke@435	312	__ jcc(Assembler::equal, is_float);
duke@435	313	__ cmpl(c_rarg1, T_DOUBLE);
duke@435	314	__ jcc(Assembler::equal, is_double);
duke@435	315
duke@435	316	// handle T_INT case
duke@435	317	__ movl(Address(c_rarg0, 0), rax);
duke@435	318
duke@435	319	__ BIND(exit);
duke@435	320
duke@435	321	// pop parameters
duke@435	322	__ leaq(rsp, rsp_after_call);
duke@435	323
duke@435	324	#ifdef ASSERT
duke@435	325	// verify that threads correspond
duke@435	326	{
duke@435	327	Label L, S;
duke@435	328	__ cmpq(r15_thread, thread);
duke@435	329	__ jcc(Assembler::notEqual, S);
duke@435	330	__ get_thread(rbx);
duke@435	331	__ cmpq(r15_thread, rbx);
duke@435	332	__ jcc(Assembler::equal, L);
duke@435	333	__ bind(S);
duke@435	334	__ jcc(Assembler::equal, L);
duke@435	335	__ stop("StubRoutines::call_stub: threads must correspond");
duke@435	336	__ bind(L);
duke@435	337	}
duke@435	338	#endif
duke@435	339
duke@435	340	// restore regs belonging to calling function
duke@435	341	__ movq(r15, r15_save);
duke@435	342	__ movq(r14, r14_save);
duke@435	343	__ movq(r13, r13_save);
duke@435	344	__ movq(r12, r12_save);
duke@435	345	__ movq(rbx, rbx_save);
duke@435	346
duke@435	347	#ifdef _WIN64
duke@435	348	__ movq(rdi, rdi_save);
duke@435	349	__ movq(rsi, rsi_save);
duke@435	350	#else
duke@435	351	__ ldmxcsr(mxcsr_save);
duke@435	352	#endif
duke@435	353
duke@435	354	// restore rsp
duke@435	355	__ addq(rsp, -rsp_after_call_off * wordSize);
duke@435	356
duke@435	357	// return
duke@435	358	__ popq(rbp);
duke@435	359	__ ret(0);
duke@435	360
duke@435	361	// handle return types different from T_INT
duke@435	362	__ BIND(is_long);
duke@435	363	__ movq(Address(c_rarg0, 0), rax);
duke@435	364	__ jmp(exit);
duke@435	365
duke@435	366	__ BIND(is_float);
duke@435	367	__ movflt(Address(c_rarg0, 0), xmm0);
duke@435	368	__ jmp(exit);
duke@435	369
duke@435	370	__ BIND(is_double);
duke@435	371	__ movdbl(Address(c_rarg0, 0), xmm0);
duke@435	372	__ jmp(exit);
duke@435	373
duke@435	374	return start;
duke@435	375	}
duke@435	376
duke@435	377	// Return point for a Java call if there's an exception thrown in
duke@435	378	// Java code. The exception is caught and transformed into a
duke@435	379	// pending exception stored in JavaThread that can be tested from
duke@435	380	// within the VM.
duke@435	381	//
duke@435	382	// Note: Usually the parameters are removed by the callee. In case
duke@435	383	// of an exception crossing an activation frame boundary, that is
duke@435	384	// not the case if the callee is compiled code => need to setup the
duke@435	385	// rsp.
duke@435	386	//
duke@435	387	// rax: exception oop
duke@435	388
duke@435	389	address generate_catch_exception() {
duke@435	390	StubCodeMark mark(this, "StubRoutines", "catch_exception");
duke@435	391	address start = __ pc();
duke@435	392
duke@435	393	// same as in generate_call_stub():
duke@435	394	const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
duke@435	395	const Address thread (rbp, thread_off * wordSize);
duke@435	396
duke@435	397	#ifdef ASSERT
duke@435	398	// verify that threads correspond
duke@435	399	{
duke@435	400	Label L, S;
duke@435	401	__ cmpq(r15_thread, thread);
duke@435	402	__ jcc(Assembler::notEqual, S);
duke@435	403	__ get_thread(rbx);
duke@435	404	__ cmpq(r15_thread, rbx);
duke@435	405	__ jcc(Assembler::equal, L);
duke@435	406	__ bind(S);
duke@435	407	__ stop("StubRoutines::catch_exception: threads must correspond");
duke@435	408	__ bind(L);
duke@435	409	}
duke@435	410	#endif
duke@435	411
duke@435	412	// set pending exception
duke@435	413	__ verify_oop(rax);
duke@435	414
duke@435	415	__ movq(Address(r15_thread, Thread::pending_exception_offset()), rax);
duke@435	416	__ lea(rscratch1, ExternalAddress((address)__FILE__));
duke@435	417	__ movq(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
duke@435	418	__ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__);
duke@435	419
duke@435	420	// complete return to VM
duke@435	421	assert(StubRoutines::_call_stub_return_address != NULL,
duke@435	422	"_call_stub_return_address must have been generated before");
duke@435	423	__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
duke@435	424
duke@435	425	return start;
duke@435	426	}
duke@435	427
duke@435	428	// Continuation point for runtime calls returning with a pending
duke@435	429	// exception. The pending exception check happened in the runtime
duke@435	430	// or native call stub. The pending exception in Thread is
duke@435	431	// converted into a Java-level exception.
duke@435	432	//
duke@435	433	// Contract with Java-level exception handlers:
duke@435	434	// rax: exception
duke@435	435	// rdx: throwing pc
duke@435	436	//
duke@435	437	// NOTE: At entry of this stub, exception-pc must be on stack !!
duke@435	438
duke@435	439	address generate_forward_exception() {
duke@435	440	StubCodeMark mark(this, "StubRoutines", "forward exception");
duke@435	441	address start = __ pc();
duke@435	442
duke@435	443	// Upon entry, the sp points to the return address returning into
duke@435	444	// Java (interpreted or compiled) code; i.e., the return address
duke@435	445	// becomes the throwing pc.
duke@435	446	//
duke@435	447	// Arguments pushed before the runtime call are still on the stack
duke@435	448	// but the exception handler will reset the stack pointer ->
duke@435	449	// ignore them. A potential result in registers can be ignored as
duke@435	450	// well.
duke@435	451
duke@435	452	#ifdef ASSERT
duke@435	453	// make sure this code is only executed if there is a pending exception
duke@435	454	{
duke@435	455	Label L;
duke@435	456	__ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL);
duke@435	457	__ jcc(Assembler::notEqual, L);
duke@435	458	__ stop("StubRoutines::forward exception: no pending exception (1)");
duke@435	459	__ bind(L);
duke@435	460	}
duke@435	461	#endif
duke@435	462
duke@435	463	// compute exception handler into rbx
duke@435	464	__ movq(c_rarg0, Address(rsp, 0));
duke@435	465	BLOCK_COMMENT("call exception_handler_for_return_address");
duke@435	466	__ call_VM_leaf(CAST_FROM_FN_PTR(address,
duke@435	467	SharedRuntime::exception_handler_for_return_address),
duke@435	468	c_rarg0);
duke@435	469	__ movq(rbx, rax);
duke@435	470
duke@435	471	// setup rax & rdx, remove return address & clear pending exception
duke@435	472	__ popq(rdx);
duke@435	473	__ movq(rax, Address(r15_thread, Thread::pending_exception_offset()));
duke@435	474	__ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD);
duke@435	475
duke@435	476	#ifdef ASSERT
duke@435	477	// make sure exception is set
duke@435	478	{
duke@435	479	Label L;
duke@435	480	__ testq(rax, rax);
duke@435	481	__ jcc(Assembler::notEqual, L);
duke@435	482	__ stop("StubRoutines::forward exception: no pending exception (2)");
duke@435	483	__ bind(L);
duke@435	484	}
duke@435	485	#endif
duke@435	486
duke@435	487	// continue at exception handler (return address removed)
duke@435	488	// rax: exception
duke@435	489	// rbx: exception handler
duke@435	490	// rdx: throwing pc
duke@435	491	__ verify_oop(rax);
duke@435	492	__ jmp(rbx);
duke@435	493
duke@435	494	return start;
duke@435	495	}
duke@435	496
duke@435	497	// Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
duke@435	498	//
duke@435	499	// Arguments :
duke@435	500	// c_rarg0: exchange_value
duke@435	501	// c_rarg0: dest
duke@435	502	//
duke@435	503	// Result:
duke@435	504	// *dest <- ex, return (orig *dest)
duke@435	505	address generate_atomic_xchg() {
duke@435	506	StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
duke@435	507	address start = __ pc();
duke@435	508
duke@435	509	__ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
duke@435	510	__ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
duke@435	511	__ ret(0);
duke@435	512
duke@435	513	return start;
duke@435	514	}
duke@435	515
duke@435	516	// Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
duke@435	517	//
duke@435	518	// Arguments :
duke@435	519	// c_rarg0: exchange_value
duke@435	520	// c_rarg1: dest
duke@435	521	//
duke@435	522	// Result:
duke@435	523	// *dest <- ex, return (orig *dest)
duke@435	524	address generate_atomic_xchg_ptr() {
duke@435	525	StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
duke@435	526	address start = __ pc();
duke@435	527
duke@435	528	__ movq(rax, c_rarg0); // Copy to eax we need a return value anyhow
duke@435	529	__ xchgq(rax, Address(c_rarg1, 0)); // automatic LOCK
duke@435	530	__ ret(0);
duke@435	531
duke@435	532	return start;
duke@435	533	}
duke@435	534
duke@435	535	// Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
duke@435	536	// jint compare_value)
duke@435	537	//
duke@435	538	// Arguments :
duke@435	539	// c_rarg0: exchange_value
duke@435	540	// c_rarg1: dest
duke@435	541	// c_rarg2: compare_value
duke@435	542	//
duke@435	543	// Result:
duke@435	544	// if ( compare_value == *dest ) {
duke@435	545	// *dest = exchange_value
duke@435	546	// return compare_value;
duke@435	547	// else
duke@435	548	// return *dest;
duke@435	549	address generate_atomic_cmpxchg() {
duke@435	550	StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
duke@435	551	address start = __ pc();
duke@435	552
duke@435	553	__ movl(rax, c_rarg2);
duke@435	554	if ( os::is_MP() ) __ lock();
duke@435	555	__ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
duke@435	556	__ ret(0);
duke@435	557
duke@435	558	return start;
duke@435	559	}
duke@435	560
duke@435	561	// Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
duke@435	562	// volatile jlong* dest,
duke@435	563	// jlong compare_value)
duke@435	564	// Arguments :
duke@435	565	// c_rarg0: exchange_value
duke@435	566	// c_rarg1: dest
duke@435	567	// c_rarg2: compare_value
duke@435	568	//
duke@435	569	// Result:
duke@435	570	// if ( compare_value == *dest ) {
duke@435	571	// *dest = exchange_value
duke@435	572	// return compare_value;
duke@435	573	// else
duke@435	574	// return *dest;
duke@435	575	address generate_atomic_cmpxchg_long() {
duke@435	576	StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
duke@435	577	address start = __ pc();
duke@435	578
duke@435	579	__ movq(rax, c_rarg2);
duke@435	580	if ( os::is_MP() ) __ lock();
duke@435	581	__ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
duke@435	582	__ ret(0);
duke@435	583
duke@435	584	return start;
duke@435	585	}
duke@435	586
duke@435	587	// Support for jint atomic::add(jint add_value, volatile jint* dest)
duke@435	588	//
duke@435	589	// Arguments :
duke@435	590	// c_rarg0: add_value
duke@435	591	// c_rarg1: dest
duke@435	592	//
duke@435	593	// Result:
duke@435	594	// *dest += add_value
duke@435	595	// return *dest;
duke@435	596	address generate_atomic_add() {
duke@435	597	StubCodeMark mark(this, "StubRoutines", "atomic_add");
duke@435	598	address start = __ pc();
duke@435	599
duke@435	600	__ movl(rax, c_rarg0);
duke@435	601	if ( os::is_MP() ) __ lock();
duke@435	602	__ xaddl(Address(c_rarg1, 0), c_rarg0);
duke@435	603	__ addl(rax, c_rarg0);
duke@435	604	__ ret(0);
duke@435	605
duke@435	606	return start;
duke@435	607	}
duke@435	608
duke@435	609	// Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
duke@435	610	//
duke@435	611	// Arguments :
duke@435	612	// c_rarg0: add_value
duke@435	613	// c_rarg1: dest
duke@435	614	//
duke@435	615	// Result:
duke@435	616	// *dest += add_value
duke@435	617	// return *dest;
duke@435	618	address generate_atomic_add_ptr() {
duke@435	619	StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
duke@435	620	address start = __ pc();
duke@435	621
duke@435	622	__ movq(rax, c_rarg0); // Copy to eax we need a return value anyhow
duke@435	623	if ( os::is_MP() ) __ lock();
duke@435	624	__ xaddl(Address(c_rarg1, 0), c_rarg0);
duke@435	625	__ addl(rax, c_rarg0);
duke@435	626	__ ret(0);
duke@435	627
duke@435	628	return start;
duke@435	629	}
duke@435	630
duke@435	631	// Support for intptr_t OrderAccess::fence()
duke@435	632	//
duke@435	633	// Arguments :
duke@435	634	//
duke@435	635	// Result:
duke@435	636	address generate_orderaccess_fence() {
duke@435	637	StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
duke@435	638	address start = __ pc();
duke@435	639	__ mfence();
duke@435	640	__ ret(0);
duke@435	641
duke@435	642	return start;
duke@435	643	}
duke@435	644
duke@435	645	// Support for intptr_t get_previous_fp()
duke@435	646	//
duke@435	647	// This routine is used to find the previous frame pointer for the
duke@435	648	// caller (current_frame_guess). This is used as part of debugging
duke@435	649	// ps() is seemingly lost trying to find frames.
duke@435	650	// This code assumes that caller current_frame_guess) has a frame.
duke@435	651	address generate_get_previous_fp() {
duke@435	652	StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
duke@435	653	const Address old_fp(rbp, 0);
duke@435	654	const Address older_fp(rax, 0);
duke@435	655	address start = __ pc();
duke@435	656
duke@435	657	__ enter();
duke@435	658	__ movq(rax, old_fp); // callers fp
duke@435	659	__ movq(rax, older_fp); // the frame for ps()
duke@435	660	__ popq(rbp);
duke@435	661	__ ret(0);
duke@435	662
duke@435	663	return start;
duke@435	664	}
duke@435	665
duke@435	666	//----------------------------------------------------------------------------------------------------
duke@435	667	// Support for void verify_mxcsr()
duke@435	668	//
duke@435	669	// This routine is used with -Xcheck:jni to verify that native
duke@435	670	// JNI code does not return to Java code without restoring the
duke@435	671	// MXCSR register to our expected state.
duke@435	672
duke@435	673	address generate_verify_mxcsr() {
duke@435	674	StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
duke@435	675	address start = __ pc();
duke@435	676
duke@435	677	const Address mxcsr_save(rsp, 0);
duke@435	678
duke@435	679	if (CheckJNICalls) {
duke@435	680	Label ok_ret;
duke@435	681	__ pushq(rax);
duke@435	682	__ subq(rsp, wordSize); // allocate a temp location
duke@435	683	__ stmxcsr(mxcsr_save);
duke@435	684	__ movl(rax, mxcsr_save);
duke@435	685	__ andl(rax, MXCSR_MASK); // Only check control and mask bits
duke@435	686	__ cmpl(rax, *(int *)(StubRoutines::amd64::mxcsr_std()));
duke@435	687	__ jcc(Assembler::equal, ok_ret);
duke@435	688
duke@435	689	__ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
duke@435	690
duke@435	691	__ ldmxcsr(ExternalAddress(StubRoutines::amd64::mxcsr_std()));
duke@435	692
duke@435	693	__ bind(ok_ret);
duke@435	694	__ addq(rsp, wordSize);
duke@435	695	__ popq(rax);
duke@435	696	}
duke@435	697
duke@435	698	__ ret(0);
duke@435	699
duke@435	700	return start;
duke@435	701	}
duke@435	702
duke@435	703	address generate_f2i_fixup() {
duke@435	704	StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
duke@435	705	Address inout(rsp, 5 * wordSize); // return address + 4 saves
duke@435	706
duke@435	707	address start = __ pc();
duke@435	708
duke@435	709	Label L;
duke@435	710
duke@435	711	__ pushq(rax);
duke@435	712	__ pushq(c_rarg3);
duke@435	713	__ pushq(c_rarg2);
duke@435	714	__ pushq(c_rarg1);
duke@435	715
duke@435	716	__ movl(rax, 0x7f800000);
duke@435	717	__ xorl(c_rarg3, c_rarg3);
duke@435	718	__ movl(c_rarg2, inout);
duke@435	719	__ movl(c_rarg1, c_rarg2);
duke@435	720	__ andl(c_rarg1, 0x7fffffff);
duke@435	721	__ cmpl(rax, c_rarg1); // NaN? -> 0
duke@435	722	__ jcc(Assembler::negative, L);
duke@435	723	__ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
duke@435	724	__ movl(c_rarg3, 0x80000000);
duke@435	725	__ movl(rax, 0x7fffffff);
duke@435	726	__ cmovl(Assembler::positive, c_rarg3, rax);
duke@435	727
duke@435	728	__ bind(L);
duke@435	729	__ movq(inout, c_rarg3);
duke@435	730
duke@435	731	__ popq(c_rarg1);
duke@435	732	__ popq(c_rarg2);
duke@435	733	__ popq(c_rarg3);
duke@435	734	__ popq(rax);
duke@435	735
duke@435	736	__ ret(0);
duke@435	737
duke@435	738	return start;
duke@435	739	}
duke@435	740
duke@435	741	address generate_f2l_fixup() {
duke@435	742	StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
duke@435	743	Address inout(rsp, 5 * wordSize); // return address + 4 saves
duke@435	744	address start = __ pc();
duke@435	745
duke@435	746	Label L;
duke@435	747
duke@435	748	__ pushq(rax);
duke@435	749	__ pushq(c_rarg3);
duke@435	750	__ pushq(c_rarg2);
duke@435	751	__ pushq(c_rarg1);
duke@435	752
duke@435	753	__ movl(rax, 0x7f800000);
duke@435	754	__ xorl(c_rarg3, c_rarg3);
duke@435	755	__ movl(c_rarg2, inout);
duke@435	756	__ movl(c_rarg1, c_rarg2);
duke@435	757	__ andl(c_rarg1, 0x7fffffff);
duke@435	758	__ cmpl(rax, c_rarg1); // NaN? -> 0
duke@435	759	__ jcc(Assembler::negative, L);
duke@435	760	__ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
duke@435	761	__ mov64(c_rarg3, 0x8000000000000000);
duke@435	762	__ mov64(rax, 0x7fffffffffffffff);
duke@435	763	__ cmovq(Assembler::positive, c_rarg3, rax);
duke@435	764
duke@435	765	__ bind(L);
duke@435	766	__ movq(inout, c_rarg3);
duke@435	767
duke@435	768	__ popq(c_rarg1);
duke@435	769	__ popq(c_rarg2);
duke@435	770	__ popq(c_rarg3);
duke@435	771	__ popq(rax);
duke@435	772
duke@435	773	__ ret(0);
duke@435	774
duke@435	775	return start;
duke@435	776	}
duke@435	777
duke@435	778	address generate_d2i_fixup() {
duke@435	779	StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
duke@435	780	Address inout(rsp, 6 * wordSize); // return address + 5 saves
duke@435	781
duke@435	782	address start = __ pc();
duke@435	783
duke@435	784	Label L;
duke@435	785
duke@435	786	__ pushq(rax);
duke@435	787	__ pushq(c_rarg3);
duke@435	788	__ pushq(c_rarg2);
duke@435	789	__ pushq(c_rarg1);
duke@435	790	__ pushq(c_rarg0);
duke@435	791
duke@435	792	__ movl(rax, 0x7ff00000);
duke@435	793	__ movq(c_rarg2, inout);
duke@435	794	__ movl(c_rarg3, c_rarg2);
duke@435	795	__ movq(c_rarg1, c_rarg2);
duke@435	796	__ movq(c_rarg0, c_rarg2);
duke@435	797	__ negl(c_rarg3);
duke@435	798	__ shrq(c_rarg1, 0x20);
duke@435	799	__ orl(c_rarg3, c_rarg2);
duke@435	800	__ andl(c_rarg1, 0x7fffffff);
duke@435	801	__ xorl(c_rarg2, c_rarg2);
duke@435	802	__ shrl(c_rarg3, 0x1f);
duke@435	803	__ orl(c_rarg1, c_rarg3);
duke@435	804	__ cmpl(rax, c_rarg1);
duke@435	805	__ jcc(Assembler::negative, L); // NaN -> 0
duke@435	806	__ testq(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
duke@435	807	__ movl(c_rarg2, 0x80000000);
duke@435	808	__ movl(rax, 0x7fffffff);
duke@435	809	__ cmovl(Assembler::positive, c_rarg2, rax);
duke@435	810
duke@435	811	__ bind(L);
duke@435	812	__ movq(inout, c_rarg2);
duke@435	813
duke@435	814	__ popq(c_rarg0);
duke@435	815	__ popq(c_rarg1);
duke@435	816	__ popq(c_rarg2);
duke@435	817	__ popq(c_rarg3);
duke@435	818	__ popq(rax);
duke@435	819
duke@435	820	__ ret(0);
duke@435	821
duke@435	822	return start;
duke@435	823	}
duke@435	824
duke@435	825	address generate_d2l_fixup() {
duke@435	826	StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
duke@435	827	Address inout(rsp, 6 * wordSize); // return address + 5 saves
duke@435	828
duke@435	829	address start = __ pc();
duke@435	830
duke@435	831	Label L;
duke@435	832
duke@435	833	__ pushq(rax);
duke@435	834	__ pushq(c_rarg3);
duke@435	835	__ pushq(c_rarg2);
duke@435	836	__ pushq(c_rarg1);
duke@435	837	__ pushq(c_rarg0);
duke@435	838
duke@435	839	__ movl(rax, 0x7ff00000);
duke@435	840	__ movq(c_rarg2, inout);
duke@435	841	__ movl(c_rarg3, c_rarg2);
duke@435	842	__ movq(c_rarg1, c_rarg2);
duke@435	843	__ movq(c_rarg0, c_rarg2);
duke@435	844	__ negl(c_rarg3);
duke@435	845	__ shrq(c_rarg1, 0x20);
duke@435	846	__ orl(c_rarg3, c_rarg2);
duke@435	847	__ andl(c_rarg1, 0x7fffffff);
duke@435	848	__ xorl(c_rarg2, c_rarg2);
duke@435	849	__ shrl(c_rarg3, 0x1f);
duke@435	850	__ orl(c_rarg1, c_rarg3);
duke@435	851	__ cmpl(rax, c_rarg1);
duke@435	852	__ jcc(Assembler::negative, L); // NaN -> 0
duke@435	853	__ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
duke@435	854	__ mov64(c_rarg2, 0x8000000000000000);
duke@435	855	__ mov64(rax, 0x7fffffffffffffff);
duke@435	856	__ cmovq(Assembler::positive, c_rarg2, rax);
duke@435	857
duke@435	858	__ bind(L);
duke@435	859	__ movq(inout, c_rarg2);
duke@435	860
duke@435	861	__ popq(c_rarg0);
duke@435	862	__ popq(c_rarg1);
duke@435	863	__ popq(c_rarg2);
duke@435	864	__ popq(c_rarg3);
duke@435	865	__ popq(rax);
duke@435	866
duke@435	867	__ ret(0);
duke@435	868
duke@435	869	return start;
duke@435	870	}
duke@435	871
duke@435	872	address generate_fp_mask(const char *stub_name, int64_t mask) {
duke@435	873	StubCodeMark mark(this, "StubRoutines", stub_name);
duke@435	874
duke@435	875	__ align(16);
duke@435	876	address start = __ pc();
duke@435	877
duke@435	878	__ emit_data64( mask, relocInfo::none );
duke@435	879	__ emit_data64( mask, relocInfo::none );
duke@435	880
duke@435	881	return start;
duke@435	882	}
duke@435	883
duke@435	884	// The following routine generates a subroutine to throw an
duke@435	885	// asynchronous UnknownError when an unsafe access gets a fault that
duke@435	886	// could not be reasonably prevented by the programmer. (Example:
duke@435	887	// SIGBUS/OBJERR.)
duke@435	888	address generate_handler_for_unsafe_access() {
duke@435	889	StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
duke@435	890	address start = __ pc();
duke@435	891
duke@435	892	__ pushq(0); // hole for return address-to-be
duke@435	893	__ pushaq(); // push registers
duke@435	894	Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
duke@435	895
duke@435	896	__ subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	897	BLOCK_COMMENT("call handle_unsafe_access");
duke@435	898	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
duke@435	899	__ addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	900
duke@435	901	__ movq(next_pc, rax); // stuff next address
duke@435	902	__ popaq();
duke@435	903	__ ret(0); // jump to next address
duke@435	904
duke@435	905	return start;
duke@435	906	}
duke@435	907
duke@435	908	// Non-destructive plausibility checks for oops
duke@435	909	//
duke@435	910	// Arguments:
duke@435	911	// all args on stack!
duke@435	912	//
duke@435	913	// Stack after saving c_rarg3:
duke@435	914	// [tos + 0]: saved c_rarg3
duke@435	915	// [tos + 1]: saved c_rarg2
kvn@559	916	// [tos + 2]: saved r12 (several TemplateTable methods use it)
kvn@559	917	// [tos + 3]: saved flags
kvn@559	918	// [tos + 4]: return address
kvn@559	919	// * [tos + 5]: error message (char*)
kvn@559	920	// * [tos + 6]: object to verify (oop)
kvn@559	921	// * [tos + 7]: saved rax - saved by caller and bashed
duke@435	922	// * = popped on exit
duke@435	923	address generate_verify_oop() {
duke@435	924	StubCodeMark mark(this, "StubRoutines", "verify_oop");
duke@435	925	address start = __ pc();
duke@435	926
duke@435	927	Label exit, error;
duke@435	928
duke@435	929	__ pushfq();
duke@435	930	__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
duke@435	931
kvn@559	932	__ pushq(r12);
kvn@559	933
duke@435	934	// save c_rarg2 and c_rarg3
duke@435	935	__ pushq(c_rarg2);
duke@435	936	__ pushq(c_rarg3);
duke@435	937
kvn@559	938	enum {
kvn@559	939	// After previous pushes.
kvn@559	940	oop_to_verify = 6 * wordSize,
kvn@559	941	saved_rax = 7 * wordSize,
kvn@559	942
kvn@559	943	// Before the call to MacroAssembler::debug(), see below.
kvn@559	944	return_addr = 16 * wordSize,
kvn@559	945	error_msg = 17 * wordSize
kvn@559	946	};
kvn@559	947
duke@435	948	// get object
kvn@559	949	__ movq(rax, Address(rsp, oop_to_verify));
duke@435	950
duke@435	951	// make sure object is 'reasonable'
duke@435	952	__ testq(rax, rax);
duke@435	953	__ jcc(Assembler::zero, exit); // if obj is NULL it is OK
duke@435	954	// Check if the oop is in the right area of memory
duke@435	955	__ movq(c_rarg2, rax);
duke@435	956	__ movptr(c_rarg3, (int64_t) Universe::verify_oop_mask());
duke@435	957	__ andq(c_rarg2, c_rarg3);
duke@435	958	__ movptr(c_rarg3, (int64_t) Universe::verify_oop_bits());
duke@435	959	__ cmpq(c_rarg2, c_rarg3);
duke@435	960	__ jcc(Assembler::notZero, error);
duke@435	961
kvn@559	962	// set r12 to heapbase for load_klass()
kvn@559	963	__ reinit_heapbase();
kvn@559	964
duke@435	965	// make sure klass is 'reasonable'
coleenp@548	966	__ load_klass(rax, rax); // get klass
duke@435	967	__ testq(rax, rax);
duke@435	968	__ jcc(Assembler::zero, error); // if klass is NULL it is broken
duke@435	969	// Check if the klass is in the right area of memory
duke@435	970	__ movq(c_rarg2, rax);
duke@435	971	__ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask());
duke@435	972	__ andq(c_rarg2, c_rarg3);
duke@435	973	__ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits());
duke@435	974	__ cmpq(c_rarg2, c_rarg3);
duke@435	975	__ jcc(Assembler::notZero, error);
duke@435	976
duke@435	977	// make sure klass' klass is 'reasonable'
coleenp@548	978	__ load_klass(rax, rax);
duke@435	979	__ testq(rax, rax);
duke@435	980	__ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
duke@435	981	// Check if the klass' klass is in the right area of memory
duke@435	982	__ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask());
duke@435	983	__ andq(rax, c_rarg3);
duke@435	984	__ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits());
duke@435	985	__ cmpq(rax, c_rarg3);
duke@435	986	__ jcc(Assembler::notZero, error);
duke@435	987
duke@435	988	// return if everything seems ok
duke@435	989	__ bind(exit);
kvn@559	990	__ movq(rax, Address(rsp, saved_rax)); // get saved rax back
kvn@559	991	__ popq(c_rarg3); // restore c_rarg3
kvn@559	992	__ popq(c_rarg2); // restore c_rarg2
kvn@559	993	__ popq(r12); // restore r12
duke@435	994	__ popfq(); // restore flags
duke@435	995	__ ret(3 * wordSize); // pop caller saved stuff
duke@435	996
duke@435	997	// handle errors
duke@435	998	__ bind(error);
kvn@559	999	__ movq(rax, Address(rsp, saved_rax)); // get saved rax back
kvn@559	1000	__ popq(c_rarg3); // get saved c_rarg3 back
kvn@559	1001	__ popq(c_rarg2); // get saved c_rarg2 back
kvn@559	1002	__ popq(r12); // get saved r12 back
duke@435	1003	__ popfq(); // get saved flags off stack --
duke@435	1004	// will be ignored
duke@435	1005
duke@435	1006	__ pushaq(); // push registers
duke@435	1007	// (rip is already
duke@435	1008	// already pushed)
kvn@559	1009	// debug(char* msg, int64_t pc, int64_t regs[])
duke@435	1010	// We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
duke@435	1011	// pushed all the registers, so now the stack looks like:
duke@435	1012	// [tos + 0] 16 saved registers
duke@435	1013	// [tos + 16] return address
kvn@559	1014	// * [tos + 17] error message (char*)
kvn@559	1015	// * [tos + 18] object to verify (oop)
kvn@559	1016	// * [tos + 19] saved rax - saved by caller and bashed
kvn@559	1017	// * = popped on exit
kvn@559	1018
kvn@559	1019	__ movq(c_rarg0, Address(rsp, error_msg)); // pass address of error message
kvn@559	1020	__ movq(c_rarg1, Address(rsp, return_addr)); // pass return address
kvn@559	1021	__ movq(c_rarg2, rsp); // pass address of regs on stack
duke@435	1022	__ movq(r12, rsp); // remember rsp
duke@435	1023	__ subq(rsp, frame::arg_reg_save_area_bytes);// windows
duke@435	1024	__ andq(rsp, -16); // align stack as required by ABI
duke@435	1025	BLOCK_COMMENT("call MacroAssembler::debug");
duke@435	1026	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
duke@435	1027	__ movq(rsp, r12); // restore rsp
kvn@559	1028	__ popaq(); // pop registers (includes r12)
duke@435	1029	__ ret(3 * wordSize); // pop caller saved stuff
duke@435	1030
duke@435	1031	return start;
duke@435	1032	}
duke@435	1033
duke@435	1034	static address disjoint_byte_copy_entry;
duke@435	1035	static address disjoint_short_copy_entry;
duke@435	1036	static address disjoint_int_copy_entry;
duke@435	1037	static address disjoint_long_copy_entry;
duke@435	1038	static address disjoint_oop_copy_entry;
duke@435	1039
duke@435	1040	static address byte_copy_entry;
duke@435	1041	static address short_copy_entry;
duke@435	1042	static address int_copy_entry;
duke@435	1043	static address long_copy_entry;
duke@435	1044	static address oop_copy_entry;
duke@435	1045
duke@435	1046	static address checkcast_copy_entry;
duke@435	1047
duke@435	1048	//
duke@435	1049	// Verify that a register contains clean 32-bits positive value
duke@435	1050	// (high 32-bits are 0) so it could be used in 64-bits shifts.
duke@435	1051	//
duke@435	1052	// Input:
duke@435	1053	// Rint - 32-bits value
duke@435	1054	// Rtmp - scratch
duke@435	1055	//
duke@435	1056	void assert_clean_int(Register Rint, Register Rtmp) {
duke@435	1057	#ifdef ASSERT
duke@435	1058	Label L;
duke@435	1059	assert_different_registers(Rtmp, Rint);
duke@435	1060	__ movslq(Rtmp, Rint);
duke@435	1061	__ cmpq(Rtmp, Rint);
kvn@559	1062	__ jcc(Assembler::equal, L);
duke@435	1063	__ stop("high 32-bits of int value are not 0");
duke@435	1064	__ bind(L);
duke@435	1065	#endif
duke@435	1066	}
duke@435	1067
duke@435	1068	// Generate overlap test for array copy stubs
duke@435	1069	//
duke@435	1070	// Input:
duke@435	1071	// c_rarg0 - from
duke@435	1072	// c_rarg1 - to
duke@435	1073	// c_rarg2 - element count
duke@435	1074	//
duke@435	1075	// Output:
duke@435	1076	// rax - &from[element count - 1]
duke@435	1077	//
duke@435	1078	void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
duke@435	1079	assert(no_overlap_target != NULL, "must be generated");
duke@435	1080	array_overlap_test(no_overlap_target, NULL, sf);
duke@435	1081	}
duke@435	1082	void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
duke@435	1083	array_overlap_test(NULL, &L_no_overlap, sf);
duke@435	1084	}
duke@435	1085	void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
duke@435	1086	const Register from = c_rarg0;
duke@435	1087	const Register to = c_rarg1;
duke@435	1088	const Register count = c_rarg2;
duke@435	1089	const Register end_from = rax;
duke@435	1090
duke@435	1091	__ cmpq(to, from);
duke@435	1092	__ leaq(end_from, Address(from, count, sf, 0));
duke@435	1093	if (NOLp == NULL) {
duke@435	1094	ExternalAddress no_overlap(no_overlap_target);
duke@435	1095	__ jump_cc(Assembler::belowEqual, no_overlap);
duke@435	1096	__ cmpq(to, end_from);
duke@435	1097	__ jump_cc(Assembler::aboveEqual, no_overlap);
duke@435	1098	} else {
duke@435	1099	__ jcc(Assembler::belowEqual, (*NOLp));
duke@435	1100	__ cmpq(to, end_from);
duke@435	1101	__ jcc(Assembler::aboveEqual, (*NOLp));
duke@435	1102	}
duke@435	1103	}
duke@435	1104
duke@435	1105	// Shuffle first three arg regs on Windows into Linux/Solaris locations.
duke@435	1106	//
duke@435	1107	// Outputs:
duke@435	1108	// rdi - rcx
duke@435	1109	// rsi - rdx
duke@435	1110	// rdx - r8
duke@435	1111	// rcx - r9
duke@435	1112	//
duke@435	1113	// Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
duke@435	1114	// are non-volatile. r9 and r10 should not be used by the caller.
duke@435	1115	//
duke@435	1116	void setup_arg_regs(int nargs = 3) {
duke@435	1117	const Register saved_rdi = r9;
duke@435	1118	const Register saved_rsi = r10;
duke@435	1119	assert(nargs == 3 || nargs == 4, "else fix");
duke@435	1120	#ifdef _WIN64
duke@435	1121	assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
duke@435	1122	"unexpected argument registers");
duke@435	1123	if (nargs >= 4)
duke@435	1124	__ movq(rax, r9); // r9 is also saved_rdi
duke@435	1125	__ movq(saved_rdi, rdi);
duke@435	1126	__ movq(saved_rsi, rsi);
duke@435	1127	__ movq(rdi, rcx); // c_rarg0
duke@435	1128	__ movq(rsi, rdx); // c_rarg1
duke@435	1129	__ movq(rdx, r8); // c_rarg2
duke@435	1130	if (nargs >= 4)
duke@435	1131	__ movq(rcx, rax); // c_rarg3 (via rax)
duke@435	1132	#else
duke@435	1133	assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
duke@435	1134	"unexpected argument registers");
duke@435	1135	#endif
duke@435	1136	}
duke@435	1137
duke@435	1138	void restore_arg_regs() {
duke@435	1139	const Register saved_rdi = r9;
duke@435	1140	const Register saved_rsi = r10;
duke@435	1141	#ifdef _WIN64
duke@435	1142	__ movq(rdi, saved_rdi);
duke@435	1143	__ movq(rsi, saved_rsi);
duke@435	1144	#endif
duke@435	1145	}
duke@435	1146
duke@435	1147	// Generate code for an array write pre barrier
duke@435	1148	//
duke@435	1149	// addr - starting address
duke@435	1150	// count - element count
duke@435	1151	//
duke@435	1152	// Destroy no registers!
duke@435	1153	//
duke@435	1154	void gen_write_ref_array_pre_barrier(Register addr, Register count) {
duke@435	1155	#if 0 // G1 - only
duke@435	1156	assert_different_registers(addr, c_rarg1);
duke@435	1157	assert_different_registers(count, c_rarg0);
duke@435	1158	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	1159	switch (bs->kind()) {
duke@435	1160	case BarrierSet::G1SATBCT:
duke@435	1161	case BarrierSet::G1SATBCTLogging:
duke@435	1162	{
duke@435	1163	__ pushaq(); // push registers
duke@435	1164	__ movq(c_rarg0, addr);
duke@435	1165	__ movq(c_rarg1, count);
duke@435	1166	__ call(RuntimeAddress(BarrierSet::static_write_ref_array_pre));
duke@435	1167	__ popaq();
duke@435	1168	}
duke@435	1169	break;
duke@435	1170	case BarrierSet::CardTableModRef:
duke@435	1171	case BarrierSet::CardTableExtension:
duke@435	1172	case BarrierSet::ModRef:
duke@435	1173	break;
duke@435	1174	default :
duke@435	1175	ShouldNotReachHere();
duke@435	1176
duke@435	1177	}
duke@435	1178	#endif // 0 G1 - only
duke@435	1179	}
duke@435	1180
duke@435	1181	//
duke@435	1182	// Generate code for an array write post barrier
duke@435	1183	//
duke@435	1184	// Input:
duke@435	1185	// start - register containing starting address of destination array
duke@435	1186	// end - register containing ending address of destination array
duke@435	1187	// scratch - scratch register
duke@435	1188	//
duke@435	1189	// The input registers are overwritten.
duke@435	1190	// The ending address is inclusive.
duke@435	1191	void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) {
duke@435	1192	assert_different_registers(start, end, scratch);
duke@435	1193	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	1194	switch (bs->kind()) {
duke@435	1195	#if 0 // G1 - only
duke@435	1196	case BarrierSet::G1SATBCT:
duke@435	1197	case BarrierSet::G1SATBCTLogging:
duke@435	1198
duke@435	1199	{
duke@435	1200	__ pushaq(); // push registers (overkill)
duke@435	1201	// must compute element count unless barrier set interface is changed (other platforms supply count)
duke@435	1202	assert_different_registers(start, end, scratch);
duke@435	1203	__ leaq(scratch, Address(end, wordSize));
duke@435	1204	__ subq(scratch, start);
duke@435	1205	__ shrq(scratch, LogBytesPerWord);
duke@435	1206	__ movq(c_rarg0, start);
duke@435	1207	__ movq(c_rarg1, scratch);
duke@435	1208	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post));
duke@435	1209	__ popaq();
duke@435	1210	}
duke@435	1211	break;
duke@435	1212	#endif // 0 G1 - only
duke@435	1213	case BarrierSet::CardTableModRef:
duke@435	1214	case BarrierSet::CardTableExtension:
duke@435	1215	{
duke@435	1216	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	1217	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	1218
duke@435	1219	Label L_loop;
duke@435	1220
duke@435	1221	__ shrq(start, CardTableModRefBS::card_shift);
duke@435	1222	__ shrq(end, CardTableModRefBS::card_shift);
duke@435	1223	__ subq(end, start); // number of bytes to copy
duke@435	1224
duke@435	1225	const Register count = end; // 'end' register contains bytes count now
duke@435	1226	__ lea(scratch, ExternalAddress((address)ct->byte_map_base));
duke@435	1227	__ addq(start, scratch);
duke@435	1228	__ BIND(L_loop);
duke@435	1229	__ movb(Address(start, count, Address::times_1), 0);
duke@435	1230	__ decrementq(count);
duke@435	1231	__ jcc(Assembler::greaterEqual, L_loop);
duke@435	1232	}
duke@435	1233	}
duke@435	1234	}
duke@435	1235
duke@435	1236	// Copy big chunks forward
duke@435	1237	//
duke@435	1238	// Inputs:
duke@435	1239	// end_from - source arrays end address
duke@435	1240	// end_to - destination array end address
duke@435	1241	// qword_count - 64-bits element count, negative
duke@435	1242	// to - scratch
duke@435	1243	// L_copy_32_bytes - entry label
duke@435	1244	// L_copy_8_bytes - exit label
duke@435	1245	//
duke@435	1246	void copy_32_bytes_forward(Register end_from, Register end_to,
duke@435	1247	Register qword_count, Register to,
duke@435	1248	Label& L_copy_32_bytes, Label& L_copy_8_bytes) {
duke@435	1249	DEBUG_ONLY(__ stop("enter at entry label, not here"));
duke@435	1250	Label L_loop;
duke@435	1251	__ align(16);
duke@435	1252	__ BIND(L_loop);
duke@435	1253	__ movq(to, Address(end_from, qword_count, Address::times_8, -24));
duke@435	1254	__ movq(Address(end_to, qword_count, Address::times_8, -24), to);
duke@435	1255	__ movq(to, Address(end_from, qword_count, Address::times_8, -16));
duke@435	1256	__ movq(Address(end_to, qword_count, Address::times_8, -16), to);
duke@435	1257	__ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
duke@435	1258	__ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
duke@435	1259	__ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
duke@435	1260	__ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
duke@435	1261	__ BIND(L_copy_32_bytes);
duke@435	1262	__ addq(qword_count, 4);
duke@435	1263	__ jcc(Assembler::lessEqual, L_loop);
duke@435	1264	__ subq(qword_count, 4);
duke@435	1265	__ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
duke@435	1266	}
duke@435	1267
duke@435	1268
duke@435	1269	// Copy big chunks backward
duke@435	1270	//
duke@435	1271	// Inputs:
duke@435	1272	// from - source arrays address
duke@435	1273	// dest - destination array address
duke@435	1274	// qword_count - 64-bits element count
duke@435	1275	// to - scratch
duke@435	1276	// L_copy_32_bytes - entry label
duke@435	1277	// L_copy_8_bytes - exit label
duke@435	1278	//
duke@435	1279	void copy_32_bytes_backward(Register from, Register dest,
duke@435	1280	Register qword_count, Register to,
duke@435	1281	Label& L_copy_32_bytes, Label& L_copy_8_bytes) {
duke@435	1282	DEBUG_ONLY(__ stop("enter at entry label, not here"));
duke@435	1283	Label L_loop;
duke@435	1284	__ align(16);
duke@435	1285	__ BIND(L_loop);
duke@435	1286	__ movq(to, Address(from, qword_count, Address::times_8, 24));
duke@435	1287	__ movq(Address(dest, qword_count, Address::times_8, 24), to);
duke@435	1288	__ movq(to, Address(from, qword_count, Address::times_8, 16));
duke@435	1289	__ movq(Address(dest, qword_count, Address::times_8, 16), to);
duke@435	1290	__ movq(to, Address(from, qword_count, Address::times_8, 8));
duke@435	1291	__ movq(Address(dest, qword_count, Address::times_8, 8), to);
duke@435	1292	__ movq(to, Address(from, qword_count, Address::times_8, 0));
duke@435	1293	__ movq(Address(dest, qword_count, Address::times_8, 0), to);
duke@435	1294	__ BIND(L_copy_32_bytes);
duke@435	1295	__ subq(qword_count, 4);
duke@435	1296	__ jcc(Assembler::greaterEqual, L_loop);
duke@435	1297	__ addq(qword_count, 4);
duke@435	1298	__ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
duke@435	1299	}
duke@435	1300
duke@435	1301
duke@435	1302	// Arguments:
duke@435	1303	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1304	// ignored
duke@435	1305	// name - stub name string
duke@435	1306	//
duke@435	1307	// Inputs:
duke@435	1308	// c_rarg0 - source array address
duke@435	1309	// c_rarg1 - destination array address
duke@435	1310	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1311	//
duke@435	1312	// If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
duke@435	1313	// we let the hardware handle it. The one to eight bytes within words,
duke@435	1314	// dwords or qwords that span cache line boundaries will still be loaded
duke@435	1315	// and stored atomically.
duke@435	1316	//
duke@435	1317	// Side Effects:
duke@435	1318	// disjoint_byte_copy_entry is set to the no-overlap entry point
duke@435	1319	// used by generate_conjoint_byte_copy().
duke@435	1320	//
duke@435	1321	address generate_disjoint_byte_copy(bool aligned, const char *name) {
duke@435	1322	__ align(CodeEntryAlignment);
duke@435	1323	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1324	address start = __ pc();
duke@435	1325
duke@435	1326	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
duke@435	1327	Label L_copy_byte, L_exit;
duke@435	1328	const Register from = rdi; // source array address
duke@435	1329	const Register to = rsi; // destination array address
duke@435	1330	const Register count = rdx; // elements count
duke@435	1331	const Register byte_count = rcx;
duke@435	1332	const Register qword_count = count;
duke@435	1333	const Register end_from = from; // source array end address
duke@435	1334	const Register end_to = to; // destination array end address
duke@435	1335	// End pointers are inclusive, and if count is not zero they point
duke@435	1336	// to the last unit copied: end_to[0] := end_from[0]
duke@435	1337
duke@435	1338	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1339	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1340
duke@435	1341	disjoint_byte_copy_entry = __ pc();
duke@435	1342	BLOCK_COMMENT("Entry:");
duke@435	1343	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1344
duke@435	1345	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1346	// r9 and r10 may be used to save non-volatile registers
duke@435	1347
duke@435	1348	// 'from', 'to' and 'count' are now valid
duke@435	1349	__ movq(byte_count, count);
duke@435	1350	__ shrq(count, 3); // count => qword_count
duke@435	1351
duke@435	1352	// Copy from low to high addresses. Use 'to' as scratch.
duke@435	1353	__ leaq(end_from, Address(from, qword_count, Address::times_8, -8));
duke@435	1354	__ leaq(end_to, Address(to, qword_count, Address::times_8, -8));
duke@435	1355	__ negq(qword_count); // make the count negative
duke@435	1356	__ jmp(L_copy_32_bytes);
duke@435	1357
duke@435	1358	// Copy trailing qwords
duke@435	1359	__ BIND(L_copy_8_bytes);
duke@435	1360	__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
duke@435	1361	__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
duke@435	1362	__ incrementq(qword_count);
duke@435	1363	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1364
duke@435	1365	// Check for and copy trailing dword
duke@435	1366	__ BIND(L_copy_4_bytes);
duke@435	1367	__ testq(byte_count, 4);
duke@435	1368	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1369	__ movl(rax, Address(end_from, 8));
duke@435	1370	__ movl(Address(end_to, 8), rax);
duke@435	1371
duke@435	1372	__ addq(end_from, 4);
duke@435	1373	__ addq(end_to, 4);
duke@435	1374
duke@435	1375	// Check for and copy trailing word
duke@435	1376	__ BIND(L_copy_2_bytes);
duke@435	1377	__ testq(byte_count, 2);
duke@435	1378	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1379	__ movw(rax, Address(end_from, 8));
duke@435	1380	__ movw(Address(end_to, 8), rax);
duke@435	1381
duke@435	1382	__ addq(end_from, 2);
duke@435	1383	__ addq(end_to, 2);
duke@435	1384
duke@435	1385	// Check for and copy trailing byte
duke@435	1386	__ BIND(L_copy_byte);
duke@435	1387	__ testq(byte_count, 1);
duke@435	1388	__ jccb(Assembler::zero, L_exit);
duke@435	1389	__ movb(rax, Address(end_from, 8));
duke@435	1390	__ movb(Address(end_to, 8), rax);
duke@435	1391
duke@435	1392	__ BIND(L_exit);
duke@435	1393	inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
duke@435	1394	restore_arg_regs();
duke@435	1395	__ xorq(rax, rax); // return 0
duke@435	1396	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1397	__ ret(0);
duke@435	1398
duke@435	1399	// Copy in 32-bytes chunks
duke@435	1400	copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1401	__ jmp(L_copy_4_bytes);
duke@435	1402
duke@435	1403	return start;
duke@435	1404	}
duke@435	1405
duke@435	1406	// Arguments:
duke@435	1407	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1408	// ignored
duke@435	1409	// name - stub name string
duke@435	1410	//
duke@435	1411	// Inputs:
duke@435	1412	// c_rarg0 - source array address
duke@435	1413	// c_rarg1 - destination array address
duke@435	1414	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1415	//
duke@435	1416	// If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
duke@435	1417	// we let the hardware handle it. The one to eight bytes within words,
duke@435	1418	// dwords or qwords that span cache line boundaries will still be loaded
duke@435	1419	// and stored atomically.
duke@435	1420	//
duke@435	1421	address generate_conjoint_byte_copy(bool aligned, const char *name) {
duke@435	1422	__ align(CodeEntryAlignment);
duke@435	1423	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1424	address start = __ pc();
duke@435	1425
duke@435	1426	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
duke@435	1427	const Register from = rdi; // source array address
duke@435	1428	const Register to = rsi; // destination array address
duke@435	1429	const Register count = rdx; // elements count
duke@435	1430	const Register byte_count = rcx;
duke@435	1431	const Register qword_count = count;
duke@435	1432
duke@435	1433	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1434	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1435
duke@435	1436	byte_copy_entry = __ pc();
duke@435	1437	BLOCK_COMMENT("Entry:");
duke@435	1438	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1439
duke@435	1440	array_overlap_test(disjoint_byte_copy_entry, Address::times_1);
duke@435	1441	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1442	// r9 and r10 may be used to save non-volatile registers
duke@435	1443
duke@435	1444	// 'from', 'to' and 'count' are now valid
duke@435	1445	__ movq(byte_count, count);
duke@435	1446	__ shrq(count, 3); // count => qword_count
duke@435	1447
duke@435	1448	// Copy from high to low addresses.
duke@435	1449
duke@435	1450	// Check for and copy trailing byte
duke@435	1451	__ testq(byte_count, 1);
duke@435	1452	__ jcc(Assembler::zero, L_copy_2_bytes);
duke@435	1453	__ movb(rax, Address(from, byte_count, Address::times_1, -1));
duke@435	1454	__ movb(Address(to, byte_count, Address::times_1, -1), rax);
duke@435	1455	__ decrementq(byte_count); // Adjust for possible trailing word
duke@435	1456
duke@435	1457	// Check for and copy trailing word
duke@435	1458	__ BIND(L_copy_2_bytes);
duke@435	1459	__ testq(byte_count, 2);
duke@435	1460	__ jcc(Assembler::zero, L_copy_4_bytes);
duke@435	1461	__ movw(rax, Address(from, byte_count, Address::times_1, -2));
duke@435	1462	__ movw(Address(to, byte_count, Address::times_1, -2), rax);
duke@435	1463
duke@435	1464	// Check for and copy trailing dword
duke@435	1465	__ BIND(L_copy_4_bytes);
duke@435	1466	__ testq(byte_count, 4);
duke@435	1467	__ jcc(Assembler::zero, L_copy_32_bytes);
duke@435	1468	__ movl(rax, Address(from, qword_count, Address::times_8));
duke@435	1469	__ movl(Address(to, qword_count, Address::times_8), rax);
duke@435	1470	__ jmp(L_copy_32_bytes);
duke@435	1471
duke@435	1472	// Copy trailing qwords
duke@435	1473	__ BIND(L_copy_8_bytes);
duke@435	1474	__ movq(rax, Address(from, qword_count, Address::times_8, -8));
duke@435	1475	__ movq(Address(to, qword_count, Address::times_8, -8), rax);
duke@435	1476	__ decrementq(qword_count);
duke@435	1477	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1478
duke@435	1479	inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
duke@435	1480	restore_arg_regs();
duke@435	1481	__ xorq(rax, rax); // return 0
duke@435	1482	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1483	__ ret(0);
duke@435	1484
duke@435	1485	// Copy in 32-bytes chunks
duke@435	1486	copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1487
duke@435	1488	inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
duke@435	1489	restore_arg_regs();
duke@435	1490	__ xorq(rax, rax); // return 0
duke@435	1491	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1492	__ ret(0);
duke@435	1493
duke@435	1494	return start;
duke@435	1495	}
duke@435	1496
duke@435	1497	// Arguments:
duke@435	1498	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1499	// ignored
duke@435	1500	// name - stub name string
duke@435	1501	//
duke@435	1502	// Inputs:
duke@435	1503	// c_rarg0 - source array address
duke@435	1504	// c_rarg1 - destination array address
duke@435	1505	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1506	//
duke@435	1507	// If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
duke@435	1508	// let the hardware handle it. The two or four words within dwords
duke@435	1509	// or qwords that span cache line boundaries will still be loaded
duke@435	1510	// and stored atomically.
duke@435	1511	//
duke@435	1512	// Side Effects:
duke@435	1513	// disjoint_short_copy_entry is set to the no-overlap entry point
duke@435	1514	// used by generate_conjoint_short_copy().
duke@435	1515	//
duke@435	1516	address generate_disjoint_short_copy(bool aligned, const char *name) {
duke@435	1517	__ align(CodeEntryAlignment);
duke@435	1518	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1519	address start = __ pc();
duke@435	1520
duke@435	1521	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
duke@435	1522	const Register from = rdi; // source array address
duke@435	1523	const Register to = rsi; // destination array address
duke@435	1524	const Register count = rdx; // elements count
duke@435	1525	const Register word_count = rcx;
duke@435	1526	const Register qword_count = count;
duke@435	1527	const Register end_from = from; // source array end address
duke@435	1528	const Register end_to = to; // destination array end address
duke@435	1529	// End pointers are inclusive, and if count is not zero they point
duke@435	1530	// to the last unit copied: end_to[0] := end_from[0]
duke@435	1531
duke@435	1532	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1533	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1534
duke@435	1535	disjoint_short_copy_entry = __ pc();
duke@435	1536	BLOCK_COMMENT("Entry:");
duke@435	1537	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1538
duke@435	1539	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1540	// r9 and r10 may be used to save non-volatile registers
duke@435	1541
duke@435	1542	// 'from', 'to' and 'count' are now valid
duke@435	1543	__ movq(word_count, count);
duke@435	1544	__ shrq(count, 2); // count => qword_count
duke@435	1545
duke@435	1546	// Copy from low to high addresses. Use 'to' as scratch.
duke@435	1547	__ leaq(end_from, Address(from, qword_count, Address::times_8, -8));
duke@435	1548	__ leaq(end_to, Address(to, qword_count, Address::times_8, -8));
duke@435	1549	__ negq(qword_count);
duke@435	1550	__ jmp(L_copy_32_bytes);
duke@435	1551
duke@435	1552	// Copy trailing qwords
duke@435	1553	__ BIND(L_copy_8_bytes);
duke@435	1554	__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
duke@435	1555	__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
duke@435	1556	__ incrementq(qword_count);
duke@435	1557	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1558
duke@435	1559	// Original 'dest' is trashed, so we can't use it as a
duke@435	1560	// base register for a possible trailing word copy
duke@435	1561
duke@435	1562	// Check for and copy trailing dword
duke@435	1563	__ BIND(L_copy_4_bytes);
duke@435	1564	__ testq(word_count, 2);
duke@435	1565	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1566	__ movl(rax, Address(end_from, 8));
duke@435	1567	__ movl(Address(end_to, 8), rax);
duke@435	1568
duke@435	1569	__ addq(end_from, 4);
duke@435	1570	__ addq(end_to, 4);
duke@435	1571
duke@435	1572	// Check for and copy trailing word
duke@435	1573	__ BIND(L_copy_2_bytes);
duke@435	1574	__ testq(word_count, 1);
duke@435	1575	__ jccb(Assembler::zero, L_exit);
duke@435	1576	__ movw(rax, Address(end_from, 8));
duke@435	1577	__ movw(Address(end_to, 8), rax);
duke@435	1578
duke@435	1579	__ BIND(L_exit);
duke@435	1580	inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
duke@435	1581	restore_arg_regs();
duke@435	1582	__ xorq(rax, rax); // return 0
duke@435	1583	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1584	__ ret(0);
duke@435	1585
duke@435	1586	// Copy in 32-bytes chunks
duke@435	1587	copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1588	__ jmp(L_copy_4_bytes);
duke@435	1589
duke@435	1590	return start;
duke@435	1591	}
duke@435	1592
duke@435	1593	// Arguments:
duke@435	1594	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1595	// ignored
duke@435	1596	// name - stub name string
duke@435	1597	//
duke@435	1598	// Inputs:
duke@435	1599	// c_rarg0 - source array address
duke@435	1600	// c_rarg1 - destination array address
duke@435	1601	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1602	//
duke@435	1603	// If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
duke@435	1604	// let the hardware handle it. The two or four words within dwords
duke@435	1605	// or qwords that span cache line boundaries will still be loaded
duke@435	1606	// and stored atomically.
duke@435	1607	//
duke@435	1608	address generate_conjoint_short_copy(bool aligned, const char *name) {
duke@435	1609	__ align(CodeEntryAlignment);
duke@435	1610	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1611	address start = __ pc();
duke@435	1612
duke@435	1613	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes;
duke@435	1614	const Register from = rdi; // source array address
duke@435	1615	const Register to = rsi; // destination array address
duke@435	1616	const Register count = rdx; // elements count
duke@435	1617	const Register word_count = rcx;
duke@435	1618	const Register qword_count = count;
duke@435	1619
duke@435	1620	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1621	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1622
duke@435	1623	short_copy_entry = __ pc();
duke@435	1624	BLOCK_COMMENT("Entry:");
duke@435	1625	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1626
duke@435	1627	array_overlap_test(disjoint_short_copy_entry, Address::times_2);
duke@435	1628	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1629	// r9 and r10 may be used to save non-volatile registers
duke@435	1630
duke@435	1631	// 'from', 'to' and 'count' are now valid
duke@435	1632	__ movq(word_count, count);
duke@435	1633	__ shrq(count, 2); // count => qword_count
duke@435	1634
duke@435	1635	// Copy from high to low addresses. Use 'to' as scratch.
duke@435	1636
duke@435	1637	// Check for and copy trailing word
duke@435	1638	__ testq(word_count, 1);
duke@435	1639	__ jccb(Assembler::zero, L_copy_4_bytes);
duke@435	1640	__ movw(rax, Address(from, word_count, Address::times_2, -2));
duke@435	1641	__ movw(Address(to, word_count, Address::times_2, -2), rax);
duke@435	1642
duke@435	1643	// Check for and copy trailing dword
duke@435	1644	__ BIND(L_copy_4_bytes);
duke@435	1645	__ testq(word_count, 2);
duke@435	1646	__ jcc(Assembler::zero, L_copy_32_bytes);
duke@435	1647	__ movl(rax, Address(from, qword_count, Address::times_8));
duke@435	1648	__ movl(Address(to, qword_count, Address::times_8), rax);
duke@435	1649	__ jmp(L_copy_32_bytes);
duke@435	1650
duke@435	1651	// Copy trailing qwords
duke@435	1652	__ BIND(L_copy_8_bytes);
duke@435	1653	__ movq(rax, Address(from, qword_count, Address::times_8, -8));
duke@435	1654	__ movq(Address(to, qword_count, Address::times_8, -8), rax);
duke@435	1655	__ decrementq(qword_count);
duke@435	1656	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1657
duke@435	1658	inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
duke@435	1659	restore_arg_regs();
duke@435	1660	__ xorq(rax, rax); // return 0
duke@435	1661	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1662	__ ret(0);
duke@435	1663
duke@435	1664	// Copy in 32-bytes chunks
duke@435	1665	copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1666
duke@435	1667	inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
duke@435	1668	restore_arg_regs();
duke@435	1669	__ xorq(rax, rax); // return 0
duke@435	1670	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1671	__ ret(0);
duke@435	1672
duke@435	1673	return start;
duke@435	1674	}
duke@435	1675
duke@435	1676	// Arguments:
duke@435	1677	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1678	// ignored
coleenp@548	1679	// is_oop - true => oop array, so generate store check code
duke@435	1680	// name - stub name string
duke@435	1681	//
duke@435	1682	// Inputs:
duke@435	1683	// c_rarg0 - source array address
duke@435	1684	// c_rarg1 - destination array address
duke@435	1685	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1686	//
duke@435	1687	// If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
duke@435	1688	// the hardware handle it. The two dwords within qwords that span
duke@435	1689	// cache line boundaries will still be loaded and stored atomicly.
duke@435	1690	//
duke@435	1691	// Side Effects:
duke@435	1692	// disjoint_int_copy_entry is set to the no-overlap entry point
coleenp@548	1693	// used by generate_conjoint_int_oop_copy().
duke@435	1694	//
coleenp@548	1695	address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) {
duke@435	1696	__ align(CodeEntryAlignment);
duke@435	1697	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1698	address start = __ pc();
duke@435	1699
duke@435	1700	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
duke@435	1701	const Register from = rdi; // source array address
duke@435	1702	const Register to = rsi; // destination array address
duke@435	1703	const Register count = rdx; // elements count
duke@435	1704	const Register dword_count = rcx;
duke@435	1705	const Register qword_count = count;
duke@435	1706	const Register end_from = from; // source array end address
duke@435	1707	const Register end_to = to; // destination array end address
coleenp@548	1708	const Register saved_to = r11; // saved destination array address
duke@435	1709	// End pointers are inclusive, and if count is not zero they point
duke@435	1710	// to the last unit copied: end_to[0] := end_from[0]
duke@435	1711
duke@435	1712	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1713	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1714
coleenp@548	1715	(is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry) = __ pc();
coleenp@548	1716
coleenp@548	1717	if (is_oop) {
coleenp@548	1718	// no registers are destroyed by this call
coleenp@548	1719	gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
coleenp@548	1720	}
coleenp@548	1721
duke@435	1722	BLOCK_COMMENT("Entry:");
duke@435	1723	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1724
duke@435	1725	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1726	// r9 and r10 may be used to save non-volatile registers
duke@435	1727
coleenp@548	1728	if (is_oop) {
coleenp@548	1729	__ movq(saved_to, to);
coleenp@548	1730	}
coleenp@548	1731
duke@435	1732	// 'from', 'to' and 'count' are now valid
duke@435	1733	__ movq(dword_count, count);
duke@435	1734	__ shrq(count, 1); // count => qword_count
duke@435	1735
duke@435	1736	// Copy from low to high addresses. Use 'to' as scratch.
duke@435	1737	__ leaq(end_from, Address(from, qword_count, Address::times_8, -8));
duke@435	1738	__ leaq(end_to, Address(to, qword_count, Address::times_8, -8));
duke@435	1739	__ negq(qword_count);
duke@435	1740	__ jmp(L_copy_32_bytes);
duke@435	1741
duke@435	1742	// Copy trailing qwords
duke@435	1743	__ BIND(L_copy_8_bytes);
duke@435	1744	__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
duke@435	1745	__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
duke@435	1746	__ incrementq(qword_count);
duke@435	1747	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1748
duke@435	1749	// Check for and copy trailing dword
duke@435	1750	__ BIND(L_copy_4_bytes);
duke@435	1751	__ testq(dword_count, 1); // Only byte test since the value is 0 or 1
duke@435	1752	__ jccb(Assembler::zero, L_exit);
duke@435	1753	__ movl(rax, Address(end_from, 8));
duke@435	1754	__ movl(Address(end_to, 8), rax);
duke@435	1755
duke@435	1756	__ BIND(L_exit);
coleenp@548	1757	if (is_oop) {
coleenp@548	1758	__ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4));
coleenp@548	1759	gen_write_ref_array_post_barrier(saved_to, end_to, rax);
coleenp@548	1760	}
duke@435	1761	inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
duke@435	1762	restore_arg_regs();
duke@435	1763	__ xorq(rax, rax); // return 0
duke@435	1764	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1765	__ ret(0);
duke@435	1766
duke@435	1767	// Copy 32-bytes chunks
duke@435	1768	copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1769	__ jmp(L_copy_4_bytes);
duke@435	1770
duke@435	1771	return start;
duke@435	1772	}
duke@435	1773
duke@435	1774	// Arguments:
duke@435	1775	// aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
duke@435	1776	// ignored
coleenp@548	1777	// is_oop - true => oop array, so generate store check code
duke@435	1778	// name - stub name string
duke@435	1779	//
duke@435	1780	// Inputs:
duke@435	1781	// c_rarg0 - source array address
duke@435	1782	// c_rarg1 - destination array address
duke@435	1783	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1784	//
duke@435	1785	// If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
duke@435	1786	// the hardware handle it. The two dwords within qwords that span
duke@435	1787	// cache line boundaries will still be loaded and stored atomicly.
duke@435	1788	//
coleenp@548	1789	address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) {
duke@435	1790	__ align(CodeEntryAlignment);
duke@435	1791	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1792	address start = __ pc();
duke@435	1793
coleenp@548	1794	Label L_copy_32_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
duke@435	1795	const Register from = rdi; // source array address
duke@435	1796	const Register to = rsi; // destination array address
duke@435	1797	const Register count = rdx; // elements count
duke@435	1798	const Register dword_count = rcx;
duke@435	1799	const Register qword_count = count;
duke@435	1800
duke@435	1801	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1802	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1803
coleenp@548	1804	if (is_oop) {
coleenp@548	1805	// no registers are destroyed by this call
coleenp@548	1806	gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
coleenp@548	1807	}
coleenp@548	1808
coleenp@548	1809	(is_oop ? oop_copy_entry : int_copy_entry) = __ pc();
duke@435	1810	BLOCK_COMMENT("Entry:");
duke@435	1811	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1812
coleenp@548	1813	array_overlap_test(is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry,
coleenp@548	1814	Address::times_4);
duke@435	1815	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1816	// r9 and r10 may be used to save non-volatile registers
duke@435	1817
coleenp@548	1818	assert_clean_int(count, rax); // Make sure 'count' is clean int.
duke@435	1819	// 'from', 'to' and 'count' are now valid
duke@435	1820	__ movq(dword_count, count);
duke@435	1821	__ shrq(count, 1); // count => qword_count
duke@435	1822
duke@435	1823	// Copy from high to low addresses. Use 'to' as scratch.
duke@435	1824
duke@435	1825	// Check for and copy trailing dword
duke@435	1826	__ testq(dword_count, 1);
duke@435	1827	__ jcc(Assembler::zero, L_copy_32_bytes);
duke@435	1828	__ movl(rax, Address(from, dword_count, Address::times_4, -4));
duke@435	1829	__ movl(Address(to, dword_count, Address::times_4, -4), rax);
duke@435	1830	__ jmp(L_copy_32_bytes);
duke@435	1831
duke@435	1832	// Copy trailing qwords
duke@435	1833	__ BIND(L_copy_8_bytes);
duke@435	1834	__ movq(rax, Address(from, qword_count, Address::times_8, -8));
duke@435	1835	__ movq(Address(to, qword_count, Address::times_8, -8), rax);
duke@435	1836	__ decrementq(qword_count);
duke@435	1837	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1838
duke@435	1839	inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
coleenp@548	1840	if (is_oop) {
coleenp@548	1841	__ jmp(L_exit);
coleenp@548	1842	}
duke@435	1843	restore_arg_regs();
duke@435	1844	__ xorq(rax, rax); // return 0
duke@435	1845	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1846	__ ret(0);
duke@435	1847
duke@435	1848	// Copy in 32-bytes chunks
duke@435	1849	copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1850
coleenp@548	1851	inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
coleenp@548	1852	__ bind(L_exit);
coleenp@548	1853	if (is_oop) {
coleenp@548	1854	Register end_to = rdx;
coleenp@548	1855	__ leaq(end_to, Address(to, dword_count, Address::times_4, -4));
coleenp@548	1856	gen_write_ref_array_post_barrier(to, end_to, rax);
coleenp@548	1857	}
duke@435	1858	restore_arg_regs();
duke@435	1859	__ xorq(rax, rax); // return 0
duke@435	1860	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1861	__ ret(0);
duke@435	1862
duke@435	1863	return start;
duke@435	1864	}
duke@435	1865
duke@435	1866	// Arguments:
duke@435	1867	// aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
duke@435	1868	// ignored
duke@435	1869	// is_oop - true => oop array, so generate store check code
duke@435	1870	// name - stub name string
duke@435	1871	//
duke@435	1872	// Inputs:
duke@435	1873	// c_rarg0 - source array address
duke@435	1874	// c_rarg1 - destination array address
duke@435	1875	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1876	//
coleenp@548	1877	// Side Effects:
duke@435	1878	// disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
duke@435	1879	// no-overlap entry point used by generate_conjoint_long_oop_copy().
duke@435	1880	//
duke@435	1881	address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) {
duke@435	1882	__ align(CodeEntryAlignment);
duke@435	1883	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1884	address start = __ pc();
duke@435	1885
duke@435	1886	Label L_copy_32_bytes, L_copy_8_bytes, L_exit;
duke@435	1887	const Register from = rdi; // source array address
duke@435	1888	const Register to = rsi; // destination array address
duke@435	1889	const Register qword_count = rdx; // elements count
duke@435	1890	const Register end_from = from; // source array end address
duke@435	1891	const Register end_to = rcx; // destination array end address
duke@435	1892	const Register saved_to = to;
duke@435	1893	// End pointers are inclusive, and if count is not zero they point
duke@435	1894	// to the last unit copied: end_to[0] := end_from[0]
duke@435	1895
duke@435	1896	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1897	// Save no-overlap entry point for generate_conjoint_long_oop_copy()
duke@435	1898	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1899
duke@435	1900	if (is_oop) {
duke@435	1901	disjoint_oop_copy_entry = __ pc();
duke@435	1902	// no registers are destroyed by this call
duke@435	1903	gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
duke@435	1904	} else {
duke@435	1905	disjoint_long_copy_entry = __ pc();
duke@435	1906	}
duke@435	1907	BLOCK_COMMENT("Entry:");
duke@435	1908	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1909
duke@435	1910	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1911	// r9 and r10 may be used to save non-volatile registers
duke@435	1912
duke@435	1913	// 'from', 'to' and 'qword_count' are now valid
duke@435	1914
duke@435	1915	// Copy from low to high addresses. Use 'to' as scratch.
duke@435	1916	__ leaq(end_from, Address(from, qword_count, Address::times_8, -8));
coleenp@548	1917	__ leaq(end_to, Address(to, qword_count, Address::times_8, -8));
duke@435	1918	__ negq(qword_count);
duke@435	1919	__ jmp(L_copy_32_bytes);
duke@435	1920
duke@435	1921	// Copy trailing qwords
duke@435	1922	__ BIND(L_copy_8_bytes);
duke@435	1923	__ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
duke@435	1924	__ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
duke@435	1925	__ incrementq(qword_count);
duke@435	1926	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	1927
duke@435	1928	if (is_oop) {
duke@435	1929	__ jmp(L_exit);
duke@435	1930	} else {
duke@435	1931	inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
duke@435	1932	restore_arg_regs();
duke@435	1933	__ xorq(rax, rax); // return 0
duke@435	1934	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1935	__ ret(0);
duke@435	1936	}
duke@435	1937
duke@435	1938	// Copy 64-byte chunks
duke@435	1939	copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	1940
duke@435	1941	if (is_oop) {
duke@435	1942	__ BIND(L_exit);
duke@435	1943	gen_write_ref_array_post_barrier(saved_to, end_to, rax);
duke@435	1944	inc_counter_np(SharedRuntime::_oop_array_copy_ctr);
duke@435	1945	} else {
duke@435	1946	inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
duke@435	1947	}
duke@435	1948	restore_arg_regs();
duke@435	1949	__ xorq(rax, rax); // return 0
duke@435	1950	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1951	__ ret(0);
duke@435	1952
duke@435	1953	return start;
duke@435	1954	}
duke@435	1955
duke@435	1956	// Arguments:
duke@435	1957	// aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
duke@435	1958	// ignored
duke@435	1959	// is_oop - true => oop array, so generate store check code
duke@435	1960	// name - stub name string
duke@435	1961	//
duke@435	1962	// Inputs:
duke@435	1963	// c_rarg0 - source array address
duke@435	1964	// c_rarg1 - destination array address
duke@435	1965	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	1966	//
duke@435	1967	address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) {
duke@435	1968	__ align(CodeEntryAlignment);
duke@435	1969	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1970	address start = __ pc();
duke@435	1971
duke@435	1972	Label L_copy_32_bytes, L_copy_8_bytes, L_exit;
duke@435	1973	const Register from = rdi; // source array address
duke@435	1974	const Register to = rsi; // destination array address
duke@435	1975	const Register qword_count = rdx; // elements count
duke@435	1976	const Register saved_count = rcx;
duke@435	1977
duke@435	1978	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1979	assert_clean_int(c_rarg2, rax); // Make sure 'count' is clean int.
duke@435	1980
duke@435	1981	address disjoint_copy_entry = NULL;
duke@435	1982	if (is_oop) {
coleenp@548	1983	assert(!UseCompressedOops, "shouldn't be called for compressed oops");
duke@435	1984	disjoint_copy_entry = disjoint_oop_copy_entry;
duke@435	1985	oop_copy_entry = __ pc();
coleenp@548	1986	array_overlap_test(disjoint_oop_copy_entry, Address::times_8);
duke@435	1987	} else {
duke@435	1988	disjoint_copy_entry = disjoint_long_copy_entry;
duke@435	1989	long_copy_entry = __ pc();
coleenp@548	1990	array_overlap_test(disjoint_long_copy_entry, Address::times_8);
duke@435	1991	}
duke@435	1992	BLOCK_COMMENT("Entry:");
duke@435	1993	// caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
duke@435	1994
duke@435	1995	array_overlap_test(disjoint_copy_entry, Address::times_8);
duke@435	1996	setup_arg_regs(); // from => rdi, to => rsi, count => rdx
duke@435	1997	// r9 and r10 may be used to save non-volatile registers
duke@435	1998
duke@435	1999	// 'from', 'to' and 'qword_count' are now valid
duke@435	2000
duke@435	2001	if (is_oop) {
duke@435	2002	// Save to and count for store barrier
duke@435	2003	__ movq(saved_count, qword_count);
duke@435	2004	// No registers are destroyed by this call
duke@435	2005	gen_write_ref_array_pre_barrier(to, saved_count);
duke@435	2006	}
duke@435	2007
duke@435	2008	__ jmp(L_copy_32_bytes);
duke@435	2009
duke@435	2010	// Copy trailing qwords
duke@435	2011	__ BIND(L_copy_8_bytes);
duke@435	2012	__ movq(rax, Address(from, qword_count, Address::times_8, -8));
duke@435	2013	__ movq(Address(to, qword_count, Address::times_8, -8), rax);
duke@435	2014	__ decrementq(qword_count);
duke@435	2015	__ jcc(Assembler::notZero, L_copy_8_bytes);
duke@435	2016
duke@435	2017	if (is_oop) {
duke@435	2018	__ jmp(L_exit);
duke@435	2019	} else {
duke@435	2020	inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
duke@435	2021	restore_arg_regs();
duke@435	2022	__ xorq(rax, rax); // return 0
duke@435	2023	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2024	__ ret(0);
duke@435	2025	}
duke@435	2026
duke@435	2027	// Copy in 32-bytes chunks
duke@435	2028	copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
duke@435	2029
duke@435	2030	if (is_oop) {
duke@435	2031	__ BIND(L_exit);
duke@435	2032	__ leaq(rcx, Address(to, saved_count, Address::times_8, -8));
duke@435	2033	gen_write_ref_array_post_barrier(to, rcx, rax);
duke@435	2034	inc_counter_np(SharedRuntime::_oop_array_copy_ctr);
duke@435	2035	} else {
duke@435	2036	inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
duke@435	2037	}
duke@435	2038	restore_arg_regs();
duke@435	2039	__ xorq(rax, rax); // return 0
duke@435	2040	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2041	__ ret(0);
duke@435	2042
duke@435	2043	return start;
duke@435	2044	}
duke@435	2045
duke@435	2046
duke@435	2047	// Helper for generating a dynamic type check.
duke@435	2048	// Smashes no registers.
duke@435	2049	void generate_type_check(Register sub_klass,
duke@435	2050	Register super_check_offset,
duke@435	2051	Register super_klass,
duke@435	2052	Label& L_success) {
duke@435	2053	assert_different_registers(sub_klass, super_check_offset, super_klass);
duke@435	2054
duke@435	2055	BLOCK_COMMENT("type_check:");
duke@435	2056
duke@435	2057	Label L_miss;
duke@435	2058
duke@435	2059	// a couple of useful fields in sub_klass:
duke@435	2060	int ss_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	2061	Klass::secondary_supers_offset_in_bytes());
duke@435	2062	int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	2063	Klass::secondary_super_cache_offset_in_bytes());
duke@435	2064	Address secondary_supers_addr(sub_klass, ss_offset);
duke@435	2065	Address super_cache_addr( sub_klass, sc_offset);
duke@435	2066
duke@435	2067	// if the pointers are equal, we are done (e.g., String[] elements)
duke@435	2068	__ cmpq(super_klass, sub_klass);
duke@435	2069	__ jcc(Assembler::equal, L_success);
duke@435	2070
duke@435	2071	// check the supertype display:
duke@435	2072	Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
duke@435	2073	__ cmpq(super_klass, super_check_addr); // test the super type
duke@435	2074	__ jcc(Assembler::equal, L_success);
duke@435	2075
duke@435	2076	// if it was a primary super, we can just fail immediately
duke@435	2077	__ cmpl(super_check_offset, sc_offset);
duke@435	2078	__ jcc(Assembler::notEqual, L_miss);
duke@435	2079
duke@435	2080	// Now do a linear scan of the secondary super-klass chain.
duke@435	2081	// The repne_scan instruction uses fixed registers, which we must spill.
duke@435	2082	// (We need a couple more temps in any case.)
duke@435	2083	// This code is rarely used, so simplicity is a virtue here.
duke@435	2084	inc_counter_np(SharedRuntime::_partial_subtype_ctr);
duke@435	2085	{
duke@435	2086	__ pushq(rax);
duke@435	2087	__ pushq(rcx);
duke@435	2088	__ pushq(rdi);
duke@435	2089	assert_different_registers(sub_klass, super_klass, rax, rcx, rdi);
duke@435	2090
duke@435	2091	__ movq(rdi, secondary_supers_addr);
duke@435	2092	// Load the array length.
duke@435	2093	__ movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes()));
duke@435	2094	// Skip to start of data.
duke@435	2095	__ addq(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
duke@435	2096	// Scan rcx words at [rdi] for occurance of rax
duke@435	2097	// Set NZ/Z based on last compare
duke@435	2098	__ movq(rax, super_klass);
coleenp@548	2099	if (UseCompressedOops) {
coleenp@548	2100	// Compare against compressed form. Don't need to uncompress because
coleenp@548	2101	// looks like orig rax is restored in popq below.
coleenp@548	2102	__ encode_heap_oop(rax);
coleenp@548	2103	__ repne_scanl();
coleenp@548	2104	} else {
coleenp@548	2105	__ repne_scanq();
coleenp@548	2106	}
duke@435	2107
duke@435	2108	// Unspill the temp. registers:
duke@435	2109	__ popq(rdi);
duke@435	2110	__ popq(rcx);
duke@435	2111	__ popq(rax);
duke@435	2112
duke@435	2113	__ jcc(Assembler::notEqual, L_miss);
duke@435	2114	}
duke@435	2115
duke@435	2116	// Success. Cache the super we found and proceed in triumph.
duke@435	2117	__ movq(super_cache_addr, super_klass); // note: rax is dead
duke@435	2118	__ jmp(L_success);
duke@435	2119
duke@435	2120	// Fall through on failure!
duke@435	2121	__ BIND(L_miss);
duke@435	2122	}
duke@435	2123
duke@435	2124	//
duke@435	2125	// Generate checkcasting array copy stub
duke@435	2126	//
duke@435	2127	// Input:
duke@435	2128	// c_rarg0 - source array address
duke@435	2129	// c_rarg1 - destination array address
duke@435	2130	// c_rarg2 - element count, treated as ssize_t, can be zero
duke@435	2131	// c_rarg3 - size_t ckoff (super_check_offset)
duke@435	2132	// not Win64
duke@435	2133	// c_rarg4 - oop ckval (super_klass)
duke@435	2134	// Win64
duke@435	2135	// rsp+40 - oop ckval (super_klass)
duke@435	2136	//
duke@435	2137	// Output:
duke@435	2138	// rax == 0 - success
duke@435	2139	// rax == -1^K - failure, where K is partial transfer count
duke@435	2140	//
duke@435	2141	address generate_checkcast_copy(const char *name) {
duke@435	2142
duke@435	2143	Label L_load_element, L_store_element, L_do_card_marks, L_done;
duke@435	2144
duke@435	2145	// Input registers (after setup_arg_regs)
duke@435	2146	const Register from = rdi; // source array address
duke@435	2147	const Register to = rsi; // destination array address
duke@435	2148	const Register length = rdx; // elements count
duke@435	2149	const Register ckoff = rcx; // super_check_offset
duke@435	2150	const Register ckval = r8; // super_klass
duke@435	2151
duke@435	2152	// Registers used as temps (r13, r14 are save-on-entry)
duke@435	2153	const Register end_from = from; // source array end address
duke@435	2154	const Register end_to = r13; // destination array end address
duke@435	2155	const Register count = rdx; // -(count_remaining)
duke@435	2156	const Register r14_length = r14; // saved copy of length
duke@435	2157	// End pointers are inclusive, and if length is not zero they point
duke@435	2158	// to the last unit copied: end_to[0] := end_from[0]
duke@435	2159
duke@435	2160	const Register rax_oop = rax; // actual oop copied
duke@435	2161	const Register r11_klass = r11; // oop._klass
duke@435	2162
duke@435	2163	//---------------------------------------------------------------
duke@435	2164	// Assembler stub will be used for this call to arraycopy
duke@435	2165	// if the two arrays are subtypes of Object[] but the
duke@435	2166	// destination array type is not equal to or a supertype
duke@435	2167	// of the source type. Each element must be separately
duke@435	2168	// checked.
duke@435	2169
duke@435	2170	__ align(CodeEntryAlignment);
duke@435	2171	StubCodeMark mark(this, "StubRoutines", name);
duke@435	2172	address start = __ pc();
duke@435	2173
duke@435	2174	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2175
duke@435	2176	checkcast_copy_entry = __ pc();
duke@435	2177	BLOCK_COMMENT("Entry:");
duke@435	2178
duke@435	2179	#ifdef ASSERT
duke@435	2180	// caller guarantees that the arrays really are different
duke@435	2181	// otherwise, we would have to make conjoint checks
duke@435	2182	{ Label L;
coleenp@548	2183	array_overlap_test(L, TIMES_OOP);
duke@435	2184	__ stop("checkcast_copy within a single array");
duke@435	2185	__ bind(L);
duke@435	2186	}
duke@435	2187	#endif //ASSERT
duke@435	2188
duke@435	2189	// allocate spill slots for r13, r14
duke@435	2190	enum {
duke@435	2191	saved_r13_offset,
duke@435	2192	saved_r14_offset,
duke@435	2193	saved_rbp_offset,
duke@435	2194	saved_rip_offset,
duke@435	2195	saved_rarg0_offset
duke@435	2196	};
duke@435	2197	__ subq(rsp, saved_rbp_offset * wordSize);
duke@435	2198	__ movq(Address(rsp, saved_r13_offset * wordSize), r13);
duke@435	2199	__ movq(Address(rsp, saved_r14_offset * wordSize), r14);
duke@435	2200	setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
duke@435	2201	// ckoff => rcx, ckval => r8
duke@435	2202	// r9 and r10 may be used to save non-volatile registers
duke@435	2203	#ifdef _WIN64
duke@435	2204	// last argument (#4) is on stack on Win64
duke@435	2205	const int ckval_offset = saved_rarg0_offset + 4;
duke@435	2206	__ movq(ckval, Address(rsp, ckval_offset * wordSize));
duke@435	2207	#endif
duke@435	2208
duke@435	2209	// check that int operands are properly extended to size_t
duke@435	2210	assert_clean_int(length, rax);
duke@435	2211	assert_clean_int(ckoff, rax);
duke@435	2212
duke@435	2213	#ifdef ASSERT
duke@435	2214	BLOCK_COMMENT("assert consistent ckoff/ckval");
duke@435	2215	// The ckoff and ckval must be mutually consistent,
duke@435	2216	// even though caller generates both.
duke@435	2217	{ Label L;
duke@435	2218	int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	2219	Klass::super_check_offset_offset_in_bytes());
duke@435	2220	__ cmpl(ckoff, Address(ckval, sco_offset));
duke@435	2221	__ jcc(Assembler::equal, L);
duke@435	2222	__ stop("super_check_offset inconsistent");
duke@435	2223	__ bind(L);
duke@435	2224	}
duke@435	2225	#endif //ASSERT
duke@435	2226
duke@435	2227	// Loop-invariant addresses. They are exclusive end pointers.
coleenp@548	2228	Address end_from_addr(from, length, TIMES_OOP, 0);
coleenp@548	2229	Address end_to_addr(to, length, TIMES_OOP, 0);
duke@435	2230	// Loop-variant addresses. They assume post-incremented count < 0.
coleenp@548	2231	Address from_element_addr(end_from, count, TIMES_OOP, 0);
coleenp@548	2232	Address to_element_addr(end_to, count, TIMES_OOP, 0);
duke@435	2233
duke@435	2234	gen_write_ref_array_pre_barrier(to, count);
duke@435	2235
duke@435	2236	// Copy from low to high addresses, indexed from the end of each array.
duke@435	2237	__ leaq(end_from, end_from_addr);
duke@435	2238	__ leaq(end_to, end_to_addr);
duke@435	2239	__ movq(r14_length, length); // save a copy of the length
duke@435	2240	assert(length == count, ""); // else fix next line:
duke@435	2241	__ negq(count); // negate and test the length
duke@435	2242	__ jcc(Assembler::notZero, L_load_element);
duke@435	2243
duke@435	2244	// Empty array: Nothing to do.
duke@435	2245	__ xorq(rax, rax); // return 0 on (trivial) success
duke@435	2246	__ jmp(L_done);
duke@435	2247
duke@435	2248	// ======== begin loop ========
duke@435	2249	// (Loop is rotated; its entry is L_load_element.)
duke@435	2250	// Loop control:
duke@435	2251	// for (count = -count; count != 0; count++)
duke@435	2252	// Base pointers src, dst are biased by 8*(count-1),to last element.
duke@435	2253	__ align(16);
duke@435	2254
duke@435	2255	__ BIND(L_store_element);
coleenp@548	2256	__ store_heap_oop(to_element_addr, rax_oop); // store the oop
duke@435	2257	__ incrementq(count); // increment the count toward zero
duke@435	2258	__ jcc(Assembler::zero, L_do_card_marks);
duke@435	2259
duke@435	2260	// ======== loop entry is here ========
duke@435	2261	__ BIND(L_load_element);
coleenp@548	2262	__ load_heap_oop(rax_oop, from_element_addr); // load the oop
duke@435	2263	__ testq(rax_oop, rax_oop);
duke@435	2264	__ jcc(Assembler::zero, L_store_element);
duke@435	2265
coleenp@548	2266	__ load_klass(r11_klass, rax_oop);// query the object klass
duke@435	2267	generate_type_check(r11_klass, ckoff, ckval, L_store_element);
duke@435	2268	// ======== end loop ========
duke@435	2269
duke@435	2270	// It was a real error; we must depend on the caller to finish the job.
duke@435	2271	// Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
duke@435	2272	// Emit GC store barriers for the oops we have copied (r14 + rdx),
duke@435	2273	// and report their number to the caller.
duke@435	2274	assert_different_registers(rax, r14_length, count, to, end_to, rcx);
duke@435	2275	__ leaq(end_to, to_element_addr);
duke@435	2276	gen_write_ref_array_post_barrier(to, end_to, rcx);
duke@435	2277	__ movq(rax, r14_length); // original oops
duke@435	2278	__ addq(rax, count); // K = (original - remaining) oops
duke@435	2279	__ notq(rax); // report (-1^K) to caller
duke@435	2280	__ jmp(L_done);
duke@435	2281
duke@435	2282	// Come here on success only.
duke@435	2283	__ BIND(L_do_card_marks);
duke@435	2284	__ addq(end_to, -wordSize); // make an inclusive end pointer
duke@435	2285	gen_write_ref_array_post_barrier(to, end_to, rcx);
duke@435	2286	__ xorq(rax, rax); // return 0 on success
duke@435	2287
duke@435	2288	// Common exit point (success or failure).
duke@435	2289	__ BIND(L_done);
duke@435	2290	__ movq(r13, Address(rsp, saved_r13_offset * wordSize));
duke@435	2291	__ movq(r14, Address(rsp, saved_r14_offset * wordSize));
duke@435	2292	inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
duke@435	2293	restore_arg_regs();
duke@435	2294	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2295	__ ret(0);
duke@435	2296
duke@435	2297	return start;
duke@435	2298	}
duke@435	2299
duke@435	2300	//
duke@435	2301	// Generate 'unsafe' array copy stub
duke@435	2302	// Though just as safe as the other stubs, it takes an unscaled
duke@435	2303	// size_t argument instead of an element count.
duke@435	2304	//
duke@435	2305	// Input:
duke@435	2306	// c_rarg0 - source array address
duke@435	2307	// c_rarg1 - destination array address
duke@435	2308	// c_rarg2 - byte count, treated as ssize_t, can be zero
duke@435	2309	//
duke@435	2310	// Examines the alignment of the operands and dispatches
duke@435	2311	// to a long, int, short, or byte copy loop.
duke@435	2312	//
duke@435	2313	address generate_unsafe_copy(const char *name) {
duke@435	2314
duke@435	2315	Label L_long_aligned, L_int_aligned, L_short_aligned;
duke@435	2316
duke@435	2317	// Input registers (before setup_arg_regs)
duke@435	2318	const Register from = c_rarg0; // source array address
duke@435	2319	const Register to = c_rarg1; // destination array address
duke@435	2320	const Register size = c_rarg2; // byte count (size_t)
duke@435	2321
duke@435	2322	// Register used as a temp
duke@435	2323	const Register bits = rax; // test copy of low bits
duke@435	2324
duke@435	2325	__ align(CodeEntryAlignment);
duke@435	2326	StubCodeMark mark(this, "StubRoutines", name);
duke@435	2327	address start = __ pc();
duke@435	2328
duke@435	2329	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2330
duke@435	2331	// bump this on entry, not on exit:
duke@435	2332	inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
duke@435	2333
duke@435	2334	__ movq(bits, from);
duke@435	2335	__ orq(bits, to);
duke@435	2336	__ orq(bits, size);
duke@435	2337
duke@435	2338	__ testb(bits, BytesPerLong-1);
duke@435	2339	__ jccb(Assembler::zero, L_long_aligned);
duke@435	2340
duke@435	2341	__ testb(bits, BytesPerInt-1);
duke@435	2342	__ jccb(Assembler::zero, L_int_aligned);
duke@435	2343
duke@435	2344	__ testb(bits, BytesPerShort-1);
duke@435	2345	__ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
duke@435	2346
duke@435	2347	__ BIND(L_short_aligned);
duke@435	2348	__ shrq(size, LogBytesPerShort); // size => short_count
duke@435	2349	__ jump(RuntimeAddress(short_copy_entry));
duke@435	2350
duke@435	2351	__ BIND(L_int_aligned);
duke@435	2352	__ shrq(size, LogBytesPerInt); // size => int_count
duke@435	2353	__ jump(RuntimeAddress(int_copy_entry));
duke@435	2354
duke@435	2355	__ BIND(L_long_aligned);
duke@435	2356	__ shrq(size, LogBytesPerLong); // size => qword_count
duke@435	2357	__ jump(RuntimeAddress(long_copy_entry));
duke@435	2358
duke@435	2359	return start;
duke@435	2360	}
duke@435	2361
duke@435	2362	// Perform range checks on the proposed arraycopy.
duke@435	2363	// Kills temp, but nothing else.
duke@435	2364	// Also, clean the sign bits of src_pos and dst_pos.
duke@435	2365	void arraycopy_range_checks(Register src, // source array oop (c_rarg0)
duke@435	2366	Register src_pos, // source position (c_rarg1)
duke@435	2367	Register dst, // destination array oo (c_rarg2)
duke@435	2368	Register dst_pos, // destination position (c_rarg3)
duke@435	2369	Register length,
duke@435	2370	Register temp,
duke@435	2371	Label& L_failed) {
duke@435	2372	BLOCK_COMMENT("arraycopy_range_checks:");
duke@435	2373
duke@435	2374	// if (src_pos + length > arrayOop(src)->length()) FAIL;
duke@435	2375	__ movl(temp, length);
duke@435	2376	__ addl(temp, src_pos); // src_pos + length
duke@435	2377	__ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
duke@435	2378	__ jcc(Assembler::above, L_failed);
duke@435	2379
duke@435	2380	// if (dst_pos + length > arrayOop(dst)->length()) FAIL;
duke@435	2381	__ movl(temp, length);
duke@435	2382	__ addl(temp, dst_pos); // dst_pos + length
duke@435	2383	__ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
duke@435	2384	__ jcc(Assembler::above, L_failed);
duke@435	2385
duke@435	2386	// Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
duke@435	2387	// Move with sign extension can be used since they are positive.
duke@435	2388	__ movslq(src_pos, src_pos);
duke@435	2389	__ movslq(dst_pos, dst_pos);
duke@435	2390
duke@435	2391	BLOCK_COMMENT("arraycopy_range_checks done");
duke@435	2392	}
duke@435	2393
duke@435	2394	//
duke@435	2395	// Generate generic array copy stubs
duke@435	2396	//
duke@435	2397	// Input:
duke@435	2398	// c_rarg0 - src oop
duke@435	2399	// c_rarg1 - src_pos (32-bits)
duke@435	2400	// c_rarg2 - dst oop
duke@435	2401	// c_rarg3 - dst_pos (32-bits)
duke@435	2402	// not Win64
duke@435	2403	// c_rarg4 - element count (32-bits)
duke@435	2404	// Win64
duke@435	2405	// rsp+40 - element count (32-bits)
duke@435	2406	//
duke@435	2407	// Output:
duke@435	2408	// rax == 0 - success
duke@435	2409	// rax == -1^K - failure, where K is partial transfer count
duke@435	2410	//
duke@435	2411	address generate_generic_copy(const char *name) {
duke@435	2412
duke@435	2413	Label L_failed, L_failed_0, L_objArray;
duke@435	2414	Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
duke@435	2415
duke@435	2416	// Input registers
duke@435	2417	const Register src = c_rarg0; // source array oop
duke@435	2418	const Register src_pos = c_rarg1; // source position
duke@435	2419	const Register dst = c_rarg2; // destination array oop
duke@435	2420	const Register dst_pos = c_rarg3; // destination position
duke@435	2421	// elements count is on stack on Win64
duke@435	2422	#ifdef _WIN64
duke@435	2423	#define C_RARG4 Address(rsp, 6 * wordSize)
duke@435	2424	#else
duke@435	2425	#define C_RARG4 c_rarg4
duke@435	2426	#endif
duke@435	2427
duke@435	2428	{ int modulus = CodeEntryAlignment;
duke@435	2429	int target = modulus - 5; // 5 = sizeof jmp(L_failed)
duke@435	2430	int advance = target - (__ offset() % modulus);
duke@435	2431	if (advance < 0) advance += modulus;
duke@435	2432	if (advance > 0) __ nop(advance);
duke@435	2433	}
duke@435	2434	StubCodeMark mark(this, "StubRoutines", name);
duke@435	2435
duke@435	2436	// Short-hop target to L_failed. Makes for denser prologue code.
duke@435	2437	__ BIND(L_failed_0);
duke@435	2438	__ jmp(L_failed);
duke@435	2439	assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
duke@435	2440
duke@435	2441	__ align(CodeEntryAlignment);
duke@435	2442	address start = __ pc();
duke@435	2443
duke@435	2444	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2445
duke@435	2446	// bump this on entry, not on exit:
duke@435	2447	inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
duke@435	2448
duke@435	2449	//-----------------------------------------------------------------------
duke@435	2450	// Assembler stub will be used for this call to arraycopy
duke@435	2451	// if the following conditions are met:
duke@435	2452	//
duke@435	2453	// (1) src and dst must not be null.
duke@435	2454	// (2) src_pos must not be negative.
duke@435	2455	// (3) dst_pos must not be negative.
duke@435	2456	// (4) length must not be negative.
duke@435	2457	// (5) src klass and dst klass should be the same and not NULL.
duke@435	2458	// (6) src and dst should be arrays.
duke@435	2459	// (7) src_pos + length must not exceed length of src.
duke@435	2460	// (8) dst_pos + length must not exceed length of dst.
duke@435	2461	//
duke@435	2462
duke@435	2463	// if (src == NULL) return -1;
duke@435	2464	__ testq(src, src); // src oop
duke@435	2465	size_t j1off = __ offset();
duke@435	2466	__ jccb(Assembler::zero, L_failed_0);
duke@435	2467
duke@435	2468	// if (src_pos < 0) return -1;
duke@435	2469	__ testl(src_pos, src_pos); // src_pos (32-bits)
duke@435	2470	__ jccb(Assembler::negative, L_failed_0);
duke@435	2471
duke@435	2472	// if (dst == NULL) return -1;
duke@435	2473	__ testq(dst, dst); // dst oop
duke@435	2474	__ jccb(Assembler::zero, L_failed_0);
duke@435	2475
duke@435	2476	// if (dst_pos < 0) return -1;
duke@435	2477	__ testl(dst_pos, dst_pos); // dst_pos (32-bits)
duke@435	2478	size_t j4off = __ offset();
duke@435	2479	__ jccb(Assembler::negative, L_failed_0);
duke@435	2480
duke@435	2481	// The first four tests are very dense code,
duke@435	2482	// but not quite dense enough to put four
duke@435	2483	// jumps in a 16-byte instruction fetch buffer.
duke@435	2484	// That's good, because some branch predicters
duke@435	2485	// do not like jumps so close together.
duke@435	2486	// Make sure of this.
duke@435	2487	guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
duke@435	2488
duke@435	2489	// registers used as temp
duke@435	2490	const Register r11_length = r11; // elements count to copy
duke@435	2491	const Register r10_src_klass = r10; // array klass
coleenp@548	2492	const Register r9_dst_klass = r9; // dest array klass
duke@435	2493
duke@435	2494	// if (length < 0) return -1;
duke@435	2495	__ movl(r11_length, C_RARG4); // length (elements count, 32-bits value)
duke@435	2496	__ testl(r11_length, r11_length);
duke@435	2497	__ jccb(Assembler::negative, L_failed_0);
duke@435	2498
coleenp@548	2499	__ load_klass(r10_src_klass, src);
duke@435	2500	#ifdef ASSERT
duke@435	2501	// assert(src->klass() != NULL);
duke@435	2502	BLOCK_COMMENT("assert klasses not null");
duke@435	2503	{ Label L1, L2;
duke@435	2504	__ testq(r10_src_klass, r10_src_klass);
duke@435	2505	__ jcc(Assembler::notZero, L2); // it is broken if klass is NULL
duke@435	2506	__ bind(L1);
duke@435	2507	__ stop("broken null klass");
duke@435	2508	__ bind(L2);
coleenp@548	2509	__ load_klass(r9_dst_klass, dst);
coleenp@548	2510	__ cmpq(r9_dst_klass, 0);
duke@435	2511	__ jcc(Assembler::equal, L1); // this would be broken also
duke@435	2512	BLOCK_COMMENT("assert done");
duke@435	2513	}
duke@435	2514	#endif
duke@435	2515
duke@435	2516	// Load layout helper (32-bits)
duke@435	2517	//
duke@435	2518	// |array_tag| | header_size | element_type | |log2_element_size|
duke@435	2519	// 32 30 24 16 8 2 0
duke@435	2520	//
duke@435	2521	// array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
duke@435	2522	//
duke@435	2523
duke@435	2524	int lh_offset = klassOopDesc::header_size() * HeapWordSize +
duke@435	2525	Klass::layout_helper_offset_in_bytes();
duke@435	2526
duke@435	2527	const Register rax_lh = rax; // layout helper
duke@435	2528
duke@435	2529	__ movl(rax_lh, Address(r10_src_klass, lh_offset));
duke@435	2530
duke@435	2531	// Handle objArrays completely differently...
duke@435	2532	jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
duke@435	2533	__ cmpl(rax_lh, objArray_lh);
duke@435	2534	__ jcc(Assembler::equal, L_objArray);
duke@435	2535
duke@435	2536	// if (src->klass() != dst->klass()) return -1;
coleenp@548	2537	__ load_klass(r9_dst_klass, dst);
coleenp@548	2538	__ cmpq(r10_src_klass, r9_dst_klass);
duke@435	2539	__ jcc(Assembler::notEqual, L_failed);
duke@435	2540
duke@435	2541	// if (!src->is_Array()) return -1;
duke@435	2542	__ cmpl(rax_lh, Klass::_lh_neutral_value);
duke@435	2543	__ jcc(Assembler::greaterEqual, L_failed);
duke@435	2544
duke@435	2545	// At this point, it is known to be a typeArray (array_tag 0x3).
duke@435	2546	#ifdef ASSERT
duke@435	2547	{ Label L;
duke@435	2548	__ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
duke@435	2549	__ jcc(Assembler::greaterEqual, L);
duke@435	2550	__ stop("must be a primitive array");
duke@435	2551	__ bind(L);
duke@435	2552	}
duke@435	2553	#endif
duke@435	2554
duke@435	2555	arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
duke@435	2556	r10, L_failed);
duke@435	2557
duke@435	2558	// typeArrayKlass
duke@435	2559	//
duke@435	2560	// src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
duke@435	2561	// dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
duke@435	2562	//
duke@435	2563
duke@435	2564	const Register r10_offset = r10; // array offset
duke@435	2565	const Register rax_elsize = rax_lh; // element size
duke@435	2566
duke@435	2567	__ movl(r10_offset, rax_lh);
duke@435	2568	__ shrl(r10_offset, Klass::_lh_header_size_shift);
duke@435	2569	__ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset
duke@435	2570	__ addq(src, r10_offset); // src array offset
duke@435	2571	__ addq(dst, r10_offset); // dst array offset
duke@435	2572	BLOCK_COMMENT("choose copy loop based on element size");
duke@435	2573	__ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
duke@435	2574
duke@435	2575	// next registers should be set before the jump to corresponding stub
duke@435	2576	const Register from = c_rarg0; // source array address
duke@435	2577	const Register to = c_rarg1; // destination array address
duke@435	2578	const Register count = c_rarg2; // elements count
duke@435	2579
duke@435	2580	// 'from', 'to', 'count' registers should be set in such order
duke@435	2581	// since they are the same as 'src', 'src_pos', 'dst'.
duke@435	2582
duke@435	2583	__ BIND(L_copy_bytes);
duke@435	2584	__ cmpl(rax_elsize, 0);
duke@435	2585	__ jccb(Assembler::notEqual, L_copy_shorts);
duke@435	2586	__ leaq(from, Address(src, src_pos, Address::times_1, 0));// src_addr
duke@435	2587	__ leaq(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr
duke@435	2588	__ movslq(count, r11_length); // length
duke@435	2589	__ jump(RuntimeAddress(byte_copy_entry));
duke@435	2590
duke@435	2591	__ BIND(L_copy_shorts);
duke@435	2592	__ cmpl(rax_elsize, LogBytesPerShort);
duke@435	2593	__ jccb(Assembler::notEqual, L_copy_ints);
duke@435	2594	__ leaq(from, Address(src, src_pos, Address::times_2, 0));// src_addr
duke@435	2595	__ leaq(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr
duke@435	2596	__ movslq(count, r11_length); // length
duke@435	2597	__ jump(RuntimeAddress(short_copy_entry));
duke@435	2598
duke@435	2599	__ BIND(L_copy_ints);
duke@435	2600	__ cmpl(rax_elsize, LogBytesPerInt);
duke@435	2601	__ jccb(Assembler::notEqual, L_copy_longs);
duke@435	2602	__ leaq(from, Address(src, src_pos, Address::times_4, 0));// src_addr
duke@435	2603	__ leaq(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr
duke@435	2604	__ movslq(count, r11_length); // length
duke@435	2605	__ jump(RuntimeAddress(int_copy_entry));
duke@435	2606
duke@435	2607	__ BIND(L_copy_longs);
duke@435	2608	#ifdef ASSERT
duke@435	2609	{ Label L;
duke@435	2610	__ cmpl(rax_elsize, LogBytesPerLong);
duke@435	2611	__ jcc(Assembler::equal, L);
duke@435	2612	__ stop("must be long copy, but elsize is wrong");
duke@435	2613	__ bind(L);
duke@435	2614	}
duke@435	2615	#endif
duke@435	2616	__ leaq(from, Address(src, src_pos, Address::times_8, 0));// src_addr
duke@435	2617	__ leaq(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr
duke@435	2618	__ movslq(count, r11_length); // length
duke@435	2619	__ jump(RuntimeAddress(long_copy_entry));
duke@435	2620
duke@435	2621	// objArrayKlass
duke@435	2622	__ BIND(L_objArray);
duke@435	2623	// live at this point: r10_src_klass, src[_pos], dst[_pos]
duke@435	2624
duke@435	2625	Label L_plain_copy, L_checkcast_copy;
duke@435	2626	// test array classes for subtyping
coleenp@548	2627	__ load_klass(r9_dst_klass, dst);
coleenp@548	2628	__ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality
duke@435	2629	__ jcc(Assembler::notEqual, L_checkcast_copy);
duke@435	2630
duke@435	2631	// Identically typed arrays can be copied without element-wise checks.
duke@435	2632	arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
duke@435	2633	r10, L_failed);
duke@435	2634
coleenp@548	2635	__ leaq(from, Address(src, src_pos, TIMES_OOP,
duke@435	2636	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
coleenp@548	2637	__ leaq(to, Address(dst, dst_pos, TIMES_OOP,
coleenp@548	2638	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
duke@435	2639	__ movslq(count, r11_length); // length
duke@435	2640	__ BIND(L_plain_copy);
duke@435	2641	__ jump(RuntimeAddress(oop_copy_entry));
duke@435	2642
duke@435	2643	__ BIND(L_checkcast_copy);
duke@435	2644	// live at this point: r10_src_klass, !r11_length
duke@435	2645	{
duke@435	2646	// assert(r11_length == C_RARG4); // will reload from here
duke@435	2647	Register r11_dst_klass = r11;
coleenp@548	2648	__ load_klass(r11_dst_klass, dst);
duke@435	2649
duke@435	2650	// Before looking at dst.length, make sure dst is also an objArray.
duke@435	2651	__ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh);
duke@435	2652	__ jcc(Assembler::notEqual, L_failed);
duke@435	2653
duke@435	2654	// It is safe to examine both src.length and dst.length.
duke@435	2655	#ifndef _WIN64
duke@435	2656	arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4,
duke@435	2657	rax, L_failed);
duke@435	2658	#else
duke@435	2659	__ movl(r11_length, C_RARG4); // reload
duke@435	2660	arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
duke@435	2661	rax, L_failed);
coleenp@548	2662	__ load_klass(r11_dst_klass, dst); // reload
duke@435	2663	#endif
duke@435	2664
duke@435	2665	// Marshal the base address arguments now, freeing registers.
coleenp@548	2666	__ leaq(from, Address(src, src_pos, TIMES_OOP,
duke@435	2667	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
coleenp@548	2668	__ leaq(to, Address(dst, dst_pos, TIMES_OOP,
duke@435	2669	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
duke@435	2670	__ movl(count, C_RARG4); // length (reloaded)
duke@435	2671	Register sco_temp = c_rarg3; // this register is free now
duke@435	2672	assert_different_registers(from, to, count, sco_temp,
duke@435	2673	r11_dst_klass, r10_src_klass);
duke@435	2674	assert_clean_int(count, sco_temp);
duke@435	2675
duke@435	2676	// Generate the type check.
duke@435	2677	int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	2678	Klass::super_check_offset_offset_in_bytes());
duke@435	2679	__ movl(sco_temp, Address(r11_dst_klass, sco_offset));
duke@435	2680	assert_clean_int(sco_temp, rax);
duke@435	2681	generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
duke@435	2682
duke@435	2683	// Fetch destination element klass from the objArrayKlass header.
duke@435	2684	int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	2685	objArrayKlass::element_klass_offset_in_bytes());
duke@435	2686	__ movq(r11_dst_klass, Address(r11_dst_klass, ek_offset));
duke@435	2687	__ movl(sco_temp, Address(r11_dst_klass, sco_offset));
duke@435	2688	assert_clean_int(sco_temp, rax);
duke@435	2689
duke@435	2690	// the checkcast_copy loop needs two extra arguments:
duke@435	2691	assert(c_rarg3 == sco_temp, "#3 already in place");
duke@435	2692	__ movq(C_RARG4, r11_dst_klass); // dst.klass.element_klass
duke@435	2693	__ jump(RuntimeAddress(checkcast_copy_entry));
duke@435	2694	}
duke@435	2695
duke@435	2696	__ BIND(L_failed);
duke@435	2697	__ xorq(rax, rax);
duke@435	2698	__ notq(rax); // return -1
duke@435	2699	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2700	__ ret(0);
duke@435	2701
duke@435	2702	return start;
duke@435	2703	}
duke@435	2704
duke@435	2705	#undef length_arg
duke@435	2706
duke@435	2707	void generate_arraycopy_stubs() {
duke@435	2708	// Call the conjoint generation methods immediately after
duke@435	2709	// the disjoint ones so that short branches from the former
duke@435	2710	// to the latter can be generated.
duke@435	2711	StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy");
duke@435	2712	StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, "jbyte_arraycopy");
duke@435	2713
duke@435	2714	StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy");
duke@435	2715	StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, "jshort_arraycopy");
duke@435	2716
coleenp@548	2717	StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, false, "jint_disjoint_arraycopy");
coleenp@548	2718	StubRoutines::_jint_arraycopy = generate_conjoint_int_oop_copy(false, false, "jint_arraycopy");
duke@435	2719
duke@435	2720	StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, false, "jlong_disjoint_arraycopy");
duke@435	2721	StubRoutines::_jlong_arraycopy = generate_conjoint_long_oop_copy(false, false, "jlong_arraycopy");
duke@435	2722
coleenp@548	2723
coleenp@548	2724	if (UseCompressedOops) {
coleenp@548	2725	StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_int_oop_copy(false, true, "oop_disjoint_arraycopy");
coleenp@548	2726	StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, "oop_arraycopy");
coleenp@548	2727	} else {
coleenp@548	2728	StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, "oop_disjoint_arraycopy");
coleenp@548	2729	StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, "oop_arraycopy");
coleenp@548	2730	}
duke@435	2731
duke@435	2732	StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy");
duke@435	2733	StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy");
duke@435	2734	StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy");
duke@435	2735
duke@435	2736	// We don't generate specialized code for HeapWord-aligned source
duke@435	2737	// arrays, so just use the code we've already generated
duke@435	2738	StubRoutines::_arrayof_jbyte_disjoint_arraycopy = StubRoutines::_jbyte_disjoint_arraycopy;
duke@435	2739	StubRoutines::_arrayof_jbyte_arraycopy = StubRoutines::_jbyte_arraycopy;
duke@435	2740
duke@435	2741	StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
duke@435	2742	StubRoutines::_arrayof_jshort_arraycopy = StubRoutines::_jshort_arraycopy;
duke@435	2743
duke@435	2744	StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy;
duke@435	2745	StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
duke@435	2746
duke@435	2747	StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy;
duke@435	2748	StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
duke@435	2749
duke@435	2750	StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy;
duke@435	2751	StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
duke@435	2752	}
duke@435	2753
duke@435	2754	#undef __
duke@435	2755	#define __ masm->
duke@435	2756
duke@435	2757	// Continuation point for throwing of implicit exceptions that are
duke@435	2758	// not handled in the current activation. Fabricates an exception
duke@435	2759	// oop and initiates normal exception dispatching in this
duke@435	2760	// frame. Since we need to preserve callee-saved values (currently
duke@435	2761	// only for C2, but done for C1 as well) we need a callee-saved oop
duke@435	2762	// map and therefore have to make these stubs into RuntimeStubs
duke@435	2763	// rather than BufferBlobs. If the compiler needs all registers to
duke@435	2764	// be preserved between the fault point and the exception handler
duke@435	2765	// then it must assume responsibility for that in
duke@435	2766	// AbstractCompiler::continuation_for_implicit_null_exception or
duke@435	2767	// continuation_for_implicit_division_by_zero_exception. All other
duke@435	2768	// implicit exceptions (e.g., NullPointerException or
duke@435	2769	// AbstractMethodError on entry) are either at call sites or
duke@435	2770	// otherwise assume that stack unwinding will be initiated, so
duke@435	2771	// caller saved registers were assumed volatile in the compiler.
duke@435	2772	address generate_throw_exception(const char* name,
duke@435	2773	address runtime_entry,
duke@435	2774	bool restore_saved_exception_pc) {
duke@435	2775	// Information about frame layout at time of blocking runtime call.
duke@435	2776	// Note that we only have to preserve callee-saved registers since
duke@435	2777	// the compilers are responsible for supplying a continuation point
duke@435	2778	// if they expect all registers to be preserved.
duke@435	2779	enum layout {
duke@435	2780	rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
duke@435	2781	rbp_off2,
duke@435	2782	return_off,
duke@435	2783	return_off2,
duke@435	2784	framesize // inclusive of return address
duke@435	2785	};
duke@435	2786
duke@435	2787	int insts_size = 512;
duke@435	2788	int locs_size = 64;
duke@435	2789
duke@435	2790	CodeBuffer code(name, insts_size, locs_size);
duke@435	2791	OopMapSet* oop_maps = new OopMapSet();
duke@435	2792	MacroAssembler* masm = new MacroAssembler(&code);
duke@435	2793
duke@435	2794	address start = __ pc();
duke@435	2795
duke@435	2796	// This is an inlined and slightly modified version of call_VM
duke@435	2797	// which has the ability to fetch the return PC out of
duke@435	2798	// thread-local storage and also sets up last_Java_sp slightly
duke@435	2799	// differently than the real call_VM
duke@435	2800	if (restore_saved_exception_pc) {
duke@435	2801	__ movq(rax,
duke@435	2802	Address(r15_thread,
duke@435	2803	in_bytes(JavaThread::saved_exception_pc_offset())));
duke@435	2804	__ pushq(rax);
duke@435	2805	}
duke@435	2806
duke@435	2807	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2808
duke@435	2809	assert(is_even(framesize/2), "sp not 16-byte aligned");
duke@435	2810
duke@435	2811	// return address and rbp are already in place
duke@435	2812	__ subq(rsp, (framesize-4) << LogBytesPerInt); // prolog
duke@435	2813
duke@435	2814	int frame_complete = __ pc() - start;
duke@435	2815
duke@435	2816	// Set up last_Java_sp and last_Java_fp
duke@435	2817	__ set_last_Java_frame(rsp, rbp, NULL);
duke@435	2818
duke@435	2819	// Call runtime
duke@435	2820	__ movq(c_rarg0, r15_thread);
duke@435	2821	BLOCK_COMMENT("call runtime_entry");
duke@435	2822	__ call(RuntimeAddress(runtime_entry));
duke@435	2823
duke@435	2824	// Generate oop map
duke@435	2825	OopMap* map = new OopMap(framesize, 0);
duke@435	2826
duke@435	2827	oop_maps->add_gc_map(__ pc() - start, map);
duke@435	2828
duke@435	2829	__ reset_last_Java_frame(true, false);
duke@435	2830
duke@435	2831	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2832
duke@435	2833	// check for pending exceptions
duke@435	2834	#ifdef ASSERT
duke@435	2835	Label L;
duke@435	2836	__ cmpq(Address(r15_thread, Thread::pending_exception_offset()),
duke@435	2837	(int) NULL);
duke@435	2838	__ jcc(Assembler::notEqual, L);
duke@435	2839	__ should_not_reach_here();
duke@435	2840	__ bind(L);
duke@435	2841	#endif // ASSERT
duke@435	2842	__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	2843
duke@435	2844
duke@435	2845	// codeBlob framesize is in words (not VMRegImpl::slot_size)
duke@435	2846	RuntimeStub* stub =
duke@435	2847	RuntimeStub::new_runtime_stub(name,
duke@435	2848	&code,
duke@435	2849	frame_complete,
duke@435	2850	(framesize >> (LogBytesPerWord - LogBytesPerInt)),
duke@435	2851	oop_maps, false);
duke@435	2852	return stub->entry_point();
duke@435	2853	}
duke@435	2854
duke@435	2855	// Initialization
duke@435	2856	void generate_initial() {
duke@435	2857	// Generates all stubs and initializes the entry points
duke@435	2858
duke@435	2859	// This platform-specific stub is needed by generate_call_stub()
duke@435	2860	StubRoutines::amd64::_mxcsr_std = generate_fp_mask("mxcsr_std", 0x0000000000001F80);
duke@435	2861
duke@435	2862	// entry points that exist in all platforms Note: This is code
duke@435	2863	// that could be shared among different platforms - however the
duke@435	2864	// benefit seems to be smaller than the disadvantage of having a
duke@435	2865	// much more complicated generator structure. See also comment in
duke@435	2866	// stubRoutines.hpp.
duke@435	2867
duke@435	2868	StubRoutines::_forward_exception_entry = generate_forward_exception();
duke@435	2869
duke@435	2870	StubRoutines::_call_stub_entry =
duke@435	2871	generate_call_stub(StubRoutines::_call_stub_return_address);
duke@435	2872
duke@435	2873	// is referenced by megamorphic call
duke@435	2874	StubRoutines::_catch_exception_entry = generate_catch_exception();
duke@435	2875
duke@435	2876	// atomic calls
duke@435	2877	StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
duke@435	2878	StubRoutines::_atomic_xchg_ptr_entry = generate_atomic_xchg_ptr();
duke@435	2879	StubRoutines::_atomic_cmpxchg_entry = generate_atomic_cmpxchg();
duke@435	2880	StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
duke@435	2881	StubRoutines::_atomic_add_entry = generate_atomic_add();
duke@435	2882	StubRoutines::_atomic_add_ptr_entry = generate_atomic_add_ptr();
duke@435	2883	StubRoutines::_fence_entry = generate_orderaccess_fence();
duke@435	2884
duke@435	2885	StubRoutines::_handler_for_unsafe_access_entry =
duke@435	2886	generate_handler_for_unsafe_access();
duke@435	2887
duke@435	2888	// platform dependent
duke@435	2889	StubRoutines::amd64::_get_previous_fp_entry = generate_get_previous_fp();
duke@435	2890
duke@435	2891	StubRoutines::amd64::_verify_mxcsr_entry = generate_verify_mxcsr();
duke@435	2892	}
duke@435	2893
duke@435	2894	void generate_all() {
duke@435	2895	// Generates all stubs and initializes the entry points
duke@435	2896
duke@435	2897	// These entry points require SharedInfo::stack0 to be set up in
duke@435	2898	// non-core builds and need to be relocatable, so they each
duke@435	2899	// fabricate a RuntimeStub internally.
duke@435	2900	StubRoutines::_throw_AbstractMethodError_entry =
duke@435	2901	generate_throw_exception("AbstractMethodError throw_exception",
duke@435	2902	CAST_FROM_FN_PTR(address,
duke@435	2903	SharedRuntime::
duke@435	2904	throw_AbstractMethodError),
duke@435	2905	false);
duke@435	2906
dcubed@451	2907	StubRoutines::_throw_IncompatibleClassChangeError_entry =
dcubed@451	2908	generate_throw_exception("IncompatibleClassChangeError throw_exception",
dcubed@451	2909	CAST_FROM_FN_PTR(address,
dcubed@451	2910	SharedRuntime::
dcubed@451	2911	throw_IncompatibleClassChangeError),
dcubed@451	2912	false);
dcubed@451	2913
duke@435	2914	StubRoutines::_throw_ArithmeticException_entry =
duke@435	2915	generate_throw_exception("ArithmeticException throw_exception",
duke@435	2916	CAST_FROM_FN_PTR(address,
duke@435	2917	SharedRuntime::
duke@435	2918	throw_ArithmeticException),
duke@435	2919	true);
duke@435	2920
duke@435	2921	StubRoutines::_throw_NullPointerException_entry =
duke@435	2922	generate_throw_exception("NullPointerException throw_exception",
duke@435	2923	CAST_FROM_FN_PTR(address,
duke@435	2924	SharedRuntime::
duke@435	2925	throw_NullPointerException),
duke@435	2926	true);
duke@435	2927
duke@435	2928	StubRoutines::_throw_NullPointerException_at_call_entry =
duke@435	2929	generate_throw_exception("NullPointerException at call throw_exception",
duke@435	2930	CAST_FROM_FN_PTR(address,
duke@435	2931	SharedRuntime::
duke@435	2932	throw_NullPointerException_at_call),
duke@435	2933	false);
duke@435	2934
duke@435	2935	StubRoutines::_throw_StackOverflowError_entry =
duke@435	2936	generate_throw_exception("StackOverflowError throw_exception",
duke@435	2937	CAST_FROM_FN_PTR(address,
duke@435	2938	SharedRuntime::
duke@435	2939	throw_StackOverflowError),
duke@435	2940	false);
duke@435	2941
duke@435	2942	// entry points that are platform specific
duke@435	2943	StubRoutines::amd64::_f2i_fixup = generate_f2i_fixup();
duke@435	2944	StubRoutines::amd64::_f2l_fixup = generate_f2l_fixup();
duke@435	2945	StubRoutines::amd64::_d2i_fixup = generate_d2i_fixup();
duke@435	2946	StubRoutines::amd64::_d2l_fixup = generate_d2l_fixup();
duke@435	2947
duke@435	2948	StubRoutines::amd64::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF);
duke@435	2949	StubRoutines::amd64::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000);
duke@435	2950	StubRoutines::amd64::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
duke@435	2951	StubRoutines::amd64::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
duke@435	2952
duke@435	2953	// support for verify_oop (must happen after universe_init)
duke@435	2954	StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
duke@435	2955
duke@435	2956	// arraycopy stubs used by compilers
duke@435	2957	generate_arraycopy_stubs();
duke@435	2958	}
duke@435	2959
duke@435	2960	public:
duke@435	2961	StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
duke@435	2962	if (all) {
duke@435	2963	generate_all();
duke@435	2964	} else {
duke@435	2965	generate_initial();
duke@435	2966	}
duke@435	2967	}
duke@435	2968	}; // end class declaration
duke@435	2969
duke@435	2970	address StubGenerator::disjoint_byte_copy_entry = NULL;
duke@435	2971	address StubGenerator::disjoint_short_copy_entry = NULL;
duke@435	2972	address StubGenerator::disjoint_int_copy_entry = NULL;
duke@435	2973	address StubGenerator::disjoint_long_copy_entry = NULL;
duke@435	2974	address StubGenerator::disjoint_oop_copy_entry = NULL;
duke@435	2975
duke@435	2976	address StubGenerator::byte_copy_entry = NULL;
duke@435	2977	address StubGenerator::short_copy_entry = NULL;
duke@435	2978	address StubGenerator::int_copy_entry = NULL;
duke@435	2979	address StubGenerator::long_copy_entry = NULL;
duke@435	2980	address StubGenerator::oop_copy_entry = NULL;
duke@435	2981
duke@435	2982	address StubGenerator::checkcast_copy_entry = NULL;
duke@435	2983
duke@435	2984	void StubGenerator_generate(CodeBuffer* code, bool all) {
duke@435	2985	StubGenerator g(code, all);
duke@435	2986	}