• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/stubGenerator_x86_32.cpp@9eba43136cb5 (annotated)

src/cpu/x86/vm/stubGenerator_x86_32.cpp

Tue, 16 Mar 2010 11:52:17 +0100

author

twisti

date

Tue, 16 Mar 2010 11:52:17 +0100

changeset 1734

9eba43136cb5

parent 1730

3cf667df43ef

child 1800

6476042f815c

permissions

-rw-r--r--

6934494: JSR 292 MethodHandles adapters should be generated into their own CodeBlob
Summary: Passing a null pointer to an InvokeDynamic function call should lead to a NullPointerException.
Reviewed-by: kvn, never

duke@435	1	/*
never@1609	2	* Copyright 1999-2010 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_stubGenerator_x86_32.cpp.incl"
duke@435	27
duke@435	28	// Declaration and definition of StubGenerator (no .hpp file).
duke@435	29	// For a more detailed description of the stub routine structure
duke@435	30	// see the comment in stubRoutines.hpp
duke@435	31
duke@435	32	#define __ _masm->
never@739	33	#define a__ ((Assembler*)_masm)->
duke@435	34
duke@435	35	#ifdef PRODUCT
duke@435	36	#define BLOCK_COMMENT(str) /* nothing */
duke@435	37	#else
duke@435	38	#define BLOCK_COMMENT(str) __ block_comment(str)
duke@435	39	#endif
duke@435	40
duke@435	41	#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
duke@435	42
duke@435	43	const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
duke@435	44	const int FPU_CNTRL_WRD_MASK = 0xFFFF;
duke@435	45
duke@435	46	// -------------------------------------------------------------------------------------------------------------------------
duke@435	47	// Stub Code definitions
duke@435	48
duke@435	49	static address handle_unsafe_access() {
duke@435	50	JavaThread* thread = JavaThread::current();
duke@435	51	address pc = thread->saved_exception_pc();
duke@435	52	// pc is the instruction which we must emulate
duke@435	53	// doing a no-op is fine: return garbage from the load
duke@435	54	// therefore, compute npc
duke@435	55	address npc = Assembler::locate_next_instruction(pc);
duke@435	56
duke@435	57	// request an async exception
duke@435	58	thread->set_pending_unsafe_access_error();
duke@435	59
duke@435	60	// return address of next instruction to execute
duke@435	61	return npc;
duke@435	62	}
duke@435	63
duke@435	64	class StubGenerator: public StubCodeGenerator {
duke@435	65	private:
duke@435	66
duke@435	67	#ifdef PRODUCT
duke@435	68	#define inc_counter_np(counter) (0)
duke@435	69	#else
duke@435	70	void inc_counter_np_(int& counter) {
never@739	71	__ incrementl(ExternalAddress((address)&counter));
duke@435	72	}
duke@435	73	#define inc_counter_np(counter) \
duke@435	74	BLOCK_COMMENT("inc_counter " #counter); \
duke@435	75	inc_counter_np_(counter);
duke@435	76	#endif //PRODUCT
duke@435	77
duke@435	78	void inc_copy_counter_np(BasicType t) {
duke@435	79	#ifndef PRODUCT
duke@435	80	switch (t) {
duke@435	81	case T_BYTE: inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
duke@435	82	case T_SHORT: inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
duke@435	83	case T_INT: inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
duke@435	84	case T_LONG: inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
duke@435	85	case T_OBJECT: inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
duke@435	86	}
duke@435	87	ShouldNotReachHere();
duke@435	88	#endif //PRODUCT
duke@435	89	}
duke@435	90
duke@435	91	//------------------------------------------------------------------------------------------------------------------------
duke@435	92	// Call stubs are used to call Java from C
duke@435	93	//
duke@435	94	// [ return_from_Java ] <--- rsp
duke@435	95	// [ argument word n ]
duke@435	96	// ...
duke@435	97	// -N [ argument word 1 ]
duke@435	98	// -7 [ Possible padding for stack alignment ]
duke@435	99	// -6 [ Possible padding for stack alignment ]
duke@435	100	// -5 [ Possible padding for stack alignment ]
duke@435	101	// -4 [ mxcsr save ] <--- rsp_after_call
duke@435	102	// -3 [ saved rbx, ]
duke@435	103	// -2 [ saved rsi ]
duke@435	104	// -1 [ saved rdi ]
duke@435	105	// 0 [ saved rbp, ] <--- rbp,
duke@435	106	// 1 [ return address ]
duke@435	107	// 2 [ ptr. to call wrapper ]
duke@435	108	// 3 [ result ]
duke@435	109	// 4 [ result_type ]
duke@435	110	// 5 [ method ]
duke@435	111	// 6 [ entry_point ]
duke@435	112	// 7 [ parameters ]
duke@435	113	// 8 [ parameter_size ]
duke@435	114	// 9 [ thread ]
duke@435	115
duke@435	116
duke@435	117	address generate_call_stub(address& return_address) {
duke@435	118	StubCodeMark mark(this, "StubRoutines", "call_stub");
duke@435	119	address start = __ pc();
duke@435	120
duke@435	121	// stub code parameters / addresses
duke@435	122	assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
duke@435	123	bool sse_save = false;
duke@435	124	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
duke@435	125	const int locals_count_in_bytes (4*wordSize);
duke@435	126	const Address mxcsr_save (rbp, -4 * wordSize);
duke@435	127	const Address saved_rbx (rbp, -3 * wordSize);
duke@435	128	const Address saved_rsi (rbp, -2 * wordSize);
duke@435	129	const Address saved_rdi (rbp, -1 * wordSize);
duke@435	130	const Address result (rbp, 3 * wordSize);
duke@435	131	const Address result_type (rbp, 4 * wordSize);
duke@435	132	const Address method (rbp, 5 * wordSize);
duke@435	133	const Address entry_point (rbp, 6 * wordSize);
duke@435	134	const Address parameters (rbp, 7 * wordSize);
duke@435	135	const Address parameter_size(rbp, 8 * wordSize);
duke@435	136	const Address thread (rbp, 9 * wordSize); // same as in generate_catch_exception()!
duke@435	137	sse_save = UseSSE > 0;
duke@435	138
duke@435	139	// stub code
duke@435	140	__ enter();
never@739	141	__ movptr(rcx, parameter_size); // parameter counter
never@739	142	__ shlptr(rcx, Interpreter::logStackElementSize()); // convert parameter count to bytes
never@739	143	__ addptr(rcx, locals_count_in_bytes); // reserve space for register saves
never@739	144	__ subptr(rsp, rcx);
never@739	145	__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
duke@435	146
duke@435	147	// save rdi, rsi, & rbx, according to C calling conventions
never@739	148	__ movptr(saved_rdi, rdi);
never@739	149	__ movptr(saved_rsi, rsi);
never@739	150	__ movptr(saved_rbx, rbx);
duke@435	151	// save and initialize %mxcsr
duke@435	152	if (sse_save) {
duke@435	153	Label skip_ldmx;
duke@435	154	__ stmxcsr(mxcsr_save);
duke@435	155	__ movl(rax, mxcsr_save);
duke@435	156	__ andl(rax, MXCSR_MASK); // Only check control and mask bits
duke@435	157	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
duke@435	158	__ cmp32(rax, mxcsr_std);
duke@435	159	__ jcc(Assembler::equal, skip_ldmx);
duke@435	160	__ ldmxcsr(mxcsr_std);
duke@435	161	__ bind(skip_ldmx);
duke@435	162	}
duke@435	163
duke@435	164	// make sure the control word is correct.
duke@435	165	__ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
duke@435	166
duke@435	167	#ifdef ASSERT
duke@435	168	// make sure we have no pending exceptions
duke@435	169	{ Label L;
never@739	170	__ movptr(rcx, thread);
never@739	171	__ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	172	__ jcc(Assembler::equal, L);
duke@435	173	__ stop("StubRoutines::call_stub: entered with pending exception");
duke@435	174	__ bind(L);
duke@435	175	}
duke@435	176	#endif
duke@435	177
duke@435	178	// pass parameters if any
duke@435	179	BLOCK_COMMENT("pass parameters if any");
duke@435	180	Label parameters_done;
duke@435	181	__ movl(rcx, parameter_size); // parameter counter
duke@435	182	__ testl(rcx, rcx);
duke@435	183	__ jcc(Assembler::zero, parameters_done);
duke@435	184
duke@435	185	// parameter passing loop
duke@435	186
duke@435	187	Label loop;
duke@435	188	// Copy Java parameters in reverse order (receiver last)
duke@435	189	// Note that the argument order is inverted in the process
duke@435	190	// source is rdx[rcx: N-1..0]
duke@435	191	// dest is rsp[rbx: 0..N-1]
duke@435	192
never@739	193	__ movptr(rdx, parameters); // parameter pointer
never@739	194	__ xorptr(rbx, rbx);
duke@435	195
duke@435	196	__ BIND(loop);
duke@435	197	if (TaggedStackInterpreter) {
never@739	198	__ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(),
duke@435	199	-2*wordSize)); // get tag
never@739	200	__ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
duke@435	201	Interpreter::expr_tag_offset_in_bytes(0)), rax); // store tag
duke@435	202	}
duke@435	203
duke@435	204	// get parameter
never@739	205	__ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
never@739	206	__ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
duke@435	207	Interpreter::expr_offset_in_bytes(0)), rax); // store parameter
duke@435	208	__ increment(rbx);
duke@435	209	__ decrement(rcx);
duke@435	210	__ jcc(Assembler::notZero, loop);
duke@435	211
duke@435	212	// call Java function
duke@435	213	__ BIND(parameters_done);
never@739	214	__ movptr(rbx, method); // get methodOop
never@739	215	__ movptr(rax, entry_point); // get entry_point
never@739	216	__ mov(rsi, rsp); // set sender sp
duke@435	217	BLOCK_COMMENT("call Java function");
duke@435	218	__ call(rax);
duke@435	219
duke@435	220	BLOCK_COMMENT("call_stub_return_address:");
duke@435	221	return_address = __ pc();
duke@435	222
duke@435	223	Label common_return;
duke@435	224
duke@435	225	__ BIND(common_return);
duke@435	226
duke@435	227	// store result depending on type
duke@435	228	// (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
never@739	229	__ movptr(rdi, result);
duke@435	230	Label is_long, is_float, is_double, exit;
duke@435	231	__ movl(rsi, result_type);
duke@435	232	__ cmpl(rsi, T_LONG);
duke@435	233	__ jcc(Assembler::equal, is_long);
duke@435	234	__ cmpl(rsi, T_FLOAT);
duke@435	235	__ jcc(Assembler::equal, is_float);
duke@435	236	__ cmpl(rsi, T_DOUBLE);
duke@435	237	__ jcc(Assembler::equal, is_double);
duke@435	238
duke@435	239	// handle T_INT case
duke@435	240	__ movl(Address(rdi, 0), rax);
duke@435	241	__ BIND(exit);
duke@435	242
duke@435	243	// check that FPU stack is empty
duke@435	244	__ verify_FPU(0, "generate_call_stub");
duke@435	245
duke@435	246	// pop parameters
never@739	247	__ lea(rsp, rsp_after_call);
duke@435	248
duke@435	249	// restore %mxcsr
duke@435	250	if (sse_save) {
duke@435	251	__ ldmxcsr(mxcsr_save);
duke@435	252	}
duke@435	253
duke@435	254	// restore rdi, rsi and rbx,
never@739	255	__ movptr(rbx, saved_rbx);
never@739	256	__ movptr(rsi, saved_rsi);
never@739	257	__ movptr(rdi, saved_rdi);
never@739	258	__ addptr(rsp, 4*wordSize);
duke@435	259
duke@435	260	// return
never@739	261	__ pop(rbp);
duke@435	262	__ ret(0);
duke@435	263
duke@435	264	// handle return types different from T_INT
duke@435	265	__ BIND(is_long);
duke@435	266	__ movl(Address(rdi, 0 * wordSize), rax);
duke@435	267	__ movl(Address(rdi, 1 * wordSize), rdx);
duke@435	268	__ jmp(exit);
duke@435	269
duke@435	270	__ BIND(is_float);
duke@435	271	// interpreter uses xmm0 for return values
duke@435	272	if (UseSSE >= 1) {
duke@435	273	__ movflt(Address(rdi, 0), xmm0);
duke@435	274	} else {
duke@435	275	__ fstp_s(Address(rdi, 0));
duke@435	276	}
duke@435	277	__ jmp(exit);
duke@435	278
duke@435	279	__ BIND(is_double);
duke@435	280	// interpreter uses xmm0 for return values
duke@435	281	if (UseSSE >= 2) {
duke@435	282	__ movdbl(Address(rdi, 0), xmm0);
duke@435	283	} else {
duke@435	284	__ fstp_d(Address(rdi, 0));
duke@435	285	}
duke@435	286	__ jmp(exit);
duke@435	287
duke@435	288	// If we call compiled code directly from the call stub we will
duke@435	289	// need to adjust the return back to the call stub to a specialized
duke@435	290	// piece of code that can handle compiled results and cleaning the fpu
duke@435	291	// stack. compiled code will be set to return here instead of the
duke@435	292	// return above that handles interpreter returns.
duke@435	293
duke@435	294	BLOCK_COMMENT("call_stub_compiled_return:");
never@739	295	StubRoutines::x86::set_call_stub_compiled_return( __ pc());
duke@435	296
duke@435	297	#ifdef COMPILER2
duke@435	298	if (UseSSE >= 2) {
duke@435	299	__ verify_FPU(0, "call_stub_compiled_return");
duke@435	300	} else {
duke@435	301	for (int i = 1; i < 8; i++) {
duke@435	302	__ ffree(i);
duke@435	303	}
duke@435	304
duke@435	305	// UseSSE <= 1 so double result should be left on TOS
duke@435	306	__ movl(rsi, result_type);
duke@435	307	__ cmpl(rsi, T_DOUBLE);
duke@435	308	__ jcc(Assembler::equal, common_return);
duke@435	309	if (UseSSE == 0) {
duke@435	310	// UseSSE == 0 so float result should be left on TOS
duke@435	311	__ cmpl(rsi, T_FLOAT);
duke@435	312	__ jcc(Assembler::equal, common_return);
duke@435	313	}
duke@435	314	__ ffree(0);
duke@435	315	}
duke@435	316	#endif /* COMPILER2 */
duke@435	317	__ jmp(common_return);
duke@435	318
duke@435	319	return start;
duke@435	320	}
duke@435	321
duke@435	322
duke@435	323	//------------------------------------------------------------------------------------------------------------------------
duke@435	324	// Return point for a Java call if there's an exception thrown in Java code.
duke@435	325	// The exception is caught and transformed into a pending exception stored in
duke@435	326	// JavaThread that can be tested from within the VM.
duke@435	327	//
duke@435	328	// Note: Usually the parameters are removed by the callee. In case of an exception
duke@435	329	// crossing an activation frame boundary, that is not the case if the callee
duke@435	330	// is compiled code => need to setup the rsp.
duke@435	331	//
duke@435	332	// rax,: exception oop
duke@435	333
duke@435	334	address generate_catch_exception() {
duke@435	335	StubCodeMark mark(this, "StubRoutines", "catch_exception");
duke@435	336	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
duke@435	337	const Address thread (rbp, 9 * wordSize); // same as in generate_call_stub()!
duke@435	338	address start = __ pc();
duke@435	339
duke@435	340	// get thread directly
never@739	341	__ movptr(rcx, thread);
duke@435	342	#ifdef ASSERT
duke@435	343	// verify that threads correspond
duke@435	344	{ Label L;
duke@435	345	__ get_thread(rbx);
never@739	346	__ cmpptr(rbx, rcx);
duke@435	347	__ jcc(Assembler::equal, L);
duke@435	348	__ stop("StubRoutines::catch_exception: threads must correspond");
duke@435	349	__ bind(L);
duke@435	350	}
duke@435	351	#endif
duke@435	352	// set pending exception
duke@435	353	__ verify_oop(rax);
never@739	354	__ movptr(Address(rcx, Thread::pending_exception_offset()), rax );
duke@435	355	__ lea(Address(rcx, Thread::exception_file_offset ()),
duke@435	356	ExternalAddress((address)__FILE__));
duke@435	357	__ movl(Address(rcx, Thread::exception_line_offset ()), __LINE__ );
duke@435	358	// complete return to VM
duke@435	359	assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
duke@435	360	__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
duke@435	361
duke@435	362	return start;
duke@435	363	}
duke@435	364
duke@435	365
duke@435	366	//------------------------------------------------------------------------------------------------------------------------
duke@435	367	// Continuation point for runtime calls returning with a pending exception.
duke@435	368	// The pending exception check happened in the runtime or native call stub.
duke@435	369	// The pending exception in Thread is converted into a Java-level exception.
duke@435	370	//
duke@435	371	// Contract with Java-level exception handlers:
twisti@1730	372	// rax: exception
duke@435	373	// rdx: throwing pc
duke@435	374	//
duke@435	375	// NOTE: At entry of this stub, exception-pc must be on stack !!
duke@435	376
duke@435	377	address generate_forward_exception() {
duke@435	378	StubCodeMark mark(this, "StubRoutines", "forward exception");
duke@435	379	address start = __ pc();
twisti@1730	380	const Register thread = rcx;
twisti@1730	381
twisti@1730	382	// other registers used in this stub
twisti@1730	383	const Register exception_oop = rax;
twisti@1730	384	const Register handler_addr = rbx;
twisti@1730	385	const Register exception_pc = rdx;
duke@435	386
duke@435	387	// Upon entry, the sp points to the return address returning into Java
duke@435	388	// (interpreted or compiled) code; i.e., the return address becomes the
duke@435	389	// throwing pc.
duke@435	390	//
duke@435	391	// Arguments pushed before the runtime call are still on the stack but
duke@435	392	// the exception handler will reset the stack pointer -> ignore them.
duke@435	393	// A potential result in registers can be ignored as well.
duke@435	394
duke@435	395	#ifdef ASSERT
duke@435	396	// make sure this code is only executed if there is a pending exception
duke@435	397	{ Label L;
twisti@1730	398	__ get_thread(thread);
twisti@1730	399	__ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	400	__ jcc(Assembler::notEqual, L);
duke@435	401	__ stop("StubRoutines::forward exception: no pending exception (1)");
duke@435	402	__ bind(L);
duke@435	403	}
duke@435	404	#endif
duke@435	405
duke@435	406	// compute exception handler into rbx,
twisti@1730	407	__ get_thread(thread);
twisti@1730	408	__ movptr(exception_pc, Address(rsp, 0));
duke@435	409	BLOCK_COMMENT("call exception_handler_for_return_address");
twisti@1730	410	__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
twisti@1730	411	__ mov(handler_addr, rax);
duke@435	412
twisti@1730	413	// setup rax & rdx, remove return address & clear pending exception
twisti@1730	414	__ get_thread(thread);
twisti@1730	415	__ pop(exception_pc);
twisti@1730	416	__ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
twisti@1730	417	__ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
duke@435	418
duke@435	419	#ifdef ASSERT
duke@435	420	// make sure exception is set
duke@435	421	{ Label L;
twisti@1730	422	__ testptr(exception_oop, exception_oop);
duke@435	423	__ jcc(Assembler::notEqual, L);
duke@435	424	__ stop("StubRoutines::forward exception: no pending exception (2)");
duke@435	425	__ bind(L);
duke@435	426	}
duke@435	427	#endif
duke@435	428
twisti@1730	429	// Verify that there is really a valid exception in RAX.
twisti@1730	430	__ verify_oop(exception_oop);
twisti@1730	431
twisti@1730	432	// Restore SP from BP if the exception PC is a MethodHandle call site.
twisti@1730	433	__ cmpl(Address(thread, JavaThread::is_method_handle_exception_offset()), 0);
twisti@1730	434	__ cmovptr(Assembler::notEqual, rsp, rbp);
twisti@1730	435
duke@435	436	// continue at exception handler (return address removed)
twisti@1730	437	// rax: exception
twisti@1730	438	// rbx: exception handler
duke@435	439	// rdx: throwing pc
twisti@1730	440	__ jmp(handler_addr);
duke@435	441
duke@435	442	return start;
duke@435	443	}
duke@435	444
duke@435	445
duke@435	446	//----------------------------------------------------------------------------------------------------
duke@435	447	// Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
duke@435	448	//
duke@435	449	// xchg exists as far back as 8086, lock needed for MP only
duke@435	450	// Stack layout immediately after call:
duke@435	451	//
duke@435	452	// 0 [ret addr ] <--- rsp
duke@435	453	// 1 [ ex ]
duke@435	454	// 2 [ dest ]
duke@435	455	//
duke@435	456	// Result: *dest <- ex, return (old *dest)
duke@435	457	//
duke@435	458	// Note: win32 does not currently use this code
duke@435	459
duke@435	460	address generate_atomic_xchg() {
duke@435	461	StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
duke@435	462	address start = __ pc();
duke@435	463
never@739	464	__ push(rdx);
duke@435	465	Address exchange(rsp, 2 * wordSize);
duke@435	466	Address dest_addr(rsp, 3 * wordSize);
duke@435	467	__ movl(rax, exchange);
never@739	468	__ movptr(rdx, dest_addr);
never@739	469	__ xchgl(rax, Address(rdx, 0));
never@739	470	__ pop(rdx);
duke@435	471	__ ret(0);
duke@435	472
duke@435	473	return start;
duke@435	474	}
duke@435	475
duke@435	476	//----------------------------------------------------------------------------------------------------
duke@435	477	// Support for void verify_mxcsr()
duke@435	478	//
duke@435	479	// This routine is used with -Xcheck:jni to verify that native
duke@435	480	// JNI code does not return to Java code without restoring the
duke@435	481	// MXCSR register to our expected state.
duke@435	482
duke@435	483
duke@435	484	address generate_verify_mxcsr() {
duke@435	485	StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
duke@435	486	address start = __ pc();
duke@435	487
duke@435	488	const Address mxcsr_save(rsp, 0);
duke@435	489
duke@435	490	if (CheckJNICalls && UseSSE > 0 ) {
duke@435	491	Label ok_ret;
duke@435	492	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
never@739	493	__ push(rax);
never@739	494	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	495	__ stmxcsr(mxcsr_save);
duke@435	496	__ movl(rax, mxcsr_save);
duke@435	497	__ andl(rax, MXCSR_MASK);
duke@435	498	__ cmp32(rax, mxcsr_std);
duke@435	499	__ jcc(Assembler::equal, ok_ret);
duke@435	500
duke@435	501	__ warn("MXCSR changed by native JNI code.");
duke@435	502
duke@435	503	__ ldmxcsr(mxcsr_std);
duke@435	504
duke@435	505	__ bind(ok_ret);
never@739	506	__ addptr(rsp, wordSize);
never@739	507	__ pop(rax);
duke@435	508	}
duke@435	509
duke@435	510	__ ret(0);
duke@435	511
duke@435	512	return start;
duke@435	513	}
duke@435	514
duke@435	515
duke@435	516	//---------------------------------------------------------------------------
duke@435	517	// Support for void verify_fpu_cntrl_wrd()
duke@435	518	//
duke@435	519	// This routine is used with -Xcheck:jni to verify that native
duke@435	520	// JNI code does not return to Java code without restoring the
duke@435	521	// FP control word to our expected state.
duke@435	522
duke@435	523	address generate_verify_fpu_cntrl_wrd() {
duke@435	524	StubCodeMark mark(this, "StubRoutines", "verify_spcw");
duke@435	525	address start = __ pc();
duke@435	526
duke@435	527	const Address fpu_cntrl_wrd_save(rsp, 0);
duke@435	528
duke@435	529	if (CheckJNICalls) {
duke@435	530	Label ok_ret;
never@739	531	__ push(rax);
never@739	532	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	533	__ fnstcw(fpu_cntrl_wrd_save);
duke@435	534	__ movl(rax, fpu_cntrl_wrd_save);
duke@435	535	__ andl(rax, FPU_CNTRL_WRD_MASK);
duke@435	536	ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
duke@435	537	__ cmp32(rax, fpu_std);
duke@435	538	__ jcc(Assembler::equal, ok_ret);
duke@435	539
duke@435	540	__ warn("Floating point control word changed by native JNI code.");
duke@435	541
duke@435	542	__ fldcw(fpu_std);
duke@435	543
duke@435	544	__ bind(ok_ret);
never@739	545	__ addptr(rsp, wordSize);
never@739	546	__ pop(rax);
duke@435	547	}
duke@435	548
duke@435	549	__ ret(0);
duke@435	550
duke@435	551	return start;
duke@435	552	}
duke@435	553
duke@435	554	//---------------------------------------------------------------------------
duke@435	555	// Wrapper for slow-case handling of double-to-integer conversion
duke@435	556	// d2i or f2i fast case failed either because it is nan or because
duke@435	557	// of under/overflow.
duke@435	558	// Input: FPU TOS: float value
duke@435	559	// Output: rax, (rdx): integer (long) result
duke@435	560
duke@435	561	address generate_d2i_wrapper(BasicType t, address fcn) {
duke@435	562	StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
duke@435	563	address start = __ pc();
duke@435	564
duke@435	565	// Capture info about frame layout
duke@435	566	enum layout { FPUState_off = 0,
duke@435	567	rbp_off = FPUStateSizeInWords,
duke@435	568	rdi_off,
duke@435	569	rsi_off,
duke@435	570	rcx_off,
duke@435	571	rbx_off,
duke@435	572	saved_argument_off,
duke@435	573	saved_argument_off2, // 2nd half of double
duke@435	574	framesize
duke@435	575	};
duke@435	576
duke@435	577	assert(FPUStateSizeInWords == 27, "update stack layout");
duke@435	578
duke@435	579	// Save outgoing argument to stack across push_FPU_state()
never@739	580	__ subptr(rsp, wordSize * 2);
duke@435	581	__ fstp_d(Address(rsp, 0));
duke@435	582
duke@435	583	// Save CPU & FPU state
never@739	584	__ push(rbx);
never@739	585	__ push(rcx);
never@739	586	__ push(rsi);
never@739	587	__ push(rdi);
never@739	588	__ push(rbp);
duke@435	589	__ push_FPU_state();
duke@435	590
duke@435	591	// push_FPU_state() resets the FP top of stack
duke@435	592	// Load original double into FP top of stack
duke@435	593	__ fld_d(Address(rsp, saved_argument_off * wordSize));
duke@435	594	// Store double into stack as outgoing argument
never@739	595	__ subptr(rsp, wordSize*2);
duke@435	596	__ fst_d(Address(rsp, 0));
duke@435	597
duke@435	598	// Prepare FPU for doing math in C-land
duke@435	599	__ empty_FPU_stack();
duke@435	600	// Call the C code to massage the double. Result in EAX
duke@435	601	if (t == T_INT)
duke@435	602	{ BLOCK_COMMENT("SharedRuntime::d2i"); }
duke@435	603	else if (t == T_LONG)
duke@435	604	{ BLOCK_COMMENT("SharedRuntime::d2l"); }
duke@435	605	__ call_VM_leaf( fcn, 2 );
duke@435	606
duke@435	607	// Restore CPU & FPU state
duke@435	608	__ pop_FPU_state();
never@739	609	__ pop(rbp);
never@739	610	__ pop(rdi);
never@739	611	__ pop(rsi);
never@739	612	__ pop(rcx);
never@739	613	__ pop(rbx);
never@739	614	__ addptr(rsp, wordSize * 2);
duke@435	615
duke@435	616	__ ret(0);
duke@435	617
duke@435	618	return start;
duke@435	619	}
duke@435	620
duke@435	621
duke@435	622	//---------------------------------------------------------------------------
duke@435	623	// The following routine generates a subroutine to throw an asynchronous
duke@435	624	// UnknownError when an unsafe access gets a fault that could not be
duke@435	625	// reasonably prevented by the programmer. (Example: SIGBUS/OBJERR.)
duke@435	626	address generate_handler_for_unsafe_access() {
duke@435	627	StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
duke@435	628	address start = __ pc();
duke@435	629
never@739	630	__ push(0); // hole for return address-to-be
never@739	631	__ pusha(); // push registers
duke@435	632	Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
duke@435	633	BLOCK_COMMENT("call handle_unsafe_access");
duke@435	634	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
never@739	635	__ movptr(next_pc, rax); // stuff next address
never@739	636	__ popa();
duke@435	637	__ ret(0); // jump to next address
duke@435	638
duke@435	639	return start;
duke@435	640	}
duke@435	641
duke@435	642
duke@435	643	//----------------------------------------------------------------------------------------------------
duke@435	644	// Non-destructive plausibility checks for oops
duke@435	645
duke@435	646	address generate_verify_oop() {
duke@435	647	StubCodeMark mark(this, "StubRoutines", "verify_oop");
duke@435	648	address start = __ pc();
duke@435	649
duke@435	650	// Incoming arguments on stack after saving rax,:
duke@435	651	//
duke@435	652	// [tos ]: saved rdx
duke@435	653	// [tos + 1]: saved EFLAGS
duke@435	654	// [tos + 2]: return address
duke@435	655	// [tos + 3]: char* error message
duke@435	656	// [tos + 4]: oop object to verify
duke@435	657	// [tos + 5]: saved rax, - saved by caller and bashed
duke@435	658
duke@435	659	Label exit, error;
never@739	660	__ pushf();
never@739	661	__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
never@739	662	__ push(rdx); // save rdx
duke@435	663	// make sure object is 'reasonable'
never@739	664	__ movptr(rax, Address(rsp, 4 * wordSize)); // get object
never@739	665	__ testptr(rax, rax);
duke@435	666	__ jcc(Assembler::zero, exit); // if obj is NULL it is ok
duke@435	667
duke@435	668	// Check if the oop is in the right area of memory
duke@435	669	const int oop_mask = Universe::verify_oop_mask();
duke@435	670	const int oop_bits = Universe::verify_oop_bits();
never@739	671	__ mov(rdx, rax);
never@739	672	__ andptr(rdx, oop_mask);
never@739	673	__ cmpptr(rdx, oop_bits);
duke@435	674	__ jcc(Assembler::notZero, error);
duke@435	675
duke@435	676	// make sure klass is 'reasonable'
never@739	677	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
never@739	678	__ testptr(rax, rax);
duke@435	679	__ jcc(Assembler::zero, error); // if klass is NULL it is broken
duke@435	680
duke@435	681	// Check if the klass is in the right area of memory
duke@435	682	const int klass_mask = Universe::verify_klass_mask();
duke@435	683	const int klass_bits = Universe::verify_klass_bits();
never@739	684	__ mov(rdx, rax);
never@739	685	__ andptr(rdx, klass_mask);
never@739	686	__ cmpptr(rdx, klass_bits);
duke@435	687	__ jcc(Assembler::notZero, error);
duke@435	688
duke@435	689	// make sure klass' klass is 'reasonable'
never@739	690	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass
never@739	691	__ testptr(rax, rax);
duke@435	692	__ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
duke@435	693
never@739	694	__ mov(rdx, rax);
never@739	695	__ andptr(rdx, klass_mask);
never@739	696	__ cmpptr(rdx, klass_bits);
duke@435	697	__ jcc(Assembler::notZero, error); // if klass not in right area
duke@435	698	// of memory it is broken too.
duke@435	699
duke@435	700	// return if everything seems ok
duke@435	701	__ bind(exit);
never@739	702	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	703	__ pop(rdx); // restore rdx
never@739	704	__ popf(); // restore EFLAGS
duke@435	705	__ ret(3 * wordSize); // pop arguments
duke@435	706
duke@435	707	// handle errors
duke@435	708	__ bind(error);
never@739	709	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	710	__ pop(rdx); // get saved rdx back
never@739	711	__ popf(); // get saved EFLAGS off stack -- will be ignored
never@739	712	__ pusha(); // push registers (eip = return address & msg are already pushed)
duke@435	713	BLOCK_COMMENT("call MacroAssembler::debug");
never@739	714	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
never@739	715	__ popa();
duke@435	716	__ ret(3 * wordSize); // pop arguments
duke@435	717	return start;
duke@435	718	}
duke@435	719
duke@435	720	//
duke@435	721	// Generate pre-barrier for array stores
duke@435	722	//
duke@435	723	// Input:
duke@435	724	// start - starting address
ysr@1280	725	// count - element count
duke@435	726	void gen_write_ref_array_pre_barrier(Register start, Register count) {
duke@435	727	assert_different_registers(start, count);
duke@435	728	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	729	switch (bs->kind()) {
duke@435	730	case BarrierSet::G1SATBCT:
duke@435	731	case BarrierSet::G1SATBCTLogging:
duke@435	732	{
never@739	733	__ pusha(); // push registers
apetrusenko@1627	734	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
apetrusenko@1627	735	start, count);
never@739	736	__ popa();
duke@435	737	}
duke@435	738	break;
duke@435	739	case BarrierSet::CardTableModRef:
duke@435	740	case BarrierSet::CardTableExtension:
duke@435	741	case BarrierSet::ModRef:
duke@435	742	break;
duke@435	743	default :
duke@435	744	ShouldNotReachHere();
duke@435	745
duke@435	746	}
duke@435	747	}
duke@435	748
duke@435	749
duke@435	750	//
duke@435	751	// Generate a post-barrier for an array store
duke@435	752	//
duke@435	753	// start - starting address
duke@435	754	// count - element count
duke@435	755	//
duke@435	756	// The two input registers are overwritten.
duke@435	757	//
duke@435	758	void gen_write_ref_array_post_barrier(Register start, Register count) {
duke@435	759	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	760	assert_different_registers(start, count);
duke@435	761	switch (bs->kind()) {
duke@435	762	case BarrierSet::G1SATBCT:
duke@435	763	case BarrierSet::G1SATBCTLogging:
duke@435	764	{
never@739	765	__ pusha(); // push registers
apetrusenko@1627	766	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
apetrusenko@1627	767	start, count);
never@739	768	__ popa();
duke@435	769	}
duke@435	770	break;
duke@435	771
duke@435	772	case BarrierSet::CardTableModRef:
duke@435	773	case BarrierSet::CardTableExtension:
duke@435	774	{
duke@435	775	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	776	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	777
duke@435	778	Label L_loop;
duke@435	779	const Register end = count; // elements count; end == start+count-1
duke@435	780	assert_different_registers(start, end);
duke@435	781
never@739	782	__ lea(end, Address(start, count, Address::times_ptr, -wordSize));
never@739	783	__ shrptr(start, CardTableModRefBS::card_shift);
never@739	784	__ shrptr(end, CardTableModRefBS::card_shift);
never@739	785	__ subptr(end, start); // end --> count
duke@435	786	__ BIND(L_loop);
never@684	787	intptr_t disp = (intptr_t) ct->byte_map_base;
never@684	788	Address cardtable(start, count, Address::times_1, disp);
never@684	789	__ movb(cardtable, 0);
duke@435	790	__ decrement(count);
duke@435	791	__ jcc(Assembler::greaterEqual, L_loop);
duke@435	792	}
duke@435	793	break;
duke@435	794	case BarrierSet::ModRef:
duke@435	795	break;
duke@435	796	default :
duke@435	797	ShouldNotReachHere();
duke@435	798
duke@435	799	}
duke@435	800	}
duke@435	801
kvn@840	802
kvn@840	803	// Copy 64 bytes chunks
kvn@840	804	//
kvn@840	805	// Inputs:
kvn@840	806	// from - source array address
kvn@840	807	// to_from - destination array address - from
kvn@840	808	// qword_count - 8-bytes element count, negative
kvn@840	809	//
kvn@840	810	void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	811	assert( UseSSE >= 2, "supported cpu only" );
kvn@840	812	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
kvn@840	813	// Copy 64-byte chunks
kvn@840	814	__ jmpb(L_copy_64_bytes);
kvn@840	815	__ align(16);
kvn@840	816	__ BIND(L_copy_64_bytes_loop);
kvn@840	817
kvn@840	818	if(UseUnalignedLoadStores) {
kvn@840	819	__ movdqu(xmm0, Address(from, 0));
kvn@840	820	__ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	821	__ movdqu(xmm1, Address(from, 16));
kvn@840	822	__ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
kvn@840	823	__ movdqu(xmm2, Address(from, 32));
kvn@840	824	__ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
kvn@840	825	__ movdqu(xmm3, Address(from, 48));
kvn@840	826	__ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
kvn@840	827
kvn@840	828	} else {
kvn@840	829	__ movq(xmm0, Address(from, 0));
kvn@840	830	__ movq(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	831	__ movq(xmm1, Address(from, 8));
kvn@840	832	__ movq(Address(from, to_from, Address::times_1, 8), xmm1);
kvn@840	833	__ movq(xmm2, Address(from, 16));
kvn@840	834	__ movq(Address(from, to_from, Address::times_1, 16), xmm2);
kvn@840	835	__ movq(xmm3, Address(from, 24));
kvn@840	836	__ movq(Address(from, to_from, Address::times_1, 24), xmm3);
kvn@840	837	__ movq(xmm4, Address(from, 32));
kvn@840	838	__ movq(Address(from, to_from, Address::times_1, 32), xmm4);
kvn@840	839	__ movq(xmm5, Address(from, 40));
kvn@840	840	__ movq(Address(from, to_from, Address::times_1, 40), xmm5);
kvn@840	841	__ movq(xmm6, Address(from, 48));
kvn@840	842	__ movq(Address(from, to_from, Address::times_1, 48), xmm6);
kvn@840	843	__ movq(xmm7, Address(from, 56));
kvn@840	844	__ movq(Address(from, to_from, Address::times_1, 56), xmm7);
kvn@840	845	}
kvn@840	846
kvn@840	847	__ addl(from, 64);
kvn@840	848	__ BIND(L_copy_64_bytes);
kvn@840	849	__ subl(qword_count, 8);
kvn@840	850	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
kvn@840	851	__ addl(qword_count, 8);
kvn@840	852	__ jccb(Assembler::zero, L_exit);
kvn@840	853	//
kvn@840	854	// length is too short, just copy qwords
kvn@840	855	//
kvn@840	856	__ BIND(L_copy_8_bytes);
kvn@840	857	__ movq(xmm0, Address(from, 0));
kvn@840	858	__ movq(Address(from, to_from, Address::times_1), xmm0);
kvn@840	859	__ addl(from, 8);
kvn@840	860	__ decrement(qword_count);
kvn@840	861	__ jcc(Assembler::greater, L_copy_8_bytes);
kvn@840	862	__ BIND(L_exit);
kvn@840	863	}
kvn@840	864
duke@435	865	// Copy 64 bytes chunks
duke@435	866	//
duke@435	867	// Inputs:
duke@435	868	// from - source array address
duke@435	869	// to_from - destination array address - from
duke@435	870	// qword_count - 8-bytes element count, negative
duke@435	871	//
duke@435	872	void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	873	assert( VM_Version::supports_mmx(), "supported cpu only" );
duke@435	874	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
duke@435	875	// Copy 64-byte chunks
duke@435	876	__ jmpb(L_copy_64_bytes);
duke@435	877	__ align(16);
duke@435	878	__ BIND(L_copy_64_bytes_loop);
duke@435	879	__ movq(mmx0, Address(from, 0));
duke@435	880	__ movq(mmx1, Address(from, 8));
duke@435	881	__ movq(mmx2, Address(from, 16));
duke@435	882	__ movq(Address(from, to_from, Address::times_1, 0), mmx0);
duke@435	883	__ movq(mmx3, Address(from, 24));
duke@435	884	__ movq(Address(from, to_from, Address::times_1, 8), mmx1);
duke@435	885	__ movq(mmx4, Address(from, 32));
duke@435	886	__ movq(Address(from, to_from, Address::times_1, 16), mmx2);
duke@435	887	__ movq(mmx5, Address(from, 40));
duke@435	888	__ movq(Address(from, to_from, Address::times_1, 24), mmx3);
duke@435	889	__ movq(mmx6, Address(from, 48));
duke@435	890	__ movq(Address(from, to_from, Address::times_1, 32), mmx4);
duke@435	891	__ movq(mmx7, Address(from, 56));
duke@435	892	__ movq(Address(from, to_from, Address::times_1, 40), mmx5);
duke@435	893	__ movq(Address(from, to_from, Address::times_1, 48), mmx6);
duke@435	894	__ movq(Address(from, to_from, Address::times_1, 56), mmx7);
never@739	895	__ addptr(from, 64);
duke@435	896	__ BIND(L_copy_64_bytes);
duke@435	897	__ subl(qword_count, 8);
duke@435	898	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
duke@435	899	__ addl(qword_count, 8);
duke@435	900	__ jccb(Assembler::zero, L_exit);
duke@435	901	//
duke@435	902	// length is too short, just copy qwords
duke@435	903	//
duke@435	904	__ BIND(L_copy_8_bytes);
duke@435	905	__ movq(mmx0, Address(from, 0));
duke@435	906	__ movq(Address(from, to_from, Address::times_1), mmx0);
never@739	907	__ addptr(from, 8);
duke@435	908	__ decrement(qword_count);
duke@435	909	__ jcc(Assembler::greater, L_copy_8_bytes);
duke@435	910	__ BIND(L_exit);
duke@435	911	__ emms();
duke@435	912	}
duke@435	913
duke@435	914	address generate_disjoint_copy(BasicType t, bool aligned,
duke@435	915	Address::ScaleFactor sf,
duke@435	916	address* entry, const char *name) {
duke@435	917	__ align(CodeEntryAlignment);
duke@435	918	StubCodeMark mark(this, "StubRoutines", name);
duke@435	919	address start = __ pc();
duke@435	920
duke@435	921	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	922	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
duke@435	923
never@739	924	int shift = Address::times_ptr - sf;
duke@435	925
duke@435	926	const Register from = rsi; // source array address
duke@435	927	const Register to = rdi; // destination array address
duke@435	928	const Register count = rcx; // elements count
duke@435	929	const Register to_from = to; // (to - from)
duke@435	930	const Register saved_to = rdx; // saved destination array address
duke@435	931
duke@435	932	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	933	__ push(rsi);
never@739	934	__ push(rdi);
never@739	935	__ movptr(from , Address(rsp, 12+ 4));
never@739	936	__ movptr(to , Address(rsp, 12+ 8));
duke@435	937	__ movl(count, Address(rsp, 12+ 12));
duke@435	938	if (t == T_OBJECT) {
duke@435	939	__ testl(count, count);
duke@435	940	__ jcc(Assembler::zero, L_0_count);
duke@435	941	gen_write_ref_array_pre_barrier(to, count);
never@739	942	__ mov(saved_to, to); // save 'to'
duke@435	943	}
duke@435	944
duke@435	945	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	946	BLOCK_COMMENT("Entry:");
duke@435	947
never@739	948	__ subptr(to, from); // to --> to_from
duke@435	949	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	950	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
kvn@840	951	if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
duke@435	952	// align source address at 4 bytes address boundary
duke@435	953	if (t == T_BYTE) {
duke@435	954	// One byte misalignment happens only for byte arrays
duke@435	955	__ testl(from, 1);
duke@435	956	__ jccb(Assembler::zero, L_skip_align1);
duke@435	957	__ movb(rax, Address(from, 0));
duke@435	958	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	959	__ increment(from);
duke@435	960	__ decrement(count);
duke@435	961	__ BIND(L_skip_align1);
duke@435	962	}
duke@435	963	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	964	__ testl(from, 2);
duke@435	965	__ jccb(Assembler::zero, L_skip_align2);
duke@435	966	__ movw(rax, Address(from, 0));
duke@435	967	__ movw(Address(from, to_from, Address::times_1, 0), rax);
never@739	968	__ addptr(from, 2);
duke@435	969	__ subl(count, 1<<(shift-1));
duke@435	970	__ BIND(L_skip_align2);
duke@435	971	}
duke@435	972	if (!VM_Version::supports_mmx()) {
never@739	973	__ mov(rax, count); // save 'count'
never@739	974	__ shrl(count, shift); // bytes count
never@739	975	__ addptr(to_from, from);// restore 'to'
never@739	976	__ rep_mov();
never@739	977	__ subptr(to_from, from);// restore 'to_from'
never@739	978	__ mov(count, rax); // restore 'count'
duke@435	979	__ jmpb(L_copy_2_bytes); // all dwords were copied
duke@435	980	} else {
kvn@840	981	if (!UseUnalignedLoadStores) {
kvn@840	982	// align to 8 bytes, we know we are 4 byte aligned to start
kvn@840	983	__ testptr(from, 4);
kvn@840	984	__ jccb(Assembler::zero, L_copy_64_bytes);
kvn@840	985	__ movl(rax, Address(from, 0));
kvn@840	986	__ movl(Address(from, to_from, Address::times_1, 0), rax);
kvn@840	987	__ addptr(from, 4);
kvn@840	988	__ subl(count, 1<<shift);
kvn@840	989	}
duke@435	990	__ BIND(L_copy_64_bytes);
never@739	991	__ mov(rax, count);
duke@435	992	__ shrl(rax, shift+1); // 8 bytes chunk count
duke@435	993	//
duke@435	994	// Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
duke@435	995	//
kvn@840	996	if (UseXMMForArrayCopy) {
kvn@840	997	xmm_copy_forward(from, to_from, rax);
kvn@840	998	} else {
kvn@840	999	mmx_copy_forward(from, to_from, rax);
kvn@840	1000	}
duke@435	1001	}
duke@435	1002	// copy tailing dword
duke@435	1003	__ BIND(L_copy_4_bytes);
duke@435	1004	__ testl(count, 1<<shift);
duke@435	1005	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1006	__ movl(rax, Address(from, 0));
duke@435	1007	__ movl(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1008	if (t == T_BYTE || t == T_SHORT) {
never@739	1009	__ addptr(from, 4);
duke@435	1010	__ BIND(L_copy_2_bytes);
duke@435	1011	// copy tailing word
duke@435	1012	__ testl(count, 1<<(shift-1));
duke@435	1013	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1014	__ movw(rax, Address(from, 0));
duke@435	1015	__ movw(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1016	if (t == T_BYTE) {
never@739	1017	__ addptr(from, 2);
duke@435	1018	__ BIND(L_copy_byte);
duke@435	1019	// copy tailing byte
duke@435	1020	__ testl(count, 1);
duke@435	1021	__ jccb(Assembler::zero, L_exit);
duke@435	1022	__ movb(rax, Address(from, 0));
duke@435	1023	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1024	__ BIND(L_exit);
duke@435	1025	} else {
duke@435	1026	__ BIND(L_copy_byte);
duke@435	1027	}
duke@435	1028	} else {
duke@435	1029	__ BIND(L_copy_2_bytes);
duke@435	1030	}
duke@435	1031
duke@435	1032	if (t == T_OBJECT) {
duke@435	1033	__ movl(count, Address(rsp, 12+12)); // reread 'count'
never@739	1034	__ mov(to, saved_to); // restore 'to'
duke@435	1035	gen_write_ref_array_post_barrier(to, count);
duke@435	1036	__ BIND(L_0_count);
duke@435	1037	}
duke@435	1038	inc_copy_counter_np(t);
never@739	1039	__ pop(rdi);
never@739	1040	__ pop(rsi);
duke@435	1041	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1042	__ xorptr(rax, rax); // return 0
duke@435	1043	__ ret(0);
duke@435	1044	return start;
duke@435	1045	}
duke@435	1046
duke@435	1047
duke@435	1048	address generate_conjoint_copy(BasicType t, bool aligned,
duke@435	1049	Address::ScaleFactor sf,
duke@435	1050	address nooverlap_target,
duke@435	1051	address* entry, const char *name) {
duke@435	1052	__ align(CodeEntryAlignment);
duke@435	1053	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1054	address start = __ pc();
duke@435	1055
duke@435	1056	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	1057	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1058
never@739	1059	int shift = Address::times_ptr - sf;
duke@435	1060
duke@435	1061	const Register src = rax; // source array address
duke@435	1062	const Register dst = rdx; // destination array address
duke@435	1063	const Register from = rsi; // source array address
duke@435	1064	const Register to = rdi; // destination array address
duke@435	1065	const Register count = rcx; // elements count
duke@435	1066	const Register end = rax; // array end address
duke@435	1067
duke@435	1068	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1069	__ push(rsi);
never@739	1070	__ push(rdi);
never@739	1071	__ movptr(src , Address(rsp, 12+ 4)); // from
never@739	1072	__ movptr(dst , Address(rsp, 12+ 8)); // to
never@739	1073	__ movl2ptr(count, Address(rsp, 12+12)); // count
duke@435	1074	if (t == T_OBJECT) {
duke@435	1075	gen_write_ref_array_pre_barrier(dst, count);
duke@435	1076	}
duke@435	1077
duke@435	1078	if (entry != NULL) {
duke@435	1079	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1080	BLOCK_COMMENT("Entry:");
duke@435	1081	}
duke@435	1082
duke@435	1083	if (t == T_OBJECT) {
duke@435	1084	__ testl(count, count);
duke@435	1085	__ jcc(Assembler::zero, L_0_count);
duke@435	1086	}
never@739	1087	__ mov(from, src);
never@739	1088	__ mov(to , dst);
duke@435	1089
duke@435	1090	// arrays overlap test
duke@435	1091	RuntimeAddress nooverlap(nooverlap_target);
never@739	1092	__ cmpptr(dst, src);
never@739	1093	__ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
duke@435	1094	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1095	__ cmpptr(dst, end);
duke@435	1096	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1097
duke@435	1098	// copy from high to low
duke@435	1099	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1100	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
duke@435	1101	if (t == T_BYTE || t == T_SHORT) {
duke@435	1102	// Align the end of destination array at 4 bytes address boundary
never@739	1103	__ lea(end, Address(dst, count, sf, 0));
duke@435	1104	if (t == T_BYTE) {
duke@435	1105	// One byte misalignment happens only for byte arrays
duke@435	1106	__ testl(end, 1);
duke@435	1107	__ jccb(Assembler::zero, L_skip_align1);
duke@435	1108	__ decrement(count);
duke@435	1109	__ movb(rdx, Address(from, count, sf, 0));
duke@435	1110	__ movb(Address(to, count, sf, 0), rdx);
duke@435	1111	__ BIND(L_skip_align1);
duke@435	1112	}
duke@435	1113	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	1114	__ testl(end, 2);
duke@435	1115	__ jccb(Assembler::zero, L_skip_align2);
never@739	1116	__ subptr(count, 1<<(shift-1));
duke@435	1117	__ movw(rdx, Address(from, count, sf, 0));
duke@435	1118	__ movw(Address(to, count, sf, 0), rdx);
duke@435	1119	__ BIND(L_skip_align2);
duke@435	1120	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1121	__ jcc(Assembler::below, L_copy_4_bytes);
duke@435	1122	}
duke@435	1123
duke@435	1124	if (!VM_Version::supports_mmx()) {
duke@435	1125	__ std();
never@739	1126	__ mov(rax, count); // Save 'count'
never@739	1127	__ mov(rdx, to); // Save 'to'
never@739	1128	__ lea(rsi, Address(from, count, sf, -4));
never@739	1129	__ lea(rdi, Address(to , count, sf, -4));
never@739	1130	__ shrptr(count, shift); // bytes count
never@739	1131	__ rep_mov();
duke@435	1132	__ cld();
never@739	1133	__ mov(count, rax); // restore 'count'
duke@435	1134	__ andl(count, (1<<shift)-1); // mask the number of rest elements
never@739	1135	__ movptr(from, Address(rsp, 12+4)); // reread 'from'
never@739	1136	__ mov(to, rdx); // restore 'to'
duke@435	1137	__ jmpb(L_copy_2_bytes); // all dword were copied
duke@435	1138	} else {
duke@435	1139	// Align to 8 bytes the end of array. It is aligned to 4 bytes already.
never@739	1140	__ testptr(end, 4);
duke@435	1141	__ jccb(Assembler::zero, L_copy_8_bytes);
duke@435	1142	__ subl(count, 1<<shift);
duke@435	1143	__ movl(rdx, Address(from, count, sf, 0));
duke@435	1144	__ movl(Address(to, count, sf, 0), rdx);
duke@435	1145	__ jmpb(L_copy_8_bytes);
duke@435	1146
duke@435	1147	__ align(16);
duke@435	1148	// Move 8 bytes
duke@435	1149	__ BIND(L_copy_8_bytes_loop);
kvn@840	1150	if (UseXMMForArrayCopy) {
kvn@840	1151	__ movq(xmm0, Address(from, count, sf, 0));
kvn@840	1152	__ movq(Address(to, count, sf, 0), xmm0);
kvn@840	1153	} else {
kvn@840	1154	__ movq(mmx0, Address(from, count, sf, 0));
kvn@840	1155	__ movq(Address(to, count, sf, 0), mmx0);
kvn@840	1156	}
duke@435	1157	__ BIND(L_copy_8_bytes);
duke@435	1158	__ subl(count, 2<<shift);
duke@435	1159	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1160	__ addl(count, 2<<shift);
kvn@840	1161	if (!UseXMMForArrayCopy) {
kvn@840	1162	__ emms();
kvn@840	1163	}
duke@435	1164	}
duke@435	1165	__ BIND(L_copy_4_bytes);
duke@435	1166	// copy prefix qword
duke@435	1167	__ testl(count, 1<<shift);
duke@435	1168	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1169	__ movl(rdx, Address(from, count, sf, -4));
duke@435	1170	__ movl(Address(to, count, sf, -4), rdx);
duke@435	1171
duke@435	1172	if (t == T_BYTE || t == T_SHORT) {
duke@435	1173	__ subl(count, (1<<shift));
duke@435	1174	__ BIND(L_copy_2_bytes);
duke@435	1175	// copy prefix dword
duke@435	1176	__ testl(count, 1<<(shift-1));
duke@435	1177	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1178	__ movw(rdx, Address(from, count, sf, -2));
duke@435	1179	__ movw(Address(to, count, sf, -2), rdx);
duke@435	1180	if (t == T_BYTE) {
duke@435	1181	__ subl(count, 1<<(shift-1));
duke@435	1182	__ BIND(L_copy_byte);
duke@435	1183	// copy prefix byte
duke@435	1184	__ testl(count, 1);
duke@435	1185	__ jccb(Assembler::zero, L_exit);
duke@435	1186	__ movb(rdx, Address(from, 0));
duke@435	1187	__ movb(Address(to, 0), rdx);
duke@435	1188	__ BIND(L_exit);
duke@435	1189	} else {
duke@435	1190	__ BIND(L_copy_byte);
duke@435	1191	}
duke@435	1192	} else {
duke@435	1193	__ BIND(L_copy_2_bytes);
duke@435	1194	}
duke@435	1195	if (t == T_OBJECT) {
never@739	1196	__ movl2ptr(count, Address(rsp, 12+12)); // reread count
duke@435	1197	gen_write_ref_array_post_barrier(to, count);
duke@435	1198	__ BIND(L_0_count);
duke@435	1199	}
duke@435	1200	inc_copy_counter_np(t);
never@739	1201	__ pop(rdi);
never@739	1202	__ pop(rsi);
duke@435	1203	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1204	__ xorptr(rax, rax); // return 0
duke@435	1205	__ ret(0);
duke@435	1206	return start;
duke@435	1207	}
duke@435	1208
duke@435	1209
duke@435	1210	address generate_disjoint_long_copy(address* entry, const char *name) {
duke@435	1211	__ align(CodeEntryAlignment);
duke@435	1212	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1213	address start = __ pc();
duke@435	1214
duke@435	1215	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1216	const Register from = rax; // source array address
duke@435	1217	const Register to = rdx; // destination array address
duke@435	1218	const Register count = rcx; // elements count
duke@435	1219	const Register to_from = rdx; // (to - from)
duke@435	1220
duke@435	1221	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1222	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1223	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1224	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1225
duke@435	1226	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	1227	BLOCK_COMMENT("Entry:");
duke@435	1228
never@739	1229	__ subptr(to, from); // to --> to_from
duke@435	1230	if (VM_Version::supports_mmx()) {
kvn@840	1231	if (UseXMMForArrayCopy) {
kvn@840	1232	xmm_copy_forward(from, to_from, count);
kvn@840	1233	} else {
kvn@840	1234	mmx_copy_forward(from, to_from, count);
kvn@840	1235	}
duke@435	1236	} else {
duke@435	1237	__ jmpb(L_copy_8_bytes);
duke@435	1238	__ align(16);
duke@435	1239	__ BIND(L_copy_8_bytes_loop);
duke@435	1240	__ fild_d(Address(from, 0));
duke@435	1241	__ fistp_d(Address(from, to_from, Address::times_1));
never@739	1242	__ addptr(from, 8);
duke@435	1243	__ BIND(L_copy_8_bytes);
duke@435	1244	__ decrement(count);
duke@435	1245	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1246	}
duke@435	1247	inc_copy_counter_np(T_LONG);
duke@435	1248	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1249	__ xorptr(rax, rax); // return 0
duke@435	1250	__ ret(0);
duke@435	1251	return start;
duke@435	1252	}
duke@435	1253
duke@435	1254	address generate_conjoint_long_copy(address nooverlap_target,
duke@435	1255	address* entry, const char *name) {
duke@435	1256	__ align(CodeEntryAlignment);
duke@435	1257	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1258	address start = __ pc();
duke@435	1259
duke@435	1260	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1261	const Register from = rax; // source array address
duke@435	1262	const Register to = rdx; // destination array address
duke@435	1263	const Register count = rcx; // elements count
duke@435	1264	const Register end_from = rax; // source array end address
duke@435	1265
duke@435	1266	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1267	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1268	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1269	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1270
duke@435	1271	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1272	BLOCK_COMMENT("Entry:");
duke@435	1273
duke@435	1274	// arrays overlap test
never@739	1275	__ cmpptr(to, from);
duke@435	1276	RuntimeAddress nooverlap(nooverlap_target);
duke@435	1277	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1278	__ lea(end_from, Address(from, count, Address::times_8, 0));
never@739	1279	__ cmpptr(to, end_from);
never@739	1280	__ movptr(from, Address(rsp, 8)); // from
duke@435	1281	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1282
duke@435	1283	__ jmpb(L_copy_8_bytes);
duke@435	1284
duke@435	1285	__ align(16);
duke@435	1286	__ BIND(L_copy_8_bytes_loop);
duke@435	1287	if (VM_Version::supports_mmx()) {
kvn@840	1288	if (UseXMMForArrayCopy) {
kvn@840	1289	__ movq(xmm0, Address(from, count, Address::times_8));
kvn@840	1290	__ movq(Address(to, count, Address::times_8), xmm0);
kvn@840	1291	} else {
kvn@840	1292	__ movq(mmx0, Address(from, count, Address::times_8));
kvn@840	1293	__ movq(Address(to, count, Address::times_8), mmx0);
kvn@840	1294	}
duke@435	1295	} else {
duke@435	1296	__ fild_d(Address(from, count, Address::times_8));
duke@435	1297	__ fistp_d(Address(to, count, Address::times_8));
duke@435	1298	}
duke@435	1299	__ BIND(L_copy_8_bytes);
duke@435	1300	__ decrement(count);
duke@435	1301	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1302
kvn@840	1303	if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
duke@435	1304	__ emms();
duke@435	1305	}
duke@435	1306	inc_copy_counter_np(T_LONG);
duke@435	1307	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1308	__ xorptr(rax, rax); // return 0
duke@435	1309	__ ret(0);
duke@435	1310	return start;
duke@435	1311	}
duke@435	1312
duke@435	1313
duke@435	1314	// Helper for generating a dynamic type check.
duke@435	1315	// The sub_klass must be one of {rbx, rdx, rsi}.
duke@435	1316	// The temp is killed.
duke@435	1317	void generate_type_check(Register sub_klass,
duke@435	1318	Address& super_check_offset_addr,
duke@435	1319	Address& super_klass_addr,
duke@435	1320	Register temp,
jrose@1079	1321	Label* L_success, Label* L_failure) {
duke@435	1322	BLOCK_COMMENT("type_check:");
duke@435	1323
duke@435	1324	Label L_fallthrough;
jrose@1079	1325	#define LOCAL_JCC(assembler_con, label_ptr) \
jrose@1079	1326	if (label_ptr != NULL) __ jcc(assembler_con, *(label_ptr)); \
jrose@1079	1327	else __ jcc(assembler_con, L_fallthrough) /*omit semi*/
duke@435	1328
jrose@1079	1329	// The following is a strange variation of the fast path which requires
jrose@1079	1330	// one less register, because needed values are on the argument stack.
jrose@1079	1331	// __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
jrose@1079	1332	// L_success, L_failure, NULL);
duke@435	1333	assert_different_registers(sub_klass, temp);
duke@435	1334
duke@435	1335	int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1336	Klass::secondary_super_cache_offset_in_bytes());
duke@435	1337
duke@435	1338	// if the pointers are equal, we are done (e.g., String[] elements)
never@739	1339	__ cmpptr(sub_klass, super_klass_addr);
jrose@1079	1340	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1341
duke@435	1342	// check the supertype display:
never@739	1343	__ movl2ptr(temp, super_check_offset_addr);
duke@435	1344	Address super_check_addr(sub_klass, temp, Address::times_1, 0);
never@739	1345	__ movptr(temp, super_check_addr); // load displayed supertype
never@739	1346	__ cmpptr(temp, super_klass_addr); // test the super type
jrose@1079	1347	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1348
duke@435	1349	// if it was a primary super, we can just fail immediately
duke@435	1350	__ cmpl(super_check_offset_addr, sc_offset);
jrose@1079	1351	LOCAL_JCC(Assembler::notEqual, L_failure);
duke@435	1352
jrose@1079	1353	// The repne_scan instruction uses fixed registers, which will get spilled.
jrose@1079	1354	// We happen to know this works best when super_klass is in rax.
jrose@1079	1355	Register super_klass = temp;
jrose@1079	1356	__ movptr(super_klass, super_klass_addr);
jrose@1079	1357	__ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
jrose@1079	1358	L_success, L_failure);
duke@435	1359
jrose@1079	1360	__ bind(L_fallthrough);
duke@435	1361
jrose@1079	1362	if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
jrose@1079	1363	if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
duke@435	1364
jrose@1079	1365	#undef LOCAL_JCC
duke@435	1366	}
duke@435	1367
duke@435	1368	//
duke@435	1369	// Generate checkcasting array copy stub
duke@435	1370	//
duke@435	1371	// Input:
duke@435	1372	// 4(rsp) - source array address
duke@435	1373	// 8(rsp) - destination array address
duke@435	1374	// 12(rsp) - element count, can be zero
duke@435	1375	// 16(rsp) - size_t ckoff (super_check_offset)
duke@435	1376	// 20(rsp) - oop ckval (super_klass)
duke@435	1377	//
duke@435	1378	// Output:
duke@435	1379	// rax, == 0 - success
duke@435	1380	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1381	//
duke@435	1382	address generate_checkcast_copy(const char *name, address* entry) {
duke@435	1383	__ align(CodeEntryAlignment);
duke@435	1384	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1385	address start = __ pc();
duke@435	1386
duke@435	1387	Label L_load_element, L_store_element, L_do_card_marks, L_done;
duke@435	1388
duke@435	1389	// register use:
duke@435	1390	// rax, rdx, rcx -- loop control (end_from, end_to, count)
duke@435	1391	// rdi, rsi -- element access (oop, klass)
duke@435	1392	// rbx, -- temp
duke@435	1393	const Register from = rax; // source array address
duke@435	1394	const Register to = rdx; // destination array address
duke@435	1395	const Register length = rcx; // elements count
duke@435	1396	const Register elem = rdi; // each oop copied
duke@435	1397	const Register elem_klass = rsi; // each elem._klass (sub_klass)
duke@435	1398	const Register temp = rbx; // lone remaining temp
duke@435	1399
duke@435	1400	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1401
never@739	1402	__ push(rsi);
never@739	1403	__ push(rdi);
never@739	1404	__ push(rbx);
duke@435	1405
duke@435	1406	Address from_arg(rsp, 16+ 4); // from
duke@435	1407	Address to_arg(rsp, 16+ 8); // to
duke@435	1408	Address length_arg(rsp, 16+12); // elements count
duke@435	1409	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1410	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1411
duke@435	1412	// Load up:
never@739	1413	__ movptr(from, from_arg);
never@739	1414	__ movptr(to, to_arg);
never@739	1415	__ movl2ptr(length, length_arg);
duke@435	1416
duke@435	1417	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1418	BLOCK_COMMENT("Entry:");
duke@435	1419
duke@435	1420	//---------------------------------------------------------------
duke@435	1421	// Assembler stub will be used for this call to arraycopy
duke@435	1422	// if the two arrays are subtypes of Object[] but the
duke@435	1423	// destination array type is not equal to or a supertype
duke@435	1424	// of the source type. Each element must be separately
duke@435	1425	// checked.
duke@435	1426
duke@435	1427	// Loop-invariant addresses. They are exclusive end pointers.
never@739	1428	Address end_from_addr(from, length, Address::times_ptr, 0);
never@739	1429	Address end_to_addr(to, length, Address::times_ptr, 0);
duke@435	1430
duke@435	1431	Register end_from = from; // re-use
duke@435	1432	Register end_to = to; // re-use
duke@435	1433	Register count = length; // re-use
duke@435	1434
duke@435	1435	// Loop-variant addresses. They assume post-incremented count < 0.
never@739	1436	Address from_element_addr(end_from, count, Address::times_ptr, 0);
never@739	1437	Address to_element_addr(end_to, count, Address::times_ptr, 0);
duke@435	1438	Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
duke@435	1439
duke@435	1440	// Copy from low to high addresses, indexed from the end of each array.
ysr@777	1441	gen_write_ref_array_pre_barrier(to, count);
never@739	1442	__ lea(end_from, end_from_addr);
never@739	1443	__ lea(end_to, end_to_addr);
duke@435	1444	assert(length == count, ""); // else fix next line:
never@739	1445	__ negptr(count); // negate and test the length
duke@435	1446	__ jccb(Assembler::notZero, L_load_element);
duke@435	1447
duke@435	1448	// Empty array: Nothing to do.
never@739	1449	__ xorptr(rax, rax); // return 0 on (trivial) success
duke@435	1450	__ jmp(L_done);
duke@435	1451
duke@435	1452	// ======== begin loop ========
duke@435	1453	// (Loop is rotated; its entry is L_load_element.)
duke@435	1454	// Loop control:
duke@435	1455	// for (count = -count; count != 0; count++)
duke@435	1456	// Base pointers src, dst are biased by 8*count,to last element.
duke@435	1457	__ align(16);
duke@435	1458
duke@435	1459	__ BIND(L_store_element);
never@739	1460	__ movptr(to_element_addr, elem); // store the oop
duke@435	1461	__ increment(count); // increment the count toward zero
duke@435	1462	__ jccb(Assembler::zero, L_do_card_marks);
duke@435	1463
duke@435	1464	// ======== loop entry is here ========
duke@435	1465	__ BIND(L_load_element);
never@739	1466	__ movptr(elem, from_element_addr); // load the oop
never@739	1467	__ testptr(elem, elem);
duke@435	1468	__ jccb(Assembler::zero, L_store_element);
duke@435	1469
duke@435	1470	// (Could do a trick here: Remember last successful non-null
duke@435	1471	// element stored and make a quick oop equality check on it.)
duke@435	1472
never@739	1473	__ movptr(elem_klass, elem_klass_addr); // query the object klass
duke@435	1474	generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
duke@435	1475	&L_store_element, NULL);
duke@435	1476	// (On fall-through, we have failed the element type check.)
duke@435	1477	// ======== end loop ========
duke@435	1478
duke@435	1479	// It was a real error; we must depend on the caller to finish the job.
rasbold@454	1480	// Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
rasbold@454	1481	// Emit GC store barriers for the oops we have copied (length_arg + count),
duke@435	1482	// and report their number to the caller.
duke@435	1483	__ addl(count, length_arg); // transfers = (length - remaining)
never@739	1484	__ movl2ptr(rax, count); // save the value
never@739	1485	__ notptr(rax); // report (-1^K) to caller
never@739	1486	__ movptr(to, to_arg); // reload
duke@435	1487	assert_different_registers(to, count, rax);
duke@435	1488	gen_write_ref_array_post_barrier(to, count);
duke@435	1489	__ jmpb(L_done);
duke@435	1490
duke@435	1491	// Come here on success only.
duke@435	1492	__ BIND(L_do_card_marks);
never@739	1493	__ movl2ptr(count, length_arg);
never@739	1494	__ movptr(to, to_arg); // reload
duke@435	1495	gen_write_ref_array_post_barrier(to, count);
never@739	1496	__ xorptr(rax, rax); // return 0 on success
duke@435	1497
duke@435	1498	// Common exit point (success or failure).
duke@435	1499	__ BIND(L_done);
never@739	1500	__ pop(rbx);
never@739	1501	__ pop(rdi);
never@739	1502	__ pop(rsi);
duke@435	1503	inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
duke@435	1504	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1505	__ ret(0);
duke@435	1506
duke@435	1507	return start;
duke@435	1508	}
duke@435	1509
duke@435	1510	//
duke@435	1511	// Generate 'unsafe' array copy stub
duke@435	1512	// Though just as safe as the other stubs, it takes an unscaled
duke@435	1513	// size_t argument instead of an element count.
duke@435	1514	//
duke@435	1515	// Input:
duke@435	1516	// 4(rsp) - source array address
duke@435	1517	// 8(rsp) - destination array address
duke@435	1518	// 12(rsp) - byte count, can be zero
duke@435	1519	//
duke@435	1520	// Output:
duke@435	1521	// rax, == 0 - success
duke@435	1522	// rax, == -1 - need to call System.arraycopy
duke@435	1523	//
duke@435	1524	// Examines the alignment of the operands and dispatches
duke@435	1525	// to a long, int, short, or byte copy loop.
duke@435	1526	//
duke@435	1527	address generate_unsafe_copy(const char *name,
duke@435	1528	address byte_copy_entry,
duke@435	1529	address short_copy_entry,
duke@435	1530	address int_copy_entry,
duke@435	1531	address long_copy_entry) {
duke@435	1532
duke@435	1533	Label L_long_aligned, L_int_aligned, L_short_aligned;
duke@435	1534
duke@435	1535	__ align(CodeEntryAlignment);
duke@435	1536	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1537	address start = __ pc();
duke@435	1538
duke@435	1539	const Register from = rax; // source array address
duke@435	1540	const Register to = rdx; // destination array address
duke@435	1541	const Register count = rcx; // elements count
duke@435	1542
duke@435	1543	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1544	__ push(rsi);
never@739	1545	__ push(rdi);
duke@435	1546	Address from_arg(rsp, 12+ 4); // from
duke@435	1547	Address to_arg(rsp, 12+ 8); // to
duke@435	1548	Address count_arg(rsp, 12+12); // byte count
duke@435	1549
duke@435	1550	// Load up:
never@739	1551	__ movptr(from , from_arg);
never@739	1552	__ movptr(to , to_arg);
never@739	1553	__ movl2ptr(count, count_arg);
duke@435	1554
duke@435	1555	// bump this on entry, not on exit:
duke@435	1556	inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
duke@435	1557
duke@435	1558	const Register bits = rsi;
never@739	1559	__ mov(bits, from);
never@739	1560	__ orptr(bits, to);
never@739	1561	__ orptr(bits, count);
duke@435	1562
duke@435	1563	__ testl(bits, BytesPerLong-1);
duke@435	1564	__ jccb(Assembler::zero, L_long_aligned);
duke@435	1565
duke@435	1566	__ testl(bits, BytesPerInt-1);
duke@435	1567	__ jccb(Assembler::zero, L_int_aligned);
duke@435	1568
duke@435	1569	__ testl(bits, BytesPerShort-1);
duke@435	1570	__ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
duke@435	1571
duke@435	1572	__ BIND(L_short_aligned);
never@739	1573	__ shrptr(count, LogBytesPerShort); // size => short_count
duke@435	1574	__ movl(count_arg, count); // update 'count'
duke@435	1575	__ jump(RuntimeAddress(short_copy_entry));
duke@435	1576
duke@435	1577	__ BIND(L_int_aligned);
never@739	1578	__ shrptr(count, LogBytesPerInt); // size => int_count
duke@435	1579	__ movl(count_arg, count); // update 'count'
duke@435	1580	__ jump(RuntimeAddress(int_copy_entry));
duke@435	1581
duke@435	1582	__ BIND(L_long_aligned);
never@739	1583	__ shrptr(count, LogBytesPerLong); // size => qword_count
duke@435	1584	__ movl(count_arg, count); // update 'count'
never@739	1585	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1586	__ pop(rsi);
duke@435	1587	__ jump(RuntimeAddress(long_copy_entry));
duke@435	1588
duke@435	1589	return start;
duke@435	1590	}
duke@435	1591
duke@435	1592
duke@435	1593	// Perform range checks on the proposed arraycopy.
duke@435	1594	// Smashes src_pos and dst_pos. (Uses them up for temps.)
duke@435	1595	void arraycopy_range_checks(Register src,
duke@435	1596	Register src_pos,
duke@435	1597	Register dst,
duke@435	1598	Register dst_pos,
duke@435	1599	Address& length,
duke@435	1600	Label& L_failed) {
duke@435	1601	BLOCK_COMMENT("arraycopy_range_checks:");
duke@435	1602	const Register src_end = src_pos; // source array end position
duke@435	1603	const Register dst_end = dst_pos; // destination array end position
duke@435	1604	__ addl(src_end, length); // src_pos + length
duke@435	1605	__ addl(dst_end, length); // dst_pos + length
duke@435	1606
duke@435	1607	// if (src_pos + length > arrayOop(src)->length() ) FAIL;
duke@435	1608	__ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
duke@435	1609	__ jcc(Assembler::above, L_failed);
duke@435	1610
duke@435	1611	// if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
duke@435	1612	__ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
duke@435	1613	__ jcc(Assembler::above, L_failed);
duke@435	1614
duke@435	1615	BLOCK_COMMENT("arraycopy_range_checks done");
duke@435	1616	}
duke@435	1617
duke@435	1618
duke@435	1619	//
duke@435	1620	// Generate generic array copy stubs
duke@435	1621	//
duke@435	1622	// Input:
duke@435	1623	// 4(rsp) - src oop
duke@435	1624	// 8(rsp) - src_pos
duke@435	1625	// 12(rsp) - dst oop
duke@435	1626	// 16(rsp) - dst_pos
duke@435	1627	// 20(rsp) - element count
duke@435	1628	//
duke@435	1629	// Output:
duke@435	1630	// rax, == 0 - success
duke@435	1631	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1632	//
duke@435	1633	address generate_generic_copy(const char *name,
duke@435	1634	address entry_jbyte_arraycopy,
duke@435	1635	address entry_jshort_arraycopy,
duke@435	1636	address entry_jint_arraycopy,
duke@435	1637	address entry_oop_arraycopy,
duke@435	1638	address entry_jlong_arraycopy,
duke@435	1639	address entry_checkcast_arraycopy) {
duke@435	1640	Label L_failed, L_failed_0, L_objArray;
duke@435	1641
duke@435	1642	{ int modulus = CodeEntryAlignment;
duke@435	1643	int target = modulus - 5; // 5 = sizeof jmp(L_failed)
duke@435	1644	int advance = target - (__ offset() % modulus);
duke@435	1645	if (advance < 0) advance += modulus;
duke@435	1646	if (advance > 0) __ nop(advance);
duke@435	1647	}
duke@435	1648	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1649
duke@435	1650	// Short-hop target to L_failed. Makes for denser prologue code.
duke@435	1651	__ BIND(L_failed_0);
duke@435	1652	__ jmp(L_failed);
duke@435	1653	assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
duke@435	1654
duke@435	1655	__ align(CodeEntryAlignment);
duke@435	1656	address start = __ pc();
duke@435	1657
duke@435	1658	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1659	__ push(rsi);
never@739	1660	__ push(rdi);
duke@435	1661
duke@435	1662	// bump this on entry, not on exit:
duke@435	1663	inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
duke@435	1664
duke@435	1665	// Input values
duke@435	1666	Address SRC (rsp, 12+ 4);
duke@435	1667	Address SRC_POS (rsp, 12+ 8);
duke@435	1668	Address DST (rsp, 12+12);
duke@435	1669	Address DST_POS (rsp, 12+16);
duke@435	1670	Address LENGTH (rsp, 12+20);
duke@435	1671
duke@435	1672	//-----------------------------------------------------------------------
duke@435	1673	// Assembler stub will be used for this call to arraycopy
duke@435	1674	// if the following conditions are met:
duke@435	1675	//
duke@435	1676	// (1) src and dst must not be null.
duke@435	1677	// (2) src_pos must not be negative.
duke@435	1678	// (3) dst_pos must not be negative.
duke@435	1679	// (4) length must not be negative.
duke@435	1680	// (5) src klass and dst klass should be the same and not NULL.
duke@435	1681	// (6) src and dst should be arrays.
duke@435	1682	// (7) src_pos + length must not exceed length of src.
duke@435	1683	// (8) dst_pos + length must not exceed length of dst.
duke@435	1684	//
duke@435	1685
duke@435	1686	const Register src = rax; // source array oop
duke@435	1687	const Register src_pos = rsi;
duke@435	1688	const Register dst = rdx; // destination array oop
duke@435	1689	const Register dst_pos = rdi;
duke@435	1690	const Register length = rcx; // transfer count
duke@435	1691
duke@435	1692	// if (src == NULL) return -1;
never@739	1693	__ movptr(src, SRC); // src oop
never@739	1694	__ testptr(src, src);
duke@435	1695	__ jccb(Assembler::zero, L_failed_0);
duke@435	1696
duke@435	1697	// if (src_pos < 0) return -1;
never@739	1698	__ movl2ptr(src_pos, SRC_POS); // src_pos
duke@435	1699	__ testl(src_pos, src_pos);
duke@435	1700	__ jccb(Assembler::negative, L_failed_0);
duke@435	1701
duke@435	1702	// if (dst == NULL) return -1;
never@739	1703	__ movptr(dst, DST); // dst oop
never@739	1704	__ testptr(dst, dst);
duke@435	1705	__ jccb(Assembler::zero, L_failed_0);
duke@435	1706
duke@435	1707	// if (dst_pos < 0) return -1;
never@739	1708	__ movl2ptr(dst_pos, DST_POS); // dst_pos
duke@435	1709	__ testl(dst_pos, dst_pos);
duke@435	1710	__ jccb(Assembler::negative, L_failed_0);
duke@435	1711
duke@435	1712	// if (length < 0) return -1;
never@739	1713	__ movl2ptr(length, LENGTH); // length
duke@435	1714	__ testl(length, length);
duke@435	1715	__ jccb(Assembler::negative, L_failed_0);
duke@435	1716
duke@435	1717	// if (src->klass() == NULL) return -1;
duke@435	1718	Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
duke@435	1719	Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
duke@435	1720	const Register rcx_src_klass = rcx; // array klass
never@739	1721	__ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
duke@435	1722
duke@435	1723	#ifdef ASSERT
duke@435	1724	// assert(src->klass() != NULL);
duke@435	1725	BLOCK_COMMENT("assert klasses not null");
duke@435	1726	{ Label L1, L2;
never@739	1727	__ testptr(rcx_src_klass, rcx_src_klass);
duke@435	1728	__ jccb(Assembler::notZero, L2); // it is broken if klass is NULL
duke@435	1729	__ bind(L1);
duke@435	1730	__ stop("broken null klass");
duke@435	1731	__ bind(L2);
never@739	1732	__ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
duke@435	1733	__ jccb(Assembler::equal, L1); // this would be broken also
duke@435	1734	BLOCK_COMMENT("assert done");
duke@435	1735	}
duke@435	1736	#endif //ASSERT
duke@435	1737
duke@435	1738	// Load layout helper (32-bits)
duke@435	1739	//
duke@435	1740	// |array_tag| | header_size | element_type | |log2_element_size|
duke@435	1741	// 32 30 24 16 8 2 0
duke@435	1742	//
duke@435	1743	// array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
duke@435	1744	//
duke@435	1745
duke@435	1746	int lh_offset = klassOopDesc::header_size() * HeapWordSize +
duke@435	1747	Klass::layout_helper_offset_in_bytes();
duke@435	1748	Address src_klass_lh_addr(rcx_src_klass, lh_offset);
duke@435	1749
duke@435	1750	// Handle objArrays completely differently...
duke@435	1751	jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
duke@435	1752	__ cmpl(src_klass_lh_addr, objArray_lh);
duke@435	1753	__ jcc(Assembler::equal, L_objArray);
duke@435	1754
duke@435	1755	// if (src->klass() != dst->klass()) return -1;
never@739	1756	__ cmpptr(rcx_src_klass, dst_klass_addr);
duke@435	1757	__ jccb(Assembler::notEqual, L_failed_0);
duke@435	1758
duke@435	1759	const Register rcx_lh = rcx; // layout helper
duke@435	1760	assert(rcx_lh == rcx_src_klass, "known alias");
duke@435	1761	__ movl(rcx_lh, src_klass_lh_addr);
duke@435	1762
duke@435	1763	// if (!src->is_Array()) return -1;
duke@435	1764	__ cmpl(rcx_lh, Klass::_lh_neutral_value);
duke@435	1765	__ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
duke@435	1766
duke@435	1767	// At this point, it is known to be a typeArray (array_tag 0x3).
duke@435	1768	#ifdef ASSERT
duke@435	1769	{ Label L;
duke@435	1770	__ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
duke@435	1771	__ jcc(Assembler::greaterEqual, L); // signed cmp
duke@435	1772	__ stop("must be a primitive array");
duke@435	1773	__ bind(L);
duke@435	1774	}
duke@435	1775	#endif
duke@435	1776
duke@435	1777	assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
duke@435	1778	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1779
duke@435	1780	// typeArrayKlass
duke@435	1781	//
duke@435	1782	// src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
duke@435	1783	// dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
duke@435	1784	//
duke@435	1785	const Register rsi_offset = rsi; // array offset
duke@435	1786	const Register src_array = src; // src array offset
duke@435	1787	const Register dst_array = dst; // dst array offset
duke@435	1788	const Register rdi_elsize = rdi; // log2 element size
duke@435	1789
never@739	1790	__ mov(rsi_offset, rcx_lh);
never@739	1791	__ shrptr(rsi_offset, Klass::_lh_header_size_shift);
never@739	1792	__ andptr(rsi_offset, Klass::_lh_header_size_mask); // array_offset
never@739	1793	__ addptr(src_array, rsi_offset); // src array offset
never@739	1794	__ addptr(dst_array, rsi_offset); // dst array offset
never@739	1795	__ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
duke@435	1796
duke@435	1797	// next registers should be set before the jump to corresponding stub
duke@435	1798	const Register from = src; // source array address
duke@435	1799	const Register to = dst; // destination array address
duke@435	1800	const Register count = rcx; // elements count
duke@435	1801	// some of them should be duplicated on stack
duke@435	1802	#define FROM Address(rsp, 12+ 4)
duke@435	1803	#define TO Address(rsp, 12+ 8) // Not used now
duke@435	1804	#define COUNT Address(rsp, 12+12) // Only for oop arraycopy
duke@435	1805
duke@435	1806	BLOCK_COMMENT("scale indexes to element size");
never@739	1807	__ movl2ptr(rsi, SRC_POS); // src_pos
never@739	1808	__ shlptr(rsi); // src_pos << rcx (log2 elsize)
duke@435	1809	assert(src_array == from, "");
never@739	1810	__ addptr(from, rsi); // from = src_array + SRC_POS << log2 elsize
never@739	1811	__ movl2ptr(rdi, DST_POS); // dst_pos
never@739	1812	__ shlptr(rdi); // dst_pos << rcx (log2 elsize)
duke@435	1813	assert(dst_array == to, "");
never@739	1814	__ addptr(to, rdi); // to = dst_array + DST_POS << log2 elsize
never@739	1815	__ movptr(FROM, from); // src_addr
never@739	1816	__ mov(rdi_elsize, rcx_lh); // log2 elsize
never@739	1817	__ movl2ptr(count, LENGTH); // elements count
duke@435	1818
duke@435	1819	BLOCK_COMMENT("choose copy loop based on element size");
duke@435	1820	__ cmpl(rdi_elsize, 0);
duke@435	1821
duke@435	1822	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
duke@435	1823	__ cmpl(rdi_elsize, LogBytesPerShort);
duke@435	1824	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
duke@435	1825	__ cmpl(rdi_elsize, LogBytesPerInt);
duke@435	1826	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
duke@435	1827	#ifdef ASSERT
duke@435	1828	__ cmpl(rdi_elsize, LogBytesPerLong);
duke@435	1829	__ jccb(Assembler::notEqual, L_failed);
duke@435	1830	#endif
never@739	1831	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1832	__ pop(rsi);
duke@435	1833	__ jump(RuntimeAddress(entry_jlong_arraycopy));
duke@435	1834
duke@435	1835	__ BIND(L_failed);
never@739	1836	__ xorptr(rax, rax);
never@739	1837	__ notptr(rax); // return -1
never@739	1838	__ pop(rdi);
never@739	1839	__ pop(rsi);
duke@435	1840	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1841	__ ret(0);
duke@435	1842
duke@435	1843	// objArrayKlass
duke@435	1844	__ BIND(L_objArray);
duke@435	1845	// live at this point: rcx_src_klass, src[_pos], dst[_pos]
duke@435	1846
duke@435	1847	Label L_plain_copy, L_checkcast_copy;
duke@435	1848	// test array classes for subtyping
never@739	1849	__ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
duke@435	1850	__ jccb(Assembler::notEqual, L_checkcast_copy);
duke@435	1851
duke@435	1852	// Identically typed arrays can be copied without element-wise checks.
duke@435	1853	assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
duke@435	1854	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1855
duke@435	1856	__ BIND(L_plain_copy);
never@739	1857	__ movl2ptr(count, LENGTH); // elements count
never@739	1858	__ movl2ptr(src_pos, SRC_POS); // reload src_pos
never@739	1859	__ lea(from, Address(src, src_pos, Address::times_ptr,
never@739	1860	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
never@739	1861	__ movl2ptr(dst_pos, DST_POS); // reload dst_pos
never@739	1862	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
never@739	1863	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
never@739	1864	__ movptr(FROM, from); // src_addr
never@739	1865	__ movptr(TO, to); // dst_addr
duke@435	1866	__ movl(COUNT, count); // count
duke@435	1867	__ jump(RuntimeAddress(entry_oop_arraycopy));
duke@435	1868
duke@435	1869	__ BIND(L_checkcast_copy);
duke@435	1870	// live at this point: rcx_src_klass, dst[_pos], src[_pos]
duke@435	1871	{
duke@435	1872	// Handy offsets:
duke@435	1873	int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1874	objArrayKlass::element_klass_offset_in_bytes());
duke@435	1875	int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1876	Klass::super_check_offset_offset_in_bytes());
duke@435	1877
duke@435	1878	Register rsi_dst_klass = rsi;
duke@435	1879	Register rdi_temp = rdi;
duke@435	1880	assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
duke@435	1881	assert(rdi_temp == dst_pos, "expected alias w/ dst_pos");
duke@435	1882	Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
duke@435	1883
duke@435	1884	// Before looking at dst.length, make sure dst is also an objArray.
never@739	1885	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1886	__ cmpl(dst_klass_lh_addr, objArray_lh);
duke@435	1887	__ jccb(Assembler::notEqual, L_failed);
duke@435	1888
duke@435	1889	// It is safe to examine both src.length and dst.length.
never@739	1890	__ movl2ptr(src_pos, SRC_POS); // reload rsi
duke@435	1891	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1892	// (Now src_pos and dst_pos are killed, but not src and dst.)
duke@435	1893
duke@435	1894	// We'll need this temp (don't forget to pop it after the type check).
never@739	1895	__ push(rbx);
duke@435	1896	Register rbx_src_klass = rbx;
duke@435	1897
never@739	1898	__ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
never@739	1899	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1900	Address super_check_offset_addr(rsi_dst_klass, sco_offset);
duke@435	1901	Label L_fail_array_check;
duke@435	1902	generate_type_check(rbx_src_klass,
duke@435	1903	super_check_offset_addr, dst_klass_addr,
duke@435	1904	rdi_temp, NULL, &L_fail_array_check);
duke@435	1905	// (On fall-through, we have passed the array type check.)
never@739	1906	__ pop(rbx);
duke@435	1907	__ jmp(L_plain_copy);
duke@435	1908
duke@435	1909	__ BIND(L_fail_array_check);
duke@435	1910	// Reshuffle arguments so we can call checkcast_arraycopy:
duke@435	1911
duke@435	1912	// match initial saves for checkcast_arraycopy
never@739	1913	// push(rsi); // already done; see above
never@739	1914	// push(rdi); // already done; see above
never@739	1915	// push(rbx); // already done; see above
duke@435	1916
duke@435	1917	// Marshal outgoing arguments now, freeing registers.
duke@435	1918	Address from_arg(rsp, 16+ 4); // from
duke@435	1919	Address to_arg(rsp, 16+ 8); // to
duke@435	1920	Address length_arg(rsp, 16+12); // elements count
duke@435	1921	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1922	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1923
duke@435	1924	Address SRC_POS_arg(rsp, 16+ 8);
duke@435	1925	Address DST_POS_arg(rsp, 16+16);
duke@435	1926	Address LENGTH_arg(rsp, 16+20);
duke@435	1927	// push rbx, changed the incoming offsets (why not just use rbp,??)
duke@435	1928	// assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
duke@435	1929
never@739	1930	__ movptr(rbx, Address(rsi_dst_klass, ek_offset));
never@739	1931	__ movl2ptr(length, LENGTH_arg); // reload elements count
never@739	1932	__ movl2ptr(src_pos, SRC_POS_arg); // reload src_pos
never@739	1933	__ movl2ptr(dst_pos, DST_POS_arg); // reload dst_pos
duke@435	1934
never@739	1935	__ movptr(ckval_arg, rbx); // destination element type
duke@435	1936	__ movl(rbx, Address(rbx, sco_offset));
duke@435	1937	__ movl(ckoff_arg, rbx); // corresponding class check offset
duke@435	1938
duke@435	1939	__ movl(length_arg, length); // outgoing length argument
duke@435	1940
never@739	1941	__ lea(from, Address(src, src_pos, Address::times_ptr,
duke@435	1942	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1943	__ movptr(from_arg, from);
duke@435	1944
never@739	1945	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
duke@435	1946	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1947	__ movptr(to_arg, to);
duke@435	1948	__ jump(RuntimeAddress(entry_checkcast_arraycopy));
duke@435	1949	}
duke@435	1950
duke@435	1951	return start;
duke@435	1952	}
duke@435	1953
duke@435	1954	void generate_arraycopy_stubs() {
duke@435	1955	address entry;
duke@435	1956	address entry_jbyte_arraycopy;
duke@435	1957	address entry_jshort_arraycopy;
duke@435	1958	address entry_jint_arraycopy;
duke@435	1959	address entry_oop_arraycopy;
duke@435	1960	address entry_jlong_arraycopy;
duke@435	1961	address entry_checkcast_arraycopy;
duke@435	1962
duke@435	1963	StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
duke@435	1964	generate_disjoint_copy(T_BYTE, true, Address::times_1, &entry,
duke@435	1965	"arrayof_jbyte_disjoint_arraycopy");
duke@435	1966	StubRoutines::_arrayof_jbyte_arraycopy =
duke@435	1967	generate_conjoint_copy(T_BYTE, true, Address::times_1, entry,
duke@435	1968	NULL, "arrayof_jbyte_arraycopy");
duke@435	1969	StubRoutines::_jbyte_disjoint_arraycopy =
duke@435	1970	generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
duke@435	1971	"jbyte_disjoint_arraycopy");
duke@435	1972	StubRoutines::_jbyte_arraycopy =
duke@435	1973	generate_conjoint_copy(T_BYTE, false, Address::times_1, entry,
duke@435	1974	&entry_jbyte_arraycopy, "jbyte_arraycopy");
duke@435	1975
duke@435	1976	StubRoutines::_arrayof_jshort_disjoint_arraycopy =
duke@435	1977	generate_disjoint_copy(T_SHORT, true, Address::times_2, &entry,
duke@435	1978	"arrayof_jshort_disjoint_arraycopy");
duke@435	1979	StubRoutines::_arrayof_jshort_arraycopy =
duke@435	1980	generate_conjoint_copy(T_SHORT, true, Address::times_2, entry,
duke@435	1981	NULL, "arrayof_jshort_arraycopy");
duke@435	1982	StubRoutines::_jshort_disjoint_arraycopy =
duke@435	1983	generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
duke@435	1984	"jshort_disjoint_arraycopy");
duke@435	1985	StubRoutines::_jshort_arraycopy =
duke@435	1986	generate_conjoint_copy(T_SHORT, false, Address::times_2, entry,
duke@435	1987	&entry_jshort_arraycopy, "jshort_arraycopy");
duke@435	1988
duke@435	1989	// Next arrays are always aligned on 4 bytes at least.
duke@435	1990	StubRoutines::_jint_disjoint_arraycopy =
duke@435	1991	generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
duke@435	1992	"jint_disjoint_arraycopy");
duke@435	1993	StubRoutines::_jint_arraycopy =
duke@435	1994	generate_conjoint_copy(T_INT, true, Address::times_4, entry,
duke@435	1995	&entry_jint_arraycopy, "jint_arraycopy");
duke@435	1996
duke@435	1997	StubRoutines::_oop_disjoint_arraycopy =
never@739	1998	generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
duke@435	1999	"oop_disjoint_arraycopy");
duke@435	2000	StubRoutines::_oop_arraycopy =
never@739	2001	generate_conjoint_copy(T_OBJECT, true, Address::times_ptr, entry,
duke@435	2002	&entry_oop_arraycopy, "oop_arraycopy");
duke@435	2003
duke@435	2004	StubRoutines::_jlong_disjoint_arraycopy =
duke@435	2005	generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
duke@435	2006	StubRoutines::_jlong_arraycopy =
duke@435	2007	generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
duke@435	2008	"jlong_arraycopy");
duke@435	2009
duke@435	2010	StubRoutines::_arrayof_jint_disjoint_arraycopy =
duke@435	2011	StubRoutines::_jint_disjoint_arraycopy;
duke@435	2012	StubRoutines::_arrayof_oop_disjoint_arraycopy =
duke@435	2013	StubRoutines::_oop_disjoint_arraycopy;
duke@435	2014	StubRoutines::_arrayof_jlong_disjoint_arraycopy =
duke@435	2015	StubRoutines::_jlong_disjoint_arraycopy;
duke@435	2016
duke@435	2017	StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
duke@435	2018	StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
duke@435	2019	StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
duke@435	2020
duke@435	2021	StubRoutines::_checkcast_arraycopy =
duke@435	2022	generate_checkcast_copy("checkcast_arraycopy",
duke@435	2023	&entry_checkcast_arraycopy);
duke@435	2024
duke@435	2025	StubRoutines::_unsafe_arraycopy =
duke@435	2026	generate_unsafe_copy("unsafe_arraycopy",
duke@435	2027	entry_jbyte_arraycopy,
duke@435	2028	entry_jshort_arraycopy,
duke@435	2029	entry_jint_arraycopy,
duke@435	2030	entry_jlong_arraycopy);
duke@435	2031
duke@435	2032	StubRoutines::_generic_arraycopy =
duke@435	2033	generate_generic_copy("generic_arraycopy",
duke@435	2034	entry_jbyte_arraycopy,
duke@435	2035	entry_jshort_arraycopy,
duke@435	2036	entry_jint_arraycopy,
duke@435	2037	entry_oop_arraycopy,
duke@435	2038	entry_jlong_arraycopy,
duke@435	2039	entry_checkcast_arraycopy);
duke@435	2040	}
duke@435	2041
never@1609	2042	void generate_math_stubs() {
never@1609	2043	{
never@1609	2044	StubCodeMark mark(this, "StubRoutines", "log");
never@1609	2045	StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
never@1609	2046
never@1609	2047	__ fld_d(Address(rsp, 4));
never@1609	2048	__ flog();
never@1609	2049	__ ret(0);
never@1609	2050	}
never@1609	2051	{
never@1609	2052	StubCodeMark mark(this, "StubRoutines", "log10");
never@1609	2053	StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
never@1609	2054
never@1609	2055	__ fld_d(Address(rsp, 4));
never@1609	2056	__ flog10();
never@1609	2057	__ ret(0);
never@1609	2058	}
never@1609	2059	{
never@1609	2060	StubCodeMark mark(this, "StubRoutines", "sin");
never@1609	2061	StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
never@1609	2062
never@1609	2063	__ fld_d(Address(rsp, 4));
never@1609	2064	__ trigfunc('s');
never@1609	2065	__ ret(0);
never@1609	2066	}
never@1609	2067	{
never@1609	2068	StubCodeMark mark(this, "StubRoutines", "cos");
never@1609	2069	StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
never@1609	2070
never@1609	2071	__ fld_d(Address(rsp, 4));
never@1609	2072	__ trigfunc('c');
never@1609	2073	__ ret(0);
never@1609	2074	}
never@1609	2075	{
never@1609	2076	StubCodeMark mark(this, "StubRoutines", "tan");
never@1609	2077	StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
never@1609	2078
never@1609	2079	__ fld_d(Address(rsp, 4));
never@1609	2080	__ trigfunc('t');
never@1609	2081	__ ret(0);
never@1609	2082	}
never@1609	2083
never@1609	2084	// The intrinsic version of these seem to return the same value as
never@1609	2085	// the strict version.
never@1609	2086	StubRoutines::_intrinsic_exp = SharedRuntime::dexp;
never@1609	2087	StubRoutines::_intrinsic_pow = SharedRuntime::dpow;
never@1609	2088	}
never@1609	2089
duke@435	2090	public:
duke@435	2091	// Information about frame layout at time of blocking runtime call.
duke@435	2092	// Note that we only have to preserve callee-saved registers since
duke@435	2093	// the compilers are responsible for supplying a continuation point
duke@435	2094	// if they expect all registers to be preserved.
duke@435	2095	enum layout {
duke@435	2096	thread_off, // last_java_sp
duke@435	2097	rbp_off, // callee saved register
duke@435	2098	ret_pc,
duke@435	2099	framesize
duke@435	2100	};
duke@435	2101
duke@435	2102	private:
duke@435	2103
duke@435	2104	#undef __
duke@435	2105	#define __ masm->
duke@435	2106
duke@435	2107	//------------------------------------------------------------------------------------------------------------------------
duke@435	2108	// Continuation point for throwing of implicit exceptions that are not handled in
duke@435	2109	// the current activation. Fabricates an exception oop and initiates normal
duke@435	2110	// exception dispatching in this frame.
duke@435	2111	//
duke@435	2112	// Previously the compiler (c2) allowed for callee save registers on Java calls.
duke@435	2113	// This is no longer true after adapter frames were removed but could possibly
duke@435	2114	// be brought back in the future if the interpreter code was reworked and it
duke@435	2115	// was deemed worthwhile. The comment below was left to describe what must
duke@435	2116	// happen here if callee saves were resurrected. As it stands now this stub
duke@435	2117	// could actually be a vanilla BufferBlob and have now oopMap at all.
duke@435	2118	// Since it doesn't make much difference we've chosen to leave it the
duke@435	2119	// way it was in the callee save days and keep the comment.
duke@435	2120
duke@435	2121	// If we need to preserve callee-saved values we need a callee-saved oop map and
duke@435	2122	// therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
duke@435	2123	// If the compiler needs all registers to be preserved between the fault
duke@435	2124	// point and the exception handler then it must assume responsibility for that in
duke@435	2125	// AbstractCompiler::continuation_for_implicit_null_exception or
duke@435	2126	// continuation_for_implicit_division_by_zero_exception. All other implicit
duke@435	2127	// exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
duke@435	2128	// either at call sites or otherwise assume that stack unwinding will be initiated,
duke@435	2129	// so caller saved registers were assumed volatile in the compiler.
duke@435	2130	address generate_throw_exception(const char* name, address runtime_entry,
duke@435	2131	bool restore_saved_exception_pc) {
duke@435	2132
duke@435	2133	int insts_size = 256;
duke@435	2134	int locs_size = 32;
duke@435	2135
duke@435	2136	CodeBuffer code(name, insts_size, locs_size);
duke@435	2137	OopMapSet* oop_maps = new OopMapSet();
duke@435	2138	MacroAssembler* masm = new MacroAssembler(&code);
duke@435	2139
duke@435	2140	address start = __ pc();
duke@435	2141
duke@435	2142	// This is an inlined and slightly modified version of call_VM
duke@435	2143	// which has the ability to fetch the return PC out of
duke@435	2144	// thread-local storage and also sets up last_Java_sp slightly
duke@435	2145	// differently than the real call_VM
duke@435	2146	Register java_thread = rbx;
duke@435	2147	__ get_thread(java_thread);
duke@435	2148	if (restore_saved_exception_pc) {
never@739	2149	__ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));
never@739	2150	__ push(rax);
duke@435	2151	}
duke@435	2152
duke@435	2153	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2154
duke@435	2155	// pc and rbp, already pushed
never@739	2156	__ subptr(rsp, (framesize-2) * wordSize); // prolog
duke@435	2157
duke@435	2158	// Frame is now completed as far as size and linkage.
duke@435	2159
duke@435	2160	int frame_complete = __ pc() - start;
duke@435	2161
duke@435	2162	// push java thread (becomes first argument of C function)
never@739	2163	__ movptr(Address(rsp, thread_off * wordSize), java_thread);
duke@435	2164
duke@435	2165	// Set up last_Java_sp and last_Java_fp
duke@435	2166	__ set_last_Java_frame(java_thread, rsp, rbp, NULL);
duke@435	2167
duke@435	2168	// Call runtime
duke@435	2169	BLOCK_COMMENT("call runtime_entry");
duke@435	2170	__ call(RuntimeAddress(runtime_entry));
duke@435	2171	// Generate oop map
duke@435	2172	OopMap* map = new OopMap(framesize, 0);
duke@435	2173	oop_maps->add_gc_map(__ pc() - start, map);
duke@435	2174
duke@435	2175	// restore the thread (cannot use the pushed argument since arguments
duke@435	2176	// may be overwritten by C code generated by an optimizing compiler);
duke@435	2177	// however can use the register value directly if it is callee saved.
duke@435	2178	__ get_thread(java_thread);
duke@435	2179
duke@435	2180	__ reset_last_Java_frame(java_thread, true, false);
duke@435	2181
duke@435	2182	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2183
duke@435	2184	// check for pending exceptions
duke@435	2185	#ifdef ASSERT
duke@435	2186	Label L;
never@739	2187	__ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	2188	__ jcc(Assembler::notEqual, L);
duke@435	2189	__ should_not_reach_here();
duke@435	2190	__ bind(L);
duke@435	2191	#endif /* ASSERT */
duke@435	2192	__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	2193
duke@435	2194
duke@435	2195	RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
duke@435	2196	return stub->entry_point();
duke@435	2197	}
duke@435	2198
duke@435	2199
duke@435	2200	void create_control_words() {
duke@435	2201	// Round to nearest, 53-bit mode, exceptions masked
duke@435	2202	StubRoutines::_fpu_cntrl_wrd_std = 0x027F;
duke@435	2203	// Round to zero, 53-bit mode, exception mased
duke@435	2204	StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
duke@435	2205	// Round to nearest, 24-bit mode, exceptions masked
duke@435	2206	StubRoutines::_fpu_cntrl_wrd_24 = 0x007F;
duke@435	2207	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2208	StubRoutines::_fpu_cntrl_wrd_64 = 0x037F;
duke@435	2209	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2210	StubRoutines::_mxcsr_std = 0x1F80;
duke@435	2211	// Note: the following two constants are 80-bit values
duke@435	2212	// layout is critical for correct loading by FPU.
duke@435	2213	// Bias for strict fp multiply/divide
duke@435	2214	StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
duke@435	2215	StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
duke@435	2216	StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
duke@435	2217	// Un-Bias for strict fp multiply/divide
duke@435	2218	StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
duke@435	2219	StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
duke@435	2220	StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
duke@435	2221	}
duke@435	2222
duke@435	2223	//---------------------------------------------------------------------------
duke@435	2224	// Initialization
duke@435	2225
duke@435	2226	void generate_initial() {
duke@435	2227	// Generates all stubs and initializes the entry points
duke@435	2228
duke@435	2229	//------------------------------------------------------------------------------------------------------------------------
duke@435	2230	// entry points that exist in all platforms
duke@435	2231	// Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
duke@435	2232	// the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
duke@435	2233	StubRoutines::_forward_exception_entry = generate_forward_exception();
duke@435	2234
duke@435	2235	StubRoutines::_call_stub_entry =
duke@435	2236	generate_call_stub(StubRoutines::_call_stub_return_address);
duke@435	2237	// is referenced by megamorphic call
duke@435	2238	StubRoutines::_catch_exception_entry = generate_catch_exception();
duke@435	2239
duke@435	2240	// These are currently used by Solaris/Intel
duke@435	2241	StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
duke@435	2242
duke@435	2243	StubRoutines::_handler_for_unsafe_access_entry =
duke@435	2244	generate_handler_for_unsafe_access();
duke@435	2245
duke@435	2246	// platform dependent
duke@435	2247	create_control_words();
duke@435	2248
never@739	2249	StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
never@739	2250	StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = generate_verify_fpu_cntrl_wrd();
duke@435	2251	StubRoutines::_d2i_wrapper = generate_d2i_wrapper(T_INT,
duke@435	2252	CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
duke@435	2253	StubRoutines::_d2l_wrapper = generate_d2i_wrapper(T_LONG,
duke@435	2254	CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
duke@435	2255	}
duke@435	2256
duke@435	2257
duke@435	2258	void generate_all() {
duke@435	2259	// Generates all stubs and initializes the entry points
duke@435	2260
duke@435	2261	// These entry points require SharedInfo::stack0 to be set up in non-core builds
duke@435	2262	// and need to be relocatable, so they each fabricate a RuntimeStub internally.
duke@435	2263	StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError), false);
dcubed@451	2264	StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError), false);
duke@435	2265	StubRoutines::_throw_ArithmeticException_entry = generate_throw_exception("ArithmeticException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException), true);
duke@435	2266	StubRoutines::_throw_NullPointerException_entry = generate_throw_exception("NullPointerException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);
duke@435	2267	StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);
duke@435	2268	StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError), false);
duke@435	2269
duke@435	2270	//------------------------------------------------------------------------------------------------------------------------
duke@435	2271	// entry points that are platform specific
duke@435	2272
duke@435	2273	// support for verify_oop (must happen after universe_init)
duke@435	2274	StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
duke@435	2275
duke@435	2276	// arraycopy stubs used by compilers
duke@435	2277	generate_arraycopy_stubs();
jrose@1145	2278
never@1609	2279	generate_math_stubs();
duke@435	2280	}
duke@435	2281
duke@435	2282
duke@435	2283	public:
duke@435	2284	StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
duke@435	2285	if (all) {
duke@435	2286	generate_all();
duke@435	2287	} else {
duke@435	2288	generate_initial();
duke@435	2289	}
duke@435	2290	}
duke@435	2291	}; // end class declaration
duke@435	2292
duke@435	2293
duke@435	2294	void StubGenerator_generate(CodeBuffer* code, bool all) {
duke@435	2295	StubGenerator g(code, all);
duke@435	2296	}