• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/stubGenerator_x86_32.cpp@d257356e35f0 (annotated)

src/cpu/x86/vm/stubGenerator_x86_32.cpp

Mon, 13 Sep 2010 23:24:30 -0700

author

jrose

date

Mon, 13 Sep 2010 23:24:30 -0700

changeset 2148

d257356e35f0

parent 2118

d6f45b55c972

child 2314

f95d63e2154a

permissions

-rw-r--r--

6939224: MethodHandle.invokeGeneric needs to perform the correct set of conversions
Reviewed-by: never

duke@435	1	/*
trims@1907	2	* Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
trims@1907	19	* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
trims@1907	20	* or visit www.oracle.com if you need additional information or have any
trims@1907	21	* questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_stubGenerator_x86_32.cpp.incl"
duke@435	27
duke@435	28	// Declaration and definition of StubGenerator (no .hpp file).
duke@435	29	// For a more detailed description of the stub routine structure
duke@435	30	// see the comment in stubRoutines.hpp
duke@435	31
duke@435	32	#define __ _masm->
never@739	33	#define a__ ((Assembler*)_masm)->
duke@435	34
duke@435	35	#ifdef PRODUCT
duke@435	36	#define BLOCK_COMMENT(str) /* nothing */
duke@435	37	#else
duke@435	38	#define BLOCK_COMMENT(str) __ block_comment(str)
duke@435	39	#endif
duke@435	40
duke@435	41	#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
duke@435	42
duke@435	43	const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
duke@435	44	const int FPU_CNTRL_WRD_MASK = 0xFFFF;
duke@435	45
duke@435	46	// -------------------------------------------------------------------------------------------------------------------------
duke@435	47	// Stub Code definitions
duke@435	48
duke@435	49	static address handle_unsafe_access() {
duke@435	50	JavaThread* thread = JavaThread::current();
duke@435	51	address pc = thread->saved_exception_pc();
duke@435	52	// pc is the instruction which we must emulate
duke@435	53	// doing a no-op is fine: return garbage from the load
duke@435	54	// therefore, compute npc
duke@435	55	address npc = Assembler::locate_next_instruction(pc);
duke@435	56
duke@435	57	// request an async exception
duke@435	58	thread->set_pending_unsafe_access_error();
duke@435	59
duke@435	60	// return address of next instruction to execute
duke@435	61	return npc;
duke@435	62	}
duke@435	63
duke@435	64	class StubGenerator: public StubCodeGenerator {
duke@435	65	private:
duke@435	66
duke@435	67	#ifdef PRODUCT
duke@435	68	#define inc_counter_np(counter) (0)
duke@435	69	#else
duke@435	70	void inc_counter_np_(int& counter) {
never@739	71	__ incrementl(ExternalAddress((address)&counter));
duke@435	72	}
duke@435	73	#define inc_counter_np(counter) \
duke@435	74	BLOCK_COMMENT("inc_counter " #counter); \
duke@435	75	inc_counter_np_(counter);
duke@435	76	#endif //PRODUCT
duke@435	77
duke@435	78	void inc_copy_counter_np(BasicType t) {
duke@435	79	#ifndef PRODUCT
duke@435	80	switch (t) {
duke@435	81	case T_BYTE: inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
duke@435	82	case T_SHORT: inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
duke@435	83	case T_INT: inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
duke@435	84	case T_LONG: inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
duke@435	85	case T_OBJECT: inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
duke@435	86	}
duke@435	87	ShouldNotReachHere();
duke@435	88	#endif //PRODUCT
duke@435	89	}
duke@435	90
duke@435	91	//------------------------------------------------------------------------------------------------------------------------
duke@435	92	// Call stubs are used to call Java from C
duke@435	93	//
duke@435	94	// [ return_from_Java ] <--- rsp
duke@435	95	// [ argument word n ]
duke@435	96	// ...
duke@435	97	// -N [ argument word 1 ]
duke@435	98	// -7 [ Possible padding for stack alignment ]
duke@435	99	// -6 [ Possible padding for stack alignment ]
duke@435	100	// -5 [ Possible padding for stack alignment ]
duke@435	101	// -4 [ mxcsr save ] <--- rsp_after_call
duke@435	102	// -3 [ saved rbx, ]
duke@435	103	// -2 [ saved rsi ]
duke@435	104	// -1 [ saved rdi ]
duke@435	105	// 0 [ saved rbp, ] <--- rbp,
duke@435	106	// 1 [ return address ]
duke@435	107	// 2 [ ptr. to call wrapper ]
duke@435	108	// 3 [ result ]
duke@435	109	// 4 [ result_type ]
duke@435	110	// 5 [ method ]
duke@435	111	// 6 [ entry_point ]
duke@435	112	// 7 [ parameters ]
duke@435	113	// 8 [ parameter_size ]
duke@435	114	// 9 [ thread ]
duke@435	115
duke@435	116
duke@435	117	address generate_call_stub(address& return_address) {
duke@435	118	StubCodeMark mark(this, "StubRoutines", "call_stub");
duke@435	119	address start = __ pc();
duke@435	120
duke@435	121	// stub code parameters / addresses
duke@435	122	assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
duke@435	123	bool sse_save = false;
duke@435	124	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
duke@435	125	const int locals_count_in_bytes (4*wordSize);
duke@435	126	const Address mxcsr_save (rbp, -4 * wordSize);
duke@435	127	const Address saved_rbx (rbp, -3 * wordSize);
duke@435	128	const Address saved_rsi (rbp, -2 * wordSize);
duke@435	129	const Address saved_rdi (rbp, -1 * wordSize);
duke@435	130	const Address result (rbp, 3 * wordSize);
duke@435	131	const Address result_type (rbp, 4 * wordSize);
duke@435	132	const Address method (rbp, 5 * wordSize);
duke@435	133	const Address entry_point (rbp, 6 * wordSize);
duke@435	134	const Address parameters (rbp, 7 * wordSize);
duke@435	135	const Address parameter_size(rbp, 8 * wordSize);
duke@435	136	const Address thread (rbp, 9 * wordSize); // same as in generate_catch_exception()!
duke@435	137	sse_save = UseSSE > 0;
duke@435	138
duke@435	139	// stub code
duke@435	140	__ enter();
never@739	141	__ movptr(rcx, parameter_size); // parameter counter
twisti@1861	142	__ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
never@739	143	__ addptr(rcx, locals_count_in_bytes); // reserve space for register saves
never@739	144	__ subptr(rsp, rcx);
never@739	145	__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
duke@435	146
duke@435	147	// save rdi, rsi, & rbx, according to C calling conventions
never@739	148	__ movptr(saved_rdi, rdi);
never@739	149	__ movptr(saved_rsi, rsi);
never@739	150	__ movptr(saved_rbx, rbx);
duke@435	151	// save and initialize %mxcsr
duke@435	152	if (sse_save) {
duke@435	153	Label skip_ldmx;
duke@435	154	__ stmxcsr(mxcsr_save);
duke@435	155	__ movl(rax, mxcsr_save);
duke@435	156	__ andl(rax, MXCSR_MASK); // Only check control and mask bits
duke@435	157	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
duke@435	158	__ cmp32(rax, mxcsr_std);
duke@435	159	__ jcc(Assembler::equal, skip_ldmx);
duke@435	160	__ ldmxcsr(mxcsr_std);
duke@435	161	__ bind(skip_ldmx);
duke@435	162	}
duke@435	163
duke@435	164	// make sure the control word is correct.
duke@435	165	__ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
duke@435	166
duke@435	167	#ifdef ASSERT
duke@435	168	// make sure we have no pending exceptions
duke@435	169	{ Label L;
never@739	170	__ movptr(rcx, thread);
never@739	171	__ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	172	__ jcc(Assembler::equal, L);
duke@435	173	__ stop("StubRoutines::call_stub: entered with pending exception");
duke@435	174	__ bind(L);
duke@435	175	}
duke@435	176	#endif
duke@435	177
duke@435	178	// pass parameters if any
duke@435	179	BLOCK_COMMENT("pass parameters if any");
duke@435	180	Label parameters_done;
duke@435	181	__ movl(rcx, parameter_size); // parameter counter
duke@435	182	__ testl(rcx, rcx);
duke@435	183	__ jcc(Assembler::zero, parameters_done);
duke@435	184
duke@435	185	// parameter passing loop
duke@435	186
duke@435	187	Label loop;
duke@435	188	// Copy Java parameters in reverse order (receiver last)
duke@435	189	// Note that the argument order is inverted in the process
duke@435	190	// source is rdx[rcx: N-1..0]
duke@435	191	// dest is rsp[rbx: 0..N-1]
duke@435	192
never@739	193	__ movptr(rdx, parameters); // parameter pointer
never@739	194	__ xorptr(rbx, rbx);
duke@435	195
duke@435	196	__ BIND(loop);
duke@435	197
duke@435	198	// get parameter
never@739	199	__ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
never@739	200	__ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
duke@435	201	Interpreter::expr_offset_in_bytes(0)), rax); // store parameter
duke@435	202	__ increment(rbx);
duke@435	203	__ decrement(rcx);
duke@435	204	__ jcc(Assembler::notZero, loop);
duke@435	205
duke@435	206	// call Java function
duke@435	207	__ BIND(parameters_done);
never@739	208	__ movptr(rbx, method); // get methodOop
never@739	209	__ movptr(rax, entry_point); // get entry_point
never@739	210	__ mov(rsi, rsp); // set sender sp
duke@435	211	BLOCK_COMMENT("call Java function");
duke@435	212	__ call(rax);
duke@435	213
duke@435	214	BLOCK_COMMENT("call_stub_return_address:");
duke@435	215	return_address = __ pc();
duke@435	216
duke@435	217	Label common_return;
duke@435	218
duke@435	219	__ BIND(common_return);
duke@435	220
duke@435	221	// store result depending on type
duke@435	222	// (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
never@739	223	__ movptr(rdi, result);
duke@435	224	Label is_long, is_float, is_double, exit;
duke@435	225	__ movl(rsi, result_type);
duke@435	226	__ cmpl(rsi, T_LONG);
duke@435	227	__ jcc(Assembler::equal, is_long);
duke@435	228	__ cmpl(rsi, T_FLOAT);
duke@435	229	__ jcc(Assembler::equal, is_float);
duke@435	230	__ cmpl(rsi, T_DOUBLE);
duke@435	231	__ jcc(Assembler::equal, is_double);
duke@435	232
duke@435	233	// handle T_INT case
duke@435	234	__ movl(Address(rdi, 0), rax);
duke@435	235	__ BIND(exit);
duke@435	236
duke@435	237	// check that FPU stack is empty
duke@435	238	__ verify_FPU(0, "generate_call_stub");
duke@435	239
duke@435	240	// pop parameters
never@739	241	__ lea(rsp, rsp_after_call);
duke@435	242
duke@435	243	// restore %mxcsr
duke@435	244	if (sse_save) {
duke@435	245	__ ldmxcsr(mxcsr_save);
duke@435	246	}
duke@435	247
duke@435	248	// restore rdi, rsi and rbx,
never@739	249	__ movptr(rbx, saved_rbx);
never@739	250	__ movptr(rsi, saved_rsi);
never@739	251	__ movptr(rdi, saved_rdi);
never@739	252	__ addptr(rsp, 4*wordSize);
duke@435	253
duke@435	254	// return
never@739	255	__ pop(rbp);
duke@435	256	__ ret(0);
duke@435	257
duke@435	258	// handle return types different from T_INT
duke@435	259	__ BIND(is_long);
duke@435	260	__ movl(Address(rdi, 0 * wordSize), rax);
duke@435	261	__ movl(Address(rdi, 1 * wordSize), rdx);
duke@435	262	__ jmp(exit);
duke@435	263
duke@435	264	__ BIND(is_float);
duke@435	265	// interpreter uses xmm0 for return values
duke@435	266	if (UseSSE >= 1) {
duke@435	267	__ movflt(Address(rdi, 0), xmm0);
duke@435	268	} else {
duke@435	269	__ fstp_s(Address(rdi, 0));
duke@435	270	}
duke@435	271	__ jmp(exit);
duke@435	272
duke@435	273	__ BIND(is_double);
duke@435	274	// interpreter uses xmm0 for return values
duke@435	275	if (UseSSE >= 2) {
duke@435	276	__ movdbl(Address(rdi, 0), xmm0);
duke@435	277	} else {
duke@435	278	__ fstp_d(Address(rdi, 0));
duke@435	279	}
duke@435	280	__ jmp(exit);
duke@435	281
duke@435	282	// If we call compiled code directly from the call stub we will
duke@435	283	// need to adjust the return back to the call stub to a specialized
duke@435	284	// piece of code that can handle compiled results and cleaning the fpu
duke@435	285	// stack. compiled code will be set to return here instead of the
duke@435	286	// return above that handles interpreter returns.
duke@435	287
duke@435	288	BLOCK_COMMENT("call_stub_compiled_return:");
never@739	289	StubRoutines::x86::set_call_stub_compiled_return( __ pc());
duke@435	290
duke@435	291	#ifdef COMPILER2
duke@435	292	if (UseSSE >= 2) {
duke@435	293	__ verify_FPU(0, "call_stub_compiled_return");
duke@435	294	} else {
duke@435	295	for (int i = 1; i < 8; i++) {
duke@435	296	__ ffree(i);
duke@435	297	}
duke@435	298
duke@435	299	// UseSSE <= 1 so double result should be left on TOS
duke@435	300	__ movl(rsi, result_type);
duke@435	301	__ cmpl(rsi, T_DOUBLE);
duke@435	302	__ jcc(Assembler::equal, common_return);
duke@435	303	if (UseSSE == 0) {
duke@435	304	// UseSSE == 0 so float result should be left on TOS
duke@435	305	__ cmpl(rsi, T_FLOAT);
duke@435	306	__ jcc(Assembler::equal, common_return);
duke@435	307	}
duke@435	308	__ ffree(0);
duke@435	309	}
duke@435	310	#endif /* COMPILER2 */
duke@435	311	__ jmp(common_return);
duke@435	312
duke@435	313	return start;
duke@435	314	}
duke@435	315
duke@435	316
duke@435	317	//------------------------------------------------------------------------------------------------------------------------
duke@435	318	// Return point for a Java call if there's an exception thrown in Java code.
duke@435	319	// The exception is caught and transformed into a pending exception stored in
duke@435	320	// JavaThread that can be tested from within the VM.
duke@435	321	//
duke@435	322	// Note: Usually the parameters are removed by the callee. In case of an exception
duke@435	323	// crossing an activation frame boundary, that is not the case if the callee
duke@435	324	// is compiled code => need to setup the rsp.
duke@435	325	//
duke@435	326	// rax,: exception oop
duke@435	327
duke@435	328	address generate_catch_exception() {
duke@435	329	StubCodeMark mark(this, "StubRoutines", "catch_exception");
duke@435	330	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
duke@435	331	const Address thread (rbp, 9 * wordSize); // same as in generate_call_stub()!
duke@435	332	address start = __ pc();
duke@435	333
duke@435	334	// get thread directly
never@739	335	__ movptr(rcx, thread);
duke@435	336	#ifdef ASSERT
duke@435	337	// verify that threads correspond
duke@435	338	{ Label L;
duke@435	339	__ get_thread(rbx);
never@739	340	__ cmpptr(rbx, rcx);
duke@435	341	__ jcc(Assembler::equal, L);
duke@435	342	__ stop("StubRoutines::catch_exception: threads must correspond");
duke@435	343	__ bind(L);
duke@435	344	}
duke@435	345	#endif
duke@435	346	// set pending exception
duke@435	347	__ verify_oop(rax);
never@739	348	__ movptr(Address(rcx, Thread::pending_exception_offset()), rax );
duke@435	349	__ lea(Address(rcx, Thread::exception_file_offset ()),
duke@435	350	ExternalAddress((address)__FILE__));
duke@435	351	__ movl(Address(rcx, Thread::exception_line_offset ()), __LINE__ );
duke@435	352	// complete return to VM
duke@435	353	assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
duke@435	354	__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
duke@435	355
duke@435	356	return start;
duke@435	357	}
duke@435	358
duke@435	359
duke@435	360	//------------------------------------------------------------------------------------------------------------------------
duke@435	361	// Continuation point for runtime calls returning with a pending exception.
duke@435	362	// The pending exception check happened in the runtime or native call stub.
duke@435	363	// The pending exception in Thread is converted into a Java-level exception.
duke@435	364	//
duke@435	365	// Contract with Java-level exception handlers:
twisti@1730	366	// rax: exception
duke@435	367	// rdx: throwing pc
duke@435	368	//
duke@435	369	// NOTE: At entry of this stub, exception-pc must be on stack !!
duke@435	370
duke@435	371	address generate_forward_exception() {
duke@435	372	StubCodeMark mark(this, "StubRoutines", "forward exception");
duke@435	373	address start = __ pc();
twisti@1730	374	const Register thread = rcx;
twisti@1730	375
twisti@1730	376	// other registers used in this stub
twisti@1730	377	const Register exception_oop = rax;
twisti@1730	378	const Register handler_addr = rbx;
twisti@1730	379	const Register exception_pc = rdx;
duke@435	380
duke@435	381	// Upon entry, the sp points to the return address returning into Java
duke@435	382	// (interpreted or compiled) code; i.e., the return address becomes the
duke@435	383	// throwing pc.
duke@435	384	//
duke@435	385	// Arguments pushed before the runtime call are still on the stack but
duke@435	386	// the exception handler will reset the stack pointer -> ignore them.
duke@435	387	// A potential result in registers can be ignored as well.
duke@435	388
duke@435	389	#ifdef ASSERT
duke@435	390	// make sure this code is only executed if there is a pending exception
duke@435	391	{ Label L;
twisti@1730	392	__ get_thread(thread);
twisti@1730	393	__ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	394	__ jcc(Assembler::notEqual, L);
duke@435	395	__ stop("StubRoutines::forward exception: no pending exception (1)");
duke@435	396	__ bind(L);
duke@435	397	}
duke@435	398	#endif
duke@435	399
duke@435	400	// compute exception handler into rbx,
twisti@1730	401	__ get_thread(thread);
twisti@1730	402	__ movptr(exception_pc, Address(rsp, 0));
duke@435	403	BLOCK_COMMENT("call exception_handler_for_return_address");
twisti@1730	404	__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
twisti@1730	405	__ mov(handler_addr, rax);
duke@435	406
twisti@1730	407	// setup rax & rdx, remove return address & clear pending exception
twisti@1730	408	__ get_thread(thread);
twisti@1730	409	__ pop(exception_pc);
twisti@1730	410	__ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
twisti@1730	411	__ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
duke@435	412
duke@435	413	#ifdef ASSERT
duke@435	414	// make sure exception is set
duke@435	415	{ Label L;
twisti@1730	416	__ testptr(exception_oop, exception_oop);
duke@435	417	__ jcc(Assembler::notEqual, L);
duke@435	418	__ stop("StubRoutines::forward exception: no pending exception (2)");
duke@435	419	__ bind(L);
duke@435	420	}
duke@435	421	#endif
duke@435	422
twisti@1730	423	// Verify that there is really a valid exception in RAX.
twisti@1730	424	__ verify_oop(exception_oop);
twisti@1730	425
twisti@1730	426	// Restore SP from BP if the exception PC is a MethodHandle call site.
twisti@1803	427	__ cmpl(Address(thread, JavaThread::is_method_handle_return_offset()), 0);
twisti@1730	428	__ cmovptr(Assembler::notEqual, rsp, rbp);
twisti@1730	429
duke@435	430	// continue at exception handler (return address removed)
twisti@1730	431	// rax: exception
twisti@1730	432	// rbx: exception handler
duke@435	433	// rdx: throwing pc
twisti@1730	434	__ jmp(handler_addr);
duke@435	435
duke@435	436	return start;
duke@435	437	}
duke@435	438
duke@435	439
duke@435	440	//----------------------------------------------------------------------------------------------------
duke@435	441	// Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
duke@435	442	//
duke@435	443	// xchg exists as far back as 8086, lock needed for MP only
duke@435	444	// Stack layout immediately after call:
duke@435	445	//
duke@435	446	// 0 [ret addr ] <--- rsp
duke@435	447	// 1 [ ex ]
duke@435	448	// 2 [ dest ]
duke@435	449	//
duke@435	450	// Result: *dest <- ex, return (old *dest)
duke@435	451	//
duke@435	452	// Note: win32 does not currently use this code
duke@435	453
duke@435	454	address generate_atomic_xchg() {
duke@435	455	StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
duke@435	456	address start = __ pc();
duke@435	457
never@739	458	__ push(rdx);
duke@435	459	Address exchange(rsp, 2 * wordSize);
duke@435	460	Address dest_addr(rsp, 3 * wordSize);
duke@435	461	__ movl(rax, exchange);
never@739	462	__ movptr(rdx, dest_addr);
never@739	463	__ xchgl(rax, Address(rdx, 0));
never@739	464	__ pop(rdx);
duke@435	465	__ ret(0);
duke@435	466
duke@435	467	return start;
duke@435	468	}
duke@435	469
duke@435	470	//----------------------------------------------------------------------------------------------------
duke@435	471	// Support for void verify_mxcsr()
duke@435	472	//
duke@435	473	// This routine is used with -Xcheck:jni to verify that native
duke@435	474	// JNI code does not return to Java code without restoring the
duke@435	475	// MXCSR register to our expected state.
duke@435	476
duke@435	477
duke@435	478	address generate_verify_mxcsr() {
duke@435	479	StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
duke@435	480	address start = __ pc();
duke@435	481
duke@435	482	const Address mxcsr_save(rsp, 0);
duke@435	483
duke@435	484	if (CheckJNICalls && UseSSE > 0 ) {
duke@435	485	Label ok_ret;
duke@435	486	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
never@739	487	__ push(rax);
never@739	488	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	489	__ stmxcsr(mxcsr_save);
duke@435	490	__ movl(rax, mxcsr_save);
duke@435	491	__ andl(rax, MXCSR_MASK);
duke@435	492	__ cmp32(rax, mxcsr_std);
duke@435	493	__ jcc(Assembler::equal, ok_ret);
duke@435	494
duke@435	495	__ warn("MXCSR changed by native JNI code.");
duke@435	496
duke@435	497	__ ldmxcsr(mxcsr_std);
duke@435	498
duke@435	499	__ bind(ok_ret);
never@739	500	__ addptr(rsp, wordSize);
never@739	501	__ pop(rax);
duke@435	502	}
duke@435	503
duke@435	504	__ ret(0);
duke@435	505
duke@435	506	return start;
duke@435	507	}
duke@435	508
duke@435	509
duke@435	510	//---------------------------------------------------------------------------
duke@435	511	// Support for void verify_fpu_cntrl_wrd()
duke@435	512	//
duke@435	513	// This routine is used with -Xcheck:jni to verify that native
duke@435	514	// JNI code does not return to Java code without restoring the
duke@435	515	// FP control word to our expected state.
duke@435	516
duke@435	517	address generate_verify_fpu_cntrl_wrd() {
duke@435	518	StubCodeMark mark(this, "StubRoutines", "verify_spcw");
duke@435	519	address start = __ pc();
duke@435	520
duke@435	521	const Address fpu_cntrl_wrd_save(rsp, 0);
duke@435	522
duke@435	523	if (CheckJNICalls) {
duke@435	524	Label ok_ret;
never@739	525	__ push(rax);
never@739	526	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	527	__ fnstcw(fpu_cntrl_wrd_save);
duke@435	528	__ movl(rax, fpu_cntrl_wrd_save);
duke@435	529	__ andl(rax, FPU_CNTRL_WRD_MASK);
duke@435	530	ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
duke@435	531	__ cmp32(rax, fpu_std);
duke@435	532	__ jcc(Assembler::equal, ok_ret);
duke@435	533
duke@435	534	__ warn("Floating point control word changed by native JNI code.");
duke@435	535
duke@435	536	__ fldcw(fpu_std);
duke@435	537
duke@435	538	__ bind(ok_ret);
never@739	539	__ addptr(rsp, wordSize);
never@739	540	__ pop(rax);
duke@435	541	}
duke@435	542
duke@435	543	__ ret(0);
duke@435	544
duke@435	545	return start;
duke@435	546	}
duke@435	547
duke@435	548	//---------------------------------------------------------------------------
duke@435	549	// Wrapper for slow-case handling of double-to-integer conversion
duke@435	550	// d2i or f2i fast case failed either because it is nan or because
duke@435	551	// of under/overflow.
duke@435	552	// Input: FPU TOS: float value
duke@435	553	// Output: rax, (rdx): integer (long) result
duke@435	554
duke@435	555	address generate_d2i_wrapper(BasicType t, address fcn) {
duke@435	556	StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
duke@435	557	address start = __ pc();
duke@435	558
duke@435	559	// Capture info about frame layout
duke@435	560	enum layout { FPUState_off = 0,
duke@435	561	rbp_off = FPUStateSizeInWords,
duke@435	562	rdi_off,
duke@435	563	rsi_off,
duke@435	564	rcx_off,
duke@435	565	rbx_off,
duke@435	566	saved_argument_off,
duke@435	567	saved_argument_off2, // 2nd half of double
duke@435	568	framesize
duke@435	569	};
duke@435	570
duke@435	571	assert(FPUStateSizeInWords == 27, "update stack layout");
duke@435	572
duke@435	573	// Save outgoing argument to stack across push_FPU_state()
never@739	574	__ subptr(rsp, wordSize * 2);
duke@435	575	__ fstp_d(Address(rsp, 0));
duke@435	576
duke@435	577	// Save CPU & FPU state
never@739	578	__ push(rbx);
never@739	579	__ push(rcx);
never@739	580	__ push(rsi);
never@739	581	__ push(rdi);
never@739	582	__ push(rbp);
duke@435	583	__ push_FPU_state();
duke@435	584
duke@435	585	// push_FPU_state() resets the FP top of stack
duke@435	586	// Load original double into FP top of stack
duke@435	587	__ fld_d(Address(rsp, saved_argument_off * wordSize));
duke@435	588	// Store double into stack as outgoing argument
never@739	589	__ subptr(rsp, wordSize*2);
duke@435	590	__ fst_d(Address(rsp, 0));
duke@435	591
duke@435	592	// Prepare FPU for doing math in C-land
duke@435	593	__ empty_FPU_stack();
duke@435	594	// Call the C code to massage the double. Result in EAX
duke@435	595	if (t == T_INT)
duke@435	596	{ BLOCK_COMMENT("SharedRuntime::d2i"); }
duke@435	597	else if (t == T_LONG)
duke@435	598	{ BLOCK_COMMENT("SharedRuntime::d2l"); }
duke@435	599	__ call_VM_leaf( fcn, 2 );
duke@435	600
duke@435	601	// Restore CPU & FPU state
duke@435	602	__ pop_FPU_state();
never@739	603	__ pop(rbp);
never@739	604	__ pop(rdi);
never@739	605	__ pop(rsi);
never@739	606	__ pop(rcx);
never@739	607	__ pop(rbx);
never@739	608	__ addptr(rsp, wordSize * 2);
duke@435	609
duke@435	610	__ ret(0);
duke@435	611
duke@435	612	return start;
duke@435	613	}
duke@435	614
duke@435	615
duke@435	616	//---------------------------------------------------------------------------
duke@435	617	// The following routine generates a subroutine to throw an asynchronous
duke@435	618	// UnknownError when an unsafe access gets a fault that could not be
duke@435	619	// reasonably prevented by the programmer. (Example: SIGBUS/OBJERR.)
duke@435	620	address generate_handler_for_unsafe_access() {
duke@435	621	StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
duke@435	622	address start = __ pc();
duke@435	623
never@739	624	__ push(0); // hole for return address-to-be
never@739	625	__ pusha(); // push registers
duke@435	626	Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
duke@435	627	BLOCK_COMMENT("call handle_unsafe_access");
duke@435	628	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
never@739	629	__ movptr(next_pc, rax); // stuff next address
never@739	630	__ popa();
duke@435	631	__ ret(0); // jump to next address
duke@435	632
duke@435	633	return start;
duke@435	634	}
duke@435	635
duke@435	636
duke@435	637	//----------------------------------------------------------------------------------------------------
duke@435	638	// Non-destructive plausibility checks for oops
duke@435	639
duke@435	640	address generate_verify_oop() {
duke@435	641	StubCodeMark mark(this, "StubRoutines", "verify_oop");
duke@435	642	address start = __ pc();
duke@435	643
duke@435	644	// Incoming arguments on stack after saving rax,:
duke@435	645	//
duke@435	646	// [tos ]: saved rdx
duke@435	647	// [tos + 1]: saved EFLAGS
duke@435	648	// [tos + 2]: return address
duke@435	649	// [tos + 3]: char* error message
duke@435	650	// [tos + 4]: oop object to verify
duke@435	651	// [tos + 5]: saved rax, - saved by caller and bashed
duke@435	652
duke@435	653	Label exit, error;
never@739	654	__ pushf();
never@739	655	__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
never@739	656	__ push(rdx); // save rdx
duke@435	657	// make sure object is 'reasonable'
never@739	658	__ movptr(rax, Address(rsp, 4 * wordSize)); // get object
never@739	659	__ testptr(rax, rax);
duke@435	660	__ jcc(Assembler::zero, exit); // if obj is NULL it is ok
duke@435	661
duke@435	662	// Check if the oop is in the right area of memory
duke@435	663	const int oop_mask = Universe::verify_oop_mask();
duke@435	664	const int oop_bits = Universe::verify_oop_bits();
never@739	665	__ mov(rdx, rax);
never@739	666	__ andptr(rdx, oop_mask);
never@739	667	__ cmpptr(rdx, oop_bits);
duke@435	668	__ jcc(Assembler::notZero, error);
duke@435	669
duke@435	670	// make sure klass is 'reasonable'
never@739	671	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
never@739	672	__ testptr(rax, rax);
duke@435	673	__ jcc(Assembler::zero, error); // if klass is NULL it is broken
duke@435	674
duke@435	675	// Check if the klass is in the right area of memory
duke@435	676	const int klass_mask = Universe::verify_klass_mask();
duke@435	677	const int klass_bits = Universe::verify_klass_bits();
never@739	678	__ mov(rdx, rax);
never@739	679	__ andptr(rdx, klass_mask);
never@739	680	__ cmpptr(rdx, klass_bits);
duke@435	681	__ jcc(Assembler::notZero, error);
duke@435	682
duke@435	683	// make sure klass' klass is 'reasonable'
never@739	684	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass
never@739	685	__ testptr(rax, rax);
duke@435	686	__ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
duke@435	687
never@739	688	__ mov(rdx, rax);
never@739	689	__ andptr(rdx, klass_mask);
never@739	690	__ cmpptr(rdx, klass_bits);
duke@435	691	__ jcc(Assembler::notZero, error); // if klass not in right area
duke@435	692	// of memory it is broken too.
duke@435	693
duke@435	694	// return if everything seems ok
duke@435	695	__ bind(exit);
never@739	696	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	697	__ pop(rdx); // restore rdx
never@739	698	__ popf(); // restore EFLAGS
duke@435	699	__ ret(3 * wordSize); // pop arguments
duke@435	700
duke@435	701	// handle errors
duke@435	702	__ bind(error);
never@739	703	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	704	__ pop(rdx); // get saved rdx back
never@739	705	__ popf(); // get saved EFLAGS off stack -- will be ignored
never@739	706	__ pusha(); // push registers (eip = return address & msg are already pushed)
duke@435	707	BLOCK_COMMENT("call MacroAssembler::debug");
never@739	708	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
never@739	709	__ popa();
duke@435	710	__ ret(3 * wordSize); // pop arguments
duke@435	711	return start;
duke@435	712	}
duke@435	713
duke@435	714	//
duke@435	715	// Generate pre-barrier for array stores
duke@435	716	//
duke@435	717	// Input:
duke@435	718	// start - starting address
ysr@1280	719	// count - element count
duke@435	720	void gen_write_ref_array_pre_barrier(Register start, Register count) {
duke@435	721	assert_different_registers(start, count);
duke@435	722	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	723	switch (bs->kind()) {
duke@435	724	case BarrierSet::G1SATBCT:
duke@435	725	case BarrierSet::G1SATBCTLogging:
duke@435	726	{
never@739	727	__ pusha(); // push registers
apetrusenko@1627	728	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
apetrusenko@1627	729	start, count);
never@739	730	__ popa();
duke@435	731	}
duke@435	732	break;
duke@435	733	case BarrierSet::CardTableModRef:
duke@435	734	case BarrierSet::CardTableExtension:
duke@435	735	case BarrierSet::ModRef:
duke@435	736	break;
duke@435	737	default :
duke@435	738	ShouldNotReachHere();
duke@435	739
duke@435	740	}
duke@435	741	}
duke@435	742
duke@435	743
duke@435	744	//
duke@435	745	// Generate a post-barrier for an array store
duke@435	746	//
duke@435	747	// start - starting address
duke@435	748	// count - element count
duke@435	749	//
duke@435	750	// The two input registers are overwritten.
duke@435	751	//
duke@435	752	void gen_write_ref_array_post_barrier(Register start, Register count) {
duke@435	753	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	754	assert_different_registers(start, count);
duke@435	755	switch (bs->kind()) {
duke@435	756	case BarrierSet::G1SATBCT:
duke@435	757	case BarrierSet::G1SATBCTLogging:
duke@435	758	{
never@739	759	__ pusha(); // push registers
apetrusenko@1627	760	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
apetrusenko@1627	761	start, count);
never@739	762	__ popa();
duke@435	763	}
duke@435	764	break;
duke@435	765
duke@435	766	case BarrierSet::CardTableModRef:
duke@435	767	case BarrierSet::CardTableExtension:
duke@435	768	{
duke@435	769	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	770	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	771
duke@435	772	Label L_loop;
duke@435	773	const Register end = count; // elements count; end == start+count-1
duke@435	774	assert_different_registers(start, end);
duke@435	775
never@739	776	__ lea(end, Address(start, count, Address::times_ptr, -wordSize));
never@739	777	__ shrptr(start, CardTableModRefBS::card_shift);
never@739	778	__ shrptr(end, CardTableModRefBS::card_shift);
never@739	779	__ subptr(end, start); // end --> count
duke@435	780	__ BIND(L_loop);
never@684	781	intptr_t disp = (intptr_t) ct->byte_map_base;
never@684	782	Address cardtable(start, count, Address::times_1, disp);
never@684	783	__ movb(cardtable, 0);
duke@435	784	__ decrement(count);
duke@435	785	__ jcc(Assembler::greaterEqual, L_loop);
duke@435	786	}
duke@435	787	break;
duke@435	788	case BarrierSet::ModRef:
duke@435	789	break;
duke@435	790	default :
duke@435	791	ShouldNotReachHere();
duke@435	792
duke@435	793	}
duke@435	794	}
duke@435	795
kvn@840	796
kvn@840	797	// Copy 64 bytes chunks
kvn@840	798	//
kvn@840	799	// Inputs:
kvn@840	800	// from - source array address
kvn@840	801	// to_from - destination array address - from
kvn@840	802	// qword_count - 8-bytes element count, negative
kvn@840	803	//
kvn@840	804	void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	805	assert( UseSSE >= 2, "supported cpu only" );
kvn@840	806	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
kvn@840	807	// Copy 64-byte chunks
kvn@840	808	__ jmpb(L_copy_64_bytes);
kvn@1800	809	__ align(OptoLoopAlignment);
kvn@840	810	__ BIND(L_copy_64_bytes_loop);
kvn@840	811
kvn@840	812	if(UseUnalignedLoadStores) {
kvn@840	813	__ movdqu(xmm0, Address(from, 0));
kvn@840	814	__ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	815	__ movdqu(xmm1, Address(from, 16));
kvn@840	816	__ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
kvn@840	817	__ movdqu(xmm2, Address(from, 32));
kvn@840	818	__ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
kvn@840	819	__ movdqu(xmm3, Address(from, 48));
kvn@840	820	__ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
kvn@840	821
kvn@840	822	} else {
kvn@840	823	__ movq(xmm0, Address(from, 0));
kvn@840	824	__ movq(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	825	__ movq(xmm1, Address(from, 8));
kvn@840	826	__ movq(Address(from, to_from, Address::times_1, 8), xmm1);
kvn@840	827	__ movq(xmm2, Address(from, 16));
kvn@840	828	__ movq(Address(from, to_from, Address::times_1, 16), xmm2);
kvn@840	829	__ movq(xmm3, Address(from, 24));
kvn@840	830	__ movq(Address(from, to_from, Address::times_1, 24), xmm3);
kvn@840	831	__ movq(xmm4, Address(from, 32));
kvn@840	832	__ movq(Address(from, to_from, Address::times_1, 32), xmm4);
kvn@840	833	__ movq(xmm5, Address(from, 40));
kvn@840	834	__ movq(Address(from, to_from, Address::times_1, 40), xmm5);
kvn@840	835	__ movq(xmm6, Address(from, 48));
kvn@840	836	__ movq(Address(from, to_from, Address::times_1, 48), xmm6);
kvn@840	837	__ movq(xmm7, Address(from, 56));
kvn@840	838	__ movq(Address(from, to_from, Address::times_1, 56), xmm7);
kvn@840	839	}
kvn@840	840
kvn@840	841	__ addl(from, 64);
kvn@840	842	__ BIND(L_copy_64_bytes);
kvn@840	843	__ subl(qword_count, 8);
kvn@840	844	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
kvn@840	845	__ addl(qword_count, 8);
kvn@840	846	__ jccb(Assembler::zero, L_exit);
kvn@840	847	//
kvn@840	848	// length is too short, just copy qwords
kvn@840	849	//
kvn@840	850	__ BIND(L_copy_8_bytes);
kvn@840	851	__ movq(xmm0, Address(from, 0));
kvn@840	852	__ movq(Address(from, to_from, Address::times_1), xmm0);
kvn@840	853	__ addl(from, 8);
kvn@840	854	__ decrement(qword_count);
kvn@840	855	__ jcc(Assembler::greater, L_copy_8_bytes);
kvn@840	856	__ BIND(L_exit);
kvn@840	857	}
kvn@840	858
duke@435	859	// Copy 64 bytes chunks
duke@435	860	//
duke@435	861	// Inputs:
duke@435	862	// from - source array address
duke@435	863	// to_from - destination array address - from
duke@435	864	// qword_count - 8-bytes element count, negative
duke@435	865	//
duke@435	866	void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	867	assert( VM_Version::supports_mmx(), "supported cpu only" );
duke@435	868	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
duke@435	869	// Copy 64-byte chunks
duke@435	870	__ jmpb(L_copy_64_bytes);
kvn@1800	871	__ align(OptoLoopAlignment);
duke@435	872	__ BIND(L_copy_64_bytes_loop);
duke@435	873	__ movq(mmx0, Address(from, 0));
duke@435	874	__ movq(mmx1, Address(from, 8));
duke@435	875	__ movq(mmx2, Address(from, 16));
duke@435	876	__ movq(Address(from, to_from, Address::times_1, 0), mmx0);
duke@435	877	__ movq(mmx3, Address(from, 24));
duke@435	878	__ movq(Address(from, to_from, Address::times_1, 8), mmx1);
duke@435	879	__ movq(mmx4, Address(from, 32));
duke@435	880	__ movq(Address(from, to_from, Address::times_1, 16), mmx2);
duke@435	881	__ movq(mmx5, Address(from, 40));
duke@435	882	__ movq(Address(from, to_from, Address::times_1, 24), mmx3);
duke@435	883	__ movq(mmx6, Address(from, 48));
duke@435	884	__ movq(Address(from, to_from, Address::times_1, 32), mmx4);
duke@435	885	__ movq(mmx7, Address(from, 56));
duke@435	886	__ movq(Address(from, to_from, Address::times_1, 40), mmx5);
duke@435	887	__ movq(Address(from, to_from, Address::times_1, 48), mmx6);
duke@435	888	__ movq(Address(from, to_from, Address::times_1, 56), mmx7);
never@739	889	__ addptr(from, 64);
duke@435	890	__ BIND(L_copy_64_bytes);
duke@435	891	__ subl(qword_count, 8);
duke@435	892	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
duke@435	893	__ addl(qword_count, 8);
duke@435	894	__ jccb(Assembler::zero, L_exit);
duke@435	895	//
duke@435	896	// length is too short, just copy qwords
duke@435	897	//
duke@435	898	__ BIND(L_copy_8_bytes);
duke@435	899	__ movq(mmx0, Address(from, 0));
duke@435	900	__ movq(Address(from, to_from, Address::times_1), mmx0);
never@739	901	__ addptr(from, 8);
duke@435	902	__ decrement(qword_count);
duke@435	903	__ jcc(Assembler::greater, L_copy_8_bytes);
duke@435	904	__ BIND(L_exit);
duke@435	905	__ emms();
duke@435	906	}
duke@435	907
duke@435	908	address generate_disjoint_copy(BasicType t, bool aligned,
duke@435	909	Address::ScaleFactor sf,
duke@435	910	address* entry, const char *name) {
duke@435	911	__ align(CodeEntryAlignment);
duke@435	912	StubCodeMark mark(this, "StubRoutines", name);
duke@435	913	address start = __ pc();
duke@435	914
duke@435	915	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	916	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
duke@435	917
never@739	918	int shift = Address::times_ptr - sf;
duke@435	919
duke@435	920	const Register from = rsi; // source array address
duke@435	921	const Register to = rdi; // destination array address
duke@435	922	const Register count = rcx; // elements count
duke@435	923	const Register to_from = to; // (to - from)
duke@435	924	const Register saved_to = rdx; // saved destination array address
duke@435	925
duke@435	926	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	927	__ push(rsi);
never@739	928	__ push(rdi);
never@739	929	__ movptr(from , Address(rsp, 12+ 4));
never@739	930	__ movptr(to , Address(rsp, 12+ 8));
duke@435	931	__ movl(count, Address(rsp, 12+ 12));
duke@435	932	if (t == T_OBJECT) {
duke@435	933	__ testl(count, count);
duke@435	934	__ jcc(Assembler::zero, L_0_count);
duke@435	935	gen_write_ref_array_pre_barrier(to, count);
never@739	936	__ mov(saved_to, to); // save 'to'
duke@435	937	}
duke@435	938
duke@435	939	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	940	BLOCK_COMMENT("Entry:");
duke@435	941
never@739	942	__ subptr(to, from); // to --> to_from
duke@435	943	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	944	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
kvn@840	945	if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
duke@435	946	// align source address at 4 bytes address boundary
duke@435	947	if (t == T_BYTE) {
duke@435	948	// One byte misalignment happens only for byte arrays
duke@435	949	__ testl(from, 1);
duke@435	950	__ jccb(Assembler::zero, L_skip_align1);
duke@435	951	__ movb(rax, Address(from, 0));
duke@435	952	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	953	__ increment(from);
duke@435	954	__ decrement(count);
duke@435	955	__ BIND(L_skip_align1);
duke@435	956	}
duke@435	957	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	958	__ testl(from, 2);
duke@435	959	__ jccb(Assembler::zero, L_skip_align2);
duke@435	960	__ movw(rax, Address(from, 0));
duke@435	961	__ movw(Address(from, to_from, Address::times_1, 0), rax);
never@739	962	__ addptr(from, 2);
duke@435	963	__ subl(count, 1<<(shift-1));
duke@435	964	__ BIND(L_skip_align2);
duke@435	965	}
duke@435	966	if (!VM_Version::supports_mmx()) {
never@739	967	__ mov(rax, count); // save 'count'
never@739	968	__ shrl(count, shift); // bytes count
never@739	969	__ addptr(to_from, from);// restore 'to'
never@739	970	__ rep_mov();
never@739	971	__ subptr(to_from, from);// restore 'to_from'
never@739	972	__ mov(count, rax); // restore 'count'
duke@435	973	__ jmpb(L_copy_2_bytes); // all dwords were copied
duke@435	974	} else {
kvn@840	975	if (!UseUnalignedLoadStores) {
kvn@840	976	// align to 8 bytes, we know we are 4 byte aligned to start
kvn@840	977	__ testptr(from, 4);
kvn@840	978	__ jccb(Assembler::zero, L_copy_64_bytes);
kvn@840	979	__ movl(rax, Address(from, 0));
kvn@840	980	__ movl(Address(from, to_from, Address::times_1, 0), rax);
kvn@840	981	__ addptr(from, 4);
kvn@840	982	__ subl(count, 1<<shift);
kvn@840	983	}
duke@435	984	__ BIND(L_copy_64_bytes);
never@739	985	__ mov(rax, count);
duke@435	986	__ shrl(rax, shift+1); // 8 bytes chunk count
duke@435	987	//
duke@435	988	// Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
duke@435	989	//
kvn@840	990	if (UseXMMForArrayCopy) {
kvn@840	991	xmm_copy_forward(from, to_from, rax);
kvn@840	992	} else {
kvn@840	993	mmx_copy_forward(from, to_from, rax);
kvn@840	994	}
duke@435	995	}
duke@435	996	// copy tailing dword
duke@435	997	__ BIND(L_copy_4_bytes);
duke@435	998	__ testl(count, 1<<shift);
duke@435	999	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1000	__ movl(rax, Address(from, 0));
duke@435	1001	__ movl(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1002	if (t == T_BYTE || t == T_SHORT) {
never@739	1003	__ addptr(from, 4);
duke@435	1004	__ BIND(L_copy_2_bytes);
duke@435	1005	// copy tailing word
duke@435	1006	__ testl(count, 1<<(shift-1));
duke@435	1007	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1008	__ movw(rax, Address(from, 0));
duke@435	1009	__ movw(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1010	if (t == T_BYTE) {
never@739	1011	__ addptr(from, 2);
duke@435	1012	__ BIND(L_copy_byte);
duke@435	1013	// copy tailing byte
duke@435	1014	__ testl(count, 1);
duke@435	1015	__ jccb(Assembler::zero, L_exit);
duke@435	1016	__ movb(rax, Address(from, 0));
duke@435	1017	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1018	__ BIND(L_exit);
duke@435	1019	} else {
duke@435	1020	__ BIND(L_copy_byte);
duke@435	1021	}
duke@435	1022	} else {
duke@435	1023	__ BIND(L_copy_2_bytes);
duke@435	1024	}
duke@435	1025
duke@435	1026	if (t == T_OBJECT) {
duke@435	1027	__ movl(count, Address(rsp, 12+12)); // reread 'count'
never@739	1028	__ mov(to, saved_to); // restore 'to'
duke@435	1029	gen_write_ref_array_post_barrier(to, count);
duke@435	1030	__ BIND(L_0_count);
duke@435	1031	}
duke@435	1032	inc_copy_counter_np(t);
never@739	1033	__ pop(rdi);
never@739	1034	__ pop(rsi);
duke@435	1035	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1036	__ xorptr(rax, rax); // return 0
duke@435	1037	__ ret(0);
duke@435	1038	return start;
duke@435	1039	}
duke@435	1040
duke@435	1041
never@2118	1042	address generate_fill(BasicType t, bool aligned, const char *name) {
never@2118	1043	__ align(CodeEntryAlignment);
never@2118	1044	StubCodeMark mark(this, "StubRoutines", name);
never@2118	1045	address start = __ pc();
never@2118	1046
never@2118	1047	BLOCK_COMMENT("Entry:");
never@2118	1048
never@2118	1049	const Register to = rdi; // source array address
never@2118	1050	const Register value = rdx; // value
never@2118	1051	const Register count = rsi; // elements count
never@2118	1052
never@2118	1053	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@2118	1054	__ push(rsi);
never@2118	1055	__ push(rdi);
never@2118	1056	__ movptr(to , Address(rsp, 12+ 4));
never@2118	1057	__ movl(value, Address(rsp, 12+ 8));
never@2118	1058	__ movl(count, Address(rsp, 12+ 12));
never@2118	1059
never@2118	1060	__ generate_fill(t, aligned, to, value, count, rax, xmm0);
never@2118	1061
never@2118	1062	__ pop(rdi);
never@2118	1063	__ pop(rsi);
never@2118	1064	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@2118	1065	__ ret(0);
never@2118	1066	return start;
never@2118	1067	}
never@2118	1068
duke@435	1069	address generate_conjoint_copy(BasicType t, bool aligned,
duke@435	1070	Address::ScaleFactor sf,
duke@435	1071	address nooverlap_target,
duke@435	1072	address* entry, const char *name) {
duke@435	1073	__ align(CodeEntryAlignment);
duke@435	1074	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1075	address start = __ pc();
duke@435	1076
duke@435	1077	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	1078	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1079
never@739	1080	int shift = Address::times_ptr - sf;
duke@435	1081
duke@435	1082	const Register src = rax; // source array address
duke@435	1083	const Register dst = rdx; // destination array address
duke@435	1084	const Register from = rsi; // source array address
duke@435	1085	const Register to = rdi; // destination array address
duke@435	1086	const Register count = rcx; // elements count
duke@435	1087	const Register end = rax; // array end address
duke@435	1088
duke@435	1089	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1090	__ push(rsi);
never@739	1091	__ push(rdi);
never@739	1092	__ movptr(src , Address(rsp, 12+ 4)); // from
never@739	1093	__ movptr(dst , Address(rsp, 12+ 8)); // to
never@739	1094	__ movl2ptr(count, Address(rsp, 12+12)); // count
duke@435	1095	if (t == T_OBJECT) {
duke@435	1096	gen_write_ref_array_pre_barrier(dst, count);
duke@435	1097	}
duke@435	1098
duke@435	1099	if (entry != NULL) {
duke@435	1100	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1101	BLOCK_COMMENT("Entry:");
duke@435	1102	}
duke@435	1103
duke@435	1104	if (t == T_OBJECT) {
duke@435	1105	__ testl(count, count);
duke@435	1106	__ jcc(Assembler::zero, L_0_count);
duke@435	1107	}
never@739	1108	__ mov(from, src);
never@739	1109	__ mov(to , dst);
duke@435	1110
duke@435	1111	// arrays overlap test
duke@435	1112	RuntimeAddress nooverlap(nooverlap_target);
never@739	1113	__ cmpptr(dst, src);
never@739	1114	__ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
duke@435	1115	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1116	__ cmpptr(dst, end);
duke@435	1117	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1118
duke@435	1119	// copy from high to low
duke@435	1120	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1121	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
duke@435	1122	if (t == T_BYTE || t == T_SHORT) {
duke@435	1123	// Align the end of destination array at 4 bytes address boundary
never@739	1124	__ lea(end, Address(dst, count, sf, 0));
duke@435	1125	if (t == T_BYTE) {
duke@435	1126	// One byte misalignment happens only for byte arrays
duke@435	1127	__ testl(end, 1);
duke@435	1128	__ jccb(Assembler::zero, L_skip_align1);
duke@435	1129	__ decrement(count);
duke@435	1130	__ movb(rdx, Address(from, count, sf, 0));
duke@435	1131	__ movb(Address(to, count, sf, 0), rdx);
duke@435	1132	__ BIND(L_skip_align1);
duke@435	1133	}
duke@435	1134	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	1135	__ testl(end, 2);
duke@435	1136	__ jccb(Assembler::zero, L_skip_align2);
never@739	1137	__ subptr(count, 1<<(shift-1));
duke@435	1138	__ movw(rdx, Address(from, count, sf, 0));
duke@435	1139	__ movw(Address(to, count, sf, 0), rdx);
duke@435	1140	__ BIND(L_skip_align2);
duke@435	1141	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1142	__ jcc(Assembler::below, L_copy_4_bytes);
duke@435	1143	}
duke@435	1144
duke@435	1145	if (!VM_Version::supports_mmx()) {
duke@435	1146	__ std();
never@739	1147	__ mov(rax, count); // Save 'count'
never@739	1148	__ mov(rdx, to); // Save 'to'
never@739	1149	__ lea(rsi, Address(from, count, sf, -4));
never@739	1150	__ lea(rdi, Address(to , count, sf, -4));
never@739	1151	__ shrptr(count, shift); // bytes count
never@739	1152	__ rep_mov();
duke@435	1153	__ cld();
never@739	1154	__ mov(count, rax); // restore 'count'
duke@435	1155	__ andl(count, (1<<shift)-1); // mask the number of rest elements
never@739	1156	__ movptr(from, Address(rsp, 12+4)); // reread 'from'
never@739	1157	__ mov(to, rdx); // restore 'to'
duke@435	1158	__ jmpb(L_copy_2_bytes); // all dword were copied
duke@435	1159	} else {
duke@435	1160	// Align to 8 bytes the end of array. It is aligned to 4 bytes already.
never@739	1161	__ testptr(end, 4);
duke@435	1162	__ jccb(Assembler::zero, L_copy_8_bytes);
duke@435	1163	__ subl(count, 1<<shift);
duke@435	1164	__ movl(rdx, Address(from, count, sf, 0));
duke@435	1165	__ movl(Address(to, count, sf, 0), rdx);
duke@435	1166	__ jmpb(L_copy_8_bytes);
duke@435	1167
kvn@1800	1168	__ align(OptoLoopAlignment);
duke@435	1169	// Move 8 bytes
duke@435	1170	__ BIND(L_copy_8_bytes_loop);
kvn@840	1171	if (UseXMMForArrayCopy) {
kvn@840	1172	__ movq(xmm0, Address(from, count, sf, 0));
kvn@840	1173	__ movq(Address(to, count, sf, 0), xmm0);
kvn@840	1174	} else {
kvn@840	1175	__ movq(mmx0, Address(from, count, sf, 0));
kvn@840	1176	__ movq(Address(to, count, sf, 0), mmx0);
kvn@840	1177	}
duke@435	1178	__ BIND(L_copy_8_bytes);
duke@435	1179	__ subl(count, 2<<shift);
duke@435	1180	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1181	__ addl(count, 2<<shift);
kvn@840	1182	if (!UseXMMForArrayCopy) {
kvn@840	1183	__ emms();
kvn@840	1184	}
duke@435	1185	}
duke@435	1186	__ BIND(L_copy_4_bytes);
duke@435	1187	// copy prefix qword
duke@435	1188	__ testl(count, 1<<shift);
duke@435	1189	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1190	__ movl(rdx, Address(from, count, sf, -4));
duke@435	1191	__ movl(Address(to, count, sf, -4), rdx);
duke@435	1192
duke@435	1193	if (t == T_BYTE || t == T_SHORT) {
duke@435	1194	__ subl(count, (1<<shift));
duke@435	1195	__ BIND(L_copy_2_bytes);
duke@435	1196	// copy prefix dword
duke@435	1197	__ testl(count, 1<<(shift-1));
duke@435	1198	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1199	__ movw(rdx, Address(from, count, sf, -2));
duke@435	1200	__ movw(Address(to, count, sf, -2), rdx);
duke@435	1201	if (t == T_BYTE) {
duke@435	1202	__ subl(count, 1<<(shift-1));
duke@435	1203	__ BIND(L_copy_byte);
duke@435	1204	// copy prefix byte
duke@435	1205	__ testl(count, 1);
duke@435	1206	__ jccb(Assembler::zero, L_exit);
duke@435	1207	__ movb(rdx, Address(from, 0));
duke@435	1208	__ movb(Address(to, 0), rdx);
duke@435	1209	__ BIND(L_exit);
duke@435	1210	} else {
duke@435	1211	__ BIND(L_copy_byte);
duke@435	1212	}
duke@435	1213	} else {
duke@435	1214	__ BIND(L_copy_2_bytes);
duke@435	1215	}
duke@435	1216	if (t == T_OBJECT) {
never@739	1217	__ movl2ptr(count, Address(rsp, 12+12)); // reread count
duke@435	1218	gen_write_ref_array_post_barrier(to, count);
duke@435	1219	__ BIND(L_0_count);
duke@435	1220	}
duke@435	1221	inc_copy_counter_np(t);
never@739	1222	__ pop(rdi);
never@739	1223	__ pop(rsi);
duke@435	1224	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1225	__ xorptr(rax, rax); // return 0
duke@435	1226	__ ret(0);
duke@435	1227	return start;
duke@435	1228	}
duke@435	1229
duke@435	1230
duke@435	1231	address generate_disjoint_long_copy(address* entry, const char *name) {
duke@435	1232	__ align(CodeEntryAlignment);
duke@435	1233	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1234	address start = __ pc();
duke@435	1235
duke@435	1236	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1237	const Register from = rax; // source array address
duke@435	1238	const Register to = rdx; // destination array address
duke@435	1239	const Register count = rcx; // elements count
duke@435	1240	const Register to_from = rdx; // (to - from)
duke@435	1241
duke@435	1242	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1243	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1244	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1245	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1246
duke@435	1247	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	1248	BLOCK_COMMENT("Entry:");
duke@435	1249
never@739	1250	__ subptr(to, from); // to --> to_from
duke@435	1251	if (VM_Version::supports_mmx()) {
kvn@840	1252	if (UseXMMForArrayCopy) {
kvn@840	1253	xmm_copy_forward(from, to_from, count);
kvn@840	1254	} else {
kvn@840	1255	mmx_copy_forward(from, to_from, count);
kvn@840	1256	}
duke@435	1257	} else {
duke@435	1258	__ jmpb(L_copy_8_bytes);
kvn@1800	1259	__ align(OptoLoopAlignment);
duke@435	1260	__ BIND(L_copy_8_bytes_loop);
duke@435	1261	__ fild_d(Address(from, 0));
duke@435	1262	__ fistp_d(Address(from, to_from, Address::times_1));
never@739	1263	__ addptr(from, 8);
duke@435	1264	__ BIND(L_copy_8_bytes);
duke@435	1265	__ decrement(count);
duke@435	1266	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1267	}
duke@435	1268	inc_copy_counter_np(T_LONG);
duke@435	1269	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1270	__ xorptr(rax, rax); // return 0
duke@435	1271	__ ret(0);
duke@435	1272	return start;
duke@435	1273	}
duke@435	1274
duke@435	1275	address generate_conjoint_long_copy(address nooverlap_target,
duke@435	1276	address* entry, const char *name) {
duke@435	1277	__ align(CodeEntryAlignment);
duke@435	1278	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1279	address start = __ pc();
duke@435	1280
duke@435	1281	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1282	const Register from = rax; // source array address
duke@435	1283	const Register to = rdx; // destination array address
duke@435	1284	const Register count = rcx; // elements count
duke@435	1285	const Register end_from = rax; // source array end address
duke@435	1286
duke@435	1287	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1288	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1289	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1290	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1291
duke@435	1292	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1293	BLOCK_COMMENT("Entry:");
duke@435	1294
duke@435	1295	// arrays overlap test
never@739	1296	__ cmpptr(to, from);
duke@435	1297	RuntimeAddress nooverlap(nooverlap_target);
duke@435	1298	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1299	__ lea(end_from, Address(from, count, Address::times_8, 0));
never@739	1300	__ cmpptr(to, end_from);
never@739	1301	__ movptr(from, Address(rsp, 8)); // from
duke@435	1302	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1303
duke@435	1304	__ jmpb(L_copy_8_bytes);
duke@435	1305
kvn@1800	1306	__ align(OptoLoopAlignment);
duke@435	1307	__ BIND(L_copy_8_bytes_loop);
duke@435	1308	if (VM_Version::supports_mmx()) {
kvn@840	1309	if (UseXMMForArrayCopy) {
kvn@840	1310	__ movq(xmm0, Address(from, count, Address::times_8));
kvn@840	1311	__ movq(Address(to, count, Address::times_8), xmm0);
kvn@840	1312	} else {
kvn@840	1313	__ movq(mmx0, Address(from, count, Address::times_8));
kvn@840	1314	__ movq(Address(to, count, Address::times_8), mmx0);
kvn@840	1315	}
duke@435	1316	} else {
duke@435	1317	__ fild_d(Address(from, count, Address::times_8));
duke@435	1318	__ fistp_d(Address(to, count, Address::times_8));
duke@435	1319	}
duke@435	1320	__ BIND(L_copy_8_bytes);
duke@435	1321	__ decrement(count);
duke@435	1322	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1323
kvn@840	1324	if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
duke@435	1325	__ emms();
duke@435	1326	}
duke@435	1327	inc_copy_counter_np(T_LONG);
duke@435	1328	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1329	__ xorptr(rax, rax); // return 0
duke@435	1330	__ ret(0);
duke@435	1331	return start;
duke@435	1332	}
duke@435	1333
duke@435	1334
duke@435	1335	// Helper for generating a dynamic type check.
duke@435	1336	// The sub_klass must be one of {rbx, rdx, rsi}.
duke@435	1337	// The temp is killed.
duke@435	1338	void generate_type_check(Register sub_klass,
duke@435	1339	Address& super_check_offset_addr,
duke@435	1340	Address& super_klass_addr,
duke@435	1341	Register temp,
jrose@1079	1342	Label* L_success, Label* L_failure) {
duke@435	1343	BLOCK_COMMENT("type_check:");
duke@435	1344
duke@435	1345	Label L_fallthrough;
jrose@1079	1346	#define LOCAL_JCC(assembler_con, label_ptr) \
jrose@1079	1347	if (label_ptr != NULL) __ jcc(assembler_con, *(label_ptr)); \
jrose@1079	1348	else __ jcc(assembler_con, L_fallthrough) /*omit semi*/
duke@435	1349
jrose@1079	1350	// The following is a strange variation of the fast path which requires
jrose@1079	1351	// one less register, because needed values are on the argument stack.
jrose@1079	1352	// __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
jrose@1079	1353	// L_success, L_failure, NULL);
duke@435	1354	assert_different_registers(sub_klass, temp);
duke@435	1355
duke@435	1356	int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1357	Klass::secondary_super_cache_offset_in_bytes());
duke@435	1358
duke@435	1359	// if the pointers are equal, we are done (e.g., String[] elements)
never@739	1360	__ cmpptr(sub_klass, super_klass_addr);
jrose@1079	1361	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1362
duke@435	1363	// check the supertype display:
never@739	1364	__ movl2ptr(temp, super_check_offset_addr);
duke@435	1365	Address super_check_addr(sub_klass, temp, Address::times_1, 0);
never@739	1366	__ movptr(temp, super_check_addr); // load displayed supertype
never@739	1367	__ cmpptr(temp, super_klass_addr); // test the super type
jrose@1079	1368	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1369
duke@435	1370	// if it was a primary super, we can just fail immediately
duke@435	1371	__ cmpl(super_check_offset_addr, sc_offset);
jrose@1079	1372	LOCAL_JCC(Assembler::notEqual, L_failure);
duke@435	1373
jrose@1079	1374	// The repne_scan instruction uses fixed registers, which will get spilled.
jrose@1079	1375	// We happen to know this works best when super_klass is in rax.
jrose@1079	1376	Register super_klass = temp;
jrose@1079	1377	__ movptr(super_klass, super_klass_addr);
jrose@1079	1378	__ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
jrose@1079	1379	L_success, L_failure);
duke@435	1380
jrose@1079	1381	__ bind(L_fallthrough);
duke@435	1382
jrose@1079	1383	if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
jrose@1079	1384	if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
duke@435	1385
jrose@1079	1386	#undef LOCAL_JCC
duke@435	1387	}
duke@435	1388
duke@435	1389	//
duke@435	1390	// Generate checkcasting array copy stub
duke@435	1391	//
duke@435	1392	// Input:
duke@435	1393	// 4(rsp) - source array address
duke@435	1394	// 8(rsp) - destination array address
duke@435	1395	// 12(rsp) - element count, can be zero
duke@435	1396	// 16(rsp) - size_t ckoff (super_check_offset)
duke@435	1397	// 20(rsp) - oop ckval (super_klass)
duke@435	1398	//
duke@435	1399	// Output:
duke@435	1400	// rax, == 0 - success
duke@435	1401	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1402	//
duke@435	1403	address generate_checkcast_copy(const char *name, address* entry) {
duke@435	1404	__ align(CodeEntryAlignment);
duke@435	1405	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1406	address start = __ pc();
duke@435	1407
duke@435	1408	Label L_load_element, L_store_element, L_do_card_marks, L_done;
duke@435	1409
duke@435	1410	// register use:
duke@435	1411	// rax, rdx, rcx -- loop control (end_from, end_to, count)
duke@435	1412	// rdi, rsi -- element access (oop, klass)
duke@435	1413	// rbx, -- temp
duke@435	1414	const Register from = rax; // source array address
duke@435	1415	const Register to = rdx; // destination array address
duke@435	1416	const Register length = rcx; // elements count
duke@435	1417	const Register elem = rdi; // each oop copied
duke@435	1418	const Register elem_klass = rsi; // each elem._klass (sub_klass)
duke@435	1419	const Register temp = rbx; // lone remaining temp
duke@435	1420
duke@435	1421	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1422
never@739	1423	__ push(rsi);
never@739	1424	__ push(rdi);
never@739	1425	__ push(rbx);
duke@435	1426
duke@435	1427	Address from_arg(rsp, 16+ 4); // from
duke@435	1428	Address to_arg(rsp, 16+ 8); // to
duke@435	1429	Address length_arg(rsp, 16+12); // elements count
duke@435	1430	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1431	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1432
duke@435	1433	// Load up:
never@739	1434	__ movptr(from, from_arg);
never@739	1435	__ movptr(to, to_arg);
never@739	1436	__ movl2ptr(length, length_arg);
duke@435	1437
duke@435	1438	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1439	BLOCK_COMMENT("Entry:");
duke@435	1440
duke@435	1441	//---------------------------------------------------------------
duke@435	1442	// Assembler stub will be used for this call to arraycopy
duke@435	1443	// if the two arrays are subtypes of Object[] but the
duke@435	1444	// destination array type is not equal to or a supertype
duke@435	1445	// of the source type. Each element must be separately
duke@435	1446	// checked.
duke@435	1447
duke@435	1448	// Loop-invariant addresses. They are exclusive end pointers.
never@739	1449	Address end_from_addr(from, length, Address::times_ptr, 0);
never@739	1450	Address end_to_addr(to, length, Address::times_ptr, 0);
duke@435	1451
duke@435	1452	Register end_from = from; // re-use
duke@435	1453	Register end_to = to; // re-use
duke@435	1454	Register count = length; // re-use
duke@435	1455
duke@435	1456	// Loop-variant addresses. They assume post-incremented count < 0.
never@739	1457	Address from_element_addr(end_from, count, Address::times_ptr, 0);
never@739	1458	Address to_element_addr(end_to, count, Address::times_ptr, 0);
duke@435	1459	Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
duke@435	1460
duke@435	1461	// Copy from low to high addresses, indexed from the end of each array.
ysr@777	1462	gen_write_ref_array_pre_barrier(to, count);
never@739	1463	__ lea(end_from, end_from_addr);
never@739	1464	__ lea(end_to, end_to_addr);
duke@435	1465	assert(length == count, ""); // else fix next line:
never@739	1466	__ negptr(count); // negate and test the length
duke@435	1467	__ jccb(Assembler::notZero, L_load_element);
duke@435	1468
duke@435	1469	// Empty array: Nothing to do.
never@739	1470	__ xorptr(rax, rax); // return 0 on (trivial) success
duke@435	1471	__ jmp(L_done);
duke@435	1472
duke@435	1473	// ======== begin loop ========
duke@435	1474	// (Loop is rotated; its entry is L_load_element.)
duke@435	1475	// Loop control:
duke@435	1476	// for (count = -count; count != 0; count++)
duke@435	1477	// Base pointers src, dst are biased by 8*count,to last element.
kvn@1800	1478	__ align(OptoLoopAlignment);
duke@435	1479
duke@435	1480	__ BIND(L_store_element);
never@739	1481	__ movptr(to_element_addr, elem); // store the oop
duke@435	1482	__ increment(count); // increment the count toward zero
duke@435	1483	__ jccb(Assembler::zero, L_do_card_marks);
duke@435	1484
duke@435	1485	// ======== loop entry is here ========
duke@435	1486	__ BIND(L_load_element);
never@739	1487	__ movptr(elem, from_element_addr); // load the oop
never@739	1488	__ testptr(elem, elem);
duke@435	1489	__ jccb(Assembler::zero, L_store_element);
duke@435	1490
duke@435	1491	// (Could do a trick here: Remember last successful non-null
duke@435	1492	// element stored and make a quick oop equality check on it.)
duke@435	1493
never@739	1494	__ movptr(elem_klass, elem_klass_addr); // query the object klass
duke@435	1495	generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
duke@435	1496	&L_store_element, NULL);
duke@435	1497	// (On fall-through, we have failed the element type check.)
duke@435	1498	// ======== end loop ========
duke@435	1499
duke@435	1500	// It was a real error; we must depend on the caller to finish the job.
rasbold@454	1501	// Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
rasbold@454	1502	// Emit GC store barriers for the oops we have copied (length_arg + count),
duke@435	1503	// and report their number to the caller.
duke@435	1504	__ addl(count, length_arg); // transfers = (length - remaining)
never@739	1505	__ movl2ptr(rax, count); // save the value
never@739	1506	__ notptr(rax); // report (-1^K) to caller
never@739	1507	__ movptr(to, to_arg); // reload
duke@435	1508	assert_different_registers(to, count, rax);
duke@435	1509	gen_write_ref_array_post_barrier(to, count);
duke@435	1510	__ jmpb(L_done);
duke@435	1511
duke@435	1512	// Come here on success only.
duke@435	1513	__ BIND(L_do_card_marks);
never@739	1514	__ movl2ptr(count, length_arg);
never@739	1515	__ movptr(to, to_arg); // reload
duke@435	1516	gen_write_ref_array_post_barrier(to, count);
never@739	1517	__ xorptr(rax, rax); // return 0 on success
duke@435	1518
duke@435	1519	// Common exit point (success or failure).
duke@435	1520	__ BIND(L_done);
never@739	1521	__ pop(rbx);
never@739	1522	__ pop(rdi);
never@739	1523	__ pop(rsi);
duke@435	1524	inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
duke@435	1525	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1526	__ ret(0);
duke@435	1527
duke@435	1528	return start;
duke@435	1529	}
duke@435	1530
duke@435	1531	//
duke@435	1532	// Generate 'unsafe' array copy stub
duke@435	1533	// Though just as safe as the other stubs, it takes an unscaled
duke@435	1534	// size_t argument instead of an element count.
duke@435	1535	//
duke@435	1536	// Input:
duke@435	1537	// 4(rsp) - source array address
duke@435	1538	// 8(rsp) - destination array address
duke@435	1539	// 12(rsp) - byte count, can be zero
duke@435	1540	//
duke@435	1541	// Output:
duke@435	1542	// rax, == 0 - success
duke@435	1543	// rax, == -1 - need to call System.arraycopy
duke@435	1544	//
duke@435	1545	// Examines the alignment of the operands and dispatches
duke@435	1546	// to a long, int, short, or byte copy loop.
duke@435	1547	//
duke@435	1548	address generate_unsafe_copy(const char *name,
duke@435	1549	address byte_copy_entry,
duke@435	1550	address short_copy_entry,
duke@435	1551	address int_copy_entry,
duke@435	1552	address long_copy_entry) {
duke@435	1553
duke@435	1554	Label L_long_aligned, L_int_aligned, L_short_aligned;
duke@435	1555
duke@435	1556	__ align(CodeEntryAlignment);
duke@435	1557	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1558	address start = __ pc();
duke@435	1559
duke@435	1560	const Register from = rax; // source array address
duke@435	1561	const Register to = rdx; // destination array address
duke@435	1562	const Register count = rcx; // elements count
duke@435	1563
duke@435	1564	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1565	__ push(rsi);
never@739	1566	__ push(rdi);
duke@435	1567	Address from_arg(rsp, 12+ 4); // from
duke@435	1568	Address to_arg(rsp, 12+ 8); // to
duke@435	1569	Address count_arg(rsp, 12+12); // byte count
duke@435	1570
duke@435	1571	// Load up:
never@739	1572	__ movptr(from , from_arg);
never@739	1573	__ movptr(to , to_arg);
never@739	1574	__ movl2ptr(count, count_arg);
duke@435	1575
duke@435	1576	// bump this on entry, not on exit:
duke@435	1577	inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
duke@435	1578
duke@435	1579	const Register bits = rsi;
never@739	1580	__ mov(bits, from);
never@739	1581	__ orptr(bits, to);
never@739	1582	__ orptr(bits, count);
duke@435	1583
duke@435	1584	__ testl(bits, BytesPerLong-1);
duke@435	1585	__ jccb(Assembler::zero, L_long_aligned);
duke@435	1586
duke@435	1587	__ testl(bits, BytesPerInt-1);
duke@435	1588	__ jccb(Assembler::zero, L_int_aligned);
duke@435	1589
duke@435	1590	__ testl(bits, BytesPerShort-1);
duke@435	1591	__ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
duke@435	1592
duke@435	1593	__ BIND(L_short_aligned);
never@739	1594	__ shrptr(count, LogBytesPerShort); // size => short_count
duke@435	1595	__ movl(count_arg, count); // update 'count'
duke@435	1596	__ jump(RuntimeAddress(short_copy_entry));
duke@435	1597
duke@435	1598	__ BIND(L_int_aligned);
never@739	1599	__ shrptr(count, LogBytesPerInt); // size => int_count
duke@435	1600	__ movl(count_arg, count); // update 'count'
duke@435	1601	__ jump(RuntimeAddress(int_copy_entry));
duke@435	1602
duke@435	1603	__ BIND(L_long_aligned);
never@739	1604	__ shrptr(count, LogBytesPerLong); // size => qword_count
duke@435	1605	__ movl(count_arg, count); // update 'count'
never@739	1606	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1607	__ pop(rsi);
duke@435	1608	__ jump(RuntimeAddress(long_copy_entry));
duke@435	1609
duke@435	1610	return start;
duke@435	1611	}
duke@435	1612
duke@435	1613
duke@435	1614	// Perform range checks on the proposed arraycopy.
duke@435	1615	// Smashes src_pos and dst_pos. (Uses them up for temps.)
duke@435	1616	void arraycopy_range_checks(Register src,
duke@435	1617	Register src_pos,
duke@435	1618	Register dst,
duke@435	1619	Register dst_pos,
duke@435	1620	Address& length,
duke@435	1621	Label& L_failed) {
duke@435	1622	BLOCK_COMMENT("arraycopy_range_checks:");
duke@435	1623	const Register src_end = src_pos; // source array end position
duke@435	1624	const Register dst_end = dst_pos; // destination array end position
duke@435	1625	__ addl(src_end, length); // src_pos + length
duke@435	1626	__ addl(dst_end, length); // dst_pos + length
duke@435	1627
duke@435	1628	// if (src_pos + length > arrayOop(src)->length() ) FAIL;
duke@435	1629	__ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
duke@435	1630	__ jcc(Assembler::above, L_failed);
duke@435	1631
duke@435	1632	// if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
duke@435	1633	__ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
duke@435	1634	__ jcc(Assembler::above, L_failed);
duke@435	1635
duke@435	1636	BLOCK_COMMENT("arraycopy_range_checks done");
duke@435	1637	}
duke@435	1638
duke@435	1639
duke@435	1640	//
duke@435	1641	// Generate generic array copy stubs
duke@435	1642	//
duke@435	1643	// Input:
duke@435	1644	// 4(rsp) - src oop
duke@435	1645	// 8(rsp) - src_pos
duke@435	1646	// 12(rsp) - dst oop
duke@435	1647	// 16(rsp) - dst_pos
duke@435	1648	// 20(rsp) - element count
duke@435	1649	//
duke@435	1650	// Output:
duke@435	1651	// rax, == 0 - success
duke@435	1652	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1653	//
duke@435	1654	address generate_generic_copy(const char *name,
duke@435	1655	address entry_jbyte_arraycopy,
duke@435	1656	address entry_jshort_arraycopy,
duke@435	1657	address entry_jint_arraycopy,
duke@435	1658	address entry_oop_arraycopy,
duke@435	1659	address entry_jlong_arraycopy,
duke@435	1660	address entry_checkcast_arraycopy) {
duke@435	1661	Label L_failed, L_failed_0, L_objArray;
duke@435	1662
duke@435	1663	{ int modulus = CodeEntryAlignment;
duke@435	1664	int target = modulus - 5; // 5 = sizeof jmp(L_failed)
duke@435	1665	int advance = target - (__ offset() % modulus);
duke@435	1666	if (advance < 0) advance += modulus;
duke@435	1667	if (advance > 0) __ nop(advance);
duke@435	1668	}
duke@435	1669	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1670
duke@435	1671	// Short-hop target to L_failed. Makes for denser prologue code.
duke@435	1672	__ BIND(L_failed_0);
duke@435	1673	__ jmp(L_failed);
duke@435	1674	assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
duke@435	1675
duke@435	1676	__ align(CodeEntryAlignment);
duke@435	1677	address start = __ pc();
duke@435	1678
duke@435	1679	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1680	__ push(rsi);
never@739	1681	__ push(rdi);
duke@435	1682
duke@435	1683	// bump this on entry, not on exit:
duke@435	1684	inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
duke@435	1685
duke@435	1686	// Input values
duke@435	1687	Address SRC (rsp, 12+ 4);
duke@435	1688	Address SRC_POS (rsp, 12+ 8);
duke@435	1689	Address DST (rsp, 12+12);
duke@435	1690	Address DST_POS (rsp, 12+16);
duke@435	1691	Address LENGTH (rsp, 12+20);
duke@435	1692
duke@435	1693	//-----------------------------------------------------------------------
duke@435	1694	// Assembler stub will be used for this call to arraycopy
duke@435	1695	// if the following conditions are met:
duke@435	1696	//
duke@435	1697	// (1) src and dst must not be null.
duke@435	1698	// (2) src_pos must not be negative.
duke@435	1699	// (3) dst_pos must not be negative.
duke@435	1700	// (4) length must not be negative.
duke@435	1701	// (5) src klass and dst klass should be the same and not NULL.
duke@435	1702	// (6) src and dst should be arrays.
duke@435	1703	// (7) src_pos + length must not exceed length of src.
duke@435	1704	// (8) dst_pos + length must not exceed length of dst.
duke@435	1705	//
duke@435	1706
duke@435	1707	const Register src = rax; // source array oop
duke@435	1708	const Register src_pos = rsi;
duke@435	1709	const Register dst = rdx; // destination array oop
duke@435	1710	const Register dst_pos = rdi;
duke@435	1711	const Register length = rcx; // transfer count
duke@435	1712
duke@435	1713	// if (src == NULL) return -1;
never@739	1714	__ movptr(src, SRC); // src oop
never@739	1715	__ testptr(src, src);
duke@435	1716	__ jccb(Assembler::zero, L_failed_0);
duke@435	1717
duke@435	1718	// if (src_pos < 0) return -1;
never@739	1719	__ movl2ptr(src_pos, SRC_POS); // src_pos
duke@435	1720	__ testl(src_pos, src_pos);
duke@435	1721	__ jccb(Assembler::negative, L_failed_0);
duke@435	1722
duke@435	1723	// if (dst == NULL) return -1;
never@739	1724	__ movptr(dst, DST); // dst oop
never@739	1725	__ testptr(dst, dst);
duke@435	1726	__ jccb(Assembler::zero, L_failed_0);
duke@435	1727
duke@435	1728	// if (dst_pos < 0) return -1;
never@739	1729	__ movl2ptr(dst_pos, DST_POS); // dst_pos
duke@435	1730	__ testl(dst_pos, dst_pos);
duke@435	1731	__ jccb(Assembler::negative, L_failed_0);
duke@435	1732
duke@435	1733	// if (length < 0) return -1;
never@739	1734	__ movl2ptr(length, LENGTH); // length
duke@435	1735	__ testl(length, length);
duke@435	1736	__ jccb(Assembler::negative, L_failed_0);
duke@435	1737
duke@435	1738	// if (src->klass() == NULL) return -1;
duke@435	1739	Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
duke@435	1740	Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
duke@435	1741	const Register rcx_src_klass = rcx; // array klass
never@739	1742	__ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
duke@435	1743
duke@435	1744	#ifdef ASSERT
duke@435	1745	// assert(src->klass() != NULL);
duke@435	1746	BLOCK_COMMENT("assert klasses not null");
duke@435	1747	{ Label L1, L2;
never@739	1748	__ testptr(rcx_src_klass, rcx_src_klass);
duke@435	1749	__ jccb(Assembler::notZero, L2); // it is broken if klass is NULL
duke@435	1750	__ bind(L1);
duke@435	1751	__ stop("broken null klass");
duke@435	1752	__ bind(L2);
never@739	1753	__ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
duke@435	1754	__ jccb(Assembler::equal, L1); // this would be broken also
duke@435	1755	BLOCK_COMMENT("assert done");
duke@435	1756	}
duke@435	1757	#endif //ASSERT
duke@435	1758
duke@435	1759	// Load layout helper (32-bits)
duke@435	1760	//
duke@435	1761	// |array_tag| | header_size | element_type | |log2_element_size|
duke@435	1762	// 32 30 24 16 8 2 0
duke@435	1763	//
duke@435	1764	// array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
duke@435	1765	//
duke@435	1766
duke@435	1767	int lh_offset = klassOopDesc::header_size() * HeapWordSize +
duke@435	1768	Klass::layout_helper_offset_in_bytes();
duke@435	1769	Address src_klass_lh_addr(rcx_src_klass, lh_offset);
duke@435	1770
duke@435	1771	// Handle objArrays completely differently...
duke@435	1772	jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
duke@435	1773	__ cmpl(src_klass_lh_addr, objArray_lh);
duke@435	1774	__ jcc(Assembler::equal, L_objArray);
duke@435	1775
duke@435	1776	// if (src->klass() != dst->klass()) return -1;
never@739	1777	__ cmpptr(rcx_src_klass, dst_klass_addr);
duke@435	1778	__ jccb(Assembler::notEqual, L_failed_0);
duke@435	1779
duke@435	1780	const Register rcx_lh = rcx; // layout helper
duke@435	1781	assert(rcx_lh == rcx_src_klass, "known alias");
duke@435	1782	__ movl(rcx_lh, src_klass_lh_addr);
duke@435	1783
duke@435	1784	// if (!src->is_Array()) return -1;
duke@435	1785	__ cmpl(rcx_lh, Klass::_lh_neutral_value);
duke@435	1786	__ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
duke@435	1787
duke@435	1788	// At this point, it is known to be a typeArray (array_tag 0x3).
duke@435	1789	#ifdef ASSERT
duke@435	1790	{ Label L;
duke@435	1791	__ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
duke@435	1792	__ jcc(Assembler::greaterEqual, L); // signed cmp
duke@435	1793	__ stop("must be a primitive array");
duke@435	1794	__ bind(L);
duke@435	1795	}
duke@435	1796	#endif
duke@435	1797
duke@435	1798	assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
duke@435	1799	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1800
duke@435	1801	// typeArrayKlass
duke@435	1802	//
duke@435	1803	// src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
duke@435	1804	// dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
duke@435	1805	//
duke@435	1806	const Register rsi_offset = rsi; // array offset
duke@435	1807	const Register src_array = src; // src array offset
duke@435	1808	const Register dst_array = dst; // dst array offset
duke@435	1809	const Register rdi_elsize = rdi; // log2 element size
duke@435	1810
never@739	1811	__ mov(rsi_offset, rcx_lh);
never@739	1812	__ shrptr(rsi_offset, Klass::_lh_header_size_shift);
never@739	1813	__ andptr(rsi_offset, Klass::_lh_header_size_mask); // array_offset
never@739	1814	__ addptr(src_array, rsi_offset); // src array offset
never@739	1815	__ addptr(dst_array, rsi_offset); // dst array offset
never@739	1816	__ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
duke@435	1817
duke@435	1818	// next registers should be set before the jump to corresponding stub
duke@435	1819	const Register from = src; // source array address
duke@435	1820	const Register to = dst; // destination array address
duke@435	1821	const Register count = rcx; // elements count
duke@435	1822	// some of them should be duplicated on stack
duke@435	1823	#define FROM Address(rsp, 12+ 4)
duke@435	1824	#define TO Address(rsp, 12+ 8) // Not used now
duke@435	1825	#define COUNT Address(rsp, 12+12) // Only for oop arraycopy
duke@435	1826
duke@435	1827	BLOCK_COMMENT("scale indexes to element size");
never@739	1828	__ movl2ptr(rsi, SRC_POS); // src_pos
never@739	1829	__ shlptr(rsi); // src_pos << rcx (log2 elsize)
duke@435	1830	assert(src_array == from, "");
never@739	1831	__ addptr(from, rsi); // from = src_array + SRC_POS << log2 elsize
never@739	1832	__ movl2ptr(rdi, DST_POS); // dst_pos
never@739	1833	__ shlptr(rdi); // dst_pos << rcx (log2 elsize)
duke@435	1834	assert(dst_array == to, "");
never@739	1835	__ addptr(to, rdi); // to = dst_array + DST_POS << log2 elsize
never@739	1836	__ movptr(FROM, from); // src_addr
never@739	1837	__ mov(rdi_elsize, rcx_lh); // log2 elsize
never@739	1838	__ movl2ptr(count, LENGTH); // elements count
duke@435	1839
duke@435	1840	BLOCK_COMMENT("choose copy loop based on element size");
duke@435	1841	__ cmpl(rdi_elsize, 0);
duke@435	1842
duke@435	1843	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
duke@435	1844	__ cmpl(rdi_elsize, LogBytesPerShort);
duke@435	1845	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
duke@435	1846	__ cmpl(rdi_elsize, LogBytesPerInt);
duke@435	1847	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
duke@435	1848	#ifdef ASSERT
duke@435	1849	__ cmpl(rdi_elsize, LogBytesPerLong);
duke@435	1850	__ jccb(Assembler::notEqual, L_failed);
duke@435	1851	#endif
never@739	1852	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1853	__ pop(rsi);
duke@435	1854	__ jump(RuntimeAddress(entry_jlong_arraycopy));
duke@435	1855
duke@435	1856	__ BIND(L_failed);
never@739	1857	__ xorptr(rax, rax);
never@739	1858	__ notptr(rax); // return -1
never@739	1859	__ pop(rdi);
never@739	1860	__ pop(rsi);
duke@435	1861	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1862	__ ret(0);
duke@435	1863
duke@435	1864	// objArrayKlass
duke@435	1865	__ BIND(L_objArray);
duke@435	1866	// live at this point: rcx_src_klass, src[_pos], dst[_pos]
duke@435	1867
duke@435	1868	Label L_plain_copy, L_checkcast_copy;
duke@435	1869	// test array classes for subtyping
never@739	1870	__ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
duke@435	1871	__ jccb(Assembler::notEqual, L_checkcast_copy);
duke@435	1872
duke@435	1873	// Identically typed arrays can be copied without element-wise checks.
duke@435	1874	assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
duke@435	1875	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1876
duke@435	1877	__ BIND(L_plain_copy);
never@739	1878	__ movl2ptr(count, LENGTH); // elements count
never@739	1879	__ movl2ptr(src_pos, SRC_POS); // reload src_pos
never@739	1880	__ lea(from, Address(src, src_pos, Address::times_ptr,
never@739	1881	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
never@739	1882	__ movl2ptr(dst_pos, DST_POS); // reload dst_pos
never@739	1883	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
never@739	1884	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
never@739	1885	__ movptr(FROM, from); // src_addr
never@739	1886	__ movptr(TO, to); // dst_addr
duke@435	1887	__ movl(COUNT, count); // count
duke@435	1888	__ jump(RuntimeAddress(entry_oop_arraycopy));
duke@435	1889
duke@435	1890	__ BIND(L_checkcast_copy);
duke@435	1891	// live at this point: rcx_src_klass, dst[_pos], src[_pos]
duke@435	1892	{
duke@435	1893	// Handy offsets:
duke@435	1894	int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1895	objArrayKlass::element_klass_offset_in_bytes());
duke@435	1896	int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1897	Klass::super_check_offset_offset_in_bytes());
duke@435	1898
duke@435	1899	Register rsi_dst_klass = rsi;
duke@435	1900	Register rdi_temp = rdi;
duke@435	1901	assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
duke@435	1902	assert(rdi_temp == dst_pos, "expected alias w/ dst_pos");
duke@435	1903	Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
duke@435	1904
duke@435	1905	// Before looking at dst.length, make sure dst is also an objArray.
never@739	1906	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1907	__ cmpl(dst_klass_lh_addr, objArray_lh);
duke@435	1908	__ jccb(Assembler::notEqual, L_failed);
duke@435	1909
duke@435	1910	// It is safe to examine both src.length and dst.length.
never@739	1911	__ movl2ptr(src_pos, SRC_POS); // reload rsi
duke@435	1912	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1913	// (Now src_pos and dst_pos are killed, but not src and dst.)
duke@435	1914
duke@435	1915	// We'll need this temp (don't forget to pop it after the type check).
never@739	1916	__ push(rbx);
duke@435	1917	Register rbx_src_klass = rbx;
duke@435	1918
never@739	1919	__ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
never@739	1920	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1921	Address super_check_offset_addr(rsi_dst_klass, sco_offset);
duke@435	1922	Label L_fail_array_check;
duke@435	1923	generate_type_check(rbx_src_klass,
duke@435	1924	super_check_offset_addr, dst_klass_addr,
duke@435	1925	rdi_temp, NULL, &L_fail_array_check);
duke@435	1926	// (On fall-through, we have passed the array type check.)
never@739	1927	__ pop(rbx);
duke@435	1928	__ jmp(L_plain_copy);
duke@435	1929
duke@435	1930	__ BIND(L_fail_array_check);
duke@435	1931	// Reshuffle arguments so we can call checkcast_arraycopy:
duke@435	1932
duke@435	1933	// match initial saves for checkcast_arraycopy
never@739	1934	// push(rsi); // already done; see above
never@739	1935	// push(rdi); // already done; see above
never@739	1936	// push(rbx); // already done; see above
duke@435	1937
duke@435	1938	// Marshal outgoing arguments now, freeing registers.
duke@435	1939	Address from_arg(rsp, 16+ 4); // from
duke@435	1940	Address to_arg(rsp, 16+ 8); // to
duke@435	1941	Address length_arg(rsp, 16+12); // elements count
duke@435	1942	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1943	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1944
duke@435	1945	Address SRC_POS_arg(rsp, 16+ 8);
duke@435	1946	Address DST_POS_arg(rsp, 16+16);
duke@435	1947	Address LENGTH_arg(rsp, 16+20);
duke@435	1948	// push rbx, changed the incoming offsets (why not just use rbp,??)
duke@435	1949	// assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
duke@435	1950
never@739	1951	__ movptr(rbx, Address(rsi_dst_klass, ek_offset));
never@739	1952	__ movl2ptr(length, LENGTH_arg); // reload elements count
never@739	1953	__ movl2ptr(src_pos, SRC_POS_arg); // reload src_pos
never@739	1954	__ movl2ptr(dst_pos, DST_POS_arg); // reload dst_pos
duke@435	1955
never@739	1956	__ movptr(ckval_arg, rbx); // destination element type
duke@435	1957	__ movl(rbx, Address(rbx, sco_offset));
duke@435	1958	__ movl(ckoff_arg, rbx); // corresponding class check offset
duke@435	1959
duke@435	1960	__ movl(length_arg, length); // outgoing length argument
duke@435	1961
never@739	1962	__ lea(from, Address(src, src_pos, Address::times_ptr,
duke@435	1963	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1964	__ movptr(from_arg, from);
duke@435	1965
never@739	1966	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
duke@435	1967	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1968	__ movptr(to_arg, to);
duke@435	1969	__ jump(RuntimeAddress(entry_checkcast_arraycopy));
duke@435	1970	}
duke@435	1971
duke@435	1972	return start;
duke@435	1973	}
duke@435	1974
duke@435	1975	void generate_arraycopy_stubs() {
duke@435	1976	address entry;
duke@435	1977	address entry_jbyte_arraycopy;
duke@435	1978	address entry_jshort_arraycopy;
duke@435	1979	address entry_jint_arraycopy;
duke@435	1980	address entry_oop_arraycopy;
duke@435	1981	address entry_jlong_arraycopy;
duke@435	1982	address entry_checkcast_arraycopy;
duke@435	1983
duke@435	1984	StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
duke@435	1985	generate_disjoint_copy(T_BYTE, true, Address::times_1, &entry,
duke@435	1986	"arrayof_jbyte_disjoint_arraycopy");
duke@435	1987	StubRoutines::_arrayof_jbyte_arraycopy =
duke@435	1988	generate_conjoint_copy(T_BYTE, true, Address::times_1, entry,
duke@435	1989	NULL, "arrayof_jbyte_arraycopy");
duke@435	1990	StubRoutines::_jbyte_disjoint_arraycopy =
duke@435	1991	generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
duke@435	1992	"jbyte_disjoint_arraycopy");
duke@435	1993	StubRoutines::_jbyte_arraycopy =
duke@435	1994	generate_conjoint_copy(T_BYTE, false, Address::times_1, entry,
duke@435	1995	&entry_jbyte_arraycopy, "jbyte_arraycopy");
duke@435	1996
duke@435	1997	StubRoutines::_arrayof_jshort_disjoint_arraycopy =
duke@435	1998	generate_disjoint_copy(T_SHORT, true, Address::times_2, &entry,
duke@435	1999	"arrayof_jshort_disjoint_arraycopy");
duke@435	2000	StubRoutines::_arrayof_jshort_arraycopy =
duke@435	2001	generate_conjoint_copy(T_SHORT, true, Address::times_2, entry,
duke@435	2002	NULL, "arrayof_jshort_arraycopy");
duke@435	2003	StubRoutines::_jshort_disjoint_arraycopy =
duke@435	2004	generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
duke@435	2005	"jshort_disjoint_arraycopy");
duke@435	2006	StubRoutines::_jshort_arraycopy =
duke@435	2007	generate_conjoint_copy(T_SHORT, false, Address::times_2, entry,
duke@435	2008	&entry_jshort_arraycopy, "jshort_arraycopy");
duke@435	2009
duke@435	2010	// Next arrays are always aligned on 4 bytes at least.
duke@435	2011	StubRoutines::_jint_disjoint_arraycopy =
duke@435	2012	generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
duke@435	2013	"jint_disjoint_arraycopy");
duke@435	2014	StubRoutines::_jint_arraycopy =
duke@435	2015	generate_conjoint_copy(T_INT, true, Address::times_4, entry,
duke@435	2016	&entry_jint_arraycopy, "jint_arraycopy");
duke@435	2017
duke@435	2018	StubRoutines::_oop_disjoint_arraycopy =
never@739	2019	generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
duke@435	2020	"oop_disjoint_arraycopy");
duke@435	2021	StubRoutines::_oop_arraycopy =
never@739	2022	generate_conjoint_copy(T_OBJECT, true, Address::times_ptr, entry,
duke@435	2023	&entry_oop_arraycopy, "oop_arraycopy");
duke@435	2024
duke@435	2025	StubRoutines::_jlong_disjoint_arraycopy =
duke@435	2026	generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
duke@435	2027	StubRoutines::_jlong_arraycopy =
duke@435	2028	generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
duke@435	2029	"jlong_arraycopy");
duke@435	2030
never@2118	2031	StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
never@2118	2032	StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
never@2118	2033	StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
never@2118	2034	StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
never@2118	2035	StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
never@2118	2036	StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
never@2118	2037
duke@435	2038	StubRoutines::_arrayof_jint_disjoint_arraycopy =
duke@435	2039	StubRoutines::_jint_disjoint_arraycopy;
duke@435	2040	StubRoutines::_arrayof_oop_disjoint_arraycopy =
duke@435	2041	StubRoutines::_oop_disjoint_arraycopy;
duke@435	2042	StubRoutines::_arrayof_jlong_disjoint_arraycopy =
duke@435	2043	StubRoutines::_jlong_disjoint_arraycopy;
duke@435	2044
duke@435	2045	StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
duke@435	2046	StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
duke@435	2047	StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
duke@435	2048
duke@435	2049	StubRoutines::_checkcast_arraycopy =
duke@435	2050	generate_checkcast_copy("checkcast_arraycopy",
duke@435	2051	&entry_checkcast_arraycopy);
duke@435	2052
duke@435	2053	StubRoutines::_unsafe_arraycopy =
duke@435	2054	generate_unsafe_copy("unsafe_arraycopy",
duke@435	2055	entry_jbyte_arraycopy,
duke@435	2056	entry_jshort_arraycopy,
duke@435	2057	entry_jint_arraycopy,
duke@435	2058	entry_jlong_arraycopy);
duke@435	2059
duke@435	2060	StubRoutines::_generic_arraycopy =
duke@435	2061	generate_generic_copy("generic_arraycopy",
duke@435	2062	entry_jbyte_arraycopy,
duke@435	2063	entry_jshort_arraycopy,
duke@435	2064	entry_jint_arraycopy,
duke@435	2065	entry_oop_arraycopy,
duke@435	2066	entry_jlong_arraycopy,
duke@435	2067	entry_checkcast_arraycopy);
duke@435	2068	}
duke@435	2069
never@1609	2070	void generate_math_stubs() {
never@1609	2071	{
never@1609	2072	StubCodeMark mark(this, "StubRoutines", "log");
never@1609	2073	StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
never@1609	2074
never@1609	2075	__ fld_d(Address(rsp, 4));
never@1609	2076	__ flog();
never@1609	2077	__ ret(0);
never@1609	2078	}
never@1609	2079	{
never@1609	2080	StubCodeMark mark(this, "StubRoutines", "log10");
never@1609	2081	StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
never@1609	2082
never@1609	2083	__ fld_d(Address(rsp, 4));
never@1609	2084	__ flog10();
never@1609	2085	__ ret(0);
never@1609	2086	}
never@1609	2087	{
never@1609	2088	StubCodeMark mark(this, "StubRoutines", "sin");
never@1609	2089	StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
never@1609	2090
never@1609	2091	__ fld_d(Address(rsp, 4));
never@1609	2092	__ trigfunc('s');
never@1609	2093	__ ret(0);
never@1609	2094	}
never@1609	2095	{
never@1609	2096	StubCodeMark mark(this, "StubRoutines", "cos");
never@1609	2097	StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
never@1609	2098
never@1609	2099	__ fld_d(Address(rsp, 4));
never@1609	2100	__ trigfunc('c');
never@1609	2101	__ ret(0);
never@1609	2102	}
never@1609	2103	{
never@1609	2104	StubCodeMark mark(this, "StubRoutines", "tan");
never@1609	2105	StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
never@1609	2106
never@1609	2107	__ fld_d(Address(rsp, 4));
never@1609	2108	__ trigfunc('t');
never@1609	2109	__ ret(0);
never@1609	2110	}
never@1609	2111
never@1609	2112	// The intrinsic version of these seem to return the same value as
never@1609	2113	// the strict version.
never@1609	2114	StubRoutines::_intrinsic_exp = SharedRuntime::dexp;
never@1609	2115	StubRoutines::_intrinsic_pow = SharedRuntime::dpow;
never@1609	2116	}
never@1609	2117
duke@435	2118	public:
duke@435	2119	// Information about frame layout at time of blocking runtime call.
duke@435	2120	// Note that we only have to preserve callee-saved registers since
duke@435	2121	// the compilers are responsible for supplying a continuation point
duke@435	2122	// if they expect all registers to be preserved.
duke@435	2123	enum layout {
duke@435	2124	thread_off, // last_java_sp
duke@435	2125	rbp_off, // callee saved register
duke@435	2126	ret_pc,
duke@435	2127	framesize
duke@435	2128	};
duke@435	2129
duke@435	2130	private:
duke@435	2131
duke@435	2132	#undef __
duke@435	2133	#define __ masm->
duke@435	2134
duke@435	2135	//------------------------------------------------------------------------------------------------------------------------
duke@435	2136	// Continuation point for throwing of implicit exceptions that are not handled in
duke@435	2137	// the current activation. Fabricates an exception oop and initiates normal
duke@435	2138	// exception dispatching in this frame.
duke@435	2139	//
duke@435	2140	// Previously the compiler (c2) allowed for callee save registers on Java calls.
duke@435	2141	// This is no longer true after adapter frames were removed but could possibly
duke@435	2142	// be brought back in the future if the interpreter code was reworked and it
duke@435	2143	// was deemed worthwhile. The comment below was left to describe what must
duke@435	2144	// happen here if callee saves were resurrected. As it stands now this stub
duke@435	2145	// could actually be a vanilla BufferBlob and have now oopMap at all.
duke@435	2146	// Since it doesn't make much difference we've chosen to leave it the
duke@435	2147	// way it was in the callee save days and keep the comment.
duke@435	2148
duke@435	2149	// If we need to preserve callee-saved values we need a callee-saved oop map and
duke@435	2150	// therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
duke@435	2151	// If the compiler needs all registers to be preserved between the fault
duke@435	2152	// point and the exception handler then it must assume responsibility for that in
duke@435	2153	// AbstractCompiler::continuation_for_implicit_null_exception or
duke@435	2154	// continuation_for_implicit_division_by_zero_exception. All other implicit
duke@435	2155	// exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
duke@435	2156	// either at call sites or otherwise assume that stack unwinding will be initiated,
duke@435	2157	// so caller saved registers were assumed volatile in the compiler.
duke@435	2158	address generate_throw_exception(const char* name, address runtime_entry,
duke@435	2159	bool restore_saved_exception_pc) {
duke@435	2160
duke@435	2161	int insts_size = 256;
duke@435	2162	int locs_size = 32;
duke@435	2163
duke@435	2164	CodeBuffer code(name, insts_size, locs_size);
duke@435	2165	OopMapSet* oop_maps = new OopMapSet();
duke@435	2166	MacroAssembler* masm = new MacroAssembler(&code);
duke@435	2167
duke@435	2168	address start = __ pc();
duke@435	2169
duke@435	2170	// This is an inlined and slightly modified version of call_VM
duke@435	2171	// which has the ability to fetch the return PC out of
duke@435	2172	// thread-local storage and also sets up last_Java_sp slightly
duke@435	2173	// differently than the real call_VM
duke@435	2174	Register java_thread = rbx;
duke@435	2175	__ get_thread(java_thread);
duke@435	2176	if (restore_saved_exception_pc) {
never@739	2177	__ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));
never@739	2178	__ push(rax);
duke@435	2179	}
duke@435	2180
duke@435	2181	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2182
duke@435	2183	// pc and rbp, already pushed
never@739	2184	__ subptr(rsp, (framesize-2) * wordSize); // prolog
duke@435	2185
duke@435	2186	// Frame is now completed as far as size and linkage.
duke@435	2187
duke@435	2188	int frame_complete = __ pc() - start;
duke@435	2189
duke@435	2190	// push java thread (becomes first argument of C function)
never@739	2191	__ movptr(Address(rsp, thread_off * wordSize), java_thread);
duke@435	2192
duke@435	2193	// Set up last_Java_sp and last_Java_fp
duke@435	2194	__ set_last_Java_frame(java_thread, rsp, rbp, NULL);
duke@435	2195
duke@435	2196	// Call runtime
duke@435	2197	BLOCK_COMMENT("call runtime_entry");
duke@435	2198	__ call(RuntimeAddress(runtime_entry));
duke@435	2199	// Generate oop map
duke@435	2200	OopMap* map = new OopMap(framesize, 0);
duke@435	2201	oop_maps->add_gc_map(__ pc() - start, map);
duke@435	2202
duke@435	2203	// restore the thread (cannot use the pushed argument since arguments
duke@435	2204	// may be overwritten by C code generated by an optimizing compiler);
duke@435	2205	// however can use the register value directly if it is callee saved.
duke@435	2206	__ get_thread(java_thread);
duke@435	2207
duke@435	2208	__ reset_last_Java_frame(java_thread, true, false);
duke@435	2209
duke@435	2210	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2211
duke@435	2212	// check for pending exceptions
duke@435	2213	#ifdef ASSERT
duke@435	2214	Label L;
never@739	2215	__ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	2216	__ jcc(Assembler::notEqual, L);
duke@435	2217	__ should_not_reach_here();
duke@435	2218	__ bind(L);
duke@435	2219	#endif /* ASSERT */
duke@435	2220	__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	2221
duke@435	2222
duke@435	2223	RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
duke@435	2224	return stub->entry_point();
duke@435	2225	}
duke@435	2226
duke@435	2227
duke@435	2228	void create_control_words() {
duke@435	2229	// Round to nearest, 53-bit mode, exceptions masked
duke@435	2230	StubRoutines::_fpu_cntrl_wrd_std = 0x027F;
duke@435	2231	// Round to zero, 53-bit mode, exception mased
duke@435	2232	StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
duke@435	2233	// Round to nearest, 24-bit mode, exceptions masked
duke@435	2234	StubRoutines::_fpu_cntrl_wrd_24 = 0x007F;
duke@435	2235	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2236	StubRoutines::_fpu_cntrl_wrd_64 = 0x037F;
duke@435	2237	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2238	StubRoutines::_mxcsr_std = 0x1F80;
duke@435	2239	// Note: the following two constants are 80-bit values
duke@435	2240	// layout is critical for correct loading by FPU.
duke@435	2241	// Bias for strict fp multiply/divide
duke@435	2242	StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
duke@435	2243	StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
duke@435	2244	StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
duke@435	2245	// Un-Bias for strict fp multiply/divide
duke@435	2246	StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
duke@435	2247	StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
duke@435	2248	StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
duke@435	2249	}
duke@435	2250
duke@435	2251	//---------------------------------------------------------------------------
duke@435	2252	// Initialization
duke@435	2253
duke@435	2254	void generate_initial() {
duke@435	2255	// Generates all stubs and initializes the entry points
duke@435	2256
duke@435	2257	//------------------------------------------------------------------------------------------------------------------------
duke@435	2258	// entry points that exist in all platforms
duke@435	2259	// Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
duke@435	2260	// the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
duke@435	2261	StubRoutines::_forward_exception_entry = generate_forward_exception();
duke@435	2262
duke@435	2263	StubRoutines::_call_stub_entry =
duke@435	2264	generate_call_stub(StubRoutines::_call_stub_return_address);
duke@435	2265	// is referenced by megamorphic call
duke@435	2266	StubRoutines::_catch_exception_entry = generate_catch_exception();
duke@435	2267
duke@435	2268	// These are currently used by Solaris/Intel
duke@435	2269	StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
duke@435	2270
duke@435	2271	StubRoutines::_handler_for_unsafe_access_entry =
duke@435	2272	generate_handler_for_unsafe_access();
duke@435	2273
duke@435	2274	// platform dependent
duke@435	2275	create_control_words();
duke@435	2276
never@739	2277	StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
never@739	2278	StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = generate_verify_fpu_cntrl_wrd();
duke@435	2279	StubRoutines::_d2i_wrapper = generate_d2i_wrapper(T_INT,
duke@435	2280	CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
duke@435	2281	StubRoutines::_d2l_wrapper = generate_d2i_wrapper(T_LONG,
duke@435	2282	CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
duke@435	2283	}
duke@435	2284
duke@435	2285
duke@435	2286	void generate_all() {
duke@435	2287	// Generates all stubs and initializes the entry points
duke@435	2288
duke@435	2289	// These entry points require SharedInfo::stack0 to be set up in non-core builds
duke@435	2290	// and need to be relocatable, so they each fabricate a RuntimeStub internally.
duke@435	2291	StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError), false);
dcubed@451	2292	StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError), false);
duke@435	2293	StubRoutines::_throw_ArithmeticException_entry = generate_throw_exception("ArithmeticException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException), true);
duke@435	2294	StubRoutines::_throw_NullPointerException_entry = generate_throw_exception("NullPointerException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);
duke@435	2295	StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);
duke@435	2296	StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError), false);
duke@435	2297
duke@435	2298	//------------------------------------------------------------------------------------------------------------------------
duke@435	2299	// entry points that are platform specific
duke@435	2300
duke@435	2301	// support for verify_oop (must happen after universe_init)
duke@435	2302	StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
duke@435	2303
duke@435	2304	// arraycopy stubs used by compilers
duke@435	2305	generate_arraycopy_stubs();
jrose@1145	2306
never@1609	2307	generate_math_stubs();
duke@435	2308	}
duke@435	2309
duke@435	2310
duke@435	2311	public:
duke@435	2312	StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
duke@435	2313	if (all) {
duke@435	2314	generate_all();
duke@435	2315	} else {
duke@435	2316	generate_initial();
duke@435	2317	}
duke@435	2318	}
duke@435	2319	}; // end class declaration
duke@435	2320
duke@435	2321
duke@435	2322	void StubGenerator_generate(CodeBuffer* code, bool all) {
duke@435	2323	StubGenerator g(code, all);
duke@435	2324	}