• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/stubGenerator_x86_32.cpp@576e77447e3c (annotated)

src/cpu/x86/vm/stubGenerator_x86_32.cpp

Sun, 07 Feb 2010 12:15:06 -0800

author

kvn

date

Sun, 07 Feb 2010 12:15:06 -0800

changeset 1686

576e77447e3c

parent 1627

776fb94f33cc

child 1730

3cf667df43ef

permissions

-rw-r--r--

6923002: assert(false,"this call site should not be polymorphic")
Summary: Clear the total count when a receiver information is cleared.
Reviewed-by: never, jrose

duke@435	1	/*
never@1609	2	* Copyright 1999-2010 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_stubGenerator_x86_32.cpp.incl"
duke@435	27
duke@435	28	// Declaration and definition of StubGenerator (no .hpp file).
duke@435	29	// For a more detailed description of the stub routine structure
duke@435	30	// see the comment in stubRoutines.hpp
duke@435	31
duke@435	32	#define __ _masm->
never@739	33	#define a__ ((Assembler*)_masm)->
duke@435	34
duke@435	35	#ifdef PRODUCT
duke@435	36	#define BLOCK_COMMENT(str) /* nothing */
duke@435	37	#else
duke@435	38	#define BLOCK_COMMENT(str) __ block_comment(str)
duke@435	39	#endif
duke@435	40
duke@435	41	#define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
duke@435	42
duke@435	43	const int MXCSR_MASK = 0xFFC0; // Mask out any pending exceptions
duke@435	44	const int FPU_CNTRL_WRD_MASK = 0xFFFF;
duke@435	45
duke@435	46	// -------------------------------------------------------------------------------------------------------------------------
duke@435	47	// Stub Code definitions
duke@435	48
duke@435	49	static address handle_unsafe_access() {
duke@435	50	JavaThread* thread = JavaThread::current();
duke@435	51	address pc = thread->saved_exception_pc();
duke@435	52	// pc is the instruction which we must emulate
duke@435	53	// doing a no-op is fine: return garbage from the load
duke@435	54	// therefore, compute npc
duke@435	55	address npc = Assembler::locate_next_instruction(pc);
duke@435	56
duke@435	57	// request an async exception
duke@435	58	thread->set_pending_unsafe_access_error();
duke@435	59
duke@435	60	// return address of next instruction to execute
duke@435	61	return npc;
duke@435	62	}
duke@435	63
duke@435	64	class StubGenerator: public StubCodeGenerator {
duke@435	65	private:
duke@435	66
duke@435	67	#ifdef PRODUCT
duke@435	68	#define inc_counter_np(counter) (0)
duke@435	69	#else
duke@435	70	void inc_counter_np_(int& counter) {
never@739	71	__ incrementl(ExternalAddress((address)&counter));
duke@435	72	}
duke@435	73	#define inc_counter_np(counter) \
duke@435	74	BLOCK_COMMENT("inc_counter " #counter); \
duke@435	75	inc_counter_np_(counter);
duke@435	76	#endif //PRODUCT
duke@435	77
duke@435	78	void inc_copy_counter_np(BasicType t) {
duke@435	79	#ifndef PRODUCT
duke@435	80	switch (t) {
duke@435	81	case T_BYTE: inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
duke@435	82	case T_SHORT: inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
duke@435	83	case T_INT: inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
duke@435	84	case T_LONG: inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
duke@435	85	case T_OBJECT: inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
duke@435	86	}
duke@435	87	ShouldNotReachHere();
duke@435	88	#endif //PRODUCT
duke@435	89	}
duke@435	90
duke@435	91	//------------------------------------------------------------------------------------------------------------------------
duke@435	92	// Call stubs are used to call Java from C
duke@435	93	//
duke@435	94	// [ return_from_Java ] <--- rsp
duke@435	95	// [ argument word n ]
duke@435	96	// ...
duke@435	97	// -N [ argument word 1 ]
duke@435	98	// -7 [ Possible padding for stack alignment ]
duke@435	99	// -6 [ Possible padding for stack alignment ]
duke@435	100	// -5 [ Possible padding for stack alignment ]
duke@435	101	// -4 [ mxcsr save ] <--- rsp_after_call
duke@435	102	// -3 [ saved rbx, ]
duke@435	103	// -2 [ saved rsi ]
duke@435	104	// -1 [ saved rdi ]
duke@435	105	// 0 [ saved rbp, ] <--- rbp,
duke@435	106	// 1 [ return address ]
duke@435	107	// 2 [ ptr. to call wrapper ]
duke@435	108	// 3 [ result ]
duke@435	109	// 4 [ result_type ]
duke@435	110	// 5 [ method ]
duke@435	111	// 6 [ entry_point ]
duke@435	112	// 7 [ parameters ]
duke@435	113	// 8 [ parameter_size ]
duke@435	114	// 9 [ thread ]
duke@435	115
duke@435	116
duke@435	117	address generate_call_stub(address& return_address) {
duke@435	118	StubCodeMark mark(this, "StubRoutines", "call_stub");
duke@435	119	address start = __ pc();
duke@435	120
duke@435	121	// stub code parameters / addresses
duke@435	122	assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
duke@435	123	bool sse_save = false;
duke@435	124	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
duke@435	125	const int locals_count_in_bytes (4*wordSize);
duke@435	126	const Address mxcsr_save (rbp, -4 * wordSize);
duke@435	127	const Address saved_rbx (rbp, -3 * wordSize);
duke@435	128	const Address saved_rsi (rbp, -2 * wordSize);
duke@435	129	const Address saved_rdi (rbp, -1 * wordSize);
duke@435	130	const Address result (rbp, 3 * wordSize);
duke@435	131	const Address result_type (rbp, 4 * wordSize);
duke@435	132	const Address method (rbp, 5 * wordSize);
duke@435	133	const Address entry_point (rbp, 6 * wordSize);
duke@435	134	const Address parameters (rbp, 7 * wordSize);
duke@435	135	const Address parameter_size(rbp, 8 * wordSize);
duke@435	136	const Address thread (rbp, 9 * wordSize); // same as in generate_catch_exception()!
duke@435	137	sse_save = UseSSE > 0;
duke@435	138
duke@435	139	// stub code
duke@435	140	__ enter();
never@739	141	__ movptr(rcx, parameter_size); // parameter counter
never@739	142	__ shlptr(rcx, Interpreter::logStackElementSize()); // convert parameter count to bytes
never@739	143	__ addptr(rcx, locals_count_in_bytes); // reserve space for register saves
never@739	144	__ subptr(rsp, rcx);
never@739	145	__ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
duke@435	146
duke@435	147	// save rdi, rsi, & rbx, according to C calling conventions
never@739	148	__ movptr(saved_rdi, rdi);
never@739	149	__ movptr(saved_rsi, rsi);
never@739	150	__ movptr(saved_rbx, rbx);
duke@435	151	// save and initialize %mxcsr
duke@435	152	if (sse_save) {
duke@435	153	Label skip_ldmx;
duke@435	154	__ stmxcsr(mxcsr_save);
duke@435	155	__ movl(rax, mxcsr_save);
duke@435	156	__ andl(rax, MXCSR_MASK); // Only check control and mask bits
duke@435	157	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
duke@435	158	__ cmp32(rax, mxcsr_std);
duke@435	159	__ jcc(Assembler::equal, skip_ldmx);
duke@435	160	__ ldmxcsr(mxcsr_std);
duke@435	161	__ bind(skip_ldmx);
duke@435	162	}
duke@435	163
duke@435	164	// make sure the control word is correct.
duke@435	165	__ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
duke@435	166
duke@435	167	#ifdef ASSERT
duke@435	168	// make sure we have no pending exceptions
duke@435	169	{ Label L;
never@739	170	__ movptr(rcx, thread);
never@739	171	__ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	172	__ jcc(Assembler::equal, L);
duke@435	173	__ stop("StubRoutines::call_stub: entered with pending exception");
duke@435	174	__ bind(L);
duke@435	175	}
duke@435	176	#endif
duke@435	177
duke@435	178	// pass parameters if any
duke@435	179	BLOCK_COMMENT("pass parameters if any");
duke@435	180	Label parameters_done;
duke@435	181	__ movl(rcx, parameter_size); // parameter counter
duke@435	182	__ testl(rcx, rcx);
duke@435	183	__ jcc(Assembler::zero, parameters_done);
duke@435	184
duke@435	185	// parameter passing loop
duke@435	186
duke@435	187	Label loop;
duke@435	188	// Copy Java parameters in reverse order (receiver last)
duke@435	189	// Note that the argument order is inverted in the process
duke@435	190	// source is rdx[rcx: N-1..0]
duke@435	191	// dest is rsp[rbx: 0..N-1]
duke@435	192
never@739	193	__ movptr(rdx, parameters); // parameter pointer
never@739	194	__ xorptr(rbx, rbx);
duke@435	195
duke@435	196	__ BIND(loop);
duke@435	197	if (TaggedStackInterpreter) {
never@739	198	__ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(),
duke@435	199	-2*wordSize)); // get tag
never@739	200	__ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
duke@435	201	Interpreter::expr_tag_offset_in_bytes(0)), rax); // store tag
duke@435	202	}
duke@435	203
duke@435	204	// get parameter
never@739	205	__ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
never@739	206	__ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
duke@435	207	Interpreter::expr_offset_in_bytes(0)), rax); // store parameter
duke@435	208	__ increment(rbx);
duke@435	209	__ decrement(rcx);
duke@435	210	__ jcc(Assembler::notZero, loop);
duke@435	211
duke@435	212	// call Java function
duke@435	213	__ BIND(parameters_done);
never@739	214	__ movptr(rbx, method); // get methodOop
never@739	215	__ movptr(rax, entry_point); // get entry_point
never@739	216	__ mov(rsi, rsp); // set sender sp
duke@435	217	BLOCK_COMMENT("call Java function");
duke@435	218	__ call(rax);
duke@435	219
duke@435	220	BLOCK_COMMENT("call_stub_return_address:");
duke@435	221	return_address = __ pc();
duke@435	222
duke@435	223	Label common_return;
duke@435	224
duke@435	225	__ BIND(common_return);
duke@435	226
duke@435	227	// store result depending on type
duke@435	228	// (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
never@739	229	__ movptr(rdi, result);
duke@435	230	Label is_long, is_float, is_double, exit;
duke@435	231	__ movl(rsi, result_type);
duke@435	232	__ cmpl(rsi, T_LONG);
duke@435	233	__ jcc(Assembler::equal, is_long);
duke@435	234	__ cmpl(rsi, T_FLOAT);
duke@435	235	__ jcc(Assembler::equal, is_float);
duke@435	236	__ cmpl(rsi, T_DOUBLE);
duke@435	237	__ jcc(Assembler::equal, is_double);
duke@435	238
duke@435	239	// handle T_INT case
duke@435	240	__ movl(Address(rdi, 0), rax);
duke@435	241	__ BIND(exit);
duke@435	242
duke@435	243	// check that FPU stack is empty
duke@435	244	__ verify_FPU(0, "generate_call_stub");
duke@435	245
duke@435	246	// pop parameters
never@739	247	__ lea(rsp, rsp_after_call);
duke@435	248
duke@435	249	// restore %mxcsr
duke@435	250	if (sse_save) {
duke@435	251	__ ldmxcsr(mxcsr_save);
duke@435	252	}
duke@435	253
duke@435	254	// restore rdi, rsi and rbx,
never@739	255	__ movptr(rbx, saved_rbx);
never@739	256	__ movptr(rsi, saved_rsi);
never@739	257	__ movptr(rdi, saved_rdi);
never@739	258	__ addptr(rsp, 4*wordSize);
duke@435	259
duke@435	260	// return
never@739	261	__ pop(rbp);
duke@435	262	__ ret(0);
duke@435	263
duke@435	264	// handle return types different from T_INT
duke@435	265	__ BIND(is_long);
duke@435	266	__ movl(Address(rdi, 0 * wordSize), rax);
duke@435	267	__ movl(Address(rdi, 1 * wordSize), rdx);
duke@435	268	__ jmp(exit);
duke@435	269
duke@435	270	__ BIND(is_float);
duke@435	271	// interpreter uses xmm0 for return values
duke@435	272	if (UseSSE >= 1) {
duke@435	273	__ movflt(Address(rdi, 0), xmm0);
duke@435	274	} else {
duke@435	275	__ fstp_s(Address(rdi, 0));
duke@435	276	}
duke@435	277	__ jmp(exit);
duke@435	278
duke@435	279	__ BIND(is_double);
duke@435	280	// interpreter uses xmm0 for return values
duke@435	281	if (UseSSE >= 2) {
duke@435	282	__ movdbl(Address(rdi, 0), xmm0);
duke@435	283	} else {
duke@435	284	__ fstp_d(Address(rdi, 0));
duke@435	285	}
duke@435	286	__ jmp(exit);
duke@435	287
duke@435	288	// If we call compiled code directly from the call stub we will
duke@435	289	// need to adjust the return back to the call stub to a specialized
duke@435	290	// piece of code that can handle compiled results and cleaning the fpu
duke@435	291	// stack. compiled code will be set to return here instead of the
duke@435	292	// return above that handles interpreter returns.
duke@435	293
duke@435	294	BLOCK_COMMENT("call_stub_compiled_return:");
never@739	295	StubRoutines::x86::set_call_stub_compiled_return( __ pc());
duke@435	296
duke@435	297	#ifdef COMPILER2
duke@435	298	if (UseSSE >= 2) {
duke@435	299	__ verify_FPU(0, "call_stub_compiled_return");
duke@435	300	} else {
duke@435	301	for (int i = 1; i < 8; i++) {
duke@435	302	__ ffree(i);
duke@435	303	}
duke@435	304
duke@435	305	// UseSSE <= 1 so double result should be left on TOS
duke@435	306	__ movl(rsi, result_type);
duke@435	307	__ cmpl(rsi, T_DOUBLE);
duke@435	308	__ jcc(Assembler::equal, common_return);
duke@435	309	if (UseSSE == 0) {
duke@435	310	// UseSSE == 0 so float result should be left on TOS
duke@435	311	__ cmpl(rsi, T_FLOAT);
duke@435	312	__ jcc(Assembler::equal, common_return);
duke@435	313	}
duke@435	314	__ ffree(0);
duke@435	315	}
duke@435	316	#endif /* COMPILER2 */
duke@435	317	__ jmp(common_return);
duke@435	318
duke@435	319	return start;
duke@435	320	}
duke@435	321
duke@435	322
duke@435	323	//------------------------------------------------------------------------------------------------------------------------
duke@435	324	// Return point for a Java call if there's an exception thrown in Java code.
duke@435	325	// The exception is caught and transformed into a pending exception stored in
duke@435	326	// JavaThread that can be tested from within the VM.
duke@435	327	//
duke@435	328	// Note: Usually the parameters are removed by the callee. In case of an exception
duke@435	329	// crossing an activation frame boundary, that is not the case if the callee
duke@435	330	// is compiled code => need to setup the rsp.
duke@435	331	//
duke@435	332	// rax,: exception oop
duke@435	333
duke@435	334	address generate_catch_exception() {
duke@435	335	StubCodeMark mark(this, "StubRoutines", "catch_exception");
duke@435	336	const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
duke@435	337	const Address thread (rbp, 9 * wordSize); // same as in generate_call_stub()!
duke@435	338	address start = __ pc();
duke@435	339
duke@435	340	// get thread directly
never@739	341	__ movptr(rcx, thread);
duke@435	342	#ifdef ASSERT
duke@435	343	// verify that threads correspond
duke@435	344	{ Label L;
duke@435	345	__ get_thread(rbx);
never@739	346	__ cmpptr(rbx, rcx);
duke@435	347	__ jcc(Assembler::equal, L);
duke@435	348	__ stop("StubRoutines::catch_exception: threads must correspond");
duke@435	349	__ bind(L);
duke@435	350	}
duke@435	351	#endif
duke@435	352	// set pending exception
duke@435	353	__ verify_oop(rax);
never@739	354	__ movptr(Address(rcx, Thread::pending_exception_offset()), rax );
duke@435	355	__ lea(Address(rcx, Thread::exception_file_offset ()),
duke@435	356	ExternalAddress((address)__FILE__));
duke@435	357	__ movl(Address(rcx, Thread::exception_line_offset ()), __LINE__ );
duke@435	358	// complete return to VM
duke@435	359	assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
duke@435	360	__ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
duke@435	361
duke@435	362	return start;
duke@435	363	}
duke@435	364
duke@435	365
duke@435	366	//------------------------------------------------------------------------------------------------------------------------
duke@435	367	// Continuation point for runtime calls returning with a pending exception.
duke@435	368	// The pending exception check happened in the runtime or native call stub.
duke@435	369	// The pending exception in Thread is converted into a Java-level exception.
duke@435	370	//
duke@435	371	// Contract with Java-level exception handlers:
duke@435	372	// rax,: exception
duke@435	373	// rdx: throwing pc
duke@435	374	//
duke@435	375	// NOTE: At entry of this stub, exception-pc must be on stack !!
duke@435	376
duke@435	377	address generate_forward_exception() {
duke@435	378	StubCodeMark mark(this, "StubRoutines", "forward exception");
duke@435	379	address start = __ pc();
duke@435	380
duke@435	381	// Upon entry, the sp points to the return address returning into Java
duke@435	382	// (interpreted or compiled) code; i.e., the return address becomes the
duke@435	383	// throwing pc.
duke@435	384	//
duke@435	385	// Arguments pushed before the runtime call are still on the stack but
duke@435	386	// the exception handler will reset the stack pointer -> ignore them.
duke@435	387	// A potential result in registers can be ignored as well.
duke@435	388
duke@435	389	#ifdef ASSERT
duke@435	390	// make sure this code is only executed if there is a pending exception
duke@435	391	{ Label L;
duke@435	392	__ get_thread(rcx);
never@739	393	__ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	394	__ jcc(Assembler::notEqual, L);
duke@435	395	__ stop("StubRoutines::forward exception: no pending exception (1)");
duke@435	396	__ bind(L);
duke@435	397	}
duke@435	398	#endif
duke@435	399
duke@435	400	// compute exception handler into rbx,
never@739	401	__ movptr(rax, Address(rsp, 0));
duke@435	402	BLOCK_COMMENT("call exception_handler_for_return_address");
duke@435	403	__ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rax);
never@739	404	__ mov(rbx, rax);
duke@435	405
duke@435	406	// setup rax, & rdx, remove return address & clear pending exception
duke@435	407	__ get_thread(rcx);
never@739	408	__ pop(rdx);
never@739	409	__ movptr(rax, Address(rcx, Thread::pending_exception_offset()));
xlu@968	410	__ movptr(Address(rcx, Thread::pending_exception_offset()), NULL_WORD);
duke@435	411
duke@435	412	#ifdef ASSERT
duke@435	413	// make sure exception is set
duke@435	414	{ Label L;
never@739	415	__ testptr(rax, rax);
duke@435	416	__ jcc(Assembler::notEqual, L);
duke@435	417	__ stop("StubRoutines::forward exception: no pending exception (2)");
duke@435	418	__ bind(L);
duke@435	419	}
duke@435	420	#endif
duke@435	421
duke@435	422	// continue at exception handler (return address removed)
duke@435	423	// rax,: exception
duke@435	424	// rbx,: exception handler
duke@435	425	// rdx: throwing pc
duke@435	426	__ verify_oop(rax);
duke@435	427	__ jmp(rbx);
duke@435	428
duke@435	429	return start;
duke@435	430	}
duke@435	431
duke@435	432
duke@435	433	//----------------------------------------------------------------------------------------------------
duke@435	434	// Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
duke@435	435	//
duke@435	436	// xchg exists as far back as 8086, lock needed for MP only
duke@435	437	// Stack layout immediately after call:
duke@435	438	//
duke@435	439	// 0 [ret addr ] <--- rsp
duke@435	440	// 1 [ ex ]
duke@435	441	// 2 [ dest ]
duke@435	442	//
duke@435	443	// Result: *dest <- ex, return (old *dest)
duke@435	444	//
duke@435	445	// Note: win32 does not currently use this code
duke@435	446
duke@435	447	address generate_atomic_xchg() {
duke@435	448	StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
duke@435	449	address start = __ pc();
duke@435	450
never@739	451	__ push(rdx);
duke@435	452	Address exchange(rsp, 2 * wordSize);
duke@435	453	Address dest_addr(rsp, 3 * wordSize);
duke@435	454	__ movl(rax, exchange);
never@739	455	__ movptr(rdx, dest_addr);
never@739	456	__ xchgl(rax, Address(rdx, 0));
never@739	457	__ pop(rdx);
duke@435	458	__ ret(0);
duke@435	459
duke@435	460	return start;
duke@435	461	}
duke@435	462
duke@435	463	//----------------------------------------------------------------------------------------------------
duke@435	464	// Support for void verify_mxcsr()
duke@435	465	//
duke@435	466	// This routine is used with -Xcheck:jni to verify that native
duke@435	467	// JNI code does not return to Java code without restoring the
duke@435	468	// MXCSR register to our expected state.
duke@435	469
duke@435	470
duke@435	471	address generate_verify_mxcsr() {
duke@435	472	StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
duke@435	473	address start = __ pc();
duke@435	474
duke@435	475	const Address mxcsr_save(rsp, 0);
duke@435	476
duke@435	477	if (CheckJNICalls && UseSSE > 0 ) {
duke@435	478	Label ok_ret;
duke@435	479	ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
never@739	480	__ push(rax);
never@739	481	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	482	__ stmxcsr(mxcsr_save);
duke@435	483	__ movl(rax, mxcsr_save);
duke@435	484	__ andl(rax, MXCSR_MASK);
duke@435	485	__ cmp32(rax, mxcsr_std);
duke@435	486	__ jcc(Assembler::equal, ok_ret);
duke@435	487
duke@435	488	__ warn("MXCSR changed by native JNI code.");
duke@435	489
duke@435	490	__ ldmxcsr(mxcsr_std);
duke@435	491
duke@435	492	__ bind(ok_ret);
never@739	493	__ addptr(rsp, wordSize);
never@739	494	__ pop(rax);
duke@435	495	}
duke@435	496
duke@435	497	__ ret(0);
duke@435	498
duke@435	499	return start;
duke@435	500	}
duke@435	501
duke@435	502
duke@435	503	//---------------------------------------------------------------------------
duke@435	504	// Support for void verify_fpu_cntrl_wrd()
duke@435	505	//
duke@435	506	// This routine is used with -Xcheck:jni to verify that native
duke@435	507	// JNI code does not return to Java code without restoring the
duke@435	508	// FP control word to our expected state.
duke@435	509
duke@435	510	address generate_verify_fpu_cntrl_wrd() {
duke@435	511	StubCodeMark mark(this, "StubRoutines", "verify_spcw");
duke@435	512	address start = __ pc();
duke@435	513
duke@435	514	const Address fpu_cntrl_wrd_save(rsp, 0);
duke@435	515
duke@435	516	if (CheckJNICalls) {
duke@435	517	Label ok_ret;
never@739	518	__ push(rax);
never@739	519	__ subptr(rsp, wordSize); // allocate a temp location
duke@435	520	__ fnstcw(fpu_cntrl_wrd_save);
duke@435	521	__ movl(rax, fpu_cntrl_wrd_save);
duke@435	522	__ andl(rax, FPU_CNTRL_WRD_MASK);
duke@435	523	ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
duke@435	524	__ cmp32(rax, fpu_std);
duke@435	525	__ jcc(Assembler::equal, ok_ret);
duke@435	526
duke@435	527	__ warn("Floating point control word changed by native JNI code.");
duke@435	528
duke@435	529	__ fldcw(fpu_std);
duke@435	530
duke@435	531	__ bind(ok_ret);
never@739	532	__ addptr(rsp, wordSize);
never@739	533	__ pop(rax);
duke@435	534	}
duke@435	535
duke@435	536	__ ret(0);
duke@435	537
duke@435	538	return start;
duke@435	539	}
duke@435	540
duke@435	541	//---------------------------------------------------------------------------
duke@435	542	// Wrapper for slow-case handling of double-to-integer conversion
duke@435	543	// d2i or f2i fast case failed either because it is nan or because
duke@435	544	// of under/overflow.
duke@435	545	// Input: FPU TOS: float value
duke@435	546	// Output: rax, (rdx): integer (long) result
duke@435	547
duke@435	548	address generate_d2i_wrapper(BasicType t, address fcn) {
duke@435	549	StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
duke@435	550	address start = __ pc();
duke@435	551
duke@435	552	// Capture info about frame layout
duke@435	553	enum layout { FPUState_off = 0,
duke@435	554	rbp_off = FPUStateSizeInWords,
duke@435	555	rdi_off,
duke@435	556	rsi_off,
duke@435	557	rcx_off,
duke@435	558	rbx_off,
duke@435	559	saved_argument_off,
duke@435	560	saved_argument_off2, // 2nd half of double
duke@435	561	framesize
duke@435	562	};
duke@435	563
duke@435	564	assert(FPUStateSizeInWords == 27, "update stack layout");
duke@435	565
duke@435	566	// Save outgoing argument to stack across push_FPU_state()
never@739	567	__ subptr(rsp, wordSize * 2);
duke@435	568	__ fstp_d(Address(rsp, 0));
duke@435	569
duke@435	570	// Save CPU & FPU state
never@739	571	__ push(rbx);
never@739	572	__ push(rcx);
never@739	573	__ push(rsi);
never@739	574	__ push(rdi);
never@739	575	__ push(rbp);
duke@435	576	__ push_FPU_state();
duke@435	577
duke@435	578	// push_FPU_state() resets the FP top of stack
duke@435	579	// Load original double into FP top of stack
duke@435	580	__ fld_d(Address(rsp, saved_argument_off * wordSize));
duke@435	581	// Store double into stack as outgoing argument
never@739	582	__ subptr(rsp, wordSize*2);
duke@435	583	__ fst_d(Address(rsp, 0));
duke@435	584
duke@435	585	// Prepare FPU for doing math in C-land
duke@435	586	__ empty_FPU_stack();
duke@435	587	// Call the C code to massage the double. Result in EAX
duke@435	588	if (t == T_INT)
duke@435	589	{ BLOCK_COMMENT("SharedRuntime::d2i"); }
duke@435	590	else if (t == T_LONG)
duke@435	591	{ BLOCK_COMMENT("SharedRuntime::d2l"); }
duke@435	592	__ call_VM_leaf( fcn, 2 );
duke@435	593
duke@435	594	// Restore CPU & FPU state
duke@435	595	__ pop_FPU_state();
never@739	596	__ pop(rbp);
never@739	597	__ pop(rdi);
never@739	598	__ pop(rsi);
never@739	599	__ pop(rcx);
never@739	600	__ pop(rbx);
never@739	601	__ addptr(rsp, wordSize * 2);
duke@435	602
duke@435	603	__ ret(0);
duke@435	604
duke@435	605	return start;
duke@435	606	}
duke@435	607
duke@435	608
duke@435	609	//---------------------------------------------------------------------------
duke@435	610	// The following routine generates a subroutine to throw an asynchronous
duke@435	611	// UnknownError when an unsafe access gets a fault that could not be
duke@435	612	// reasonably prevented by the programmer. (Example: SIGBUS/OBJERR.)
duke@435	613	address generate_handler_for_unsafe_access() {
duke@435	614	StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
duke@435	615	address start = __ pc();
duke@435	616
never@739	617	__ push(0); // hole for return address-to-be
never@739	618	__ pusha(); // push registers
duke@435	619	Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
duke@435	620	BLOCK_COMMENT("call handle_unsafe_access");
duke@435	621	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
never@739	622	__ movptr(next_pc, rax); // stuff next address
never@739	623	__ popa();
duke@435	624	__ ret(0); // jump to next address
duke@435	625
duke@435	626	return start;
duke@435	627	}
duke@435	628
duke@435	629
duke@435	630	//----------------------------------------------------------------------------------------------------
duke@435	631	// Non-destructive plausibility checks for oops
duke@435	632
duke@435	633	address generate_verify_oop() {
duke@435	634	StubCodeMark mark(this, "StubRoutines", "verify_oop");
duke@435	635	address start = __ pc();
duke@435	636
duke@435	637	// Incoming arguments on stack after saving rax,:
duke@435	638	//
duke@435	639	// [tos ]: saved rdx
duke@435	640	// [tos + 1]: saved EFLAGS
duke@435	641	// [tos + 2]: return address
duke@435	642	// [tos + 3]: char* error message
duke@435	643	// [tos + 4]: oop object to verify
duke@435	644	// [tos + 5]: saved rax, - saved by caller and bashed
duke@435	645
duke@435	646	Label exit, error;
never@739	647	__ pushf();
never@739	648	__ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
never@739	649	__ push(rdx); // save rdx
duke@435	650	// make sure object is 'reasonable'
never@739	651	__ movptr(rax, Address(rsp, 4 * wordSize)); // get object
never@739	652	__ testptr(rax, rax);
duke@435	653	__ jcc(Assembler::zero, exit); // if obj is NULL it is ok
duke@435	654
duke@435	655	// Check if the oop is in the right area of memory
duke@435	656	const int oop_mask = Universe::verify_oop_mask();
duke@435	657	const int oop_bits = Universe::verify_oop_bits();
never@739	658	__ mov(rdx, rax);
never@739	659	__ andptr(rdx, oop_mask);
never@739	660	__ cmpptr(rdx, oop_bits);
duke@435	661	__ jcc(Assembler::notZero, error);
duke@435	662
duke@435	663	// make sure klass is 'reasonable'
never@739	664	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
never@739	665	__ testptr(rax, rax);
duke@435	666	__ jcc(Assembler::zero, error); // if klass is NULL it is broken
duke@435	667
duke@435	668	// Check if the klass is in the right area of memory
duke@435	669	const int klass_mask = Universe::verify_klass_mask();
duke@435	670	const int klass_bits = Universe::verify_klass_bits();
never@739	671	__ mov(rdx, rax);
never@739	672	__ andptr(rdx, klass_mask);
never@739	673	__ cmpptr(rdx, klass_bits);
duke@435	674	__ jcc(Assembler::notZero, error);
duke@435	675
duke@435	676	// make sure klass' klass is 'reasonable'
never@739	677	__ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass
never@739	678	__ testptr(rax, rax);
duke@435	679	__ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
duke@435	680
never@739	681	__ mov(rdx, rax);
never@739	682	__ andptr(rdx, klass_mask);
never@739	683	__ cmpptr(rdx, klass_bits);
duke@435	684	__ jcc(Assembler::notZero, error); // if klass not in right area
duke@435	685	// of memory it is broken too.
duke@435	686
duke@435	687	// return if everything seems ok
duke@435	688	__ bind(exit);
never@739	689	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	690	__ pop(rdx); // restore rdx
never@739	691	__ popf(); // restore EFLAGS
duke@435	692	__ ret(3 * wordSize); // pop arguments
duke@435	693
duke@435	694	// handle errors
duke@435	695	__ bind(error);
never@739	696	__ movptr(rax, Address(rsp, 5 * wordSize)); // get saved rax, back
never@739	697	__ pop(rdx); // get saved rdx back
never@739	698	__ popf(); // get saved EFLAGS off stack -- will be ignored
never@739	699	__ pusha(); // push registers (eip = return address & msg are already pushed)
duke@435	700	BLOCK_COMMENT("call MacroAssembler::debug");
never@739	701	__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
never@739	702	__ popa();
duke@435	703	__ ret(3 * wordSize); // pop arguments
duke@435	704	return start;
duke@435	705	}
duke@435	706
duke@435	707	//
duke@435	708	// Generate pre-barrier for array stores
duke@435	709	//
duke@435	710	// Input:
duke@435	711	// start - starting address
ysr@1280	712	// count - element count
duke@435	713	void gen_write_ref_array_pre_barrier(Register start, Register count) {
duke@435	714	assert_different_registers(start, count);
duke@435	715	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	716	switch (bs->kind()) {
duke@435	717	case BarrierSet::G1SATBCT:
duke@435	718	case BarrierSet::G1SATBCTLogging:
duke@435	719	{
never@739	720	__ pusha(); // push registers
apetrusenko@1627	721	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
apetrusenko@1627	722	start, count);
never@739	723	__ popa();
duke@435	724	}
duke@435	725	break;
duke@435	726	case BarrierSet::CardTableModRef:
duke@435	727	case BarrierSet::CardTableExtension:
duke@435	728	case BarrierSet::ModRef:
duke@435	729	break;
duke@435	730	default :
duke@435	731	ShouldNotReachHere();
duke@435	732
duke@435	733	}
duke@435	734	}
duke@435	735
duke@435	736
duke@435	737	//
duke@435	738	// Generate a post-barrier for an array store
duke@435	739	//
duke@435	740	// start - starting address
duke@435	741	// count - element count
duke@435	742	//
duke@435	743	// The two input registers are overwritten.
duke@435	744	//
duke@435	745	void gen_write_ref_array_post_barrier(Register start, Register count) {
duke@435	746	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	747	assert_different_registers(start, count);
duke@435	748	switch (bs->kind()) {
duke@435	749	case BarrierSet::G1SATBCT:
duke@435	750	case BarrierSet::G1SATBCTLogging:
duke@435	751	{
never@739	752	__ pusha(); // push registers
apetrusenko@1627	753	__ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
apetrusenko@1627	754	start, count);
never@739	755	__ popa();
duke@435	756	}
duke@435	757	break;
duke@435	758
duke@435	759	case BarrierSet::CardTableModRef:
duke@435	760	case BarrierSet::CardTableExtension:
duke@435	761	{
duke@435	762	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	763	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	764
duke@435	765	Label L_loop;
duke@435	766	const Register end = count; // elements count; end == start+count-1
duke@435	767	assert_different_registers(start, end);
duke@435	768
never@739	769	__ lea(end, Address(start, count, Address::times_ptr, -wordSize));
never@739	770	__ shrptr(start, CardTableModRefBS::card_shift);
never@739	771	__ shrptr(end, CardTableModRefBS::card_shift);
never@739	772	__ subptr(end, start); // end --> count
duke@435	773	__ BIND(L_loop);
never@684	774	intptr_t disp = (intptr_t) ct->byte_map_base;
never@684	775	Address cardtable(start, count, Address::times_1, disp);
never@684	776	__ movb(cardtable, 0);
duke@435	777	__ decrement(count);
duke@435	778	__ jcc(Assembler::greaterEqual, L_loop);
duke@435	779	}
duke@435	780	break;
duke@435	781	case BarrierSet::ModRef:
duke@435	782	break;
duke@435	783	default :
duke@435	784	ShouldNotReachHere();
duke@435	785
duke@435	786	}
duke@435	787	}
duke@435	788
kvn@840	789
kvn@840	790	// Copy 64 bytes chunks
kvn@840	791	//
kvn@840	792	// Inputs:
kvn@840	793	// from - source array address
kvn@840	794	// to_from - destination array address - from
kvn@840	795	// qword_count - 8-bytes element count, negative
kvn@840	796	//
kvn@840	797	void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	798	assert( UseSSE >= 2, "supported cpu only" );
kvn@840	799	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
kvn@840	800	// Copy 64-byte chunks
kvn@840	801	__ jmpb(L_copy_64_bytes);
kvn@840	802	__ align(16);
kvn@840	803	__ BIND(L_copy_64_bytes_loop);
kvn@840	804
kvn@840	805	if(UseUnalignedLoadStores) {
kvn@840	806	__ movdqu(xmm0, Address(from, 0));
kvn@840	807	__ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	808	__ movdqu(xmm1, Address(from, 16));
kvn@840	809	__ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
kvn@840	810	__ movdqu(xmm2, Address(from, 32));
kvn@840	811	__ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
kvn@840	812	__ movdqu(xmm3, Address(from, 48));
kvn@840	813	__ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
kvn@840	814
kvn@840	815	} else {
kvn@840	816	__ movq(xmm0, Address(from, 0));
kvn@840	817	__ movq(Address(from, to_from, Address::times_1, 0), xmm0);
kvn@840	818	__ movq(xmm1, Address(from, 8));
kvn@840	819	__ movq(Address(from, to_from, Address::times_1, 8), xmm1);
kvn@840	820	__ movq(xmm2, Address(from, 16));
kvn@840	821	__ movq(Address(from, to_from, Address::times_1, 16), xmm2);
kvn@840	822	__ movq(xmm3, Address(from, 24));
kvn@840	823	__ movq(Address(from, to_from, Address::times_1, 24), xmm3);
kvn@840	824	__ movq(xmm4, Address(from, 32));
kvn@840	825	__ movq(Address(from, to_from, Address::times_1, 32), xmm4);
kvn@840	826	__ movq(xmm5, Address(from, 40));
kvn@840	827	__ movq(Address(from, to_from, Address::times_1, 40), xmm5);
kvn@840	828	__ movq(xmm6, Address(from, 48));
kvn@840	829	__ movq(Address(from, to_from, Address::times_1, 48), xmm6);
kvn@840	830	__ movq(xmm7, Address(from, 56));
kvn@840	831	__ movq(Address(from, to_from, Address::times_1, 56), xmm7);
kvn@840	832	}
kvn@840	833
kvn@840	834	__ addl(from, 64);
kvn@840	835	__ BIND(L_copy_64_bytes);
kvn@840	836	__ subl(qword_count, 8);
kvn@840	837	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
kvn@840	838	__ addl(qword_count, 8);
kvn@840	839	__ jccb(Assembler::zero, L_exit);
kvn@840	840	//
kvn@840	841	// length is too short, just copy qwords
kvn@840	842	//
kvn@840	843	__ BIND(L_copy_8_bytes);
kvn@840	844	__ movq(xmm0, Address(from, 0));
kvn@840	845	__ movq(Address(from, to_from, Address::times_1), xmm0);
kvn@840	846	__ addl(from, 8);
kvn@840	847	__ decrement(qword_count);
kvn@840	848	__ jcc(Assembler::greater, L_copy_8_bytes);
kvn@840	849	__ BIND(L_exit);
kvn@840	850	}
kvn@840	851
duke@435	852	// Copy 64 bytes chunks
duke@435	853	//
duke@435	854	// Inputs:
duke@435	855	// from - source array address
duke@435	856	// to_from - destination array address - from
duke@435	857	// qword_count - 8-bytes element count, negative
duke@435	858	//
duke@435	859	void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
kvn@840	860	assert( VM_Version::supports_mmx(), "supported cpu only" );
duke@435	861	Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
duke@435	862	// Copy 64-byte chunks
duke@435	863	__ jmpb(L_copy_64_bytes);
duke@435	864	__ align(16);
duke@435	865	__ BIND(L_copy_64_bytes_loop);
duke@435	866	__ movq(mmx0, Address(from, 0));
duke@435	867	__ movq(mmx1, Address(from, 8));
duke@435	868	__ movq(mmx2, Address(from, 16));
duke@435	869	__ movq(Address(from, to_from, Address::times_1, 0), mmx0);
duke@435	870	__ movq(mmx3, Address(from, 24));
duke@435	871	__ movq(Address(from, to_from, Address::times_1, 8), mmx1);
duke@435	872	__ movq(mmx4, Address(from, 32));
duke@435	873	__ movq(Address(from, to_from, Address::times_1, 16), mmx2);
duke@435	874	__ movq(mmx5, Address(from, 40));
duke@435	875	__ movq(Address(from, to_from, Address::times_1, 24), mmx3);
duke@435	876	__ movq(mmx6, Address(from, 48));
duke@435	877	__ movq(Address(from, to_from, Address::times_1, 32), mmx4);
duke@435	878	__ movq(mmx7, Address(from, 56));
duke@435	879	__ movq(Address(from, to_from, Address::times_1, 40), mmx5);
duke@435	880	__ movq(Address(from, to_from, Address::times_1, 48), mmx6);
duke@435	881	__ movq(Address(from, to_from, Address::times_1, 56), mmx7);
never@739	882	__ addptr(from, 64);
duke@435	883	__ BIND(L_copy_64_bytes);
duke@435	884	__ subl(qword_count, 8);
duke@435	885	__ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
duke@435	886	__ addl(qword_count, 8);
duke@435	887	__ jccb(Assembler::zero, L_exit);
duke@435	888	//
duke@435	889	// length is too short, just copy qwords
duke@435	890	//
duke@435	891	__ BIND(L_copy_8_bytes);
duke@435	892	__ movq(mmx0, Address(from, 0));
duke@435	893	__ movq(Address(from, to_from, Address::times_1), mmx0);
never@739	894	__ addptr(from, 8);
duke@435	895	__ decrement(qword_count);
duke@435	896	__ jcc(Assembler::greater, L_copy_8_bytes);
duke@435	897	__ BIND(L_exit);
duke@435	898	__ emms();
duke@435	899	}
duke@435	900
duke@435	901	address generate_disjoint_copy(BasicType t, bool aligned,
duke@435	902	Address::ScaleFactor sf,
duke@435	903	address* entry, const char *name) {
duke@435	904	__ align(CodeEntryAlignment);
duke@435	905	StubCodeMark mark(this, "StubRoutines", name);
duke@435	906	address start = __ pc();
duke@435	907
duke@435	908	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	909	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
duke@435	910
never@739	911	int shift = Address::times_ptr - sf;
duke@435	912
duke@435	913	const Register from = rsi; // source array address
duke@435	914	const Register to = rdi; // destination array address
duke@435	915	const Register count = rcx; // elements count
duke@435	916	const Register to_from = to; // (to - from)
duke@435	917	const Register saved_to = rdx; // saved destination array address
duke@435	918
duke@435	919	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	920	__ push(rsi);
never@739	921	__ push(rdi);
never@739	922	__ movptr(from , Address(rsp, 12+ 4));
never@739	923	__ movptr(to , Address(rsp, 12+ 8));
duke@435	924	__ movl(count, Address(rsp, 12+ 12));
duke@435	925	if (t == T_OBJECT) {
duke@435	926	__ testl(count, count);
duke@435	927	__ jcc(Assembler::zero, L_0_count);
duke@435	928	gen_write_ref_array_pre_barrier(to, count);
never@739	929	__ mov(saved_to, to); // save 'to'
duke@435	930	}
duke@435	931
duke@435	932	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	933	BLOCK_COMMENT("Entry:");
duke@435	934
never@739	935	__ subptr(to, from); // to --> to_from
duke@435	936	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	937	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
kvn@840	938	if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
duke@435	939	// align source address at 4 bytes address boundary
duke@435	940	if (t == T_BYTE) {
duke@435	941	// One byte misalignment happens only for byte arrays
duke@435	942	__ testl(from, 1);
duke@435	943	__ jccb(Assembler::zero, L_skip_align1);
duke@435	944	__ movb(rax, Address(from, 0));
duke@435	945	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	946	__ increment(from);
duke@435	947	__ decrement(count);
duke@435	948	__ BIND(L_skip_align1);
duke@435	949	}
duke@435	950	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	951	__ testl(from, 2);
duke@435	952	__ jccb(Assembler::zero, L_skip_align2);
duke@435	953	__ movw(rax, Address(from, 0));
duke@435	954	__ movw(Address(from, to_from, Address::times_1, 0), rax);
never@739	955	__ addptr(from, 2);
duke@435	956	__ subl(count, 1<<(shift-1));
duke@435	957	__ BIND(L_skip_align2);
duke@435	958	}
duke@435	959	if (!VM_Version::supports_mmx()) {
never@739	960	__ mov(rax, count); // save 'count'
never@739	961	__ shrl(count, shift); // bytes count
never@739	962	__ addptr(to_from, from);// restore 'to'
never@739	963	__ rep_mov();
never@739	964	__ subptr(to_from, from);// restore 'to_from'
never@739	965	__ mov(count, rax); // restore 'count'
duke@435	966	__ jmpb(L_copy_2_bytes); // all dwords were copied
duke@435	967	} else {
kvn@840	968	if (!UseUnalignedLoadStores) {
kvn@840	969	// align to 8 bytes, we know we are 4 byte aligned to start
kvn@840	970	__ testptr(from, 4);
kvn@840	971	__ jccb(Assembler::zero, L_copy_64_bytes);
kvn@840	972	__ movl(rax, Address(from, 0));
kvn@840	973	__ movl(Address(from, to_from, Address::times_1, 0), rax);
kvn@840	974	__ addptr(from, 4);
kvn@840	975	__ subl(count, 1<<shift);
kvn@840	976	}
duke@435	977	__ BIND(L_copy_64_bytes);
never@739	978	__ mov(rax, count);
duke@435	979	__ shrl(rax, shift+1); // 8 bytes chunk count
duke@435	980	//
duke@435	981	// Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
duke@435	982	//
kvn@840	983	if (UseXMMForArrayCopy) {
kvn@840	984	xmm_copy_forward(from, to_from, rax);
kvn@840	985	} else {
kvn@840	986	mmx_copy_forward(from, to_from, rax);
kvn@840	987	}
duke@435	988	}
duke@435	989	// copy tailing dword
duke@435	990	__ BIND(L_copy_4_bytes);
duke@435	991	__ testl(count, 1<<shift);
duke@435	992	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	993	__ movl(rax, Address(from, 0));
duke@435	994	__ movl(Address(from, to_from, Address::times_1, 0), rax);
duke@435	995	if (t == T_BYTE || t == T_SHORT) {
never@739	996	__ addptr(from, 4);
duke@435	997	__ BIND(L_copy_2_bytes);
duke@435	998	// copy tailing word
duke@435	999	__ testl(count, 1<<(shift-1));
duke@435	1000	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1001	__ movw(rax, Address(from, 0));
duke@435	1002	__ movw(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1003	if (t == T_BYTE) {
never@739	1004	__ addptr(from, 2);
duke@435	1005	__ BIND(L_copy_byte);
duke@435	1006	// copy tailing byte
duke@435	1007	__ testl(count, 1);
duke@435	1008	__ jccb(Assembler::zero, L_exit);
duke@435	1009	__ movb(rax, Address(from, 0));
duke@435	1010	__ movb(Address(from, to_from, Address::times_1, 0), rax);
duke@435	1011	__ BIND(L_exit);
duke@435	1012	} else {
duke@435	1013	__ BIND(L_copy_byte);
duke@435	1014	}
duke@435	1015	} else {
duke@435	1016	__ BIND(L_copy_2_bytes);
duke@435	1017	}
duke@435	1018
duke@435	1019	if (t == T_OBJECT) {
duke@435	1020	__ movl(count, Address(rsp, 12+12)); // reread 'count'
never@739	1021	__ mov(to, saved_to); // restore 'to'
duke@435	1022	gen_write_ref_array_post_barrier(to, count);
duke@435	1023	__ BIND(L_0_count);
duke@435	1024	}
duke@435	1025	inc_copy_counter_np(t);
never@739	1026	__ pop(rdi);
never@739	1027	__ pop(rsi);
duke@435	1028	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1029	__ xorptr(rax, rax); // return 0
duke@435	1030	__ ret(0);
duke@435	1031	return start;
duke@435	1032	}
duke@435	1033
duke@435	1034
duke@435	1035	address generate_conjoint_copy(BasicType t, bool aligned,
duke@435	1036	Address::ScaleFactor sf,
duke@435	1037	address nooverlap_target,
duke@435	1038	address* entry, const char *name) {
duke@435	1039	__ align(CodeEntryAlignment);
duke@435	1040	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1041	address start = __ pc();
duke@435	1042
duke@435	1043	Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
duke@435	1044	Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1045
never@739	1046	int shift = Address::times_ptr - sf;
duke@435	1047
duke@435	1048	const Register src = rax; // source array address
duke@435	1049	const Register dst = rdx; // destination array address
duke@435	1050	const Register from = rsi; // source array address
duke@435	1051	const Register to = rdi; // destination array address
duke@435	1052	const Register count = rcx; // elements count
duke@435	1053	const Register end = rax; // array end address
duke@435	1054
duke@435	1055	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1056	__ push(rsi);
never@739	1057	__ push(rdi);
never@739	1058	__ movptr(src , Address(rsp, 12+ 4)); // from
never@739	1059	__ movptr(dst , Address(rsp, 12+ 8)); // to
never@739	1060	__ movl2ptr(count, Address(rsp, 12+12)); // count
duke@435	1061	if (t == T_OBJECT) {
duke@435	1062	gen_write_ref_array_pre_barrier(dst, count);
duke@435	1063	}
duke@435	1064
duke@435	1065	if (entry != NULL) {
duke@435	1066	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1067	BLOCK_COMMENT("Entry:");
duke@435	1068	}
duke@435	1069
duke@435	1070	if (t == T_OBJECT) {
duke@435	1071	__ testl(count, count);
duke@435	1072	__ jcc(Assembler::zero, L_0_count);
duke@435	1073	}
never@739	1074	__ mov(from, src);
never@739	1075	__ mov(to , dst);
duke@435	1076
duke@435	1077	// arrays overlap test
duke@435	1078	RuntimeAddress nooverlap(nooverlap_target);
never@739	1079	__ cmpptr(dst, src);
never@739	1080	__ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
duke@435	1081	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1082	__ cmpptr(dst, end);
duke@435	1083	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1084
duke@435	1085	// copy from high to low
duke@435	1086	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1087	__ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
duke@435	1088	if (t == T_BYTE || t == T_SHORT) {
duke@435	1089	// Align the end of destination array at 4 bytes address boundary
never@739	1090	__ lea(end, Address(dst, count, sf, 0));
duke@435	1091	if (t == T_BYTE) {
duke@435	1092	// One byte misalignment happens only for byte arrays
duke@435	1093	__ testl(end, 1);
duke@435	1094	__ jccb(Assembler::zero, L_skip_align1);
duke@435	1095	__ decrement(count);
duke@435	1096	__ movb(rdx, Address(from, count, sf, 0));
duke@435	1097	__ movb(Address(to, count, sf, 0), rdx);
duke@435	1098	__ BIND(L_skip_align1);
duke@435	1099	}
duke@435	1100	// Two bytes misalignment happens only for byte and short (char) arrays
duke@435	1101	__ testl(end, 2);
duke@435	1102	__ jccb(Assembler::zero, L_skip_align2);
never@739	1103	__ subptr(count, 1<<(shift-1));
duke@435	1104	__ movw(rdx, Address(from, count, sf, 0));
duke@435	1105	__ movw(Address(to, count, sf, 0), rdx);
duke@435	1106	__ BIND(L_skip_align2);
duke@435	1107	__ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
duke@435	1108	__ jcc(Assembler::below, L_copy_4_bytes);
duke@435	1109	}
duke@435	1110
duke@435	1111	if (!VM_Version::supports_mmx()) {
duke@435	1112	__ std();
never@739	1113	__ mov(rax, count); // Save 'count'
never@739	1114	__ mov(rdx, to); // Save 'to'
never@739	1115	__ lea(rsi, Address(from, count, sf, -4));
never@739	1116	__ lea(rdi, Address(to , count, sf, -4));
never@739	1117	__ shrptr(count, shift); // bytes count
never@739	1118	__ rep_mov();
duke@435	1119	__ cld();
never@739	1120	__ mov(count, rax); // restore 'count'
duke@435	1121	__ andl(count, (1<<shift)-1); // mask the number of rest elements
never@739	1122	__ movptr(from, Address(rsp, 12+4)); // reread 'from'
never@739	1123	__ mov(to, rdx); // restore 'to'
duke@435	1124	__ jmpb(L_copy_2_bytes); // all dword were copied
duke@435	1125	} else {
duke@435	1126	// Align to 8 bytes the end of array. It is aligned to 4 bytes already.
never@739	1127	__ testptr(end, 4);
duke@435	1128	__ jccb(Assembler::zero, L_copy_8_bytes);
duke@435	1129	__ subl(count, 1<<shift);
duke@435	1130	__ movl(rdx, Address(from, count, sf, 0));
duke@435	1131	__ movl(Address(to, count, sf, 0), rdx);
duke@435	1132	__ jmpb(L_copy_8_bytes);
duke@435	1133
duke@435	1134	__ align(16);
duke@435	1135	// Move 8 bytes
duke@435	1136	__ BIND(L_copy_8_bytes_loop);
kvn@840	1137	if (UseXMMForArrayCopy) {
kvn@840	1138	__ movq(xmm0, Address(from, count, sf, 0));
kvn@840	1139	__ movq(Address(to, count, sf, 0), xmm0);
kvn@840	1140	} else {
kvn@840	1141	__ movq(mmx0, Address(from, count, sf, 0));
kvn@840	1142	__ movq(Address(to, count, sf, 0), mmx0);
kvn@840	1143	}
duke@435	1144	__ BIND(L_copy_8_bytes);
duke@435	1145	__ subl(count, 2<<shift);
duke@435	1146	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1147	__ addl(count, 2<<shift);
kvn@840	1148	if (!UseXMMForArrayCopy) {
kvn@840	1149	__ emms();
kvn@840	1150	}
duke@435	1151	}
duke@435	1152	__ BIND(L_copy_4_bytes);
duke@435	1153	// copy prefix qword
duke@435	1154	__ testl(count, 1<<shift);
duke@435	1155	__ jccb(Assembler::zero, L_copy_2_bytes);
duke@435	1156	__ movl(rdx, Address(from, count, sf, -4));
duke@435	1157	__ movl(Address(to, count, sf, -4), rdx);
duke@435	1158
duke@435	1159	if (t == T_BYTE || t == T_SHORT) {
duke@435	1160	__ subl(count, (1<<shift));
duke@435	1161	__ BIND(L_copy_2_bytes);
duke@435	1162	// copy prefix dword
duke@435	1163	__ testl(count, 1<<(shift-1));
duke@435	1164	__ jccb(Assembler::zero, L_copy_byte);
duke@435	1165	__ movw(rdx, Address(from, count, sf, -2));
duke@435	1166	__ movw(Address(to, count, sf, -2), rdx);
duke@435	1167	if (t == T_BYTE) {
duke@435	1168	__ subl(count, 1<<(shift-1));
duke@435	1169	__ BIND(L_copy_byte);
duke@435	1170	// copy prefix byte
duke@435	1171	__ testl(count, 1);
duke@435	1172	__ jccb(Assembler::zero, L_exit);
duke@435	1173	__ movb(rdx, Address(from, 0));
duke@435	1174	__ movb(Address(to, 0), rdx);
duke@435	1175	__ BIND(L_exit);
duke@435	1176	} else {
duke@435	1177	__ BIND(L_copy_byte);
duke@435	1178	}
duke@435	1179	} else {
duke@435	1180	__ BIND(L_copy_2_bytes);
duke@435	1181	}
duke@435	1182	if (t == T_OBJECT) {
never@739	1183	__ movl2ptr(count, Address(rsp, 12+12)); // reread count
duke@435	1184	gen_write_ref_array_post_barrier(to, count);
duke@435	1185	__ BIND(L_0_count);
duke@435	1186	}
duke@435	1187	inc_copy_counter_np(t);
never@739	1188	__ pop(rdi);
never@739	1189	__ pop(rsi);
duke@435	1190	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1191	__ xorptr(rax, rax); // return 0
duke@435	1192	__ ret(0);
duke@435	1193	return start;
duke@435	1194	}
duke@435	1195
duke@435	1196
duke@435	1197	address generate_disjoint_long_copy(address* entry, const char *name) {
duke@435	1198	__ align(CodeEntryAlignment);
duke@435	1199	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1200	address start = __ pc();
duke@435	1201
duke@435	1202	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1203	const Register from = rax; // source array address
duke@435	1204	const Register to = rdx; // destination array address
duke@435	1205	const Register count = rcx; // elements count
duke@435	1206	const Register to_from = rdx; // (to - from)
duke@435	1207
duke@435	1208	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1209	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1210	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1211	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1212
duke@435	1213	*entry = __ pc(); // Entry point from conjoint arraycopy stub.
duke@435	1214	BLOCK_COMMENT("Entry:");
duke@435	1215
never@739	1216	__ subptr(to, from); // to --> to_from
duke@435	1217	if (VM_Version::supports_mmx()) {
kvn@840	1218	if (UseXMMForArrayCopy) {
kvn@840	1219	xmm_copy_forward(from, to_from, count);
kvn@840	1220	} else {
kvn@840	1221	mmx_copy_forward(from, to_from, count);
kvn@840	1222	}
duke@435	1223	} else {
duke@435	1224	__ jmpb(L_copy_8_bytes);
duke@435	1225	__ align(16);
duke@435	1226	__ BIND(L_copy_8_bytes_loop);
duke@435	1227	__ fild_d(Address(from, 0));
duke@435	1228	__ fistp_d(Address(from, to_from, Address::times_1));
never@739	1229	__ addptr(from, 8);
duke@435	1230	__ BIND(L_copy_8_bytes);
duke@435	1231	__ decrement(count);
duke@435	1232	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1233	}
duke@435	1234	inc_copy_counter_np(T_LONG);
duke@435	1235	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1236	__ xorptr(rax, rax); // return 0
duke@435	1237	__ ret(0);
duke@435	1238	return start;
duke@435	1239	}
duke@435	1240
duke@435	1241	address generate_conjoint_long_copy(address nooverlap_target,
duke@435	1242	address* entry, const char *name) {
duke@435	1243	__ align(CodeEntryAlignment);
duke@435	1244	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1245	address start = __ pc();
duke@435	1246
duke@435	1247	Label L_copy_8_bytes, L_copy_8_bytes_loop;
duke@435	1248	const Register from = rax; // source array address
duke@435	1249	const Register to = rdx; // destination array address
duke@435	1250	const Register count = rcx; // elements count
duke@435	1251	const Register end_from = rax; // source array end address
duke@435	1252
duke@435	1253	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1254	__ movptr(from , Address(rsp, 8+0)); // from
never@739	1255	__ movptr(to , Address(rsp, 8+4)); // to
never@739	1256	__ movl2ptr(count, Address(rsp, 8+8)); // count
duke@435	1257
duke@435	1258	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1259	BLOCK_COMMENT("Entry:");
duke@435	1260
duke@435	1261	// arrays overlap test
never@739	1262	__ cmpptr(to, from);
duke@435	1263	RuntimeAddress nooverlap(nooverlap_target);
duke@435	1264	__ jump_cc(Assembler::belowEqual, nooverlap);
never@739	1265	__ lea(end_from, Address(from, count, Address::times_8, 0));
never@739	1266	__ cmpptr(to, end_from);
never@739	1267	__ movptr(from, Address(rsp, 8)); // from
duke@435	1268	__ jump_cc(Assembler::aboveEqual, nooverlap);
duke@435	1269
duke@435	1270	__ jmpb(L_copy_8_bytes);
duke@435	1271
duke@435	1272	__ align(16);
duke@435	1273	__ BIND(L_copy_8_bytes_loop);
duke@435	1274	if (VM_Version::supports_mmx()) {
kvn@840	1275	if (UseXMMForArrayCopy) {
kvn@840	1276	__ movq(xmm0, Address(from, count, Address::times_8));
kvn@840	1277	__ movq(Address(to, count, Address::times_8), xmm0);
kvn@840	1278	} else {
kvn@840	1279	__ movq(mmx0, Address(from, count, Address::times_8));
kvn@840	1280	__ movq(Address(to, count, Address::times_8), mmx0);
kvn@840	1281	}
duke@435	1282	} else {
duke@435	1283	__ fild_d(Address(from, count, Address::times_8));
duke@435	1284	__ fistp_d(Address(to, count, Address::times_8));
duke@435	1285	}
duke@435	1286	__ BIND(L_copy_8_bytes);
duke@435	1287	__ decrement(count);
duke@435	1288	__ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
duke@435	1289
kvn@840	1290	if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
duke@435	1291	__ emms();
duke@435	1292	}
duke@435	1293	inc_copy_counter_np(T_LONG);
duke@435	1294	__ leave(); // required for proper stackwalking of RuntimeStub frame
never@739	1295	__ xorptr(rax, rax); // return 0
duke@435	1296	__ ret(0);
duke@435	1297	return start;
duke@435	1298	}
duke@435	1299
duke@435	1300
duke@435	1301	// Helper for generating a dynamic type check.
duke@435	1302	// The sub_klass must be one of {rbx, rdx, rsi}.
duke@435	1303	// The temp is killed.
duke@435	1304	void generate_type_check(Register sub_klass,
duke@435	1305	Address& super_check_offset_addr,
duke@435	1306	Address& super_klass_addr,
duke@435	1307	Register temp,
jrose@1079	1308	Label* L_success, Label* L_failure) {
duke@435	1309	BLOCK_COMMENT("type_check:");
duke@435	1310
duke@435	1311	Label L_fallthrough;
jrose@1079	1312	#define LOCAL_JCC(assembler_con, label_ptr) \
jrose@1079	1313	if (label_ptr != NULL) __ jcc(assembler_con, *(label_ptr)); \
jrose@1079	1314	else __ jcc(assembler_con, L_fallthrough) /*omit semi*/
duke@435	1315
jrose@1079	1316	// The following is a strange variation of the fast path which requires
jrose@1079	1317	// one less register, because needed values are on the argument stack.
jrose@1079	1318	// __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
jrose@1079	1319	// L_success, L_failure, NULL);
duke@435	1320	assert_different_registers(sub_klass, temp);
duke@435	1321
duke@435	1322	int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1323	Klass::secondary_super_cache_offset_in_bytes());
duke@435	1324
duke@435	1325	// if the pointers are equal, we are done (e.g., String[] elements)
never@739	1326	__ cmpptr(sub_klass, super_klass_addr);
jrose@1079	1327	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1328
duke@435	1329	// check the supertype display:
never@739	1330	__ movl2ptr(temp, super_check_offset_addr);
duke@435	1331	Address super_check_addr(sub_klass, temp, Address::times_1, 0);
never@739	1332	__ movptr(temp, super_check_addr); // load displayed supertype
never@739	1333	__ cmpptr(temp, super_klass_addr); // test the super type
jrose@1079	1334	LOCAL_JCC(Assembler::equal, L_success);
duke@435	1335
duke@435	1336	// if it was a primary super, we can just fail immediately
duke@435	1337	__ cmpl(super_check_offset_addr, sc_offset);
jrose@1079	1338	LOCAL_JCC(Assembler::notEqual, L_failure);
duke@435	1339
jrose@1079	1340	// The repne_scan instruction uses fixed registers, which will get spilled.
jrose@1079	1341	// We happen to know this works best when super_klass is in rax.
jrose@1079	1342	Register super_klass = temp;
jrose@1079	1343	__ movptr(super_klass, super_klass_addr);
jrose@1079	1344	__ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
jrose@1079	1345	L_success, L_failure);
duke@435	1346
jrose@1079	1347	__ bind(L_fallthrough);
duke@435	1348
jrose@1079	1349	if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
jrose@1079	1350	if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
duke@435	1351
jrose@1079	1352	#undef LOCAL_JCC
duke@435	1353	}
duke@435	1354
duke@435	1355	//
duke@435	1356	// Generate checkcasting array copy stub
duke@435	1357	//
duke@435	1358	// Input:
duke@435	1359	// 4(rsp) - source array address
duke@435	1360	// 8(rsp) - destination array address
duke@435	1361	// 12(rsp) - element count, can be zero
duke@435	1362	// 16(rsp) - size_t ckoff (super_check_offset)
duke@435	1363	// 20(rsp) - oop ckval (super_klass)
duke@435	1364	//
duke@435	1365	// Output:
duke@435	1366	// rax, == 0 - success
duke@435	1367	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1368	//
duke@435	1369	address generate_checkcast_copy(const char *name, address* entry) {
duke@435	1370	__ align(CodeEntryAlignment);
duke@435	1371	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1372	address start = __ pc();
duke@435	1373
duke@435	1374	Label L_load_element, L_store_element, L_do_card_marks, L_done;
duke@435	1375
duke@435	1376	// register use:
duke@435	1377	// rax, rdx, rcx -- loop control (end_from, end_to, count)
duke@435	1378	// rdi, rsi -- element access (oop, klass)
duke@435	1379	// rbx, -- temp
duke@435	1380	const Register from = rax; // source array address
duke@435	1381	const Register to = rdx; // destination array address
duke@435	1382	const Register length = rcx; // elements count
duke@435	1383	const Register elem = rdi; // each oop copied
duke@435	1384	const Register elem_klass = rsi; // each elem._klass (sub_klass)
duke@435	1385	const Register temp = rbx; // lone remaining temp
duke@435	1386
duke@435	1387	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	1388
never@739	1389	__ push(rsi);
never@739	1390	__ push(rdi);
never@739	1391	__ push(rbx);
duke@435	1392
duke@435	1393	Address from_arg(rsp, 16+ 4); // from
duke@435	1394	Address to_arg(rsp, 16+ 8); // to
duke@435	1395	Address length_arg(rsp, 16+12); // elements count
duke@435	1396	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1397	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1398
duke@435	1399	// Load up:
never@739	1400	__ movptr(from, from_arg);
never@739	1401	__ movptr(to, to_arg);
never@739	1402	__ movl2ptr(length, length_arg);
duke@435	1403
duke@435	1404	*entry = __ pc(); // Entry point from generic arraycopy stub.
duke@435	1405	BLOCK_COMMENT("Entry:");
duke@435	1406
duke@435	1407	//---------------------------------------------------------------
duke@435	1408	// Assembler stub will be used for this call to arraycopy
duke@435	1409	// if the two arrays are subtypes of Object[] but the
duke@435	1410	// destination array type is not equal to or a supertype
duke@435	1411	// of the source type. Each element must be separately
duke@435	1412	// checked.
duke@435	1413
duke@435	1414	// Loop-invariant addresses. They are exclusive end pointers.
never@739	1415	Address end_from_addr(from, length, Address::times_ptr, 0);
never@739	1416	Address end_to_addr(to, length, Address::times_ptr, 0);
duke@435	1417
duke@435	1418	Register end_from = from; // re-use
duke@435	1419	Register end_to = to; // re-use
duke@435	1420	Register count = length; // re-use
duke@435	1421
duke@435	1422	// Loop-variant addresses. They assume post-incremented count < 0.
never@739	1423	Address from_element_addr(end_from, count, Address::times_ptr, 0);
never@739	1424	Address to_element_addr(end_to, count, Address::times_ptr, 0);
duke@435	1425	Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
duke@435	1426
duke@435	1427	// Copy from low to high addresses, indexed from the end of each array.
ysr@777	1428	gen_write_ref_array_pre_barrier(to, count);
never@739	1429	__ lea(end_from, end_from_addr);
never@739	1430	__ lea(end_to, end_to_addr);
duke@435	1431	assert(length == count, ""); // else fix next line:
never@739	1432	__ negptr(count); // negate and test the length
duke@435	1433	__ jccb(Assembler::notZero, L_load_element);
duke@435	1434
duke@435	1435	// Empty array: Nothing to do.
never@739	1436	__ xorptr(rax, rax); // return 0 on (trivial) success
duke@435	1437	__ jmp(L_done);
duke@435	1438
duke@435	1439	// ======== begin loop ========
duke@435	1440	// (Loop is rotated; its entry is L_load_element.)
duke@435	1441	// Loop control:
duke@435	1442	// for (count = -count; count != 0; count++)
duke@435	1443	// Base pointers src, dst are biased by 8*count,to last element.
duke@435	1444	__ align(16);
duke@435	1445
duke@435	1446	__ BIND(L_store_element);
never@739	1447	__ movptr(to_element_addr, elem); // store the oop
duke@435	1448	__ increment(count); // increment the count toward zero
duke@435	1449	__ jccb(Assembler::zero, L_do_card_marks);
duke@435	1450
duke@435	1451	// ======== loop entry is here ========
duke@435	1452	__ BIND(L_load_element);
never@739	1453	__ movptr(elem, from_element_addr); // load the oop
never@739	1454	__ testptr(elem, elem);
duke@435	1455	__ jccb(Assembler::zero, L_store_element);
duke@435	1456
duke@435	1457	// (Could do a trick here: Remember last successful non-null
duke@435	1458	// element stored and make a quick oop equality check on it.)
duke@435	1459
never@739	1460	__ movptr(elem_klass, elem_klass_addr); // query the object klass
duke@435	1461	generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
duke@435	1462	&L_store_element, NULL);
duke@435	1463	// (On fall-through, we have failed the element type check.)
duke@435	1464	// ======== end loop ========
duke@435	1465
duke@435	1466	// It was a real error; we must depend on the caller to finish the job.
rasbold@454	1467	// Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
rasbold@454	1468	// Emit GC store barriers for the oops we have copied (length_arg + count),
duke@435	1469	// and report their number to the caller.
duke@435	1470	__ addl(count, length_arg); // transfers = (length - remaining)
never@739	1471	__ movl2ptr(rax, count); // save the value
never@739	1472	__ notptr(rax); // report (-1^K) to caller
never@739	1473	__ movptr(to, to_arg); // reload
duke@435	1474	assert_different_registers(to, count, rax);
duke@435	1475	gen_write_ref_array_post_barrier(to, count);
duke@435	1476	__ jmpb(L_done);
duke@435	1477
duke@435	1478	// Come here on success only.
duke@435	1479	__ BIND(L_do_card_marks);
never@739	1480	__ movl2ptr(count, length_arg);
never@739	1481	__ movptr(to, to_arg); // reload
duke@435	1482	gen_write_ref_array_post_barrier(to, count);
never@739	1483	__ xorptr(rax, rax); // return 0 on success
duke@435	1484
duke@435	1485	// Common exit point (success or failure).
duke@435	1486	__ BIND(L_done);
never@739	1487	__ pop(rbx);
never@739	1488	__ pop(rdi);
never@739	1489	__ pop(rsi);
duke@435	1490	inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
duke@435	1491	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1492	__ ret(0);
duke@435	1493
duke@435	1494	return start;
duke@435	1495	}
duke@435	1496
duke@435	1497	//
duke@435	1498	// Generate 'unsafe' array copy stub
duke@435	1499	// Though just as safe as the other stubs, it takes an unscaled
duke@435	1500	// size_t argument instead of an element count.
duke@435	1501	//
duke@435	1502	// Input:
duke@435	1503	// 4(rsp) - source array address
duke@435	1504	// 8(rsp) - destination array address
duke@435	1505	// 12(rsp) - byte count, can be zero
duke@435	1506	//
duke@435	1507	// Output:
duke@435	1508	// rax, == 0 - success
duke@435	1509	// rax, == -1 - need to call System.arraycopy
duke@435	1510	//
duke@435	1511	// Examines the alignment of the operands and dispatches
duke@435	1512	// to a long, int, short, or byte copy loop.
duke@435	1513	//
duke@435	1514	address generate_unsafe_copy(const char *name,
duke@435	1515	address byte_copy_entry,
duke@435	1516	address short_copy_entry,
duke@435	1517	address int_copy_entry,
duke@435	1518	address long_copy_entry) {
duke@435	1519
duke@435	1520	Label L_long_aligned, L_int_aligned, L_short_aligned;
duke@435	1521
duke@435	1522	__ align(CodeEntryAlignment);
duke@435	1523	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1524	address start = __ pc();
duke@435	1525
duke@435	1526	const Register from = rax; // source array address
duke@435	1527	const Register to = rdx; // destination array address
duke@435	1528	const Register count = rcx; // elements count
duke@435	1529
duke@435	1530	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1531	__ push(rsi);
never@739	1532	__ push(rdi);
duke@435	1533	Address from_arg(rsp, 12+ 4); // from
duke@435	1534	Address to_arg(rsp, 12+ 8); // to
duke@435	1535	Address count_arg(rsp, 12+12); // byte count
duke@435	1536
duke@435	1537	// Load up:
never@739	1538	__ movptr(from , from_arg);
never@739	1539	__ movptr(to , to_arg);
never@739	1540	__ movl2ptr(count, count_arg);
duke@435	1541
duke@435	1542	// bump this on entry, not on exit:
duke@435	1543	inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
duke@435	1544
duke@435	1545	const Register bits = rsi;
never@739	1546	__ mov(bits, from);
never@739	1547	__ orptr(bits, to);
never@739	1548	__ orptr(bits, count);
duke@435	1549
duke@435	1550	__ testl(bits, BytesPerLong-1);
duke@435	1551	__ jccb(Assembler::zero, L_long_aligned);
duke@435	1552
duke@435	1553	__ testl(bits, BytesPerInt-1);
duke@435	1554	__ jccb(Assembler::zero, L_int_aligned);
duke@435	1555
duke@435	1556	__ testl(bits, BytesPerShort-1);
duke@435	1557	__ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
duke@435	1558
duke@435	1559	__ BIND(L_short_aligned);
never@739	1560	__ shrptr(count, LogBytesPerShort); // size => short_count
duke@435	1561	__ movl(count_arg, count); // update 'count'
duke@435	1562	__ jump(RuntimeAddress(short_copy_entry));
duke@435	1563
duke@435	1564	__ BIND(L_int_aligned);
never@739	1565	__ shrptr(count, LogBytesPerInt); // size => int_count
duke@435	1566	__ movl(count_arg, count); // update 'count'
duke@435	1567	__ jump(RuntimeAddress(int_copy_entry));
duke@435	1568
duke@435	1569	__ BIND(L_long_aligned);
never@739	1570	__ shrptr(count, LogBytesPerLong); // size => qword_count
duke@435	1571	__ movl(count_arg, count); // update 'count'
never@739	1572	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1573	__ pop(rsi);
duke@435	1574	__ jump(RuntimeAddress(long_copy_entry));
duke@435	1575
duke@435	1576	return start;
duke@435	1577	}
duke@435	1578
duke@435	1579
duke@435	1580	// Perform range checks on the proposed arraycopy.
duke@435	1581	// Smashes src_pos and dst_pos. (Uses them up for temps.)
duke@435	1582	void arraycopy_range_checks(Register src,
duke@435	1583	Register src_pos,
duke@435	1584	Register dst,
duke@435	1585	Register dst_pos,
duke@435	1586	Address& length,
duke@435	1587	Label& L_failed) {
duke@435	1588	BLOCK_COMMENT("arraycopy_range_checks:");
duke@435	1589	const Register src_end = src_pos; // source array end position
duke@435	1590	const Register dst_end = dst_pos; // destination array end position
duke@435	1591	__ addl(src_end, length); // src_pos + length
duke@435	1592	__ addl(dst_end, length); // dst_pos + length
duke@435	1593
duke@435	1594	// if (src_pos + length > arrayOop(src)->length() ) FAIL;
duke@435	1595	__ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
duke@435	1596	__ jcc(Assembler::above, L_failed);
duke@435	1597
duke@435	1598	// if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
duke@435	1599	__ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
duke@435	1600	__ jcc(Assembler::above, L_failed);
duke@435	1601
duke@435	1602	BLOCK_COMMENT("arraycopy_range_checks done");
duke@435	1603	}
duke@435	1604
duke@435	1605
duke@435	1606	//
duke@435	1607	// Generate generic array copy stubs
duke@435	1608	//
duke@435	1609	// Input:
duke@435	1610	// 4(rsp) - src oop
duke@435	1611	// 8(rsp) - src_pos
duke@435	1612	// 12(rsp) - dst oop
duke@435	1613	// 16(rsp) - dst_pos
duke@435	1614	// 20(rsp) - element count
duke@435	1615	//
duke@435	1616	// Output:
duke@435	1617	// rax, == 0 - success
duke@435	1618	// rax, == -1^K - failure, where K is partial transfer count
duke@435	1619	//
duke@435	1620	address generate_generic_copy(const char *name,
duke@435	1621	address entry_jbyte_arraycopy,
duke@435	1622	address entry_jshort_arraycopy,
duke@435	1623	address entry_jint_arraycopy,
duke@435	1624	address entry_oop_arraycopy,
duke@435	1625	address entry_jlong_arraycopy,
duke@435	1626	address entry_checkcast_arraycopy) {
duke@435	1627	Label L_failed, L_failed_0, L_objArray;
duke@435	1628
duke@435	1629	{ int modulus = CodeEntryAlignment;
duke@435	1630	int target = modulus - 5; // 5 = sizeof jmp(L_failed)
duke@435	1631	int advance = target - (__ offset() % modulus);
duke@435	1632	if (advance < 0) advance += modulus;
duke@435	1633	if (advance > 0) __ nop(advance);
duke@435	1634	}
duke@435	1635	StubCodeMark mark(this, "StubRoutines", name);
duke@435	1636
duke@435	1637	// Short-hop target to L_failed. Makes for denser prologue code.
duke@435	1638	__ BIND(L_failed_0);
duke@435	1639	__ jmp(L_failed);
duke@435	1640	assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
duke@435	1641
duke@435	1642	__ align(CodeEntryAlignment);
duke@435	1643	address start = __ pc();
duke@435	1644
duke@435	1645	__ enter(); // required for proper stackwalking of RuntimeStub frame
never@739	1646	__ push(rsi);
never@739	1647	__ push(rdi);
duke@435	1648
duke@435	1649	// bump this on entry, not on exit:
duke@435	1650	inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
duke@435	1651
duke@435	1652	// Input values
duke@435	1653	Address SRC (rsp, 12+ 4);
duke@435	1654	Address SRC_POS (rsp, 12+ 8);
duke@435	1655	Address DST (rsp, 12+12);
duke@435	1656	Address DST_POS (rsp, 12+16);
duke@435	1657	Address LENGTH (rsp, 12+20);
duke@435	1658
duke@435	1659	//-----------------------------------------------------------------------
duke@435	1660	// Assembler stub will be used for this call to arraycopy
duke@435	1661	// if the following conditions are met:
duke@435	1662	//
duke@435	1663	// (1) src and dst must not be null.
duke@435	1664	// (2) src_pos must not be negative.
duke@435	1665	// (3) dst_pos must not be negative.
duke@435	1666	// (4) length must not be negative.
duke@435	1667	// (5) src klass and dst klass should be the same and not NULL.
duke@435	1668	// (6) src and dst should be arrays.
duke@435	1669	// (7) src_pos + length must not exceed length of src.
duke@435	1670	// (8) dst_pos + length must not exceed length of dst.
duke@435	1671	//
duke@435	1672
duke@435	1673	const Register src = rax; // source array oop
duke@435	1674	const Register src_pos = rsi;
duke@435	1675	const Register dst = rdx; // destination array oop
duke@435	1676	const Register dst_pos = rdi;
duke@435	1677	const Register length = rcx; // transfer count
duke@435	1678
duke@435	1679	// if (src == NULL) return -1;
never@739	1680	__ movptr(src, SRC); // src oop
never@739	1681	__ testptr(src, src);
duke@435	1682	__ jccb(Assembler::zero, L_failed_0);
duke@435	1683
duke@435	1684	// if (src_pos < 0) return -1;
never@739	1685	__ movl2ptr(src_pos, SRC_POS); // src_pos
duke@435	1686	__ testl(src_pos, src_pos);
duke@435	1687	__ jccb(Assembler::negative, L_failed_0);
duke@435	1688
duke@435	1689	// if (dst == NULL) return -1;
never@739	1690	__ movptr(dst, DST); // dst oop
never@739	1691	__ testptr(dst, dst);
duke@435	1692	__ jccb(Assembler::zero, L_failed_0);
duke@435	1693
duke@435	1694	// if (dst_pos < 0) return -1;
never@739	1695	__ movl2ptr(dst_pos, DST_POS); // dst_pos
duke@435	1696	__ testl(dst_pos, dst_pos);
duke@435	1697	__ jccb(Assembler::negative, L_failed_0);
duke@435	1698
duke@435	1699	// if (length < 0) return -1;
never@739	1700	__ movl2ptr(length, LENGTH); // length
duke@435	1701	__ testl(length, length);
duke@435	1702	__ jccb(Assembler::negative, L_failed_0);
duke@435	1703
duke@435	1704	// if (src->klass() == NULL) return -1;
duke@435	1705	Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
duke@435	1706	Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
duke@435	1707	const Register rcx_src_klass = rcx; // array klass
never@739	1708	__ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
duke@435	1709
duke@435	1710	#ifdef ASSERT
duke@435	1711	// assert(src->klass() != NULL);
duke@435	1712	BLOCK_COMMENT("assert klasses not null");
duke@435	1713	{ Label L1, L2;
never@739	1714	__ testptr(rcx_src_klass, rcx_src_klass);
duke@435	1715	__ jccb(Assembler::notZero, L2); // it is broken if klass is NULL
duke@435	1716	__ bind(L1);
duke@435	1717	__ stop("broken null klass");
duke@435	1718	__ bind(L2);
never@739	1719	__ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
duke@435	1720	__ jccb(Assembler::equal, L1); // this would be broken also
duke@435	1721	BLOCK_COMMENT("assert done");
duke@435	1722	}
duke@435	1723	#endif //ASSERT
duke@435	1724
duke@435	1725	// Load layout helper (32-bits)
duke@435	1726	//
duke@435	1727	// |array_tag| | header_size | element_type | |log2_element_size|
duke@435	1728	// 32 30 24 16 8 2 0
duke@435	1729	//
duke@435	1730	// array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
duke@435	1731	//
duke@435	1732
duke@435	1733	int lh_offset = klassOopDesc::header_size() * HeapWordSize +
duke@435	1734	Klass::layout_helper_offset_in_bytes();
duke@435	1735	Address src_klass_lh_addr(rcx_src_klass, lh_offset);
duke@435	1736
duke@435	1737	// Handle objArrays completely differently...
duke@435	1738	jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
duke@435	1739	__ cmpl(src_klass_lh_addr, objArray_lh);
duke@435	1740	__ jcc(Assembler::equal, L_objArray);
duke@435	1741
duke@435	1742	// if (src->klass() != dst->klass()) return -1;
never@739	1743	__ cmpptr(rcx_src_klass, dst_klass_addr);
duke@435	1744	__ jccb(Assembler::notEqual, L_failed_0);
duke@435	1745
duke@435	1746	const Register rcx_lh = rcx; // layout helper
duke@435	1747	assert(rcx_lh == rcx_src_klass, "known alias");
duke@435	1748	__ movl(rcx_lh, src_klass_lh_addr);
duke@435	1749
duke@435	1750	// if (!src->is_Array()) return -1;
duke@435	1751	__ cmpl(rcx_lh, Klass::_lh_neutral_value);
duke@435	1752	__ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
duke@435	1753
duke@435	1754	// At this point, it is known to be a typeArray (array_tag 0x3).
duke@435	1755	#ifdef ASSERT
duke@435	1756	{ Label L;
duke@435	1757	__ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
duke@435	1758	__ jcc(Assembler::greaterEqual, L); // signed cmp
duke@435	1759	__ stop("must be a primitive array");
duke@435	1760	__ bind(L);
duke@435	1761	}
duke@435	1762	#endif
duke@435	1763
duke@435	1764	assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
duke@435	1765	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1766
duke@435	1767	// typeArrayKlass
duke@435	1768	//
duke@435	1769	// src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
duke@435	1770	// dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
duke@435	1771	//
duke@435	1772	const Register rsi_offset = rsi; // array offset
duke@435	1773	const Register src_array = src; // src array offset
duke@435	1774	const Register dst_array = dst; // dst array offset
duke@435	1775	const Register rdi_elsize = rdi; // log2 element size
duke@435	1776
never@739	1777	__ mov(rsi_offset, rcx_lh);
never@739	1778	__ shrptr(rsi_offset, Klass::_lh_header_size_shift);
never@739	1779	__ andptr(rsi_offset, Klass::_lh_header_size_mask); // array_offset
never@739	1780	__ addptr(src_array, rsi_offset); // src array offset
never@739	1781	__ addptr(dst_array, rsi_offset); // dst array offset
never@739	1782	__ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
duke@435	1783
duke@435	1784	// next registers should be set before the jump to corresponding stub
duke@435	1785	const Register from = src; // source array address
duke@435	1786	const Register to = dst; // destination array address
duke@435	1787	const Register count = rcx; // elements count
duke@435	1788	// some of them should be duplicated on stack
duke@435	1789	#define FROM Address(rsp, 12+ 4)
duke@435	1790	#define TO Address(rsp, 12+ 8) // Not used now
duke@435	1791	#define COUNT Address(rsp, 12+12) // Only for oop arraycopy
duke@435	1792
duke@435	1793	BLOCK_COMMENT("scale indexes to element size");
never@739	1794	__ movl2ptr(rsi, SRC_POS); // src_pos
never@739	1795	__ shlptr(rsi); // src_pos << rcx (log2 elsize)
duke@435	1796	assert(src_array == from, "");
never@739	1797	__ addptr(from, rsi); // from = src_array + SRC_POS << log2 elsize
never@739	1798	__ movl2ptr(rdi, DST_POS); // dst_pos
never@739	1799	__ shlptr(rdi); // dst_pos << rcx (log2 elsize)
duke@435	1800	assert(dst_array == to, "");
never@739	1801	__ addptr(to, rdi); // to = dst_array + DST_POS << log2 elsize
never@739	1802	__ movptr(FROM, from); // src_addr
never@739	1803	__ mov(rdi_elsize, rcx_lh); // log2 elsize
never@739	1804	__ movl2ptr(count, LENGTH); // elements count
duke@435	1805
duke@435	1806	BLOCK_COMMENT("choose copy loop based on element size");
duke@435	1807	__ cmpl(rdi_elsize, 0);
duke@435	1808
duke@435	1809	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
duke@435	1810	__ cmpl(rdi_elsize, LogBytesPerShort);
duke@435	1811	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
duke@435	1812	__ cmpl(rdi_elsize, LogBytesPerInt);
duke@435	1813	__ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
duke@435	1814	#ifdef ASSERT
duke@435	1815	__ cmpl(rdi_elsize, LogBytesPerLong);
duke@435	1816	__ jccb(Assembler::notEqual, L_failed);
duke@435	1817	#endif
never@739	1818	__ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
never@739	1819	__ pop(rsi);
duke@435	1820	__ jump(RuntimeAddress(entry_jlong_arraycopy));
duke@435	1821
duke@435	1822	__ BIND(L_failed);
never@739	1823	__ xorptr(rax, rax);
never@739	1824	__ notptr(rax); // return -1
never@739	1825	__ pop(rdi);
never@739	1826	__ pop(rsi);
duke@435	1827	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	1828	__ ret(0);
duke@435	1829
duke@435	1830	// objArrayKlass
duke@435	1831	__ BIND(L_objArray);
duke@435	1832	// live at this point: rcx_src_klass, src[_pos], dst[_pos]
duke@435	1833
duke@435	1834	Label L_plain_copy, L_checkcast_copy;
duke@435	1835	// test array classes for subtyping
never@739	1836	__ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
duke@435	1837	__ jccb(Assembler::notEqual, L_checkcast_copy);
duke@435	1838
duke@435	1839	// Identically typed arrays can be copied without element-wise checks.
duke@435	1840	assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
duke@435	1841	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1842
duke@435	1843	__ BIND(L_plain_copy);
never@739	1844	__ movl2ptr(count, LENGTH); // elements count
never@739	1845	__ movl2ptr(src_pos, SRC_POS); // reload src_pos
never@739	1846	__ lea(from, Address(src, src_pos, Address::times_ptr,
never@739	1847	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
never@739	1848	__ movl2ptr(dst_pos, DST_POS); // reload dst_pos
never@739	1849	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
never@739	1850	arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
never@739	1851	__ movptr(FROM, from); // src_addr
never@739	1852	__ movptr(TO, to); // dst_addr
duke@435	1853	__ movl(COUNT, count); // count
duke@435	1854	__ jump(RuntimeAddress(entry_oop_arraycopy));
duke@435	1855
duke@435	1856	__ BIND(L_checkcast_copy);
duke@435	1857	// live at this point: rcx_src_klass, dst[_pos], src[_pos]
duke@435	1858	{
duke@435	1859	// Handy offsets:
duke@435	1860	int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1861	objArrayKlass::element_klass_offset_in_bytes());
duke@435	1862	int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
duke@435	1863	Klass::super_check_offset_offset_in_bytes());
duke@435	1864
duke@435	1865	Register rsi_dst_klass = rsi;
duke@435	1866	Register rdi_temp = rdi;
duke@435	1867	assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
duke@435	1868	assert(rdi_temp == dst_pos, "expected alias w/ dst_pos");
duke@435	1869	Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
duke@435	1870
duke@435	1871	// Before looking at dst.length, make sure dst is also an objArray.
never@739	1872	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1873	__ cmpl(dst_klass_lh_addr, objArray_lh);
duke@435	1874	__ jccb(Assembler::notEqual, L_failed);
duke@435	1875
duke@435	1876	// It is safe to examine both src.length and dst.length.
never@739	1877	__ movl2ptr(src_pos, SRC_POS); // reload rsi
duke@435	1878	arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
duke@435	1879	// (Now src_pos and dst_pos are killed, but not src and dst.)
duke@435	1880
duke@435	1881	// We'll need this temp (don't forget to pop it after the type check).
never@739	1882	__ push(rbx);
duke@435	1883	Register rbx_src_klass = rbx;
duke@435	1884
never@739	1885	__ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
never@739	1886	__ movptr(rsi_dst_klass, dst_klass_addr);
duke@435	1887	Address super_check_offset_addr(rsi_dst_klass, sco_offset);
duke@435	1888	Label L_fail_array_check;
duke@435	1889	generate_type_check(rbx_src_klass,
duke@435	1890	super_check_offset_addr, dst_klass_addr,
duke@435	1891	rdi_temp, NULL, &L_fail_array_check);
duke@435	1892	// (On fall-through, we have passed the array type check.)
never@739	1893	__ pop(rbx);
duke@435	1894	__ jmp(L_plain_copy);
duke@435	1895
duke@435	1896	__ BIND(L_fail_array_check);
duke@435	1897	// Reshuffle arguments so we can call checkcast_arraycopy:
duke@435	1898
duke@435	1899	// match initial saves for checkcast_arraycopy
never@739	1900	// push(rsi); // already done; see above
never@739	1901	// push(rdi); // already done; see above
never@739	1902	// push(rbx); // already done; see above
duke@435	1903
duke@435	1904	// Marshal outgoing arguments now, freeing registers.
duke@435	1905	Address from_arg(rsp, 16+ 4); // from
duke@435	1906	Address to_arg(rsp, 16+ 8); // to
duke@435	1907	Address length_arg(rsp, 16+12); // elements count
duke@435	1908	Address ckoff_arg(rsp, 16+16); // super_check_offset
duke@435	1909	Address ckval_arg(rsp, 16+20); // super_klass
duke@435	1910
duke@435	1911	Address SRC_POS_arg(rsp, 16+ 8);
duke@435	1912	Address DST_POS_arg(rsp, 16+16);
duke@435	1913	Address LENGTH_arg(rsp, 16+20);
duke@435	1914	// push rbx, changed the incoming offsets (why not just use rbp,??)
duke@435	1915	// assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
duke@435	1916
never@739	1917	__ movptr(rbx, Address(rsi_dst_klass, ek_offset));
never@739	1918	__ movl2ptr(length, LENGTH_arg); // reload elements count
never@739	1919	__ movl2ptr(src_pos, SRC_POS_arg); // reload src_pos
never@739	1920	__ movl2ptr(dst_pos, DST_POS_arg); // reload dst_pos
duke@435	1921
never@739	1922	__ movptr(ckval_arg, rbx); // destination element type
duke@435	1923	__ movl(rbx, Address(rbx, sco_offset));
duke@435	1924	__ movl(ckoff_arg, rbx); // corresponding class check offset
duke@435	1925
duke@435	1926	__ movl(length_arg, length); // outgoing length argument
duke@435	1927
never@739	1928	__ lea(from, Address(src, src_pos, Address::times_ptr,
duke@435	1929	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1930	__ movptr(from_arg, from);
duke@435	1931
never@739	1932	__ lea(to, Address(dst, dst_pos, Address::times_ptr,
duke@435	1933	arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
never@739	1934	__ movptr(to_arg, to);
duke@435	1935	__ jump(RuntimeAddress(entry_checkcast_arraycopy));
duke@435	1936	}
duke@435	1937
duke@435	1938	return start;
duke@435	1939	}
duke@435	1940
duke@435	1941	void generate_arraycopy_stubs() {
duke@435	1942	address entry;
duke@435	1943	address entry_jbyte_arraycopy;
duke@435	1944	address entry_jshort_arraycopy;
duke@435	1945	address entry_jint_arraycopy;
duke@435	1946	address entry_oop_arraycopy;
duke@435	1947	address entry_jlong_arraycopy;
duke@435	1948	address entry_checkcast_arraycopy;
duke@435	1949
duke@435	1950	StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
duke@435	1951	generate_disjoint_copy(T_BYTE, true, Address::times_1, &entry,
duke@435	1952	"arrayof_jbyte_disjoint_arraycopy");
duke@435	1953	StubRoutines::_arrayof_jbyte_arraycopy =
duke@435	1954	generate_conjoint_copy(T_BYTE, true, Address::times_1, entry,
duke@435	1955	NULL, "arrayof_jbyte_arraycopy");
duke@435	1956	StubRoutines::_jbyte_disjoint_arraycopy =
duke@435	1957	generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
duke@435	1958	"jbyte_disjoint_arraycopy");
duke@435	1959	StubRoutines::_jbyte_arraycopy =
duke@435	1960	generate_conjoint_copy(T_BYTE, false, Address::times_1, entry,
duke@435	1961	&entry_jbyte_arraycopy, "jbyte_arraycopy");
duke@435	1962
duke@435	1963	StubRoutines::_arrayof_jshort_disjoint_arraycopy =
duke@435	1964	generate_disjoint_copy(T_SHORT, true, Address::times_2, &entry,
duke@435	1965	"arrayof_jshort_disjoint_arraycopy");
duke@435	1966	StubRoutines::_arrayof_jshort_arraycopy =
duke@435	1967	generate_conjoint_copy(T_SHORT, true, Address::times_2, entry,
duke@435	1968	NULL, "arrayof_jshort_arraycopy");
duke@435	1969	StubRoutines::_jshort_disjoint_arraycopy =
duke@435	1970	generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
duke@435	1971	"jshort_disjoint_arraycopy");
duke@435	1972	StubRoutines::_jshort_arraycopy =
duke@435	1973	generate_conjoint_copy(T_SHORT, false, Address::times_2, entry,
duke@435	1974	&entry_jshort_arraycopy, "jshort_arraycopy");
duke@435	1975
duke@435	1976	// Next arrays are always aligned on 4 bytes at least.
duke@435	1977	StubRoutines::_jint_disjoint_arraycopy =
duke@435	1978	generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
duke@435	1979	"jint_disjoint_arraycopy");
duke@435	1980	StubRoutines::_jint_arraycopy =
duke@435	1981	generate_conjoint_copy(T_INT, true, Address::times_4, entry,
duke@435	1982	&entry_jint_arraycopy, "jint_arraycopy");
duke@435	1983
duke@435	1984	StubRoutines::_oop_disjoint_arraycopy =
never@739	1985	generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
duke@435	1986	"oop_disjoint_arraycopy");
duke@435	1987	StubRoutines::_oop_arraycopy =
never@739	1988	generate_conjoint_copy(T_OBJECT, true, Address::times_ptr, entry,
duke@435	1989	&entry_oop_arraycopy, "oop_arraycopy");
duke@435	1990
duke@435	1991	StubRoutines::_jlong_disjoint_arraycopy =
duke@435	1992	generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
duke@435	1993	StubRoutines::_jlong_arraycopy =
duke@435	1994	generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
duke@435	1995	"jlong_arraycopy");
duke@435	1996
duke@435	1997	StubRoutines::_arrayof_jint_disjoint_arraycopy =
duke@435	1998	StubRoutines::_jint_disjoint_arraycopy;
duke@435	1999	StubRoutines::_arrayof_oop_disjoint_arraycopy =
duke@435	2000	StubRoutines::_oop_disjoint_arraycopy;
duke@435	2001	StubRoutines::_arrayof_jlong_disjoint_arraycopy =
duke@435	2002	StubRoutines::_jlong_disjoint_arraycopy;
duke@435	2003
duke@435	2004	StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy;
duke@435	2005	StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy;
duke@435	2006	StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;
duke@435	2007
duke@435	2008	StubRoutines::_checkcast_arraycopy =
duke@435	2009	generate_checkcast_copy("checkcast_arraycopy",
duke@435	2010	&entry_checkcast_arraycopy);
duke@435	2011
duke@435	2012	StubRoutines::_unsafe_arraycopy =
duke@435	2013	generate_unsafe_copy("unsafe_arraycopy",
duke@435	2014	entry_jbyte_arraycopy,
duke@435	2015	entry_jshort_arraycopy,
duke@435	2016	entry_jint_arraycopy,
duke@435	2017	entry_jlong_arraycopy);
duke@435	2018
duke@435	2019	StubRoutines::_generic_arraycopy =
duke@435	2020	generate_generic_copy("generic_arraycopy",
duke@435	2021	entry_jbyte_arraycopy,
duke@435	2022	entry_jshort_arraycopy,
duke@435	2023	entry_jint_arraycopy,
duke@435	2024	entry_oop_arraycopy,
duke@435	2025	entry_jlong_arraycopy,
duke@435	2026	entry_checkcast_arraycopy);
duke@435	2027	}
duke@435	2028
never@1609	2029	void generate_math_stubs() {
never@1609	2030	{
never@1609	2031	StubCodeMark mark(this, "StubRoutines", "log");
never@1609	2032	StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
never@1609	2033
never@1609	2034	__ fld_d(Address(rsp, 4));
never@1609	2035	__ flog();
never@1609	2036	__ ret(0);
never@1609	2037	}
never@1609	2038	{
never@1609	2039	StubCodeMark mark(this, "StubRoutines", "log10");
never@1609	2040	StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
never@1609	2041
never@1609	2042	__ fld_d(Address(rsp, 4));
never@1609	2043	__ flog10();
never@1609	2044	__ ret(0);
never@1609	2045	}
never@1609	2046	{
never@1609	2047	StubCodeMark mark(this, "StubRoutines", "sin");
never@1609	2048	StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
never@1609	2049
never@1609	2050	__ fld_d(Address(rsp, 4));
never@1609	2051	__ trigfunc('s');
never@1609	2052	__ ret(0);
never@1609	2053	}
never@1609	2054	{
never@1609	2055	StubCodeMark mark(this, "StubRoutines", "cos");
never@1609	2056	StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
never@1609	2057
never@1609	2058	__ fld_d(Address(rsp, 4));
never@1609	2059	__ trigfunc('c');
never@1609	2060	__ ret(0);
never@1609	2061	}
never@1609	2062	{
never@1609	2063	StubCodeMark mark(this, "StubRoutines", "tan");
never@1609	2064	StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
never@1609	2065
never@1609	2066	__ fld_d(Address(rsp, 4));
never@1609	2067	__ trigfunc('t');
never@1609	2068	__ ret(0);
never@1609	2069	}
never@1609	2070
never@1609	2071	// The intrinsic version of these seem to return the same value as
never@1609	2072	// the strict version.
never@1609	2073	StubRoutines::_intrinsic_exp = SharedRuntime::dexp;
never@1609	2074	StubRoutines::_intrinsic_pow = SharedRuntime::dpow;
never@1609	2075	}
never@1609	2076
duke@435	2077	public:
duke@435	2078	// Information about frame layout at time of blocking runtime call.
duke@435	2079	// Note that we only have to preserve callee-saved registers since
duke@435	2080	// the compilers are responsible for supplying a continuation point
duke@435	2081	// if they expect all registers to be preserved.
duke@435	2082	enum layout {
duke@435	2083	thread_off, // last_java_sp
duke@435	2084	rbp_off, // callee saved register
duke@435	2085	ret_pc,
duke@435	2086	framesize
duke@435	2087	};
duke@435	2088
duke@435	2089	private:
duke@435	2090
duke@435	2091	#undef __
duke@435	2092	#define __ masm->
duke@435	2093
duke@435	2094	//------------------------------------------------------------------------------------------------------------------------
duke@435	2095	// Continuation point for throwing of implicit exceptions that are not handled in
duke@435	2096	// the current activation. Fabricates an exception oop and initiates normal
duke@435	2097	// exception dispatching in this frame.
duke@435	2098	//
duke@435	2099	// Previously the compiler (c2) allowed for callee save registers on Java calls.
duke@435	2100	// This is no longer true after adapter frames were removed but could possibly
duke@435	2101	// be brought back in the future if the interpreter code was reworked and it
duke@435	2102	// was deemed worthwhile. The comment below was left to describe what must
duke@435	2103	// happen here if callee saves were resurrected. As it stands now this stub
duke@435	2104	// could actually be a vanilla BufferBlob and have now oopMap at all.
duke@435	2105	// Since it doesn't make much difference we've chosen to leave it the
duke@435	2106	// way it was in the callee save days and keep the comment.
duke@435	2107
duke@435	2108	// If we need to preserve callee-saved values we need a callee-saved oop map and
duke@435	2109	// therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
duke@435	2110	// If the compiler needs all registers to be preserved between the fault
duke@435	2111	// point and the exception handler then it must assume responsibility for that in
duke@435	2112	// AbstractCompiler::continuation_for_implicit_null_exception or
duke@435	2113	// continuation_for_implicit_division_by_zero_exception. All other implicit
duke@435	2114	// exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
duke@435	2115	// either at call sites or otherwise assume that stack unwinding will be initiated,
duke@435	2116	// so caller saved registers were assumed volatile in the compiler.
duke@435	2117	address generate_throw_exception(const char* name, address runtime_entry,
duke@435	2118	bool restore_saved_exception_pc) {
duke@435	2119
duke@435	2120	int insts_size = 256;
duke@435	2121	int locs_size = 32;
duke@435	2122
duke@435	2123	CodeBuffer code(name, insts_size, locs_size);
duke@435	2124	OopMapSet* oop_maps = new OopMapSet();
duke@435	2125	MacroAssembler* masm = new MacroAssembler(&code);
duke@435	2126
duke@435	2127	address start = __ pc();
duke@435	2128
duke@435	2129	// This is an inlined and slightly modified version of call_VM
duke@435	2130	// which has the ability to fetch the return PC out of
duke@435	2131	// thread-local storage and also sets up last_Java_sp slightly
duke@435	2132	// differently than the real call_VM
duke@435	2133	Register java_thread = rbx;
duke@435	2134	__ get_thread(java_thread);
duke@435	2135	if (restore_saved_exception_pc) {
never@739	2136	__ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));
never@739	2137	__ push(rax);
duke@435	2138	}
duke@435	2139
duke@435	2140	__ enter(); // required for proper stackwalking of RuntimeStub frame
duke@435	2141
duke@435	2142	// pc and rbp, already pushed
never@739	2143	__ subptr(rsp, (framesize-2) * wordSize); // prolog
duke@435	2144
duke@435	2145	// Frame is now completed as far as size and linkage.
duke@435	2146
duke@435	2147	int frame_complete = __ pc() - start;
duke@435	2148
duke@435	2149	// push java thread (becomes first argument of C function)
never@739	2150	__ movptr(Address(rsp, thread_off * wordSize), java_thread);
duke@435	2151
duke@435	2152	// Set up last_Java_sp and last_Java_fp
duke@435	2153	__ set_last_Java_frame(java_thread, rsp, rbp, NULL);
duke@435	2154
duke@435	2155	// Call runtime
duke@435	2156	BLOCK_COMMENT("call runtime_entry");
duke@435	2157	__ call(RuntimeAddress(runtime_entry));
duke@435	2158	// Generate oop map
duke@435	2159	OopMap* map = new OopMap(framesize, 0);
duke@435	2160	oop_maps->add_gc_map(__ pc() - start, map);
duke@435	2161
duke@435	2162	// restore the thread (cannot use the pushed argument since arguments
duke@435	2163	// may be overwritten by C code generated by an optimizing compiler);
duke@435	2164	// however can use the register value directly if it is callee saved.
duke@435	2165	__ get_thread(java_thread);
duke@435	2166
duke@435	2167	__ reset_last_Java_frame(java_thread, true, false);
duke@435	2168
duke@435	2169	__ leave(); // required for proper stackwalking of RuntimeStub frame
duke@435	2170
duke@435	2171	// check for pending exceptions
duke@435	2172	#ifdef ASSERT
duke@435	2173	Label L;
never@739	2174	__ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
duke@435	2175	__ jcc(Assembler::notEqual, L);
duke@435	2176	__ should_not_reach_here();
duke@435	2177	__ bind(L);
duke@435	2178	#endif /* ASSERT */
duke@435	2179	__ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	2180
duke@435	2181
duke@435	2182	RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
duke@435	2183	return stub->entry_point();
duke@435	2184	}
duke@435	2185
duke@435	2186
duke@435	2187	void create_control_words() {
duke@435	2188	// Round to nearest, 53-bit mode, exceptions masked
duke@435	2189	StubRoutines::_fpu_cntrl_wrd_std = 0x027F;
duke@435	2190	// Round to zero, 53-bit mode, exception mased
duke@435	2191	StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
duke@435	2192	// Round to nearest, 24-bit mode, exceptions masked
duke@435	2193	StubRoutines::_fpu_cntrl_wrd_24 = 0x007F;
duke@435	2194	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2195	StubRoutines::_fpu_cntrl_wrd_64 = 0x037F;
duke@435	2196	// Round to nearest, 64-bit mode, exceptions masked
duke@435	2197	StubRoutines::_mxcsr_std = 0x1F80;
duke@435	2198	// Note: the following two constants are 80-bit values
duke@435	2199	// layout is critical for correct loading by FPU.
duke@435	2200	// Bias for strict fp multiply/divide
duke@435	2201	StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
duke@435	2202	StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
duke@435	2203	StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
duke@435	2204	// Un-Bias for strict fp multiply/divide
duke@435	2205	StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
duke@435	2206	StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
duke@435	2207	StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
duke@435	2208	}
duke@435	2209
duke@435	2210	//---------------------------------------------------------------------------
duke@435	2211	// Initialization
duke@435	2212
duke@435	2213	void generate_initial() {
duke@435	2214	// Generates all stubs and initializes the entry points
duke@435	2215
duke@435	2216	//------------------------------------------------------------------------------------------------------------------------
duke@435	2217	// entry points that exist in all platforms
duke@435	2218	// Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
duke@435	2219	// the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
duke@435	2220	StubRoutines::_forward_exception_entry = generate_forward_exception();
duke@435	2221
duke@435	2222	StubRoutines::_call_stub_entry =
duke@435	2223	generate_call_stub(StubRoutines::_call_stub_return_address);
duke@435	2224	// is referenced by megamorphic call
duke@435	2225	StubRoutines::_catch_exception_entry = generate_catch_exception();
duke@435	2226
duke@435	2227	// These are currently used by Solaris/Intel
duke@435	2228	StubRoutines::_atomic_xchg_entry = generate_atomic_xchg();
duke@435	2229
duke@435	2230	StubRoutines::_handler_for_unsafe_access_entry =
duke@435	2231	generate_handler_for_unsafe_access();
duke@435	2232
duke@435	2233	// platform dependent
duke@435	2234	create_control_words();
duke@435	2235
never@739	2236	StubRoutines::x86::_verify_mxcsr_entry = generate_verify_mxcsr();
never@739	2237	StubRoutines::x86::_verify_fpu_cntrl_wrd_entry = generate_verify_fpu_cntrl_wrd();
duke@435	2238	StubRoutines::_d2i_wrapper = generate_d2i_wrapper(T_INT,
duke@435	2239	CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
duke@435	2240	StubRoutines::_d2l_wrapper = generate_d2i_wrapper(T_LONG,
duke@435	2241	CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
duke@435	2242	}
duke@435	2243
duke@435	2244
duke@435	2245	void generate_all() {
duke@435	2246	// Generates all stubs and initializes the entry points
duke@435	2247
duke@435	2248	// These entry points require SharedInfo::stack0 to be set up in non-core builds
duke@435	2249	// and need to be relocatable, so they each fabricate a RuntimeStub internally.
duke@435	2250	StubRoutines::_throw_AbstractMethodError_entry = generate_throw_exception("AbstractMethodError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError), false);
dcubed@451	2251	StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError), false);
duke@435	2252	StubRoutines::_throw_ArithmeticException_entry = generate_throw_exception("ArithmeticException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException), true);
duke@435	2253	StubRoutines::_throw_NullPointerException_entry = generate_throw_exception("NullPointerException throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);
duke@435	2254	StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);
duke@435	2255	StubRoutines::_throw_StackOverflowError_entry = generate_throw_exception("StackOverflowError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError), false);
duke@435	2256
duke@435	2257	//------------------------------------------------------------------------------------------------------------------------
duke@435	2258	// entry points that are platform specific
duke@435	2259
duke@435	2260	// support for verify_oop (must happen after universe_init)
duke@435	2261	StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
duke@435	2262
duke@435	2263	// arraycopy stubs used by compilers
duke@435	2264	generate_arraycopy_stubs();
jrose@1145	2265
jrose@1145	2266	// generic method handle stubs
jrose@1145	2267	if (EnableMethodHandles && SystemDictionary::MethodHandle_klass() != NULL) {
jrose@1145	2268	for (MethodHandles::EntryKind ek = MethodHandles::_EK_FIRST;
jrose@1145	2269	ek < MethodHandles::_EK_LIMIT;
jrose@1145	2270	ek = MethodHandles::EntryKind(1 + (int)ek)) {
jrose@1145	2271	StubCodeMark mark(this, "MethodHandle", MethodHandles::entry_name(ek));
jrose@1145	2272	MethodHandles::generate_method_handle_stub(_masm, ek);
jrose@1145	2273	}
jrose@1145	2274	}
never@1609	2275
never@1609	2276	generate_math_stubs();
duke@435	2277	}
duke@435	2278
duke@435	2279
duke@435	2280	public:
duke@435	2281	StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
duke@435	2282	if (all) {
duke@435	2283	generate_all();
duke@435	2284	} else {
duke@435	2285	generate_initial();
duke@435	2286	}
duke@435	2287	}
duke@435	2288	}; // end class declaration
duke@435	2289
duke@435	2290
duke@435	2291	void StubGenerator_generate(CodeBuffer* code, bool all) {
duke@435	2292	StubGenerator g(code, all);
duke@435	2293	}