• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/assembler_x86_64.cpp@ba764ed4b6f2 (annotated)

src/cpu/x86/vm/assembler_x86_64.cpp

Sun, 13 Apr 2008 17:43:42 -0400

author

coleenp

date

Sun, 13 Apr 2008 17:43:42 -0400

changeset 548

ba764ed4b6f2

parent 511

d6fe2e4959d6

child 553

fb75a7673531

permissions

-rw-r--r--

6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
Summary: Compressed oops in instances, arrays, and headers. Code contributors are coleenp, phh, never, swamyv
Reviewed-by: jmasa, kamg, acorn, tbell, kvn, rasbold

duke@435	1	/*
duke@435	2	* Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_assembler_x86_64.cpp.incl"
duke@435	27
duke@435	28	// Implementation of AddressLiteral
duke@435	29
duke@435	30	AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
duke@435	31	_is_lval = false;
duke@435	32	_target = target;
duke@435	33	switch (rtype) {
duke@435	34	case relocInfo::oop_type:
duke@435	35	// Oops are a special case. Normally they would be their own section
duke@435	36	// but in cases like icBuffer they are literals in the code stream that
duke@435	37	// we don't have a section for. We use none so that we get a literal address
duke@435	38	// which is always patchable.
duke@435	39	break;
duke@435	40	case relocInfo::external_word_type:
duke@435	41	_rspec = external_word_Relocation::spec(target);
duke@435	42	break;
duke@435	43	case relocInfo::internal_word_type:
duke@435	44	_rspec = internal_word_Relocation::spec(target);
duke@435	45	break;
duke@435	46	case relocInfo::opt_virtual_call_type:
duke@435	47	_rspec = opt_virtual_call_Relocation::spec();
duke@435	48	break;
duke@435	49	case relocInfo::static_call_type:
duke@435	50	_rspec = static_call_Relocation::spec();
duke@435	51	break;
duke@435	52	case relocInfo::runtime_call_type:
duke@435	53	_rspec = runtime_call_Relocation::spec();
duke@435	54	break;
duke@435	55	case relocInfo::none:
duke@435	56	break;
duke@435	57	default:
duke@435	58	ShouldNotReachHere();
duke@435	59	break;
duke@435	60	}
duke@435	61	}
duke@435	62
duke@435	63	// Implementation of Address
duke@435	64
duke@435	65	Address Address::make_array(ArrayAddress adr) {
duke@435	66	#ifdef _LP64
duke@435	67	// Not implementable on 64bit machines
duke@435	68	// Should have been handled higher up the call chain.
duke@435	69	ShouldNotReachHere();
duke@435	70	return Address();
duke@435	71	#else
duke@435	72	AddressLiteral base = adr.base();
duke@435	73	Address index = adr.index();
duke@435	74	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	75	Address array(index._base, index._index, index._scale, (intptr_t) base.target());
duke@435	76	array._rspec = base._rspec;
duke@435	77	return array;
duke@435	78	#endif // _LP64
duke@435	79	}
duke@435	80
duke@435	81	// exceedingly dangerous constructor
duke@435	82	Address::Address(int disp, address loc, relocInfo::relocType rtype) {
duke@435	83	_base = noreg;
duke@435	84	_index = noreg;
duke@435	85	_scale = no_scale;
duke@435	86	_disp = disp;
duke@435	87	switch (rtype) {
duke@435	88	case relocInfo::external_word_type:
duke@435	89	_rspec = external_word_Relocation::spec(loc);
duke@435	90	break;
duke@435	91	case relocInfo::internal_word_type:
duke@435	92	_rspec = internal_word_Relocation::spec(loc);
duke@435	93	break;
duke@435	94	case relocInfo::runtime_call_type:
duke@435	95	// HMM
duke@435	96	_rspec = runtime_call_Relocation::spec();
duke@435	97	break;
duke@435	98	case relocInfo::none:
duke@435	99	break;
duke@435	100	default:
duke@435	101	ShouldNotReachHere();
duke@435	102	}
duke@435	103	}
duke@435	104
duke@435	105	// Convert the raw encoding form into the form expected by the constructor for
duke@435	106	// Address. An index of 4 (rsp) corresponds to having no index, so convert
duke@435	107	// that to noreg for the Address constructor.
duke@435	108	Address Address::make_raw(int base, int index, int scale, int disp) {
duke@435	109	bool valid_index = index != rsp->encoding();
duke@435	110	if (valid_index) {
duke@435	111	Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
duke@435	112	return madr;
duke@435	113	} else {
duke@435	114	Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
duke@435	115	return madr;
duke@435	116	}
duke@435	117	}
duke@435	118
duke@435	119
duke@435	120	// Implementation of Assembler
duke@435	121	int AbstractAssembler::code_fill_byte() {
duke@435	122	return (u_char)'\xF4'; // hlt
duke@435	123	}
duke@435	124
duke@435	125	// This should only be used by 64bit instructions that can use rip-relative
duke@435	126	// it cannot be used by instructions that want an immediate value.
duke@435	127
duke@435	128	bool Assembler::reachable(AddressLiteral adr) {
duke@435	129	int64_t disp;
coleenp@548	130
duke@435	131	// None will force a 64bit literal to the code stream. Likely a placeholder
duke@435	132	// for something that will be patched later and we need to certain it will
duke@435	133	// always be reachable.
duke@435	134	if (adr.reloc() == relocInfo::none) {
duke@435	135	return false;
duke@435	136	}
duke@435	137	if (adr.reloc() == relocInfo::internal_word_type) {
duke@435	138	// This should be rip relative and easily reachable.
duke@435	139	return true;
duke@435	140	}
duke@435	141	if (adr.reloc() != relocInfo::external_word_type &&
duke@435	142	adr.reloc() != relocInfo::runtime_call_type ) {
duke@435	143	return false;
duke@435	144	}
duke@435	145
duke@435	146	// Stress the correction code
duke@435	147	if (ForceUnreachable) {
duke@435	148	// Must be runtimecall reloc, see if it is in the codecache
duke@435	149	// Flipping stuff in the codecache to be unreachable causes issues
duke@435	150	// with things like inline caches where the additional instructions
duke@435	151	// are not handled.
duke@435	152	if (CodeCache::find_blob(adr._target) == NULL) {
duke@435	153	return false;
duke@435	154	}
duke@435	155	}
duke@435	156	// For external_word_type/runtime_call_type if it is reachable from where we
duke@435	157	// are now (possibly a temp buffer) and where we might end up
duke@435	158	// anywhere in the codeCache then we are always reachable.
duke@435	159	// This would have to change if we ever save/restore shared code
duke@435	160	// to be more pessimistic.
duke@435	161
duke@435	162	disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
duke@435	163	if (!is_simm32(disp)) return false;
duke@435	164	disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
duke@435	165	if (!is_simm32(disp)) return false;
duke@435	166
duke@435	167	disp = (int64_t)adr._target - ((int64_t)_code_pos + sizeof(int));
duke@435	168
duke@435	169	// Because rip relative is a disp + address_of_next_instruction and we
duke@435	170	// don't know the value of address_of_next_instruction we apply a fudge factor
duke@435	171	// to make sure we will be ok no matter the size of the instruction we get placed into.
duke@435	172	// We don't have to fudge the checks above here because they are already worst case.
duke@435	173
duke@435	174	// 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
duke@435	175	// + 4 because better safe than sorry.
duke@435	176	const int fudge = 12 + 4;
duke@435	177	if (disp < 0) {
duke@435	178	disp -= fudge;
duke@435	179	} else {
duke@435	180	disp += fudge;
duke@435	181	}
duke@435	182	return is_simm32(disp);
duke@435	183	}
duke@435	184
duke@435	185
duke@435	186	// make this go away eventually
duke@435	187	void Assembler::emit_data(jint data,
duke@435	188	relocInfo::relocType rtype,
duke@435	189	int format) {
duke@435	190	if (rtype == relocInfo::none) {
duke@435	191	emit_long(data);
duke@435	192	} else {
duke@435	193	emit_data(data, Relocation::spec_simple(rtype), format);
duke@435	194	}
duke@435	195	}
duke@435	196
duke@435	197	void Assembler::emit_data(jint data,
duke@435	198	RelocationHolder const& rspec,
duke@435	199	int format) {
duke@435	200	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	201	assert(imm64_operand != format, "must not be imm64");
duke@435	202	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	203	if (rspec.type() != relocInfo::none) {
duke@435	204	#ifdef ASSERT
duke@435	205	check_relocation(rspec, format);
duke@435	206	#endif
duke@435	207	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	208	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	209
duke@435	210	// hack. call32 is too wide for mask so use disp32
duke@435	211	if (format == call32_operand)
duke@435	212	code_section()->relocate(inst_mark(), rspec, disp32_operand);
duke@435	213	else
duke@435	214	code_section()->relocate(inst_mark(), rspec, format);
duke@435	215	}
duke@435	216	emit_long(data);
duke@435	217	}
duke@435	218
duke@435	219	void Assembler::emit_data64(jlong data,
duke@435	220	relocInfo::relocType rtype,
duke@435	221	int format) {
duke@435	222	if (rtype == relocInfo::none) {
duke@435	223	emit_long64(data);
duke@435	224	} else {
duke@435	225	emit_data64(data, Relocation::spec_simple(rtype), format);
duke@435	226	}
duke@435	227	}
duke@435	228
duke@435	229	void Assembler::emit_data64(jlong data,
duke@435	230	RelocationHolder const& rspec,
duke@435	231	int format) {
duke@435	232	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	233	assert(imm64_operand == format, "must be imm64");
duke@435	234	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	235	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	236	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	237	code_section()->relocate(inst_mark(), rspec, format);
duke@435	238	#ifdef ASSERT
duke@435	239	check_relocation(rspec, format);
duke@435	240	#endif
duke@435	241	emit_long64(data);
duke@435	242	}
duke@435	243
duke@435	244	void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
duke@435	245	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	246	assert(isByte(imm8), "not a byte");
duke@435	247	assert((op1 & 0x01) == 0, "should be 8bit operation");
duke@435	248	int dstenc = dst->encoding();
duke@435	249	if (dstenc >= 8) {
duke@435	250	dstenc -= 8;
duke@435	251	}
duke@435	252	emit_byte(op1);
duke@435	253	emit_byte(op2 | dstenc);
duke@435	254	emit_byte(imm8);
duke@435	255	}
duke@435	256
duke@435	257	void Assembler::emit_arith(int op1, int op2, Register dst, int imm32) {
duke@435	258	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	259	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	260	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	261	int dstenc = dst->encoding();
duke@435	262	if (dstenc >= 8) {
duke@435	263	dstenc -= 8;
duke@435	264	}
duke@435	265	if (is8bit(imm32)) {
duke@435	266	emit_byte(op1 | 0x02); // set sign bit
duke@435	267	emit_byte(op2 | dstenc);
duke@435	268	emit_byte(imm32 & 0xFF);
duke@435	269	} else {
duke@435	270	emit_byte(op1);
duke@435	271	emit_byte(op2 | dstenc);
duke@435	272	emit_long(imm32);
duke@435	273	}
duke@435	274	}
duke@435	275
duke@435	276	// immediate-to-memory forms
duke@435	277	void Assembler::emit_arith_operand(int op1,
duke@435	278	Register rm, Address adr,
duke@435	279	int imm32) {
duke@435	280	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	281	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	282	if (is8bit(imm32)) {
duke@435	283	emit_byte(op1 | 0x02); // set sign bit
duke@435	284	emit_operand(rm, adr, 1);
duke@435	285	emit_byte(imm32 & 0xFF);
duke@435	286	} else {
duke@435	287	emit_byte(op1);
duke@435	288	emit_operand(rm, adr, 4);
duke@435	289	emit_long(imm32);
duke@435	290	}
duke@435	291	}
duke@435	292
duke@435	293
duke@435	294	void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
duke@435	295	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	296	int dstenc = dst->encoding();
duke@435	297	int srcenc = src->encoding();
duke@435	298	if (dstenc >= 8) {
duke@435	299	dstenc -= 8;
duke@435	300	}
duke@435	301	if (srcenc >= 8) {
duke@435	302	srcenc -= 8;
duke@435	303	}
duke@435	304	emit_byte(op1);
duke@435	305	emit_byte(op2 | dstenc << 3 | srcenc);
duke@435	306	}
duke@435	307
duke@435	308	void Assembler::emit_operand(Register reg, Register base, Register index,
duke@435	309	Address::ScaleFactor scale, int disp,
duke@435	310	RelocationHolder const& rspec,
duke@435	311	int rip_relative_correction) {
duke@435	312	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	313	int regenc = reg->encoding();
duke@435	314	if (regenc >= 8) {
duke@435	315	regenc -= 8;
duke@435	316	}
duke@435	317	if (base->is_valid()) {
duke@435	318	if (index->is_valid()) {
duke@435	319	assert(scale != Address::no_scale, "inconsistent address");
duke@435	320	int indexenc = index->encoding();
duke@435	321	if (indexenc >= 8) {
duke@435	322	indexenc -= 8;
duke@435	323	}
duke@435	324	int baseenc = base->encoding();
duke@435	325	if (baseenc >= 8) {
duke@435	326	baseenc -= 8;
duke@435	327	}
duke@435	328	// [base + index*scale + disp]
duke@435	329	if (disp == 0 && rtype == relocInfo::none &&
duke@435	330	base != rbp && base != r13) {
duke@435	331	// [base + index*scale]
duke@435	332	// [00 reg 100][ss index base]
duke@435	333	assert(index != rsp, "illegal addressing mode");
duke@435	334	emit_byte(0x04 | regenc << 3);
duke@435	335	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	336	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	337	// [base + index*scale + imm8]
duke@435	338	// [01 reg 100][ss index base] imm8
duke@435	339	assert(index != rsp, "illegal addressing mode");
duke@435	340	emit_byte(0x44 | regenc << 3);
duke@435	341	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	342	emit_byte(disp & 0xFF);
duke@435	343	} else {
duke@435	344	// [base + index*scale + disp32]
duke@435	345	// [10 reg 100][ss index base] disp32
duke@435	346	assert(index != rsp, "illegal addressing mode");
duke@435	347	emit_byte(0x84 | regenc << 3);
duke@435	348	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	349	emit_data(disp, rspec, disp32_operand);
duke@435	350	}
duke@435	351	} else if (base == rsp || base == r12) {
duke@435	352	// [rsp + disp]
duke@435	353	if (disp == 0 && rtype == relocInfo::none) {
duke@435	354	// [rsp]
duke@435	355	// [00 reg 100][00 100 100]
duke@435	356	emit_byte(0x04 | regenc << 3);
duke@435	357	emit_byte(0x24);
duke@435	358	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	359	// [rsp + imm8]
duke@435	360	// [01 reg 100][00 100 100] disp8
duke@435	361	emit_byte(0x44 | regenc << 3);
duke@435	362	emit_byte(0x24);
duke@435	363	emit_byte(disp & 0xFF);
duke@435	364	} else {
duke@435	365	// [rsp + imm32]
duke@435	366	// [10 reg 100][00 100 100] disp32
duke@435	367	emit_byte(0x84 | regenc << 3);
duke@435	368	emit_byte(0x24);
duke@435	369	emit_data(disp, rspec, disp32_operand);
duke@435	370	}
duke@435	371	} else {
duke@435	372	// [base + disp]
duke@435	373	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	374	int baseenc = base->encoding();
duke@435	375	if (baseenc >= 8) {
duke@435	376	baseenc -= 8;
duke@435	377	}
duke@435	378	if (disp == 0 && rtype == relocInfo::none &&
duke@435	379	base != rbp && base != r13) {
duke@435	380	// [base]
duke@435	381	// [00 reg base]
duke@435	382	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	383	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	384	// [base + disp8]
duke@435	385	// [01 reg base] disp8
duke@435	386	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	387	emit_byte(disp & 0xFF);
duke@435	388	} else {
duke@435	389	// [base + disp32]
duke@435	390	// [10 reg base] disp32
duke@435	391	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	392	emit_data(disp, rspec, disp32_operand);
duke@435	393	}
duke@435	394	}
duke@435	395	} else {
duke@435	396	if (index->is_valid()) {
duke@435	397	assert(scale != Address::no_scale, "inconsistent address");
duke@435	398	int indexenc = index->encoding();
duke@435	399	if (indexenc >= 8) {
duke@435	400	indexenc -= 8;
duke@435	401	}
duke@435	402	// [index*scale + disp]
duke@435	403	// [00 reg 100][ss index 101] disp32
duke@435	404	assert(index != rsp, "illegal addressing mode");
duke@435	405	emit_byte(0x04 | regenc << 3);
duke@435	406	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	407	emit_data(disp, rspec, disp32_operand);
duke@435	408	#ifdef _LP64
duke@435	409	} else if (rtype != relocInfo::none ) {
duke@435	410	// [disp] RIP-RELATIVE
duke@435	411	// [00 000 101] disp32
duke@435	412
duke@435	413	emit_byte(0x05 | regenc << 3);
duke@435	414	// Note that the RIP-rel. correction applies to the generated
duke@435	415	// disp field, but _not_ to the target address in the rspec.
duke@435	416
duke@435	417	// disp was created by converting the target address minus the pc
duke@435	418	// at the start of the instruction. That needs more correction here.
duke@435	419	// intptr_t disp = target - next_ip;
duke@435	420	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	421	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	422	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	423	assert(is_simm32(adjusted),
duke@435	424	"must be 32bit offset (RIP relative address)");
duke@435	425	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	426
duke@435	427	#endif // _LP64
duke@435	428	} else {
duke@435	429	// [disp] ABSOLUTE
duke@435	430	// [00 reg 100][00 100 101] disp32
duke@435	431	emit_byte(0x04 | regenc << 3);
duke@435	432	emit_byte(0x25);
duke@435	433	emit_data(disp, rspec, disp32_operand);
duke@435	434	}
duke@435	435	}
duke@435	436	}
duke@435	437
duke@435	438	void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
duke@435	439	Address::ScaleFactor scale, int disp,
duke@435	440	RelocationHolder const& rspec,
duke@435	441	int rip_relative_correction) {
duke@435	442	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	443	int regenc = reg->encoding();
duke@435	444	if (regenc >= 8) {
duke@435	445	regenc -= 8;
duke@435	446	}
duke@435	447	if (base->is_valid()) {
duke@435	448	if (index->is_valid()) {
duke@435	449	assert(scale != Address::no_scale, "inconsistent address");
duke@435	450	int indexenc = index->encoding();
duke@435	451	if (indexenc >= 8) {
duke@435	452	indexenc -= 8;
duke@435	453	}
duke@435	454	int baseenc = base->encoding();
duke@435	455	if (baseenc >= 8) {
duke@435	456	baseenc -= 8;
duke@435	457	}
duke@435	458	// [base + index*scale + disp]
duke@435	459	if (disp == 0 && rtype == relocInfo::none &&
duke@435	460	base != rbp && base != r13) {
duke@435	461	// [base + index*scale]
duke@435	462	// [00 reg 100][ss index base]
duke@435	463	assert(index != rsp, "illegal addressing mode");
duke@435	464	emit_byte(0x04 | regenc << 3);
duke@435	465	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	466	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	467	// [base + index*scale + disp8]
duke@435	468	// [01 reg 100][ss index base] disp8
duke@435	469	assert(index != rsp, "illegal addressing mode");
duke@435	470	emit_byte(0x44 | regenc << 3);
duke@435	471	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	472	emit_byte(disp & 0xFF);
duke@435	473	} else {
duke@435	474	// [base + index*scale + disp32]
duke@435	475	// [10 reg 100][ss index base] disp32
duke@435	476	assert(index != rsp, "illegal addressing mode");
duke@435	477	emit_byte(0x84 | regenc << 3);
duke@435	478	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	479	emit_data(disp, rspec, disp32_operand);
duke@435	480	}
duke@435	481	} else if (base == rsp || base == r12) {
duke@435	482	// [rsp + disp]
duke@435	483	if (disp == 0 && rtype == relocInfo::none) {
duke@435	484	// [rsp]
duke@435	485	// [00 reg 100][00 100 100]
duke@435	486	emit_byte(0x04 | regenc << 3);
duke@435	487	emit_byte(0x24);
duke@435	488	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	489	// [rsp + imm8]
duke@435	490	// [01 reg 100][00 100 100] disp8
duke@435	491	emit_byte(0x44 | regenc << 3);
duke@435	492	emit_byte(0x24);
duke@435	493	emit_byte(disp & 0xFF);
duke@435	494	} else {
duke@435	495	// [rsp + imm32]
duke@435	496	// [10 reg 100][00 100 100] disp32
duke@435	497	emit_byte(0x84 | regenc << 3);
duke@435	498	emit_byte(0x24);
duke@435	499	emit_data(disp, rspec, disp32_operand);
duke@435	500	}
duke@435	501	} else {
duke@435	502	// [base + disp]
duke@435	503	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	504	int baseenc = base->encoding();
duke@435	505	if (baseenc >= 8) {
duke@435	506	baseenc -= 8;
duke@435	507	}
duke@435	508	if (disp == 0 && rtype == relocInfo::none &&
duke@435	509	base != rbp && base != r13) {
duke@435	510	// [base]
duke@435	511	// [00 reg base]
duke@435	512	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	513	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	514	// [base + imm8]
duke@435	515	// [01 reg base] disp8
duke@435	516	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	517	emit_byte(disp & 0xFF);
duke@435	518	} else {
duke@435	519	// [base + imm32]
duke@435	520	// [10 reg base] disp32
duke@435	521	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	522	emit_data(disp, rspec, disp32_operand);
duke@435	523	}
duke@435	524	}
duke@435	525	} else {
duke@435	526	if (index->is_valid()) {
duke@435	527	assert(scale != Address::no_scale, "inconsistent address");
duke@435	528	int indexenc = index->encoding();
duke@435	529	if (indexenc >= 8) {
duke@435	530	indexenc -= 8;
duke@435	531	}
duke@435	532	// [index*scale + disp]
duke@435	533	// [00 reg 100][ss index 101] disp32
duke@435	534	assert(index != rsp, "illegal addressing mode");
duke@435	535	emit_byte(0x04 | regenc << 3);
duke@435	536	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	537	emit_data(disp, rspec, disp32_operand);
duke@435	538	#ifdef _LP64
duke@435	539	} else if ( rtype != relocInfo::none ) {
duke@435	540	// [disp] RIP-RELATIVE
duke@435	541	// [00 reg 101] disp32
duke@435	542	emit_byte(0x05 | regenc << 3);
duke@435	543	// Note that the RIP-rel. correction applies to the generated
duke@435	544	// disp field, but _not_ to the target address in the rspec.
duke@435	545
duke@435	546	// disp was created by converting the target address minus the pc
duke@435	547	// at the start of the instruction. That needs more correction here.
duke@435	548	// intptr_t disp = target - next_ip;
duke@435	549
duke@435	550	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	551	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	552
duke@435	553	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	554	assert(is_simm32(adjusted),
duke@435	555	"must be 32bit offset (RIP relative address)");
duke@435	556	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	557	#endif // _LP64
duke@435	558	} else {
duke@435	559	// [disp] ABSOLUTE
duke@435	560	// [00 reg 100][00 100 101] disp32
duke@435	561	emit_byte(0x04 | regenc << 3);
duke@435	562	emit_byte(0x25);
duke@435	563	emit_data(disp, rspec, disp32_operand);
duke@435	564	}
duke@435	565	}
duke@435	566	}
duke@435	567
duke@435	568	// Secret local extension to Assembler::WhichOperand:
duke@435	569	#define end_pc_operand (_WhichOperand_limit)
duke@435	570
duke@435	571	address Assembler::locate_operand(address inst, WhichOperand which) {
duke@435	572	// Decode the given instruction, and return the address of
duke@435	573	// an embedded 32-bit operand word.
duke@435	574
duke@435	575	// If "which" is disp32_operand, selects the displacement portion
duke@435	576	// of an effective address specifier.
duke@435	577	// If "which" is imm64_operand, selects the trailing immediate constant.
duke@435	578	// If "which" is call32_operand, selects the displacement of a call or jump.
duke@435	579	// Caller is responsible for ensuring that there is such an operand,
duke@435	580	// and that it is 32/64 bits wide.
duke@435	581
duke@435	582	// If "which" is end_pc_operand, find the end of the instruction.
duke@435	583
duke@435	584	address ip = inst;
duke@435	585	bool is_64bit = false;
duke@435	586
duke@435	587	debug_only(bool has_disp32 = false);
duke@435	588	int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
duke@435	589
duke@435	590	again_after_prefix:
duke@435	591	switch (0xFF & *ip++) {
duke@435	592
duke@435	593	// These convenience macros generate groups of "case" labels for the switch.
duke@435	594	#define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
duke@435	595	#define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
duke@435	596	case (x)+4: case (x)+5: case (x)+6: case (x)+7
duke@435	597	#define REP16(x) REP8((x)+0): \
duke@435	598	case REP8((x)+8)
duke@435	599
duke@435	600	case CS_segment:
duke@435	601	case SS_segment:
duke@435	602	case DS_segment:
duke@435	603	case ES_segment:
duke@435	604	case FS_segment:
duke@435	605	case GS_segment:
duke@435	606	assert(0, "shouldn't have that prefix");
duke@435	607	assert(ip == inst + 1 || ip == inst + 2, "only two prefixes allowed");
duke@435	608	goto again_after_prefix;
duke@435	609
duke@435	610	case 0x67:
duke@435	611	case REX:
duke@435	612	case REX_B:
duke@435	613	case REX_X:
duke@435	614	case REX_XB:
duke@435	615	case REX_R:
duke@435	616	case REX_RB:
duke@435	617	case REX_RX:
duke@435	618	case REX_RXB:
duke@435	619	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	620	goto again_after_prefix;
duke@435	621
duke@435	622	case REX_W:
duke@435	623	case REX_WB:
duke@435	624	case REX_WX:
duke@435	625	case REX_WXB:
duke@435	626	case REX_WR:
duke@435	627	case REX_WRB:
duke@435	628	case REX_WRX:
duke@435	629	case REX_WRXB:
duke@435	630	is_64bit = true;
duke@435	631	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	632	goto again_after_prefix;
duke@435	633
duke@435	634	case 0xFF: // pushq a; decl a; incl a; call a; jmp a
duke@435	635	case 0x88: // movb a, r
duke@435	636	case 0x89: // movl a, r
duke@435	637	case 0x8A: // movb r, a
duke@435	638	case 0x8B: // movl r, a
duke@435	639	case 0x8F: // popl a
coleenp@548	640	debug_only(has_disp32 = true;)
duke@435	641	break;
duke@435	642
duke@435	643	case 0x68: // pushq #32
duke@435	644	if (which == end_pc_operand) {
duke@435	645	return ip + 4;
duke@435	646	}
duke@435	647	assert(0, "pushq has no disp32 or imm64");
duke@435	648	ShouldNotReachHere();
duke@435	649
duke@435	650	case 0x66: // movw ... (size prefix)
duke@435	651	again_after_size_prefix2:
duke@435	652	switch (0xFF & *ip++) {
duke@435	653	case REX:
duke@435	654	case REX_B:
duke@435	655	case REX_X:
duke@435	656	case REX_XB:
duke@435	657	case REX_R:
duke@435	658	case REX_RB:
duke@435	659	case REX_RX:
duke@435	660	case REX_RXB:
duke@435	661	case REX_W:
duke@435	662	case REX_WB:
duke@435	663	case REX_WX:
duke@435	664	case REX_WXB:
duke@435	665	case REX_WR:
duke@435	666	case REX_WRB:
duke@435	667	case REX_WRX:
duke@435	668	case REX_WRXB:
duke@435	669	goto again_after_size_prefix2;
duke@435	670	case 0x8B: // movw r, a
duke@435	671	case 0x89: // movw a, r
duke@435	672	break;
duke@435	673	case 0xC7: // movw a, #16
duke@435	674	tail_size = 2; // the imm16
duke@435	675	break;
duke@435	676	case 0x0F: // several SSE/SSE2 variants
duke@435	677	ip--; // reparse the 0x0F
duke@435	678	goto again_after_prefix;
duke@435	679	default:
duke@435	680	ShouldNotReachHere();
duke@435	681	}
duke@435	682	break;
duke@435	683
duke@435	684	case REP8(0xB8): // movl/q r, #32/#64(oop?)
duke@435	685	if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
duke@435	686	assert((which == call32_operand || which == imm64_operand) && is_64bit, "");
duke@435	687	return ip;
duke@435	688
duke@435	689	case 0x69: // imul r, a, #32
duke@435	690	case 0xC7: // movl a, #32(oop?)
duke@435	691	tail_size = 4;
duke@435	692	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	693	break;
duke@435	694
duke@435	695	case 0x0F: // movx..., etc.
duke@435	696	switch (0xFF & *ip++) {
duke@435	697	case 0x12: // movlps
duke@435	698	case 0x28: // movaps
duke@435	699	case 0x2E: // ucomiss
duke@435	700	case 0x2F: // comiss
duke@435	701	case 0x54: // andps
duke@435	702	case 0x57: // xorps
duke@435	703	case 0x6E: // movd
duke@435	704	case 0x7E: // movd
duke@435	705	case 0xAE: // ldmxcsr a
duke@435	706	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	707	break;
duke@435	708	case 0xAD: // shrd r, a, %cl
duke@435	709	case 0xAF: // imul r, a
duke@435	710	case 0xBE: // movsbl r, a
duke@435	711	case 0xBF: // movswl r, a
duke@435	712	case 0xB6: // movzbl r, a
duke@435	713	case 0xB7: // movzwl r, a
duke@435	714	case REP16(0x40): // cmovl cc, r, a
duke@435	715	case 0xB0: // cmpxchgb
duke@435	716	case 0xB1: // cmpxchg
duke@435	717	case 0xC1: // xaddl
duke@435	718	case 0xC7: // cmpxchg8
duke@435	719	case REP16(0x90): // setcc a
duke@435	720	debug_only(has_disp32 = true);
duke@435	721	// fall out of the switch to decode the address
duke@435	722	break;
duke@435	723	case 0xAC: // shrd r, a, #8
duke@435	724	debug_only(has_disp32 = true);
duke@435	725	tail_size = 1; // the imm8
duke@435	726	break;
duke@435	727	case REP16(0x80): // jcc rdisp32
duke@435	728	if (which == end_pc_operand) return ip + 4;
duke@435	729	assert(which == call32_operand, "jcc has no disp32 or imm64");
duke@435	730	return ip;
duke@435	731	default:
duke@435	732	ShouldNotReachHere();
duke@435	733	}
duke@435	734	break;
duke@435	735
duke@435	736	case 0x81: // addl a, #32; addl r, #32
duke@435	737	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	738	tail_size = 4;
duke@435	739	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	740	break;
duke@435	741
duke@435	742	case 0x83: // addl a, #8; addl r, #8
duke@435	743	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	744	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	745	tail_size = 1;
duke@435	746	break;
duke@435	747
duke@435	748	case 0x9B:
duke@435	749	switch (0xFF & *ip++) {
duke@435	750	case 0xD9: // fnstcw a
duke@435	751	debug_only(has_disp32 = true);
duke@435	752	break;
duke@435	753	default:
duke@435	754	ShouldNotReachHere();
duke@435	755	}
duke@435	756	break;
duke@435	757
duke@435	758	case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
duke@435	759	case REP4(0x10): // adc...
duke@435	760	case REP4(0x20): // and...
duke@435	761	case REP4(0x30): // xor...
duke@435	762	case REP4(0x08): // or...
duke@435	763	case REP4(0x18): // sbb...
duke@435	764	case REP4(0x28): // sub...
duke@435	765	case 0xF7: // mull a
duke@435	766	case 0x87: // xchg r, a
duke@435	767	debug_only(has_disp32 = true);
duke@435	768	break;
duke@435	769	case REP4(0x38): // cmp...
duke@435	770	case 0x8D: // lea r, a
duke@435	771	case 0x85: // test r, a
duke@435	772	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	773	break;
duke@435	774
duke@435	775	case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
duke@435	776	case 0xC6: // movb a, #8
duke@435	777	case 0x80: // cmpb a, #8
duke@435	778	case 0x6B: // imul r, a, #8
duke@435	779	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	780	tail_size = 1; // the imm8
duke@435	781	break;
duke@435	782
duke@435	783	case 0xE8: // call rdisp32
duke@435	784	case 0xE9: // jmp rdisp32
duke@435	785	if (which == end_pc_operand) return ip + 4;
duke@435	786	assert(which == call32_operand, "call has no disp32 or imm32");
duke@435	787	return ip;
duke@435	788
duke@435	789	case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
duke@435	790	case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
duke@435	791	case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
duke@435	792	case 0xDD: // fld_d a; fst_d a; fstp_d a
duke@435	793	case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
duke@435	794	case 0xDF: // fild_d a; fistp_d a
duke@435	795	case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
duke@435	796	case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
duke@435	797	case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
duke@435	798	debug_only(has_disp32 = true);
duke@435	799	break;
duke@435	800
duke@435	801	case 0xF3: // For SSE
duke@435	802	case 0xF2: // For SSE2
duke@435	803	switch (0xFF & *ip++) {
duke@435	804	case REX:
duke@435	805	case REX_B:
duke@435	806	case REX_X:
duke@435	807	case REX_XB:
duke@435	808	case REX_R:
duke@435	809	case REX_RB:
duke@435	810	case REX_RX:
duke@435	811	case REX_RXB:
duke@435	812	case REX_W:
duke@435	813	case REX_WB:
duke@435	814	case REX_WX:
duke@435	815	case REX_WXB:
duke@435	816	case REX_WR:
duke@435	817	case REX_WRB:
duke@435	818	case REX_WRX:
duke@435	819	case REX_WRXB:
duke@435	820	ip++;
duke@435	821	default:
duke@435	822	ip++;
duke@435	823	}
duke@435	824	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	825	break;
duke@435	826
duke@435	827	default:
duke@435	828	ShouldNotReachHere();
duke@435	829
duke@435	830	#undef REP8
duke@435	831	#undef REP16
duke@435	832	}
duke@435	833
duke@435	834	assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
duke@435	835	assert(which != imm64_operand, "instruction is not a movq reg, imm64");
duke@435	836	assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
duke@435	837
duke@435	838	// parse the output of emit_operand
duke@435	839	int op2 = 0xFF & *ip++;
duke@435	840	int base = op2 & 0x07;
duke@435	841	int op3 = -1;
duke@435	842	const int b100 = 4;
duke@435	843	const int b101 = 5;
duke@435	844	if (base == b100 && (op2 >> 6) != 3) {
duke@435	845	op3 = 0xFF & *ip++;
duke@435	846	base = op3 & 0x07; // refetch the base
duke@435	847	}
duke@435	848	// now ip points at the disp (if any)
duke@435	849
duke@435	850	switch (op2 >> 6) {
duke@435	851	case 0:
duke@435	852	// [00 reg 100][ss index base]
duke@435	853	// [00 reg 100][00 100 esp]
duke@435	854	// [00 reg base]
duke@435	855	// [00 reg 100][ss index 101][disp32]
duke@435	856	// [00 reg 101] [disp32]
duke@435	857
duke@435	858	if (base == b101) {
duke@435	859	if (which == disp32_operand)
duke@435	860	return ip; // caller wants the disp32
duke@435	861	ip += 4; // skip the disp32
duke@435	862	}
duke@435	863	break;
duke@435	864
duke@435	865	case 1:
duke@435	866	// [01 reg 100][ss index base][disp8]
duke@435	867	// [01 reg 100][00 100 esp][disp8]
duke@435	868	// [01 reg base] [disp8]
duke@435	869	ip += 1; // skip the disp8
duke@435	870	break;
duke@435	871
duke@435	872	case 2:
duke@435	873	// [10 reg 100][ss index base][disp32]
duke@435	874	// [10 reg 100][00 100 esp][disp32]
duke@435	875	// [10 reg base] [disp32]
duke@435	876	if (which == disp32_operand)
duke@435	877	return ip; // caller wants the disp32
duke@435	878	ip += 4; // skip the disp32
duke@435	879	break;
duke@435	880
duke@435	881	case 3:
duke@435	882	// [11 reg base] (not a memory addressing mode)
duke@435	883	break;
duke@435	884	}
duke@435	885
duke@435	886	if (which == end_pc_operand) {
duke@435	887	return ip + tail_size;
duke@435	888	}
duke@435	889
duke@435	890	assert(0, "fix locate_operand");
duke@435	891	return ip;
duke@435	892	}
duke@435	893
duke@435	894	address Assembler::locate_next_instruction(address inst) {
duke@435	895	// Secretly share code with locate_operand:
duke@435	896	return locate_operand(inst, end_pc_operand);
duke@435	897	}
duke@435	898
duke@435	899	#ifdef ASSERT
duke@435	900	void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
duke@435	901	address inst = inst_mark();
duke@435	902	assert(inst != NULL && inst < pc(),
duke@435	903	"must point to beginning of instruction");
duke@435	904	address opnd;
duke@435	905
duke@435	906	Relocation* r = rspec.reloc();
duke@435	907	if (r->type() == relocInfo::none) {
duke@435	908	return;
duke@435	909	} else if (r->is_call() || format == call32_operand) {
duke@435	910	opnd = locate_operand(inst, call32_operand);
duke@435	911	} else if (r->is_data()) {
duke@435	912	assert(format == imm64_operand || format == disp32_operand, "format ok");
duke@435	913	opnd = locate_operand(inst, (WhichOperand) format);
duke@435	914	} else {
duke@435	915	assert(format == 0, "cannot specify a format");
duke@435	916	return;
duke@435	917	}
duke@435	918	assert(opnd == pc(), "must put operand where relocs can find it");
duke@435	919	}
duke@435	920	#endif
duke@435	921
duke@435	922	int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
duke@435	923	if (reg_enc >= 8) {
duke@435	924	prefix(REX_B);
duke@435	925	reg_enc -= 8;
duke@435	926	} else if (byteinst && reg_enc >= 4) {
duke@435	927	prefix(REX);
duke@435	928	}
duke@435	929	return reg_enc;
duke@435	930	}
duke@435	931
duke@435	932	int Assembler::prefixq_and_encode(int reg_enc) {
duke@435	933	if (reg_enc < 8) {
duke@435	934	prefix(REX_W);
duke@435	935	} else {
duke@435	936	prefix(REX_WB);
duke@435	937	reg_enc -= 8;
duke@435	938	}
duke@435	939	return reg_enc;
duke@435	940	}
duke@435	941
duke@435	942	int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
duke@435	943	if (dst_enc < 8) {
duke@435	944	if (src_enc >= 8) {
duke@435	945	prefix(REX_B);
duke@435	946	src_enc -= 8;
duke@435	947	} else if (byteinst && src_enc >= 4) {
duke@435	948	prefix(REX);
duke@435	949	}
duke@435	950	} else {
duke@435	951	if (src_enc < 8) {
duke@435	952	prefix(REX_R);
duke@435	953	} else {
duke@435	954	prefix(REX_RB);
duke@435	955	src_enc -= 8;
duke@435	956	}
duke@435	957	dst_enc -= 8;
duke@435	958	}
duke@435	959	return dst_enc << 3 | src_enc;
duke@435	960	}
duke@435	961
duke@435	962	int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
duke@435	963	if (dst_enc < 8) {
duke@435	964	if (src_enc < 8) {
duke@435	965	prefix(REX_W);
duke@435	966	} else {
duke@435	967	prefix(REX_WB);
duke@435	968	src_enc -= 8;
duke@435	969	}
duke@435	970	} else {
duke@435	971	if (src_enc < 8) {
duke@435	972	prefix(REX_WR);
duke@435	973	} else {
duke@435	974	prefix(REX_WRB);
duke@435	975	src_enc -= 8;
duke@435	976	}
duke@435	977	dst_enc -= 8;
duke@435	978	}
duke@435	979	return dst_enc << 3 | src_enc;
duke@435	980	}
duke@435	981
duke@435	982	void Assembler::prefix(Register reg) {
duke@435	983	if (reg->encoding() >= 8) {
duke@435	984	prefix(REX_B);
duke@435	985	}
duke@435	986	}
duke@435	987
duke@435	988	void Assembler::prefix(Address adr) {
duke@435	989	if (adr.base_needs_rex()) {
duke@435	990	if (adr.index_needs_rex()) {
duke@435	991	prefix(REX_XB);
duke@435	992	} else {
duke@435	993	prefix(REX_B);
duke@435	994	}
duke@435	995	} else {
duke@435	996	if (adr.index_needs_rex()) {
duke@435	997	prefix(REX_X);
duke@435	998	}
duke@435	999	}
duke@435	1000	}
duke@435	1001
duke@435	1002	void Assembler::prefixq(Address adr) {
duke@435	1003	if (adr.base_needs_rex()) {
duke@435	1004	if (adr.index_needs_rex()) {
duke@435	1005	prefix(REX_WXB);
duke@435	1006	} else {
duke@435	1007	prefix(REX_WB);
duke@435	1008	}
duke@435	1009	} else {
duke@435	1010	if (adr.index_needs_rex()) {
duke@435	1011	prefix(REX_WX);
duke@435	1012	} else {
duke@435	1013	prefix(REX_W);
duke@435	1014	}
duke@435	1015	}
duke@435	1016	}
duke@435	1017
duke@435	1018
duke@435	1019	void Assembler::prefix(Address adr, Register reg, bool byteinst) {
duke@435	1020	if (reg->encoding() < 8) {
duke@435	1021	if (adr.base_needs_rex()) {
duke@435	1022	if (adr.index_needs_rex()) {
duke@435	1023	prefix(REX_XB);
duke@435	1024	} else {
duke@435	1025	prefix(REX_B);
duke@435	1026	}
duke@435	1027	} else {
duke@435	1028	if (adr.index_needs_rex()) {
duke@435	1029	prefix(REX_X);
duke@435	1030	} else if (reg->encoding() >= 4 ) {
duke@435	1031	prefix(REX);
duke@435	1032	}
duke@435	1033	}
duke@435	1034	} else {
duke@435	1035	if (adr.base_needs_rex()) {
duke@435	1036	if (adr.index_needs_rex()) {
duke@435	1037	prefix(REX_RXB);
duke@435	1038	} else {
duke@435	1039	prefix(REX_RB);
duke@435	1040	}
duke@435	1041	} else {
duke@435	1042	if (adr.index_needs_rex()) {
duke@435	1043	prefix(REX_RX);
duke@435	1044	} else {
duke@435	1045	prefix(REX_R);
duke@435	1046	}
duke@435	1047	}
duke@435	1048	}
duke@435	1049	}
duke@435	1050
duke@435	1051	void Assembler::prefixq(Address adr, Register src) {
duke@435	1052	if (src->encoding() < 8) {
duke@435	1053	if (adr.base_needs_rex()) {
duke@435	1054	if (adr.index_needs_rex()) {
duke@435	1055	prefix(REX_WXB);
duke@435	1056	} else {
duke@435	1057	prefix(REX_WB);
duke@435	1058	}
duke@435	1059	} else {
duke@435	1060	if (adr.index_needs_rex()) {
duke@435	1061	prefix(REX_WX);
duke@435	1062	} else {
duke@435	1063	prefix(REX_W);
duke@435	1064	}
duke@435	1065	}
duke@435	1066	} else {
duke@435	1067	if (adr.base_needs_rex()) {
duke@435	1068	if (adr.index_needs_rex()) {
duke@435	1069	prefix(REX_WRXB);
duke@435	1070	} else {
duke@435	1071	prefix(REX_WRB);
duke@435	1072	}
duke@435	1073	} else {
duke@435	1074	if (adr.index_needs_rex()) {
duke@435	1075	prefix(REX_WRX);
duke@435	1076	} else {
duke@435	1077	prefix(REX_WR);
duke@435	1078	}
duke@435	1079	}
duke@435	1080	}
duke@435	1081	}
duke@435	1082
duke@435	1083	void Assembler::prefix(Address adr, XMMRegister reg) {
duke@435	1084	if (reg->encoding() < 8) {
duke@435	1085	if (adr.base_needs_rex()) {
duke@435	1086	if (adr.index_needs_rex()) {
duke@435	1087	prefix(REX_XB);
duke@435	1088	} else {
duke@435	1089	prefix(REX_B);
duke@435	1090	}
duke@435	1091	} else {
duke@435	1092	if (adr.index_needs_rex()) {
duke@435	1093	prefix(REX_X);
duke@435	1094	}
duke@435	1095	}
duke@435	1096	} else {
duke@435	1097	if (adr.base_needs_rex()) {
duke@435	1098	if (adr.index_needs_rex()) {
duke@435	1099	prefix(REX_RXB);
duke@435	1100	} else {
duke@435	1101	prefix(REX_RB);
duke@435	1102	}
duke@435	1103	} else {
duke@435	1104	if (adr.index_needs_rex()) {
duke@435	1105	prefix(REX_RX);
duke@435	1106	} else {
duke@435	1107	prefix(REX_R);
duke@435	1108	}
duke@435	1109	}
duke@435	1110	}
duke@435	1111	}
duke@435	1112
duke@435	1113	void Assembler::emit_operand(Register reg, Address adr,
duke@435	1114	int rip_relative_correction) {
duke@435	1115	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1116	adr._rspec,
duke@435	1117	rip_relative_correction);
duke@435	1118	}
duke@435	1119
duke@435	1120	void Assembler::emit_operand(XMMRegister reg, Address adr,
duke@435	1121	int rip_relative_correction) {
duke@435	1122	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1123	adr._rspec,
duke@435	1124	rip_relative_correction);
duke@435	1125	}
duke@435	1126
duke@435	1127	void Assembler::emit_farith(int b1, int b2, int i) {
duke@435	1128	assert(isByte(b1) && isByte(b2), "wrong opcode");
duke@435	1129	assert(0 <= i && i < 8, "illegal stack offset");
duke@435	1130	emit_byte(b1);
duke@435	1131	emit_byte(b2 + i);
duke@435	1132	}
duke@435	1133
duke@435	1134	// pushad is invalid, use this instead.
duke@435	1135	// NOTE: Kills flags!!
duke@435	1136	void Assembler::pushaq() {
duke@435	1137	// we have to store original rsp. ABI says that 128 bytes
duke@435	1138	// below rsp are local scratch.
duke@435	1139	movq(Address(rsp, -5 * wordSize), rsp);
duke@435	1140
duke@435	1141	subq(rsp, 16 * wordSize);
duke@435	1142
duke@435	1143	movq(Address(rsp, 15 * wordSize), rax);
duke@435	1144	movq(Address(rsp, 14 * wordSize), rcx);
duke@435	1145	movq(Address(rsp, 13 * wordSize), rdx);
duke@435	1146	movq(Address(rsp, 12 * wordSize), rbx);
duke@435	1147	// skip rsp
duke@435	1148	movq(Address(rsp, 10 * wordSize), rbp);
duke@435	1149	movq(Address(rsp, 9 * wordSize), rsi);
duke@435	1150	movq(Address(rsp, 8 * wordSize), rdi);
duke@435	1151	movq(Address(rsp, 7 * wordSize), r8);
duke@435	1152	movq(Address(rsp, 6 * wordSize), r9);
duke@435	1153	movq(Address(rsp, 5 * wordSize), r10);
duke@435	1154	movq(Address(rsp, 4 * wordSize), r11);
duke@435	1155	movq(Address(rsp, 3 * wordSize), r12);
duke@435	1156	movq(Address(rsp, 2 * wordSize), r13);
duke@435	1157	movq(Address(rsp, wordSize), r14);
duke@435	1158	movq(Address(rsp, 0), r15);
duke@435	1159	}
duke@435	1160
duke@435	1161	// popad is invalid, use this instead
duke@435	1162	// NOTE: Kills flags!!
duke@435	1163	void Assembler::popaq() {
duke@435	1164	movq(r15, Address(rsp, 0));
duke@435	1165	movq(r14, Address(rsp, wordSize));
duke@435	1166	movq(r13, Address(rsp, 2 * wordSize));
duke@435	1167	movq(r12, Address(rsp, 3 * wordSize));
duke@435	1168	movq(r11, Address(rsp, 4 * wordSize));
duke@435	1169	movq(r10, Address(rsp, 5 * wordSize));
duke@435	1170	movq(r9, Address(rsp, 6 * wordSize));
duke@435	1171	movq(r8, Address(rsp, 7 * wordSize));
duke@435	1172	movq(rdi, Address(rsp, 8 * wordSize));
duke@435	1173	movq(rsi, Address(rsp, 9 * wordSize));
duke@435	1174	movq(rbp, Address(rsp, 10 * wordSize));
duke@435	1175	// skip rsp
duke@435	1176	movq(rbx, Address(rsp, 12 * wordSize));
duke@435	1177	movq(rdx, Address(rsp, 13 * wordSize));
duke@435	1178	movq(rcx, Address(rsp, 14 * wordSize));
duke@435	1179	movq(rax, Address(rsp, 15 * wordSize));
duke@435	1180
duke@435	1181	addq(rsp, 16 * wordSize);
duke@435	1182	}
duke@435	1183
duke@435	1184	void Assembler::pushfq() {
duke@435	1185	emit_byte(0x9C);
duke@435	1186	}
duke@435	1187
duke@435	1188	void Assembler::popfq() {
duke@435	1189	emit_byte(0x9D);
duke@435	1190	}
duke@435	1191
duke@435	1192	void Assembler::pushq(int imm32) {
duke@435	1193	emit_byte(0x68);
duke@435	1194	emit_long(imm32);
duke@435	1195	}
duke@435	1196
duke@435	1197	void Assembler::pushq(Register src) {
duke@435	1198	int encode = prefix_and_encode(src->encoding());
duke@435	1199
duke@435	1200	emit_byte(0x50 | encode);
duke@435	1201	}
duke@435	1202
duke@435	1203	void Assembler::pushq(Address src) {
duke@435	1204	InstructionMark im(this);
duke@435	1205	prefix(src);
duke@435	1206	emit_byte(0xFF);
duke@435	1207	emit_operand(rsi, src);
duke@435	1208	}
duke@435	1209
duke@435	1210	void Assembler::popq(Register dst) {
duke@435	1211	int encode = prefix_and_encode(dst->encoding());
duke@435	1212	emit_byte(0x58 | encode);
duke@435	1213	}
duke@435	1214
duke@435	1215	void Assembler::popq(Address dst) {
duke@435	1216	InstructionMark im(this);
duke@435	1217	prefix(dst);
duke@435	1218	emit_byte(0x8F);
duke@435	1219	emit_operand(rax, dst);
duke@435	1220	}
duke@435	1221
duke@435	1222	void Assembler::prefix(Prefix p) {
duke@435	1223	a_byte(p);
duke@435	1224	}
duke@435	1225
duke@435	1226	void Assembler::movb(Register dst, Address src) {
duke@435	1227	InstructionMark im(this);
duke@435	1228	prefix(src, dst, true);
duke@435	1229	emit_byte(0x8A);
duke@435	1230	emit_operand(dst, src);
duke@435	1231	}
duke@435	1232
duke@435	1233	void Assembler::movb(Address dst, int imm8) {
duke@435	1234	InstructionMark im(this);
duke@435	1235	prefix(dst);
duke@435	1236	emit_byte(0xC6);
duke@435	1237	emit_operand(rax, dst, 1);
duke@435	1238	emit_byte(imm8);
duke@435	1239	}
duke@435	1240
duke@435	1241	void Assembler::movb(Address dst, Register src) {
duke@435	1242	InstructionMark im(this);
duke@435	1243	prefix(dst, src, true);
duke@435	1244	emit_byte(0x88);
duke@435	1245	emit_operand(src, dst);
duke@435	1246	}
duke@435	1247
duke@435	1248	void Assembler::movw(Address dst, int imm16) {
duke@435	1249	InstructionMark im(this);
duke@435	1250	emit_byte(0x66); // switch to 16-bit mode
duke@435	1251	prefix(dst);
duke@435	1252	emit_byte(0xC7);
duke@435	1253	emit_operand(rax, dst, 2);
duke@435	1254	emit_word(imm16);
duke@435	1255	}
duke@435	1256
duke@435	1257	void Assembler::movw(Register dst, Address src) {
duke@435	1258	InstructionMark im(this);
duke@435	1259	emit_byte(0x66);
duke@435	1260	prefix(src, dst);
duke@435	1261	emit_byte(0x8B);
duke@435	1262	emit_operand(dst, src);
duke@435	1263	}
duke@435	1264
duke@435	1265	void Assembler::movw(Address dst, Register src) {
duke@435	1266	InstructionMark im(this);
duke@435	1267	emit_byte(0x66);
duke@435	1268	prefix(dst, src);
duke@435	1269	emit_byte(0x89);
duke@435	1270	emit_operand(src, dst);
duke@435	1271	}
duke@435	1272
duke@435	1273	// Uses zero extension.
duke@435	1274	void Assembler::movl(Register dst, int imm32) {
duke@435	1275	int encode = prefix_and_encode(dst->encoding());
duke@435	1276	emit_byte(0xB8 | encode);
duke@435	1277	emit_long(imm32);
duke@435	1278	}
duke@435	1279
duke@435	1280	void Assembler::movl(Register dst, Register src) {
duke@435	1281	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1282	emit_byte(0x8B);
duke@435	1283	emit_byte(0xC0 | encode);
duke@435	1284	}
duke@435	1285
duke@435	1286	void Assembler::movl(Register dst, Address src) {
duke@435	1287	InstructionMark im(this);
duke@435	1288	prefix(src, dst);
duke@435	1289	emit_byte(0x8B);
duke@435	1290	emit_operand(dst, src);
duke@435	1291	}
duke@435	1292
duke@435	1293	void Assembler::movl(Address dst, int imm32) {
duke@435	1294	InstructionMark im(this);
duke@435	1295	prefix(dst);
duke@435	1296	emit_byte(0xC7);
duke@435	1297	emit_operand(rax, dst, 4);
duke@435	1298	emit_long(imm32);
duke@435	1299	}
duke@435	1300
duke@435	1301	void Assembler::movl(Address dst, Register src) {
duke@435	1302	InstructionMark im(this);
duke@435	1303	prefix(dst, src);
duke@435	1304	emit_byte(0x89);
duke@435	1305	emit_operand(src, dst);
duke@435	1306	}
duke@435	1307
dcubed@485	1308	void Assembler::mov64(Register dst, intptr_t imm64) {
duke@435	1309	InstructionMark im(this);
duke@435	1310	int encode = prefixq_and_encode(dst->encoding());
duke@435	1311	emit_byte(0xB8 | encode);
duke@435	1312	emit_long64(imm64);
duke@435	1313	}
duke@435	1314
duke@435	1315	void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
duke@435	1316	InstructionMark im(this);
duke@435	1317	int encode = prefixq_and_encode(dst->encoding());
duke@435	1318	emit_byte(0xB8 | encode);
duke@435	1319	emit_data64(imm64, rspec);
duke@435	1320	}
duke@435	1321
duke@435	1322	void Assembler::movq(Register dst, Register src) {
duke@435	1323	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1324	emit_byte(0x8B);
duke@435	1325	emit_byte(0xC0 | encode);
duke@435	1326	}
duke@435	1327
duke@435	1328	void Assembler::movq(Register dst, Address src) {
duke@435	1329	InstructionMark im(this);
duke@435	1330	prefixq(src, dst);
duke@435	1331	emit_byte(0x8B);
duke@435	1332	emit_operand(dst, src);
duke@435	1333	}
duke@435	1334
dcubed@485	1335	void Assembler::mov64(Address dst, intptr_t imm32) {
duke@435	1336	assert(is_simm32(imm32), "lost bits");
duke@435	1337	InstructionMark im(this);
duke@435	1338	prefixq(dst);
duke@435	1339	emit_byte(0xC7);
duke@435	1340	emit_operand(rax, dst, 4);
duke@435	1341	emit_long(imm32);
duke@435	1342	}
duke@435	1343
duke@435	1344	void Assembler::movq(Address dst, Register src) {
duke@435	1345	InstructionMark im(this);
duke@435	1346	prefixq(dst, src);
duke@435	1347	emit_byte(0x89);
duke@435	1348	emit_operand(src, dst);
duke@435	1349	}
duke@435	1350
duke@435	1351	void Assembler::movsbl(Register dst, Address src) {
duke@435	1352	InstructionMark im(this);
duke@435	1353	prefix(src, dst);
duke@435	1354	emit_byte(0x0F);
duke@435	1355	emit_byte(0xBE);
duke@435	1356	emit_operand(dst, src);
duke@435	1357	}
duke@435	1358
duke@435	1359	void Assembler::movsbl(Register dst, Register src) {
duke@435	1360	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1361	emit_byte(0x0F);
duke@435	1362	emit_byte(0xBE);
duke@435	1363	emit_byte(0xC0 | encode);
duke@435	1364	}
duke@435	1365
duke@435	1366	void Assembler::movswl(Register dst, Address src) {
duke@435	1367	InstructionMark im(this);
duke@435	1368	prefix(src, dst);
duke@435	1369	emit_byte(0x0F);
duke@435	1370	emit_byte(0xBF);
duke@435	1371	emit_operand(dst, src);
duke@435	1372	}
duke@435	1373
duke@435	1374	void Assembler::movswl(Register dst, Register src) {
duke@435	1375	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1376	emit_byte(0x0F);
duke@435	1377	emit_byte(0xBF);
duke@435	1378	emit_byte(0xC0 | encode);
duke@435	1379	}
duke@435	1380
duke@435	1381	void Assembler::movslq(Register dst, Address src) {
duke@435	1382	InstructionMark im(this);
duke@435	1383	prefixq(src, dst);
duke@435	1384	emit_byte(0x63);
duke@435	1385	emit_operand(dst, src);
duke@435	1386	}
duke@435	1387
duke@435	1388	void Assembler::movslq(Register dst, Register src) {
duke@435	1389	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1390	emit_byte(0x63);
duke@435	1391	emit_byte(0xC0 | encode);
duke@435	1392	}
duke@435	1393
duke@435	1394	void Assembler::movzbl(Register dst, Address src) {
duke@435	1395	InstructionMark im(this);
duke@435	1396	prefix(src, dst);
duke@435	1397	emit_byte(0x0F);
duke@435	1398	emit_byte(0xB6);
duke@435	1399	emit_operand(dst, src);
duke@435	1400	}
duke@435	1401
duke@435	1402	void Assembler::movzbl(Register dst, Register src) {
duke@435	1403	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1404	emit_byte(0x0F);
duke@435	1405	emit_byte(0xB6);
duke@435	1406	emit_byte(0xC0 | encode);
duke@435	1407	}
duke@435	1408
duke@435	1409	void Assembler::movzwl(Register dst, Address src) {
duke@435	1410	InstructionMark im(this);
duke@435	1411	prefix(src, dst);
duke@435	1412	emit_byte(0x0F);
duke@435	1413	emit_byte(0xB7);
duke@435	1414	emit_operand(dst, src);
duke@435	1415	}
duke@435	1416
duke@435	1417	void Assembler::movzwl(Register dst, Register src) {
duke@435	1418	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1419	emit_byte(0x0F);
duke@435	1420	emit_byte(0xB7);
duke@435	1421	emit_byte(0xC0 | encode);
duke@435	1422	}
duke@435	1423
duke@435	1424	void Assembler::movss(XMMRegister dst, XMMRegister src) {
duke@435	1425	emit_byte(0xF3);
duke@435	1426	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1427	emit_byte(0x0F);
duke@435	1428	emit_byte(0x10);
duke@435	1429	emit_byte(0xC0 | encode);
duke@435	1430	}
duke@435	1431
duke@435	1432	void Assembler::movss(XMMRegister dst, Address src) {
duke@435	1433	InstructionMark im(this);
duke@435	1434	emit_byte(0xF3);
duke@435	1435	prefix(src, dst);
duke@435	1436	emit_byte(0x0F);
duke@435	1437	emit_byte(0x10);
duke@435	1438	emit_operand(dst, src);
duke@435	1439	}
duke@435	1440
duke@435	1441	void Assembler::movss(Address dst, XMMRegister src) {
duke@435	1442	InstructionMark im(this);
duke@435	1443	emit_byte(0xF3);
duke@435	1444	prefix(dst, src);
duke@435	1445	emit_byte(0x0F);
duke@435	1446	emit_byte(0x11);
duke@435	1447	emit_operand(src, dst);
duke@435	1448	}
duke@435	1449
duke@435	1450	void Assembler::movsd(XMMRegister dst, XMMRegister src) {
duke@435	1451	emit_byte(0xF2);
duke@435	1452	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1453	emit_byte(0x0F);
duke@435	1454	emit_byte(0x10);
duke@435	1455	emit_byte(0xC0 | encode);
duke@435	1456	}
duke@435	1457
duke@435	1458	void Assembler::movsd(XMMRegister dst, Address src) {
duke@435	1459	InstructionMark im(this);
duke@435	1460	emit_byte(0xF2);
duke@435	1461	prefix(src, dst);
duke@435	1462	emit_byte(0x0F);
duke@435	1463	emit_byte(0x10);
duke@435	1464	emit_operand(dst, src);
duke@435	1465	}
duke@435	1466
duke@435	1467	void Assembler::movsd(Address dst, XMMRegister src) {
duke@435	1468	InstructionMark im(this);
duke@435	1469	emit_byte(0xF2);
duke@435	1470	prefix(dst, src);
duke@435	1471	emit_byte(0x0F);
duke@435	1472	emit_byte(0x11);
duke@435	1473	emit_operand(src, dst);
duke@435	1474	}
duke@435	1475
duke@435	1476	// New cpus require to use movsd and movss to avoid partial register stall
duke@435	1477	// when loading from memory. But for old Opteron use movlpd instead of movsd.
duke@435	1478	// The selection is done in MacroAssembler::movdbl() and movflt().
duke@435	1479	void Assembler::movlpd(XMMRegister dst, Address src) {
duke@435	1480	InstructionMark im(this);
duke@435	1481	emit_byte(0x66);
duke@435	1482	prefix(src, dst);
duke@435	1483	emit_byte(0x0F);
duke@435	1484	emit_byte(0x12);
duke@435	1485	emit_operand(dst, src);
duke@435	1486	}
duke@435	1487
duke@435	1488	void Assembler::movapd(XMMRegister dst, XMMRegister src) {
duke@435	1489	int dstenc = dst->encoding();
duke@435	1490	int srcenc = src->encoding();
duke@435	1491	emit_byte(0x66);
duke@435	1492	if (dstenc < 8) {
duke@435	1493	if (srcenc >= 8) {
duke@435	1494	prefix(REX_B);
duke@435	1495	srcenc -= 8;
duke@435	1496	}
duke@435	1497	} else {
duke@435	1498	if (srcenc < 8) {
duke@435	1499	prefix(REX_R);
duke@435	1500	} else {
duke@435	1501	prefix(REX_RB);
duke@435	1502	srcenc -= 8;
duke@435	1503	}
duke@435	1504	dstenc -= 8;
duke@435	1505	}
duke@435	1506	emit_byte(0x0F);
duke@435	1507	emit_byte(0x28);
duke@435	1508	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1509	}
duke@435	1510
duke@435	1511	void Assembler::movaps(XMMRegister dst, XMMRegister src) {
duke@435	1512	int dstenc = dst->encoding();
duke@435	1513	int srcenc = src->encoding();
duke@435	1514	if (dstenc < 8) {
duke@435	1515	if (srcenc >= 8) {
duke@435	1516	prefix(REX_B);
duke@435	1517	srcenc -= 8;
duke@435	1518	}
duke@435	1519	} else {
duke@435	1520	if (srcenc < 8) {
duke@435	1521	prefix(REX_R);
duke@435	1522	} else {
duke@435	1523	prefix(REX_RB);
duke@435	1524	srcenc -= 8;
duke@435	1525	}
duke@435	1526	dstenc -= 8;
duke@435	1527	}
duke@435	1528	emit_byte(0x0F);
duke@435	1529	emit_byte(0x28);
duke@435	1530	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1531	}
duke@435	1532
duke@435	1533	void Assembler::movdl(XMMRegister dst, Register src) {
duke@435	1534	emit_byte(0x66);
duke@435	1535	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1536	emit_byte(0x0F);
duke@435	1537	emit_byte(0x6E);
duke@435	1538	emit_byte(0xC0 | encode);
duke@435	1539	}
duke@435	1540
duke@435	1541	void Assembler::movdl(Register dst, XMMRegister src) {
duke@435	1542	emit_byte(0x66);
duke@435	1543	// swap src/dst to get correct prefix
duke@435	1544	int encode = prefix_and_encode(src->encoding(), dst->encoding());
duke@435	1545	emit_byte(0x0F);
duke@435	1546	emit_byte(0x7E);
duke@435	1547	emit_byte(0xC0 | encode);
duke@435	1548	}
duke@435	1549
duke@435	1550	void Assembler::movdq(XMMRegister dst, Register src) {
duke@435	1551	emit_byte(0x66);
duke@435	1552	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1553	emit_byte(0x0F);
duke@435	1554	emit_byte(0x6E);
duke@435	1555	emit_byte(0xC0 | encode);
duke@435	1556	}
duke@435	1557
duke@435	1558	void Assembler::movdq(Register dst, XMMRegister src) {
duke@435	1559	emit_byte(0x66);
duke@435	1560	// swap src/dst to get correct prefix
duke@435	1561	int encode = prefixq_and_encode(src->encoding(), dst->encoding());
duke@435	1562	emit_byte(0x0F);
duke@435	1563	emit_byte(0x7E);
duke@435	1564	emit_byte(0xC0 | encode);
duke@435	1565	}
duke@435	1566
duke@435	1567	void Assembler::pxor(XMMRegister dst, Address src) {
duke@435	1568	InstructionMark im(this);
duke@435	1569	emit_byte(0x66);
duke@435	1570	prefix(src, dst);
duke@435	1571	emit_byte(0x0F);
duke@435	1572	emit_byte(0xEF);
duke@435	1573	emit_operand(dst, src);
duke@435	1574	}
duke@435	1575
duke@435	1576	void Assembler::pxor(XMMRegister dst, XMMRegister src) {
duke@435	1577	InstructionMark im(this);
duke@435	1578	emit_byte(0x66);
duke@435	1579	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1580	emit_byte(0x0F);
duke@435	1581	emit_byte(0xEF);
duke@435	1582	emit_byte(0xC0 | encode);
duke@435	1583	}
duke@435	1584
duke@435	1585	void Assembler::movdqa(XMMRegister dst, Address src) {
duke@435	1586	InstructionMark im(this);
duke@435	1587	emit_byte(0x66);
duke@435	1588	prefix(src, dst);
duke@435	1589	emit_byte(0x0F);
duke@435	1590	emit_byte(0x6F);
duke@435	1591	emit_operand(dst, src);
duke@435	1592	}
duke@435	1593
duke@435	1594	void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
duke@435	1595	emit_byte(0x66);
duke@435	1596	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1597	emit_byte(0x0F);
duke@435	1598	emit_byte(0x6F);
duke@435	1599	emit_byte(0xC0 | encode);
duke@435	1600	}
duke@435	1601
duke@435	1602	void Assembler::movdqa(Address dst, XMMRegister src) {
duke@435	1603	InstructionMark im(this);
duke@435	1604	emit_byte(0x66);
duke@435	1605	prefix(dst, src);
duke@435	1606	emit_byte(0x0F);
duke@435	1607	emit_byte(0x7F);
duke@435	1608	emit_operand(src, dst);
duke@435	1609	}
duke@435	1610
duke@435	1611	void Assembler::movq(XMMRegister dst, Address src) {
duke@435	1612	InstructionMark im(this);
duke@435	1613	emit_byte(0xF3);
duke@435	1614	prefix(src, dst);
duke@435	1615	emit_byte(0x0F);
duke@435	1616	emit_byte(0x7E);
duke@435	1617	emit_operand(dst, src);
duke@435	1618	}
duke@435	1619
duke@435	1620	void Assembler::movq(Address dst, XMMRegister src) {
duke@435	1621	InstructionMark im(this);
duke@435	1622	emit_byte(0x66);
duke@435	1623	prefix(dst, src);
duke@435	1624	emit_byte(0x0F);
duke@435	1625	emit_byte(0xD6);
duke@435	1626	emit_operand(src, dst);
duke@435	1627	}
duke@435	1628
duke@435	1629	void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1630	assert(isByte(mode), "invalid value");
duke@435	1631	emit_byte(0x66);
duke@435	1632	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1633	emit_byte(0x0F);
duke@435	1634	emit_byte(0x70);
duke@435	1635	emit_byte(0xC0 | encode);
duke@435	1636	emit_byte(mode & 0xFF);
duke@435	1637	}
duke@435	1638
duke@435	1639	void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
duke@435	1640	assert(isByte(mode), "invalid value");
duke@435	1641	InstructionMark im(this);
duke@435	1642	emit_byte(0x66);
duke@435	1643	emit_byte(0x0F);
duke@435	1644	emit_byte(0x70);
duke@435	1645	emit_operand(dst, src);
duke@435	1646	emit_byte(mode & 0xFF);
duke@435	1647	}
duke@435	1648
duke@435	1649	void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1650	assert(isByte(mode), "invalid value");
duke@435	1651	emit_byte(0xF2);
duke@435	1652	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1653	emit_byte(0x0F);
duke@435	1654	emit_byte(0x70);
duke@435	1655	emit_byte(0xC0 | encode);
duke@435	1656	emit_byte(mode & 0xFF);
duke@435	1657	}
duke@435	1658
duke@435	1659	void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
duke@435	1660	assert(isByte(mode), "invalid value");
duke@435	1661	InstructionMark im(this);
duke@435	1662	emit_byte(0xF2);
duke@435	1663	emit_byte(0x0F);
duke@435	1664	emit_byte(0x70);
duke@435	1665	emit_operand(dst, src);
duke@435	1666	emit_byte(mode & 0xFF);
duke@435	1667	}
duke@435	1668
duke@435	1669	void Assembler::cmovl(Condition cc, Register dst, Register src) {
duke@435	1670	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1671	emit_byte(0x0F);
duke@435	1672	emit_byte(0x40 | cc);
duke@435	1673	emit_byte(0xC0 | encode);
duke@435	1674	}
duke@435	1675
duke@435	1676	void Assembler::cmovl(Condition cc, Register dst, Address src) {
duke@435	1677	InstructionMark im(this);
duke@435	1678	prefix(src, dst);
duke@435	1679	emit_byte(0x0F);
duke@435	1680	emit_byte(0x40 | cc);
duke@435	1681	emit_operand(dst, src);
duke@435	1682	}
duke@435	1683
duke@435	1684	void Assembler::cmovq(Condition cc, Register dst, Register src) {
duke@435	1685	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1686	emit_byte(0x0F);
duke@435	1687	emit_byte(0x40 | cc);
duke@435	1688	emit_byte(0xC0 | encode);
duke@435	1689	}
duke@435	1690
duke@435	1691	void Assembler::cmovq(Condition cc, Register dst, Address src) {
duke@435	1692	InstructionMark im(this);
duke@435	1693	prefixq(src, dst);
duke@435	1694	emit_byte(0x0F);
duke@435	1695	emit_byte(0x40 | cc);
duke@435	1696	emit_operand(dst, src);
duke@435	1697	}
duke@435	1698
duke@435	1699	void Assembler::prefetch_prefix(Address src) {
duke@435	1700	prefix(src);
duke@435	1701	emit_byte(0x0F);
duke@435	1702	}
duke@435	1703
duke@435	1704	void Assembler::prefetcht0(Address src) {
duke@435	1705	InstructionMark im(this);
duke@435	1706	prefetch_prefix(src);
duke@435	1707	emit_byte(0x18);
duke@435	1708	emit_operand(rcx, src); // 1, src
duke@435	1709	}
duke@435	1710
duke@435	1711	void Assembler::prefetcht1(Address src) {
duke@435	1712	InstructionMark im(this);
duke@435	1713	prefetch_prefix(src);
duke@435	1714	emit_byte(0x18);
duke@435	1715	emit_operand(rdx, src); // 2, src
duke@435	1716	}
duke@435	1717
duke@435	1718	void Assembler::prefetcht2(Address src) {
duke@435	1719	InstructionMark im(this);
duke@435	1720	prefetch_prefix(src);
duke@435	1721	emit_byte(0x18);
duke@435	1722	emit_operand(rbx, src); // 3, src
duke@435	1723	}
duke@435	1724
duke@435	1725	void Assembler::prefetchnta(Address src) {
duke@435	1726	InstructionMark im(this);
duke@435	1727	prefetch_prefix(src);
duke@435	1728	emit_byte(0x18);
duke@435	1729	emit_operand(rax, src); // 0, src
duke@435	1730	}
duke@435	1731
duke@435	1732	void Assembler::prefetchw(Address src) {
duke@435	1733	InstructionMark im(this);
duke@435	1734	prefetch_prefix(src);
duke@435	1735	emit_byte(0x0D);
duke@435	1736	emit_operand(rcx, src); // 1, src
duke@435	1737	}
duke@435	1738
duke@435	1739	void Assembler::adcl(Register dst, int imm32) {
duke@435	1740	prefix(dst);
duke@435	1741	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1742	}
duke@435	1743
duke@435	1744	void Assembler::adcl(Register dst, Address src) {
duke@435	1745	InstructionMark im(this);
duke@435	1746	prefix(src, dst);
duke@435	1747	emit_byte(0x13);
duke@435	1748	emit_operand(dst, src);
duke@435	1749	}
duke@435	1750
duke@435	1751	void Assembler::adcl(Register dst, Register src) {
duke@435	1752	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1753	emit_arith(0x13, 0xC0, dst, src);
duke@435	1754	}
duke@435	1755
duke@435	1756	void Assembler::adcq(Register dst, int imm32) {
duke@435	1757	(void) prefixq_and_encode(dst->encoding());
duke@435	1758	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1759	}
duke@435	1760
duke@435	1761	void Assembler::adcq(Register dst, Address src) {
duke@435	1762	InstructionMark im(this);
duke@435	1763	prefixq(src, dst);
duke@435	1764	emit_byte(0x13);
duke@435	1765	emit_operand(dst, src);
duke@435	1766	}
duke@435	1767
duke@435	1768	void Assembler::adcq(Register dst, Register src) {
duke@435	1769	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1770	emit_arith(0x13, 0xC0, dst, src);
duke@435	1771	}
duke@435	1772
duke@435	1773	void Assembler::addl(Address dst, int imm32) {
duke@435	1774	InstructionMark im(this);
duke@435	1775	prefix(dst);
duke@435	1776	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1777	}
duke@435	1778
duke@435	1779	void Assembler::addl(Address dst, Register src) {
duke@435	1780	InstructionMark im(this);
duke@435	1781	prefix(dst, src);
duke@435	1782	emit_byte(0x01);
duke@435	1783	emit_operand(src, dst);
duke@435	1784	}
duke@435	1785
duke@435	1786	void Assembler::addl(Register dst, int imm32) {
duke@435	1787	prefix(dst);
duke@435	1788	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1789	}
duke@435	1790
duke@435	1791	void Assembler::addl(Register dst, Address src) {
duke@435	1792	InstructionMark im(this);
duke@435	1793	prefix(src, dst);
duke@435	1794	emit_byte(0x03);
duke@435	1795	emit_operand(dst, src);
duke@435	1796	}
duke@435	1797
duke@435	1798	void Assembler::addl(Register dst, Register src) {
duke@435	1799	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1800	emit_arith(0x03, 0xC0, dst, src);
duke@435	1801	}
duke@435	1802
duke@435	1803	void Assembler::addq(Address dst, int imm32) {
duke@435	1804	InstructionMark im(this);
duke@435	1805	prefixq(dst);
duke@435	1806	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1807	}
duke@435	1808
duke@435	1809	void Assembler::addq(Address dst, Register src) {
duke@435	1810	InstructionMark im(this);
duke@435	1811	prefixq(dst, src);
duke@435	1812	emit_byte(0x01);
duke@435	1813	emit_operand(src, dst);
duke@435	1814	}
duke@435	1815
duke@435	1816	void Assembler::addq(Register dst, int imm32) {
duke@435	1817	(void) prefixq_and_encode(dst->encoding());
duke@435	1818	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1819	}
duke@435	1820
duke@435	1821	void Assembler::addq(Register dst, Address src) {
duke@435	1822	InstructionMark im(this);
duke@435	1823	prefixq(src, dst);
duke@435	1824	emit_byte(0x03);
duke@435	1825	emit_operand(dst, src);
duke@435	1826	}
duke@435	1827
duke@435	1828	void Assembler::addq(Register dst, Register src) {
duke@435	1829	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1830	emit_arith(0x03, 0xC0, dst, src);
duke@435	1831	}
duke@435	1832
duke@435	1833	void Assembler::andl(Register dst, int imm32) {
duke@435	1834	prefix(dst);
duke@435	1835	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1836	}
duke@435	1837
duke@435	1838	void Assembler::andl(Register dst, Address src) {
duke@435	1839	InstructionMark im(this);
duke@435	1840	prefix(src, dst);
duke@435	1841	emit_byte(0x23);
duke@435	1842	emit_operand(dst, src);
duke@435	1843	}
duke@435	1844
duke@435	1845	void Assembler::andl(Register dst, Register src) {
duke@435	1846	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1847	emit_arith(0x23, 0xC0, dst, src);
duke@435	1848	}
duke@435	1849
duke@435	1850	void Assembler::andq(Register dst, int imm32) {
duke@435	1851	(void) prefixq_and_encode(dst->encoding());
duke@435	1852	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1853	}
duke@435	1854
duke@435	1855	void Assembler::andq(Register dst, Address src) {
duke@435	1856	InstructionMark im(this);
duke@435	1857	prefixq(src, dst);
duke@435	1858	emit_byte(0x23);
duke@435	1859	emit_operand(dst, src);
duke@435	1860	}
duke@435	1861
duke@435	1862	void Assembler::andq(Register dst, Register src) {
duke@435	1863	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1864	emit_arith(0x23, 0xC0, dst, src);
duke@435	1865	}
duke@435	1866
duke@435	1867	void Assembler::cmpb(Address dst, int imm8) {
duke@435	1868	InstructionMark im(this);
duke@435	1869	prefix(dst);
duke@435	1870	emit_byte(0x80);
duke@435	1871	emit_operand(rdi, dst, 1);
duke@435	1872	emit_byte(imm8);
duke@435	1873	}
duke@435	1874
duke@435	1875	void Assembler::cmpl(Address dst, int imm32) {
duke@435	1876	InstructionMark im(this);
duke@435	1877	prefix(dst);
duke@435	1878	emit_byte(0x81);
duke@435	1879	emit_operand(rdi, dst, 4);
duke@435	1880	emit_long(imm32);
duke@435	1881	}
duke@435	1882
duke@435	1883	void Assembler::cmpl(Register dst, int imm32) {
duke@435	1884	prefix(dst);
duke@435	1885	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1886	}
duke@435	1887
duke@435	1888	void Assembler::cmpl(Register dst, Register src) {
duke@435	1889	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1890	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1891	}
duke@435	1892
duke@435	1893	void Assembler::cmpl(Register dst, Address src) {
duke@435	1894	InstructionMark im(this);
duke@435	1895	prefix(src, dst);
duke@435	1896	emit_byte(0x3B);
duke@435	1897	emit_operand(dst, src);
duke@435	1898	}
duke@435	1899
duke@435	1900	void Assembler::cmpq(Address dst, int imm32) {
duke@435	1901	InstructionMark im(this);
duke@435	1902	prefixq(dst);
duke@435	1903	emit_byte(0x81);
duke@435	1904	emit_operand(rdi, dst, 4);
duke@435	1905	emit_long(imm32);
duke@435	1906	}
duke@435	1907
duke@435	1908	void Assembler::cmpq(Register dst, int imm32) {
duke@435	1909	(void) prefixq_and_encode(dst->encoding());
duke@435	1910	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1911	}
duke@435	1912
duke@435	1913	void Assembler::cmpq(Address dst, Register src) {
duke@435	1914	prefixq(dst, src);
duke@435	1915	emit_byte(0x3B);
duke@435	1916	emit_operand(src, dst);
duke@435	1917	}
duke@435	1918
duke@435	1919	void Assembler::cmpq(Register dst, Register src) {
duke@435	1920	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1921	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1922	}
duke@435	1923
duke@435	1924	void Assembler::cmpq(Register dst, Address src) {
duke@435	1925	InstructionMark im(this);
duke@435	1926	prefixq(src, dst);
duke@435	1927	emit_byte(0x3B);
duke@435	1928	emit_operand(dst, src);
duke@435	1929	}
duke@435	1930
duke@435	1931	void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
duke@435	1932	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1933	emit_byte(0x0F);
duke@435	1934	emit_byte(0x2E);
duke@435	1935	emit_byte(0xC0 | encode);
duke@435	1936	}
duke@435	1937
duke@435	1938	void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
duke@435	1939	emit_byte(0x66);
duke@435	1940	ucomiss(dst, src);
duke@435	1941	}
duke@435	1942
duke@435	1943	void Assembler::decl(Register dst) {
duke@435	1944	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1945	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1946	int encode = prefix_and_encode(dst->encoding());
duke@435	1947	emit_byte(0xFF);
duke@435	1948	emit_byte(0xC8 | encode);
duke@435	1949	}
duke@435	1950
duke@435	1951	void Assembler::decl(Address dst) {
duke@435	1952	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1953	InstructionMark im(this);
duke@435	1954	prefix(dst);
duke@435	1955	emit_byte(0xFF);
duke@435	1956	emit_operand(rcx, dst);
duke@435	1957	}
duke@435	1958
duke@435	1959	void Assembler::decq(Register dst) {
duke@435	1960	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1961	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1962	int encode = prefixq_and_encode(dst->encoding());
duke@435	1963	emit_byte(0xFF);
duke@435	1964	emit_byte(0xC8 | encode);
duke@435	1965	}
duke@435	1966
duke@435	1967	void Assembler::decq(Address dst) {
duke@435	1968	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1969	InstructionMark im(this);
duke@435	1970	prefixq(dst);
duke@435	1971	emit_byte(0xFF);
duke@435	1972	emit_operand(rcx, dst);
duke@435	1973	}
duke@435	1974
duke@435	1975	void Assembler::idivl(Register src) {
duke@435	1976	int encode = prefix_and_encode(src->encoding());
duke@435	1977	emit_byte(0xF7);
duke@435	1978	emit_byte(0xF8 | encode);
duke@435	1979	}
duke@435	1980
duke@435	1981	void Assembler::idivq(Register src) {
duke@435	1982	int encode = prefixq_and_encode(src->encoding());
duke@435	1983	emit_byte(0xF7);
duke@435	1984	emit_byte(0xF8 | encode);
duke@435	1985	}
duke@435	1986
duke@435	1987	void Assembler::cdql() {
duke@435	1988	emit_byte(0x99);
duke@435	1989	}
duke@435	1990
duke@435	1991	void Assembler::cdqq() {
duke@435	1992	prefix(REX_W);
duke@435	1993	emit_byte(0x99);
duke@435	1994	}
duke@435	1995
duke@435	1996	void Assembler::imull(Register dst, Register src) {
duke@435	1997	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1998	emit_byte(0x0F);
duke@435	1999	emit_byte(0xAF);
duke@435	2000	emit_byte(0xC0 | encode);
duke@435	2001	}
duke@435	2002
duke@435	2003	void Assembler::imull(Register dst, Register src, int value) {
duke@435	2004	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2005	if (is8bit(value)) {
duke@435	2006	emit_byte(0x6B);
duke@435	2007	emit_byte(0xC0 | encode);
duke@435	2008	emit_byte(value);
duke@435	2009	} else {
duke@435	2010	emit_byte(0x69);
duke@435	2011	emit_byte(0xC0 | encode);
duke@435	2012	emit_long(value);
duke@435	2013	}
duke@435	2014	}
duke@435	2015
duke@435	2016	void Assembler::imulq(Register dst, Register src) {
duke@435	2017	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2018	emit_byte(0x0F);
duke@435	2019	emit_byte(0xAF);
duke@435	2020	emit_byte(0xC0 | encode);
duke@435	2021	}
duke@435	2022
duke@435	2023	void Assembler::imulq(Register dst, Register src, int value) {
duke@435	2024	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2025	if (is8bit(value)) {
duke@435	2026	emit_byte(0x6B);
duke@435	2027	emit_byte(0xC0 | encode);
duke@435	2028	emit_byte(value);
duke@435	2029	} else {
duke@435	2030	emit_byte(0x69);
duke@435	2031	emit_byte(0xC0 | encode);
duke@435	2032	emit_long(value);
duke@435	2033	}
duke@435	2034	}
duke@435	2035
duke@435	2036	void Assembler::incl(Register dst) {
duke@435	2037	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2038	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2039	int encode = prefix_and_encode(dst->encoding());
duke@435	2040	emit_byte(0xFF);
duke@435	2041	emit_byte(0xC0 | encode);
duke@435	2042	}
duke@435	2043
duke@435	2044	void Assembler::incl(Address dst) {
duke@435	2045	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2046	InstructionMark im(this);
duke@435	2047	prefix(dst);
duke@435	2048	emit_byte(0xFF);
duke@435	2049	emit_operand(rax, dst);
duke@435	2050	}
duke@435	2051
duke@435	2052	void Assembler::incq(Register dst) {
duke@435	2053	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2054	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2055	int encode = prefixq_and_encode(dst->encoding());
duke@435	2056	emit_byte(0xFF);
duke@435	2057	emit_byte(0xC0 | encode);
duke@435	2058	}
duke@435	2059
duke@435	2060	void Assembler::incq(Address dst) {
duke@435	2061	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2062	InstructionMark im(this);
duke@435	2063	prefixq(dst);
duke@435	2064	emit_byte(0xFF);
duke@435	2065	emit_operand(rax, dst);
duke@435	2066	}
duke@435	2067
duke@435	2068	void Assembler::leal(Register dst, Address src) {
duke@435	2069	InstructionMark im(this);
duke@435	2070	emit_byte(0x67); // addr32
duke@435	2071	prefix(src, dst);
duke@435	2072	emit_byte(0x8D);
duke@435	2073	emit_operand(dst, src);
duke@435	2074	}
duke@435	2075
duke@435	2076	void Assembler::leaq(Register dst, Address src) {
duke@435	2077	InstructionMark im(this);
duke@435	2078	prefixq(src, dst);
duke@435	2079	emit_byte(0x8D);
duke@435	2080	emit_operand(dst, src);
duke@435	2081	}
duke@435	2082
duke@435	2083	void Assembler::mull(Address src) {
duke@435	2084	InstructionMark im(this);
duke@435	2085	// was missing
duke@435	2086	prefix(src);
duke@435	2087	emit_byte(0xF7);
duke@435	2088	emit_operand(rsp, src);
duke@435	2089	}
duke@435	2090
duke@435	2091	void Assembler::mull(Register src) {
duke@435	2092	// was missing
duke@435	2093	int encode = prefix_and_encode(src->encoding());
duke@435	2094	emit_byte(0xF7);
duke@435	2095	emit_byte(0xE0 | encode);
duke@435	2096	}
duke@435	2097
duke@435	2098	void Assembler::negl(Register dst) {
duke@435	2099	int encode = prefix_and_encode(dst->encoding());
duke@435	2100	emit_byte(0xF7);
duke@435	2101	emit_byte(0xD8 | encode);
duke@435	2102	}
duke@435	2103
duke@435	2104	void Assembler::negq(Register dst) {
duke@435	2105	int encode = prefixq_and_encode(dst->encoding());
duke@435	2106	emit_byte(0xF7);
duke@435	2107	emit_byte(0xD8 | encode);
duke@435	2108	}
duke@435	2109
duke@435	2110	void Assembler::notl(Register dst) {
duke@435	2111	int encode = prefix_and_encode(dst->encoding());
duke@435	2112	emit_byte(0xF7);
duke@435	2113	emit_byte(0xD0 | encode);
duke@435	2114	}
duke@435	2115
duke@435	2116	void Assembler::notq(Register dst) {
duke@435	2117	int encode = prefixq_and_encode(dst->encoding());
duke@435	2118	emit_byte(0xF7);
duke@435	2119	emit_byte(0xD0 | encode);
duke@435	2120	}
duke@435	2121
duke@435	2122	void Assembler::orl(Address dst, int imm32) {
duke@435	2123	InstructionMark im(this);
duke@435	2124	prefix(dst);
duke@435	2125	emit_byte(0x81);
duke@435	2126	emit_operand(rcx, dst, 4);
duke@435	2127	emit_long(imm32);
duke@435	2128	}
duke@435	2129
duke@435	2130	void Assembler::orl(Register dst, int imm32) {
duke@435	2131	prefix(dst);
duke@435	2132	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2133	}
duke@435	2134
duke@435	2135	void Assembler::orl(Register dst, Address src) {
duke@435	2136	InstructionMark im(this);
duke@435	2137	prefix(src, dst);
duke@435	2138	emit_byte(0x0B);
duke@435	2139	emit_operand(dst, src);
duke@435	2140	}
duke@435	2141
duke@435	2142	void Assembler::orl(Register dst, Register src) {
duke@435	2143	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2144	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2145	}
duke@435	2146
duke@435	2147	void Assembler::orq(Address dst, int imm32) {
duke@435	2148	InstructionMark im(this);
duke@435	2149	prefixq(dst);
duke@435	2150	emit_byte(0x81);
duke@435	2151	emit_operand(rcx, dst, 4);
duke@435	2152	emit_long(imm32);
duke@435	2153	}
duke@435	2154
duke@435	2155	void Assembler::orq(Register dst, int imm32) {
duke@435	2156	(void) prefixq_and_encode(dst->encoding());
duke@435	2157	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2158	}
duke@435	2159
duke@435	2160	void Assembler::orq(Register dst, Address src) {
duke@435	2161	InstructionMark im(this);
duke@435	2162	prefixq(src, dst);
duke@435	2163	emit_byte(0x0B);
duke@435	2164	emit_operand(dst, src);
duke@435	2165	}
duke@435	2166
duke@435	2167	void Assembler::orq(Register dst, Register src) {
duke@435	2168	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2169	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2170	}
duke@435	2171
duke@435	2172	void Assembler::rcll(Register dst, int imm8) {
duke@435	2173	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2174	int encode = prefix_and_encode(dst->encoding());
duke@435	2175	if (imm8 == 1) {
duke@435	2176	emit_byte(0xD1);
duke@435	2177	emit_byte(0xD0 | encode);
duke@435	2178	} else {
duke@435	2179	emit_byte(0xC1);
duke@435	2180	emit_byte(0xD0 | encode);
duke@435	2181	emit_byte(imm8);
duke@435	2182	}
duke@435	2183	}
duke@435	2184
duke@435	2185	void Assembler::rclq(Register dst, int imm8) {
duke@435	2186	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2187	int encode = prefixq_and_encode(dst->encoding());
duke@435	2188	if (imm8 == 1) {
duke@435	2189	emit_byte(0xD1);
duke@435	2190	emit_byte(0xD0 | encode);
duke@435	2191	} else {
duke@435	2192	emit_byte(0xC1);
duke@435	2193	emit_byte(0xD0 | encode);
duke@435	2194	emit_byte(imm8);
duke@435	2195	}
duke@435	2196	}
duke@435	2197
duke@435	2198	void Assembler::sarl(Register dst, int imm8) {
duke@435	2199	int encode = prefix_and_encode(dst->encoding());
duke@435	2200	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2201	if (imm8 == 1) {
duke@435	2202	emit_byte(0xD1);
duke@435	2203	emit_byte(0xF8 | encode);
duke@435	2204	} else {
duke@435	2205	emit_byte(0xC1);
duke@435	2206	emit_byte(0xF8 | encode);
duke@435	2207	emit_byte(imm8);
duke@435	2208	}
duke@435	2209	}
duke@435	2210
duke@435	2211	void Assembler::sarl(Register dst) {
duke@435	2212	int encode = prefix_and_encode(dst->encoding());
duke@435	2213	emit_byte(0xD3);
duke@435	2214	emit_byte(0xF8 | encode);
duke@435	2215	}
duke@435	2216
duke@435	2217	void Assembler::sarq(Register dst, int imm8) {
duke@435	2218	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2219	int encode = prefixq_and_encode(dst->encoding());
duke@435	2220	if (imm8 == 1) {
duke@435	2221	emit_byte(0xD1);
duke@435	2222	emit_byte(0xF8 | encode);
duke@435	2223	} else {
duke@435	2224	emit_byte(0xC1);
duke@435	2225	emit_byte(0xF8 | encode);
duke@435	2226	emit_byte(imm8);
duke@435	2227	}
duke@435	2228	}
duke@435	2229
duke@435	2230	void Assembler::sarq(Register dst) {
duke@435	2231	int encode = prefixq_and_encode(dst->encoding());
duke@435	2232	emit_byte(0xD3);
duke@435	2233	emit_byte(0xF8 | encode);
duke@435	2234	}
duke@435	2235
duke@435	2236	void Assembler::sbbl(Address dst, int imm32) {
duke@435	2237	InstructionMark im(this);
duke@435	2238	prefix(dst);
duke@435	2239	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2240	}
duke@435	2241
duke@435	2242	void Assembler::sbbl(Register dst, int imm32) {
duke@435	2243	prefix(dst);
duke@435	2244	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2245	}
duke@435	2246
duke@435	2247	void Assembler::sbbl(Register dst, Address src) {
duke@435	2248	InstructionMark im(this);
duke@435	2249	prefix(src, dst);
duke@435	2250	emit_byte(0x1B);
duke@435	2251	emit_operand(dst, src);
duke@435	2252	}
duke@435	2253
duke@435	2254	void Assembler::sbbl(Register dst, Register src) {
duke@435	2255	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2256	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2257	}
duke@435	2258
duke@435	2259	void Assembler::sbbq(Address dst, int imm32) {
duke@435	2260	InstructionMark im(this);
duke@435	2261	prefixq(dst);
duke@435	2262	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2263	}
duke@435	2264
duke@435	2265	void Assembler::sbbq(Register dst, int imm32) {
duke@435	2266	(void) prefixq_and_encode(dst->encoding());
duke@435	2267	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2268	}
duke@435	2269
duke@435	2270	void Assembler::sbbq(Register dst, Address src) {
duke@435	2271	InstructionMark im(this);
duke@435	2272	prefixq(src, dst);
duke@435	2273	emit_byte(0x1B);
duke@435	2274	emit_operand(dst, src);
duke@435	2275	}
duke@435	2276
duke@435	2277	void Assembler::sbbq(Register dst, Register src) {
duke@435	2278	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2279	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2280	}
duke@435	2281
duke@435	2282	void Assembler::shll(Register dst, int imm8) {
duke@435	2283	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2284	int encode = prefix_and_encode(dst->encoding());
duke@435	2285	if (imm8 == 1 ) {
duke@435	2286	emit_byte(0xD1);
duke@435	2287	emit_byte(0xE0 | encode);
duke@435	2288	} else {
duke@435	2289	emit_byte(0xC1);
duke@435	2290	emit_byte(0xE0 | encode);
duke@435	2291	emit_byte(imm8);
duke@435	2292	}
duke@435	2293	}
duke@435	2294
duke@435	2295	void Assembler::shll(Register dst) {
duke@435	2296	int encode = prefix_and_encode(dst->encoding());
duke@435	2297	emit_byte(0xD3);
duke@435	2298	emit_byte(0xE0 | encode);
duke@435	2299	}
duke@435	2300
duke@435	2301	void Assembler::shlq(Register dst, int imm8) {
duke@435	2302	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2303	int encode = prefixq_and_encode(dst->encoding());
duke@435	2304	if (imm8 == 1) {
duke@435	2305	emit_byte(0xD1);
duke@435	2306	emit_byte(0xE0 | encode);
duke@435	2307	} else {
duke@435	2308	emit_byte(0xC1);
duke@435	2309	emit_byte(0xE0 | encode);
duke@435	2310	emit_byte(imm8);
duke@435	2311	}
duke@435	2312	}
duke@435	2313
duke@435	2314	void Assembler::shlq(Register dst) {
duke@435	2315	int encode = prefixq_and_encode(dst->encoding());
duke@435	2316	emit_byte(0xD3);
duke@435	2317	emit_byte(0xE0 | encode);
duke@435	2318	}
duke@435	2319
duke@435	2320	void Assembler::shrl(Register dst, int imm8) {
duke@435	2321	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2322	int encode = prefix_and_encode(dst->encoding());
duke@435	2323	emit_byte(0xC1);
duke@435	2324	emit_byte(0xE8 | encode);
duke@435	2325	emit_byte(imm8);
duke@435	2326	}
duke@435	2327
duke@435	2328	void Assembler::shrl(Register dst) {
duke@435	2329	int encode = prefix_and_encode(dst->encoding());
duke@435	2330	emit_byte(0xD3);
duke@435	2331	emit_byte(0xE8 | encode);
duke@435	2332	}
duke@435	2333
duke@435	2334	void Assembler::shrq(Register dst, int imm8) {
duke@435	2335	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2336	int encode = prefixq_and_encode(dst->encoding());
duke@435	2337	emit_byte(0xC1);
duke@435	2338	emit_byte(0xE8 | encode);
duke@435	2339	emit_byte(imm8);
duke@435	2340	}
duke@435	2341
duke@435	2342	void Assembler::shrq(Register dst) {
duke@435	2343	int encode = prefixq_and_encode(dst->encoding());
duke@435	2344	emit_byte(0xD3);
duke@435	2345	emit_byte(0xE8 | encode);
duke@435	2346	}
duke@435	2347
duke@435	2348	void Assembler::subl(Address dst, int imm32) {
duke@435	2349	InstructionMark im(this);
duke@435	2350	prefix(dst);
duke@435	2351	if (is8bit(imm32)) {
duke@435	2352	emit_byte(0x83);
duke@435	2353	emit_operand(rbp, dst, 1);
duke@435	2354	emit_byte(imm32 & 0xFF);
duke@435	2355	} else {
duke@435	2356	emit_byte(0x81);
duke@435	2357	emit_operand(rbp, dst, 4);
duke@435	2358	emit_long(imm32);
duke@435	2359	}
duke@435	2360	}
duke@435	2361
duke@435	2362	void Assembler::subl(Register dst, int imm32) {
duke@435	2363	prefix(dst);
duke@435	2364	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2365	}
duke@435	2366
duke@435	2367	void Assembler::subl(Address dst, Register src) {
duke@435	2368	InstructionMark im(this);
duke@435	2369	prefix(dst, src);
duke@435	2370	emit_byte(0x29);
duke@435	2371	emit_operand(src, dst);
duke@435	2372	}
duke@435	2373
duke@435	2374	void Assembler::subl(Register dst, Address src) {
duke@435	2375	InstructionMark im(this);
duke@435	2376	prefix(src, dst);
duke@435	2377	emit_byte(0x2B);
duke@435	2378	emit_operand(dst, src);
duke@435	2379	}
duke@435	2380
duke@435	2381	void Assembler::subl(Register dst, Register src) {
duke@435	2382	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2383	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2384	}
duke@435	2385
duke@435	2386	void Assembler::subq(Address dst, int imm32) {
duke@435	2387	InstructionMark im(this);
duke@435	2388	prefixq(dst);
duke@435	2389	if (is8bit(imm32)) {
duke@435	2390	emit_byte(0x83);
duke@435	2391	emit_operand(rbp, dst, 1);
duke@435	2392	emit_byte(imm32 & 0xFF);
duke@435	2393	} else {
duke@435	2394	emit_byte(0x81);
duke@435	2395	emit_operand(rbp, dst, 4);
duke@435	2396	emit_long(imm32);
duke@435	2397	}
duke@435	2398	}
duke@435	2399
duke@435	2400	void Assembler::subq(Register dst, int imm32) {
duke@435	2401	(void) prefixq_and_encode(dst->encoding());
duke@435	2402	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2403	}
duke@435	2404
duke@435	2405	void Assembler::subq(Address dst, Register src) {
duke@435	2406	InstructionMark im(this);
duke@435	2407	prefixq(dst, src);
duke@435	2408	emit_byte(0x29);
duke@435	2409	emit_operand(src, dst);
duke@435	2410	}
duke@435	2411
duke@435	2412	void Assembler::subq(Register dst, Address src) {
duke@435	2413	InstructionMark im(this);
duke@435	2414	prefixq(src, dst);
duke@435	2415	emit_byte(0x2B);
duke@435	2416	emit_operand(dst, src);
duke@435	2417	}
duke@435	2418
duke@435	2419	void Assembler::subq(Register dst, Register src) {
duke@435	2420	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2421	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2422	}
duke@435	2423
duke@435	2424	void Assembler::testb(Register dst, int imm8) {
duke@435	2425	(void) prefix_and_encode(dst->encoding(), true);
duke@435	2426	emit_arith_b(0xF6, 0xC0, dst, imm8);
duke@435	2427	}
duke@435	2428
duke@435	2429	void Assembler::testl(Register dst, int imm32) {
duke@435	2430	// not using emit_arith because test
duke@435	2431	// doesn't support sign-extension of
duke@435	2432	// 8bit operands
duke@435	2433	int encode = dst->encoding();
duke@435	2434	if (encode == 0) {
duke@435	2435	emit_byte(0xA9);
duke@435	2436	} else {
duke@435	2437	encode = prefix_and_encode(encode);
duke@435	2438	emit_byte(0xF7);
duke@435	2439	emit_byte(0xC0 | encode);
duke@435	2440	}
duke@435	2441	emit_long(imm32);
duke@435	2442	}
duke@435	2443
duke@435	2444	void Assembler::testl(Register dst, Register src) {
duke@435	2445	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2446	emit_arith(0x85, 0xC0, dst, src);
duke@435	2447	}
duke@435	2448
duke@435	2449	void Assembler::testq(Register dst, int imm32) {
duke@435	2450	// not using emit_arith because test
duke@435	2451	// doesn't support sign-extension of
duke@435	2452	// 8bit operands
duke@435	2453	int encode = dst->encoding();
duke@435	2454	if (encode == 0) {
duke@435	2455	prefix(REX_W);
duke@435	2456	emit_byte(0xA9);
duke@435	2457	} else {
duke@435	2458	encode = prefixq_and_encode(encode);
duke@435	2459	emit_byte(0xF7);
duke@435	2460	emit_byte(0xC0 | encode);
duke@435	2461	}
duke@435	2462	emit_long(imm32);
duke@435	2463	}
duke@435	2464
duke@435	2465	void Assembler::testq(Register dst, Register src) {
duke@435	2466	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2467	emit_arith(0x85, 0xC0, dst, src);
duke@435	2468	}
duke@435	2469
duke@435	2470	void Assembler::xaddl(Address dst, Register src) {
duke@435	2471	InstructionMark im(this);
duke@435	2472	prefix(dst, src);
duke@435	2473	emit_byte(0x0F);
duke@435	2474	emit_byte(0xC1);
duke@435	2475	emit_operand(src, dst);
duke@435	2476	}
duke@435	2477
duke@435	2478	void Assembler::xaddq(Address dst, Register src) {
duke@435	2479	InstructionMark im(this);
duke@435	2480	prefixq(dst, src);
duke@435	2481	emit_byte(0x0F);
duke@435	2482	emit_byte(0xC1);
duke@435	2483	emit_operand(src, dst);
duke@435	2484	}
duke@435	2485
duke@435	2486	void Assembler::xorl(Register dst, int imm32) {
duke@435	2487	prefix(dst);
duke@435	2488	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2489	}
duke@435	2490
duke@435	2491	void Assembler::xorl(Register dst, Register src) {
duke@435	2492	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2493	emit_arith(0x33, 0xC0, dst, src);
duke@435	2494	}
duke@435	2495
duke@435	2496	void Assembler::xorl(Register dst, Address src) {
duke@435	2497	InstructionMark im(this);
duke@435	2498	prefix(src, dst);
duke@435	2499	emit_byte(0x33);
duke@435	2500	emit_operand(dst, src);
duke@435	2501	}
duke@435	2502
duke@435	2503	void Assembler::xorq(Register dst, int imm32) {
duke@435	2504	(void) prefixq_and_encode(dst->encoding());
duke@435	2505	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2506	}
duke@435	2507
duke@435	2508	void Assembler::xorq(Register dst, Register src) {
duke@435	2509	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2510	emit_arith(0x33, 0xC0, dst, src);
duke@435	2511	}
duke@435	2512
duke@435	2513	void Assembler::xorq(Register dst, Address src) {
duke@435	2514	InstructionMark im(this);
duke@435	2515	prefixq(src, dst);
duke@435	2516	emit_byte(0x33);
duke@435	2517	emit_operand(dst, src);
duke@435	2518	}
duke@435	2519
duke@435	2520	void Assembler::bswapl(Register reg) {
duke@435	2521	int encode = prefix_and_encode(reg->encoding());
duke@435	2522	emit_byte(0x0F);
duke@435	2523	emit_byte(0xC8 | encode);
duke@435	2524	}
duke@435	2525
duke@435	2526	void Assembler::bswapq(Register reg) {
duke@435	2527	int encode = prefixq_and_encode(reg->encoding());
duke@435	2528	emit_byte(0x0F);
duke@435	2529	emit_byte(0xC8 | encode);
duke@435	2530	}
duke@435	2531
duke@435	2532	void Assembler::lock() {
duke@435	2533	emit_byte(0xF0);
duke@435	2534	}
duke@435	2535
duke@435	2536	void Assembler::xchgl(Register dst, Address src) {
duke@435	2537	InstructionMark im(this);
duke@435	2538	prefix(src, dst);
duke@435	2539	emit_byte(0x87);
duke@435	2540	emit_operand(dst, src);
duke@435	2541	}
duke@435	2542
duke@435	2543	void Assembler::xchgl(Register dst, Register src) {
duke@435	2544	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2545	emit_byte(0x87);
duke@435	2546	emit_byte(0xc0 | encode);
duke@435	2547	}
duke@435	2548
duke@435	2549	void Assembler::xchgq(Register dst, Address src) {
duke@435	2550	InstructionMark im(this);
duke@435	2551	prefixq(src, dst);
duke@435	2552	emit_byte(0x87);
duke@435	2553	emit_operand(dst, src);
duke@435	2554	}
duke@435	2555
duke@435	2556	void Assembler::xchgq(Register dst, Register src) {
duke@435	2557	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2558	emit_byte(0x87);
duke@435	2559	emit_byte(0xc0 | encode);
duke@435	2560	}
duke@435	2561
duke@435	2562	void Assembler::cmpxchgl(Register reg, Address adr) {
duke@435	2563	InstructionMark im(this);
duke@435	2564	prefix(adr, reg);
duke@435	2565	emit_byte(0x0F);
duke@435	2566	emit_byte(0xB1);
duke@435	2567	emit_operand(reg, adr);
duke@435	2568	}
duke@435	2569
duke@435	2570	void Assembler::cmpxchgq(Register reg, Address adr) {
duke@435	2571	InstructionMark im(this);
duke@435	2572	prefixq(adr, reg);
duke@435	2573	emit_byte(0x0F);
duke@435	2574	emit_byte(0xB1);
duke@435	2575	emit_operand(reg, adr);
duke@435	2576	}
duke@435	2577
duke@435	2578	void Assembler::hlt() {
duke@435	2579	emit_byte(0xF4);
duke@435	2580	}
duke@435	2581
duke@435	2582
duke@435	2583	void Assembler::addr_nop_4() {
duke@435	2584	// 4 bytes: NOP DWORD PTR [EAX+0]
duke@435	2585	emit_byte(0x0F);
duke@435	2586	emit_byte(0x1F);
duke@435	2587	emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
duke@435	2588	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2589	}
duke@435	2590
duke@435	2591	void Assembler::addr_nop_5() {
duke@435	2592	// 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
duke@435	2593	emit_byte(0x0F);
duke@435	2594	emit_byte(0x1F);
duke@435	2595	emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
duke@435	2596	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2597	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2598	}
duke@435	2599
duke@435	2600	void Assembler::addr_nop_7() {
duke@435	2601	// 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
duke@435	2602	emit_byte(0x0F);
duke@435	2603	emit_byte(0x1F);
duke@435	2604	emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
duke@435	2605	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2606	}
duke@435	2607
duke@435	2608	void Assembler::addr_nop_8() {
duke@435	2609	// 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
duke@435	2610	emit_byte(0x0F);
duke@435	2611	emit_byte(0x1F);
duke@435	2612	emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
duke@435	2613	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2614	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2615	}
duke@435	2616
duke@435	2617	void Assembler::nop(int i) {
duke@435	2618	assert(i > 0, " ");
duke@435	2619	if (UseAddressNop && VM_Version::is_intel()) {
duke@435	2620	//
duke@435	2621	// Using multi-bytes nops "0x0F 0x1F [address]" for Intel
duke@435	2622	// 1: 0x90
duke@435	2623	// 2: 0x66 0x90
duke@435	2624	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2625	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2626	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2627	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2628	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2629	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2630	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2631	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2632	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2633
duke@435	2634	// The rest coding is Intel specific - don't use consecutive address nops
duke@435	2635
duke@435	2636	// 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2637	// 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2638	// 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2639	// 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2640
duke@435	2641	while(i >= 15) {
duke@435	2642	// For Intel don't generate consecutive addess nops (mix with regular nops)
duke@435	2643	i -= 15;
duke@435	2644	emit_byte(0x66); // size prefix
duke@435	2645	emit_byte(0x66); // size prefix
duke@435	2646	emit_byte(0x66); // size prefix
duke@435	2647	addr_nop_8();
duke@435	2648	emit_byte(0x66); // size prefix
duke@435	2649	emit_byte(0x66); // size prefix
duke@435	2650	emit_byte(0x66); // size prefix
duke@435	2651	emit_byte(0x90); // nop
duke@435	2652	}
duke@435	2653	switch (i) {
duke@435	2654	case 14:
duke@435	2655	emit_byte(0x66); // size prefix
duke@435	2656	case 13:
duke@435	2657	emit_byte(0x66); // size prefix
duke@435	2658	case 12:
duke@435	2659	addr_nop_8();
duke@435	2660	emit_byte(0x66); // size prefix
duke@435	2661	emit_byte(0x66); // size prefix
duke@435	2662	emit_byte(0x66); // size prefix
duke@435	2663	emit_byte(0x90); // nop
duke@435	2664	break;
duke@435	2665	case 11:
duke@435	2666	emit_byte(0x66); // size prefix
duke@435	2667	case 10:
duke@435	2668	emit_byte(0x66); // size prefix
duke@435	2669	case 9:
duke@435	2670	emit_byte(0x66); // size prefix
duke@435	2671	case 8:
duke@435	2672	addr_nop_8();
duke@435	2673	break;
duke@435	2674	case 7:
duke@435	2675	addr_nop_7();
duke@435	2676	break;
duke@435	2677	case 6:
duke@435	2678	emit_byte(0x66); // size prefix
duke@435	2679	case 5:
duke@435	2680	addr_nop_5();
duke@435	2681	break;
duke@435	2682	case 4:
duke@435	2683	addr_nop_4();
duke@435	2684	break;
duke@435	2685	case 3:
duke@435	2686	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2687	emit_byte(0x66); // size prefix
duke@435	2688	case 2:
duke@435	2689	emit_byte(0x66); // size prefix
duke@435	2690	case 1:
duke@435	2691	emit_byte(0x90); // nop
duke@435	2692	break;
duke@435	2693	default:
duke@435	2694	assert(i == 0, " ");
duke@435	2695	}
duke@435	2696	return;
duke@435	2697	}
duke@435	2698	if (UseAddressNop && VM_Version::is_amd()) {
duke@435	2699	//
duke@435	2700	// Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
duke@435	2701	// 1: 0x90
duke@435	2702	// 2: 0x66 0x90
duke@435	2703	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2704	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2705	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2706	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2707	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2708	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2709	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2710	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2711	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2712
duke@435	2713	// The rest coding is AMD specific - use consecutive address nops
duke@435	2714
duke@435	2715	// 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2716	// 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2717	// 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2718	// 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2719	// 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2720	// Size prefixes (0x66) are added for larger sizes
duke@435	2721
duke@435	2722	while(i >= 22) {
duke@435	2723	i -= 11;
duke@435	2724	emit_byte(0x66); // size prefix
duke@435	2725	emit_byte(0x66); // size prefix
duke@435	2726	emit_byte(0x66); // size prefix
duke@435	2727	addr_nop_8();
duke@435	2728	}
duke@435	2729	// Generate first nop for size between 21-12
duke@435	2730	switch (i) {
duke@435	2731	case 21:
duke@435	2732	i -= 1;
duke@435	2733	emit_byte(0x66); // size prefix
duke@435	2734	case 20:
duke@435	2735	case 19:
duke@435	2736	i -= 1;
duke@435	2737	emit_byte(0x66); // size prefix
duke@435	2738	case 18:
duke@435	2739	case 17:
duke@435	2740	i -= 1;
duke@435	2741	emit_byte(0x66); // size prefix
duke@435	2742	case 16:
duke@435	2743	case 15:
duke@435	2744	i -= 8;
duke@435	2745	addr_nop_8();
duke@435	2746	break;
duke@435	2747	case 14:
duke@435	2748	case 13:
duke@435	2749	i -= 7;
duke@435	2750	addr_nop_7();
duke@435	2751	break;
duke@435	2752	case 12:
duke@435	2753	i -= 6;
duke@435	2754	emit_byte(0x66); // size prefix
duke@435	2755	addr_nop_5();
duke@435	2756	break;
duke@435	2757	default:
duke@435	2758	assert(i < 12, " ");
duke@435	2759	}
duke@435	2760
duke@435	2761	// Generate second nop for size between 11-1
duke@435	2762	switch (i) {
duke@435	2763	case 11:
duke@435	2764	emit_byte(0x66); // size prefix
duke@435	2765	case 10:
duke@435	2766	emit_byte(0x66); // size prefix
duke@435	2767	case 9:
duke@435	2768	emit_byte(0x66); // size prefix
duke@435	2769	case 8:
duke@435	2770	addr_nop_8();
duke@435	2771	break;
duke@435	2772	case 7:
duke@435	2773	addr_nop_7();
duke@435	2774	break;
duke@435	2775	case 6:
duke@435	2776	emit_byte(0x66); // size prefix
duke@435	2777	case 5:
duke@435	2778	addr_nop_5();
duke@435	2779	break;
duke@435	2780	case 4:
duke@435	2781	addr_nop_4();
duke@435	2782	break;
duke@435	2783	case 3:
duke@435	2784	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2785	emit_byte(0x66); // size prefix
duke@435	2786	case 2:
duke@435	2787	emit_byte(0x66); // size prefix
duke@435	2788	case 1:
duke@435	2789	emit_byte(0x90); // nop
duke@435	2790	break;
duke@435	2791	default:
duke@435	2792	assert(i == 0, " ");
duke@435	2793	}
duke@435	2794	return;
duke@435	2795	}
duke@435	2796
duke@435	2797	// Using nops with size prefixes "0x66 0x90".
duke@435	2798	// From AMD Optimization Guide:
duke@435	2799	// 1: 0x90
duke@435	2800	// 2: 0x66 0x90
duke@435	2801	// 3: 0x66 0x66 0x90
duke@435	2802	// 4: 0x66 0x66 0x66 0x90
duke@435	2803	// 5: 0x66 0x66 0x90 0x66 0x90
duke@435	2804	// 6: 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2805	// 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2806	// 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
duke@435	2807	// 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2808	// 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2809	//
duke@435	2810	while(i > 12) {
duke@435	2811	i -= 4;
duke@435	2812	emit_byte(0x66); // size prefix
duke@435	2813	emit_byte(0x66);
duke@435	2814	emit_byte(0x66);
duke@435	2815	emit_byte(0x90); // nop
duke@435	2816	}
duke@435	2817	// 1 - 12 nops
duke@435	2818	if(i > 8) {
duke@435	2819	if(i > 9) {
duke@435	2820	i -= 1;
duke@435	2821	emit_byte(0x66);
duke@435	2822	}
duke@435	2823	i -= 3;
duke@435	2824	emit_byte(0x66);
duke@435	2825	emit_byte(0x66);
duke@435	2826	emit_byte(0x90);
duke@435	2827	}
duke@435	2828	// 1 - 8 nops
duke@435	2829	if(i > 4) {
duke@435	2830	if(i > 6) {
duke@435	2831	i -= 1;
duke@435	2832	emit_byte(0x66);
duke@435	2833	}
duke@435	2834	i -= 3;
duke@435	2835	emit_byte(0x66);
duke@435	2836	emit_byte(0x66);
duke@435	2837	emit_byte(0x90);
duke@435	2838	}
duke@435	2839	switch (i) {
duke@435	2840	case 4:
duke@435	2841	emit_byte(0x66);
duke@435	2842	case 3:
duke@435	2843	emit_byte(0x66);
duke@435	2844	case 2:
duke@435	2845	emit_byte(0x66);
duke@435	2846	case 1:
duke@435	2847	emit_byte(0x90);
duke@435	2848	break;
duke@435	2849	default:
duke@435	2850	assert(i == 0, " ");
duke@435	2851	}
duke@435	2852	}
duke@435	2853
duke@435	2854	void Assembler::ret(int imm16) {
duke@435	2855	if (imm16 == 0) {
duke@435	2856	emit_byte(0xC3);
duke@435	2857	} else {
duke@435	2858	emit_byte(0xC2);
duke@435	2859	emit_word(imm16);
duke@435	2860	}
duke@435	2861	}
duke@435	2862
duke@435	2863	// copies a single word from [esi] to [edi]
duke@435	2864	void Assembler::smovl() {
duke@435	2865	emit_byte(0xA5);
duke@435	2866	}
duke@435	2867
duke@435	2868	// copies data from [rsi] to [rdi] using rcx words (m32)
duke@435	2869	void Assembler::rep_movl() {
duke@435	2870	// REP
duke@435	2871	emit_byte(0xF3);
duke@435	2872	// MOVSL
duke@435	2873	emit_byte(0xA5);
duke@435	2874	}
duke@435	2875
duke@435	2876	// copies data from [rsi] to [rdi] using rcx double words (m64)
duke@435	2877	void Assembler::rep_movq() {
duke@435	2878	// REP
duke@435	2879	emit_byte(0xF3);
duke@435	2880	// MOVSQ
duke@435	2881	prefix(REX_W);
duke@435	2882	emit_byte(0xA5);
duke@435	2883	}
duke@435	2884
duke@435	2885	// sets rcx double words (m64) with rax value at [rdi]
duke@435	2886	void Assembler::rep_set() {
duke@435	2887	// REP
duke@435	2888	emit_byte(0xF3);
duke@435	2889	// STOSQ
duke@435	2890	prefix(REX_W);
duke@435	2891	emit_byte(0xAB);
duke@435	2892	}
duke@435	2893
duke@435	2894	// scans rcx double words (m64) at [rdi] for occurance of rax
coleenp@548	2895	void Assembler::repne_scanq() {
duke@435	2896	// REPNE/REPNZ
duke@435	2897	emit_byte(0xF2);
duke@435	2898	// SCASQ
duke@435	2899	prefix(REX_W);
duke@435	2900	emit_byte(0xAF);
duke@435	2901	}
duke@435	2902
coleenp@548	2903	void Assembler::repne_scanl() {
coleenp@548	2904	// REPNE/REPNZ
coleenp@548	2905	emit_byte(0xF2);
coleenp@548	2906	// SCASL
coleenp@548	2907	emit_byte(0xAF);
coleenp@548	2908	}
coleenp@548	2909
coleenp@548	2910
duke@435	2911	void Assembler::setb(Condition cc, Register dst) {
duke@435	2912	assert(0 <= cc && cc < 16, "illegal cc");
duke@435	2913	int encode = prefix_and_encode(dst->encoding(), true);
duke@435	2914	emit_byte(0x0F);
duke@435	2915	emit_byte(0x90 | cc);
duke@435	2916	emit_byte(0xC0 | encode);
duke@435	2917	}
duke@435	2918
duke@435	2919	void Assembler::clflush(Address adr) {
duke@435	2920	prefix(adr);
duke@435	2921	emit_byte(0x0F);
duke@435	2922	emit_byte(0xAE);
duke@435	2923	emit_operand(rdi, adr);
duke@435	2924	}
duke@435	2925
duke@435	2926	void Assembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	2927	if (L.is_bound()) {
duke@435	2928	const int long_size = 5;
duke@435	2929	int offs = (int)( target(L) - pc() );
duke@435	2930	assert(offs <= 0, "assembler error");
duke@435	2931	InstructionMark im(this);
duke@435	2932	// 1110 1000 #32-bit disp
duke@435	2933	emit_byte(0xE8);
duke@435	2934	emit_data(offs - long_size, rtype, disp32_operand);
duke@435	2935	} else {
duke@435	2936	InstructionMark im(this);
duke@435	2937	// 1110 1000 #32-bit disp
duke@435	2938	L.add_patch_at(code(), locator());
duke@435	2939
duke@435	2940	emit_byte(0xE8);
duke@435	2941	emit_data(int(0), rtype, disp32_operand);
duke@435	2942	}
duke@435	2943	}
duke@435	2944
duke@435	2945	void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
duke@435	2946	assert(entry != NULL, "call most probably wrong");
duke@435	2947	InstructionMark im(this);
duke@435	2948	emit_byte(0xE8);
duke@435	2949	intptr_t disp = entry - (_code_pos + sizeof(int32_t));
duke@435	2950	assert(is_simm32(disp), "must be 32bit offset (call2)");
duke@435	2951	// Technically, should use call32_operand, but this format is
duke@435	2952	// implied by the fact that we're emitting a call instruction.
duke@435	2953	emit_data((int) disp, rspec, disp32_operand);
duke@435	2954	}
duke@435	2955
duke@435	2956
duke@435	2957	void Assembler::call(Register dst) {
duke@435	2958	// This was originally using a 32bit register encoding
duke@435	2959	// and surely we want 64bit!
duke@435	2960	// this is a 32bit encoding but in 64bit mode the default
duke@435	2961	// operand size is 64bit so there is no need for the
duke@435	2962	// wide prefix. So prefix only happens if we use the
duke@435	2963	// new registers. Much like push/pop.
duke@435	2964	int encode = prefixq_and_encode(dst->encoding());
duke@435	2965	emit_byte(0xFF);
duke@435	2966	emit_byte(0xD0 | encode);
duke@435	2967	}
duke@435	2968
duke@435	2969	void Assembler::call(Address adr) {
duke@435	2970	InstructionMark im(this);
duke@435	2971	prefix(adr);
duke@435	2972	emit_byte(0xFF);
duke@435	2973	emit_operand(rdx, adr);
duke@435	2974	}
duke@435	2975
duke@435	2976	void Assembler::jmp(Register reg) {
duke@435	2977	int encode = prefix_and_encode(reg->encoding());
duke@435	2978	emit_byte(0xFF);
duke@435	2979	emit_byte(0xE0 | encode);
duke@435	2980	}
duke@435	2981
duke@435	2982	void Assembler::jmp(Address adr) {
duke@435	2983	InstructionMark im(this);
duke@435	2984	prefix(adr);
duke@435	2985	emit_byte(0xFF);
duke@435	2986	emit_operand(rsp, adr);
duke@435	2987	}
duke@435	2988
duke@435	2989	void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
duke@435	2990	InstructionMark im(this);
duke@435	2991	emit_byte(0xE9);
duke@435	2992	assert(dest != NULL, "must have a target");
duke@435	2993	intptr_t disp = dest - (_code_pos + sizeof(int32_t));
duke@435	2994	assert(is_simm32(disp), "must be 32bit offset (jmp)");
duke@435	2995	emit_data(disp, rspec.reloc(), call32_operand);
duke@435	2996	}
duke@435	2997
duke@435	2998	void Assembler::jmp(Label& L, relocInfo::relocType rtype) {
duke@435	2999	if (L.is_bound()) {
duke@435	3000	address entry = target(L);
duke@435	3001	assert(entry != NULL, "jmp most probably wrong");
duke@435	3002	InstructionMark im(this);
duke@435	3003	const int short_size = 2;
duke@435	3004	const int long_size = 5;
duke@435	3005	intptr_t offs = entry - _code_pos;
duke@435	3006	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3007	emit_byte(0xEB);
duke@435	3008	emit_byte((offs - short_size) & 0xFF);
duke@435	3009	} else {
duke@435	3010	emit_byte(0xE9);
duke@435	3011	emit_long(offs - long_size);
duke@435	3012	}
duke@435	3013	} else {
duke@435	3014	// By default, forward jumps are always 32-bit displacements, since
duke@435	3015	// we can't yet know where the label will be bound. If you're sure that
duke@435	3016	// the forward jump will not run beyond 256 bytes, use jmpb to
duke@435	3017	// force an 8-bit displacement.
duke@435	3018	InstructionMark im(this);
duke@435	3019	relocate(rtype);
duke@435	3020	L.add_patch_at(code(), locator());
duke@435	3021	emit_byte(0xE9);
duke@435	3022	emit_long(0);
duke@435	3023	}
duke@435	3024	}
duke@435	3025
duke@435	3026	void Assembler::jmpb(Label& L) {
duke@435	3027	if (L.is_bound()) {
duke@435	3028	const int short_size = 2;
duke@435	3029	address entry = target(L);
duke@435	3030	assert(is8bit((entry - _code_pos) + short_size),
duke@435	3031	"Dispacement too large for a short jmp");
duke@435	3032	assert(entry != NULL, "jmp most probably wrong");
duke@435	3033	intptr_t offs = entry - _code_pos;
duke@435	3034	emit_byte(0xEB);
duke@435	3035	emit_byte((offs - short_size) & 0xFF);
duke@435	3036	} else {
duke@435	3037	InstructionMark im(this);
duke@435	3038	L.add_patch_at(code(), locator());
duke@435	3039	emit_byte(0xEB);
duke@435	3040	emit_byte(0);
duke@435	3041	}
duke@435	3042	}
duke@435	3043
duke@435	3044	void Assembler::jcc(Condition cc, Label& L, relocInfo::relocType rtype) {
duke@435	3045	InstructionMark im(this);
duke@435	3046	relocate(rtype);
duke@435	3047	assert((0 <= cc) && (cc < 16), "illegal cc");
duke@435	3048	if (L.is_bound()) {
duke@435	3049	address dst = target(L);
duke@435	3050	assert(dst != NULL, "jcc most probably wrong");
duke@435	3051
duke@435	3052	const int short_size = 2;
duke@435	3053	const int long_size = 6;
duke@435	3054	intptr_t offs = (intptr_t)dst - (intptr_t)_code_pos;
duke@435	3055	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3056	// 0111 tttn #8-bit disp
duke@435	3057	emit_byte(0x70 | cc);
duke@435	3058	emit_byte((offs - short_size) & 0xFF);
duke@435	3059	} else {
duke@435	3060	// 0000 1111 1000 tttn #32-bit disp
duke@435	3061	assert(is_simm32(offs - long_size),
duke@435	3062	"must be 32bit offset (call4)");
duke@435	3063	emit_byte(0x0F);
duke@435	3064	emit_byte(0x80 | cc);
duke@435	3065	emit_long(offs - long_size);
duke@435	3066	}
duke@435	3067	} else {
duke@435	3068	// Note: could eliminate cond. jumps to this jump if condition
duke@435	3069	// is the same however, seems to be rather unlikely case.
duke@435	3070	// Note: use jccb() if label to be bound is very close to get
duke@435	3071	// an 8-bit displacement
duke@435	3072	L.add_patch_at(code(), locator());
duke@435	3073	emit_byte(0x0F);
duke@435	3074	emit_byte(0x80 | cc);
duke@435	3075	emit_long(0);
duke@435	3076	}
duke@435	3077	}
duke@435	3078
duke@435	3079	void Assembler::jccb(Condition cc, Label& L) {
duke@435	3080	if (L.is_bound()) {
duke@435	3081	const int short_size = 2;
duke@435	3082	const int long_size = 6;
duke@435	3083	address entry = target(L);
duke@435	3084	assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),
duke@435	3085	"Dispacement too large for a short jmp");
duke@435	3086	intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;
duke@435	3087	// 0111 tttn #8-bit disp
duke@435	3088	emit_byte(0x70 | cc);
duke@435	3089	emit_byte((offs - short_size) & 0xFF);
duke@435	3090	} else {
duke@435	3091	InstructionMark im(this);
duke@435	3092	L.add_patch_at(code(), locator());
duke@435	3093	emit_byte(0x70 | cc);
duke@435	3094	emit_byte(0);
duke@435	3095	}
duke@435	3096	}
duke@435	3097
duke@435	3098	// FP instructions
duke@435	3099
duke@435	3100	void Assembler::fxsave(Address dst) {
duke@435	3101	prefixq(dst);
duke@435	3102	emit_byte(0x0F);
duke@435	3103	emit_byte(0xAE);
duke@435	3104	emit_operand(as_Register(0), dst);
duke@435	3105	}
duke@435	3106
duke@435	3107	void Assembler::fxrstor(Address src) {
duke@435	3108	prefixq(src);
duke@435	3109	emit_byte(0x0F);
duke@435	3110	emit_byte(0xAE);
duke@435	3111	emit_operand(as_Register(1), src);
duke@435	3112	}
duke@435	3113
duke@435	3114	void Assembler::ldmxcsr(Address src) {
duke@435	3115	InstructionMark im(this);
duke@435	3116	prefix(src);
duke@435	3117	emit_byte(0x0F);
duke@435	3118	emit_byte(0xAE);
duke@435	3119	emit_operand(as_Register(2), src);
duke@435	3120	}
duke@435	3121
duke@435	3122	void Assembler::stmxcsr(Address dst) {
duke@435	3123	InstructionMark im(this);
duke@435	3124	prefix(dst);
duke@435	3125	emit_byte(0x0F);
duke@435	3126	emit_byte(0xAE);
duke@435	3127	emit_operand(as_Register(3), dst);
duke@435	3128	}
duke@435	3129
duke@435	3130	void Assembler::addss(XMMRegister dst, XMMRegister src) {
duke@435	3131	emit_byte(0xF3);
duke@435	3132	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3133	emit_byte(0x0F);
duke@435	3134	emit_byte(0x58);
duke@435	3135	emit_byte(0xC0 | encode);
duke@435	3136	}
duke@435	3137
duke@435	3138	void Assembler::addss(XMMRegister dst, Address src) {
duke@435	3139	InstructionMark im(this);
duke@435	3140	emit_byte(0xF3);
duke@435	3141	prefix(src, dst);
duke@435	3142	emit_byte(0x0F);
duke@435	3143	emit_byte(0x58);
duke@435	3144	emit_operand(dst, src);
duke@435	3145	}
duke@435	3146
duke@435	3147	void Assembler::subss(XMMRegister dst, XMMRegister src) {
duke@435	3148	emit_byte(0xF3);
duke@435	3149	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3150	emit_byte(0x0F);
duke@435	3151	emit_byte(0x5C);
duke@435	3152	emit_byte(0xC0 | encode);
duke@435	3153	}
duke@435	3154
duke@435	3155	void Assembler::subss(XMMRegister dst, Address src) {
duke@435	3156	InstructionMark im(this);
duke@435	3157	emit_byte(0xF3);
duke@435	3158	prefix(src, dst);
duke@435	3159	emit_byte(0x0F);
duke@435	3160	emit_byte(0x5C);
duke@435	3161	emit_operand(dst, src);
duke@435	3162	}
duke@435	3163
duke@435	3164	void Assembler::mulss(XMMRegister dst, XMMRegister src) {
duke@435	3165	emit_byte(0xF3);
duke@435	3166	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3167	emit_byte(0x0F);
duke@435	3168	emit_byte(0x59);
duke@435	3169	emit_byte(0xC0 | encode);
duke@435	3170	}
duke@435	3171
duke@435	3172	void Assembler::mulss(XMMRegister dst, Address src) {
duke@435	3173	InstructionMark im(this);
duke@435	3174	emit_byte(0xF3);
duke@435	3175	prefix(src, dst);
duke@435	3176	emit_byte(0x0F);
duke@435	3177	emit_byte(0x59);
duke@435	3178	emit_operand(dst, src);
duke@435	3179	}
duke@435	3180
duke@435	3181	void Assembler::divss(XMMRegister dst, XMMRegister src) {
duke@435	3182	emit_byte(0xF3);
duke@435	3183	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3184	emit_byte(0x0F);
duke@435	3185	emit_byte(0x5E);
duke@435	3186	emit_byte(0xC0 | encode);
duke@435	3187	}
duke@435	3188
duke@435	3189	void Assembler::divss(XMMRegister dst, Address src) {
duke@435	3190	InstructionMark im(this);
duke@435	3191	emit_byte(0xF3);
duke@435	3192	prefix(src, dst);
duke@435	3193	emit_byte(0x0F);
duke@435	3194	emit_byte(0x5E);
duke@435	3195	emit_operand(dst, src);
duke@435	3196	}
duke@435	3197
duke@435	3198	void Assembler::addsd(XMMRegister dst, XMMRegister src) {
duke@435	3199	emit_byte(0xF2);
duke@435	3200	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3201	emit_byte(0x0F);
duke@435	3202	emit_byte(0x58);
duke@435	3203	emit_byte(0xC0 | encode);
duke@435	3204	}
duke@435	3205
duke@435	3206	void Assembler::addsd(XMMRegister dst, Address src) {
duke@435	3207	InstructionMark im(this);
duke@435	3208	emit_byte(0xF2);
duke@435	3209	prefix(src, dst);
duke@435	3210	emit_byte(0x0F);
duke@435	3211	emit_byte(0x58);
duke@435	3212	emit_operand(dst, src);
duke@435	3213	}
duke@435	3214
duke@435	3215	void Assembler::subsd(XMMRegister dst, XMMRegister src) {
duke@435	3216	emit_byte(0xF2);
duke@435	3217	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3218	emit_byte(0x0F);
duke@435	3219	emit_byte(0x5C);
duke@435	3220	emit_byte(0xC0 | encode);
duke@435	3221	}
duke@435	3222
duke@435	3223	void Assembler::subsd(XMMRegister dst, Address src) {
duke@435	3224	InstructionMark im(this);
duke@435	3225	emit_byte(0xF2);
duke@435	3226	prefix(src, dst);
duke@435	3227	emit_byte(0x0F);
duke@435	3228	emit_byte(0x5C);
duke@435	3229	emit_operand(dst, src);
duke@435	3230	}
duke@435	3231
duke@435	3232	void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
duke@435	3233	emit_byte(0xF2);
duke@435	3234	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3235	emit_byte(0x0F);
duke@435	3236	emit_byte(0x59);
duke@435	3237	emit_byte(0xC0 | encode);
duke@435	3238	}
duke@435	3239
duke@435	3240	void Assembler::mulsd(XMMRegister dst, Address src) {
duke@435	3241	InstructionMark im(this);
duke@435	3242	emit_byte(0xF2);
duke@435	3243	prefix(src, dst);
duke@435	3244	emit_byte(0x0F);
duke@435	3245	emit_byte(0x59);
duke@435	3246	emit_operand(dst, src);
duke@435	3247	}
duke@435	3248
duke@435	3249	void Assembler::divsd(XMMRegister dst, XMMRegister src) {
duke@435	3250	emit_byte(0xF2);
duke@435	3251	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3252	emit_byte(0x0F);
duke@435	3253	emit_byte(0x5E);
duke@435	3254	emit_byte(0xC0 | encode);
duke@435	3255	}
duke@435	3256
duke@435	3257	void Assembler::divsd(XMMRegister dst, Address src) {
duke@435	3258	InstructionMark im(this);
duke@435	3259	emit_byte(0xF2);
duke@435	3260	prefix(src, dst);
duke@435	3261	emit_byte(0x0F);
duke@435	3262	emit_byte(0x5E);
duke@435	3263	emit_operand(dst, src);
duke@435	3264	}
duke@435	3265
duke@435	3266	void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
duke@435	3267	emit_byte(0xF2);
duke@435	3268	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3269	emit_byte(0x0F);
duke@435	3270	emit_byte(0x51);
duke@435	3271	emit_byte(0xC0 | encode);
duke@435	3272	}
duke@435	3273
duke@435	3274	void Assembler::sqrtsd(XMMRegister dst, Address src) {
duke@435	3275	InstructionMark im(this);
duke@435	3276	emit_byte(0xF2);
duke@435	3277	prefix(src, dst);
duke@435	3278	emit_byte(0x0F);
duke@435	3279	emit_byte(0x51);
duke@435	3280	emit_operand(dst, src);
duke@435	3281	}
duke@435	3282
duke@435	3283	void Assembler::xorps(XMMRegister dst, XMMRegister src) {
duke@435	3284	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3285	emit_byte(0x0F);
duke@435	3286	emit_byte(0x57);
duke@435	3287	emit_byte(0xC0 | encode);
duke@435	3288	}
duke@435	3289
duke@435	3290	void Assembler::xorps(XMMRegister dst, Address src) {
duke@435	3291	InstructionMark im(this);
duke@435	3292	prefix(src, dst);
duke@435	3293	emit_byte(0x0F);
duke@435	3294	emit_byte(0x57);
duke@435	3295	emit_operand(dst, src);
duke@435	3296	}
duke@435	3297
duke@435	3298	void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
duke@435	3299	emit_byte(0x66);
duke@435	3300	xorps(dst, src);
duke@435	3301	}
duke@435	3302
duke@435	3303	void Assembler::xorpd(XMMRegister dst, Address src) {
duke@435	3304	InstructionMark im(this);
duke@435	3305	emit_byte(0x66);
duke@435	3306	prefix(src, dst);
duke@435	3307	emit_byte(0x0F);
duke@435	3308	emit_byte(0x57);
duke@435	3309	emit_operand(dst, src);
duke@435	3310	}
duke@435	3311
duke@435	3312	void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
duke@435	3313	emit_byte(0xF3);
duke@435	3314	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3315	emit_byte(0x0F);
duke@435	3316	emit_byte(0x2A);
duke@435	3317	emit_byte(0xC0 | encode);
duke@435	3318	}
duke@435	3319
duke@435	3320	void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
duke@435	3321	emit_byte(0xF3);
duke@435	3322	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3323	emit_byte(0x0F);
duke@435	3324	emit_byte(0x2A);
duke@435	3325	emit_byte(0xC0 | encode);
duke@435	3326	}
duke@435	3327
duke@435	3328	void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
duke@435	3329	emit_byte(0xF2);
duke@435	3330	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3331	emit_byte(0x0F);
duke@435	3332	emit_byte(0x2A);
duke@435	3333	emit_byte(0xC0 | encode);
duke@435	3334	}
duke@435	3335
duke@435	3336	void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
duke@435	3337	emit_byte(0xF2);
duke@435	3338	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3339	emit_byte(0x0F);
duke@435	3340	emit_byte(0x2A);
duke@435	3341	emit_byte(0xC0 | encode);
duke@435	3342	}
duke@435	3343
duke@435	3344	void Assembler::cvttss2sil(Register dst, XMMRegister src) {
duke@435	3345	emit_byte(0xF3);
duke@435	3346	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3347	emit_byte(0x0F);
duke@435	3348	emit_byte(0x2C);
duke@435	3349	emit_byte(0xC0 | encode);
duke@435	3350	}
duke@435	3351
duke@435	3352	void Assembler::cvttss2siq(Register dst, XMMRegister src) {
duke@435	3353	emit_byte(0xF3);
duke@435	3354	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3355	emit_byte(0x0F);
duke@435	3356	emit_byte(0x2C);
duke@435	3357	emit_byte(0xC0 | encode);
duke@435	3358	}
duke@435	3359
duke@435	3360	void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
duke@435	3361	emit_byte(0xF2);
duke@435	3362	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3363	emit_byte(0x0F);
duke@435	3364	emit_byte(0x2C);
duke@435	3365	emit_byte(0xC0 | encode);
duke@435	3366	}
duke@435	3367
duke@435	3368	void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
duke@435	3369	emit_byte(0xF2);
duke@435	3370	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3371	emit_byte(0x0F);
duke@435	3372	emit_byte(0x2C);
duke@435	3373	emit_byte(0xC0 | encode);
duke@435	3374	}
duke@435	3375
duke@435	3376	void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
duke@435	3377	emit_byte(0xF3);
duke@435	3378	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3379	emit_byte(0x0F);
duke@435	3380	emit_byte(0x5A);
duke@435	3381	emit_byte(0xC0 | encode);
duke@435	3382	}
duke@435	3383
kvn@506	3384	void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
kvn@506	3385	emit_byte(0xF3);
kvn@506	3386	int encode = prefix_and_encode(dst->encoding(), src->encoding());
kvn@506	3387	emit_byte(0x0F);
kvn@506	3388	emit_byte(0xE6);
kvn@506	3389	emit_byte(0xC0 | encode);
kvn@506	3390	}
kvn@506	3391
kvn@506	3392	void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
kvn@506	3393	int encode = prefix_and_encode(dst->encoding(), src->encoding());
kvn@506	3394	emit_byte(0x0F);
kvn@506	3395	emit_byte(0x5B);
kvn@506	3396	emit_byte(0xC0 | encode);
kvn@506	3397	}
kvn@506	3398
duke@435	3399	void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
duke@435	3400	emit_byte(0xF2);
duke@435	3401	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3402	emit_byte(0x0F);
duke@435	3403	emit_byte(0x5A);
duke@435	3404	emit_byte(0xC0 | encode);
duke@435	3405	}
duke@435	3406
duke@435	3407	void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
duke@435	3408	emit_byte(0x66);
duke@435	3409	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3410	emit_byte(0x0F);
duke@435	3411	emit_byte(0x60);
duke@435	3412	emit_byte(0xC0 | encode);
duke@435	3413	}
duke@435	3414
duke@435	3415	// Implementation of MacroAssembler
duke@435	3416
duke@435	3417	// On 32 bit it returns a vanilla displacement on 64 bit is a rip relative displacement
duke@435	3418	Address MacroAssembler::as_Address(AddressLiteral adr) {
duke@435	3419	assert(!adr.is_lval(), "must be rval");
duke@435	3420	assert(reachable(adr), "must be");
duke@435	3421	return Address((int)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
duke@435	3422	}
duke@435	3423
duke@435	3424	Address MacroAssembler::as_Address(ArrayAddress adr) {
duke@435	3425	#ifdef _LP64
duke@435	3426	AddressLiteral base = adr.base();
duke@435	3427	lea(rscratch1, base);
duke@435	3428	Address index = adr.index();
duke@435	3429	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	3430	Address array(rscratch1, index._index, index._scale, index._disp);
duke@435	3431	return array;
duke@435	3432	#else
duke@435	3433	return Address::make_array(adr);
duke@435	3434	#endif // _LP64
duke@435	3435
duke@435	3436	}
duke@435	3437
duke@435	3438	void MacroAssembler::fat_nop() {
duke@435	3439	// A 5 byte nop that is safe for patching (see patch_verified_entry)
duke@435	3440	// Recommened sequence from 'Software Optimization Guide for the AMD
duke@435	3441	// Hammer Processor'
duke@435	3442	emit_byte(0x66);
duke@435	3443	emit_byte(0x66);
duke@435	3444	emit_byte(0x90);
duke@435	3445	emit_byte(0x66);
duke@435	3446	emit_byte(0x90);
duke@435	3447	}
duke@435	3448
duke@435	3449	static Assembler::Condition reverse[] = {
duke@435	3450	Assembler::noOverflow /* overflow = 0x0 */ ,
duke@435	3451	Assembler::overflow /* noOverflow = 0x1 */ ,
duke@435	3452	Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
duke@435	3453	Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
duke@435	3454	Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
duke@435	3455	Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ ,
duke@435	3456	Assembler::above /* belowEqual = 0x6 */ ,
duke@435	3457	Assembler::belowEqual /* above = 0x7 */ ,
duke@435	3458	Assembler::positive /* negative = 0x8 */ ,
duke@435	3459	Assembler::negative /* positive = 0x9 */ ,
duke@435	3460	Assembler::noParity /* parity = 0xa */ ,
duke@435	3461	Assembler::parity /* noParity = 0xb */ ,
duke@435	3462	Assembler::greaterEqual /* less = 0xc */ ,
duke@435	3463	Assembler::less /* greaterEqual = 0xd */ ,
duke@435	3464	Assembler::greater /* lessEqual = 0xe */ ,
duke@435	3465	Assembler::lessEqual /* greater = 0xf, */
duke@435	3466
duke@435	3467	};
duke@435	3468
duke@435	3469	// 32bit can do a case table jump in one instruction but we no longer allow the base
duke@435	3470	// to be installed in the Address class
duke@435	3471	void MacroAssembler::jump(ArrayAddress entry) {
duke@435	3472	#ifdef _LP64
duke@435	3473	lea(rscratch1, entry.base());
duke@435	3474	Address dispatch = entry.index();
duke@435	3475	assert(dispatch._base == noreg, "must be");
duke@435	3476	dispatch._base = rscratch1;
duke@435	3477	jmp(dispatch);
duke@435	3478	#else
duke@435	3479	jmp(as_Address(entry));
duke@435	3480	#endif // _LP64
duke@435	3481	}
duke@435	3482
duke@435	3483	void MacroAssembler::jump(AddressLiteral dst) {
duke@435	3484	if (reachable(dst)) {
duke@435	3485	jmp_literal(dst.target(), dst.rspec());
duke@435	3486	} else {
duke@435	3487	lea(rscratch1, dst);
duke@435	3488	jmp(rscratch1);
duke@435	3489	}
duke@435	3490	}
duke@435	3491
duke@435	3492	void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
duke@435	3493	if (reachable(dst)) {
duke@435	3494	InstructionMark im(this);
duke@435	3495	relocate(dst.reloc());
duke@435	3496	const int short_size = 2;
duke@435	3497	const int long_size = 6;
duke@435	3498	int offs = (intptr_t)dst.target() - ((intptr_t)_code_pos);
duke@435	3499	if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3500	// 0111 tttn #8-bit disp
duke@435	3501	emit_byte(0x70 | cc);
duke@435	3502	emit_byte((offs - short_size) & 0xFF);
duke@435	3503	} else {
duke@435	3504	// 0000 1111 1000 tttn #32-bit disp
duke@435	3505	emit_byte(0x0F);
duke@435	3506	emit_byte(0x80 | cc);
duke@435	3507	emit_long(offs - long_size);
duke@435	3508	}
duke@435	3509	} else {
duke@435	3510	#ifdef ASSERT
duke@435	3511	warning("reversing conditional branch");
duke@435	3512	#endif /* ASSERT */
duke@435	3513	Label skip;
duke@435	3514	jccb(reverse[cc], skip);
duke@435	3515	lea(rscratch1, dst);
duke@435	3516	Assembler::jmp(rscratch1);
duke@435	3517	bind(skip);
duke@435	3518	}
duke@435	3519	}
duke@435	3520
duke@435	3521	// Wouldn't need if AddressLiteral version had new name
duke@435	3522	void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	3523	Assembler::call(L, rtype);
duke@435	3524	}
duke@435	3525
duke@435	3526	// Wouldn't need if AddressLiteral version had new name
duke@435	3527	void MacroAssembler::call(Register entry) {
duke@435	3528	Assembler::call(entry);
duke@435	3529	}
duke@435	3530
duke@435	3531	void MacroAssembler::call(AddressLiteral entry) {
duke@435	3532	if (reachable(entry)) {
duke@435	3533	Assembler::call_literal(entry.target(), entry.rspec());
duke@435	3534	} else {
duke@435	3535	lea(rscratch1, entry);
duke@435	3536	Assembler::call(rscratch1);
duke@435	3537	}
duke@435	3538	}
duke@435	3539
duke@435	3540	void MacroAssembler::cmp8(AddressLiteral src1, int8_t src2) {
duke@435	3541	if (reachable(src1)) {
duke@435	3542	cmpb(as_Address(src1), src2);
duke@435	3543	} else {
duke@435	3544	lea(rscratch1, src1);
duke@435	3545	cmpb(Address(rscratch1, 0), src2);
duke@435	3546	}
duke@435	3547	}
duke@435	3548
duke@435	3549	void MacroAssembler::cmp32(AddressLiteral src1, int32_t src2) {
duke@435	3550	if (reachable(src1)) {
duke@435	3551	cmpl(as_Address(src1), src2);
duke@435	3552	} else {
duke@435	3553	lea(rscratch1, src1);
duke@435	3554	cmpl(Address(rscratch1, 0), src2);
duke@435	3555	}
duke@435	3556	}
duke@435	3557
duke@435	3558	void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
duke@435	3559	if (reachable(src2)) {
duke@435	3560	cmpl(src1, as_Address(src2));
duke@435	3561	} else {
duke@435	3562	lea(rscratch1, src2);
duke@435	3563	cmpl(src1, Address(rscratch1, 0));
duke@435	3564	}
duke@435	3565	}
duke@435	3566
duke@435	3567	void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
duke@435	3568	#ifdef _LP64
duke@435	3569	if (src2.is_lval()) {
duke@435	3570	movptr(rscratch1, src2);
duke@435	3571	Assembler::cmpq(src1, rscratch1);
duke@435	3572	} else if (reachable(src2)) {
duke@435	3573	cmpq(src1, as_Address(src2));
duke@435	3574	} else {
duke@435	3575	lea(rscratch1, src2);
duke@435	3576	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3577	}
duke@435	3578	#else
duke@435	3579	if (src2.is_lval()) {
duke@435	3580	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3581	} else {
duke@435	3582	cmpl(src1, as_Address(src2));
duke@435	3583	}
duke@435	3584	#endif // _LP64
duke@435	3585	}
duke@435	3586
duke@435	3587	void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
duke@435	3588	assert(src2.is_lval(), "not a mem-mem compare");
duke@435	3589	#ifdef _LP64
duke@435	3590	// moves src2's literal address
duke@435	3591	movptr(rscratch1, src2);
duke@435	3592	Assembler::cmpq(src1, rscratch1);
duke@435	3593	#else
duke@435	3594	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3595	#endif // _LP64
duke@435	3596	}
duke@435	3597
duke@435	3598	void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
duke@435	3599	assert(!src2.is_lval(), "should use cmpptr");
duke@435	3600
duke@435	3601	if (reachable(src2)) {
duke@435	3602	#ifdef _LP64
duke@435	3603	cmpq(src1, as_Address(src2));
duke@435	3604	#else
duke@435	3605	ShouldNotReachHere();
duke@435	3606	#endif // _LP64
duke@435	3607	} else {
duke@435	3608	lea(rscratch1, src2);
duke@435	3609	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3610	}
duke@435	3611	}
duke@435	3612
duke@435	3613	void MacroAssembler::cmpxchgptr(Register reg, AddressLiteral adr) {
duke@435	3614	if (reachable(adr)) {
duke@435	3615	#ifdef _LP64
duke@435	3616	cmpxchgq(reg, as_Address(adr));
duke@435	3617	#else
duke@435	3618	cmpxchgl(reg, as_Address(adr));
duke@435	3619	#endif // _LP64
duke@435	3620	} else {
duke@435	3621	lea(rscratch1, adr);
duke@435	3622	cmpxchgq(reg, Address(rscratch1, 0));
duke@435	3623	}
duke@435	3624	}
duke@435	3625
duke@435	3626	void MacroAssembler::incrementl(AddressLiteral dst) {
duke@435	3627	if (reachable(dst)) {
duke@435	3628	incrementl(as_Address(dst));
duke@435	3629	} else {
duke@435	3630	lea(rscratch1, dst);
duke@435	3631	incrementl(Address(rscratch1, 0));
duke@435	3632	}
duke@435	3633	}
duke@435	3634
duke@435	3635	void MacroAssembler::incrementl(ArrayAddress dst) {
duke@435	3636	incrementl(as_Address(dst));
duke@435	3637	}
duke@435	3638
duke@435	3639	void MacroAssembler::lea(Register dst, Address src) {
duke@435	3640	#ifdef _LP64
duke@435	3641	leaq(dst, src);
duke@435	3642	#else
duke@435	3643	leal(dst, src);
duke@435	3644	#endif // _LP64
duke@435	3645	}
duke@435	3646
duke@435	3647	void MacroAssembler::lea(Register dst, AddressLiteral src) {
duke@435	3648	#ifdef _LP64
duke@435	3649	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3650	#else
duke@435	3651	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3652	#endif // _LP64
duke@435	3653	}
duke@435	3654
duke@435	3655	void MacroAssembler::mov32(AddressLiteral dst, Register src) {
duke@435	3656	if (reachable(dst)) {
duke@435	3657	movl(as_Address(dst), src);
duke@435	3658	} else {
duke@435	3659	lea(rscratch1, dst);
duke@435	3660	movl(Address(rscratch1, 0), src);
duke@435	3661	}
duke@435	3662	}
duke@435	3663
duke@435	3664	void MacroAssembler::mov32(Register dst, AddressLiteral src) {
duke@435	3665	if (reachable(src)) {
duke@435	3666	movl(dst, as_Address(src));
duke@435	3667	} else {
duke@435	3668	lea(rscratch1, src);
duke@435	3669	movl(dst, Address(rscratch1, 0));
duke@435	3670	}
duke@435	3671	}
duke@435	3672
duke@435	3673	void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
duke@435	3674	if (reachable(src)) {
duke@435	3675	if (UseXmmLoadAndClearUpper) {
duke@435	3676	movsd (dst, as_Address(src));
duke@435	3677	} else {
duke@435	3678	movlpd(dst, as_Address(src));
duke@435	3679	}
duke@435	3680	} else {
duke@435	3681	lea(rscratch1, src);
duke@435	3682	if (UseXmmLoadAndClearUpper) {
duke@435	3683	movsd (dst, Address(rscratch1, 0));
duke@435	3684	} else {
duke@435	3685	movlpd(dst, Address(rscratch1, 0));
duke@435	3686	}
duke@435	3687	}
duke@435	3688	}
duke@435	3689
duke@435	3690	void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
duke@435	3691	if (reachable(src)) {
duke@435	3692	movss(dst, as_Address(src));
duke@435	3693	} else {
duke@435	3694	lea(rscratch1, src);
duke@435	3695	movss(dst, Address(rscratch1, 0));
duke@435	3696	}
duke@435	3697	}
duke@435	3698
duke@435	3699	void MacroAssembler::movoop(Register dst, jobject obj) {
duke@435	3700	mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3701	}
duke@435	3702
duke@435	3703	void MacroAssembler::movoop(Address dst, jobject obj) {
duke@435	3704	mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3705	movq(dst, rscratch1);
duke@435	3706	}
duke@435	3707
duke@435	3708	void MacroAssembler::movptr(Register dst, AddressLiteral src) {
duke@435	3709	#ifdef _LP64
duke@435	3710	if (src.is_lval()) {
duke@435	3711	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3712	} else {
duke@435	3713	if (reachable(src)) {
duke@435	3714	movq(dst, as_Address(src));
duke@435	3715	} else {
duke@435	3716	lea(rscratch1, src);
duke@435	3717	movq(dst, Address(rscratch1,0));
duke@435	3718	}
duke@435	3719	}
duke@435	3720	#else
duke@435	3721	if (src.is_lval()) {
duke@435	3722	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3723	} else {
duke@435	3724	movl(dst, as_Address(src));
duke@435	3725	}
duke@435	3726	#endif // LP64
duke@435	3727	}
duke@435	3728
duke@435	3729	void MacroAssembler::movptr(ArrayAddress dst, Register src) {
duke@435	3730	#ifdef _LP64
duke@435	3731	movq(as_Address(dst), src);
duke@435	3732	#else
duke@435	3733	movl(as_Address(dst), src);
duke@435	3734	#endif // _LP64
duke@435	3735	}
duke@435	3736
duke@435	3737	void MacroAssembler::pushoop(jobject obj) {
duke@435	3738	#ifdef _LP64
duke@435	3739	movoop(rscratch1, obj);
duke@435	3740	pushq(rscratch1);
duke@435	3741	#else
duke@435	3742	push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3743	#endif // _LP64
duke@435	3744	}
duke@435	3745
duke@435	3746	void MacroAssembler::pushptr(AddressLiteral src) {
duke@435	3747	#ifdef _LP64
duke@435	3748	lea(rscratch1, src);
duke@435	3749	if (src.is_lval()) {
duke@435	3750	pushq(rscratch1);
duke@435	3751	} else {
duke@435	3752	pushq(Address(rscratch1, 0));
duke@435	3753	}
duke@435	3754	#else
duke@435	3755	if (src.is_lval()) {
duke@435	3756	push_literal((int32_t)src.target(), src.rspec());
duke@435	3757	else {
duke@435	3758	pushl(as_Address(src));
duke@435	3759	}
duke@435	3760	#endif // _LP64
duke@435	3761	}
duke@435	3762
duke@435	3763	void MacroAssembler::ldmxcsr(AddressLiteral src) {
duke@435	3764	if (reachable(src)) {
duke@435	3765	Assembler::ldmxcsr(as_Address(src));
duke@435	3766	} else {
duke@435	3767	lea(rscratch1, src);
duke@435	3768	Assembler::ldmxcsr(Address(rscratch1, 0));
duke@435	3769	}
duke@435	3770	}
duke@435	3771
duke@435	3772	void MacroAssembler::movlpd(XMMRegister dst, AddressLiteral src) {
duke@435	3773	if (reachable(src)) {
duke@435	3774	movlpd(dst, as_Address(src));
duke@435	3775	} else {
duke@435	3776	lea(rscratch1, src);
duke@435	3777	movlpd(dst, Address(rscratch1, 0));
duke@435	3778	}
duke@435	3779	}
duke@435	3780
duke@435	3781	void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
duke@435	3782	if (reachable(src)) {
duke@435	3783	movss(dst, as_Address(src));
duke@435	3784	} else {
duke@435	3785	lea(rscratch1, src);
duke@435	3786	movss(dst, Address(rscratch1, 0));
duke@435	3787	}
duke@435	3788	}
duke@435	3789	void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
duke@435	3790	if (reachable(src)) {
duke@435	3791	xorpd(dst, as_Address(src));
duke@435	3792	} else {
duke@435	3793	lea(rscratch1, src);
duke@435	3794	xorpd(dst, Address(rscratch1, 0));
duke@435	3795	}
duke@435	3796	}
duke@435	3797
duke@435	3798	void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
duke@435	3799	if (reachable(src)) {
duke@435	3800	xorps(dst, as_Address(src));
duke@435	3801	} else {
duke@435	3802	lea(rscratch1, src);
duke@435	3803	xorps(dst, Address(rscratch1, 0));
duke@435	3804	}
duke@435	3805	}
duke@435	3806
duke@435	3807	void MacroAssembler::null_check(Register reg, int offset) {
duke@435	3808	if (needs_explicit_null_check(offset)) {
duke@435	3809	// provoke OS NULL exception if reg = NULL by
duke@435	3810	// accessing M[reg] w/o changing any (non-CC) registers
duke@435	3811	cmpq(rax, Address(reg, 0));
duke@435	3812	// Note: should probably use testl(rax, Address(reg, 0));
duke@435	3813	// may be shorter code (however, this version of
duke@435	3814	// testl needs to be implemented first)
duke@435	3815	} else {
duke@435	3816	// nothing to do, (later) access of M[reg + offset]
duke@435	3817	// will provoke OS NULL exception if reg = NULL
duke@435	3818	}
duke@435	3819	}
duke@435	3820
duke@435	3821	int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
duke@435	3822	int off = offset();
duke@435	3823	movzbl(dst, src);
duke@435	3824	return off;
duke@435	3825	}
duke@435	3826
duke@435	3827	int MacroAssembler::load_unsigned_word(Register dst, Address src) {
duke@435	3828	int off = offset();
duke@435	3829	movzwl(dst, src);
duke@435	3830	return off;
duke@435	3831	}
duke@435	3832
duke@435	3833	int MacroAssembler::load_signed_byte(Register dst, Address src) {
duke@435	3834	int off = offset();
duke@435	3835	movsbl(dst, src);
duke@435	3836	return off;
duke@435	3837	}
duke@435	3838
duke@435	3839	int MacroAssembler::load_signed_word(Register dst, Address src) {
duke@435	3840	int off = offset();
duke@435	3841	movswl(dst, src);
duke@435	3842	return off;
duke@435	3843	}
duke@435	3844
duke@435	3845	void MacroAssembler::incrementl(Register reg, int value) {
duke@435	3846	if (value == min_jint) { addl(reg, value); return; }
duke@435	3847	if (value < 0) { decrementl(reg, -value); return; }
duke@435	3848	if (value == 0) { ; return; }
duke@435	3849	if (value == 1 && UseIncDec) { incl(reg) ; return; }
duke@435	3850	/* else */ { addl(reg, value) ; return; }
duke@435	3851	}
duke@435	3852
duke@435	3853	void MacroAssembler::decrementl(Register reg, int value) {
duke@435	3854	if (value == min_jint) { subl(reg, value); return; }
duke@435	3855	if (value < 0) { incrementl(reg, -value); return; }
duke@435	3856	if (value == 0) { ; return; }
duke@435	3857	if (value == 1 && UseIncDec) { decl(reg) ; return; }
duke@435	3858	/* else */ { subl(reg, value) ; return; }
duke@435	3859	}
duke@435	3860
duke@435	3861	void MacroAssembler::incrementq(Register reg, int value) {
duke@435	3862	if (value == min_jint) { addq(reg, value); return; }
duke@435	3863	if (value < 0) { decrementq(reg, -value); return; }
duke@435	3864	if (value == 0) { ; return; }
duke@435	3865	if (value == 1 && UseIncDec) { incq(reg) ; return; }
duke@435	3866	/* else */ { addq(reg, value) ; return; }
duke@435	3867	}
duke@435	3868
duke@435	3869	void MacroAssembler::decrementq(Register reg, int value) {
duke@435	3870	if (value == min_jint) { subq(reg, value); return; }
duke@435	3871	if (value < 0) { incrementq(reg, -value); return; }
duke@435	3872	if (value == 0) { ; return; }
duke@435	3873	if (value == 1 && UseIncDec) { decq(reg) ; return; }
duke@435	3874	/* else */ { subq(reg, value) ; return; }
duke@435	3875	}
duke@435	3876
duke@435	3877	void MacroAssembler::incrementl(Address dst, int value) {
duke@435	3878	if (value == min_jint) { addl(dst, value); return; }
duke@435	3879	if (value < 0) { decrementl(dst, -value); return; }
duke@435	3880	if (value == 0) { ; return; }
duke@435	3881	if (value == 1 && UseIncDec) { incl(dst) ; return; }
duke@435	3882	/* else */ { addl(dst, value) ; return; }
duke@435	3883	}
duke@435	3884
duke@435	3885	void MacroAssembler::decrementl(Address dst, int value) {
duke@435	3886	if (value == min_jint) { subl(dst, value); return; }
duke@435	3887	if (value < 0) { incrementl(dst, -value); return; }
duke@435	3888	if (value == 0) { ; return; }
duke@435	3889	if (value == 1 && UseIncDec) { decl(dst) ; return; }
duke@435	3890	/* else */ { subl(dst, value) ; return; }
duke@435	3891	}
duke@435	3892
duke@435	3893	void MacroAssembler::incrementq(Address dst, int value) {
duke@435	3894	if (value == min_jint) { addq(dst, value); return; }
duke@435	3895	if (value < 0) { decrementq(dst, -value); return; }
duke@435	3896	if (value == 0) { ; return; }
duke@435	3897	if (value == 1 && UseIncDec) { incq(dst) ; return; }
duke@435	3898	/* else */ { addq(dst, value) ; return; }
duke@435	3899	}
duke@435	3900
duke@435	3901	void MacroAssembler::decrementq(Address dst, int value) {
duke@435	3902	if (value == min_jint) { subq(dst, value); return; }
duke@435	3903	if (value < 0) { incrementq(dst, -value); return; }
duke@435	3904	if (value == 0) { ; return; }
duke@435	3905	if (value == 1 && UseIncDec) { decq(dst) ; return; }
duke@435	3906	/* else */ { subq(dst, value) ; return; }
duke@435	3907	}
duke@435	3908
duke@435	3909	void MacroAssembler::align(int modulus) {
duke@435	3910	if (offset() % modulus != 0) {
duke@435	3911	nop(modulus - (offset() % modulus));
duke@435	3912	}
duke@435	3913	}
duke@435	3914
duke@435	3915	void MacroAssembler::enter() {
duke@435	3916	pushq(rbp);
duke@435	3917	movq(rbp, rsp);
duke@435	3918	}
duke@435	3919
duke@435	3920	void MacroAssembler::leave() {
duke@435	3921	emit_byte(0xC9); // LEAVE
duke@435	3922	}
duke@435	3923
duke@435	3924	// C++ bool manipulation
duke@435	3925
duke@435	3926	void MacroAssembler::movbool(Register dst, Address src) {
duke@435	3927	if(sizeof(bool) == 1)
duke@435	3928	movb(dst, src);
duke@435	3929	else if(sizeof(bool) == 2)
duke@435	3930	movw(dst, src);
duke@435	3931	else if(sizeof(bool) == 4)
duke@435	3932	movl(dst, src);
duke@435	3933	else {
duke@435	3934	// unsupported
duke@435	3935	ShouldNotReachHere();
duke@435	3936	}
duke@435	3937	}
duke@435	3938
duke@435	3939	void MacroAssembler::movbool(Address dst, bool boolconst) {
duke@435	3940	if(sizeof(bool) == 1)
duke@435	3941	movb(dst, (int) boolconst);
duke@435	3942	else if(sizeof(bool) == 2)
duke@435	3943	movw(dst, (int) boolconst);
duke@435	3944	else if(sizeof(bool) == 4)
duke@435	3945	movl(dst, (int) boolconst);
duke@435	3946	else {
duke@435	3947	// unsupported
duke@435	3948	ShouldNotReachHere();
duke@435	3949	}
duke@435	3950	}
duke@435	3951
duke@435	3952	void MacroAssembler::movbool(Address dst, Register src) {
duke@435	3953	if(sizeof(bool) == 1)
duke@435	3954	movb(dst, src);
duke@435	3955	else if(sizeof(bool) == 2)
duke@435	3956	movw(dst, src);
duke@435	3957	else if(sizeof(bool) == 4)
duke@435	3958	movl(dst, src);
duke@435	3959	else {
duke@435	3960	// unsupported
duke@435	3961	ShouldNotReachHere();
duke@435	3962	}
duke@435	3963	}
duke@435	3964
duke@435	3965	void MacroAssembler::testbool(Register dst) {
duke@435	3966	if(sizeof(bool) == 1)
duke@435	3967	testb(dst, (int) 0xff);
duke@435	3968	else if(sizeof(bool) == 2) {
duke@435	3969	// need testw impl
duke@435	3970	ShouldNotReachHere();
duke@435	3971	} else if(sizeof(bool) == 4)
duke@435	3972	testl(dst, dst);
duke@435	3973	else {
duke@435	3974	// unsupported
duke@435	3975	ShouldNotReachHere();
duke@435	3976	}
duke@435	3977	}
duke@435	3978
duke@435	3979	void MacroAssembler::set_last_Java_frame(Register last_java_sp,
duke@435	3980	Register last_java_fp,
duke@435	3981	address last_java_pc) {
duke@435	3982	// determine last_java_sp register
duke@435	3983	if (!last_java_sp->is_valid()) {
duke@435	3984	last_java_sp = rsp;
duke@435	3985	}
duke@435	3986
duke@435	3987	// last_java_fp is optional
duke@435	3988	if (last_java_fp->is_valid()) {
duke@435	3989	movq(Address(r15_thread, JavaThread::last_Java_fp_offset()),
duke@435	3990	last_java_fp);
duke@435	3991	}
duke@435	3992
duke@435	3993	// last_java_pc is optional
duke@435	3994	if (last_java_pc != NULL) {
duke@435	3995	Address java_pc(r15_thread,
duke@435	3996	JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
duke@435	3997	lea(rscratch1, InternalAddress(last_java_pc));
duke@435	3998	movq(java_pc, rscratch1);
duke@435	3999	}
duke@435	4000
duke@435	4001	movq(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
duke@435	4002	}
duke@435	4003
duke@435	4004	void MacroAssembler::reset_last_Java_frame(bool clear_fp,
duke@435	4005	bool clear_pc) {
duke@435	4006	// we must set sp to zero to clear frame
duke@435	4007	movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
duke@435	4008	// must clear fp, so that compiled frames are not confused; it is
duke@435	4009	// possible that we need it only for debugging
duke@435	4010	if (clear_fp) {
duke@435	4011	movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
duke@435	4012	}
duke@435	4013
duke@435	4014	if (clear_pc) {
duke@435	4015	movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
duke@435	4016	}
duke@435	4017	}
duke@435	4018
duke@435	4019
duke@435	4020	// Implementation of call_VM versions
duke@435	4021
duke@435	4022	void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
duke@435	4023	Label L, E;
duke@435	4024
duke@435	4025	#ifdef _WIN64
duke@435	4026	// Windows always allocates space for it's register args
duke@435	4027	assert(num_args <= 4, "only register arguments supported");
duke@435	4028	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4029	#endif
duke@435	4030
duke@435	4031	// Align stack if necessary
duke@435	4032	testl(rsp, 15);
duke@435	4033	jcc(Assembler::zero, L);
duke@435	4034
duke@435	4035	subq(rsp, 8);
duke@435	4036	{
duke@435	4037	call(RuntimeAddress(entry_point));
duke@435	4038	}
duke@435	4039	addq(rsp, 8);
duke@435	4040	jmp(E);
duke@435	4041
duke@435	4042	bind(L);
duke@435	4043	{
duke@435	4044	call(RuntimeAddress(entry_point));
duke@435	4045	}
duke@435	4046
duke@435	4047	bind(E);
duke@435	4048
duke@435	4049	#ifdef _WIN64
duke@435	4050	// restore stack pointer
duke@435	4051	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4052	#endif
duke@435	4053
duke@435	4054	}
duke@435	4055
duke@435	4056
duke@435	4057	void MacroAssembler::call_VM_base(Register oop_result,
duke@435	4058	Register java_thread,
duke@435	4059	Register last_java_sp,
duke@435	4060	address entry_point,
duke@435	4061	int num_args,
duke@435	4062	bool check_exceptions) {
duke@435	4063	// determine last_java_sp register
duke@435	4064	if (!last_java_sp->is_valid()) {
duke@435	4065	last_java_sp = rsp;
duke@435	4066	}
duke@435	4067
duke@435	4068	// debugging support
duke@435	4069	assert(num_args >= 0, "cannot have negative number of arguments");
duke@435	4070	assert(r15_thread != oop_result,
duke@435	4071	"cannot use the same register for java_thread & oop_result");
duke@435	4072	assert(r15_thread != last_java_sp,
duke@435	4073	"cannot use the same register for java_thread & last_java_sp");
duke@435	4074
duke@435	4075	// set last Java frame before call
duke@435	4076
duke@435	4077	// This sets last_Java_fp which is only needed from interpreted frames
duke@435	4078	// and should really be done only from the interp_masm version before
duke@435	4079	// calling the underlying call_VM. That doesn't happen yet so we set
duke@435	4080	// last_Java_fp here even though some callers don't need it and
duke@435	4081	// also clear it below.
duke@435	4082	set_last_Java_frame(last_java_sp, rbp, NULL);
duke@435	4083
duke@435	4084	{
duke@435	4085	Label L, E;
duke@435	4086
duke@435	4087	// Align stack if necessary
duke@435	4088	#ifdef _WIN64
duke@435	4089	assert(num_args <= 4, "only register arguments supported");
duke@435	4090	// Windows always allocates space for it's register args
duke@435	4091	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4092	#endif
duke@435	4093	testl(rsp, 15);
duke@435	4094	jcc(Assembler::zero, L);
duke@435	4095
duke@435	4096	subq(rsp, 8);
duke@435	4097	{
duke@435	4098	call(RuntimeAddress(entry_point));
duke@435	4099	}
duke@435	4100	addq(rsp, 8);
duke@435	4101	jmp(E);
duke@435	4102
duke@435	4103
duke@435	4104	bind(L);
duke@435	4105	{
duke@435	4106	call(RuntimeAddress(entry_point));
duke@435	4107	}
duke@435	4108
duke@435	4109	bind(E);
duke@435	4110
duke@435	4111	#ifdef _WIN64
duke@435	4112	// restore stack pointer
duke@435	4113	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4114	#endif
duke@435	4115	}
duke@435	4116
duke@435	4117	#ifdef ASSERT
duke@435	4118	pushq(rax);
duke@435	4119	{
duke@435	4120	Label L;
duke@435	4121	get_thread(rax);
duke@435	4122	cmpq(r15_thread, rax);
duke@435	4123	jcc(Assembler::equal, L);
duke@435	4124	stop("MacroAssembler::call_VM_base: register not callee saved?");
duke@435	4125	bind(L);
duke@435	4126	}
duke@435	4127	popq(rax);
duke@435	4128	#endif
duke@435	4129
duke@435	4130	// reset last Java frame
duke@435	4131	// This really shouldn't have to clear fp set note above at the
duke@435	4132	// call to set_last_Java_frame
duke@435	4133	reset_last_Java_frame(true, false);
duke@435	4134
duke@435	4135	check_and_handle_popframe(noreg);
duke@435	4136	check_and_handle_earlyret(noreg);
duke@435	4137
duke@435	4138	if (check_exceptions) {
duke@435	4139	cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL);
duke@435	4140	// This used to conditionally jump to forward_exception however it is
duke@435	4141	// possible if we relocate that the branch will not reach. So we must jump
duke@435	4142	// around so we can always reach
duke@435	4143	Label ok;
duke@435	4144	jcc(Assembler::equal, ok);
duke@435	4145	jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	4146	bind(ok);
duke@435	4147	}
duke@435	4148
duke@435	4149	// get oop result if there is one and reset the value in the thread
duke@435	4150	if (oop_result->is_valid()) {
duke@435	4151	movq(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
duke@435	4152	movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
duke@435	4153	verify_oop(oop_result);
duke@435	4154	}
duke@435	4155	}
duke@435	4156
duke@435	4157	void MacroAssembler::check_and_handle_popframe(Register java_thread) {}
duke@435	4158	void MacroAssembler::check_and_handle_earlyret(Register java_thread) {}
duke@435	4159
duke@435	4160	void MacroAssembler::call_VM_helper(Register oop_result,
duke@435	4161	address entry_point,
duke@435	4162	int num_args,
duke@435	4163	bool check_exceptions) {
duke@435	4164	// Java thread becomes first argument of C function
duke@435	4165	movq(c_rarg0, r15_thread);
duke@435	4166
duke@435	4167	// We've pushed one address, correct last_Java_sp
duke@435	4168	leaq(rax, Address(rsp, wordSize));
duke@435	4169
duke@435	4170	call_VM_base(oop_result, noreg, rax, entry_point, num_args,
duke@435	4171	check_exceptions);
duke@435	4172	}
duke@435	4173
duke@435	4174
duke@435	4175	void MacroAssembler::call_VM(Register oop_result,
duke@435	4176	address entry_point,
duke@435	4177	bool check_exceptions) {
duke@435	4178	Label C, E;
duke@435	4179	Assembler::call(C, relocInfo::none);
duke@435	4180	jmp(E);
duke@435	4181
duke@435	4182	bind(C);
duke@435	4183	call_VM_helper(oop_result, entry_point, 0, check_exceptions);
duke@435	4184	ret(0);
duke@435	4185
duke@435	4186	bind(E);
duke@435	4187	}
duke@435	4188
duke@435	4189
duke@435	4190	void MacroAssembler::call_VM(Register oop_result,
duke@435	4191	address entry_point,
duke@435	4192	Register arg_1,
duke@435	4193	bool check_exceptions) {
duke@435	4194	assert(rax != arg_1, "smashed argument");
duke@435	4195	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4196
duke@435	4197	Label C, E;
duke@435	4198	Assembler::call(C, relocInfo::none);
duke@435	4199	jmp(E);
duke@435	4200
duke@435	4201	bind(C);
duke@435	4202	// c_rarg0 is reserved for thread
duke@435	4203	if (c_rarg1 != arg_1) {
duke@435	4204	movq(c_rarg1, arg_1);
duke@435	4205	}
duke@435	4206	call_VM_helper(oop_result, entry_point, 1, check_exceptions);
duke@435	4207	ret(0);
duke@435	4208
duke@435	4209	bind(E);
duke@435	4210	}
duke@435	4211
duke@435	4212	void MacroAssembler::call_VM(Register oop_result,
duke@435	4213	address entry_point,
duke@435	4214	Register arg_1,
duke@435	4215	Register arg_2,
duke@435	4216	bool check_exceptions) {
duke@435	4217	assert(rax != arg_1, "smashed argument");
duke@435	4218	assert(rax != arg_2, "smashed argument");
duke@435	4219	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4220	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4221	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4222	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4223
duke@435	4224	Label C, E;
duke@435	4225	Assembler::call(C, relocInfo::none);
duke@435	4226	jmp(E);
duke@435	4227
duke@435	4228	bind(C);
duke@435	4229	// c_rarg0 is reserved for thread
duke@435	4230	if (c_rarg1 != arg_1) {
duke@435	4231	movq(c_rarg1, arg_1);
duke@435	4232	}
duke@435	4233	if (c_rarg2 != arg_2) {
duke@435	4234	movq(c_rarg2, arg_2);
duke@435	4235	}
duke@435	4236	call_VM_helper(oop_result, entry_point, 2, check_exceptions);
duke@435	4237	ret(0);
duke@435	4238
duke@435	4239	bind(E);
duke@435	4240	}
duke@435	4241
duke@435	4242
duke@435	4243	void MacroAssembler::call_VM(Register oop_result,
duke@435	4244	address entry_point,
duke@435	4245	Register arg_1,
duke@435	4246	Register arg_2,
duke@435	4247	Register arg_3,
duke@435	4248	bool check_exceptions) {
duke@435	4249	assert(rax != arg_1, "smashed argument");
duke@435	4250	assert(rax != arg_2, "smashed argument");
duke@435	4251	assert(rax != arg_3, "smashed argument");
duke@435	4252	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4253	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4254	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4255	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4256	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4257	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4258	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4259	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4260	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4261
duke@435	4262	Label C, E;
duke@435	4263	Assembler::call(C, relocInfo::none);
duke@435	4264	jmp(E);
duke@435	4265
duke@435	4266	bind(C);
duke@435	4267	// c_rarg0 is reserved for thread
duke@435	4268	if (c_rarg1 != arg_1) {
duke@435	4269	movq(c_rarg1, arg_1);
duke@435	4270	}
duke@435	4271	if (c_rarg2 != arg_2) {
duke@435	4272	movq(c_rarg2, arg_2);
duke@435	4273	}
duke@435	4274	if (c_rarg3 != arg_3) {
duke@435	4275	movq(c_rarg3, arg_3);
duke@435	4276	}
duke@435	4277	call_VM_helper(oop_result, entry_point, 3, check_exceptions);
duke@435	4278	ret(0);
duke@435	4279
duke@435	4280	bind(E);
duke@435	4281	}
duke@435	4282
duke@435	4283	void MacroAssembler::call_VM(Register oop_result,
duke@435	4284	Register last_java_sp,
duke@435	4285	address entry_point,
duke@435	4286	int num_args,
duke@435	4287	bool check_exceptions) {
duke@435	4288	call_VM_base(oop_result, noreg, last_java_sp, entry_point, num_args,
duke@435	4289	check_exceptions);
duke@435	4290	}
duke@435	4291
duke@435	4292	void MacroAssembler::call_VM(Register oop_result,
duke@435	4293	Register last_java_sp,
duke@435	4294	address entry_point,
duke@435	4295	Register arg_1,
duke@435	4296	bool check_exceptions) {
duke@435	4297	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4298	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4299	// c_rarg0 is reserved for thread
duke@435	4300	if (c_rarg1 != arg_1) {
duke@435	4301	movq(c_rarg1, arg_1);
duke@435	4302	}
duke@435	4303	call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
duke@435	4304	}
duke@435	4305
duke@435	4306	void MacroAssembler::call_VM(Register oop_result,
duke@435	4307	Register last_java_sp,
duke@435	4308	address entry_point,
duke@435	4309	Register arg_1,
duke@435	4310	Register arg_2,
duke@435	4311	bool check_exceptions) {
duke@435	4312	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4313	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4314	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4315	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4316	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4317	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4318	// c_rarg0 is reserved for thread
duke@435	4319	if (c_rarg1 != arg_1) {
duke@435	4320	movq(c_rarg1, arg_1);
duke@435	4321	}
duke@435	4322	if (c_rarg2 != arg_2) {
duke@435	4323	movq(c_rarg2, arg_2);
duke@435	4324	}
duke@435	4325	call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
duke@435	4326	}
duke@435	4327
duke@435	4328
duke@435	4329	void MacroAssembler::call_VM(Register oop_result,
duke@435	4330	Register last_java_sp,
duke@435	4331	address entry_point,
duke@435	4332	Register arg_1,
duke@435	4333	Register arg_2,
duke@435	4334	Register arg_3,
duke@435	4335	bool check_exceptions) {
duke@435	4336	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4337	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4338	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4339	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4340	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4341	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4342	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4343	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4344	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4345	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4346	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4347	assert(c_rarg3 != last_java_sp, "smashed argument");
duke@435	4348	// c_rarg0 is reserved for thread
duke@435	4349	if (c_rarg1 != arg_1) {
duke@435	4350	movq(c_rarg1, arg_1);
duke@435	4351	}
duke@435	4352	if (c_rarg2 != arg_2) {
duke@435	4353	movq(c_rarg2, arg_2);
duke@435	4354	}
duke@435	4355	if (c_rarg3 != arg_3) {
duke@435	4356	movq(c_rarg2, arg_3);
duke@435	4357	}
duke@435	4358	call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
duke@435	4359	}
duke@435	4360
duke@435	4361	void MacroAssembler::call_VM_leaf(address entry_point, int num_args) {
duke@435	4362	call_VM_leaf_base(entry_point, num_args);
duke@435	4363	}
duke@435	4364
duke@435	4365	void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
duke@435	4366	if (c_rarg0 != arg_1) {
duke@435	4367	movq(c_rarg0, arg_1);
duke@435	4368	}
duke@435	4369	call_VM_leaf(entry_point, 1);
duke@435	4370	}
duke@435	4371
duke@435	4372	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4373	Register arg_1,
duke@435	4374	Register arg_2) {
duke@435	4375	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4376	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4377	if (c_rarg0 != arg_1) {
duke@435	4378	movq(c_rarg0, arg_1);
duke@435	4379	}
duke@435	4380	if (c_rarg1 != arg_2) {
duke@435	4381	movq(c_rarg1, arg_2);
duke@435	4382	}
duke@435	4383	call_VM_leaf(entry_point, 2);
duke@435	4384	}
duke@435	4385
duke@435	4386	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4387	Register arg_1,
duke@435	4388	Register arg_2,
duke@435	4389	Register arg_3) {
duke@435	4390	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4391	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4392	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4393	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4394	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4395	assert(c_rarg2 != arg_2, "smashed argument");
duke@435	4396	if (c_rarg0 != arg_1) {
duke@435	4397	movq(c_rarg0, arg_1);
duke@435	4398	}
duke@435	4399	if (c_rarg1 != arg_2) {
duke@435	4400	movq(c_rarg1, arg_2);
duke@435	4401	}
duke@435	4402	if (c_rarg2 != arg_3) {
duke@435	4403	movq(c_rarg2, arg_3);
duke@435	4404	}
duke@435	4405	call_VM_leaf(entry_point, 3);
duke@435	4406	}
duke@435	4407
duke@435	4408
duke@435	4409	// Calls to C land
duke@435	4410	//
duke@435	4411	// When entering C land, the rbp & rsp of the last Java frame have to
duke@435	4412	// be recorded in the (thread-local) JavaThread object. When leaving C
duke@435	4413	// land, the last Java fp has to be reset to 0. This is required to
duke@435	4414	// allow proper stack traversal.
duke@435	4415	void MacroAssembler::store_check(Register obj) {
duke@435	4416	// Does a store check for the oop in register obj. The content of
duke@435	4417	// register obj is destroyed afterwards.
duke@435	4418	store_check_part_1(obj);
duke@435	4419	store_check_part_2(obj);
duke@435	4420	}
duke@435	4421
duke@435	4422	void MacroAssembler::store_check(Register obj, Address dst) {
duke@435	4423	store_check(obj);
duke@435	4424	}
duke@435	4425
duke@435	4426	// split the store check operation so that other instructions can be
duke@435	4427	// scheduled inbetween
duke@435	4428	void MacroAssembler::store_check_part_1(Register obj) {
duke@435	4429	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4430	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4431	shrq(obj, CardTableModRefBS::card_shift);
duke@435	4432	}
duke@435	4433
duke@435	4434	void MacroAssembler::store_check_part_2(Register obj) {
duke@435	4435	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4436	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4437	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	4438	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	4439	ExternalAddress cardtable((address)ct->byte_map_base);
duke@435	4440	Address index(noreg, obj, Address::times_1);
duke@435	4441	movb(as_Address(ArrayAddress(cardtable, index)), 0);
duke@435	4442	}
duke@435	4443
duke@435	4444	void MacroAssembler::c2bool(Register x) {
duke@435	4445	// implements x == 0 ? 0 : 1
duke@435	4446	// note: must only look at least-significant byte of x
duke@435	4447	// since C-style booleans are stored in one byte
duke@435	4448	// only! (was bug)
duke@435	4449	andl(x, 0xFF);
duke@435	4450	setb(Assembler::notZero, x);
duke@435	4451	}
duke@435	4452
duke@435	4453	int MacroAssembler::corrected_idivl(Register reg) {
duke@435	4454	// Full implementation of Java idiv and irem; checks for special
duke@435	4455	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4456	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4457	// for implicit exceptions.
duke@435	4458	//
duke@435	4459	// normal case special case
duke@435	4460	//
duke@435	4461	// input : eax: dividend min_int
duke@435	4462	// reg: divisor (may not be eax/edx) -1
duke@435	4463	//
duke@435	4464	// output: eax: quotient (= eax idiv reg) min_int
duke@435	4465	// edx: remainder (= eax irem reg) 0
duke@435	4466	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4467	const int min_int = 0x80000000;
duke@435	4468	Label normal_case, special_case;
duke@435	4469
duke@435	4470	// check for special case
duke@435	4471	cmpl(rax, min_int);
duke@435	4472	jcc(Assembler::notEqual, normal_case);
duke@435	4473	xorl(rdx, rdx); // prepare edx for possible special case (where
duke@435	4474	// remainder = 0)
duke@435	4475	cmpl(reg, -1);
duke@435	4476	jcc(Assembler::equal, special_case);
duke@435	4477
duke@435	4478	// handle normal case
duke@435	4479	bind(normal_case);
duke@435	4480	cdql();
duke@435	4481	int idivl_offset = offset();
duke@435	4482	idivl(reg);
duke@435	4483
duke@435	4484	// normal and special case exit
duke@435	4485	bind(special_case);
duke@435	4486
duke@435	4487	return idivl_offset;
duke@435	4488	}
duke@435	4489
duke@435	4490	int MacroAssembler::corrected_idivq(Register reg) {
duke@435	4491	// Full implementation of Java ldiv and lrem; checks for special
duke@435	4492	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4493	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4494	// for implicit exceptions.
duke@435	4495	//
duke@435	4496	// normal case special case
duke@435	4497	//
duke@435	4498	// input : rax: dividend min_long
duke@435	4499	// reg: divisor (may not be eax/edx) -1
duke@435	4500	//
duke@435	4501	// output: rax: quotient (= rax idiv reg) min_long
duke@435	4502	// rdx: remainder (= rax irem reg) 0
duke@435	4503	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4504	static const int64_t min_long = 0x8000000000000000;
duke@435	4505	Label normal_case, special_case;
duke@435	4506
duke@435	4507	// check for special case
duke@435	4508	cmp64(rax, ExternalAddress((address) &min_long));
duke@435	4509	jcc(Assembler::notEqual, normal_case);
duke@435	4510	xorl(rdx, rdx); // prepare rdx for possible special case (where
duke@435	4511	// remainder = 0)
duke@435	4512	cmpq(reg, -1);
duke@435	4513	jcc(Assembler::equal, special_case);
duke@435	4514
duke@435	4515	// handle normal case
duke@435	4516	bind(normal_case);
duke@435	4517	cdqq();
duke@435	4518	int idivq_offset = offset();
duke@435	4519	idivq(reg);
duke@435	4520
duke@435	4521	// normal and special case exit
duke@435	4522	bind(special_case);
duke@435	4523
duke@435	4524	return idivq_offset;
duke@435	4525	}
duke@435	4526
duke@435	4527	void MacroAssembler::push_IU_state() {
duke@435	4528	pushfq(); // Push flags first because pushaq kills them
duke@435	4529	subq(rsp, 8); // Make sure rsp stays 16-byte aligned
duke@435	4530	pushaq();
duke@435	4531	}
duke@435	4532
duke@435	4533	void MacroAssembler::pop_IU_state() {
duke@435	4534	popaq();
duke@435	4535	addq(rsp, 8);
duke@435	4536	popfq();
duke@435	4537	}
duke@435	4538
duke@435	4539	void MacroAssembler::push_FPU_state() {
duke@435	4540	subq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4541	fxsave(Address(rsp, 0));
duke@435	4542	}
duke@435	4543
duke@435	4544	void MacroAssembler::pop_FPU_state() {
duke@435	4545	fxrstor(Address(rsp, 0));
duke@435	4546	addq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4547	}
duke@435	4548
duke@435	4549	// Save Integer and Float state
duke@435	4550	// Warning: Stack must be 16 byte aligned
duke@435	4551	void MacroAssembler::push_CPU_state() {
duke@435	4552	push_IU_state();
duke@435	4553	push_FPU_state();
duke@435	4554	}
duke@435	4555
duke@435	4556	void MacroAssembler::pop_CPU_state() {
duke@435	4557	pop_FPU_state();
duke@435	4558	pop_IU_state();
duke@435	4559	}
duke@435	4560
duke@435	4561	void MacroAssembler::sign_extend_short(Register reg) {
duke@435	4562	movswl(reg, reg);
duke@435	4563	}
duke@435	4564
duke@435	4565	void MacroAssembler::sign_extend_byte(Register reg) {
duke@435	4566	movsbl(reg, reg);
duke@435	4567	}
duke@435	4568
duke@435	4569	void MacroAssembler::division_with_shift(Register reg, int shift_value) {
duke@435	4570	assert (shift_value > 0, "illegal shift value");
duke@435	4571	Label _is_positive;
duke@435	4572	testl (reg, reg);
duke@435	4573	jcc (Assembler::positive, _is_positive);
duke@435	4574	int offset = (1 << shift_value) - 1 ;
duke@435	4575
duke@435	4576	if (offset == 1) {
duke@435	4577	incrementl(reg);
duke@435	4578	} else {
duke@435	4579	addl(reg, offset);
duke@435	4580	}
duke@435	4581
duke@435	4582	bind (_is_positive);
duke@435	4583	sarl(reg, shift_value);
duke@435	4584	}
duke@435	4585
duke@435	4586	void MacroAssembler::round_to_l(Register reg, int modulus) {
duke@435	4587	addl(reg, modulus - 1);
duke@435	4588	andl(reg, -modulus);
duke@435	4589	}
duke@435	4590
duke@435	4591	void MacroAssembler::round_to_q(Register reg, int modulus) {
duke@435	4592	addq(reg, modulus - 1);
duke@435	4593	andq(reg, -modulus);
duke@435	4594	}
duke@435	4595
duke@435	4596	void MacroAssembler::verify_oop(Register reg, const char* s) {
duke@435	4597	if (!VerifyOops) {
duke@435	4598	return;
duke@435	4599	}
duke@435	4600
duke@435	4601	// Pass register number to verify_oop_subroutine
duke@435	4602	char* b = new char[strlen(s) + 50];
duke@435	4603	sprintf(b, "verify_oop: %s: %s", reg->name(), s);
duke@435	4604
duke@435	4605	pushq(rax); // save rax, restored by receiver
duke@435	4606
duke@435	4607	// pass args on stack, only touch rax
duke@435	4608	pushq(reg);
duke@435	4609	// avoid using pushptr, as it modifies scratch registers
duke@435	4610	// and our contract is not to modify anything
duke@435	4611	ExternalAddress buffer((address)b);
duke@435	4612	movptr(rax, buffer.addr());
duke@435	4613	pushq(rax);
duke@435	4614
duke@435	4615	// call indirectly to solve generation ordering problem
duke@435	4616	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4617	call(rax); // no alignment requirement
duke@435	4618	// everything popped by receiver
duke@435	4619	}
duke@435	4620
duke@435	4621	void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
duke@435	4622	if (!VerifyOops) return;
duke@435	4623	// Pass register number to verify_oop_subroutine
duke@435	4624	char* b = new char[strlen(s) + 50];
duke@435	4625	sprintf(b, "verify_oop_addr: %s", s);
duke@435	4626	pushq(rax); // save rax
duke@435	4627	movq(addr, rax);
duke@435	4628	pushq(rax); // pass register argument
duke@435	4629
duke@435	4630
duke@435	4631	// avoid using pushptr, as it modifies scratch registers
duke@435	4632	// and our contract is not to modify anything
duke@435	4633	ExternalAddress buffer((address)b);
duke@435	4634	movptr(rax, buffer.addr());
duke@435	4635	pushq(rax);
duke@435	4636
duke@435	4637	// call indirectly to solve generation ordering problem
duke@435	4638	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4639	call(rax); // no alignment requirement
duke@435	4640	// everything popped by receiver
duke@435	4641	}
duke@435	4642
duke@435	4643
duke@435	4644	void MacroAssembler::stop(const char* msg) {
duke@435	4645	address rip = pc();
duke@435	4646	pushaq(); // get regs on stack
duke@435	4647	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4648	lea(c_rarg1, InternalAddress(rip));
duke@435	4649	movq(c_rarg2, rsp); // pass pointer to regs array
duke@435	4650	andq(rsp, -16); // align stack as required by ABI
duke@435	4651	call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
duke@435	4652	hlt();
duke@435	4653	}
duke@435	4654
duke@435	4655	void MacroAssembler::warn(const char* msg) {
duke@435	4656	pushq(r12);
duke@435	4657	movq(r12, rsp);
duke@435	4658	andq(rsp, -16); // align stack as required by push_CPU_state and call
duke@435	4659
duke@435	4660	push_CPU_state(); // keeps alignment at 16 bytes
duke@435	4661	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4662	call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
duke@435	4663	pop_CPU_state();
duke@435	4664
duke@435	4665	movq(rsp, r12);
duke@435	4666	popq(r12);
duke@435	4667	}
duke@435	4668
duke@435	4669	void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
duke@435	4670	// In order to get locks to work, we need to fake a in_VM state
duke@435	4671	if (ShowMessageBoxOnError ) {
duke@435	4672	JavaThread* thread = JavaThread::current();
duke@435	4673	JavaThreadState saved_state = thread->thread_state();
duke@435	4674	thread->set_thread_state(_thread_in_vm);
duke@435	4675	#ifndef PRODUCT
duke@435	4676	if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
coleenp@548	4677	ttyLocker ttyl;
duke@435	4678	BytecodeCounter::print();
duke@435	4679	}
duke@435	4680	#endif
duke@435	4681	// To see where a verify_oop failed, get $ebx+40/X for this frame.
duke@435	4682	// XXX correct this offset for amd64
duke@435	4683	// This is the value of eip which points to where verify_oop will return.
duke@435	4684	if (os::message_box(msg, "Execution stopped, print registers?")) {
coleenp@548	4685	ttyLocker ttyl;
duke@435	4686	tty->print_cr("rip = 0x%016lx", pc);
duke@435	4687	tty->print_cr("rax = 0x%016lx", regs[15]);
duke@435	4688	tty->print_cr("rbx = 0x%016lx", regs[12]);
duke@435	4689	tty->print_cr("rcx = 0x%016lx", regs[14]);
duke@435	4690	tty->print_cr("rdx = 0x%016lx", regs[13]);
duke@435	4691	tty->print_cr("rdi = 0x%016lx", regs[8]);
duke@435	4692	tty->print_cr("rsi = 0x%016lx", regs[9]);
duke@435	4693	tty->print_cr("rbp = 0x%016lx", regs[10]);
duke@435	4694	tty->print_cr("rsp = 0x%016lx", regs[11]);
duke@435	4695	tty->print_cr("r8 = 0x%016lx", regs[7]);
duke@435	4696	tty->print_cr("r9 = 0x%016lx", regs[6]);
duke@435	4697	tty->print_cr("r10 = 0x%016lx", regs[5]);
duke@435	4698	tty->print_cr("r11 = 0x%016lx", regs[4]);
duke@435	4699	tty->print_cr("r12 = 0x%016lx", regs[3]);
duke@435	4700	tty->print_cr("r13 = 0x%016lx", regs[2]);
duke@435	4701	tty->print_cr("r14 = 0x%016lx", regs[1]);
duke@435	4702	tty->print_cr("r15 = 0x%016lx", regs[0]);
duke@435	4703	BREAKPOINT;
duke@435	4704	}
duke@435	4705	ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
duke@435	4706	} else {
coleenp@548	4707	ttyLocker ttyl;
duke@435	4708	::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
duke@435	4709	msg);
duke@435	4710	}
duke@435	4711	}
duke@435	4712
duke@435	4713	void MacroAssembler::os_breakpoint() {
duke@435	4714	// instead of directly emitting a breakpoint, call os:breakpoint for
duke@435	4715	// better debugability
duke@435	4716	// This shouldn't need alignment, it's an empty function
duke@435	4717	call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
duke@435	4718	}
duke@435	4719
duke@435	4720	// Write serialization page so VM thread can do a pseudo remote membar.
duke@435	4721	// We use the current thread pointer to calculate a thread specific
duke@435	4722	// offset to write to within the page. This minimizes bus traffic
duke@435	4723	// due to cache line collision.
duke@435	4724	void MacroAssembler::serialize_memory(Register thread,
duke@435	4725	Register tmp) {
duke@435	4726
duke@435	4727	movl(tmp, thread);
duke@435	4728	shrl(tmp, os::get_serialize_page_shift_count());
duke@435	4729	andl(tmp, (os::vm_page_size() - sizeof(int)));
duke@435	4730
duke@435	4731	Address index(noreg, tmp, Address::times_1);
duke@435	4732	ExternalAddress page(os::get_memory_serialize_page());
duke@435	4733
duke@435	4734	movptr(ArrayAddress(page, index), tmp);
duke@435	4735	}
duke@435	4736
duke@435	4737	void MacroAssembler::verify_tlab() {
duke@435	4738	#ifdef ASSERT
duke@435	4739	if (UseTLAB) {
duke@435	4740	Label next, ok;
duke@435	4741	Register t1 = rsi;
duke@435	4742
duke@435	4743	pushq(t1);
duke@435	4744
duke@435	4745	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4746	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset())));
duke@435	4747	jcc(Assembler::aboveEqual, next);
duke@435	4748	stop("assert(top >= start)");
duke@435	4749	should_not_reach_here();
duke@435	4750
duke@435	4751	bind(next);
duke@435	4752	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4753	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4754	jcc(Assembler::aboveEqual, ok);
duke@435	4755	stop("assert(top <= end)");
duke@435	4756	should_not_reach_here();
duke@435	4757
duke@435	4758	bind(ok);
duke@435	4759
duke@435	4760	popq(t1);
duke@435	4761	}
duke@435	4762	#endif
duke@435	4763	}
duke@435	4764
duke@435	4765	// Defines obj, preserves var_size_in_bytes
duke@435	4766	void MacroAssembler::eden_allocate(Register obj,
duke@435	4767	Register var_size_in_bytes,
duke@435	4768	int con_size_in_bytes,
duke@435	4769	Register t1,
duke@435	4770	Label& slow_case) {
duke@435	4771	assert(obj == rax, "obj must be in rax for cmpxchg");
duke@435	4772	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4773	Register end = t1;
duke@435	4774	Label retry;
duke@435	4775	bind(retry);
duke@435	4776	ExternalAddress heap_top((address) Universe::heap()->top_addr());
duke@435	4777	movptr(obj, heap_top);
duke@435	4778	if (var_size_in_bytes == noreg) {
duke@435	4779	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4780	} else {
duke@435	4781	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4782	}
duke@435	4783	// if end < obj then we wrapped around => object too long => slow case
duke@435	4784	cmpq(end, obj);
duke@435	4785	jcc(Assembler::below, slow_case);
duke@435	4786	cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
duke@435	4787
duke@435	4788	jcc(Assembler::above, slow_case);
duke@435	4789	// Compare obj with the top addr, and if still equal, store the new
duke@435	4790	// top addr in end at the address of the top addr pointer. Sets ZF
duke@435	4791	// if was equal, and clears it otherwise. Use lock prefix for
duke@435	4792	// atomicity on MPs.
duke@435	4793	if (os::is_MP()) {
duke@435	4794	lock();
duke@435	4795	}
duke@435	4796	cmpxchgptr(end, heap_top);
duke@435	4797	// if someone beat us on the allocation, try again, otherwise continue
duke@435	4798	jcc(Assembler::notEqual, retry);
duke@435	4799	}
duke@435	4800
duke@435	4801	// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
duke@435	4802	void MacroAssembler::tlab_allocate(Register obj,
duke@435	4803	Register var_size_in_bytes,
duke@435	4804	int con_size_in_bytes,
duke@435	4805	Register t1,
duke@435	4806	Register t2,
duke@435	4807	Label& slow_case) {
duke@435	4808	assert_different_registers(obj, t1, t2);
duke@435	4809	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4810	Register end = t2;
duke@435	4811
duke@435	4812	verify_tlab();
duke@435	4813
duke@435	4814	movq(obj, Address(r15_thread, JavaThread::tlab_top_offset()));
duke@435	4815	if (var_size_in_bytes == noreg) {
duke@435	4816	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4817	} else {
duke@435	4818	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4819	}
duke@435	4820	cmpq(end, Address(r15_thread, JavaThread::tlab_end_offset()));
duke@435	4821	jcc(Assembler::above, slow_case);
duke@435	4822
duke@435	4823	// update the tlab top pointer
duke@435	4824	movq(Address(r15_thread, JavaThread::tlab_top_offset()), end);
duke@435	4825
duke@435	4826	// recover var_size_in_bytes if necessary
duke@435	4827	if (var_size_in_bytes == end) {
duke@435	4828	subq(var_size_in_bytes, obj);
duke@435	4829	}
duke@435	4830	verify_tlab();
duke@435	4831	}
duke@435	4832
duke@435	4833	// Preserves rbx and rdx.
duke@435	4834	void MacroAssembler::tlab_refill(Label& retry,
duke@435	4835	Label& try_eden,
duke@435	4836	Label& slow_case) {
duke@435	4837	Register top = rax;
duke@435	4838	Register t1 = rcx;
duke@435	4839	Register t2 = rsi;
duke@435	4840	Register t3 = r10;
duke@435	4841	Register thread_reg = r15_thread;
duke@435	4842	assert_different_registers(top, thread_reg, t1, t2, t3,
duke@435	4843	/* preserve: */ rbx, rdx);
duke@435	4844	Label do_refill, discard_tlab;
duke@435	4845
duke@435	4846	if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
duke@435	4847	// No allocation in the shared eden.
duke@435	4848	jmp(slow_case);
duke@435	4849	}
duke@435	4850
duke@435	4851	movq(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4852	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4853
duke@435	4854	// calculate amount of free space
duke@435	4855	subq(t1, top);
duke@435	4856	shrq(t1, LogHeapWordSize);
duke@435	4857
duke@435	4858	// Retain tlab and allocate object in shared space if
duke@435	4859	// the amount free in the tlab is too large to discard.
duke@435	4860	cmpq(t1, Address(thread_reg, // size_t
duke@435	4861	in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
duke@435	4862	jcc(Assembler::lessEqual, discard_tlab);
duke@435	4863
duke@435	4864	// Retain
duke@435	4865	mov64(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment());
duke@435	4866	addq(Address(thread_reg, // size_t
duke@435	4867	in_bytes(JavaThread::tlab_refill_waste_limit_offset())),
duke@435	4868	t2);
duke@435	4869	if (TLABStats) {
duke@435	4870	// increment number of slow_allocations
duke@435	4871	addl(Address(thread_reg, // unsigned int
duke@435	4872	in_bytes(JavaThread::tlab_slow_allocations_offset())),
duke@435	4873	1);
duke@435	4874	}
duke@435	4875	jmp(try_eden);
duke@435	4876
duke@435	4877	bind(discard_tlab);
duke@435	4878	if (TLABStats) {
duke@435	4879	// increment number of refills
duke@435	4880	addl(Address(thread_reg, // unsigned int
duke@435	4881	in_bytes(JavaThread::tlab_number_of_refills_offset())),
duke@435	4882	1);
duke@435	4883	// accumulate wastage -- t1 is amount free in tlab
duke@435	4884	addl(Address(thread_reg, // unsigned int
duke@435	4885	in_bytes(JavaThread::tlab_fast_refill_waste_offset())),
duke@435	4886	t1);
duke@435	4887	}
duke@435	4888
duke@435	4889	// if tlab is currently allocated (top or end != null) then
duke@435	4890	// fill [top, end + alignment_reserve) with array object
duke@435	4891	testq(top, top);
duke@435	4892	jcc(Assembler::zero, do_refill);
duke@435	4893
duke@435	4894	// set up the mark word
duke@435	4895	mov64(t3, (int64_t) markOopDesc::prototype()->copy_set_hash(0x2));
duke@435	4896	movq(Address(top, oopDesc::mark_offset_in_bytes()), t3);
duke@435	4897	// set the length to the remaining space
duke@435	4898	subq(t1, typeArrayOopDesc::header_size(T_INT));
duke@435	4899	addq(t1, (int)ThreadLocalAllocBuffer::alignment_reserve());
duke@435	4900	shlq(t1, log2_intptr(HeapWordSize / sizeof(jint)));
duke@435	4901	movq(Address(top, arrayOopDesc::length_offset_in_bytes()), t1);
duke@435	4902	// set klass to intArrayKlass
duke@435	4903	movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr()));
coleenp@548	4904	store_klass(top, t1);
duke@435	4905
duke@435	4906	// refill the tlab with an eden allocation
duke@435	4907	bind(do_refill);
duke@435	4908	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4909	shlq(t1, LogHeapWordSize);
duke@435	4910	// add object_size ??
duke@435	4911	eden_allocate(top, t1, 0, t2, slow_case);
duke@435	4912
duke@435	4913	// Check that t1 was preserved in eden_allocate.
duke@435	4914	#ifdef ASSERT
duke@435	4915	if (UseTLAB) {
duke@435	4916	Label ok;
duke@435	4917	Register tsize = rsi;
duke@435	4918	assert_different_registers(tsize, thread_reg, t1);
duke@435	4919	pushq(tsize);
duke@435	4920	movq(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4921	shlq(tsize, LogHeapWordSize);
duke@435	4922	cmpq(t1, tsize);
duke@435	4923	jcc(Assembler::equal, ok);
duke@435	4924	stop("assert(t1 != tlab size)");
duke@435	4925	should_not_reach_here();
duke@435	4926
duke@435	4927	bind(ok);
duke@435	4928	popq(tsize);
duke@435	4929	}
duke@435	4930	#endif
duke@435	4931	movq(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
duke@435	4932	movq(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
duke@435	4933	addq(top, t1);
duke@435	4934	subq(top, (int)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
duke@435	4935	movq(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
duke@435	4936	verify_tlab();
duke@435	4937	jmp(retry);
duke@435	4938	}
duke@435	4939
duke@435	4940
duke@435	4941	int MacroAssembler::biased_locking_enter(Register lock_reg, Register obj_reg, Register swap_reg, Register tmp_reg,
duke@435	4942	bool swap_reg_contains_mark,
duke@435	4943	Label& done, Label* slow_case,
duke@435	4944	BiasedLockingCounters* counters) {
duke@435	4945	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	4946	assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
duke@435	4947	assert(tmp_reg != noreg, "tmp_reg must be supplied");
duke@435	4948	assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
duke@435	4949	assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
duke@435	4950	Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
duke@435	4951	Address saved_mark_addr(lock_reg, 0);
duke@435	4952
duke@435	4953	if (PrintBiasedLockingStatistics && counters == NULL)
duke@435	4954	counters = BiasedLocking::counters();
duke@435	4955
duke@435	4956	// Biased locking
duke@435	4957	// See whether the lock is currently biased toward our thread and
duke@435	4958	// whether the epoch is still valid
duke@435	4959	// Note that the runtime guarantees sufficient alignment of JavaThread
duke@435	4960	// pointers to allow age to be placed into low bits
duke@435	4961	// First check to see whether biasing is even enabled for this object
duke@435	4962	Label cas_label;
duke@435	4963	int null_check_offset = -1;
duke@435	4964	if (!swap_reg_contains_mark) {
duke@435	4965	null_check_offset = offset();
duke@435	4966	movq(swap_reg, mark_addr);
duke@435	4967	}
duke@435	4968	movq(tmp_reg, swap_reg);
duke@435	4969	andq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	4970	cmpq(tmp_reg, markOopDesc::biased_lock_pattern);
duke@435	4971	jcc(Assembler::notEqual, cas_label);
duke@435	4972	// The bias pattern is present in the object's header. Need to check
duke@435	4973	// whether the bias owner and the epoch are both still current.
coleenp@548	4974	load_klass(tmp_reg, obj_reg);
duke@435	4975	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	4976	orq(tmp_reg, r15_thread);
duke@435	4977	xorq(tmp_reg, swap_reg);
duke@435	4978	andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place));
duke@435	4979	if (counters != NULL) {
duke@435	4980	cond_inc32(Assembler::zero,
duke@435	4981	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	4982	}
duke@435	4983	jcc(Assembler::equal, done);
duke@435	4984
duke@435	4985	Label try_revoke_bias;
duke@435	4986	Label try_rebias;
duke@435	4987
duke@435	4988	// At this point we know that the header has the bias pattern and
duke@435	4989	// that we are not the bias owner in the current epoch. We need to
duke@435	4990	// figure out more details about the state of the header in order to
duke@435	4991	// know what operations can be legally performed on the object's
duke@435	4992	// header.
duke@435	4993
duke@435	4994	// If the low three bits in the xor result aren't clear, that means
duke@435	4995	// the prototype header is no longer biased and we have to revoke
duke@435	4996	// the bias on this object.
duke@435	4997	testq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	4998	jcc(Assembler::notZero, try_revoke_bias);
duke@435	4999
duke@435	5000	// Biasing is still enabled for this data type. See whether the
duke@435	5001	// epoch of the current bias is still valid, meaning that the epoch
duke@435	5002	// bits of the mark word are equal to the epoch bits of the
duke@435	5003	// prototype header. (Note that the prototype header's epoch bits
duke@435	5004	// only change at a safepoint.) If not, attempt to rebias the object
duke@435	5005	// toward the current thread. Note that we must be absolutely sure
duke@435	5006	// that the current epoch is invalid in order to do this because
duke@435	5007	// otherwise the manipulations it performs on the mark word are
duke@435	5008	// illegal.
duke@435	5009	testq(tmp_reg, markOopDesc::epoch_mask_in_place);
duke@435	5010	jcc(Assembler::notZero, try_rebias);
duke@435	5011
duke@435	5012	// The epoch of the current bias is still valid but we know nothing
duke@435	5013	// about the owner; it might be set or it might be clear. Try to
duke@435	5014	// acquire the bias of the object using an atomic operation. If this
duke@435	5015	// fails we will go in to the runtime to revoke the object's bias.
duke@435	5016	// Note that we first construct the presumed unbiased header so we
duke@435	5017	// don't accidentally blow away another thread's valid bias.
duke@435	5018	andq(swap_reg,
duke@435	5019	markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
duke@435	5020	movq(tmp_reg, swap_reg);
duke@435	5021	orq(tmp_reg, r15_thread);
duke@435	5022	if (os::is_MP()) {
duke@435	5023	lock();
duke@435	5024	}
duke@435	5025	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5026	// If the biasing toward our thread failed, this means that
duke@435	5027	// another thread succeeded in biasing it toward itself and we
duke@435	5028	// need to revoke that bias. The revocation will occur in the
duke@435	5029	// interpreter runtime in the slow case.
duke@435	5030	if (counters != NULL) {
duke@435	5031	cond_inc32(Assembler::zero,
duke@435	5032	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	5033	}
duke@435	5034	if (slow_case != NULL) {
duke@435	5035	jcc(Assembler::notZero, *slow_case);
duke@435	5036	}
duke@435	5037	jmp(done);
duke@435	5038
duke@435	5039	bind(try_rebias);
duke@435	5040	// At this point we know the epoch has expired, meaning that the
duke@435	5041	// current "bias owner", if any, is actually invalid. Under these
duke@435	5042	// circumstances _only_, we are allowed to use the current header's
duke@435	5043	// value as the comparison value when doing the cas to acquire the
duke@435	5044	// bias in the current epoch. In other words, we allow transfer of
duke@435	5045	// the bias from one thread to another directly in this situation.
duke@435	5046	//
duke@435	5047	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5048	// bits in this situation. Should attempt to preserve them.
coleenp@548	5049	load_klass(tmp_reg, obj_reg);
duke@435	5050	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5051	orq(tmp_reg, r15_thread);
duke@435	5052	if (os::is_MP()) {
duke@435	5053	lock();
duke@435	5054	}
duke@435	5055	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5056	// If the biasing toward our thread failed, then another thread
duke@435	5057	// succeeded in biasing it toward itself and we need to revoke that
duke@435	5058	// bias. The revocation will occur in the runtime in the slow case.
duke@435	5059	if (counters != NULL) {
duke@435	5060	cond_inc32(Assembler::zero,
duke@435	5061	ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
duke@435	5062	}
duke@435	5063	if (slow_case != NULL) {
duke@435	5064	jcc(Assembler::notZero, *slow_case);
duke@435	5065	}
duke@435	5066	jmp(done);
duke@435	5067
duke@435	5068	bind(try_revoke_bias);
duke@435	5069	// The prototype mark in the klass doesn't have the bias bit set any
duke@435	5070	// more, indicating that objects of this data type are not supposed
duke@435	5071	// to be biased any more. We are going to try to reset the mark of
duke@435	5072	// this object to the prototype value and fall through to the
duke@435	5073	// CAS-based locking scheme. Note that if our CAS fails, it means
duke@435	5074	// that another thread raced us for the privilege of revoking the
duke@435	5075	// bias of this particular object, so it's okay to continue in the
duke@435	5076	// normal locking code.
duke@435	5077	//
duke@435	5078	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5079	// bits in this situation. Should attempt to preserve them.
coleenp@548	5080	load_klass(tmp_reg, obj_reg);
duke@435	5081	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5082	if (os::is_MP()) {
duke@435	5083	lock();
duke@435	5084	}
duke@435	5085	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5086	// Fall through to the normal CAS-based lock, because no matter what
duke@435	5087	// the result of the above CAS, some thread must have succeeded in
duke@435	5088	// removing the bias bit from the object's header.
duke@435	5089	if (counters != NULL) {
duke@435	5090	cond_inc32(Assembler::zero,
duke@435	5091	ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
duke@435	5092	}
duke@435	5093
duke@435	5094	bind(cas_label);
duke@435	5095
duke@435	5096	return null_check_offset;
duke@435	5097	}
duke@435	5098
duke@435	5099
duke@435	5100	void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
duke@435	5101	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	5102
duke@435	5103	// Check for biased locking unlock case, which is a no-op
duke@435	5104	// Note: we do not have to check the thread ID for two reasons.
duke@435	5105	// First, the interpreter checks for IllegalMonitorStateException at
duke@435	5106	// a higher level. Second, if the bias was revoked while we held the
duke@435	5107	// lock, the object could not be rebiased toward another thread, so
duke@435	5108	// the bias bit would be clear.
duke@435	5109	movq(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
duke@435	5110	andq(temp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	5111	cmpq(temp_reg, markOopDesc::biased_lock_pattern);
duke@435	5112	jcc(Assembler::equal, done);
duke@435	5113	}
duke@435	5114
duke@435	5115
coleenp@548	5116	void MacroAssembler::load_klass(Register dst, Register src) {
coleenp@548	5117	if (UseCompressedOops) {
coleenp@548	5118	movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
coleenp@548	5119	decode_heap_oop_not_null(dst);
coleenp@548	5120	} else {
coleenp@548	5121	movq(dst, Address(src, oopDesc::klass_offset_in_bytes()));
coleenp@548	5122	}
coleenp@548	5123	}
coleenp@548	5124
coleenp@548	5125	void MacroAssembler::store_klass(Register dst, Register src) {
coleenp@548	5126	if (UseCompressedOops) {
coleenp@548	5127	encode_heap_oop_not_null(src);
coleenp@548	5128	// zero the entire klass field first as the gap needs to be zeroed too.
coleenp@548	5129	movptr(Address(dst, oopDesc::klass_offset_in_bytes()), NULL_WORD);
coleenp@548	5130	movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
coleenp@548	5131	} else {
coleenp@548	5132	movq(Address(dst, oopDesc::klass_offset_in_bytes()), src);
coleenp@548	5133	}
coleenp@548	5134	}
coleenp@548	5135
coleenp@548	5136	void MacroAssembler::load_heap_oop(Register dst, Address src) {
coleenp@548	5137	if (UseCompressedOops) {
coleenp@548	5138	movl(dst, src);
coleenp@548	5139	decode_heap_oop(dst);
coleenp@548	5140	} else {
coleenp@548	5141	movq(dst, src);
coleenp@548	5142	}
coleenp@548	5143	}
coleenp@548	5144
coleenp@548	5145	void MacroAssembler::store_heap_oop(Address dst, Register src) {
coleenp@548	5146	if (UseCompressedOops) {
coleenp@548	5147	assert(!dst.uses(src), "not enough registers");
coleenp@548	5148	encode_heap_oop(src);
coleenp@548	5149	movl(dst, src);
coleenp@548	5150	} else {
coleenp@548	5151	movq(dst, src);
coleenp@548	5152	}
coleenp@548	5153	}
coleenp@548	5154
coleenp@548	5155	// Algorithm must match oop.inline.hpp encode_heap_oop.
coleenp@548	5156	void MacroAssembler::encode_heap_oop(Register r) {
coleenp@548	5157	assert (UseCompressedOops, "should be compressed");
coleenp@548	5158	#ifdef ASSERT
coleenp@548	5159	Label ok;
coleenp@548	5160	pushq(rscratch1); // cmpptr trashes rscratch1
coleenp@548	5161	cmpptr(r12_heapbase, ExternalAddress((address)Universe::heap_base_addr()));
coleenp@548	5162	jcc(Assembler::equal, ok);
coleenp@548	5163	stop("MacroAssembler::encode_heap_oop: heap base corrupted?");
coleenp@548	5164	bind(ok);
coleenp@548	5165	popq(rscratch1);
coleenp@548	5166	#endif
coleenp@548	5167	verify_oop(r);
coleenp@548	5168	testq(r, r);
coleenp@548	5169	cmovq(Assembler::equal, r, r12_heapbase);
coleenp@548	5170	subq(r, r12_heapbase);
coleenp@548	5171	shrq(r, LogMinObjAlignmentInBytes);
coleenp@548	5172	}
coleenp@548	5173
coleenp@548	5174	void MacroAssembler::encode_heap_oop_not_null(Register r) {
coleenp@548	5175	assert (UseCompressedOops, "should be compressed");
coleenp@548	5176	#ifdef ASSERT
coleenp@548	5177	Label ok;
coleenp@548	5178	testq(r, r);
coleenp@548	5179	jcc(Assembler::notEqual, ok);
coleenp@548	5180	stop("null oop passed to encode_heap_oop_not_null");
coleenp@548	5181	bind(ok);
coleenp@548	5182	#endif
coleenp@548	5183	verify_oop(r);
coleenp@548	5184	subq(r, r12_heapbase);
coleenp@548	5185	shrq(r, LogMinObjAlignmentInBytes);
coleenp@548	5186	}
coleenp@548	5187
coleenp@548	5188	void MacroAssembler::decode_heap_oop(Register r) {
coleenp@548	5189	assert (UseCompressedOops, "should be compressed");
coleenp@548	5190	#ifdef ASSERT
coleenp@548	5191	Label ok;
coleenp@548	5192	pushq(rscratch1);
coleenp@548	5193	cmpptr(r12_heapbase,
coleenp@548	5194	ExternalAddress((address)Universe::heap_base_addr()));
coleenp@548	5195	jcc(Assembler::equal, ok);
coleenp@548	5196	stop("MacroAssembler::decode_heap_oop: heap base corrupted?");
coleenp@548	5197	bind(ok);
coleenp@548	5198	popq(rscratch1);
coleenp@548	5199	#endif
coleenp@548	5200
coleenp@548	5201	Label done;
coleenp@548	5202	shlq(r, LogMinObjAlignmentInBytes);
coleenp@548	5203	jccb(Assembler::equal, done);
coleenp@548	5204	addq(r, r12_heapbase);
coleenp@548	5205	#if 0
coleenp@548	5206	// alternate decoding probably a wash.
coleenp@548	5207	testq(r, r);
coleenp@548	5208	jccb(Assembler::equal, done);
coleenp@548	5209	leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
coleenp@548	5210	#endif
coleenp@548	5211	bind(done);
coleenp@548	5212	verify_oop(r);
coleenp@548	5213	}
coleenp@548	5214
coleenp@548	5215	void MacroAssembler::decode_heap_oop_not_null(Register r) {
coleenp@548	5216	assert (UseCompressedOops, "should only be used for compressed headers");
coleenp@548	5217	// Cannot assert, unverified entry point counts instructions (see .ad file)
coleenp@548	5218	// vtableStubs also counts instructions in pd_code_size_limit.
coleenp@548	5219	assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
coleenp@548	5220	leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
coleenp@548	5221	}
coleenp@548	5222
duke@435	5223	Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
duke@435	5224	switch (cond) {
duke@435	5225	// Note some conditions are synonyms for others
duke@435	5226	case Assembler::zero: return Assembler::notZero;
duke@435	5227	case Assembler::notZero: return Assembler::zero;
duke@435	5228	case Assembler::less: return Assembler::greaterEqual;
duke@435	5229	case Assembler::lessEqual: return Assembler::greater;
duke@435	5230	case Assembler::greater: return Assembler::lessEqual;
duke@435	5231	case Assembler::greaterEqual: return Assembler::less;
duke@435	5232	case Assembler::below: return Assembler::aboveEqual;
duke@435	5233	case Assembler::belowEqual: return Assembler::above;
duke@435	5234	case Assembler::above: return Assembler::belowEqual;
duke@435	5235	case Assembler::aboveEqual: return Assembler::below;
duke@435	5236	case Assembler::overflow: return Assembler::noOverflow;
duke@435	5237	case Assembler::noOverflow: return Assembler::overflow;
duke@435	5238	case Assembler::negative: return Assembler::positive;
duke@435	5239	case Assembler::positive: return Assembler::negative;
duke@435	5240	case Assembler::parity: return Assembler::noParity;
duke@435	5241	case Assembler::noParity: return Assembler::parity;
duke@435	5242	}
duke@435	5243	ShouldNotReachHere(); return Assembler::overflow;
duke@435	5244	}
duke@435	5245
duke@435	5246
duke@435	5247	void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
duke@435	5248	Condition negated_cond = negate_condition(cond);
duke@435	5249	Label L;
duke@435	5250	jcc(negated_cond, L);
duke@435	5251	atomic_incl(counter_addr);
duke@435	5252	bind(L);
duke@435	5253	}
duke@435	5254
duke@435	5255	void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
duke@435	5256	pushfq();
duke@435	5257	if (os::is_MP())
duke@435	5258	lock();
duke@435	5259	incrementl(counter_addr);
duke@435	5260	popfq();
duke@435	5261	}
duke@435	5262
duke@435	5263	SkipIfEqual::SkipIfEqual(
duke@435	5264	MacroAssembler* masm, const bool* flag_addr, bool value) {
duke@435	5265	_masm = masm;
duke@435	5266	_masm->cmp8(ExternalAddress((address)flag_addr), value);
duke@435	5267	_masm->jcc(Assembler::equal, _label);
duke@435	5268	}
duke@435	5269
duke@435	5270	SkipIfEqual::~SkipIfEqual() {
duke@435	5271	_masm->bind(_label);
duke@435	5272	}
duke@435	5273
duke@435	5274	void MacroAssembler::bang_stack_size(Register size, Register tmp) {
duke@435	5275	movq(tmp, rsp);
duke@435	5276	// Bang stack for total size given plus shadow page size.
duke@435	5277	// Bang one page at a time because large size can bang beyond yellow and
duke@435	5278	// red zones.
duke@435	5279	Label loop;
duke@435	5280	bind(loop);
duke@435	5281	movl(Address(tmp, (-os::vm_page_size())), size );
duke@435	5282	subq(tmp, os::vm_page_size());
duke@435	5283	subl(size, os::vm_page_size());
duke@435	5284	jcc(Assembler::greater, loop);
duke@435	5285
duke@435	5286	// Bang down shadow pages too.
duke@435	5287	// The -1 because we already subtracted 1 page.
duke@435	5288	for (int i = 0; i< StackShadowPages-1; i++) {
duke@435	5289	movq(Address(tmp, (-i*os::vm_page_size())), size );
duke@435	5290	}
duke@435	5291	}
coleenp@548	5292
coleenp@548	5293	void MacroAssembler::reinit_heapbase() {
coleenp@548	5294	if (UseCompressedOops) {
coleenp@548	5295	movptr(r12_heapbase, ExternalAddress((address)Universe::heap_base_addr()));
coleenp@548	5296	}
coleenp@548	5297	}