• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/assembler_x86_64.cpp@feeb96a45707 (annotated)

src/cpu/x86/vm/assembler_x86_64.cpp

Wed, 28 May 2008 21:06:24 -0700

author

coleenp

date

Wed, 28 May 2008 21:06:24 -0700

changeset 602

feeb96a45707

parent 599

c436414a719e

child 614

1f809e010142

child 617

44abbb0d4c18

permissions

-rw-r--r--

6696264: assert("narrow oop can never be zero") for GCBasher & ParNewGC
Summary: decouple set_klass() with zeroing the gap when compressed.
Reviewed-by: kvn, ysr, jrose

duke@435	1	/*
duke@435	2	* Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_assembler_x86_64.cpp.incl"
duke@435	27
duke@435	28	// Implementation of AddressLiteral
duke@435	29
duke@435	30	AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
duke@435	31	_is_lval = false;
duke@435	32	_target = target;
duke@435	33	switch (rtype) {
duke@435	34	case relocInfo::oop_type:
duke@435	35	// Oops are a special case. Normally they would be their own section
duke@435	36	// but in cases like icBuffer they are literals in the code stream that
duke@435	37	// we don't have a section for. We use none so that we get a literal address
duke@435	38	// which is always patchable.
duke@435	39	break;
duke@435	40	case relocInfo::external_word_type:
duke@435	41	_rspec = external_word_Relocation::spec(target);
duke@435	42	break;
duke@435	43	case relocInfo::internal_word_type:
duke@435	44	_rspec = internal_word_Relocation::spec(target);
duke@435	45	break;
duke@435	46	case relocInfo::opt_virtual_call_type:
duke@435	47	_rspec = opt_virtual_call_Relocation::spec();
duke@435	48	break;
duke@435	49	case relocInfo::static_call_type:
duke@435	50	_rspec = static_call_Relocation::spec();
duke@435	51	break;
duke@435	52	case relocInfo::runtime_call_type:
duke@435	53	_rspec = runtime_call_Relocation::spec();
duke@435	54	break;
duke@435	55	case relocInfo::none:
duke@435	56	break;
duke@435	57	default:
duke@435	58	ShouldNotReachHere();
duke@435	59	break;
duke@435	60	}
duke@435	61	}
duke@435	62
duke@435	63	// Implementation of Address
duke@435	64
duke@435	65	Address Address::make_array(ArrayAddress adr) {
duke@435	66	#ifdef _LP64
duke@435	67	// Not implementable on 64bit machines
duke@435	68	// Should have been handled higher up the call chain.
duke@435	69	ShouldNotReachHere();
duke@435	70	return Address();
duke@435	71	#else
duke@435	72	AddressLiteral base = adr.base();
duke@435	73	Address index = adr.index();
duke@435	74	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	75	Address array(index._base, index._index, index._scale, (intptr_t) base.target());
duke@435	76	array._rspec = base._rspec;
duke@435	77	return array;
duke@435	78	#endif // _LP64
duke@435	79	}
duke@435	80
duke@435	81	// exceedingly dangerous constructor
duke@435	82	Address::Address(int disp, address loc, relocInfo::relocType rtype) {
duke@435	83	_base = noreg;
duke@435	84	_index = noreg;
duke@435	85	_scale = no_scale;
duke@435	86	_disp = disp;
duke@435	87	switch (rtype) {
duke@435	88	case relocInfo::external_word_type:
duke@435	89	_rspec = external_word_Relocation::spec(loc);
duke@435	90	break;
duke@435	91	case relocInfo::internal_word_type:
duke@435	92	_rspec = internal_word_Relocation::spec(loc);
duke@435	93	break;
duke@435	94	case relocInfo::runtime_call_type:
duke@435	95	// HMM
duke@435	96	_rspec = runtime_call_Relocation::spec();
duke@435	97	break;
duke@435	98	case relocInfo::none:
duke@435	99	break;
duke@435	100	default:
duke@435	101	ShouldNotReachHere();
duke@435	102	}
duke@435	103	}
duke@435	104
duke@435	105	// Convert the raw encoding form into the form expected by the constructor for
duke@435	106	// Address. An index of 4 (rsp) corresponds to having no index, so convert
duke@435	107	// that to noreg for the Address constructor.
duke@435	108	Address Address::make_raw(int base, int index, int scale, int disp) {
duke@435	109	bool valid_index = index != rsp->encoding();
duke@435	110	if (valid_index) {
duke@435	111	Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
duke@435	112	return madr;
duke@435	113	} else {
duke@435	114	Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
duke@435	115	return madr;
duke@435	116	}
duke@435	117	}
duke@435	118
duke@435	119
duke@435	120	// Implementation of Assembler
duke@435	121	int AbstractAssembler::code_fill_byte() {
duke@435	122	return (u_char)'\xF4'; // hlt
duke@435	123	}
duke@435	124
duke@435	125	// This should only be used by 64bit instructions that can use rip-relative
duke@435	126	// it cannot be used by instructions that want an immediate value.
duke@435	127
duke@435	128	bool Assembler::reachable(AddressLiteral adr) {
duke@435	129	int64_t disp;
coleenp@548	130
duke@435	131	// None will force a 64bit literal to the code stream. Likely a placeholder
duke@435	132	// for something that will be patched later and we need to certain it will
duke@435	133	// always be reachable.
duke@435	134	if (adr.reloc() == relocInfo::none) {
duke@435	135	return false;
duke@435	136	}
duke@435	137	if (adr.reloc() == relocInfo::internal_word_type) {
duke@435	138	// This should be rip relative and easily reachable.
duke@435	139	return true;
duke@435	140	}
duke@435	141	if (adr.reloc() != relocInfo::external_word_type &&
duke@435	142	adr.reloc() != relocInfo::runtime_call_type ) {
duke@435	143	return false;
duke@435	144	}
duke@435	145
duke@435	146	// Stress the correction code
duke@435	147	if (ForceUnreachable) {
duke@435	148	// Must be runtimecall reloc, see if it is in the codecache
duke@435	149	// Flipping stuff in the codecache to be unreachable causes issues
duke@435	150	// with things like inline caches where the additional instructions
duke@435	151	// are not handled.
duke@435	152	if (CodeCache::find_blob(adr._target) == NULL) {
duke@435	153	return false;
duke@435	154	}
duke@435	155	}
duke@435	156	// For external_word_type/runtime_call_type if it is reachable from where we
duke@435	157	// are now (possibly a temp buffer) and where we might end up
duke@435	158	// anywhere in the codeCache then we are always reachable.
duke@435	159	// This would have to change if we ever save/restore shared code
duke@435	160	// to be more pessimistic.
duke@435	161
duke@435	162	disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
duke@435	163	if (!is_simm32(disp)) return false;
duke@435	164	disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
duke@435	165	if (!is_simm32(disp)) return false;
duke@435	166
duke@435	167	disp = (int64_t)adr._target - ((int64_t)_code_pos + sizeof(int));
duke@435	168
duke@435	169	// Because rip relative is a disp + address_of_next_instruction and we
duke@435	170	// don't know the value of address_of_next_instruction we apply a fudge factor
duke@435	171	// to make sure we will be ok no matter the size of the instruction we get placed into.
duke@435	172	// We don't have to fudge the checks above here because they are already worst case.
duke@435	173
duke@435	174	// 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
duke@435	175	// + 4 because better safe than sorry.
duke@435	176	const int fudge = 12 + 4;
duke@435	177	if (disp < 0) {
duke@435	178	disp -= fudge;
duke@435	179	} else {
duke@435	180	disp += fudge;
duke@435	181	}
duke@435	182	return is_simm32(disp);
duke@435	183	}
duke@435	184
duke@435	185
duke@435	186	// make this go away eventually
duke@435	187	void Assembler::emit_data(jint data,
duke@435	188	relocInfo::relocType rtype,
duke@435	189	int format) {
duke@435	190	if (rtype == relocInfo::none) {
duke@435	191	emit_long(data);
duke@435	192	} else {
duke@435	193	emit_data(data, Relocation::spec_simple(rtype), format);
duke@435	194	}
duke@435	195	}
duke@435	196
duke@435	197	void Assembler::emit_data(jint data,
duke@435	198	RelocationHolder const& rspec,
duke@435	199	int format) {
duke@435	200	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	201	assert(imm64_operand != format, "must not be imm64");
duke@435	202	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	203	if (rspec.type() != relocInfo::none) {
duke@435	204	#ifdef ASSERT
duke@435	205	check_relocation(rspec, format);
duke@435	206	#endif
duke@435	207	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	208	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	209
duke@435	210	// hack. call32 is too wide for mask so use disp32
duke@435	211	if (format == call32_operand)
duke@435	212	code_section()->relocate(inst_mark(), rspec, disp32_operand);
duke@435	213	else
duke@435	214	code_section()->relocate(inst_mark(), rspec, format);
duke@435	215	}
duke@435	216	emit_long(data);
duke@435	217	}
duke@435	218
duke@435	219	void Assembler::emit_data64(jlong data,
duke@435	220	relocInfo::relocType rtype,
duke@435	221	int format) {
duke@435	222	if (rtype == relocInfo::none) {
duke@435	223	emit_long64(data);
duke@435	224	} else {
duke@435	225	emit_data64(data, Relocation::spec_simple(rtype), format);
duke@435	226	}
duke@435	227	}
duke@435	228
duke@435	229	void Assembler::emit_data64(jlong data,
duke@435	230	RelocationHolder const& rspec,
duke@435	231	int format) {
duke@435	232	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	233	assert(imm64_operand == format, "must be imm64");
duke@435	234	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	235	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	236	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	237	code_section()->relocate(inst_mark(), rspec, format);
duke@435	238	#ifdef ASSERT
duke@435	239	check_relocation(rspec, format);
duke@435	240	#endif
duke@435	241	emit_long64(data);
duke@435	242	}
duke@435	243
duke@435	244	void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
duke@435	245	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	246	assert(isByte(imm8), "not a byte");
duke@435	247	assert((op1 & 0x01) == 0, "should be 8bit operation");
duke@435	248	int dstenc = dst->encoding();
duke@435	249	if (dstenc >= 8) {
duke@435	250	dstenc -= 8;
duke@435	251	}
duke@435	252	emit_byte(op1);
duke@435	253	emit_byte(op2 | dstenc);
duke@435	254	emit_byte(imm8);
duke@435	255	}
duke@435	256
duke@435	257	void Assembler::emit_arith(int op1, int op2, Register dst, int imm32) {
duke@435	258	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	259	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	260	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	261	int dstenc = dst->encoding();
duke@435	262	if (dstenc >= 8) {
duke@435	263	dstenc -= 8;
duke@435	264	}
duke@435	265	if (is8bit(imm32)) {
duke@435	266	emit_byte(op1 | 0x02); // set sign bit
duke@435	267	emit_byte(op2 | dstenc);
duke@435	268	emit_byte(imm32 & 0xFF);
duke@435	269	} else {
duke@435	270	emit_byte(op1);
duke@435	271	emit_byte(op2 | dstenc);
duke@435	272	emit_long(imm32);
duke@435	273	}
duke@435	274	}
duke@435	275
duke@435	276	// immediate-to-memory forms
duke@435	277	void Assembler::emit_arith_operand(int op1,
duke@435	278	Register rm, Address adr,
duke@435	279	int imm32) {
duke@435	280	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	281	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	282	if (is8bit(imm32)) {
duke@435	283	emit_byte(op1 | 0x02); // set sign bit
duke@435	284	emit_operand(rm, adr, 1);
duke@435	285	emit_byte(imm32 & 0xFF);
duke@435	286	} else {
duke@435	287	emit_byte(op1);
duke@435	288	emit_operand(rm, adr, 4);
duke@435	289	emit_long(imm32);
duke@435	290	}
duke@435	291	}
duke@435	292
duke@435	293
duke@435	294	void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
duke@435	295	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	296	int dstenc = dst->encoding();
duke@435	297	int srcenc = src->encoding();
duke@435	298	if (dstenc >= 8) {
duke@435	299	dstenc -= 8;
duke@435	300	}
duke@435	301	if (srcenc >= 8) {
duke@435	302	srcenc -= 8;
duke@435	303	}
duke@435	304	emit_byte(op1);
duke@435	305	emit_byte(op2 | dstenc << 3 | srcenc);
duke@435	306	}
duke@435	307
duke@435	308	void Assembler::emit_operand(Register reg, Register base, Register index,
duke@435	309	Address::ScaleFactor scale, int disp,
duke@435	310	RelocationHolder const& rspec,
duke@435	311	int rip_relative_correction) {
duke@435	312	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	313	int regenc = reg->encoding();
duke@435	314	if (regenc >= 8) {
duke@435	315	regenc -= 8;
duke@435	316	}
duke@435	317	if (base->is_valid()) {
duke@435	318	if (index->is_valid()) {
duke@435	319	assert(scale != Address::no_scale, "inconsistent address");
duke@435	320	int indexenc = index->encoding();
duke@435	321	if (indexenc >= 8) {
duke@435	322	indexenc -= 8;
duke@435	323	}
duke@435	324	int baseenc = base->encoding();
duke@435	325	if (baseenc >= 8) {
duke@435	326	baseenc -= 8;
duke@435	327	}
duke@435	328	// [base + index*scale + disp]
duke@435	329	if (disp == 0 && rtype == relocInfo::none &&
duke@435	330	base != rbp && base != r13) {
duke@435	331	// [base + index*scale]
duke@435	332	// [00 reg 100][ss index base]
duke@435	333	assert(index != rsp, "illegal addressing mode");
duke@435	334	emit_byte(0x04 | regenc << 3);
duke@435	335	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	336	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	337	// [base + index*scale + imm8]
duke@435	338	// [01 reg 100][ss index base] imm8
duke@435	339	assert(index != rsp, "illegal addressing mode");
duke@435	340	emit_byte(0x44 | regenc << 3);
duke@435	341	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	342	emit_byte(disp & 0xFF);
duke@435	343	} else {
duke@435	344	// [base + index*scale + disp32]
duke@435	345	// [10 reg 100][ss index base] disp32
duke@435	346	assert(index != rsp, "illegal addressing mode");
duke@435	347	emit_byte(0x84 | regenc << 3);
duke@435	348	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	349	emit_data(disp, rspec, disp32_operand);
duke@435	350	}
duke@435	351	} else if (base == rsp || base == r12) {
duke@435	352	// [rsp + disp]
duke@435	353	if (disp == 0 && rtype == relocInfo::none) {
duke@435	354	// [rsp]
duke@435	355	// [00 reg 100][00 100 100]
duke@435	356	emit_byte(0x04 | regenc << 3);
duke@435	357	emit_byte(0x24);
duke@435	358	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	359	// [rsp + imm8]
duke@435	360	// [01 reg 100][00 100 100] disp8
duke@435	361	emit_byte(0x44 | regenc << 3);
duke@435	362	emit_byte(0x24);
duke@435	363	emit_byte(disp & 0xFF);
duke@435	364	} else {
duke@435	365	// [rsp + imm32]
duke@435	366	// [10 reg 100][00 100 100] disp32
duke@435	367	emit_byte(0x84 | regenc << 3);
duke@435	368	emit_byte(0x24);
duke@435	369	emit_data(disp, rspec, disp32_operand);
duke@435	370	}
duke@435	371	} else {
duke@435	372	// [base + disp]
duke@435	373	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	374	int baseenc = base->encoding();
duke@435	375	if (baseenc >= 8) {
duke@435	376	baseenc -= 8;
duke@435	377	}
duke@435	378	if (disp == 0 && rtype == relocInfo::none &&
duke@435	379	base != rbp && base != r13) {
duke@435	380	// [base]
duke@435	381	// [00 reg base]
duke@435	382	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	383	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	384	// [base + disp8]
duke@435	385	// [01 reg base] disp8
duke@435	386	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	387	emit_byte(disp & 0xFF);
duke@435	388	} else {
duke@435	389	// [base + disp32]
duke@435	390	// [10 reg base] disp32
duke@435	391	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	392	emit_data(disp, rspec, disp32_operand);
duke@435	393	}
duke@435	394	}
duke@435	395	} else {
duke@435	396	if (index->is_valid()) {
duke@435	397	assert(scale != Address::no_scale, "inconsistent address");
duke@435	398	int indexenc = index->encoding();
duke@435	399	if (indexenc >= 8) {
duke@435	400	indexenc -= 8;
duke@435	401	}
duke@435	402	// [index*scale + disp]
duke@435	403	// [00 reg 100][ss index 101] disp32
duke@435	404	assert(index != rsp, "illegal addressing mode");
duke@435	405	emit_byte(0x04 | regenc << 3);
duke@435	406	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	407	emit_data(disp, rspec, disp32_operand);
duke@435	408	#ifdef _LP64
duke@435	409	} else if (rtype != relocInfo::none ) {
duke@435	410	// [disp] RIP-RELATIVE
duke@435	411	// [00 000 101] disp32
duke@435	412
duke@435	413	emit_byte(0x05 | regenc << 3);
duke@435	414	// Note that the RIP-rel. correction applies to the generated
duke@435	415	// disp field, but _not_ to the target address in the rspec.
duke@435	416
duke@435	417	// disp was created by converting the target address minus the pc
duke@435	418	// at the start of the instruction. That needs more correction here.
duke@435	419	// intptr_t disp = target - next_ip;
duke@435	420	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	421	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	422	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	423	assert(is_simm32(adjusted),
duke@435	424	"must be 32bit offset (RIP relative address)");
duke@435	425	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	426
duke@435	427	#endif // _LP64
duke@435	428	} else {
duke@435	429	// [disp] ABSOLUTE
duke@435	430	// [00 reg 100][00 100 101] disp32
duke@435	431	emit_byte(0x04 | regenc << 3);
duke@435	432	emit_byte(0x25);
duke@435	433	emit_data(disp, rspec, disp32_operand);
duke@435	434	}
duke@435	435	}
duke@435	436	}
duke@435	437
duke@435	438	void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
duke@435	439	Address::ScaleFactor scale, int disp,
duke@435	440	RelocationHolder const& rspec,
duke@435	441	int rip_relative_correction) {
duke@435	442	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	443	int regenc = reg->encoding();
duke@435	444	if (regenc >= 8) {
duke@435	445	regenc -= 8;
duke@435	446	}
duke@435	447	if (base->is_valid()) {
duke@435	448	if (index->is_valid()) {
duke@435	449	assert(scale != Address::no_scale, "inconsistent address");
duke@435	450	int indexenc = index->encoding();
duke@435	451	if (indexenc >= 8) {
duke@435	452	indexenc -= 8;
duke@435	453	}
duke@435	454	int baseenc = base->encoding();
duke@435	455	if (baseenc >= 8) {
duke@435	456	baseenc -= 8;
duke@435	457	}
duke@435	458	// [base + index*scale + disp]
duke@435	459	if (disp == 0 && rtype == relocInfo::none &&
duke@435	460	base != rbp && base != r13) {
duke@435	461	// [base + index*scale]
duke@435	462	// [00 reg 100][ss index base]
duke@435	463	assert(index != rsp, "illegal addressing mode");
duke@435	464	emit_byte(0x04 | regenc << 3);
duke@435	465	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	466	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	467	// [base + index*scale + disp8]
duke@435	468	// [01 reg 100][ss index base] disp8
duke@435	469	assert(index != rsp, "illegal addressing mode");
duke@435	470	emit_byte(0x44 | regenc << 3);
duke@435	471	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	472	emit_byte(disp & 0xFF);
duke@435	473	} else {
duke@435	474	// [base + index*scale + disp32]
duke@435	475	// [10 reg 100][ss index base] disp32
duke@435	476	assert(index != rsp, "illegal addressing mode");
duke@435	477	emit_byte(0x84 | regenc << 3);
duke@435	478	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	479	emit_data(disp, rspec, disp32_operand);
duke@435	480	}
duke@435	481	} else if (base == rsp || base == r12) {
duke@435	482	// [rsp + disp]
duke@435	483	if (disp == 0 && rtype == relocInfo::none) {
duke@435	484	// [rsp]
duke@435	485	// [00 reg 100][00 100 100]
duke@435	486	emit_byte(0x04 | regenc << 3);
duke@435	487	emit_byte(0x24);
duke@435	488	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	489	// [rsp + imm8]
duke@435	490	// [01 reg 100][00 100 100] disp8
duke@435	491	emit_byte(0x44 | regenc << 3);
duke@435	492	emit_byte(0x24);
duke@435	493	emit_byte(disp & 0xFF);
duke@435	494	} else {
duke@435	495	// [rsp + imm32]
duke@435	496	// [10 reg 100][00 100 100] disp32
duke@435	497	emit_byte(0x84 | regenc << 3);
duke@435	498	emit_byte(0x24);
duke@435	499	emit_data(disp, rspec, disp32_operand);
duke@435	500	}
duke@435	501	} else {
duke@435	502	// [base + disp]
duke@435	503	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	504	int baseenc = base->encoding();
duke@435	505	if (baseenc >= 8) {
duke@435	506	baseenc -= 8;
duke@435	507	}
duke@435	508	if (disp == 0 && rtype == relocInfo::none &&
duke@435	509	base != rbp && base != r13) {
duke@435	510	// [base]
duke@435	511	// [00 reg base]
duke@435	512	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	513	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	514	// [base + imm8]
duke@435	515	// [01 reg base] disp8
duke@435	516	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	517	emit_byte(disp & 0xFF);
duke@435	518	} else {
duke@435	519	// [base + imm32]
duke@435	520	// [10 reg base] disp32
duke@435	521	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	522	emit_data(disp, rspec, disp32_operand);
duke@435	523	}
duke@435	524	}
duke@435	525	} else {
duke@435	526	if (index->is_valid()) {
duke@435	527	assert(scale != Address::no_scale, "inconsistent address");
duke@435	528	int indexenc = index->encoding();
duke@435	529	if (indexenc >= 8) {
duke@435	530	indexenc -= 8;
duke@435	531	}
duke@435	532	// [index*scale + disp]
duke@435	533	// [00 reg 100][ss index 101] disp32
duke@435	534	assert(index != rsp, "illegal addressing mode");
duke@435	535	emit_byte(0x04 | regenc << 3);
duke@435	536	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	537	emit_data(disp, rspec, disp32_operand);
duke@435	538	#ifdef _LP64
duke@435	539	} else if ( rtype != relocInfo::none ) {
duke@435	540	// [disp] RIP-RELATIVE
duke@435	541	// [00 reg 101] disp32
duke@435	542	emit_byte(0x05 | regenc << 3);
duke@435	543	// Note that the RIP-rel. correction applies to the generated
duke@435	544	// disp field, but _not_ to the target address in the rspec.
duke@435	545
duke@435	546	// disp was created by converting the target address minus the pc
duke@435	547	// at the start of the instruction. That needs more correction here.
duke@435	548	// intptr_t disp = target - next_ip;
duke@435	549
duke@435	550	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	551	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	552
duke@435	553	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	554	assert(is_simm32(adjusted),
duke@435	555	"must be 32bit offset (RIP relative address)");
duke@435	556	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	557	#endif // _LP64
duke@435	558	} else {
duke@435	559	// [disp] ABSOLUTE
duke@435	560	// [00 reg 100][00 100 101] disp32
duke@435	561	emit_byte(0x04 | regenc << 3);
duke@435	562	emit_byte(0x25);
duke@435	563	emit_data(disp, rspec, disp32_operand);
duke@435	564	}
duke@435	565	}
duke@435	566	}
duke@435	567
duke@435	568	// Secret local extension to Assembler::WhichOperand:
duke@435	569	#define end_pc_operand (_WhichOperand_limit)
duke@435	570
duke@435	571	address Assembler::locate_operand(address inst, WhichOperand which) {
duke@435	572	// Decode the given instruction, and return the address of
duke@435	573	// an embedded 32-bit operand word.
duke@435	574
duke@435	575	// If "which" is disp32_operand, selects the displacement portion
duke@435	576	// of an effective address specifier.
duke@435	577	// If "which" is imm64_operand, selects the trailing immediate constant.
duke@435	578	// If "which" is call32_operand, selects the displacement of a call or jump.
duke@435	579	// Caller is responsible for ensuring that there is such an operand,
duke@435	580	// and that it is 32/64 bits wide.
duke@435	581
duke@435	582	// If "which" is end_pc_operand, find the end of the instruction.
duke@435	583
duke@435	584	address ip = inst;
duke@435	585	bool is_64bit = false;
duke@435	586
duke@435	587	debug_only(bool has_disp32 = false);
duke@435	588	int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
duke@435	589
duke@435	590	again_after_prefix:
duke@435	591	switch (0xFF & *ip++) {
duke@435	592
duke@435	593	// These convenience macros generate groups of "case" labels for the switch.
duke@435	594	#define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
duke@435	595	#define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
duke@435	596	case (x)+4: case (x)+5: case (x)+6: case (x)+7
duke@435	597	#define REP16(x) REP8((x)+0): \
duke@435	598	case REP8((x)+8)
duke@435	599
duke@435	600	case CS_segment:
duke@435	601	case SS_segment:
duke@435	602	case DS_segment:
duke@435	603	case ES_segment:
duke@435	604	case FS_segment:
duke@435	605	case GS_segment:
duke@435	606	assert(0, "shouldn't have that prefix");
duke@435	607	assert(ip == inst + 1 || ip == inst + 2, "only two prefixes allowed");
duke@435	608	goto again_after_prefix;
duke@435	609
duke@435	610	case 0x67:
duke@435	611	case REX:
duke@435	612	case REX_B:
duke@435	613	case REX_X:
duke@435	614	case REX_XB:
duke@435	615	case REX_R:
duke@435	616	case REX_RB:
duke@435	617	case REX_RX:
duke@435	618	case REX_RXB:
duke@435	619	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	620	goto again_after_prefix;
duke@435	621
duke@435	622	case REX_W:
duke@435	623	case REX_WB:
duke@435	624	case REX_WX:
duke@435	625	case REX_WXB:
duke@435	626	case REX_WR:
duke@435	627	case REX_WRB:
duke@435	628	case REX_WRX:
duke@435	629	case REX_WRXB:
duke@435	630	is_64bit = true;
duke@435	631	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	632	goto again_after_prefix;
duke@435	633
duke@435	634	case 0xFF: // pushq a; decl a; incl a; call a; jmp a
duke@435	635	case 0x88: // movb a, r
duke@435	636	case 0x89: // movl a, r
duke@435	637	case 0x8A: // movb r, a
duke@435	638	case 0x8B: // movl r, a
duke@435	639	case 0x8F: // popl a
coleenp@548	640	debug_only(has_disp32 = true;)
duke@435	641	break;
duke@435	642
duke@435	643	case 0x68: // pushq #32
duke@435	644	if (which == end_pc_operand) {
duke@435	645	return ip + 4;
duke@435	646	}
duke@435	647	assert(0, "pushq has no disp32 or imm64");
duke@435	648	ShouldNotReachHere();
duke@435	649
duke@435	650	case 0x66: // movw ... (size prefix)
duke@435	651	again_after_size_prefix2:
duke@435	652	switch (0xFF & *ip++) {
duke@435	653	case REX:
duke@435	654	case REX_B:
duke@435	655	case REX_X:
duke@435	656	case REX_XB:
duke@435	657	case REX_R:
duke@435	658	case REX_RB:
duke@435	659	case REX_RX:
duke@435	660	case REX_RXB:
duke@435	661	case REX_W:
duke@435	662	case REX_WB:
duke@435	663	case REX_WX:
duke@435	664	case REX_WXB:
duke@435	665	case REX_WR:
duke@435	666	case REX_WRB:
duke@435	667	case REX_WRX:
duke@435	668	case REX_WRXB:
duke@435	669	goto again_after_size_prefix2;
duke@435	670	case 0x8B: // movw r, a
duke@435	671	case 0x89: // movw a, r
duke@435	672	break;
duke@435	673	case 0xC7: // movw a, #16
duke@435	674	tail_size = 2; // the imm16
duke@435	675	break;
duke@435	676	case 0x0F: // several SSE/SSE2 variants
duke@435	677	ip--; // reparse the 0x0F
duke@435	678	goto again_after_prefix;
duke@435	679	default:
duke@435	680	ShouldNotReachHere();
duke@435	681	}
duke@435	682	break;
duke@435	683
duke@435	684	case REP8(0xB8): // movl/q r, #32/#64(oop?)
duke@435	685	if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
kvn@599	686	assert((which == call32_operand || which == imm64_operand) && is_64bit ||
kvn@599	687	which == narrow_oop_operand && !is_64bit, "");
duke@435	688	return ip;
duke@435	689
duke@435	690	case 0x69: // imul r, a, #32
duke@435	691	case 0xC7: // movl a, #32(oop?)
duke@435	692	tail_size = 4;
duke@435	693	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	694	break;
duke@435	695
duke@435	696	case 0x0F: // movx..., etc.
duke@435	697	switch (0xFF & *ip++) {
duke@435	698	case 0x12: // movlps
duke@435	699	case 0x28: // movaps
duke@435	700	case 0x2E: // ucomiss
duke@435	701	case 0x2F: // comiss
duke@435	702	case 0x54: // andps
duke@435	703	case 0x57: // xorps
duke@435	704	case 0x6E: // movd
duke@435	705	case 0x7E: // movd
duke@435	706	case 0xAE: // ldmxcsr a
duke@435	707	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	708	break;
duke@435	709	case 0xAD: // shrd r, a, %cl
duke@435	710	case 0xAF: // imul r, a
duke@435	711	case 0xBE: // movsbl r, a
duke@435	712	case 0xBF: // movswl r, a
duke@435	713	case 0xB6: // movzbl r, a
duke@435	714	case 0xB7: // movzwl r, a
duke@435	715	case REP16(0x40): // cmovl cc, r, a
duke@435	716	case 0xB0: // cmpxchgb
duke@435	717	case 0xB1: // cmpxchg
duke@435	718	case 0xC1: // xaddl
duke@435	719	case 0xC7: // cmpxchg8
duke@435	720	case REP16(0x90): // setcc a
duke@435	721	debug_only(has_disp32 = true);
duke@435	722	// fall out of the switch to decode the address
duke@435	723	break;
duke@435	724	case 0xAC: // shrd r, a, #8
duke@435	725	debug_only(has_disp32 = true);
duke@435	726	tail_size = 1; // the imm8
duke@435	727	break;
duke@435	728	case REP16(0x80): // jcc rdisp32
duke@435	729	if (which == end_pc_operand) return ip + 4;
duke@435	730	assert(which == call32_operand, "jcc has no disp32 or imm64");
duke@435	731	return ip;
duke@435	732	default:
duke@435	733	ShouldNotReachHere();
duke@435	734	}
duke@435	735	break;
duke@435	736
duke@435	737	case 0x81: // addl a, #32; addl r, #32
duke@435	738	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	739	tail_size = 4;
duke@435	740	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	741	break;
duke@435	742
duke@435	743	case 0x83: // addl a, #8; addl r, #8
duke@435	744	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	745	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	746	tail_size = 1;
duke@435	747	break;
duke@435	748
duke@435	749	case 0x9B:
duke@435	750	switch (0xFF & *ip++) {
duke@435	751	case 0xD9: // fnstcw a
duke@435	752	debug_only(has_disp32 = true);
duke@435	753	break;
duke@435	754	default:
duke@435	755	ShouldNotReachHere();
duke@435	756	}
duke@435	757	break;
duke@435	758
duke@435	759	case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
duke@435	760	case REP4(0x10): // adc...
duke@435	761	case REP4(0x20): // and...
duke@435	762	case REP4(0x30): // xor...
duke@435	763	case REP4(0x08): // or...
duke@435	764	case REP4(0x18): // sbb...
duke@435	765	case REP4(0x28): // sub...
duke@435	766	case 0xF7: // mull a
duke@435	767	case 0x87: // xchg r, a
duke@435	768	debug_only(has_disp32 = true);
duke@435	769	break;
duke@435	770	case REP4(0x38): // cmp...
duke@435	771	case 0x8D: // lea r, a
duke@435	772	case 0x85: // test r, a
duke@435	773	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	774	break;
duke@435	775
duke@435	776	case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
duke@435	777	case 0xC6: // movb a, #8
duke@435	778	case 0x80: // cmpb a, #8
duke@435	779	case 0x6B: // imul r, a, #8
duke@435	780	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	781	tail_size = 1; // the imm8
duke@435	782	break;
duke@435	783
duke@435	784	case 0xE8: // call rdisp32
duke@435	785	case 0xE9: // jmp rdisp32
duke@435	786	if (which == end_pc_operand) return ip + 4;
duke@435	787	assert(which == call32_operand, "call has no disp32 or imm32");
duke@435	788	return ip;
duke@435	789
duke@435	790	case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
duke@435	791	case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
duke@435	792	case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
duke@435	793	case 0xDD: // fld_d a; fst_d a; fstp_d a
duke@435	794	case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
duke@435	795	case 0xDF: // fild_d a; fistp_d a
duke@435	796	case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
duke@435	797	case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
duke@435	798	case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
duke@435	799	debug_only(has_disp32 = true);
duke@435	800	break;
duke@435	801
duke@435	802	case 0xF3: // For SSE
duke@435	803	case 0xF2: // For SSE2
duke@435	804	switch (0xFF & *ip++) {
duke@435	805	case REX:
duke@435	806	case REX_B:
duke@435	807	case REX_X:
duke@435	808	case REX_XB:
duke@435	809	case REX_R:
duke@435	810	case REX_RB:
duke@435	811	case REX_RX:
duke@435	812	case REX_RXB:
duke@435	813	case REX_W:
duke@435	814	case REX_WB:
duke@435	815	case REX_WX:
duke@435	816	case REX_WXB:
duke@435	817	case REX_WR:
duke@435	818	case REX_WRB:
duke@435	819	case REX_WRX:
duke@435	820	case REX_WRXB:
duke@435	821	ip++;
duke@435	822	default:
duke@435	823	ip++;
duke@435	824	}
duke@435	825	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	826	break;
duke@435	827
duke@435	828	default:
duke@435	829	ShouldNotReachHere();
duke@435	830
duke@435	831	#undef REP8
duke@435	832	#undef REP16
duke@435	833	}
duke@435	834
duke@435	835	assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
duke@435	836	assert(which != imm64_operand, "instruction is not a movq reg, imm64");
duke@435	837	assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
duke@435	838
duke@435	839	// parse the output of emit_operand
duke@435	840	int op2 = 0xFF & *ip++;
duke@435	841	int base = op2 & 0x07;
duke@435	842	int op3 = -1;
duke@435	843	const int b100 = 4;
duke@435	844	const int b101 = 5;
duke@435	845	if (base == b100 && (op2 >> 6) != 3) {
duke@435	846	op3 = 0xFF & *ip++;
duke@435	847	base = op3 & 0x07; // refetch the base
duke@435	848	}
duke@435	849	// now ip points at the disp (if any)
duke@435	850
duke@435	851	switch (op2 >> 6) {
duke@435	852	case 0:
duke@435	853	// [00 reg 100][ss index base]
duke@435	854	// [00 reg 100][00 100 esp]
duke@435	855	// [00 reg base]
duke@435	856	// [00 reg 100][ss index 101][disp32]
duke@435	857	// [00 reg 101] [disp32]
duke@435	858
duke@435	859	if (base == b101) {
duke@435	860	if (which == disp32_operand)
duke@435	861	return ip; // caller wants the disp32
duke@435	862	ip += 4; // skip the disp32
duke@435	863	}
duke@435	864	break;
duke@435	865
duke@435	866	case 1:
duke@435	867	// [01 reg 100][ss index base][disp8]
duke@435	868	// [01 reg 100][00 100 esp][disp8]
duke@435	869	// [01 reg base] [disp8]
duke@435	870	ip += 1; // skip the disp8
duke@435	871	break;
duke@435	872
duke@435	873	case 2:
duke@435	874	// [10 reg 100][ss index base][disp32]
duke@435	875	// [10 reg 100][00 100 esp][disp32]
duke@435	876	// [10 reg base] [disp32]
duke@435	877	if (which == disp32_operand)
duke@435	878	return ip; // caller wants the disp32
duke@435	879	ip += 4; // skip the disp32
duke@435	880	break;
duke@435	881
duke@435	882	case 3:
duke@435	883	// [11 reg base] (not a memory addressing mode)
duke@435	884	break;
duke@435	885	}
duke@435	886
duke@435	887	if (which == end_pc_operand) {
duke@435	888	return ip + tail_size;
duke@435	889	}
duke@435	890
duke@435	891	assert(0, "fix locate_operand");
duke@435	892	return ip;
duke@435	893	}
duke@435	894
duke@435	895	address Assembler::locate_next_instruction(address inst) {
duke@435	896	// Secretly share code with locate_operand:
duke@435	897	return locate_operand(inst, end_pc_operand);
duke@435	898	}
duke@435	899
duke@435	900	#ifdef ASSERT
duke@435	901	void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
duke@435	902	address inst = inst_mark();
duke@435	903	assert(inst != NULL && inst < pc(),
duke@435	904	"must point to beginning of instruction");
duke@435	905	address opnd;
duke@435	906
duke@435	907	Relocation* r = rspec.reloc();
duke@435	908	if (r->type() == relocInfo::none) {
duke@435	909	return;
duke@435	910	} else if (r->is_call() || format == call32_operand) {
duke@435	911	opnd = locate_operand(inst, call32_operand);
duke@435	912	} else if (r->is_data()) {
kvn@599	913	assert(format == imm64_operand || format == disp32_operand ||
kvn@599	914	format == narrow_oop_operand, "format ok");
duke@435	915	opnd = locate_operand(inst, (WhichOperand) format);
duke@435	916	} else {
duke@435	917	assert(format == 0, "cannot specify a format");
duke@435	918	return;
duke@435	919	}
duke@435	920	assert(opnd == pc(), "must put operand where relocs can find it");
duke@435	921	}
duke@435	922	#endif
duke@435	923
duke@435	924	int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
duke@435	925	if (reg_enc >= 8) {
duke@435	926	prefix(REX_B);
duke@435	927	reg_enc -= 8;
duke@435	928	} else if (byteinst && reg_enc >= 4) {
duke@435	929	prefix(REX);
duke@435	930	}
duke@435	931	return reg_enc;
duke@435	932	}
duke@435	933
duke@435	934	int Assembler::prefixq_and_encode(int reg_enc) {
duke@435	935	if (reg_enc < 8) {
duke@435	936	prefix(REX_W);
duke@435	937	} else {
duke@435	938	prefix(REX_WB);
duke@435	939	reg_enc -= 8;
duke@435	940	}
duke@435	941	return reg_enc;
duke@435	942	}
duke@435	943
duke@435	944	int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
duke@435	945	if (dst_enc < 8) {
duke@435	946	if (src_enc >= 8) {
duke@435	947	prefix(REX_B);
duke@435	948	src_enc -= 8;
duke@435	949	} else if (byteinst && src_enc >= 4) {
duke@435	950	prefix(REX);
duke@435	951	}
duke@435	952	} else {
duke@435	953	if (src_enc < 8) {
duke@435	954	prefix(REX_R);
duke@435	955	} else {
duke@435	956	prefix(REX_RB);
duke@435	957	src_enc -= 8;
duke@435	958	}
duke@435	959	dst_enc -= 8;
duke@435	960	}
duke@435	961	return dst_enc << 3 | src_enc;
duke@435	962	}
duke@435	963
duke@435	964	int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
duke@435	965	if (dst_enc < 8) {
duke@435	966	if (src_enc < 8) {
duke@435	967	prefix(REX_W);
duke@435	968	} else {
duke@435	969	prefix(REX_WB);
duke@435	970	src_enc -= 8;
duke@435	971	}
duke@435	972	} else {
duke@435	973	if (src_enc < 8) {
duke@435	974	prefix(REX_WR);
duke@435	975	} else {
duke@435	976	prefix(REX_WRB);
duke@435	977	src_enc -= 8;
duke@435	978	}
duke@435	979	dst_enc -= 8;
duke@435	980	}
duke@435	981	return dst_enc << 3 | src_enc;
duke@435	982	}
duke@435	983
duke@435	984	void Assembler::prefix(Register reg) {
duke@435	985	if (reg->encoding() >= 8) {
duke@435	986	prefix(REX_B);
duke@435	987	}
duke@435	988	}
duke@435	989
duke@435	990	void Assembler::prefix(Address adr) {
duke@435	991	if (adr.base_needs_rex()) {
duke@435	992	if (adr.index_needs_rex()) {
duke@435	993	prefix(REX_XB);
duke@435	994	} else {
duke@435	995	prefix(REX_B);
duke@435	996	}
duke@435	997	} else {
duke@435	998	if (adr.index_needs_rex()) {
duke@435	999	prefix(REX_X);
duke@435	1000	}
duke@435	1001	}
duke@435	1002	}
duke@435	1003
duke@435	1004	void Assembler::prefixq(Address adr) {
duke@435	1005	if (adr.base_needs_rex()) {
duke@435	1006	if (adr.index_needs_rex()) {
duke@435	1007	prefix(REX_WXB);
duke@435	1008	} else {
duke@435	1009	prefix(REX_WB);
duke@435	1010	}
duke@435	1011	} else {
duke@435	1012	if (adr.index_needs_rex()) {
duke@435	1013	prefix(REX_WX);
duke@435	1014	} else {
duke@435	1015	prefix(REX_W);
duke@435	1016	}
duke@435	1017	}
duke@435	1018	}
duke@435	1019
duke@435	1020
duke@435	1021	void Assembler::prefix(Address adr, Register reg, bool byteinst) {
duke@435	1022	if (reg->encoding() < 8) {
duke@435	1023	if (adr.base_needs_rex()) {
duke@435	1024	if (adr.index_needs_rex()) {
duke@435	1025	prefix(REX_XB);
duke@435	1026	} else {
duke@435	1027	prefix(REX_B);
duke@435	1028	}
duke@435	1029	} else {
duke@435	1030	if (adr.index_needs_rex()) {
duke@435	1031	prefix(REX_X);
duke@435	1032	} else if (reg->encoding() >= 4 ) {
duke@435	1033	prefix(REX);
duke@435	1034	}
duke@435	1035	}
duke@435	1036	} else {
duke@435	1037	if (adr.base_needs_rex()) {
duke@435	1038	if (adr.index_needs_rex()) {
duke@435	1039	prefix(REX_RXB);
duke@435	1040	} else {
duke@435	1041	prefix(REX_RB);
duke@435	1042	}
duke@435	1043	} else {
duke@435	1044	if (adr.index_needs_rex()) {
duke@435	1045	prefix(REX_RX);
duke@435	1046	} else {
duke@435	1047	prefix(REX_R);
duke@435	1048	}
duke@435	1049	}
duke@435	1050	}
duke@435	1051	}
duke@435	1052
duke@435	1053	void Assembler::prefixq(Address adr, Register src) {
duke@435	1054	if (src->encoding() < 8) {
duke@435	1055	if (adr.base_needs_rex()) {
duke@435	1056	if (adr.index_needs_rex()) {
duke@435	1057	prefix(REX_WXB);
duke@435	1058	} else {
duke@435	1059	prefix(REX_WB);
duke@435	1060	}
duke@435	1061	} else {
duke@435	1062	if (adr.index_needs_rex()) {
duke@435	1063	prefix(REX_WX);
duke@435	1064	} else {
duke@435	1065	prefix(REX_W);
duke@435	1066	}
duke@435	1067	}
duke@435	1068	} else {
duke@435	1069	if (adr.base_needs_rex()) {
duke@435	1070	if (adr.index_needs_rex()) {
duke@435	1071	prefix(REX_WRXB);
duke@435	1072	} else {
duke@435	1073	prefix(REX_WRB);
duke@435	1074	}
duke@435	1075	} else {
duke@435	1076	if (adr.index_needs_rex()) {
duke@435	1077	prefix(REX_WRX);
duke@435	1078	} else {
duke@435	1079	prefix(REX_WR);
duke@435	1080	}
duke@435	1081	}
duke@435	1082	}
duke@435	1083	}
duke@435	1084
duke@435	1085	void Assembler::prefix(Address adr, XMMRegister reg) {
duke@435	1086	if (reg->encoding() < 8) {
duke@435	1087	if (adr.base_needs_rex()) {
duke@435	1088	if (adr.index_needs_rex()) {
duke@435	1089	prefix(REX_XB);
duke@435	1090	} else {
duke@435	1091	prefix(REX_B);
duke@435	1092	}
duke@435	1093	} else {
duke@435	1094	if (adr.index_needs_rex()) {
duke@435	1095	prefix(REX_X);
duke@435	1096	}
duke@435	1097	}
duke@435	1098	} else {
duke@435	1099	if (adr.base_needs_rex()) {
duke@435	1100	if (adr.index_needs_rex()) {
duke@435	1101	prefix(REX_RXB);
duke@435	1102	} else {
duke@435	1103	prefix(REX_RB);
duke@435	1104	}
duke@435	1105	} else {
duke@435	1106	if (adr.index_needs_rex()) {
duke@435	1107	prefix(REX_RX);
duke@435	1108	} else {
duke@435	1109	prefix(REX_R);
duke@435	1110	}
duke@435	1111	}
duke@435	1112	}
duke@435	1113	}
duke@435	1114
duke@435	1115	void Assembler::emit_operand(Register reg, Address adr,
duke@435	1116	int rip_relative_correction) {
duke@435	1117	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1118	adr._rspec,
duke@435	1119	rip_relative_correction);
duke@435	1120	}
duke@435	1121
duke@435	1122	void Assembler::emit_operand(XMMRegister reg, Address adr,
duke@435	1123	int rip_relative_correction) {
duke@435	1124	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1125	adr._rspec,
duke@435	1126	rip_relative_correction);
duke@435	1127	}
duke@435	1128
duke@435	1129	void Assembler::emit_farith(int b1, int b2, int i) {
duke@435	1130	assert(isByte(b1) && isByte(b2), "wrong opcode");
duke@435	1131	assert(0 <= i && i < 8, "illegal stack offset");
duke@435	1132	emit_byte(b1);
duke@435	1133	emit_byte(b2 + i);
duke@435	1134	}
duke@435	1135
duke@435	1136	// pushad is invalid, use this instead.
duke@435	1137	// NOTE: Kills flags!!
duke@435	1138	void Assembler::pushaq() {
duke@435	1139	// we have to store original rsp. ABI says that 128 bytes
duke@435	1140	// below rsp are local scratch.
duke@435	1141	movq(Address(rsp, -5 * wordSize), rsp);
duke@435	1142
duke@435	1143	subq(rsp, 16 * wordSize);
duke@435	1144
duke@435	1145	movq(Address(rsp, 15 * wordSize), rax);
duke@435	1146	movq(Address(rsp, 14 * wordSize), rcx);
duke@435	1147	movq(Address(rsp, 13 * wordSize), rdx);
duke@435	1148	movq(Address(rsp, 12 * wordSize), rbx);
duke@435	1149	// skip rsp
duke@435	1150	movq(Address(rsp, 10 * wordSize), rbp);
duke@435	1151	movq(Address(rsp, 9 * wordSize), rsi);
duke@435	1152	movq(Address(rsp, 8 * wordSize), rdi);
duke@435	1153	movq(Address(rsp, 7 * wordSize), r8);
duke@435	1154	movq(Address(rsp, 6 * wordSize), r9);
duke@435	1155	movq(Address(rsp, 5 * wordSize), r10);
duke@435	1156	movq(Address(rsp, 4 * wordSize), r11);
duke@435	1157	movq(Address(rsp, 3 * wordSize), r12);
duke@435	1158	movq(Address(rsp, 2 * wordSize), r13);
duke@435	1159	movq(Address(rsp, wordSize), r14);
duke@435	1160	movq(Address(rsp, 0), r15);
duke@435	1161	}
duke@435	1162
duke@435	1163	// popad is invalid, use this instead
duke@435	1164	// NOTE: Kills flags!!
duke@435	1165	void Assembler::popaq() {
duke@435	1166	movq(r15, Address(rsp, 0));
duke@435	1167	movq(r14, Address(rsp, wordSize));
duke@435	1168	movq(r13, Address(rsp, 2 * wordSize));
duke@435	1169	movq(r12, Address(rsp, 3 * wordSize));
duke@435	1170	movq(r11, Address(rsp, 4 * wordSize));
duke@435	1171	movq(r10, Address(rsp, 5 * wordSize));
duke@435	1172	movq(r9, Address(rsp, 6 * wordSize));
duke@435	1173	movq(r8, Address(rsp, 7 * wordSize));
duke@435	1174	movq(rdi, Address(rsp, 8 * wordSize));
duke@435	1175	movq(rsi, Address(rsp, 9 * wordSize));
duke@435	1176	movq(rbp, Address(rsp, 10 * wordSize));
duke@435	1177	// skip rsp
duke@435	1178	movq(rbx, Address(rsp, 12 * wordSize));
duke@435	1179	movq(rdx, Address(rsp, 13 * wordSize));
duke@435	1180	movq(rcx, Address(rsp, 14 * wordSize));
duke@435	1181	movq(rax, Address(rsp, 15 * wordSize));
duke@435	1182
duke@435	1183	addq(rsp, 16 * wordSize);
duke@435	1184	}
duke@435	1185
duke@435	1186	void Assembler::pushfq() {
duke@435	1187	emit_byte(0x9C);
duke@435	1188	}
duke@435	1189
duke@435	1190	void Assembler::popfq() {
duke@435	1191	emit_byte(0x9D);
duke@435	1192	}
duke@435	1193
duke@435	1194	void Assembler::pushq(int imm32) {
duke@435	1195	emit_byte(0x68);
duke@435	1196	emit_long(imm32);
duke@435	1197	}
duke@435	1198
duke@435	1199	void Assembler::pushq(Register src) {
duke@435	1200	int encode = prefix_and_encode(src->encoding());
duke@435	1201
duke@435	1202	emit_byte(0x50 | encode);
duke@435	1203	}
duke@435	1204
duke@435	1205	void Assembler::pushq(Address src) {
duke@435	1206	InstructionMark im(this);
duke@435	1207	prefix(src);
duke@435	1208	emit_byte(0xFF);
duke@435	1209	emit_operand(rsi, src);
duke@435	1210	}
duke@435	1211
duke@435	1212	void Assembler::popq(Register dst) {
duke@435	1213	int encode = prefix_and_encode(dst->encoding());
duke@435	1214	emit_byte(0x58 | encode);
duke@435	1215	}
duke@435	1216
duke@435	1217	void Assembler::popq(Address dst) {
duke@435	1218	InstructionMark im(this);
duke@435	1219	prefix(dst);
duke@435	1220	emit_byte(0x8F);
duke@435	1221	emit_operand(rax, dst);
duke@435	1222	}
duke@435	1223
duke@435	1224	void Assembler::prefix(Prefix p) {
duke@435	1225	a_byte(p);
duke@435	1226	}
duke@435	1227
duke@435	1228	void Assembler::movb(Register dst, Address src) {
duke@435	1229	InstructionMark im(this);
duke@435	1230	prefix(src, dst, true);
duke@435	1231	emit_byte(0x8A);
duke@435	1232	emit_operand(dst, src);
duke@435	1233	}
duke@435	1234
duke@435	1235	void Assembler::movb(Address dst, int imm8) {
duke@435	1236	InstructionMark im(this);
duke@435	1237	prefix(dst);
duke@435	1238	emit_byte(0xC6);
duke@435	1239	emit_operand(rax, dst, 1);
duke@435	1240	emit_byte(imm8);
duke@435	1241	}
duke@435	1242
duke@435	1243	void Assembler::movb(Address dst, Register src) {
duke@435	1244	InstructionMark im(this);
duke@435	1245	prefix(dst, src, true);
duke@435	1246	emit_byte(0x88);
duke@435	1247	emit_operand(src, dst);
duke@435	1248	}
duke@435	1249
duke@435	1250	void Assembler::movw(Address dst, int imm16) {
duke@435	1251	InstructionMark im(this);
duke@435	1252	emit_byte(0x66); // switch to 16-bit mode
duke@435	1253	prefix(dst);
duke@435	1254	emit_byte(0xC7);
duke@435	1255	emit_operand(rax, dst, 2);
duke@435	1256	emit_word(imm16);
duke@435	1257	}
duke@435	1258
duke@435	1259	void Assembler::movw(Register dst, Address src) {
duke@435	1260	InstructionMark im(this);
duke@435	1261	emit_byte(0x66);
duke@435	1262	prefix(src, dst);
duke@435	1263	emit_byte(0x8B);
duke@435	1264	emit_operand(dst, src);
duke@435	1265	}
duke@435	1266
duke@435	1267	void Assembler::movw(Address dst, Register src) {
duke@435	1268	InstructionMark im(this);
duke@435	1269	emit_byte(0x66);
duke@435	1270	prefix(dst, src);
duke@435	1271	emit_byte(0x89);
duke@435	1272	emit_operand(src, dst);
duke@435	1273	}
duke@435	1274
duke@435	1275	// Uses zero extension.
duke@435	1276	void Assembler::movl(Register dst, int imm32) {
duke@435	1277	int encode = prefix_and_encode(dst->encoding());
duke@435	1278	emit_byte(0xB8 | encode);
duke@435	1279	emit_long(imm32);
duke@435	1280	}
duke@435	1281
duke@435	1282	void Assembler::movl(Register dst, Register src) {
duke@435	1283	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1284	emit_byte(0x8B);
duke@435	1285	emit_byte(0xC0 | encode);
duke@435	1286	}
duke@435	1287
duke@435	1288	void Assembler::movl(Register dst, Address src) {
duke@435	1289	InstructionMark im(this);
duke@435	1290	prefix(src, dst);
duke@435	1291	emit_byte(0x8B);
duke@435	1292	emit_operand(dst, src);
duke@435	1293	}
duke@435	1294
duke@435	1295	void Assembler::movl(Address dst, int imm32) {
duke@435	1296	InstructionMark im(this);
duke@435	1297	prefix(dst);
duke@435	1298	emit_byte(0xC7);
duke@435	1299	emit_operand(rax, dst, 4);
duke@435	1300	emit_long(imm32);
duke@435	1301	}
duke@435	1302
duke@435	1303	void Assembler::movl(Address dst, Register src) {
duke@435	1304	InstructionMark im(this);
duke@435	1305	prefix(dst, src);
duke@435	1306	emit_byte(0x89);
duke@435	1307	emit_operand(src, dst);
duke@435	1308	}
duke@435	1309
dcubed@485	1310	void Assembler::mov64(Register dst, intptr_t imm64) {
duke@435	1311	InstructionMark im(this);
duke@435	1312	int encode = prefixq_and_encode(dst->encoding());
duke@435	1313	emit_byte(0xB8 | encode);
duke@435	1314	emit_long64(imm64);
duke@435	1315	}
duke@435	1316
duke@435	1317	void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
duke@435	1318	InstructionMark im(this);
duke@435	1319	int encode = prefixq_and_encode(dst->encoding());
duke@435	1320	emit_byte(0xB8 | encode);
duke@435	1321	emit_data64(imm64, rspec);
duke@435	1322	}
duke@435	1323
duke@435	1324	void Assembler::movq(Register dst, Register src) {
duke@435	1325	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1326	emit_byte(0x8B);
duke@435	1327	emit_byte(0xC0 | encode);
duke@435	1328	}
duke@435	1329
duke@435	1330	void Assembler::movq(Register dst, Address src) {
duke@435	1331	InstructionMark im(this);
duke@435	1332	prefixq(src, dst);
duke@435	1333	emit_byte(0x8B);
duke@435	1334	emit_operand(dst, src);
duke@435	1335	}
duke@435	1336
dcubed@485	1337	void Assembler::mov64(Address dst, intptr_t imm32) {
duke@435	1338	assert(is_simm32(imm32), "lost bits");
duke@435	1339	InstructionMark im(this);
duke@435	1340	prefixq(dst);
duke@435	1341	emit_byte(0xC7);
duke@435	1342	emit_operand(rax, dst, 4);
duke@435	1343	emit_long(imm32);
duke@435	1344	}
duke@435	1345
duke@435	1346	void Assembler::movq(Address dst, Register src) {
duke@435	1347	InstructionMark im(this);
duke@435	1348	prefixq(dst, src);
duke@435	1349	emit_byte(0x89);
duke@435	1350	emit_operand(src, dst);
duke@435	1351	}
duke@435	1352
duke@435	1353	void Assembler::movsbl(Register dst, Address src) {
duke@435	1354	InstructionMark im(this);
duke@435	1355	prefix(src, dst);
duke@435	1356	emit_byte(0x0F);
duke@435	1357	emit_byte(0xBE);
duke@435	1358	emit_operand(dst, src);
duke@435	1359	}
duke@435	1360
duke@435	1361	void Assembler::movsbl(Register dst, Register src) {
duke@435	1362	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1363	emit_byte(0x0F);
duke@435	1364	emit_byte(0xBE);
duke@435	1365	emit_byte(0xC0 | encode);
duke@435	1366	}
duke@435	1367
duke@435	1368	void Assembler::movswl(Register dst, Address src) {
duke@435	1369	InstructionMark im(this);
duke@435	1370	prefix(src, dst);
duke@435	1371	emit_byte(0x0F);
duke@435	1372	emit_byte(0xBF);
duke@435	1373	emit_operand(dst, src);
duke@435	1374	}
duke@435	1375
duke@435	1376	void Assembler::movswl(Register dst, Register src) {
duke@435	1377	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1378	emit_byte(0x0F);
duke@435	1379	emit_byte(0xBF);
duke@435	1380	emit_byte(0xC0 | encode);
duke@435	1381	}
duke@435	1382
duke@435	1383	void Assembler::movslq(Register dst, Address src) {
duke@435	1384	InstructionMark im(this);
duke@435	1385	prefixq(src, dst);
duke@435	1386	emit_byte(0x63);
duke@435	1387	emit_operand(dst, src);
duke@435	1388	}
duke@435	1389
duke@435	1390	void Assembler::movslq(Register dst, Register src) {
duke@435	1391	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1392	emit_byte(0x63);
duke@435	1393	emit_byte(0xC0 | encode);
duke@435	1394	}
duke@435	1395
duke@435	1396	void Assembler::movzbl(Register dst, Address src) {
duke@435	1397	InstructionMark im(this);
duke@435	1398	prefix(src, dst);
duke@435	1399	emit_byte(0x0F);
duke@435	1400	emit_byte(0xB6);
duke@435	1401	emit_operand(dst, src);
duke@435	1402	}
duke@435	1403
duke@435	1404	void Assembler::movzbl(Register dst, Register src) {
duke@435	1405	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1406	emit_byte(0x0F);
duke@435	1407	emit_byte(0xB6);
duke@435	1408	emit_byte(0xC0 | encode);
duke@435	1409	}
duke@435	1410
duke@435	1411	void Assembler::movzwl(Register dst, Address src) {
duke@435	1412	InstructionMark im(this);
duke@435	1413	prefix(src, dst);
duke@435	1414	emit_byte(0x0F);
duke@435	1415	emit_byte(0xB7);
duke@435	1416	emit_operand(dst, src);
duke@435	1417	}
duke@435	1418
duke@435	1419	void Assembler::movzwl(Register dst, Register src) {
duke@435	1420	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1421	emit_byte(0x0F);
duke@435	1422	emit_byte(0xB7);
duke@435	1423	emit_byte(0xC0 | encode);
duke@435	1424	}
duke@435	1425
duke@435	1426	void Assembler::movss(XMMRegister dst, XMMRegister src) {
duke@435	1427	emit_byte(0xF3);
duke@435	1428	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1429	emit_byte(0x0F);
duke@435	1430	emit_byte(0x10);
duke@435	1431	emit_byte(0xC0 | encode);
duke@435	1432	}
duke@435	1433
duke@435	1434	void Assembler::movss(XMMRegister dst, Address src) {
duke@435	1435	InstructionMark im(this);
duke@435	1436	emit_byte(0xF3);
duke@435	1437	prefix(src, dst);
duke@435	1438	emit_byte(0x0F);
duke@435	1439	emit_byte(0x10);
duke@435	1440	emit_operand(dst, src);
duke@435	1441	}
duke@435	1442
duke@435	1443	void Assembler::movss(Address dst, XMMRegister src) {
duke@435	1444	InstructionMark im(this);
duke@435	1445	emit_byte(0xF3);
duke@435	1446	prefix(dst, src);
duke@435	1447	emit_byte(0x0F);
duke@435	1448	emit_byte(0x11);
duke@435	1449	emit_operand(src, dst);
duke@435	1450	}
duke@435	1451
duke@435	1452	void Assembler::movsd(XMMRegister dst, XMMRegister src) {
duke@435	1453	emit_byte(0xF2);
duke@435	1454	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1455	emit_byte(0x0F);
duke@435	1456	emit_byte(0x10);
duke@435	1457	emit_byte(0xC0 | encode);
duke@435	1458	}
duke@435	1459
duke@435	1460	void Assembler::movsd(XMMRegister dst, Address src) {
duke@435	1461	InstructionMark im(this);
duke@435	1462	emit_byte(0xF2);
duke@435	1463	prefix(src, dst);
duke@435	1464	emit_byte(0x0F);
duke@435	1465	emit_byte(0x10);
duke@435	1466	emit_operand(dst, src);
duke@435	1467	}
duke@435	1468
duke@435	1469	void Assembler::movsd(Address dst, XMMRegister src) {
duke@435	1470	InstructionMark im(this);
duke@435	1471	emit_byte(0xF2);
duke@435	1472	prefix(dst, src);
duke@435	1473	emit_byte(0x0F);
duke@435	1474	emit_byte(0x11);
duke@435	1475	emit_operand(src, dst);
duke@435	1476	}
duke@435	1477
duke@435	1478	// New cpus require to use movsd and movss to avoid partial register stall
duke@435	1479	// when loading from memory. But for old Opteron use movlpd instead of movsd.
duke@435	1480	// The selection is done in MacroAssembler::movdbl() and movflt().
duke@435	1481	void Assembler::movlpd(XMMRegister dst, Address src) {
duke@435	1482	InstructionMark im(this);
duke@435	1483	emit_byte(0x66);
duke@435	1484	prefix(src, dst);
duke@435	1485	emit_byte(0x0F);
duke@435	1486	emit_byte(0x12);
duke@435	1487	emit_operand(dst, src);
duke@435	1488	}
duke@435	1489
duke@435	1490	void Assembler::movapd(XMMRegister dst, XMMRegister src) {
duke@435	1491	int dstenc = dst->encoding();
duke@435	1492	int srcenc = src->encoding();
duke@435	1493	emit_byte(0x66);
duke@435	1494	if (dstenc < 8) {
duke@435	1495	if (srcenc >= 8) {
duke@435	1496	prefix(REX_B);
duke@435	1497	srcenc -= 8;
duke@435	1498	}
duke@435	1499	} else {
duke@435	1500	if (srcenc < 8) {
duke@435	1501	prefix(REX_R);
duke@435	1502	} else {
duke@435	1503	prefix(REX_RB);
duke@435	1504	srcenc -= 8;
duke@435	1505	}
duke@435	1506	dstenc -= 8;
duke@435	1507	}
duke@435	1508	emit_byte(0x0F);
duke@435	1509	emit_byte(0x28);
duke@435	1510	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1511	}
duke@435	1512
duke@435	1513	void Assembler::movaps(XMMRegister dst, XMMRegister src) {
duke@435	1514	int dstenc = dst->encoding();
duke@435	1515	int srcenc = src->encoding();
duke@435	1516	if (dstenc < 8) {
duke@435	1517	if (srcenc >= 8) {
duke@435	1518	prefix(REX_B);
duke@435	1519	srcenc -= 8;
duke@435	1520	}
duke@435	1521	} else {
duke@435	1522	if (srcenc < 8) {
duke@435	1523	prefix(REX_R);
duke@435	1524	} else {
duke@435	1525	prefix(REX_RB);
duke@435	1526	srcenc -= 8;
duke@435	1527	}
duke@435	1528	dstenc -= 8;
duke@435	1529	}
duke@435	1530	emit_byte(0x0F);
duke@435	1531	emit_byte(0x28);
duke@435	1532	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1533	}
duke@435	1534
duke@435	1535	void Assembler::movdl(XMMRegister dst, Register src) {
duke@435	1536	emit_byte(0x66);
duke@435	1537	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1538	emit_byte(0x0F);
duke@435	1539	emit_byte(0x6E);
duke@435	1540	emit_byte(0xC0 | encode);
duke@435	1541	}
duke@435	1542
duke@435	1543	void Assembler::movdl(Register dst, XMMRegister src) {
duke@435	1544	emit_byte(0x66);
duke@435	1545	// swap src/dst to get correct prefix
duke@435	1546	int encode = prefix_and_encode(src->encoding(), dst->encoding());
duke@435	1547	emit_byte(0x0F);
duke@435	1548	emit_byte(0x7E);
duke@435	1549	emit_byte(0xC0 | encode);
duke@435	1550	}
duke@435	1551
duke@435	1552	void Assembler::movdq(XMMRegister dst, Register src) {
duke@435	1553	emit_byte(0x66);
duke@435	1554	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1555	emit_byte(0x0F);
duke@435	1556	emit_byte(0x6E);
duke@435	1557	emit_byte(0xC0 | encode);
duke@435	1558	}
duke@435	1559
duke@435	1560	void Assembler::movdq(Register dst, XMMRegister src) {
duke@435	1561	emit_byte(0x66);
duke@435	1562	// swap src/dst to get correct prefix
duke@435	1563	int encode = prefixq_and_encode(src->encoding(), dst->encoding());
duke@435	1564	emit_byte(0x0F);
duke@435	1565	emit_byte(0x7E);
duke@435	1566	emit_byte(0xC0 | encode);
duke@435	1567	}
duke@435	1568
duke@435	1569	void Assembler::pxor(XMMRegister dst, Address src) {
duke@435	1570	InstructionMark im(this);
duke@435	1571	emit_byte(0x66);
duke@435	1572	prefix(src, dst);
duke@435	1573	emit_byte(0x0F);
duke@435	1574	emit_byte(0xEF);
duke@435	1575	emit_operand(dst, src);
duke@435	1576	}
duke@435	1577
duke@435	1578	void Assembler::pxor(XMMRegister dst, XMMRegister src) {
duke@435	1579	InstructionMark im(this);
duke@435	1580	emit_byte(0x66);
duke@435	1581	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1582	emit_byte(0x0F);
duke@435	1583	emit_byte(0xEF);
duke@435	1584	emit_byte(0xC0 | encode);
duke@435	1585	}
duke@435	1586
duke@435	1587	void Assembler::movdqa(XMMRegister dst, Address src) {
duke@435	1588	InstructionMark im(this);
duke@435	1589	emit_byte(0x66);
duke@435	1590	prefix(src, dst);
duke@435	1591	emit_byte(0x0F);
duke@435	1592	emit_byte(0x6F);
duke@435	1593	emit_operand(dst, src);
duke@435	1594	}
duke@435	1595
duke@435	1596	void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
duke@435	1597	emit_byte(0x66);
duke@435	1598	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1599	emit_byte(0x0F);
duke@435	1600	emit_byte(0x6F);
duke@435	1601	emit_byte(0xC0 | encode);
duke@435	1602	}
duke@435	1603
duke@435	1604	void Assembler::movdqa(Address dst, XMMRegister src) {
duke@435	1605	InstructionMark im(this);
duke@435	1606	emit_byte(0x66);
duke@435	1607	prefix(dst, src);
duke@435	1608	emit_byte(0x0F);
duke@435	1609	emit_byte(0x7F);
duke@435	1610	emit_operand(src, dst);
duke@435	1611	}
duke@435	1612
duke@435	1613	void Assembler::movq(XMMRegister dst, Address src) {
duke@435	1614	InstructionMark im(this);
duke@435	1615	emit_byte(0xF3);
duke@435	1616	prefix(src, dst);
duke@435	1617	emit_byte(0x0F);
duke@435	1618	emit_byte(0x7E);
duke@435	1619	emit_operand(dst, src);
duke@435	1620	}
duke@435	1621
duke@435	1622	void Assembler::movq(Address dst, XMMRegister src) {
duke@435	1623	InstructionMark im(this);
duke@435	1624	emit_byte(0x66);
duke@435	1625	prefix(dst, src);
duke@435	1626	emit_byte(0x0F);
duke@435	1627	emit_byte(0xD6);
duke@435	1628	emit_operand(src, dst);
duke@435	1629	}
duke@435	1630
duke@435	1631	void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1632	assert(isByte(mode), "invalid value");
duke@435	1633	emit_byte(0x66);
duke@435	1634	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1635	emit_byte(0x0F);
duke@435	1636	emit_byte(0x70);
duke@435	1637	emit_byte(0xC0 | encode);
duke@435	1638	emit_byte(mode & 0xFF);
duke@435	1639	}
duke@435	1640
duke@435	1641	void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
duke@435	1642	assert(isByte(mode), "invalid value");
duke@435	1643	InstructionMark im(this);
duke@435	1644	emit_byte(0x66);
duke@435	1645	emit_byte(0x0F);
duke@435	1646	emit_byte(0x70);
duke@435	1647	emit_operand(dst, src);
duke@435	1648	emit_byte(mode & 0xFF);
duke@435	1649	}
duke@435	1650
duke@435	1651	void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1652	assert(isByte(mode), "invalid value");
duke@435	1653	emit_byte(0xF2);
duke@435	1654	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1655	emit_byte(0x0F);
duke@435	1656	emit_byte(0x70);
duke@435	1657	emit_byte(0xC0 | encode);
duke@435	1658	emit_byte(mode & 0xFF);
duke@435	1659	}
duke@435	1660
duke@435	1661	void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
duke@435	1662	assert(isByte(mode), "invalid value");
duke@435	1663	InstructionMark im(this);
duke@435	1664	emit_byte(0xF2);
duke@435	1665	emit_byte(0x0F);
duke@435	1666	emit_byte(0x70);
duke@435	1667	emit_operand(dst, src);
duke@435	1668	emit_byte(mode & 0xFF);
duke@435	1669	}
duke@435	1670
duke@435	1671	void Assembler::cmovl(Condition cc, Register dst, Register src) {
duke@435	1672	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1673	emit_byte(0x0F);
duke@435	1674	emit_byte(0x40 | cc);
duke@435	1675	emit_byte(0xC0 | encode);
duke@435	1676	}
duke@435	1677
duke@435	1678	void Assembler::cmovl(Condition cc, Register dst, Address src) {
duke@435	1679	InstructionMark im(this);
duke@435	1680	prefix(src, dst);
duke@435	1681	emit_byte(0x0F);
duke@435	1682	emit_byte(0x40 | cc);
duke@435	1683	emit_operand(dst, src);
duke@435	1684	}
duke@435	1685
duke@435	1686	void Assembler::cmovq(Condition cc, Register dst, Register src) {
duke@435	1687	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1688	emit_byte(0x0F);
duke@435	1689	emit_byte(0x40 | cc);
duke@435	1690	emit_byte(0xC0 | encode);
duke@435	1691	}
duke@435	1692
duke@435	1693	void Assembler::cmovq(Condition cc, Register dst, Address src) {
duke@435	1694	InstructionMark im(this);
duke@435	1695	prefixq(src, dst);
duke@435	1696	emit_byte(0x0F);
duke@435	1697	emit_byte(0x40 | cc);
duke@435	1698	emit_operand(dst, src);
duke@435	1699	}
duke@435	1700
duke@435	1701	void Assembler::prefetch_prefix(Address src) {
duke@435	1702	prefix(src);
duke@435	1703	emit_byte(0x0F);
duke@435	1704	}
duke@435	1705
duke@435	1706	void Assembler::prefetcht0(Address src) {
duke@435	1707	InstructionMark im(this);
duke@435	1708	prefetch_prefix(src);
duke@435	1709	emit_byte(0x18);
duke@435	1710	emit_operand(rcx, src); // 1, src
duke@435	1711	}
duke@435	1712
duke@435	1713	void Assembler::prefetcht1(Address src) {
duke@435	1714	InstructionMark im(this);
duke@435	1715	prefetch_prefix(src);
duke@435	1716	emit_byte(0x18);
duke@435	1717	emit_operand(rdx, src); // 2, src
duke@435	1718	}
duke@435	1719
duke@435	1720	void Assembler::prefetcht2(Address src) {
duke@435	1721	InstructionMark im(this);
duke@435	1722	prefetch_prefix(src);
duke@435	1723	emit_byte(0x18);
duke@435	1724	emit_operand(rbx, src); // 3, src
duke@435	1725	}
duke@435	1726
duke@435	1727	void Assembler::prefetchnta(Address src) {
duke@435	1728	InstructionMark im(this);
duke@435	1729	prefetch_prefix(src);
duke@435	1730	emit_byte(0x18);
duke@435	1731	emit_operand(rax, src); // 0, src
duke@435	1732	}
duke@435	1733
duke@435	1734	void Assembler::prefetchw(Address src) {
duke@435	1735	InstructionMark im(this);
duke@435	1736	prefetch_prefix(src);
duke@435	1737	emit_byte(0x0D);
duke@435	1738	emit_operand(rcx, src); // 1, src
duke@435	1739	}
duke@435	1740
duke@435	1741	void Assembler::adcl(Register dst, int imm32) {
duke@435	1742	prefix(dst);
duke@435	1743	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1744	}
duke@435	1745
duke@435	1746	void Assembler::adcl(Register dst, Address src) {
duke@435	1747	InstructionMark im(this);
duke@435	1748	prefix(src, dst);
duke@435	1749	emit_byte(0x13);
duke@435	1750	emit_operand(dst, src);
duke@435	1751	}
duke@435	1752
duke@435	1753	void Assembler::adcl(Register dst, Register src) {
duke@435	1754	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1755	emit_arith(0x13, 0xC0, dst, src);
duke@435	1756	}
duke@435	1757
duke@435	1758	void Assembler::adcq(Register dst, int imm32) {
duke@435	1759	(void) prefixq_and_encode(dst->encoding());
duke@435	1760	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1761	}
duke@435	1762
duke@435	1763	void Assembler::adcq(Register dst, Address src) {
duke@435	1764	InstructionMark im(this);
duke@435	1765	prefixq(src, dst);
duke@435	1766	emit_byte(0x13);
duke@435	1767	emit_operand(dst, src);
duke@435	1768	}
duke@435	1769
duke@435	1770	void Assembler::adcq(Register dst, Register src) {
duke@435	1771	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1772	emit_arith(0x13, 0xC0, dst, src);
duke@435	1773	}
duke@435	1774
duke@435	1775	void Assembler::addl(Address dst, int imm32) {
duke@435	1776	InstructionMark im(this);
duke@435	1777	prefix(dst);
duke@435	1778	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1779	}
duke@435	1780
duke@435	1781	void Assembler::addl(Address dst, Register src) {
duke@435	1782	InstructionMark im(this);
duke@435	1783	prefix(dst, src);
duke@435	1784	emit_byte(0x01);
duke@435	1785	emit_operand(src, dst);
duke@435	1786	}
duke@435	1787
duke@435	1788	void Assembler::addl(Register dst, int imm32) {
duke@435	1789	prefix(dst);
duke@435	1790	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1791	}
duke@435	1792
duke@435	1793	void Assembler::addl(Register dst, Address src) {
duke@435	1794	InstructionMark im(this);
duke@435	1795	prefix(src, dst);
duke@435	1796	emit_byte(0x03);
duke@435	1797	emit_operand(dst, src);
duke@435	1798	}
duke@435	1799
duke@435	1800	void Assembler::addl(Register dst, Register src) {
duke@435	1801	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1802	emit_arith(0x03, 0xC0, dst, src);
duke@435	1803	}
duke@435	1804
duke@435	1805	void Assembler::addq(Address dst, int imm32) {
duke@435	1806	InstructionMark im(this);
duke@435	1807	prefixq(dst);
duke@435	1808	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1809	}
duke@435	1810
duke@435	1811	void Assembler::addq(Address dst, Register src) {
duke@435	1812	InstructionMark im(this);
duke@435	1813	prefixq(dst, src);
duke@435	1814	emit_byte(0x01);
duke@435	1815	emit_operand(src, dst);
duke@435	1816	}
duke@435	1817
duke@435	1818	void Assembler::addq(Register dst, int imm32) {
duke@435	1819	(void) prefixq_and_encode(dst->encoding());
duke@435	1820	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1821	}
duke@435	1822
duke@435	1823	void Assembler::addq(Register dst, Address src) {
duke@435	1824	InstructionMark im(this);
duke@435	1825	prefixq(src, dst);
duke@435	1826	emit_byte(0x03);
duke@435	1827	emit_operand(dst, src);
duke@435	1828	}
duke@435	1829
duke@435	1830	void Assembler::addq(Register dst, Register src) {
duke@435	1831	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1832	emit_arith(0x03, 0xC0, dst, src);
duke@435	1833	}
duke@435	1834
duke@435	1835	void Assembler::andl(Register dst, int imm32) {
duke@435	1836	prefix(dst);
duke@435	1837	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1838	}
duke@435	1839
duke@435	1840	void Assembler::andl(Register dst, Address src) {
duke@435	1841	InstructionMark im(this);
duke@435	1842	prefix(src, dst);
duke@435	1843	emit_byte(0x23);
duke@435	1844	emit_operand(dst, src);
duke@435	1845	}
duke@435	1846
duke@435	1847	void Assembler::andl(Register dst, Register src) {
duke@435	1848	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1849	emit_arith(0x23, 0xC0, dst, src);
duke@435	1850	}
duke@435	1851
duke@435	1852	void Assembler::andq(Register dst, int imm32) {
duke@435	1853	(void) prefixq_and_encode(dst->encoding());
duke@435	1854	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1855	}
duke@435	1856
duke@435	1857	void Assembler::andq(Register dst, Address src) {
duke@435	1858	InstructionMark im(this);
duke@435	1859	prefixq(src, dst);
duke@435	1860	emit_byte(0x23);
duke@435	1861	emit_operand(dst, src);
duke@435	1862	}
duke@435	1863
duke@435	1864	void Assembler::andq(Register dst, Register src) {
duke@435	1865	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1866	emit_arith(0x23, 0xC0, dst, src);
duke@435	1867	}
duke@435	1868
duke@435	1869	void Assembler::cmpb(Address dst, int imm8) {
duke@435	1870	InstructionMark im(this);
duke@435	1871	prefix(dst);
duke@435	1872	emit_byte(0x80);
duke@435	1873	emit_operand(rdi, dst, 1);
duke@435	1874	emit_byte(imm8);
duke@435	1875	}
duke@435	1876
duke@435	1877	void Assembler::cmpl(Address dst, int imm32) {
duke@435	1878	InstructionMark im(this);
duke@435	1879	prefix(dst);
duke@435	1880	emit_byte(0x81);
duke@435	1881	emit_operand(rdi, dst, 4);
duke@435	1882	emit_long(imm32);
duke@435	1883	}
duke@435	1884
duke@435	1885	void Assembler::cmpl(Register dst, int imm32) {
duke@435	1886	prefix(dst);
duke@435	1887	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1888	}
duke@435	1889
duke@435	1890	void Assembler::cmpl(Register dst, Register src) {
duke@435	1891	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1892	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1893	}
duke@435	1894
duke@435	1895	void Assembler::cmpl(Register dst, Address src) {
duke@435	1896	InstructionMark im(this);
duke@435	1897	prefix(src, dst);
duke@435	1898	emit_byte(0x3B);
duke@435	1899	emit_operand(dst, src);
duke@435	1900	}
duke@435	1901
duke@435	1902	void Assembler::cmpq(Address dst, int imm32) {
duke@435	1903	InstructionMark im(this);
duke@435	1904	prefixq(dst);
duke@435	1905	emit_byte(0x81);
duke@435	1906	emit_operand(rdi, dst, 4);
duke@435	1907	emit_long(imm32);
duke@435	1908	}
duke@435	1909
duke@435	1910	void Assembler::cmpq(Register dst, int imm32) {
duke@435	1911	(void) prefixq_and_encode(dst->encoding());
duke@435	1912	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1913	}
duke@435	1914
duke@435	1915	void Assembler::cmpq(Address dst, Register src) {
duke@435	1916	prefixq(dst, src);
duke@435	1917	emit_byte(0x3B);
duke@435	1918	emit_operand(src, dst);
duke@435	1919	}
duke@435	1920
duke@435	1921	void Assembler::cmpq(Register dst, Register src) {
duke@435	1922	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1923	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1924	}
duke@435	1925
duke@435	1926	void Assembler::cmpq(Register dst, Address src) {
duke@435	1927	InstructionMark im(this);
duke@435	1928	prefixq(src, dst);
duke@435	1929	emit_byte(0x3B);
duke@435	1930	emit_operand(dst, src);
duke@435	1931	}
duke@435	1932
duke@435	1933	void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
duke@435	1934	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1935	emit_byte(0x0F);
duke@435	1936	emit_byte(0x2E);
duke@435	1937	emit_byte(0xC0 | encode);
duke@435	1938	}
duke@435	1939
duke@435	1940	void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
duke@435	1941	emit_byte(0x66);
duke@435	1942	ucomiss(dst, src);
duke@435	1943	}
duke@435	1944
duke@435	1945	void Assembler::decl(Register dst) {
duke@435	1946	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1947	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1948	int encode = prefix_and_encode(dst->encoding());
duke@435	1949	emit_byte(0xFF);
duke@435	1950	emit_byte(0xC8 | encode);
duke@435	1951	}
duke@435	1952
duke@435	1953	void Assembler::decl(Address dst) {
duke@435	1954	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1955	InstructionMark im(this);
duke@435	1956	prefix(dst);
duke@435	1957	emit_byte(0xFF);
duke@435	1958	emit_operand(rcx, dst);
duke@435	1959	}
duke@435	1960
duke@435	1961	void Assembler::decq(Register dst) {
duke@435	1962	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1963	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1964	int encode = prefixq_and_encode(dst->encoding());
duke@435	1965	emit_byte(0xFF);
duke@435	1966	emit_byte(0xC8 | encode);
duke@435	1967	}
duke@435	1968
duke@435	1969	void Assembler::decq(Address dst) {
duke@435	1970	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1971	InstructionMark im(this);
duke@435	1972	prefixq(dst);
duke@435	1973	emit_byte(0xFF);
duke@435	1974	emit_operand(rcx, dst);
duke@435	1975	}
duke@435	1976
duke@435	1977	void Assembler::idivl(Register src) {
duke@435	1978	int encode = prefix_and_encode(src->encoding());
duke@435	1979	emit_byte(0xF7);
duke@435	1980	emit_byte(0xF8 | encode);
duke@435	1981	}
duke@435	1982
duke@435	1983	void Assembler::idivq(Register src) {
duke@435	1984	int encode = prefixq_and_encode(src->encoding());
duke@435	1985	emit_byte(0xF7);
duke@435	1986	emit_byte(0xF8 | encode);
duke@435	1987	}
duke@435	1988
duke@435	1989	void Assembler::cdql() {
duke@435	1990	emit_byte(0x99);
duke@435	1991	}
duke@435	1992
duke@435	1993	void Assembler::cdqq() {
duke@435	1994	prefix(REX_W);
duke@435	1995	emit_byte(0x99);
duke@435	1996	}
duke@435	1997
duke@435	1998	void Assembler::imull(Register dst, Register src) {
duke@435	1999	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2000	emit_byte(0x0F);
duke@435	2001	emit_byte(0xAF);
duke@435	2002	emit_byte(0xC0 | encode);
duke@435	2003	}
duke@435	2004
duke@435	2005	void Assembler::imull(Register dst, Register src, int value) {
duke@435	2006	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2007	if (is8bit(value)) {
duke@435	2008	emit_byte(0x6B);
duke@435	2009	emit_byte(0xC0 | encode);
duke@435	2010	emit_byte(value);
duke@435	2011	} else {
duke@435	2012	emit_byte(0x69);
duke@435	2013	emit_byte(0xC0 | encode);
duke@435	2014	emit_long(value);
duke@435	2015	}
duke@435	2016	}
duke@435	2017
duke@435	2018	void Assembler::imulq(Register dst, Register src) {
duke@435	2019	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2020	emit_byte(0x0F);
duke@435	2021	emit_byte(0xAF);
duke@435	2022	emit_byte(0xC0 | encode);
duke@435	2023	}
duke@435	2024
duke@435	2025	void Assembler::imulq(Register dst, Register src, int value) {
duke@435	2026	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2027	if (is8bit(value)) {
duke@435	2028	emit_byte(0x6B);
duke@435	2029	emit_byte(0xC0 | encode);
duke@435	2030	emit_byte(value);
duke@435	2031	} else {
duke@435	2032	emit_byte(0x69);
duke@435	2033	emit_byte(0xC0 | encode);
duke@435	2034	emit_long(value);
duke@435	2035	}
duke@435	2036	}
duke@435	2037
duke@435	2038	void Assembler::incl(Register dst) {
duke@435	2039	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2040	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2041	int encode = prefix_and_encode(dst->encoding());
duke@435	2042	emit_byte(0xFF);
duke@435	2043	emit_byte(0xC0 | encode);
duke@435	2044	}
duke@435	2045
duke@435	2046	void Assembler::incl(Address dst) {
duke@435	2047	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2048	InstructionMark im(this);
duke@435	2049	prefix(dst);
duke@435	2050	emit_byte(0xFF);
duke@435	2051	emit_operand(rax, dst);
duke@435	2052	}
duke@435	2053
duke@435	2054	void Assembler::incq(Register dst) {
duke@435	2055	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2056	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2057	int encode = prefixq_and_encode(dst->encoding());
duke@435	2058	emit_byte(0xFF);
duke@435	2059	emit_byte(0xC0 | encode);
duke@435	2060	}
duke@435	2061
duke@435	2062	void Assembler::incq(Address dst) {
duke@435	2063	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2064	InstructionMark im(this);
duke@435	2065	prefixq(dst);
duke@435	2066	emit_byte(0xFF);
duke@435	2067	emit_operand(rax, dst);
duke@435	2068	}
duke@435	2069
duke@435	2070	void Assembler::leal(Register dst, Address src) {
duke@435	2071	InstructionMark im(this);
duke@435	2072	emit_byte(0x67); // addr32
duke@435	2073	prefix(src, dst);
duke@435	2074	emit_byte(0x8D);
duke@435	2075	emit_operand(dst, src);
duke@435	2076	}
duke@435	2077
duke@435	2078	void Assembler::leaq(Register dst, Address src) {
duke@435	2079	InstructionMark im(this);
duke@435	2080	prefixq(src, dst);
duke@435	2081	emit_byte(0x8D);
duke@435	2082	emit_operand(dst, src);
duke@435	2083	}
duke@435	2084
duke@435	2085	void Assembler::mull(Address src) {
duke@435	2086	InstructionMark im(this);
duke@435	2087	// was missing
duke@435	2088	prefix(src);
duke@435	2089	emit_byte(0xF7);
duke@435	2090	emit_operand(rsp, src);
duke@435	2091	}
duke@435	2092
duke@435	2093	void Assembler::mull(Register src) {
duke@435	2094	// was missing
duke@435	2095	int encode = prefix_and_encode(src->encoding());
duke@435	2096	emit_byte(0xF7);
duke@435	2097	emit_byte(0xE0 | encode);
duke@435	2098	}
duke@435	2099
duke@435	2100	void Assembler::negl(Register dst) {
duke@435	2101	int encode = prefix_and_encode(dst->encoding());
duke@435	2102	emit_byte(0xF7);
duke@435	2103	emit_byte(0xD8 | encode);
duke@435	2104	}
duke@435	2105
duke@435	2106	void Assembler::negq(Register dst) {
duke@435	2107	int encode = prefixq_and_encode(dst->encoding());
duke@435	2108	emit_byte(0xF7);
duke@435	2109	emit_byte(0xD8 | encode);
duke@435	2110	}
duke@435	2111
duke@435	2112	void Assembler::notl(Register dst) {
duke@435	2113	int encode = prefix_and_encode(dst->encoding());
duke@435	2114	emit_byte(0xF7);
duke@435	2115	emit_byte(0xD0 | encode);
duke@435	2116	}
duke@435	2117
duke@435	2118	void Assembler::notq(Register dst) {
duke@435	2119	int encode = prefixq_and_encode(dst->encoding());
duke@435	2120	emit_byte(0xF7);
duke@435	2121	emit_byte(0xD0 | encode);
duke@435	2122	}
duke@435	2123
duke@435	2124	void Assembler::orl(Address dst, int imm32) {
duke@435	2125	InstructionMark im(this);
duke@435	2126	prefix(dst);
duke@435	2127	emit_byte(0x81);
duke@435	2128	emit_operand(rcx, dst, 4);
duke@435	2129	emit_long(imm32);
duke@435	2130	}
duke@435	2131
duke@435	2132	void Assembler::orl(Register dst, int imm32) {
duke@435	2133	prefix(dst);
duke@435	2134	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2135	}
duke@435	2136
duke@435	2137	void Assembler::orl(Register dst, Address src) {
duke@435	2138	InstructionMark im(this);
duke@435	2139	prefix(src, dst);
duke@435	2140	emit_byte(0x0B);
duke@435	2141	emit_operand(dst, src);
duke@435	2142	}
duke@435	2143
duke@435	2144	void Assembler::orl(Register dst, Register src) {
duke@435	2145	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2146	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2147	}
duke@435	2148
duke@435	2149	void Assembler::orq(Address dst, int imm32) {
duke@435	2150	InstructionMark im(this);
duke@435	2151	prefixq(dst);
duke@435	2152	emit_byte(0x81);
duke@435	2153	emit_operand(rcx, dst, 4);
duke@435	2154	emit_long(imm32);
duke@435	2155	}
duke@435	2156
duke@435	2157	void Assembler::orq(Register dst, int imm32) {
duke@435	2158	(void) prefixq_and_encode(dst->encoding());
duke@435	2159	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2160	}
duke@435	2161
duke@435	2162	void Assembler::orq(Register dst, Address src) {
duke@435	2163	InstructionMark im(this);
duke@435	2164	prefixq(src, dst);
duke@435	2165	emit_byte(0x0B);
duke@435	2166	emit_operand(dst, src);
duke@435	2167	}
duke@435	2168
duke@435	2169	void Assembler::orq(Register dst, Register src) {
duke@435	2170	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2171	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2172	}
duke@435	2173
duke@435	2174	void Assembler::rcll(Register dst, int imm8) {
duke@435	2175	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2176	int encode = prefix_and_encode(dst->encoding());
duke@435	2177	if (imm8 == 1) {
duke@435	2178	emit_byte(0xD1);
duke@435	2179	emit_byte(0xD0 | encode);
duke@435	2180	} else {
duke@435	2181	emit_byte(0xC1);
duke@435	2182	emit_byte(0xD0 | encode);
duke@435	2183	emit_byte(imm8);
duke@435	2184	}
duke@435	2185	}
duke@435	2186
duke@435	2187	void Assembler::rclq(Register dst, int imm8) {
duke@435	2188	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2189	int encode = prefixq_and_encode(dst->encoding());
duke@435	2190	if (imm8 == 1) {
duke@435	2191	emit_byte(0xD1);
duke@435	2192	emit_byte(0xD0 | encode);
duke@435	2193	} else {
duke@435	2194	emit_byte(0xC1);
duke@435	2195	emit_byte(0xD0 | encode);
duke@435	2196	emit_byte(imm8);
duke@435	2197	}
duke@435	2198	}
duke@435	2199
duke@435	2200	void Assembler::sarl(Register dst, int imm8) {
duke@435	2201	int encode = prefix_and_encode(dst->encoding());
duke@435	2202	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2203	if (imm8 == 1) {
duke@435	2204	emit_byte(0xD1);
duke@435	2205	emit_byte(0xF8 | encode);
duke@435	2206	} else {
duke@435	2207	emit_byte(0xC1);
duke@435	2208	emit_byte(0xF8 | encode);
duke@435	2209	emit_byte(imm8);
duke@435	2210	}
duke@435	2211	}
duke@435	2212
duke@435	2213	void Assembler::sarl(Register dst) {
duke@435	2214	int encode = prefix_and_encode(dst->encoding());
duke@435	2215	emit_byte(0xD3);
duke@435	2216	emit_byte(0xF8 | encode);
duke@435	2217	}
duke@435	2218
duke@435	2219	void Assembler::sarq(Register dst, int imm8) {
duke@435	2220	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2221	int encode = prefixq_and_encode(dst->encoding());
duke@435	2222	if (imm8 == 1) {
duke@435	2223	emit_byte(0xD1);
duke@435	2224	emit_byte(0xF8 | encode);
duke@435	2225	} else {
duke@435	2226	emit_byte(0xC1);
duke@435	2227	emit_byte(0xF8 | encode);
duke@435	2228	emit_byte(imm8);
duke@435	2229	}
duke@435	2230	}
duke@435	2231
duke@435	2232	void Assembler::sarq(Register dst) {
duke@435	2233	int encode = prefixq_and_encode(dst->encoding());
duke@435	2234	emit_byte(0xD3);
duke@435	2235	emit_byte(0xF8 | encode);
duke@435	2236	}
duke@435	2237
duke@435	2238	void Assembler::sbbl(Address dst, int imm32) {
duke@435	2239	InstructionMark im(this);
duke@435	2240	prefix(dst);
duke@435	2241	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2242	}
duke@435	2243
duke@435	2244	void Assembler::sbbl(Register dst, int imm32) {
duke@435	2245	prefix(dst);
duke@435	2246	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2247	}
duke@435	2248
duke@435	2249	void Assembler::sbbl(Register dst, Address src) {
duke@435	2250	InstructionMark im(this);
duke@435	2251	prefix(src, dst);
duke@435	2252	emit_byte(0x1B);
duke@435	2253	emit_operand(dst, src);
duke@435	2254	}
duke@435	2255
duke@435	2256	void Assembler::sbbl(Register dst, Register src) {
duke@435	2257	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2258	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2259	}
duke@435	2260
duke@435	2261	void Assembler::sbbq(Address dst, int imm32) {
duke@435	2262	InstructionMark im(this);
duke@435	2263	prefixq(dst);
duke@435	2264	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2265	}
duke@435	2266
duke@435	2267	void Assembler::sbbq(Register dst, int imm32) {
duke@435	2268	(void) prefixq_and_encode(dst->encoding());
duke@435	2269	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2270	}
duke@435	2271
duke@435	2272	void Assembler::sbbq(Register dst, Address src) {
duke@435	2273	InstructionMark im(this);
duke@435	2274	prefixq(src, dst);
duke@435	2275	emit_byte(0x1B);
duke@435	2276	emit_operand(dst, src);
duke@435	2277	}
duke@435	2278
duke@435	2279	void Assembler::sbbq(Register dst, Register src) {
duke@435	2280	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2281	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2282	}
duke@435	2283
duke@435	2284	void Assembler::shll(Register dst, int imm8) {
duke@435	2285	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2286	int encode = prefix_and_encode(dst->encoding());
duke@435	2287	if (imm8 == 1 ) {
duke@435	2288	emit_byte(0xD1);
duke@435	2289	emit_byte(0xE0 | encode);
duke@435	2290	} else {
duke@435	2291	emit_byte(0xC1);
duke@435	2292	emit_byte(0xE0 | encode);
duke@435	2293	emit_byte(imm8);
duke@435	2294	}
duke@435	2295	}
duke@435	2296
duke@435	2297	void Assembler::shll(Register dst) {
duke@435	2298	int encode = prefix_and_encode(dst->encoding());
duke@435	2299	emit_byte(0xD3);
duke@435	2300	emit_byte(0xE0 | encode);
duke@435	2301	}
duke@435	2302
duke@435	2303	void Assembler::shlq(Register dst, int imm8) {
duke@435	2304	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2305	int encode = prefixq_and_encode(dst->encoding());
duke@435	2306	if (imm8 == 1) {
duke@435	2307	emit_byte(0xD1);
duke@435	2308	emit_byte(0xE0 | encode);
duke@435	2309	} else {
duke@435	2310	emit_byte(0xC1);
duke@435	2311	emit_byte(0xE0 | encode);
duke@435	2312	emit_byte(imm8);
duke@435	2313	}
duke@435	2314	}
duke@435	2315
duke@435	2316	void Assembler::shlq(Register dst) {
duke@435	2317	int encode = prefixq_and_encode(dst->encoding());
duke@435	2318	emit_byte(0xD3);
duke@435	2319	emit_byte(0xE0 | encode);
duke@435	2320	}
duke@435	2321
duke@435	2322	void Assembler::shrl(Register dst, int imm8) {
duke@435	2323	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2324	int encode = prefix_and_encode(dst->encoding());
duke@435	2325	emit_byte(0xC1);
duke@435	2326	emit_byte(0xE8 | encode);
duke@435	2327	emit_byte(imm8);
duke@435	2328	}
duke@435	2329
duke@435	2330	void Assembler::shrl(Register dst) {
duke@435	2331	int encode = prefix_and_encode(dst->encoding());
duke@435	2332	emit_byte(0xD3);
duke@435	2333	emit_byte(0xE8 | encode);
duke@435	2334	}
duke@435	2335
duke@435	2336	void Assembler::shrq(Register dst, int imm8) {
duke@435	2337	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2338	int encode = prefixq_and_encode(dst->encoding());
duke@435	2339	emit_byte(0xC1);
duke@435	2340	emit_byte(0xE8 | encode);
duke@435	2341	emit_byte(imm8);
duke@435	2342	}
duke@435	2343
duke@435	2344	void Assembler::shrq(Register dst) {
duke@435	2345	int encode = prefixq_and_encode(dst->encoding());
duke@435	2346	emit_byte(0xD3);
duke@435	2347	emit_byte(0xE8 | encode);
duke@435	2348	}
duke@435	2349
duke@435	2350	void Assembler::subl(Address dst, int imm32) {
duke@435	2351	InstructionMark im(this);
duke@435	2352	prefix(dst);
duke@435	2353	if (is8bit(imm32)) {
duke@435	2354	emit_byte(0x83);
duke@435	2355	emit_operand(rbp, dst, 1);
duke@435	2356	emit_byte(imm32 & 0xFF);
duke@435	2357	} else {
duke@435	2358	emit_byte(0x81);
duke@435	2359	emit_operand(rbp, dst, 4);
duke@435	2360	emit_long(imm32);
duke@435	2361	}
duke@435	2362	}
duke@435	2363
duke@435	2364	void Assembler::subl(Register dst, int imm32) {
duke@435	2365	prefix(dst);
duke@435	2366	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2367	}
duke@435	2368
duke@435	2369	void Assembler::subl(Address dst, Register src) {
duke@435	2370	InstructionMark im(this);
duke@435	2371	prefix(dst, src);
duke@435	2372	emit_byte(0x29);
duke@435	2373	emit_operand(src, dst);
duke@435	2374	}
duke@435	2375
duke@435	2376	void Assembler::subl(Register dst, Address src) {
duke@435	2377	InstructionMark im(this);
duke@435	2378	prefix(src, dst);
duke@435	2379	emit_byte(0x2B);
duke@435	2380	emit_operand(dst, src);
duke@435	2381	}
duke@435	2382
duke@435	2383	void Assembler::subl(Register dst, Register src) {
duke@435	2384	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2385	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2386	}
duke@435	2387
duke@435	2388	void Assembler::subq(Address dst, int imm32) {
duke@435	2389	InstructionMark im(this);
duke@435	2390	prefixq(dst);
duke@435	2391	if (is8bit(imm32)) {
duke@435	2392	emit_byte(0x83);
duke@435	2393	emit_operand(rbp, dst, 1);
duke@435	2394	emit_byte(imm32 & 0xFF);
duke@435	2395	} else {
duke@435	2396	emit_byte(0x81);
duke@435	2397	emit_operand(rbp, dst, 4);
duke@435	2398	emit_long(imm32);
duke@435	2399	}
duke@435	2400	}
duke@435	2401
duke@435	2402	void Assembler::subq(Register dst, int imm32) {
duke@435	2403	(void) prefixq_and_encode(dst->encoding());
duke@435	2404	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2405	}
duke@435	2406
duke@435	2407	void Assembler::subq(Address dst, Register src) {
duke@435	2408	InstructionMark im(this);
duke@435	2409	prefixq(dst, src);
duke@435	2410	emit_byte(0x29);
duke@435	2411	emit_operand(src, dst);
duke@435	2412	}
duke@435	2413
duke@435	2414	void Assembler::subq(Register dst, Address src) {
duke@435	2415	InstructionMark im(this);
duke@435	2416	prefixq(src, dst);
duke@435	2417	emit_byte(0x2B);
duke@435	2418	emit_operand(dst, src);
duke@435	2419	}
duke@435	2420
duke@435	2421	void Assembler::subq(Register dst, Register src) {
duke@435	2422	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2423	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2424	}
duke@435	2425
duke@435	2426	void Assembler::testb(Register dst, int imm8) {
duke@435	2427	(void) prefix_and_encode(dst->encoding(), true);
duke@435	2428	emit_arith_b(0xF6, 0xC0, dst, imm8);
duke@435	2429	}
duke@435	2430
duke@435	2431	void Assembler::testl(Register dst, int imm32) {
duke@435	2432	// not using emit_arith because test
duke@435	2433	// doesn't support sign-extension of
duke@435	2434	// 8bit operands
duke@435	2435	int encode = dst->encoding();
duke@435	2436	if (encode == 0) {
duke@435	2437	emit_byte(0xA9);
duke@435	2438	} else {
duke@435	2439	encode = prefix_and_encode(encode);
duke@435	2440	emit_byte(0xF7);
duke@435	2441	emit_byte(0xC0 | encode);
duke@435	2442	}
duke@435	2443	emit_long(imm32);
duke@435	2444	}
duke@435	2445
duke@435	2446	void Assembler::testl(Register dst, Register src) {
duke@435	2447	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2448	emit_arith(0x85, 0xC0, dst, src);
duke@435	2449	}
duke@435	2450
duke@435	2451	void Assembler::testq(Register dst, int imm32) {
duke@435	2452	// not using emit_arith because test
duke@435	2453	// doesn't support sign-extension of
duke@435	2454	// 8bit operands
duke@435	2455	int encode = dst->encoding();
duke@435	2456	if (encode == 0) {
duke@435	2457	prefix(REX_W);
duke@435	2458	emit_byte(0xA9);
duke@435	2459	} else {
duke@435	2460	encode = prefixq_and_encode(encode);
duke@435	2461	emit_byte(0xF7);
duke@435	2462	emit_byte(0xC0 | encode);
duke@435	2463	}
duke@435	2464	emit_long(imm32);
duke@435	2465	}
duke@435	2466
duke@435	2467	void Assembler::testq(Register dst, Register src) {
duke@435	2468	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2469	emit_arith(0x85, 0xC0, dst, src);
duke@435	2470	}
duke@435	2471
duke@435	2472	void Assembler::xaddl(Address dst, Register src) {
duke@435	2473	InstructionMark im(this);
duke@435	2474	prefix(dst, src);
duke@435	2475	emit_byte(0x0F);
duke@435	2476	emit_byte(0xC1);
duke@435	2477	emit_operand(src, dst);
duke@435	2478	}
duke@435	2479
duke@435	2480	void Assembler::xaddq(Address dst, Register src) {
duke@435	2481	InstructionMark im(this);
duke@435	2482	prefixq(dst, src);
duke@435	2483	emit_byte(0x0F);
duke@435	2484	emit_byte(0xC1);
duke@435	2485	emit_operand(src, dst);
duke@435	2486	}
duke@435	2487
duke@435	2488	void Assembler::xorl(Register dst, int imm32) {
duke@435	2489	prefix(dst);
duke@435	2490	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2491	}
duke@435	2492
duke@435	2493	void Assembler::xorl(Register dst, Register src) {
duke@435	2494	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2495	emit_arith(0x33, 0xC0, dst, src);
duke@435	2496	}
duke@435	2497
duke@435	2498	void Assembler::xorl(Register dst, Address src) {
duke@435	2499	InstructionMark im(this);
duke@435	2500	prefix(src, dst);
duke@435	2501	emit_byte(0x33);
duke@435	2502	emit_operand(dst, src);
duke@435	2503	}
duke@435	2504
duke@435	2505	void Assembler::xorq(Register dst, int imm32) {
duke@435	2506	(void) prefixq_and_encode(dst->encoding());
duke@435	2507	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2508	}
duke@435	2509
duke@435	2510	void Assembler::xorq(Register dst, Register src) {
duke@435	2511	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2512	emit_arith(0x33, 0xC0, dst, src);
duke@435	2513	}
duke@435	2514
duke@435	2515	void Assembler::xorq(Register dst, Address src) {
duke@435	2516	InstructionMark im(this);
duke@435	2517	prefixq(src, dst);
duke@435	2518	emit_byte(0x33);
duke@435	2519	emit_operand(dst, src);
duke@435	2520	}
duke@435	2521
duke@435	2522	void Assembler::bswapl(Register reg) {
duke@435	2523	int encode = prefix_and_encode(reg->encoding());
duke@435	2524	emit_byte(0x0F);
duke@435	2525	emit_byte(0xC8 | encode);
duke@435	2526	}
duke@435	2527
duke@435	2528	void Assembler::bswapq(Register reg) {
duke@435	2529	int encode = prefixq_and_encode(reg->encoding());
duke@435	2530	emit_byte(0x0F);
duke@435	2531	emit_byte(0xC8 | encode);
duke@435	2532	}
duke@435	2533
duke@435	2534	void Assembler::lock() {
duke@435	2535	emit_byte(0xF0);
duke@435	2536	}
duke@435	2537
duke@435	2538	void Assembler::xchgl(Register dst, Address src) {
duke@435	2539	InstructionMark im(this);
duke@435	2540	prefix(src, dst);
duke@435	2541	emit_byte(0x87);
duke@435	2542	emit_operand(dst, src);
duke@435	2543	}
duke@435	2544
duke@435	2545	void Assembler::xchgl(Register dst, Register src) {
duke@435	2546	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2547	emit_byte(0x87);
duke@435	2548	emit_byte(0xc0 | encode);
duke@435	2549	}
duke@435	2550
duke@435	2551	void Assembler::xchgq(Register dst, Address src) {
duke@435	2552	InstructionMark im(this);
duke@435	2553	prefixq(src, dst);
duke@435	2554	emit_byte(0x87);
duke@435	2555	emit_operand(dst, src);
duke@435	2556	}
duke@435	2557
duke@435	2558	void Assembler::xchgq(Register dst, Register src) {
duke@435	2559	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2560	emit_byte(0x87);
duke@435	2561	emit_byte(0xc0 | encode);
duke@435	2562	}
duke@435	2563
duke@435	2564	void Assembler::cmpxchgl(Register reg, Address adr) {
duke@435	2565	InstructionMark im(this);
duke@435	2566	prefix(adr, reg);
duke@435	2567	emit_byte(0x0F);
duke@435	2568	emit_byte(0xB1);
duke@435	2569	emit_operand(reg, adr);
duke@435	2570	}
duke@435	2571
duke@435	2572	void Assembler::cmpxchgq(Register reg, Address adr) {
duke@435	2573	InstructionMark im(this);
duke@435	2574	prefixq(adr, reg);
duke@435	2575	emit_byte(0x0F);
duke@435	2576	emit_byte(0xB1);
duke@435	2577	emit_operand(reg, adr);
duke@435	2578	}
duke@435	2579
duke@435	2580	void Assembler::hlt() {
duke@435	2581	emit_byte(0xF4);
duke@435	2582	}
duke@435	2583
duke@435	2584
duke@435	2585	void Assembler::addr_nop_4() {
duke@435	2586	// 4 bytes: NOP DWORD PTR [EAX+0]
duke@435	2587	emit_byte(0x0F);
duke@435	2588	emit_byte(0x1F);
duke@435	2589	emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
duke@435	2590	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2591	}
duke@435	2592
duke@435	2593	void Assembler::addr_nop_5() {
duke@435	2594	// 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
duke@435	2595	emit_byte(0x0F);
duke@435	2596	emit_byte(0x1F);
duke@435	2597	emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
duke@435	2598	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2599	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2600	}
duke@435	2601
duke@435	2602	void Assembler::addr_nop_7() {
duke@435	2603	// 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
duke@435	2604	emit_byte(0x0F);
duke@435	2605	emit_byte(0x1F);
duke@435	2606	emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
duke@435	2607	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2608	}
duke@435	2609
duke@435	2610	void Assembler::addr_nop_8() {
duke@435	2611	// 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
duke@435	2612	emit_byte(0x0F);
duke@435	2613	emit_byte(0x1F);
duke@435	2614	emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
duke@435	2615	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2616	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2617	}
duke@435	2618
duke@435	2619	void Assembler::nop(int i) {
duke@435	2620	assert(i > 0, " ");
duke@435	2621	if (UseAddressNop && VM_Version::is_intel()) {
duke@435	2622	//
duke@435	2623	// Using multi-bytes nops "0x0F 0x1F [address]" for Intel
duke@435	2624	// 1: 0x90
duke@435	2625	// 2: 0x66 0x90
duke@435	2626	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2627	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2628	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2629	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2630	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2631	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2632	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2633	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2634	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2635
duke@435	2636	// The rest coding is Intel specific - don't use consecutive address nops
duke@435	2637
duke@435	2638	// 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2639	// 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2640	// 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2641	// 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2642
duke@435	2643	while(i >= 15) {
duke@435	2644	// For Intel don't generate consecutive addess nops (mix with regular nops)
duke@435	2645	i -= 15;
duke@435	2646	emit_byte(0x66); // size prefix
duke@435	2647	emit_byte(0x66); // size prefix
duke@435	2648	emit_byte(0x66); // size prefix
duke@435	2649	addr_nop_8();
duke@435	2650	emit_byte(0x66); // size prefix
duke@435	2651	emit_byte(0x66); // size prefix
duke@435	2652	emit_byte(0x66); // size prefix
duke@435	2653	emit_byte(0x90); // nop
duke@435	2654	}
duke@435	2655	switch (i) {
duke@435	2656	case 14:
duke@435	2657	emit_byte(0x66); // size prefix
duke@435	2658	case 13:
duke@435	2659	emit_byte(0x66); // size prefix
duke@435	2660	case 12:
duke@435	2661	addr_nop_8();
duke@435	2662	emit_byte(0x66); // size prefix
duke@435	2663	emit_byte(0x66); // size prefix
duke@435	2664	emit_byte(0x66); // size prefix
duke@435	2665	emit_byte(0x90); // nop
duke@435	2666	break;
duke@435	2667	case 11:
duke@435	2668	emit_byte(0x66); // size prefix
duke@435	2669	case 10:
duke@435	2670	emit_byte(0x66); // size prefix
duke@435	2671	case 9:
duke@435	2672	emit_byte(0x66); // size prefix
duke@435	2673	case 8:
duke@435	2674	addr_nop_8();
duke@435	2675	break;
duke@435	2676	case 7:
duke@435	2677	addr_nop_7();
duke@435	2678	break;
duke@435	2679	case 6:
duke@435	2680	emit_byte(0x66); // size prefix
duke@435	2681	case 5:
duke@435	2682	addr_nop_5();
duke@435	2683	break;
duke@435	2684	case 4:
duke@435	2685	addr_nop_4();
duke@435	2686	break;
duke@435	2687	case 3:
duke@435	2688	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2689	emit_byte(0x66); // size prefix
duke@435	2690	case 2:
duke@435	2691	emit_byte(0x66); // size prefix
duke@435	2692	case 1:
duke@435	2693	emit_byte(0x90); // nop
duke@435	2694	break;
duke@435	2695	default:
duke@435	2696	assert(i == 0, " ");
duke@435	2697	}
duke@435	2698	return;
duke@435	2699	}
duke@435	2700	if (UseAddressNop && VM_Version::is_amd()) {
duke@435	2701	//
duke@435	2702	// Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
duke@435	2703	// 1: 0x90
duke@435	2704	// 2: 0x66 0x90
duke@435	2705	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2706	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2707	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2708	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2709	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2710	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2711	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2712	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2713	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2714
duke@435	2715	// The rest coding is AMD specific - use consecutive address nops
duke@435	2716
duke@435	2717	// 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2718	// 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2719	// 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2720	// 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2721	// 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2722	// Size prefixes (0x66) are added for larger sizes
duke@435	2723
duke@435	2724	while(i >= 22) {
duke@435	2725	i -= 11;
duke@435	2726	emit_byte(0x66); // size prefix
duke@435	2727	emit_byte(0x66); // size prefix
duke@435	2728	emit_byte(0x66); // size prefix
duke@435	2729	addr_nop_8();
duke@435	2730	}
duke@435	2731	// Generate first nop for size between 21-12
duke@435	2732	switch (i) {
duke@435	2733	case 21:
duke@435	2734	i -= 1;
duke@435	2735	emit_byte(0x66); // size prefix
duke@435	2736	case 20:
duke@435	2737	case 19:
duke@435	2738	i -= 1;
duke@435	2739	emit_byte(0x66); // size prefix
duke@435	2740	case 18:
duke@435	2741	case 17:
duke@435	2742	i -= 1;
duke@435	2743	emit_byte(0x66); // size prefix
duke@435	2744	case 16:
duke@435	2745	case 15:
duke@435	2746	i -= 8;
duke@435	2747	addr_nop_8();
duke@435	2748	break;
duke@435	2749	case 14:
duke@435	2750	case 13:
duke@435	2751	i -= 7;
duke@435	2752	addr_nop_7();
duke@435	2753	break;
duke@435	2754	case 12:
duke@435	2755	i -= 6;
duke@435	2756	emit_byte(0x66); // size prefix
duke@435	2757	addr_nop_5();
duke@435	2758	break;
duke@435	2759	default:
duke@435	2760	assert(i < 12, " ");
duke@435	2761	}
duke@435	2762
duke@435	2763	// Generate second nop for size between 11-1
duke@435	2764	switch (i) {
duke@435	2765	case 11:
duke@435	2766	emit_byte(0x66); // size prefix
duke@435	2767	case 10:
duke@435	2768	emit_byte(0x66); // size prefix
duke@435	2769	case 9:
duke@435	2770	emit_byte(0x66); // size prefix
duke@435	2771	case 8:
duke@435	2772	addr_nop_8();
duke@435	2773	break;
duke@435	2774	case 7:
duke@435	2775	addr_nop_7();
duke@435	2776	break;
duke@435	2777	case 6:
duke@435	2778	emit_byte(0x66); // size prefix
duke@435	2779	case 5:
duke@435	2780	addr_nop_5();
duke@435	2781	break;
duke@435	2782	case 4:
duke@435	2783	addr_nop_4();
duke@435	2784	break;
duke@435	2785	case 3:
duke@435	2786	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2787	emit_byte(0x66); // size prefix
duke@435	2788	case 2:
duke@435	2789	emit_byte(0x66); // size prefix
duke@435	2790	case 1:
duke@435	2791	emit_byte(0x90); // nop
duke@435	2792	break;
duke@435	2793	default:
duke@435	2794	assert(i == 0, " ");
duke@435	2795	}
duke@435	2796	return;
duke@435	2797	}
duke@435	2798
duke@435	2799	// Using nops with size prefixes "0x66 0x90".
duke@435	2800	// From AMD Optimization Guide:
duke@435	2801	// 1: 0x90
duke@435	2802	// 2: 0x66 0x90
duke@435	2803	// 3: 0x66 0x66 0x90
duke@435	2804	// 4: 0x66 0x66 0x66 0x90
duke@435	2805	// 5: 0x66 0x66 0x90 0x66 0x90
duke@435	2806	// 6: 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2807	// 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2808	// 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
duke@435	2809	// 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2810	// 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2811	//
duke@435	2812	while(i > 12) {
duke@435	2813	i -= 4;
duke@435	2814	emit_byte(0x66); // size prefix
duke@435	2815	emit_byte(0x66);
duke@435	2816	emit_byte(0x66);
duke@435	2817	emit_byte(0x90); // nop
duke@435	2818	}
duke@435	2819	// 1 - 12 nops
duke@435	2820	if(i > 8) {
duke@435	2821	if(i > 9) {
duke@435	2822	i -= 1;
duke@435	2823	emit_byte(0x66);
duke@435	2824	}
duke@435	2825	i -= 3;
duke@435	2826	emit_byte(0x66);
duke@435	2827	emit_byte(0x66);
duke@435	2828	emit_byte(0x90);
duke@435	2829	}
duke@435	2830	// 1 - 8 nops
duke@435	2831	if(i > 4) {
duke@435	2832	if(i > 6) {
duke@435	2833	i -= 1;
duke@435	2834	emit_byte(0x66);
duke@435	2835	}
duke@435	2836	i -= 3;
duke@435	2837	emit_byte(0x66);
duke@435	2838	emit_byte(0x66);
duke@435	2839	emit_byte(0x90);
duke@435	2840	}
duke@435	2841	switch (i) {
duke@435	2842	case 4:
duke@435	2843	emit_byte(0x66);
duke@435	2844	case 3:
duke@435	2845	emit_byte(0x66);
duke@435	2846	case 2:
duke@435	2847	emit_byte(0x66);
duke@435	2848	case 1:
duke@435	2849	emit_byte(0x90);
duke@435	2850	break;
duke@435	2851	default:
duke@435	2852	assert(i == 0, " ");
duke@435	2853	}
duke@435	2854	}
duke@435	2855
duke@435	2856	void Assembler::ret(int imm16) {
duke@435	2857	if (imm16 == 0) {
duke@435	2858	emit_byte(0xC3);
duke@435	2859	} else {
duke@435	2860	emit_byte(0xC2);
duke@435	2861	emit_word(imm16);
duke@435	2862	}
duke@435	2863	}
duke@435	2864
duke@435	2865	// copies a single word from [esi] to [edi]
duke@435	2866	void Assembler::smovl() {
duke@435	2867	emit_byte(0xA5);
duke@435	2868	}
duke@435	2869
duke@435	2870	// copies data from [rsi] to [rdi] using rcx words (m32)
duke@435	2871	void Assembler::rep_movl() {
duke@435	2872	// REP
duke@435	2873	emit_byte(0xF3);
duke@435	2874	// MOVSL
duke@435	2875	emit_byte(0xA5);
duke@435	2876	}
duke@435	2877
duke@435	2878	// copies data from [rsi] to [rdi] using rcx double words (m64)
duke@435	2879	void Assembler::rep_movq() {
duke@435	2880	// REP
duke@435	2881	emit_byte(0xF3);
duke@435	2882	// MOVSQ
duke@435	2883	prefix(REX_W);
duke@435	2884	emit_byte(0xA5);
duke@435	2885	}
duke@435	2886
duke@435	2887	// sets rcx double words (m64) with rax value at [rdi]
duke@435	2888	void Assembler::rep_set() {
duke@435	2889	// REP
duke@435	2890	emit_byte(0xF3);
duke@435	2891	// STOSQ
duke@435	2892	prefix(REX_W);
duke@435	2893	emit_byte(0xAB);
duke@435	2894	}
duke@435	2895
duke@435	2896	// scans rcx double words (m64) at [rdi] for occurance of rax
coleenp@548	2897	void Assembler::repne_scanq() {
duke@435	2898	// REPNE/REPNZ
duke@435	2899	emit_byte(0xF2);
duke@435	2900	// SCASQ
duke@435	2901	prefix(REX_W);
duke@435	2902	emit_byte(0xAF);
duke@435	2903	}
duke@435	2904
coleenp@548	2905	void Assembler::repne_scanl() {
coleenp@548	2906	// REPNE/REPNZ
coleenp@548	2907	emit_byte(0xF2);
coleenp@548	2908	// SCASL
coleenp@548	2909	emit_byte(0xAF);
coleenp@548	2910	}
coleenp@548	2911
coleenp@548	2912
duke@435	2913	void Assembler::setb(Condition cc, Register dst) {
duke@435	2914	assert(0 <= cc && cc < 16, "illegal cc");
duke@435	2915	int encode = prefix_and_encode(dst->encoding(), true);
duke@435	2916	emit_byte(0x0F);
duke@435	2917	emit_byte(0x90 | cc);
duke@435	2918	emit_byte(0xC0 | encode);
duke@435	2919	}
duke@435	2920
duke@435	2921	void Assembler::clflush(Address adr) {
duke@435	2922	prefix(adr);
duke@435	2923	emit_byte(0x0F);
duke@435	2924	emit_byte(0xAE);
duke@435	2925	emit_operand(rdi, adr);
duke@435	2926	}
duke@435	2927
duke@435	2928	void Assembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	2929	if (L.is_bound()) {
duke@435	2930	const int long_size = 5;
duke@435	2931	int offs = (int)( target(L) - pc() );
duke@435	2932	assert(offs <= 0, "assembler error");
duke@435	2933	InstructionMark im(this);
duke@435	2934	// 1110 1000 #32-bit disp
duke@435	2935	emit_byte(0xE8);
duke@435	2936	emit_data(offs - long_size, rtype, disp32_operand);
duke@435	2937	} else {
duke@435	2938	InstructionMark im(this);
duke@435	2939	// 1110 1000 #32-bit disp
duke@435	2940	L.add_patch_at(code(), locator());
duke@435	2941
duke@435	2942	emit_byte(0xE8);
duke@435	2943	emit_data(int(0), rtype, disp32_operand);
duke@435	2944	}
duke@435	2945	}
duke@435	2946
duke@435	2947	void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
duke@435	2948	assert(entry != NULL, "call most probably wrong");
duke@435	2949	InstructionMark im(this);
duke@435	2950	emit_byte(0xE8);
duke@435	2951	intptr_t disp = entry - (_code_pos + sizeof(int32_t));
duke@435	2952	assert(is_simm32(disp), "must be 32bit offset (call2)");
duke@435	2953	// Technically, should use call32_operand, but this format is
duke@435	2954	// implied by the fact that we're emitting a call instruction.
duke@435	2955	emit_data((int) disp, rspec, disp32_operand);
duke@435	2956	}
duke@435	2957
duke@435	2958
duke@435	2959	void Assembler::call(Register dst) {
duke@435	2960	// This was originally using a 32bit register encoding
duke@435	2961	// and surely we want 64bit!
duke@435	2962	// this is a 32bit encoding but in 64bit mode the default
duke@435	2963	// operand size is 64bit so there is no need for the
duke@435	2964	// wide prefix. So prefix only happens if we use the
duke@435	2965	// new registers. Much like push/pop.
duke@435	2966	int encode = prefixq_and_encode(dst->encoding());
duke@435	2967	emit_byte(0xFF);
duke@435	2968	emit_byte(0xD0 | encode);
duke@435	2969	}
duke@435	2970
duke@435	2971	void Assembler::call(Address adr) {
duke@435	2972	InstructionMark im(this);
duke@435	2973	prefix(adr);
duke@435	2974	emit_byte(0xFF);
duke@435	2975	emit_operand(rdx, adr);
duke@435	2976	}
duke@435	2977
duke@435	2978	void Assembler::jmp(Register reg) {
duke@435	2979	int encode = prefix_and_encode(reg->encoding());
duke@435	2980	emit_byte(0xFF);
duke@435	2981	emit_byte(0xE0 | encode);
duke@435	2982	}
duke@435	2983
duke@435	2984	void Assembler::jmp(Address adr) {
duke@435	2985	InstructionMark im(this);
duke@435	2986	prefix(adr);
duke@435	2987	emit_byte(0xFF);
duke@435	2988	emit_operand(rsp, adr);
duke@435	2989	}
duke@435	2990
duke@435	2991	void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
duke@435	2992	InstructionMark im(this);
duke@435	2993	emit_byte(0xE9);
duke@435	2994	assert(dest != NULL, "must have a target");
duke@435	2995	intptr_t disp = dest - (_code_pos + sizeof(int32_t));
duke@435	2996	assert(is_simm32(disp), "must be 32bit offset (jmp)");
duke@435	2997	emit_data(disp, rspec.reloc(), call32_operand);
duke@435	2998	}
duke@435	2999
duke@435	3000	void Assembler::jmp(Label& L, relocInfo::relocType rtype) {
duke@435	3001	if (L.is_bound()) {
duke@435	3002	address entry = target(L);
duke@435	3003	assert(entry != NULL, "jmp most probably wrong");
duke@435	3004	InstructionMark im(this);
duke@435	3005	const int short_size = 2;
duke@435	3006	const int long_size = 5;
duke@435	3007	intptr_t offs = entry - _code_pos;
duke@435	3008	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3009	emit_byte(0xEB);
duke@435	3010	emit_byte((offs - short_size) & 0xFF);
duke@435	3011	} else {
duke@435	3012	emit_byte(0xE9);
duke@435	3013	emit_long(offs - long_size);
duke@435	3014	}
duke@435	3015	} else {
duke@435	3016	// By default, forward jumps are always 32-bit displacements, since
duke@435	3017	// we can't yet know where the label will be bound. If you're sure that
duke@435	3018	// the forward jump will not run beyond 256 bytes, use jmpb to
duke@435	3019	// force an 8-bit displacement.
duke@435	3020	InstructionMark im(this);
duke@435	3021	relocate(rtype);
duke@435	3022	L.add_patch_at(code(), locator());
duke@435	3023	emit_byte(0xE9);
duke@435	3024	emit_long(0);
duke@435	3025	}
duke@435	3026	}
duke@435	3027
duke@435	3028	void Assembler::jmpb(Label& L) {
duke@435	3029	if (L.is_bound()) {
duke@435	3030	const int short_size = 2;
duke@435	3031	address entry = target(L);
duke@435	3032	assert(is8bit((entry - _code_pos) + short_size),
duke@435	3033	"Dispacement too large for a short jmp");
duke@435	3034	assert(entry != NULL, "jmp most probably wrong");
duke@435	3035	intptr_t offs = entry - _code_pos;
duke@435	3036	emit_byte(0xEB);
duke@435	3037	emit_byte((offs - short_size) & 0xFF);
duke@435	3038	} else {
duke@435	3039	InstructionMark im(this);
duke@435	3040	L.add_patch_at(code(), locator());
duke@435	3041	emit_byte(0xEB);
duke@435	3042	emit_byte(0);
duke@435	3043	}
duke@435	3044	}
duke@435	3045
duke@435	3046	void Assembler::jcc(Condition cc, Label& L, relocInfo::relocType rtype) {
duke@435	3047	InstructionMark im(this);
duke@435	3048	relocate(rtype);
duke@435	3049	assert((0 <= cc) && (cc < 16), "illegal cc");
duke@435	3050	if (L.is_bound()) {
duke@435	3051	address dst = target(L);
duke@435	3052	assert(dst != NULL, "jcc most probably wrong");
duke@435	3053
duke@435	3054	const int short_size = 2;
duke@435	3055	const int long_size = 6;
duke@435	3056	intptr_t offs = (intptr_t)dst - (intptr_t)_code_pos;
duke@435	3057	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3058	// 0111 tttn #8-bit disp
duke@435	3059	emit_byte(0x70 | cc);
duke@435	3060	emit_byte((offs - short_size) & 0xFF);
duke@435	3061	} else {
duke@435	3062	// 0000 1111 1000 tttn #32-bit disp
duke@435	3063	assert(is_simm32(offs - long_size),
duke@435	3064	"must be 32bit offset (call4)");
duke@435	3065	emit_byte(0x0F);
duke@435	3066	emit_byte(0x80 | cc);
duke@435	3067	emit_long(offs - long_size);
duke@435	3068	}
duke@435	3069	} else {
duke@435	3070	// Note: could eliminate cond. jumps to this jump if condition
duke@435	3071	// is the same however, seems to be rather unlikely case.
duke@435	3072	// Note: use jccb() if label to be bound is very close to get
duke@435	3073	// an 8-bit displacement
duke@435	3074	L.add_patch_at(code(), locator());
duke@435	3075	emit_byte(0x0F);
duke@435	3076	emit_byte(0x80 | cc);
duke@435	3077	emit_long(0);
duke@435	3078	}
duke@435	3079	}
duke@435	3080
duke@435	3081	void Assembler::jccb(Condition cc, Label& L) {
duke@435	3082	if (L.is_bound()) {
duke@435	3083	const int short_size = 2;
duke@435	3084	const int long_size = 6;
duke@435	3085	address entry = target(L);
duke@435	3086	assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),
duke@435	3087	"Dispacement too large for a short jmp");
duke@435	3088	intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;
duke@435	3089	// 0111 tttn #8-bit disp
duke@435	3090	emit_byte(0x70 | cc);
duke@435	3091	emit_byte((offs - short_size) & 0xFF);
duke@435	3092	} else {
duke@435	3093	InstructionMark im(this);
duke@435	3094	L.add_patch_at(code(), locator());
duke@435	3095	emit_byte(0x70 | cc);
duke@435	3096	emit_byte(0);
duke@435	3097	}
duke@435	3098	}
duke@435	3099
duke@435	3100	// FP instructions
duke@435	3101
duke@435	3102	void Assembler::fxsave(Address dst) {
duke@435	3103	prefixq(dst);
duke@435	3104	emit_byte(0x0F);
duke@435	3105	emit_byte(0xAE);
duke@435	3106	emit_operand(as_Register(0), dst);
duke@435	3107	}
duke@435	3108
duke@435	3109	void Assembler::fxrstor(Address src) {
duke@435	3110	prefixq(src);
duke@435	3111	emit_byte(0x0F);
duke@435	3112	emit_byte(0xAE);
duke@435	3113	emit_operand(as_Register(1), src);
duke@435	3114	}
duke@435	3115
duke@435	3116	void Assembler::ldmxcsr(Address src) {
duke@435	3117	InstructionMark im(this);
duke@435	3118	prefix(src);
duke@435	3119	emit_byte(0x0F);
duke@435	3120	emit_byte(0xAE);
duke@435	3121	emit_operand(as_Register(2), src);
duke@435	3122	}
duke@435	3123
duke@435	3124	void Assembler::stmxcsr(Address dst) {
duke@435	3125	InstructionMark im(this);
duke@435	3126	prefix(dst);
duke@435	3127	emit_byte(0x0F);
duke@435	3128	emit_byte(0xAE);
duke@435	3129	emit_operand(as_Register(3), dst);
duke@435	3130	}
duke@435	3131
duke@435	3132	void Assembler::addss(XMMRegister dst, XMMRegister src) {
duke@435	3133	emit_byte(0xF3);
duke@435	3134	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3135	emit_byte(0x0F);
duke@435	3136	emit_byte(0x58);
duke@435	3137	emit_byte(0xC0 | encode);
duke@435	3138	}
duke@435	3139
duke@435	3140	void Assembler::addss(XMMRegister dst, Address src) {
duke@435	3141	InstructionMark im(this);
duke@435	3142	emit_byte(0xF3);
duke@435	3143	prefix(src, dst);
duke@435	3144	emit_byte(0x0F);
duke@435	3145	emit_byte(0x58);
duke@435	3146	emit_operand(dst, src);
duke@435	3147	}
duke@435	3148
duke@435	3149	void Assembler::subss(XMMRegister dst, XMMRegister src) {
duke@435	3150	emit_byte(0xF3);
duke@435	3151	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3152	emit_byte(0x0F);
duke@435	3153	emit_byte(0x5C);
duke@435	3154	emit_byte(0xC0 | encode);
duke@435	3155	}
duke@435	3156
duke@435	3157	void Assembler::subss(XMMRegister dst, Address src) {
duke@435	3158	InstructionMark im(this);
duke@435	3159	emit_byte(0xF3);
duke@435	3160	prefix(src, dst);
duke@435	3161	emit_byte(0x0F);
duke@435	3162	emit_byte(0x5C);
duke@435	3163	emit_operand(dst, src);
duke@435	3164	}
duke@435	3165
duke@435	3166	void Assembler::mulss(XMMRegister dst, XMMRegister src) {
duke@435	3167	emit_byte(0xF3);
duke@435	3168	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3169	emit_byte(0x0F);
duke@435	3170	emit_byte(0x59);
duke@435	3171	emit_byte(0xC0 | encode);
duke@435	3172	}
duke@435	3173
duke@435	3174	void Assembler::mulss(XMMRegister dst, Address src) {
duke@435	3175	InstructionMark im(this);
duke@435	3176	emit_byte(0xF3);
duke@435	3177	prefix(src, dst);
duke@435	3178	emit_byte(0x0F);
duke@435	3179	emit_byte(0x59);
duke@435	3180	emit_operand(dst, src);
duke@435	3181	}
duke@435	3182
duke@435	3183	void Assembler::divss(XMMRegister dst, XMMRegister src) {
duke@435	3184	emit_byte(0xF3);
duke@435	3185	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3186	emit_byte(0x0F);
duke@435	3187	emit_byte(0x5E);
duke@435	3188	emit_byte(0xC0 | encode);
duke@435	3189	}
duke@435	3190
duke@435	3191	void Assembler::divss(XMMRegister dst, Address src) {
duke@435	3192	InstructionMark im(this);
duke@435	3193	emit_byte(0xF3);
duke@435	3194	prefix(src, dst);
duke@435	3195	emit_byte(0x0F);
duke@435	3196	emit_byte(0x5E);
duke@435	3197	emit_operand(dst, src);
duke@435	3198	}
duke@435	3199
duke@435	3200	void Assembler::addsd(XMMRegister dst, XMMRegister src) {
duke@435	3201	emit_byte(0xF2);
duke@435	3202	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3203	emit_byte(0x0F);
duke@435	3204	emit_byte(0x58);
duke@435	3205	emit_byte(0xC0 | encode);
duke@435	3206	}
duke@435	3207
duke@435	3208	void Assembler::addsd(XMMRegister dst, Address src) {
duke@435	3209	InstructionMark im(this);
duke@435	3210	emit_byte(0xF2);
duke@435	3211	prefix(src, dst);
duke@435	3212	emit_byte(0x0F);
duke@435	3213	emit_byte(0x58);
duke@435	3214	emit_operand(dst, src);
duke@435	3215	}
duke@435	3216
duke@435	3217	void Assembler::subsd(XMMRegister dst, XMMRegister src) {
duke@435	3218	emit_byte(0xF2);
duke@435	3219	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3220	emit_byte(0x0F);
duke@435	3221	emit_byte(0x5C);
duke@435	3222	emit_byte(0xC0 | encode);
duke@435	3223	}
duke@435	3224
duke@435	3225	void Assembler::subsd(XMMRegister dst, Address src) {
duke@435	3226	InstructionMark im(this);
duke@435	3227	emit_byte(0xF2);
duke@435	3228	prefix(src, dst);
duke@435	3229	emit_byte(0x0F);
duke@435	3230	emit_byte(0x5C);
duke@435	3231	emit_operand(dst, src);
duke@435	3232	}
duke@435	3233
duke@435	3234	void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
duke@435	3235	emit_byte(0xF2);
duke@435	3236	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3237	emit_byte(0x0F);
duke@435	3238	emit_byte(0x59);
duke@435	3239	emit_byte(0xC0 | encode);
duke@435	3240	}
duke@435	3241
duke@435	3242	void Assembler::mulsd(XMMRegister dst, Address src) {
duke@435	3243	InstructionMark im(this);
duke@435	3244	emit_byte(0xF2);
duke@435	3245	prefix(src, dst);
duke@435	3246	emit_byte(0x0F);
duke@435	3247	emit_byte(0x59);
duke@435	3248	emit_operand(dst, src);
duke@435	3249	}
duke@435	3250
duke@435	3251	void Assembler::divsd(XMMRegister dst, XMMRegister src) {
duke@435	3252	emit_byte(0xF2);
duke@435	3253	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3254	emit_byte(0x0F);
duke@435	3255	emit_byte(0x5E);
duke@435	3256	emit_byte(0xC0 | encode);
duke@435	3257	}
duke@435	3258
duke@435	3259	void Assembler::divsd(XMMRegister dst, Address src) {
duke@435	3260	InstructionMark im(this);
duke@435	3261	emit_byte(0xF2);
duke@435	3262	prefix(src, dst);
duke@435	3263	emit_byte(0x0F);
duke@435	3264	emit_byte(0x5E);
duke@435	3265	emit_operand(dst, src);
duke@435	3266	}
duke@435	3267
duke@435	3268	void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
duke@435	3269	emit_byte(0xF2);
duke@435	3270	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3271	emit_byte(0x0F);
duke@435	3272	emit_byte(0x51);
duke@435	3273	emit_byte(0xC0 | encode);
duke@435	3274	}
duke@435	3275
duke@435	3276	void Assembler::sqrtsd(XMMRegister dst, Address src) {
duke@435	3277	InstructionMark im(this);
duke@435	3278	emit_byte(0xF2);
duke@435	3279	prefix(src, dst);
duke@435	3280	emit_byte(0x0F);
duke@435	3281	emit_byte(0x51);
duke@435	3282	emit_operand(dst, src);
duke@435	3283	}
duke@435	3284
duke@435	3285	void Assembler::xorps(XMMRegister dst, XMMRegister src) {
duke@435	3286	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3287	emit_byte(0x0F);
duke@435	3288	emit_byte(0x57);
duke@435	3289	emit_byte(0xC0 | encode);
duke@435	3290	}
duke@435	3291
duke@435	3292	void Assembler::xorps(XMMRegister dst, Address src) {
duke@435	3293	InstructionMark im(this);
duke@435	3294	prefix(src, dst);
duke@435	3295	emit_byte(0x0F);
duke@435	3296	emit_byte(0x57);
duke@435	3297	emit_operand(dst, src);
duke@435	3298	}
duke@435	3299
duke@435	3300	void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
duke@435	3301	emit_byte(0x66);
duke@435	3302	xorps(dst, src);
duke@435	3303	}
duke@435	3304
duke@435	3305	void Assembler::xorpd(XMMRegister dst, Address src) {
duke@435	3306	InstructionMark im(this);
duke@435	3307	emit_byte(0x66);
duke@435	3308	prefix(src, dst);
duke@435	3309	emit_byte(0x0F);
duke@435	3310	emit_byte(0x57);
duke@435	3311	emit_operand(dst, src);
duke@435	3312	}
duke@435	3313
duke@435	3314	void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
duke@435	3315	emit_byte(0xF3);
duke@435	3316	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3317	emit_byte(0x0F);
duke@435	3318	emit_byte(0x2A);
duke@435	3319	emit_byte(0xC0 | encode);
duke@435	3320	}
duke@435	3321
duke@435	3322	void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
duke@435	3323	emit_byte(0xF3);
duke@435	3324	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3325	emit_byte(0x0F);
duke@435	3326	emit_byte(0x2A);
duke@435	3327	emit_byte(0xC0 | encode);
duke@435	3328	}
duke@435	3329
duke@435	3330	void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
duke@435	3331	emit_byte(0xF2);
duke@435	3332	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3333	emit_byte(0x0F);
duke@435	3334	emit_byte(0x2A);
duke@435	3335	emit_byte(0xC0 | encode);
duke@435	3336	}
duke@435	3337
duke@435	3338	void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
duke@435	3339	emit_byte(0xF2);
duke@435	3340	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3341	emit_byte(0x0F);
duke@435	3342	emit_byte(0x2A);
duke@435	3343	emit_byte(0xC0 | encode);
duke@435	3344	}
duke@435	3345
duke@435	3346	void Assembler::cvttss2sil(Register dst, XMMRegister src) {
duke@435	3347	emit_byte(0xF3);
duke@435	3348	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3349	emit_byte(0x0F);
duke@435	3350	emit_byte(0x2C);
duke@435	3351	emit_byte(0xC0 | encode);
duke@435	3352	}
duke@435	3353
duke@435	3354	void Assembler::cvttss2siq(Register dst, XMMRegister src) {
duke@435	3355	emit_byte(0xF3);
duke@435	3356	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3357	emit_byte(0x0F);
duke@435	3358	emit_byte(0x2C);
duke@435	3359	emit_byte(0xC0 | encode);
duke@435	3360	}
duke@435	3361
duke@435	3362	void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
duke@435	3363	emit_byte(0xF2);
duke@435	3364	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3365	emit_byte(0x0F);
duke@435	3366	emit_byte(0x2C);
duke@435	3367	emit_byte(0xC0 | encode);
duke@435	3368	}
duke@435	3369
duke@435	3370	void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
duke@435	3371	emit_byte(0xF2);
duke@435	3372	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3373	emit_byte(0x0F);
duke@435	3374	emit_byte(0x2C);
duke@435	3375	emit_byte(0xC0 | encode);
duke@435	3376	}
duke@435	3377
duke@435	3378	void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
duke@435	3379	emit_byte(0xF3);
duke@435	3380	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3381	emit_byte(0x0F);
duke@435	3382	emit_byte(0x5A);
duke@435	3383	emit_byte(0xC0 | encode);
duke@435	3384	}
duke@435	3385
kvn@506	3386	void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
kvn@506	3387	emit_byte(0xF3);
kvn@506	3388	int encode = prefix_and_encode(dst->encoding(), src->encoding());
kvn@506	3389	emit_byte(0x0F);
kvn@506	3390	emit_byte(0xE6);
kvn@506	3391	emit_byte(0xC0 | encode);
kvn@506	3392	}
kvn@506	3393
kvn@506	3394	void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
kvn@506	3395	int encode = prefix_and_encode(dst->encoding(), src->encoding());
kvn@506	3396	emit_byte(0x0F);
kvn@506	3397	emit_byte(0x5B);
kvn@506	3398	emit_byte(0xC0 | encode);
kvn@506	3399	}
kvn@506	3400
duke@435	3401	void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
duke@435	3402	emit_byte(0xF2);
duke@435	3403	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3404	emit_byte(0x0F);
duke@435	3405	emit_byte(0x5A);
duke@435	3406	emit_byte(0xC0 | encode);
duke@435	3407	}
duke@435	3408
duke@435	3409	void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
duke@435	3410	emit_byte(0x66);
duke@435	3411	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3412	emit_byte(0x0F);
duke@435	3413	emit_byte(0x60);
duke@435	3414	emit_byte(0xC0 | encode);
duke@435	3415	}
duke@435	3416
duke@435	3417	// Implementation of MacroAssembler
duke@435	3418
duke@435	3419	// On 32 bit it returns a vanilla displacement on 64 bit is a rip relative displacement
duke@435	3420	Address MacroAssembler::as_Address(AddressLiteral adr) {
duke@435	3421	assert(!adr.is_lval(), "must be rval");
duke@435	3422	assert(reachable(adr), "must be");
duke@435	3423	return Address((int)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
duke@435	3424	}
duke@435	3425
duke@435	3426	Address MacroAssembler::as_Address(ArrayAddress adr) {
duke@435	3427	#ifdef _LP64
duke@435	3428	AddressLiteral base = adr.base();
duke@435	3429	lea(rscratch1, base);
duke@435	3430	Address index = adr.index();
duke@435	3431	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	3432	Address array(rscratch1, index._index, index._scale, index._disp);
duke@435	3433	return array;
duke@435	3434	#else
duke@435	3435	return Address::make_array(adr);
duke@435	3436	#endif // _LP64
duke@435	3437
duke@435	3438	}
duke@435	3439
duke@435	3440	void MacroAssembler::fat_nop() {
duke@435	3441	// A 5 byte nop that is safe for patching (see patch_verified_entry)
duke@435	3442	// Recommened sequence from 'Software Optimization Guide for the AMD
duke@435	3443	// Hammer Processor'
duke@435	3444	emit_byte(0x66);
duke@435	3445	emit_byte(0x66);
duke@435	3446	emit_byte(0x90);
duke@435	3447	emit_byte(0x66);
duke@435	3448	emit_byte(0x90);
duke@435	3449	}
duke@435	3450
duke@435	3451	static Assembler::Condition reverse[] = {
duke@435	3452	Assembler::noOverflow /* overflow = 0x0 */ ,
duke@435	3453	Assembler::overflow /* noOverflow = 0x1 */ ,
duke@435	3454	Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
duke@435	3455	Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
duke@435	3456	Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
duke@435	3457	Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ ,
duke@435	3458	Assembler::above /* belowEqual = 0x6 */ ,
duke@435	3459	Assembler::belowEqual /* above = 0x7 */ ,
duke@435	3460	Assembler::positive /* negative = 0x8 */ ,
duke@435	3461	Assembler::negative /* positive = 0x9 */ ,
duke@435	3462	Assembler::noParity /* parity = 0xa */ ,
duke@435	3463	Assembler::parity /* noParity = 0xb */ ,
duke@435	3464	Assembler::greaterEqual /* less = 0xc */ ,
duke@435	3465	Assembler::less /* greaterEqual = 0xd */ ,
duke@435	3466	Assembler::greater /* lessEqual = 0xe */ ,
duke@435	3467	Assembler::lessEqual /* greater = 0xf, */
duke@435	3468
duke@435	3469	};
duke@435	3470
duke@435	3471	// 32bit can do a case table jump in one instruction but we no longer allow the base
duke@435	3472	// to be installed in the Address class
duke@435	3473	void MacroAssembler::jump(ArrayAddress entry) {
duke@435	3474	#ifdef _LP64
duke@435	3475	lea(rscratch1, entry.base());
duke@435	3476	Address dispatch = entry.index();
duke@435	3477	assert(dispatch._base == noreg, "must be");
duke@435	3478	dispatch._base = rscratch1;
duke@435	3479	jmp(dispatch);
duke@435	3480	#else
duke@435	3481	jmp(as_Address(entry));
duke@435	3482	#endif // _LP64
duke@435	3483	}
duke@435	3484
duke@435	3485	void MacroAssembler::jump(AddressLiteral dst) {
duke@435	3486	if (reachable(dst)) {
duke@435	3487	jmp_literal(dst.target(), dst.rspec());
duke@435	3488	} else {
duke@435	3489	lea(rscratch1, dst);
duke@435	3490	jmp(rscratch1);
duke@435	3491	}
duke@435	3492	}
duke@435	3493
duke@435	3494	void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
duke@435	3495	if (reachable(dst)) {
duke@435	3496	InstructionMark im(this);
duke@435	3497	relocate(dst.reloc());
duke@435	3498	const int short_size = 2;
duke@435	3499	const int long_size = 6;
duke@435	3500	int offs = (intptr_t)dst.target() - ((intptr_t)_code_pos);
duke@435	3501	if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3502	// 0111 tttn #8-bit disp
duke@435	3503	emit_byte(0x70 | cc);
duke@435	3504	emit_byte((offs - short_size) & 0xFF);
duke@435	3505	} else {
duke@435	3506	// 0000 1111 1000 tttn #32-bit disp
duke@435	3507	emit_byte(0x0F);
duke@435	3508	emit_byte(0x80 | cc);
duke@435	3509	emit_long(offs - long_size);
duke@435	3510	}
duke@435	3511	} else {
duke@435	3512	#ifdef ASSERT
duke@435	3513	warning("reversing conditional branch");
duke@435	3514	#endif /* ASSERT */
duke@435	3515	Label skip;
duke@435	3516	jccb(reverse[cc], skip);
duke@435	3517	lea(rscratch1, dst);
duke@435	3518	Assembler::jmp(rscratch1);
duke@435	3519	bind(skip);
duke@435	3520	}
duke@435	3521	}
duke@435	3522
duke@435	3523	// Wouldn't need if AddressLiteral version had new name
duke@435	3524	void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	3525	Assembler::call(L, rtype);
duke@435	3526	}
duke@435	3527
duke@435	3528	// Wouldn't need if AddressLiteral version had new name
duke@435	3529	void MacroAssembler::call(Register entry) {
duke@435	3530	Assembler::call(entry);
duke@435	3531	}
duke@435	3532
duke@435	3533	void MacroAssembler::call(AddressLiteral entry) {
duke@435	3534	if (reachable(entry)) {
duke@435	3535	Assembler::call_literal(entry.target(), entry.rspec());
duke@435	3536	} else {
duke@435	3537	lea(rscratch1, entry);
duke@435	3538	Assembler::call(rscratch1);
duke@435	3539	}
duke@435	3540	}
duke@435	3541
duke@435	3542	void MacroAssembler::cmp8(AddressLiteral src1, int8_t src2) {
duke@435	3543	if (reachable(src1)) {
duke@435	3544	cmpb(as_Address(src1), src2);
duke@435	3545	} else {
duke@435	3546	lea(rscratch1, src1);
duke@435	3547	cmpb(Address(rscratch1, 0), src2);
duke@435	3548	}
duke@435	3549	}
duke@435	3550
duke@435	3551	void MacroAssembler::cmp32(AddressLiteral src1, int32_t src2) {
duke@435	3552	if (reachable(src1)) {
duke@435	3553	cmpl(as_Address(src1), src2);
duke@435	3554	} else {
duke@435	3555	lea(rscratch1, src1);
duke@435	3556	cmpl(Address(rscratch1, 0), src2);
duke@435	3557	}
duke@435	3558	}
duke@435	3559
duke@435	3560	void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
duke@435	3561	if (reachable(src2)) {
duke@435	3562	cmpl(src1, as_Address(src2));
duke@435	3563	} else {
duke@435	3564	lea(rscratch1, src2);
duke@435	3565	cmpl(src1, Address(rscratch1, 0));
duke@435	3566	}
duke@435	3567	}
duke@435	3568
duke@435	3569	void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
duke@435	3570	#ifdef _LP64
duke@435	3571	if (src2.is_lval()) {
duke@435	3572	movptr(rscratch1, src2);
duke@435	3573	Assembler::cmpq(src1, rscratch1);
duke@435	3574	} else if (reachable(src2)) {
duke@435	3575	cmpq(src1, as_Address(src2));
duke@435	3576	} else {
duke@435	3577	lea(rscratch1, src2);
duke@435	3578	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3579	}
duke@435	3580	#else
duke@435	3581	if (src2.is_lval()) {
duke@435	3582	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3583	} else {
duke@435	3584	cmpl(src1, as_Address(src2));
duke@435	3585	}
duke@435	3586	#endif // _LP64
duke@435	3587	}
duke@435	3588
duke@435	3589	void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
duke@435	3590	assert(src2.is_lval(), "not a mem-mem compare");
duke@435	3591	#ifdef _LP64
duke@435	3592	// moves src2's literal address
duke@435	3593	movptr(rscratch1, src2);
duke@435	3594	Assembler::cmpq(src1, rscratch1);
duke@435	3595	#else
duke@435	3596	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3597	#endif // _LP64
duke@435	3598	}
duke@435	3599
duke@435	3600	void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
duke@435	3601	assert(!src2.is_lval(), "should use cmpptr");
duke@435	3602
duke@435	3603	if (reachable(src2)) {
duke@435	3604	#ifdef _LP64
duke@435	3605	cmpq(src1, as_Address(src2));
duke@435	3606	#else
duke@435	3607	ShouldNotReachHere();
duke@435	3608	#endif // _LP64
duke@435	3609	} else {
duke@435	3610	lea(rscratch1, src2);
duke@435	3611	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3612	}
duke@435	3613	}
duke@435	3614
duke@435	3615	void MacroAssembler::cmpxchgptr(Register reg, AddressLiteral adr) {
duke@435	3616	if (reachable(adr)) {
duke@435	3617	#ifdef _LP64
duke@435	3618	cmpxchgq(reg, as_Address(adr));
duke@435	3619	#else
duke@435	3620	cmpxchgl(reg, as_Address(adr));
duke@435	3621	#endif // _LP64
duke@435	3622	} else {
duke@435	3623	lea(rscratch1, adr);
duke@435	3624	cmpxchgq(reg, Address(rscratch1, 0));
duke@435	3625	}
duke@435	3626	}
duke@435	3627
duke@435	3628	void MacroAssembler::incrementl(AddressLiteral dst) {
duke@435	3629	if (reachable(dst)) {
duke@435	3630	incrementl(as_Address(dst));
duke@435	3631	} else {
duke@435	3632	lea(rscratch1, dst);
duke@435	3633	incrementl(Address(rscratch1, 0));
duke@435	3634	}
duke@435	3635	}
duke@435	3636
duke@435	3637	void MacroAssembler::incrementl(ArrayAddress dst) {
duke@435	3638	incrementl(as_Address(dst));
duke@435	3639	}
duke@435	3640
duke@435	3641	void MacroAssembler::lea(Register dst, Address src) {
duke@435	3642	#ifdef _LP64
duke@435	3643	leaq(dst, src);
duke@435	3644	#else
duke@435	3645	leal(dst, src);
duke@435	3646	#endif // _LP64
duke@435	3647	}
duke@435	3648
duke@435	3649	void MacroAssembler::lea(Register dst, AddressLiteral src) {
duke@435	3650	#ifdef _LP64
duke@435	3651	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3652	#else
duke@435	3653	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3654	#endif // _LP64
duke@435	3655	}
duke@435	3656
duke@435	3657	void MacroAssembler::mov32(AddressLiteral dst, Register src) {
duke@435	3658	if (reachable(dst)) {
duke@435	3659	movl(as_Address(dst), src);
duke@435	3660	} else {
duke@435	3661	lea(rscratch1, dst);
duke@435	3662	movl(Address(rscratch1, 0), src);
duke@435	3663	}
duke@435	3664	}
duke@435	3665
duke@435	3666	void MacroAssembler::mov32(Register dst, AddressLiteral src) {
duke@435	3667	if (reachable(src)) {
duke@435	3668	movl(dst, as_Address(src));
duke@435	3669	} else {
duke@435	3670	lea(rscratch1, src);
duke@435	3671	movl(dst, Address(rscratch1, 0));
duke@435	3672	}
duke@435	3673	}
duke@435	3674
duke@435	3675	void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
duke@435	3676	if (reachable(src)) {
duke@435	3677	if (UseXmmLoadAndClearUpper) {
duke@435	3678	movsd (dst, as_Address(src));
duke@435	3679	} else {
duke@435	3680	movlpd(dst, as_Address(src));
duke@435	3681	}
duke@435	3682	} else {
duke@435	3683	lea(rscratch1, src);
duke@435	3684	if (UseXmmLoadAndClearUpper) {
duke@435	3685	movsd (dst, Address(rscratch1, 0));
duke@435	3686	} else {
duke@435	3687	movlpd(dst, Address(rscratch1, 0));
duke@435	3688	}
duke@435	3689	}
duke@435	3690	}
duke@435	3691
duke@435	3692	void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
duke@435	3693	if (reachable(src)) {
duke@435	3694	movss(dst, as_Address(src));
duke@435	3695	} else {
duke@435	3696	lea(rscratch1, src);
duke@435	3697	movss(dst, Address(rscratch1, 0));
duke@435	3698	}
duke@435	3699	}
duke@435	3700
duke@435	3701	void MacroAssembler::movoop(Register dst, jobject obj) {
duke@435	3702	mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3703	}
duke@435	3704
duke@435	3705	void MacroAssembler::movoop(Address dst, jobject obj) {
duke@435	3706	mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3707	movq(dst, rscratch1);
duke@435	3708	}
duke@435	3709
duke@435	3710	void MacroAssembler::movptr(Register dst, AddressLiteral src) {
duke@435	3711	#ifdef _LP64
duke@435	3712	if (src.is_lval()) {
duke@435	3713	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3714	} else {
duke@435	3715	if (reachable(src)) {
duke@435	3716	movq(dst, as_Address(src));
duke@435	3717	} else {
duke@435	3718	lea(rscratch1, src);
duke@435	3719	movq(dst, Address(rscratch1,0));
duke@435	3720	}
duke@435	3721	}
duke@435	3722	#else
duke@435	3723	if (src.is_lval()) {
duke@435	3724	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3725	} else {
duke@435	3726	movl(dst, as_Address(src));
duke@435	3727	}
duke@435	3728	#endif // LP64
duke@435	3729	}
duke@435	3730
duke@435	3731	void MacroAssembler::movptr(ArrayAddress dst, Register src) {
duke@435	3732	#ifdef _LP64
duke@435	3733	movq(as_Address(dst), src);
duke@435	3734	#else
duke@435	3735	movl(as_Address(dst), src);
duke@435	3736	#endif // _LP64
duke@435	3737	}
duke@435	3738
duke@435	3739	void MacroAssembler::pushoop(jobject obj) {
duke@435	3740	#ifdef _LP64
duke@435	3741	movoop(rscratch1, obj);
duke@435	3742	pushq(rscratch1);
duke@435	3743	#else
duke@435	3744	push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3745	#endif // _LP64
duke@435	3746	}
duke@435	3747
duke@435	3748	void MacroAssembler::pushptr(AddressLiteral src) {
duke@435	3749	#ifdef _LP64
duke@435	3750	lea(rscratch1, src);
duke@435	3751	if (src.is_lval()) {
duke@435	3752	pushq(rscratch1);
duke@435	3753	} else {
duke@435	3754	pushq(Address(rscratch1, 0));
duke@435	3755	}
duke@435	3756	#else
duke@435	3757	if (src.is_lval()) {
duke@435	3758	push_literal((int32_t)src.target(), src.rspec());
duke@435	3759	else {
duke@435	3760	pushl(as_Address(src));
duke@435	3761	}
duke@435	3762	#endif // _LP64
duke@435	3763	}
duke@435	3764
duke@435	3765	void MacroAssembler::ldmxcsr(AddressLiteral src) {
duke@435	3766	if (reachable(src)) {
duke@435	3767	Assembler::ldmxcsr(as_Address(src));
duke@435	3768	} else {
duke@435	3769	lea(rscratch1, src);
duke@435	3770	Assembler::ldmxcsr(Address(rscratch1, 0));
duke@435	3771	}
duke@435	3772	}
duke@435	3773
duke@435	3774	void MacroAssembler::movlpd(XMMRegister dst, AddressLiteral src) {
duke@435	3775	if (reachable(src)) {
duke@435	3776	movlpd(dst, as_Address(src));
duke@435	3777	} else {
duke@435	3778	lea(rscratch1, src);
duke@435	3779	movlpd(dst, Address(rscratch1, 0));
duke@435	3780	}
duke@435	3781	}
duke@435	3782
duke@435	3783	void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
duke@435	3784	if (reachable(src)) {
duke@435	3785	movss(dst, as_Address(src));
duke@435	3786	} else {
duke@435	3787	lea(rscratch1, src);
duke@435	3788	movss(dst, Address(rscratch1, 0));
duke@435	3789	}
duke@435	3790	}
duke@435	3791	void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
duke@435	3792	if (reachable(src)) {
duke@435	3793	xorpd(dst, as_Address(src));
duke@435	3794	} else {
duke@435	3795	lea(rscratch1, src);
duke@435	3796	xorpd(dst, Address(rscratch1, 0));
duke@435	3797	}
duke@435	3798	}
duke@435	3799
duke@435	3800	void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
duke@435	3801	if (reachable(src)) {
duke@435	3802	xorps(dst, as_Address(src));
duke@435	3803	} else {
duke@435	3804	lea(rscratch1, src);
duke@435	3805	xorps(dst, Address(rscratch1, 0));
duke@435	3806	}
duke@435	3807	}
duke@435	3808
duke@435	3809	void MacroAssembler::null_check(Register reg, int offset) {
duke@435	3810	if (needs_explicit_null_check(offset)) {
duke@435	3811	// provoke OS NULL exception if reg = NULL by
duke@435	3812	// accessing M[reg] w/o changing any (non-CC) registers
duke@435	3813	cmpq(rax, Address(reg, 0));
duke@435	3814	// Note: should probably use testl(rax, Address(reg, 0));
duke@435	3815	// may be shorter code (however, this version of
duke@435	3816	// testl needs to be implemented first)
duke@435	3817	} else {
duke@435	3818	// nothing to do, (later) access of M[reg + offset]
duke@435	3819	// will provoke OS NULL exception if reg = NULL
duke@435	3820	}
duke@435	3821	}
duke@435	3822
duke@435	3823	int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
duke@435	3824	int off = offset();
duke@435	3825	movzbl(dst, src);
duke@435	3826	return off;
duke@435	3827	}
duke@435	3828
duke@435	3829	int MacroAssembler::load_unsigned_word(Register dst, Address src) {
duke@435	3830	int off = offset();
duke@435	3831	movzwl(dst, src);
duke@435	3832	return off;
duke@435	3833	}
duke@435	3834
duke@435	3835	int MacroAssembler::load_signed_byte(Register dst, Address src) {
duke@435	3836	int off = offset();
duke@435	3837	movsbl(dst, src);
duke@435	3838	return off;
duke@435	3839	}
duke@435	3840
duke@435	3841	int MacroAssembler::load_signed_word(Register dst, Address src) {
duke@435	3842	int off = offset();
duke@435	3843	movswl(dst, src);
duke@435	3844	return off;
duke@435	3845	}
duke@435	3846
duke@435	3847	void MacroAssembler::incrementl(Register reg, int value) {
duke@435	3848	if (value == min_jint) { addl(reg, value); return; }
duke@435	3849	if (value < 0) { decrementl(reg, -value); return; }
duke@435	3850	if (value == 0) { ; return; }
duke@435	3851	if (value == 1 && UseIncDec) { incl(reg) ; return; }
duke@435	3852	/* else */ { addl(reg, value) ; return; }
duke@435	3853	}
duke@435	3854
duke@435	3855	void MacroAssembler::decrementl(Register reg, int value) {
duke@435	3856	if (value == min_jint) { subl(reg, value); return; }
duke@435	3857	if (value < 0) { incrementl(reg, -value); return; }
duke@435	3858	if (value == 0) { ; return; }
duke@435	3859	if (value == 1 && UseIncDec) { decl(reg) ; return; }
duke@435	3860	/* else */ { subl(reg, value) ; return; }
duke@435	3861	}
duke@435	3862
duke@435	3863	void MacroAssembler::incrementq(Register reg, int value) {
duke@435	3864	if (value == min_jint) { addq(reg, value); return; }
duke@435	3865	if (value < 0) { decrementq(reg, -value); return; }
duke@435	3866	if (value == 0) { ; return; }
duke@435	3867	if (value == 1 && UseIncDec) { incq(reg) ; return; }
duke@435	3868	/* else */ { addq(reg, value) ; return; }
duke@435	3869	}
duke@435	3870
duke@435	3871	void MacroAssembler::decrementq(Register reg, int value) {
duke@435	3872	if (value == min_jint) { subq(reg, value); return; }
duke@435	3873	if (value < 0) { incrementq(reg, -value); return; }
duke@435	3874	if (value == 0) { ; return; }
duke@435	3875	if (value == 1 && UseIncDec) { decq(reg) ; return; }
duke@435	3876	/* else */ { subq(reg, value) ; return; }
duke@435	3877	}
duke@435	3878
duke@435	3879	void MacroAssembler::incrementl(Address dst, int value) {
duke@435	3880	if (value == min_jint) { addl(dst, value); return; }
duke@435	3881	if (value < 0) { decrementl(dst, -value); return; }
duke@435	3882	if (value == 0) { ; return; }
duke@435	3883	if (value == 1 && UseIncDec) { incl(dst) ; return; }
duke@435	3884	/* else */ { addl(dst, value) ; return; }
duke@435	3885	}
duke@435	3886
duke@435	3887	void MacroAssembler::decrementl(Address dst, int value) {
duke@435	3888	if (value == min_jint) { subl(dst, value); return; }
duke@435	3889	if (value < 0) { incrementl(dst, -value); return; }
duke@435	3890	if (value == 0) { ; return; }
duke@435	3891	if (value == 1 && UseIncDec) { decl(dst) ; return; }
duke@435	3892	/* else */ { subl(dst, value) ; return; }
duke@435	3893	}
duke@435	3894
duke@435	3895	void MacroAssembler::incrementq(Address dst, int value) {
duke@435	3896	if (value == min_jint) { addq(dst, value); return; }
duke@435	3897	if (value < 0) { decrementq(dst, -value); return; }
duke@435	3898	if (value == 0) { ; return; }
duke@435	3899	if (value == 1 && UseIncDec) { incq(dst) ; return; }
duke@435	3900	/* else */ { addq(dst, value) ; return; }
duke@435	3901	}
duke@435	3902
duke@435	3903	void MacroAssembler::decrementq(Address dst, int value) {
duke@435	3904	if (value == min_jint) { subq(dst, value); return; }
duke@435	3905	if (value < 0) { incrementq(dst, -value); return; }
duke@435	3906	if (value == 0) { ; return; }
duke@435	3907	if (value == 1 && UseIncDec) { decq(dst) ; return; }
duke@435	3908	/* else */ { subq(dst, value) ; return; }
duke@435	3909	}
duke@435	3910
duke@435	3911	void MacroAssembler::align(int modulus) {
duke@435	3912	if (offset() % modulus != 0) {
duke@435	3913	nop(modulus - (offset() % modulus));
duke@435	3914	}
duke@435	3915	}
duke@435	3916
duke@435	3917	void MacroAssembler::enter() {
duke@435	3918	pushq(rbp);
duke@435	3919	movq(rbp, rsp);
duke@435	3920	}
duke@435	3921
duke@435	3922	void MacroAssembler::leave() {
duke@435	3923	emit_byte(0xC9); // LEAVE
duke@435	3924	}
duke@435	3925
duke@435	3926	// C++ bool manipulation
duke@435	3927
duke@435	3928	void MacroAssembler::movbool(Register dst, Address src) {
duke@435	3929	if(sizeof(bool) == 1)
duke@435	3930	movb(dst, src);
duke@435	3931	else if(sizeof(bool) == 2)
duke@435	3932	movw(dst, src);
duke@435	3933	else if(sizeof(bool) == 4)
duke@435	3934	movl(dst, src);
duke@435	3935	else {
duke@435	3936	// unsupported
duke@435	3937	ShouldNotReachHere();
duke@435	3938	}
duke@435	3939	}
duke@435	3940
duke@435	3941	void MacroAssembler::movbool(Address dst, bool boolconst) {
duke@435	3942	if(sizeof(bool) == 1)
duke@435	3943	movb(dst, (int) boolconst);
duke@435	3944	else if(sizeof(bool) == 2)
duke@435	3945	movw(dst, (int) boolconst);
duke@435	3946	else if(sizeof(bool) == 4)
duke@435	3947	movl(dst, (int) boolconst);
duke@435	3948	else {
duke@435	3949	// unsupported
duke@435	3950	ShouldNotReachHere();
duke@435	3951	}
duke@435	3952	}
duke@435	3953
duke@435	3954	void MacroAssembler::movbool(Address dst, Register src) {
duke@435	3955	if(sizeof(bool) == 1)
duke@435	3956	movb(dst, src);
duke@435	3957	else if(sizeof(bool) == 2)
duke@435	3958	movw(dst, src);
duke@435	3959	else if(sizeof(bool) == 4)
duke@435	3960	movl(dst, src);
duke@435	3961	else {
duke@435	3962	// unsupported
duke@435	3963	ShouldNotReachHere();
duke@435	3964	}
duke@435	3965	}
duke@435	3966
duke@435	3967	void MacroAssembler::testbool(Register dst) {
duke@435	3968	if(sizeof(bool) == 1)
duke@435	3969	testb(dst, (int) 0xff);
duke@435	3970	else if(sizeof(bool) == 2) {
duke@435	3971	// need testw impl
duke@435	3972	ShouldNotReachHere();
duke@435	3973	} else if(sizeof(bool) == 4)
duke@435	3974	testl(dst, dst);
duke@435	3975	else {
duke@435	3976	// unsupported
duke@435	3977	ShouldNotReachHere();
duke@435	3978	}
duke@435	3979	}
duke@435	3980
duke@435	3981	void MacroAssembler::set_last_Java_frame(Register last_java_sp,
duke@435	3982	Register last_java_fp,
duke@435	3983	address last_java_pc) {
duke@435	3984	// determine last_java_sp register
duke@435	3985	if (!last_java_sp->is_valid()) {
duke@435	3986	last_java_sp = rsp;
duke@435	3987	}
duke@435	3988
duke@435	3989	// last_java_fp is optional
duke@435	3990	if (last_java_fp->is_valid()) {
duke@435	3991	movq(Address(r15_thread, JavaThread::last_Java_fp_offset()),
duke@435	3992	last_java_fp);
duke@435	3993	}
duke@435	3994
duke@435	3995	// last_java_pc is optional
duke@435	3996	if (last_java_pc != NULL) {
duke@435	3997	Address java_pc(r15_thread,
duke@435	3998	JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
duke@435	3999	lea(rscratch1, InternalAddress(last_java_pc));
duke@435	4000	movq(java_pc, rscratch1);
duke@435	4001	}
duke@435	4002
duke@435	4003	movq(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
duke@435	4004	}
duke@435	4005
duke@435	4006	void MacroAssembler::reset_last_Java_frame(bool clear_fp,
duke@435	4007	bool clear_pc) {
duke@435	4008	// we must set sp to zero to clear frame
duke@435	4009	movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
duke@435	4010	// must clear fp, so that compiled frames are not confused; it is
duke@435	4011	// possible that we need it only for debugging
duke@435	4012	if (clear_fp) {
duke@435	4013	movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
duke@435	4014	}
duke@435	4015
duke@435	4016	if (clear_pc) {
duke@435	4017	movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
duke@435	4018	}
duke@435	4019	}
duke@435	4020
duke@435	4021
duke@435	4022	// Implementation of call_VM versions
duke@435	4023
duke@435	4024	void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
duke@435	4025	Label L, E;
duke@435	4026
duke@435	4027	#ifdef _WIN64
duke@435	4028	// Windows always allocates space for it's register args
duke@435	4029	assert(num_args <= 4, "only register arguments supported");
duke@435	4030	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4031	#endif
duke@435	4032
duke@435	4033	// Align stack if necessary
duke@435	4034	testl(rsp, 15);
duke@435	4035	jcc(Assembler::zero, L);
duke@435	4036
duke@435	4037	subq(rsp, 8);
duke@435	4038	{
duke@435	4039	call(RuntimeAddress(entry_point));
duke@435	4040	}
duke@435	4041	addq(rsp, 8);
duke@435	4042	jmp(E);
duke@435	4043
duke@435	4044	bind(L);
duke@435	4045	{
duke@435	4046	call(RuntimeAddress(entry_point));
duke@435	4047	}
duke@435	4048
duke@435	4049	bind(E);
duke@435	4050
duke@435	4051	#ifdef _WIN64
duke@435	4052	// restore stack pointer
duke@435	4053	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4054	#endif
duke@435	4055
duke@435	4056	}
duke@435	4057
duke@435	4058
duke@435	4059	void MacroAssembler::call_VM_base(Register oop_result,
duke@435	4060	Register java_thread,
duke@435	4061	Register last_java_sp,
duke@435	4062	address entry_point,
duke@435	4063	int num_args,
duke@435	4064	bool check_exceptions) {
duke@435	4065	// determine last_java_sp register
duke@435	4066	if (!last_java_sp->is_valid()) {
duke@435	4067	last_java_sp = rsp;
duke@435	4068	}
duke@435	4069
duke@435	4070	// debugging support
duke@435	4071	assert(num_args >= 0, "cannot have negative number of arguments");
duke@435	4072	assert(r15_thread != oop_result,
duke@435	4073	"cannot use the same register for java_thread & oop_result");
duke@435	4074	assert(r15_thread != last_java_sp,
duke@435	4075	"cannot use the same register for java_thread & last_java_sp");
duke@435	4076
duke@435	4077	// set last Java frame before call
duke@435	4078
duke@435	4079	// This sets last_Java_fp which is only needed from interpreted frames
duke@435	4080	// and should really be done only from the interp_masm version before
duke@435	4081	// calling the underlying call_VM. That doesn't happen yet so we set
duke@435	4082	// last_Java_fp here even though some callers don't need it and
duke@435	4083	// also clear it below.
duke@435	4084	set_last_Java_frame(last_java_sp, rbp, NULL);
duke@435	4085
duke@435	4086	{
duke@435	4087	Label L, E;
duke@435	4088
duke@435	4089	// Align stack if necessary
duke@435	4090	#ifdef _WIN64
duke@435	4091	assert(num_args <= 4, "only register arguments supported");
duke@435	4092	// Windows always allocates space for it's register args
duke@435	4093	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4094	#endif
duke@435	4095	testl(rsp, 15);
duke@435	4096	jcc(Assembler::zero, L);
duke@435	4097
duke@435	4098	subq(rsp, 8);
duke@435	4099	{
duke@435	4100	call(RuntimeAddress(entry_point));
duke@435	4101	}
duke@435	4102	addq(rsp, 8);
duke@435	4103	jmp(E);
duke@435	4104
duke@435	4105
duke@435	4106	bind(L);
duke@435	4107	{
duke@435	4108	call(RuntimeAddress(entry_point));
duke@435	4109	}
duke@435	4110
duke@435	4111	bind(E);
duke@435	4112
duke@435	4113	#ifdef _WIN64
duke@435	4114	// restore stack pointer
duke@435	4115	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4116	#endif
duke@435	4117	}
duke@435	4118
duke@435	4119	#ifdef ASSERT
duke@435	4120	pushq(rax);
duke@435	4121	{
duke@435	4122	Label L;
duke@435	4123	get_thread(rax);
duke@435	4124	cmpq(r15_thread, rax);
duke@435	4125	jcc(Assembler::equal, L);
duke@435	4126	stop("MacroAssembler::call_VM_base: register not callee saved?");
duke@435	4127	bind(L);
duke@435	4128	}
duke@435	4129	popq(rax);
duke@435	4130	#endif
duke@435	4131
duke@435	4132	// reset last Java frame
duke@435	4133	// This really shouldn't have to clear fp set note above at the
duke@435	4134	// call to set_last_Java_frame
duke@435	4135	reset_last_Java_frame(true, false);
duke@435	4136
duke@435	4137	check_and_handle_popframe(noreg);
duke@435	4138	check_and_handle_earlyret(noreg);
duke@435	4139
duke@435	4140	if (check_exceptions) {
duke@435	4141	cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL);
duke@435	4142	// This used to conditionally jump to forward_exception however it is
duke@435	4143	// possible if we relocate that the branch will not reach. So we must jump
duke@435	4144	// around so we can always reach
duke@435	4145	Label ok;
duke@435	4146	jcc(Assembler::equal, ok);
duke@435	4147	jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	4148	bind(ok);
duke@435	4149	}
duke@435	4150
duke@435	4151	// get oop result if there is one and reset the value in the thread
duke@435	4152	if (oop_result->is_valid()) {
duke@435	4153	movq(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
duke@435	4154	movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
kvn@559	4155	verify_oop(oop_result, "broken oop in call_VM_base");
duke@435	4156	}
duke@435	4157	}
duke@435	4158
duke@435	4159	void MacroAssembler::check_and_handle_popframe(Register java_thread) {}
duke@435	4160	void MacroAssembler::check_and_handle_earlyret(Register java_thread) {}
duke@435	4161
duke@435	4162	void MacroAssembler::call_VM_helper(Register oop_result,
duke@435	4163	address entry_point,
duke@435	4164	int num_args,
duke@435	4165	bool check_exceptions) {
duke@435	4166	// Java thread becomes first argument of C function
duke@435	4167	movq(c_rarg0, r15_thread);
duke@435	4168
duke@435	4169	// We've pushed one address, correct last_Java_sp
duke@435	4170	leaq(rax, Address(rsp, wordSize));
duke@435	4171
duke@435	4172	call_VM_base(oop_result, noreg, rax, entry_point, num_args,
duke@435	4173	check_exceptions);
duke@435	4174	}
duke@435	4175
duke@435	4176
duke@435	4177	void MacroAssembler::call_VM(Register oop_result,
duke@435	4178	address entry_point,
duke@435	4179	bool check_exceptions) {
duke@435	4180	Label C, E;
duke@435	4181	Assembler::call(C, relocInfo::none);
duke@435	4182	jmp(E);
duke@435	4183
duke@435	4184	bind(C);
duke@435	4185	call_VM_helper(oop_result, entry_point, 0, check_exceptions);
duke@435	4186	ret(0);
duke@435	4187
duke@435	4188	bind(E);
duke@435	4189	}
duke@435	4190
duke@435	4191
duke@435	4192	void MacroAssembler::call_VM(Register oop_result,
duke@435	4193	address entry_point,
duke@435	4194	Register arg_1,
duke@435	4195	bool check_exceptions) {
duke@435	4196	assert(rax != arg_1, "smashed argument");
duke@435	4197	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4198
duke@435	4199	Label C, E;
duke@435	4200	Assembler::call(C, relocInfo::none);
duke@435	4201	jmp(E);
duke@435	4202
duke@435	4203	bind(C);
duke@435	4204	// c_rarg0 is reserved for thread
duke@435	4205	if (c_rarg1 != arg_1) {
duke@435	4206	movq(c_rarg1, arg_1);
duke@435	4207	}
duke@435	4208	call_VM_helper(oop_result, entry_point, 1, check_exceptions);
duke@435	4209	ret(0);
duke@435	4210
duke@435	4211	bind(E);
duke@435	4212	}
duke@435	4213
duke@435	4214	void MacroAssembler::call_VM(Register oop_result,
duke@435	4215	address entry_point,
duke@435	4216	Register arg_1,
duke@435	4217	Register arg_2,
duke@435	4218	bool check_exceptions) {
duke@435	4219	assert(rax != arg_1, "smashed argument");
duke@435	4220	assert(rax != arg_2, "smashed argument");
duke@435	4221	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4222	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4223	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4224	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4225
duke@435	4226	Label C, E;
duke@435	4227	Assembler::call(C, relocInfo::none);
duke@435	4228	jmp(E);
duke@435	4229
duke@435	4230	bind(C);
duke@435	4231	// c_rarg0 is reserved for thread
duke@435	4232	if (c_rarg1 != arg_1) {
duke@435	4233	movq(c_rarg1, arg_1);
duke@435	4234	}
duke@435	4235	if (c_rarg2 != arg_2) {
duke@435	4236	movq(c_rarg2, arg_2);
duke@435	4237	}
duke@435	4238	call_VM_helper(oop_result, entry_point, 2, check_exceptions);
duke@435	4239	ret(0);
duke@435	4240
duke@435	4241	bind(E);
duke@435	4242	}
duke@435	4243
duke@435	4244
duke@435	4245	void MacroAssembler::call_VM(Register oop_result,
duke@435	4246	address entry_point,
duke@435	4247	Register arg_1,
duke@435	4248	Register arg_2,
duke@435	4249	Register arg_3,
duke@435	4250	bool check_exceptions) {
duke@435	4251	assert(rax != arg_1, "smashed argument");
duke@435	4252	assert(rax != arg_2, "smashed argument");
duke@435	4253	assert(rax != arg_3, "smashed argument");
duke@435	4254	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4255	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4256	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4257	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4258	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4259	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4260	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4261	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4262	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4263
duke@435	4264	Label C, E;
duke@435	4265	Assembler::call(C, relocInfo::none);
duke@435	4266	jmp(E);
duke@435	4267
duke@435	4268	bind(C);
duke@435	4269	// c_rarg0 is reserved for thread
duke@435	4270	if (c_rarg1 != arg_1) {
duke@435	4271	movq(c_rarg1, arg_1);
duke@435	4272	}
duke@435	4273	if (c_rarg2 != arg_2) {
duke@435	4274	movq(c_rarg2, arg_2);
duke@435	4275	}
duke@435	4276	if (c_rarg3 != arg_3) {
duke@435	4277	movq(c_rarg3, arg_3);
duke@435	4278	}
duke@435	4279	call_VM_helper(oop_result, entry_point, 3, check_exceptions);
duke@435	4280	ret(0);
duke@435	4281
duke@435	4282	bind(E);
duke@435	4283	}
duke@435	4284
duke@435	4285	void MacroAssembler::call_VM(Register oop_result,
duke@435	4286	Register last_java_sp,
duke@435	4287	address entry_point,
duke@435	4288	int num_args,
duke@435	4289	bool check_exceptions) {
duke@435	4290	call_VM_base(oop_result, noreg, last_java_sp, entry_point, num_args,
duke@435	4291	check_exceptions);
duke@435	4292	}
duke@435	4293
duke@435	4294	void MacroAssembler::call_VM(Register oop_result,
duke@435	4295	Register last_java_sp,
duke@435	4296	address entry_point,
duke@435	4297	Register arg_1,
duke@435	4298	bool check_exceptions) {
duke@435	4299	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4300	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4301	// c_rarg0 is reserved for thread
duke@435	4302	if (c_rarg1 != arg_1) {
duke@435	4303	movq(c_rarg1, arg_1);
duke@435	4304	}
duke@435	4305	call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
duke@435	4306	}
duke@435	4307
duke@435	4308	void MacroAssembler::call_VM(Register oop_result,
duke@435	4309	Register last_java_sp,
duke@435	4310	address entry_point,
duke@435	4311	Register arg_1,
duke@435	4312	Register arg_2,
duke@435	4313	bool check_exceptions) {
duke@435	4314	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4315	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4316	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4317	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4318	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4319	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4320	// c_rarg0 is reserved for thread
duke@435	4321	if (c_rarg1 != arg_1) {
duke@435	4322	movq(c_rarg1, arg_1);
duke@435	4323	}
duke@435	4324	if (c_rarg2 != arg_2) {
duke@435	4325	movq(c_rarg2, arg_2);
duke@435	4326	}
duke@435	4327	call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
duke@435	4328	}
duke@435	4329
duke@435	4330
duke@435	4331	void MacroAssembler::call_VM(Register oop_result,
duke@435	4332	Register last_java_sp,
duke@435	4333	address entry_point,
duke@435	4334	Register arg_1,
duke@435	4335	Register arg_2,
duke@435	4336	Register arg_3,
duke@435	4337	bool check_exceptions) {
duke@435	4338	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4339	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4340	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4341	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4342	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4343	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4344	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4345	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4346	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4347	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4348	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4349	assert(c_rarg3 != last_java_sp, "smashed argument");
duke@435	4350	// c_rarg0 is reserved for thread
duke@435	4351	if (c_rarg1 != arg_1) {
duke@435	4352	movq(c_rarg1, arg_1);
duke@435	4353	}
duke@435	4354	if (c_rarg2 != arg_2) {
duke@435	4355	movq(c_rarg2, arg_2);
duke@435	4356	}
duke@435	4357	if (c_rarg3 != arg_3) {
duke@435	4358	movq(c_rarg2, arg_3);
duke@435	4359	}
duke@435	4360	call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
duke@435	4361	}
duke@435	4362
duke@435	4363	void MacroAssembler::call_VM_leaf(address entry_point, int num_args) {
duke@435	4364	call_VM_leaf_base(entry_point, num_args);
duke@435	4365	}
duke@435	4366
duke@435	4367	void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
duke@435	4368	if (c_rarg0 != arg_1) {
duke@435	4369	movq(c_rarg0, arg_1);
duke@435	4370	}
duke@435	4371	call_VM_leaf(entry_point, 1);
duke@435	4372	}
duke@435	4373
duke@435	4374	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4375	Register arg_1,
duke@435	4376	Register arg_2) {
duke@435	4377	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4378	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4379	if (c_rarg0 != arg_1) {
duke@435	4380	movq(c_rarg0, arg_1);
duke@435	4381	}
duke@435	4382	if (c_rarg1 != arg_2) {
duke@435	4383	movq(c_rarg1, arg_2);
duke@435	4384	}
duke@435	4385	call_VM_leaf(entry_point, 2);
duke@435	4386	}
duke@435	4387
duke@435	4388	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4389	Register arg_1,
duke@435	4390	Register arg_2,
duke@435	4391	Register arg_3) {
duke@435	4392	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4393	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4394	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4395	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4396	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4397	assert(c_rarg2 != arg_2, "smashed argument");
duke@435	4398	if (c_rarg0 != arg_1) {
duke@435	4399	movq(c_rarg0, arg_1);
duke@435	4400	}
duke@435	4401	if (c_rarg1 != arg_2) {
duke@435	4402	movq(c_rarg1, arg_2);
duke@435	4403	}
duke@435	4404	if (c_rarg2 != arg_3) {
duke@435	4405	movq(c_rarg2, arg_3);
duke@435	4406	}
duke@435	4407	call_VM_leaf(entry_point, 3);
duke@435	4408	}
duke@435	4409
duke@435	4410
duke@435	4411	// Calls to C land
duke@435	4412	//
duke@435	4413	// When entering C land, the rbp & rsp of the last Java frame have to
duke@435	4414	// be recorded in the (thread-local) JavaThread object. When leaving C
duke@435	4415	// land, the last Java fp has to be reset to 0. This is required to
duke@435	4416	// allow proper stack traversal.
duke@435	4417	void MacroAssembler::store_check(Register obj) {
duke@435	4418	// Does a store check for the oop in register obj. The content of
duke@435	4419	// register obj is destroyed afterwards.
duke@435	4420	store_check_part_1(obj);
duke@435	4421	store_check_part_2(obj);
duke@435	4422	}
duke@435	4423
duke@435	4424	void MacroAssembler::store_check(Register obj, Address dst) {
duke@435	4425	store_check(obj);
duke@435	4426	}
duke@435	4427
duke@435	4428	// split the store check operation so that other instructions can be
duke@435	4429	// scheduled inbetween
duke@435	4430	void MacroAssembler::store_check_part_1(Register obj) {
duke@435	4431	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4432	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4433	shrq(obj, CardTableModRefBS::card_shift);
duke@435	4434	}
duke@435	4435
duke@435	4436	void MacroAssembler::store_check_part_2(Register obj) {
duke@435	4437	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4438	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4439	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	4440	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
sgoldman@552	4441
sgoldman@552	4442	// The calculation for byte_map_base is as follows:
sgoldman@552	4443	// byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
sgoldman@552	4444	// So this essentially converts an address to a displacement and
sgoldman@552	4445	// it will never need to be relocated. On 64bit however the value may be too
sgoldman@552	4446	// large for a 32bit displacement
sgoldman@552	4447
sgoldman@552	4448	intptr_t disp = (intptr_t) ct->byte_map_base;
sgoldman@552	4449	if (is_simm32(disp)) {
sgoldman@552	4450	Address cardtable(noreg, obj, Address::times_1, disp);
sgoldman@552	4451	movb(cardtable, 0);
sgoldman@552	4452	} else {
sgoldman@552	4453	// By doing it as an ExternalAddress disp could be converted to a rip-relative
sgoldman@552	4454	// displacement and done in a single instruction given favorable mapping and
sgoldman@552	4455	// a smarter version of as_Address. Worst case it is two instructions which
sgoldman@552	4456	// is no worse off then loading disp into a register and doing as a simple
sgoldman@552	4457	// Address() as above.
sgoldman@552	4458	// We can't do as ExternalAddress as the only style since if disp == 0 we'll
sgoldman@552	4459	// assert since NULL isn't acceptable in a reloci (see 6644928). In any case
sgoldman@552	4460	// in some cases we'll get a single instruction version.
sgoldman@552	4461
sgoldman@552	4462	ExternalAddress cardtable((address)disp);
sgoldman@552	4463	Address index(noreg, obj, Address::times_1);
sgoldman@552	4464	movb(as_Address(ArrayAddress(cardtable, index)), 0);
sgoldman@552	4465	}
sgoldman@552	4466
duke@435	4467	}
duke@435	4468
duke@435	4469	void MacroAssembler::c2bool(Register x) {
duke@435	4470	// implements x == 0 ? 0 : 1
duke@435	4471	// note: must only look at least-significant byte of x
duke@435	4472	// since C-style booleans are stored in one byte
duke@435	4473	// only! (was bug)
duke@435	4474	andl(x, 0xFF);
duke@435	4475	setb(Assembler::notZero, x);
duke@435	4476	}
duke@435	4477
duke@435	4478	int MacroAssembler::corrected_idivl(Register reg) {
duke@435	4479	// Full implementation of Java idiv and irem; checks for special
duke@435	4480	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4481	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4482	// for implicit exceptions.
duke@435	4483	//
duke@435	4484	// normal case special case
duke@435	4485	//
duke@435	4486	// input : eax: dividend min_int
duke@435	4487	// reg: divisor (may not be eax/edx) -1
duke@435	4488	//
duke@435	4489	// output: eax: quotient (= eax idiv reg) min_int
duke@435	4490	// edx: remainder (= eax irem reg) 0
duke@435	4491	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4492	const int min_int = 0x80000000;
duke@435	4493	Label normal_case, special_case;
duke@435	4494
duke@435	4495	// check for special case
duke@435	4496	cmpl(rax, min_int);
duke@435	4497	jcc(Assembler::notEqual, normal_case);
duke@435	4498	xorl(rdx, rdx); // prepare edx for possible special case (where
duke@435	4499	// remainder = 0)
duke@435	4500	cmpl(reg, -1);
duke@435	4501	jcc(Assembler::equal, special_case);
duke@435	4502
duke@435	4503	// handle normal case
duke@435	4504	bind(normal_case);
duke@435	4505	cdql();
duke@435	4506	int idivl_offset = offset();
duke@435	4507	idivl(reg);
duke@435	4508
duke@435	4509	// normal and special case exit
duke@435	4510	bind(special_case);
duke@435	4511
duke@435	4512	return idivl_offset;
duke@435	4513	}
duke@435	4514
duke@435	4515	int MacroAssembler::corrected_idivq(Register reg) {
duke@435	4516	// Full implementation of Java ldiv and lrem; checks for special
duke@435	4517	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4518	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4519	// for implicit exceptions.
duke@435	4520	//
duke@435	4521	// normal case special case
duke@435	4522	//
duke@435	4523	// input : rax: dividend min_long
duke@435	4524	// reg: divisor (may not be eax/edx) -1
duke@435	4525	//
duke@435	4526	// output: rax: quotient (= rax idiv reg) min_long
duke@435	4527	// rdx: remainder (= rax irem reg) 0
duke@435	4528	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4529	static const int64_t min_long = 0x8000000000000000;
duke@435	4530	Label normal_case, special_case;
duke@435	4531
duke@435	4532	// check for special case
duke@435	4533	cmp64(rax, ExternalAddress((address) &min_long));
duke@435	4534	jcc(Assembler::notEqual, normal_case);
duke@435	4535	xorl(rdx, rdx); // prepare rdx for possible special case (where
duke@435	4536	// remainder = 0)
duke@435	4537	cmpq(reg, -1);
duke@435	4538	jcc(Assembler::equal, special_case);
duke@435	4539
duke@435	4540	// handle normal case
duke@435	4541	bind(normal_case);
duke@435	4542	cdqq();
duke@435	4543	int idivq_offset = offset();
duke@435	4544	idivq(reg);
duke@435	4545
duke@435	4546	// normal and special case exit
duke@435	4547	bind(special_case);
duke@435	4548
duke@435	4549	return idivq_offset;
duke@435	4550	}
duke@435	4551
duke@435	4552	void MacroAssembler::push_IU_state() {
duke@435	4553	pushfq(); // Push flags first because pushaq kills them
duke@435	4554	subq(rsp, 8); // Make sure rsp stays 16-byte aligned
duke@435	4555	pushaq();
duke@435	4556	}
duke@435	4557
duke@435	4558	void MacroAssembler::pop_IU_state() {
duke@435	4559	popaq();
duke@435	4560	addq(rsp, 8);
duke@435	4561	popfq();
duke@435	4562	}
duke@435	4563
duke@435	4564	void MacroAssembler::push_FPU_state() {
duke@435	4565	subq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4566	fxsave(Address(rsp, 0));
duke@435	4567	}
duke@435	4568
duke@435	4569	void MacroAssembler::pop_FPU_state() {
duke@435	4570	fxrstor(Address(rsp, 0));
duke@435	4571	addq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4572	}
duke@435	4573
duke@435	4574	// Save Integer and Float state
duke@435	4575	// Warning: Stack must be 16 byte aligned
duke@435	4576	void MacroAssembler::push_CPU_state() {
duke@435	4577	push_IU_state();
duke@435	4578	push_FPU_state();
duke@435	4579	}
duke@435	4580
duke@435	4581	void MacroAssembler::pop_CPU_state() {
duke@435	4582	pop_FPU_state();
duke@435	4583	pop_IU_state();
duke@435	4584	}
duke@435	4585
duke@435	4586	void MacroAssembler::sign_extend_short(Register reg) {
duke@435	4587	movswl(reg, reg);
duke@435	4588	}
duke@435	4589
duke@435	4590	void MacroAssembler::sign_extend_byte(Register reg) {
duke@435	4591	movsbl(reg, reg);
duke@435	4592	}
duke@435	4593
duke@435	4594	void MacroAssembler::division_with_shift(Register reg, int shift_value) {
duke@435	4595	assert (shift_value > 0, "illegal shift value");
duke@435	4596	Label _is_positive;
duke@435	4597	testl (reg, reg);
duke@435	4598	jcc (Assembler::positive, _is_positive);
duke@435	4599	int offset = (1 << shift_value) - 1 ;
duke@435	4600
duke@435	4601	if (offset == 1) {
duke@435	4602	incrementl(reg);
duke@435	4603	} else {
duke@435	4604	addl(reg, offset);
duke@435	4605	}
duke@435	4606
duke@435	4607	bind (_is_positive);
duke@435	4608	sarl(reg, shift_value);
duke@435	4609	}
duke@435	4610
duke@435	4611	void MacroAssembler::round_to_l(Register reg, int modulus) {
duke@435	4612	addl(reg, modulus - 1);
duke@435	4613	andl(reg, -modulus);
duke@435	4614	}
duke@435	4615
duke@435	4616	void MacroAssembler::round_to_q(Register reg, int modulus) {
duke@435	4617	addq(reg, modulus - 1);
duke@435	4618	andq(reg, -modulus);
duke@435	4619	}
duke@435	4620
duke@435	4621	void MacroAssembler::verify_oop(Register reg, const char* s) {
duke@435	4622	if (!VerifyOops) {
duke@435	4623	return;
duke@435	4624	}
duke@435	4625
duke@435	4626	// Pass register number to verify_oop_subroutine
duke@435	4627	char* b = new char[strlen(s) + 50];
duke@435	4628	sprintf(b, "verify_oop: %s: %s", reg->name(), s);
duke@435	4629
duke@435	4630	pushq(rax); // save rax, restored by receiver
duke@435	4631
duke@435	4632	// pass args on stack, only touch rax
duke@435	4633	pushq(reg);
duke@435	4634	// avoid using pushptr, as it modifies scratch registers
duke@435	4635	// and our contract is not to modify anything
duke@435	4636	ExternalAddress buffer((address)b);
duke@435	4637	movptr(rax, buffer.addr());
duke@435	4638	pushq(rax);
duke@435	4639
duke@435	4640	// call indirectly to solve generation ordering problem
duke@435	4641	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4642	call(rax); // no alignment requirement
duke@435	4643	// everything popped by receiver
duke@435	4644	}
duke@435	4645
duke@435	4646	void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
duke@435	4647	if (!VerifyOops) return;
duke@435	4648	// Pass register number to verify_oop_subroutine
duke@435	4649	char* b = new char[strlen(s) + 50];
duke@435	4650	sprintf(b, "verify_oop_addr: %s", s);
duke@435	4651	pushq(rax); // save rax
duke@435	4652	movq(addr, rax);
duke@435	4653	pushq(rax); // pass register argument
duke@435	4654
duke@435	4655
duke@435	4656	// avoid using pushptr, as it modifies scratch registers
duke@435	4657	// and our contract is not to modify anything
duke@435	4658	ExternalAddress buffer((address)b);
duke@435	4659	movptr(rax, buffer.addr());
duke@435	4660	pushq(rax);
duke@435	4661
duke@435	4662	// call indirectly to solve generation ordering problem
duke@435	4663	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4664	call(rax); // no alignment requirement
duke@435	4665	// everything popped by receiver
duke@435	4666	}
duke@435	4667
duke@435	4668
duke@435	4669	void MacroAssembler::stop(const char* msg) {
duke@435	4670	address rip = pc();
duke@435	4671	pushaq(); // get regs on stack
duke@435	4672	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4673	lea(c_rarg1, InternalAddress(rip));
duke@435	4674	movq(c_rarg2, rsp); // pass pointer to regs array
duke@435	4675	andq(rsp, -16); // align stack as required by ABI
duke@435	4676	call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
duke@435	4677	hlt();
duke@435	4678	}
duke@435	4679
duke@435	4680	void MacroAssembler::warn(const char* msg) {
duke@435	4681	pushq(r12);
duke@435	4682	movq(r12, rsp);
duke@435	4683	andq(rsp, -16); // align stack as required by push_CPU_state and call
duke@435	4684
duke@435	4685	push_CPU_state(); // keeps alignment at 16 bytes
duke@435	4686	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4687	call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
duke@435	4688	pop_CPU_state();
duke@435	4689
duke@435	4690	movq(rsp, r12);
duke@435	4691	popq(r12);
duke@435	4692	}
duke@435	4693
kvn@559	4694	#ifndef PRODUCT
kvn@559	4695	extern "C" void findpc(intptr_t x);
kvn@559	4696	#endif
kvn@559	4697
duke@435	4698	void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
duke@435	4699	// In order to get locks to work, we need to fake a in_VM state
duke@435	4700	if (ShowMessageBoxOnError ) {
duke@435	4701	JavaThread* thread = JavaThread::current();
duke@435	4702	JavaThreadState saved_state = thread->thread_state();
duke@435	4703	thread->set_thread_state(_thread_in_vm);
duke@435	4704	#ifndef PRODUCT
duke@435	4705	if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
coleenp@548	4706	ttyLocker ttyl;
duke@435	4707	BytecodeCounter::print();
duke@435	4708	}
duke@435	4709	#endif
duke@435	4710	// To see where a verify_oop failed, get $ebx+40/X for this frame.
duke@435	4711	// XXX correct this offset for amd64
duke@435	4712	// This is the value of eip which points to where verify_oop will return.
duke@435	4713	if (os::message_box(msg, "Execution stopped, print registers?")) {
coleenp@548	4714	ttyLocker ttyl;
duke@435	4715	tty->print_cr("rip = 0x%016lx", pc);
kvn@559	4716	#ifndef PRODUCT
kvn@559	4717	tty->cr();
kvn@559	4718	findpc(pc);
kvn@559	4719	tty->cr();
kvn@559	4720	#endif
duke@435	4721	tty->print_cr("rax = 0x%016lx", regs[15]);
duke@435	4722	tty->print_cr("rbx = 0x%016lx", regs[12]);
duke@435	4723	tty->print_cr("rcx = 0x%016lx", regs[14]);
duke@435	4724	tty->print_cr("rdx = 0x%016lx", regs[13]);
duke@435	4725	tty->print_cr("rdi = 0x%016lx", regs[8]);
duke@435	4726	tty->print_cr("rsi = 0x%016lx", regs[9]);
duke@435	4727	tty->print_cr("rbp = 0x%016lx", regs[10]);
duke@435	4728	tty->print_cr("rsp = 0x%016lx", regs[11]);
duke@435	4729	tty->print_cr("r8 = 0x%016lx", regs[7]);
duke@435	4730	tty->print_cr("r9 = 0x%016lx", regs[6]);
duke@435	4731	tty->print_cr("r10 = 0x%016lx", regs[5]);
duke@435	4732	tty->print_cr("r11 = 0x%016lx", regs[4]);
duke@435	4733	tty->print_cr("r12 = 0x%016lx", regs[3]);
duke@435	4734	tty->print_cr("r13 = 0x%016lx", regs[2]);
duke@435	4735	tty->print_cr("r14 = 0x%016lx", regs[1]);
duke@435	4736	tty->print_cr("r15 = 0x%016lx", regs[0]);
duke@435	4737	BREAKPOINT;
duke@435	4738	}
duke@435	4739	ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
duke@435	4740	} else {
coleenp@548	4741	ttyLocker ttyl;
duke@435	4742	::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
duke@435	4743	msg);
duke@435	4744	}
duke@435	4745	}
duke@435	4746
duke@435	4747	void MacroAssembler::os_breakpoint() {
duke@435	4748	// instead of directly emitting a breakpoint, call os:breakpoint for
duke@435	4749	// better debugability
duke@435	4750	// This shouldn't need alignment, it's an empty function
duke@435	4751	call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
duke@435	4752	}
duke@435	4753
duke@435	4754	// Write serialization page so VM thread can do a pseudo remote membar.
duke@435	4755	// We use the current thread pointer to calculate a thread specific
duke@435	4756	// offset to write to within the page. This minimizes bus traffic
duke@435	4757	// due to cache line collision.
duke@435	4758	void MacroAssembler::serialize_memory(Register thread,
duke@435	4759	Register tmp) {
duke@435	4760
duke@435	4761	movl(tmp, thread);
duke@435	4762	shrl(tmp, os::get_serialize_page_shift_count());
duke@435	4763	andl(tmp, (os::vm_page_size() - sizeof(int)));
duke@435	4764
duke@435	4765	Address index(noreg, tmp, Address::times_1);
duke@435	4766	ExternalAddress page(os::get_memory_serialize_page());
duke@435	4767
duke@435	4768	movptr(ArrayAddress(page, index), tmp);
duke@435	4769	}
duke@435	4770
duke@435	4771	void MacroAssembler::verify_tlab() {
duke@435	4772	#ifdef ASSERT
duke@435	4773	if (UseTLAB) {
duke@435	4774	Label next, ok;
duke@435	4775	Register t1 = rsi;
duke@435	4776
duke@435	4777	pushq(t1);
duke@435	4778
duke@435	4779	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4780	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset())));
duke@435	4781	jcc(Assembler::aboveEqual, next);
duke@435	4782	stop("assert(top >= start)");
duke@435	4783	should_not_reach_here();
duke@435	4784
duke@435	4785	bind(next);
duke@435	4786	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4787	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4788	jcc(Assembler::aboveEqual, ok);
duke@435	4789	stop("assert(top <= end)");
duke@435	4790	should_not_reach_here();
duke@435	4791
duke@435	4792	bind(ok);
duke@435	4793
duke@435	4794	popq(t1);
duke@435	4795	}
duke@435	4796	#endif
duke@435	4797	}
duke@435	4798
duke@435	4799	// Defines obj, preserves var_size_in_bytes
duke@435	4800	void MacroAssembler::eden_allocate(Register obj,
duke@435	4801	Register var_size_in_bytes,
duke@435	4802	int con_size_in_bytes,
duke@435	4803	Register t1,
duke@435	4804	Label& slow_case) {
duke@435	4805	assert(obj == rax, "obj must be in rax for cmpxchg");
duke@435	4806	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4807	Register end = t1;
duke@435	4808	Label retry;
duke@435	4809	bind(retry);
duke@435	4810	ExternalAddress heap_top((address) Universe::heap()->top_addr());
duke@435	4811	movptr(obj, heap_top);
duke@435	4812	if (var_size_in_bytes == noreg) {
duke@435	4813	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4814	} else {
duke@435	4815	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4816	}
duke@435	4817	// if end < obj then we wrapped around => object too long => slow case
duke@435	4818	cmpq(end, obj);
duke@435	4819	jcc(Assembler::below, slow_case);
duke@435	4820	cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
duke@435	4821
duke@435	4822	jcc(Assembler::above, slow_case);
duke@435	4823	// Compare obj with the top addr, and if still equal, store the new
duke@435	4824	// top addr in end at the address of the top addr pointer. Sets ZF
duke@435	4825	// if was equal, and clears it otherwise. Use lock prefix for
duke@435	4826	// atomicity on MPs.
duke@435	4827	if (os::is_MP()) {
duke@435	4828	lock();
duke@435	4829	}
duke@435	4830	cmpxchgptr(end, heap_top);
duke@435	4831	// if someone beat us on the allocation, try again, otherwise continue
duke@435	4832	jcc(Assembler::notEqual, retry);
duke@435	4833	}
duke@435	4834
duke@435	4835	// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
duke@435	4836	void MacroAssembler::tlab_allocate(Register obj,
duke@435	4837	Register var_size_in_bytes,
duke@435	4838	int con_size_in_bytes,
duke@435	4839	Register t1,
duke@435	4840	Register t2,
duke@435	4841	Label& slow_case) {
duke@435	4842	assert_different_registers(obj, t1, t2);
duke@435	4843	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4844	Register end = t2;
duke@435	4845
duke@435	4846	verify_tlab();
duke@435	4847
duke@435	4848	movq(obj, Address(r15_thread, JavaThread::tlab_top_offset()));
duke@435	4849	if (var_size_in_bytes == noreg) {
duke@435	4850	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4851	} else {
duke@435	4852	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4853	}
duke@435	4854	cmpq(end, Address(r15_thread, JavaThread::tlab_end_offset()));
duke@435	4855	jcc(Assembler::above, slow_case);
duke@435	4856
duke@435	4857	// update the tlab top pointer
duke@435	4858	movq(Address(r15_thread, JavaThread::tlab_top_offset()), end);
duke@435	4859
duke@435	4860	// recover var_size_in_bytes if necessary
duke@435	4861	if (var_size_in_bytes == end) {
duke@435	4862	subq(var_size_in_bytes, obj);
duke@435	4863	}
duke@435	4864	verify_tlab();
duke@435	4865	}
duke@435	4866
duke@435	4867	// Preserves rbx and rdx.
duke@435	4868	void MacroAssembler::tlab_refill(Label& retry,
duke@435	4869	Label& try_eden,
duke@435	4870	Label& slow_case) {
duke@435	4871	Register top = rax;
duke@435	4872	Register t1 = rcx;
duke@435	4873	Register t2 = rsi;
duke@435	4874	Register t3 = r10;
duke@435	4875	Register thread_reg = r15_thread;
duke@435	4876	assert_different_registers(top, thread_reg, t1, t2, t3,
duke@435	4877	/* preserve: */ rbx, rdx);
duke@435	4878	Label do_refill, discard_tlab;
duke@435	4879
duke@435	4880	if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
duke@435	4881	// No allocation in the shared eden.
duke@435	4882	jmp(slow_case);
duke@435	4883	}
duke@435	4884
duke@435	4885	movq(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4886	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4887
duke@435	4888	// calculate amount of free space
duke@435	4889	subq(t1, top);
duke@435	4890	shrq(t1, LogHeapWordSize);
duke@435	4891
duke@435	4892	// Retain tlab and allocate object in shared space if
duke@435	4893	// the amount free in the tlab is too large to discard.
duke@435	4894	cmpq(t1, Address(thread_reg, // size_t
duke@435	4895	in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
duke@435	4896	jcc(Assembler::lessEqual, discard_tlab);
duke@435	4897
duke@435	4898	// Retain
duke@435	4899	mov64(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment());
duke@435	4900	addq(Address(thread_reg, // size_t
duke@435	4901	in_bytes(JavaThread::tlab_refill_waste_limit_offset())),
duke@435	4902	t2);
duke@435	4903	if (TLABStats) {
duke@435	4904	// increment number of slow_allocations
duke@435	4905	addl(Address(thread_reg, // unsigned int
duke@435	4906	in_bytes(JavaThread::tlab_slow_allocations_offset())),
duke@435	4907	1);
duke@435	4908	}
duke@435	4909	jmp(try_eden);
duke@435	4910
duke@435	4911	bind(discard_tlab);
duke@435	4912	if (TLABStats) {
duke@435	4913	// increment number of refills
duke@435	4914	addl(Address(thread_reg, // unsigned int
duke@435	4915	in_bytes(JavaThread::tlab_number_of_refills_offset())),
duke@435	4916	1);
duke@435	4917	// accumulate wastage -- t1 is amount free in tlab
duke@435	4918	addl(Address(thread_reg, // unsigned int
duke@435	4919	in_bytes(JavaThread::tlab_fast_refill_waste_offset())),
duke@435	4920	t1);
duke@435	4921	}
duke@435	4922
duke@435	4923	// if tlab is currently allocated (top or end != null) then
duke@435	4924	// fill [top, end + alignment_reserve) with array object
duke@435	4925	testq(top, top);
duke@435	4926	jcc(Assembler::zero, do_refill);
duke@435	4927
duke@435	4928	// set up the mark word
duke@435	4929	mov64(t3, (int64_t) markOopDesc::prototype()->copy_set_hash(0x2));
duke@435	4930	movq(Address(top, oopDesc::mark_offset_in_bytes()), t3);
duke@435	4931	// set the length to the remaining space
duke@435	4932	subq(t1, typeArrayOopDesc::header_size(T_INT));
duke@435	4933	addq(t1, (int)ThreadLocalAllocBuffer::alignment_reserve());
duke@435	4934	shlq(t1, log2_intptr(HeapWordSize / sizeof(jint)));
duke@435	4935	movq(Address(top, arrayOopDesc::length_offset_in_bytes()), t1);
duke@435	4936	// set klass to intArrayKlass
duke@435	4937	movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr()));
coleenp@602	4938	// store klass last. concurrent gcs assumes klass length is valid if
coleenp@602	4939	// klass field is not null.
coleenp@548	4940	store_klass(top, t1);
duke@435	4941
duke@435	4942	// refill the tlab with an eden allocation
duke@435	4943	bind(do_refill);
duke@435	4944	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4945	shlq(t1, LogHeapWordSize);
duke@435	4946	// add object_size ??
duke@435	4947	eden_allocate(top, t1, 0, t2, slow_case);
duke@435	4948
duke@435	4949	// Check that t1 was preserved in eden_allocate.
duke@435	4950	#ifdef ASSERT
duke@435	4951	if (UseTLAB) {
duke@435	4952	Label ok;
duke@435	4953	Register tsize = rsi;
duke@435	4954	assert_different_registers(tsize, thread_reg, t1);
duke@435	4955	pushq(tsize);
duke@435	4956	movq(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4957	shlq(tsize, LogHeapWordSize);
duke@435	4958	cmpq(t1, tsize);
duke@435	4959	jcc(Assembler::equal, ok);
duke@435	4960	stop("assert(t1 != tlab size)");
duke@435	4961	should_not_reach_here();
duke@435	4962
duke@435	4963	bind(ok);
duke@435	4964	popq(tsize);
duke@435	4965	}
duke@435	4966	#endif
duke@435	4967	movq(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
duke@435	4968	movq(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
duke@435	4969	addq(top, t1);
duke@435	4970	subq(top, (int)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
duke@435	4971	movq(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
duke@435	4972	verify_tlab();
duke@435	4973	jmp(retry);
duke@435	4974	}
duke@435	4975
duke@435	4976
duke@435	4977	int MacroAssembler::biased_locking_enter(Register lock_reg, Register obj_reg, Register swap_reg, Register tmp_reg,
duke@435	4978	bool swap_reg_contains_mark,
duke@435	4979	Label& done, Label* slow_case,
duke@435	4980	BiasedLockingCounters* counters) {
duke@435	4981	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	4982	assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
duke@435	4983	assert(tmp_reg != noreg, "tmp_reg must be supplied");
duke@435	4984	assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
duke@435	4985	assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
duke@435	4986	Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
duke@435	4987	Address saved_mark_addr(lock_reg, 0);
duke@435	4988
duke@435	4989	if (PrintBiasedLockingStatistics && counters == NULL)
duke@435	4990	counters = BiasedLocking::counters();
duke@435	4991
duke@435	4992	// Biased locking
duke@435	4993	// See whether the lock is currently biased toward our thread and
duke@435	4994	// whether the epoch is still valid
duke@435	4995	// Note that the runtime guarantees sufficient alignment of JavaThread
duke@435	4996	// pointers to allow age to be placed into low bits
duke@435	4997	// First check to see whether biasing is even enabled for this object
duke@435	4998	Label cas_label;
duke@435	4999	int null_check_offset = -1;
duke@435	5000	if (!swap_reg_contains_mark) {
duke@435	5001	null_check_offset = offset();
duke@435	5002	movq(swap_reg, mark_addr);
duke@435	5003	}
duke@435	5004	movq(tmp_reg, swap_reg);
duke@435	5005	andq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	5006	cmpq(tmp_reg, markOopDesc::biased_lock_pattern);
duke@435	5007	jcc(Assembler::notEqual, cas_label);
duke@435	5008	// The bias pattern is present in the object's header. Need to check
duke@435	5009	// whether the bias owner and the epoch are both still current.
coleenp@548	5010	load_klass(tmp_reg, obj_reg);
duke@435	5011	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5012	orq(tmp_reg, r15_thread);
duke@435	5013	xorq(tmp_reg, swap_reg);
duke@435	5014	andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place));
duke@435	5015	if (counters != NULL) {
duke@435	5016	cond_inc32(Assembler::zero,
duke@435	5017	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	5018	}
duke@435	5019	jcc(Assembler::equal, done);
duke@435	5020
duke@435	5021	Label try_revoke_bias;
duke@435	5022	Label try_rebias;
duke@435	5023
duke@435	5024	// At this point we know that the header has the bias pattern and
duke@435	5025	// that we are not the bias owner in the current epoch. We need to
duke@435	5026	// figure out more details about the state of the header in order to
duke@435	5027	// know what operations can be legally performed on the object's
duke@435	5028	// header.
duke@435	5029
duke@435	5030	// If the low three bits in the xor result aren't clear, that means
duke@435	5031	// the prototype header is no longer biased and we have to revoke
duke@435	5032	// the bias on this object.
duke@435	5033	testq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	5034	jcc(Assembler::notZero, try_revoke_bias);
duke@435	5035
duke@435	5036	// Biasing is still enabled for this data type. See whether the
duke@435	5037	// epoch of the current bias is still valid, meaning that the epoch
duke@435	5038	// bits of the mark word are equal to the epoch bits of the
duke@435	5039	// prototype header. (Note that the prototype header's epoch bits
duke@435	5040	// only change at a safepoint.) If not, attempt to rebias the object
duke@435	5041	// toward the current thread. Note that we must be absolutely sure
duke@435	5042	// that the current epoch is invalid in order to do this because
duke@435	5043	// otherwise the manipulations it performs on the mark word are
duke@435	5044	// illegal.
duke@435	5045	testq(tmp_reg, markOopDesc::epoch_mask_in_place);
duke@435	5046	jcc(Assembler::notZero, try_rebias);
duke@435	5047
duke@435	5048	// The epoch of the current bias is still valid but we know nothing
duke@435	5049	// about the owner; it might be set or it might be clear. Try to
duke@435	5050	// acquire the bias of the object using an atomic operation. If this
duke@435	5051	// fails we will go in to the runtime to revoke the object's bias.
duke@435	5052	// Note that we first construct the presumed unbiased header so we
duke@435	5053	// don't accidentally blow away another thread's valid bias.
duke@435	5054	andq(swap_reg,
duke@435	5055	markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
duke@435	5056	movq(tmp_reg, swap_reg);
duke@435	5057	orq(tmp_reg, r15_thread);
duke@435	5058	if (os::is_MP()) {
duke@435	5059	lock();
duke@435	5060	}
duke@435	5061	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5062	// If the biasing toward our thread failed, this means that
duke@435	5063	// another thread succeeded in biasing it toward itself and we
duke@435	5064	// need to revoke that bias. The revocation will occur in the
duke@435	5065	// interpreter runtime in the slow case.
duke@435	5066	if (counters != NULL) {
duke@435	5067	cond_inc32(Assembler::zero,
duke@435	5068	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	5069	}
duke@435	5070	if (slow_case != NULL) {
duke@435	5071	jcc(Assembler::notZero, *slow_case);
duke@435	5072	}
duke@435	5073	jmp(done);
duke@435	5074
duke@435	5075	bind(try_rebias);
duke@435	5076	// At this point we know the epoch has expired, meaning that the
duke@435	5077	// current "bias owner", if any, is actually invalid. Under these
duke@435	5078	// circumstances _only_, we are allowed to use the current header's
duke@435	5079	// value as the comparison value when doing the cas to acquire the
duke@435	5080	// bias in the current epoch. In other words, we allow transfer of
duke@435	5081	// the bias from one thread to another directly in this situation.
duke@435	5082	//
duke@435	5083	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5084	// bits in this situation. Should attempt to preserve them.
coleenp@548	5085	load_klass(tmp_reg, obj_reg);
duke@435	5086	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5087	orq(tmp_reg, r15_thread);
duke@435	5088	if (os::is_MP()) {
duke@435	5089	lock();
duke@435	5090	}
duke@435	5091	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5092	// If the biasing toward our thread failed, then another thread
duke@435	5093	// succeeded in biasing it toward itself and we need to revoke that
duke@435	5094	// bias. The revocation will occur in the runtime in the slow case.
duke@435	5095	if (counters != NULL) {
duke@435	5096	cond_inc32(Assembler::zero,
duke@435	5097	ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
duke@435	5098	}
duke@435	5099	if (slow_case != NULL) {
duke@435	5100	jcc(Assembler::notZero, *slow_case);
duke@435	5101	}
duke@435	5102	jmp(done);
duke@435	5103
duke@435	5104	bind(try_revoke_bias);
duke@435	5105	// The prototype mark in the klass doesn't have the bias bit set any
duke@435	5106	// more, indicating that objects of this data type are not supposed
duke@435	5107	// to be biased any more. We are going to try to reset the mark of
duke@435	5108	// this object to the prototype value and fall through to the
duke@435	5109	// CAS-based locking scheme. Note that if our CAS fails, it means
duke@435	5110	// that another thread raced us for the privilege of revoking the
duke@435	5111	// bias of this particular object, so it's okay to continue in the
duke@435	5112	// normal locking code.
duke@435	5113	//
duke@435	5114	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5115	// bits in this situation. Should attempt to preserve them.
coleenp@548	5116	load_klass(tmp_reg, obj_reg);
duke@435	5117	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5118	if (os::is_MP()) {
duke@435	5119	lock();
duke@435	5120	}
duke@435	5121	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5122	// Fall through to the normal CAS-based lock, because no matter what
duke@435	5123	// the result of the above CAS, some thread must have succeeded in
duke@435	5124	// removing the bias bit from the object's header.
duke@435	5125	if (counters != NULL) {
duke@435	5126	cond_inc32(Assembler::zero,
duke@435	5127	ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
duke@435	5128	}
duke@435	5129
duke@435	5130	bind(cas_label);
duke@435	5131
duke@435	5132	return null_check_offset;
duke@435	5133	}
duke@435	5134
duke@435	5135
duke@435	5136	void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
duke@435	5137	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	5138
duke@435	5139	// Check for biased locking unlock case, which is a no-op
duke@435	5140	// Note: we do not have to check the thread ID for two reasons.
duke@435	5141	// First, the interpreter checks for IllegalMonitorStateException at
duke@435	5142	// a higher level. Second, if the bias was revoked while we held the
duke@435	5143	// lock, the object could not be rebiased toward another thread, so
duke@435	5144	// the bias bit would be clear.
duke@435	5145	movq(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
duke@435	5146	andq(temp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	5147	cmpq(temp_reg, markOopDesc::biased_lock_pattern);
duke@435	5148	jcc(Assembler::equal, done);
duke@435	5149	}
duke@435	5150
duke@435	5151
coleenp@548	5152	void MacroAssembler::load_klass(Register dst, Register src) {
coleenp@548	5153	if (UseCompressedOops) {
coleenp@548	5154	movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
coleenp@548	5155	decode_heap_oop_not_null(dst);
coleenp@548	5156	} else {
coleenp@548	5157	movq(dst, Address(src, oopDesc::klass_offset_in_bytes()));
coleenp@548	5158	}
coleenp@548	5159	}
coleenp@548	5160
coleenp@548	5161	void MacroAssembler::store_klass(Register dst, Register src) {
coleenp@548	5162	if (UseCompressedOops) {
coleenp@548	5163	encode_heap_oop_not_null(src);
coleenp@602	5164	movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
coleenp@602	5165	} else {
coleenp@602	5166	movq(Address(dst, oopDesc::klass_offset_in_bytes()), src);
coleenp@548	5167	}
coleenp@602	5168	}
coleenp@602	5169
coleenp@602	5170	void MacroAssembler::store_klass_gap(Register dst, Register src) {
coleenp@602	5171	if (UseCompressedOops) {
coleenp@602	5172	// Store to klass gap in destination
coleenp@602	5173	movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
coleenp@602	5174	}
coleenp@548	5175	}
coleenp@548	5176
coleenp@548	5177	void MacroAssembler::load_heap_oop(Register dst, Address src) {
coleenp@548	5178	if (UseCompressedOops) {
coleenp@548	5179	movl(dst, src);
coleenp@548	5180	decode_heap_oop(dst);
coleenp@548	5181	} else {
coleenp@548	5182	movq(dst, src);
coleenp@548	5183	}
coleenp@548	5184	}
coleenp@548	5185
coleenp@548	5186	void MacroAssembler::store_heap_oop(Address dst, Register src) {
coleenp@548	5187	if (UseCompressedOops) {
coleenp@548	5188	assert(!dst.uses(src), "not enough registers");
coleenp@548	5189	encode_heap_oop(src);
coleenp@548	5190	movl(dst, src);
coleenp@548	5191	} else {
coleenp@548	5192	movq(dst, src);
coleenp@548	5193	}
coleenp@548	5194	}
coleenp@548	5195
coleenp@548	5196	// Algorithm must match oop.inline.hpp encode_heap_oop.
coleenp@548	5197	void MacroAssembler::encode_heap_oop(Register r) {
coleenp@548	5198	assert (UseCompressedOops, "should be compressed");
coleenp@548	5199	#ifdef ASSERT
kvn@599	5200	if (CheckCompressedOops) {
kvn@599	5201	Label ok;
kvn@599	5202	pushq(rscratch1); // cmpptr trashes rscratch1
kvn@599	5203	cmpptr(r12_heapbase, ExternalAddress((address)Universe::heap_base_addr()));
kvn@599	5204	jcc(Assembler::equal, ok);
kvn@599	5205	stop("MacroAssembler::encode_heap_oop: heap base corrupted?");
kvn@599	5206	bind(ok);
kvn@599	5207	popq(rscratch1);
kvn@599	5208	}
coleenp@548	5209	#endif
kvn@559	5210	verify_oop(r, "broken oop in encode_heap_oop");
coleenp@548	5211	testq(r, r);
coleenp@548	5212	cmovq(Assembler::equal, r, r12_heapbase);
coleenp@548	5213	subq(r, r12_heapbase);
coleenp@548	5214	shrq(r, LogMinObjAlignmentInBytes);
coleenp@548	5215	}
coleenp@548	5216
coleenp@548	5217	void MacroAssembler::encode_heap_oop_not_null(Register r) {
coleenp@548	5218	assert (UseCompressedOops, "should be compressed");
coleenp@548	5219	#ifdef ASSERT
kvn@599	5220	if (CheckCompressedOops) {
kvn@599	5221	Label ok;
kvn@599	5222	testq(r, r);
kvn@599	5223	jcc(Assembler::notEqual, ok);
kvn@599	5224	stop("null oop passed to encode_heap_oop_not_null");
kvn@599	5225	bind(ok);
kvn@599	5226	}
coleenp@548	5227	#endif
kvn@559	5228	verify_oop(r, "broken oop in encode_heap_oop_not_null");
coleenp@548	5229	subq(r, r12_heapbase);
coleenp@548	5230	shrq(r, LogMinObjAlignmentInBytes);
coleenp@548	5231	}
coleenp@548	5232
kvn@559	5233	void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
kvn@559	5234	assert (UseCompressedOops, "should be compressed");
kvn@559	5235	#ifdef ASSERT
kvn@599	5236	if (CheckCompressedOops) {
kvn@599	5237	Label ok;
kvn@599	5238	testq(src, src);
kvn@599	5239	jcc(Assembler::notEqual, ok);
kvn@599	5240	stop("null oop passed to encode_heap_oop_not_null2");
kvn@599	5241	bind(ok);
kvn@599	5242	}
kvn@559	5243	#endif
kvn@559	5244	verify_oop(src, "broken oop in encode_heap_oop_not_null2");
kvn@559	5245	if (dst != src) {
kvn@559	5246	movq(dst, src);
kvn@559	5247	}
kvn@559	5248	subq(dst, r12_heapbase);
kvn@559	5249	shrq(dst, LogMinObjAlignmentInBytes);
kvn@559	5250	}
kvn@559	5251
coleenp@548	5252	void MacroAssembler::decode_heap_oop(Register r) {
coleenp@548	5253	assert (UseCompressedOops, "should be compressed");
coleenp@548	5254	#ifdef ASSERT
kvn@599	5255	if (CheckCompressedOops) {
kvn@599	5256	Label ok;
kvn@599	5257	pushq(rscratch1);
kvn@599	5258	cmpptr(r12_heapbase,
kvn@599	5259	ExternalAddress((address)Universe::heap_base_addr()));
kvn@599	5260	jcc(Assembler::equal, ok);
kvn@599	5261	stop("MacroAssembler::decode_heap_oop: heap base corrupted?");
kvn@599	5262	bind(ok);
kvn@599	5263	popq(rscratch1);
kvn@599	5264	}
coleenp@548	5265	#endif
coleenp@548	5266
coleenp@548	5267	Label done;
coleenp@548	5268	shlq(r, LogMinObjAlignmentInBytes);
coleenp@548	5269	jccb(Assembler::equal, done);
coleenp@548	5270	addq(r, r12_heapbase);
coleenp@548	5271	#if 0
coleenp@548	5272	// alternate decoding probably a wash.
coleenp@548	5273	testq(r, r);
coleenp@548	5274	jccb(Assembler::equal, done);
coleenp@548	5275	leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
coleenp@548	5276	#endif
coleenp@548	5277	bind(done);
kvn@559	5278	verify_oop(r, "broken oop in decode_heap_oop");
coleenp@548	5279	}
coleenp@548	5280
coleenp@548	5281	void MacroAssembler::decode_heap_oop_not_null(Register r) {
coleenp@548	5282	assert (UseCompressedOops, "should only be used for compressed headers");
coleenp@548	5283	// Cannot assert, unverified entry point counts instructions (see .ad file)
coleenp@548	5284	// vtableStubs also counts instructions in pd_code_size_limit.
coleenp@548	5285	assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
coleenp@548	5286	leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
coleenp@548	5287	}
coleenp@548	5288
kvn@559	5289	void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
kvn@559	5290	assert (UseCompressedOops, "should only be used for compressed headers");
kvn@559	5291	// Cannot assert, unverified entry point counts instructions (see .ad file)
kvn@559	5292	// vtableStubs also counts instructions in pd_code_size_limit.
kvn@559	5293	assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
kvn@559	5294	leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
kvn@559	5295	}
kvn@559	5296
kvn@599	5297	void MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
kvn@599	5298	assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
kvn@599	5299	int oop_index = oop_recorder()->find_index(obj);
kvn@599	5300	RelocationHolder rspec = oop_Relocation::spec(oop_index);
kvn@599	5301
kvn@599	5302	// movl dst,obj
kvn@599	5303	InstructionMark im(this);
kvn@599	5304	int encode = prefix_and_encode(dst->encoding());
kvn@599	5305	emit_byte(0xB8 | encode);
kvn@599	5306	emit_data(oop_index, rspec, narrow_oop_operand);
kvn@599	5307	}
kvn@599	5308
kvn@599	5309
duke@435	5310	Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
duke@435	5311	switch (cond) {
duke@435	5312	// Note some conditions are synonyms for others
duke@435	5313	case Assembler::zero: return Assembler::notZero;
duke@435	5314	case Assembler::notZero: return Assembler::zero;
duke@435	5315	case Assembler::less: return Assembler::greaterEqual;
duke@435	5316	case Assembler::lessEqual: return Assembler::greater;
duke@435	5317	case Assembler::greater: return Assembler::lessEqual;
duke@435	5318	case Assembler::greaterEqual: return Assembler::less;
duke@435	5319	case Assembler::below: return Assembler::aboveEqual;
duke@435	5320	case Assembler::belowEqual: return Assembler::above;
duke@435	5321	case Assembler::above: return Assembler::belowEqual;
duke@435	5322	case Assembler::aboveEqual: return Assembler::below;
duke@435	5323	case Assembler::overflow: return Assembler::noOverflow;
duke@435	5324	case Assembler::noOverflow: return Assembler::overflow;
duke@435	5325	case Assembler::negative: return Assembler::positive;
duke@435	5326	case Assembler::positive: return Assembler::negative;
duke@435	5327	case Assembler::parity: return Assembler::noParity;
duke@435	5328	case Assembler::noParity: return Assembler::parity;
duke@435	5329	}
duke@435	5330	ShouldNotReachHere(); return Assembler::overflow;
duke@435	5331	}
duke@435	5332
duke@435	5333
duke@435	5334	void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
duke@435	5335	Condition negated_cond = negate_condition(cond);
duke@435	5336	Label L;
duke@435	5337	jcc(negated_cond, L);
duke@435	5338	atomic_incl(counter_addr);
duke@435	5339	bind(L);
duke@435	5340	}
duke@435	5341
duke@435	5342	void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
duke@435	5343	pushfq();
duke@435	5344	if (os::is_MP())
duke@435	5345	lock();
duke@435	5346	incrementl(counter_addr);
duke@435	5347	popfq();
duke@435	5348	}
duke@435	5349
duke@435	5350	SkipIfEqual::SkipIfEqual(
duke@435	5351	MacroAssembler* masm, const bool* flag_addr, bool value) {
duke@435	5352	_masm = masm;
duke@435	5353	_masm->cmp8(ExternalAddress((address)flag_addr), value);
duke@435	5354	_masm->jcc(Assembler::equal, _label);
duke@435	5355	}
duke@435	5356
duke@435	5357	SkipIfEqual::~SkipIfEqual() {
duke@435	5358	_masm->bind(_label);
duke@435	5359	}
duke@435	5360
duke@435	5361	void MacroAssembler::bang_stack_size(Register size, Register tmp) {
duke@435	5362	movq(tmp, rsp);
duke@435	5363	// Bang stack for total size given plus shadow page size.
duke@435	5364	// Bang one page at a time because large size can bang beyond yellow and
duke@435	5365	// red zones.
duke@435	5366	Label loop;
duke@435	5367	bind(loop);
duke@435	5368	movl(Address(tmp, (-os::vm_page_size())), size );
duke@435	5369	subq(tmp, os::vm_page_size());
duke@435	5370	subl(size, os::vm_page_size());
duke@435	5371	jcc(Assembler::greater, loop);
duke@435	5372
duke@435	5373	// Bang down shadow pages too.
duke@435	5374	// The -1 because we already subtracted 1 page.
duke@435	5375	for (int i = 0; i< StackShadowPages-1; i++) {
duke@435	5376	movq(Address(tmp, (-i*os::vm_page_size())), size );
duke@435	5377	}
duke@435	5378	}
coleenp@548	5379
coleenp@548	5380	void MacroAssembler::reinit_heapbase() {
coleenp@548	5381	if (UseCompressedOops) {
coleenp@548	5382	movptr(r12_heapbase, ExternalAddress((address)Universe::heap_base_addr()));
coleenp@548	5383	}
coleenp@548	5384	}