• file
• revisions
• annotate
• diff
• comparison
• raw

src/cpu/x86/vm/assembler_x86_64.cpp@a61af66fc99e (annotated)

src/cpu/x86/vm/assembler_x86_64.cpp

Sat, 01 Dec 2007 00:00:00 +0000

author

duke

date

Sat, 01 Dec 2007 00:00:00 +0000

changeset 435

a61af66fc99e

child 485

485d403e94e1

child 506

3d62cb85208d

permissions

-rw-r--r--

Initial load

duke@435	1	/*
duke@435	2	* Copyright 2003-2007 Sun Microsystems, Inc. All Rights Reserved.
duke@435	3	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
duke@435	4	*
duke@435	5	* This code is free software; you can redistribute it and/or modify it
duke@435	6	* under the terms of the GNU General Public License version 2 only, as
duke@435	7	* published by the Free Software Foundation.
duke@435	8	*
duke@435	9	* This code is distributed in the hope that it will be useful, but WITHOUT
duke@435	10	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
duke@435	11	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
duke@435	12	* version 2 for more details (a copy is included in the LICENSE file that
duke@435	13	* accompanied this code).
duke@435	14	*
duke@435	15	* You should have received a copy of the GNU General Public License version
duke@435	16	* 2 along with this work; if not, write to the Free Software Foundation,
duke@435	17	* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
duke@435	18	*
duke@435	19	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
duke@435	20	* CA 95054 USA or visit www.sun.com if you need additional information or
duke@435	21	* have any questions.
duke@435	22	*
duke@435	23	*/
duke@435	24
duke@435	25	#include "incls/_precompiled.incl"
duke@435	26	#include "incls/_assembler_x86_64.cpp.incl"
duke@435	27
duke@435	28	// Implementation of AddressLiteral
duke@435	29
duke@435	30	AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
duke@435	31	_is_lval = false;
duke@435	32	_target = target;
duke@435	33	switch (rtype) {
duke@435	34	case relocInfo::oop_type:
duke@435	35	// Oops are a special case. Normally they would be their own section
duke@435	36	// but in cases like icBuffer they are literals in the code stream that
duke@435	37	// we don't have a section for. We use none so that we get a literal address
duke@435	38	// which is always patchable.
duke@435	39	break;
duke@435	40	case relocInfo::external_word_type:
duke@435	41	_rspec = external_word_Relocation::spec(target);
duke@435	42	break;
duke@435	43	case relocInfo::internal_word_type:
duke@435	44	_rspec = internal_word_Relocation::spec(target);
duke@435	45	break;
duke@435	46	case relocInfo::opt_virtual_call_type:
duke@435	47	_rspec = opt_virtual_call_Relocation::spec();
duke@435	48	break;
duke@435	49	case relocInfo::static_call_type:
duke@435	50	_rspec = static_call_Relocation::spec();
duke@435	51	break;
duke@435	52	case relocInfo::runtime_call_type:
duke@435	53	_rspec = runtime_call_Relocation::spec();
duke@435	54	break;
duke@435	55	case relocInfo::none:
duke@435	56	break;
duke@435	57	default:
duke@435	58	ShouldNotReachHere();
duke@435	59	break;
duke@435	60	}
duke@435	61	}
duke@435	62
duke@435	63	// Implementation of Address
duke@435	64
duke@435	65	Address Address::make_array(ArrayAddress adr) {
duke@435	66	#ifdef _LP64
duke@435	67	// Not implementable on 64bit machines
duke@435	68	// Should have been handled higher up the call chain.
duke@435	69	ShouldNotReachHere();
duke@435	70	return Address();
duke@435	71	#else
duke@435	72	AddressLiteral base = adr.base();
duke@435	73	Address index = adr.index();
duke@435	74	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	75	Address array(index._base, index._index, index._scale, (intptr_t) base.target());
duke@435	76	array._rspec = base._rspec;
duke@435	77	return array;
duke@435	78	#endif // _LP64
duke@435	79	}
duke@435	80
duke@435	81	// exceedingly dangerous constructor
duke@435	82	Address::Address(int disp, address loc, relocInfo::relocType rtype) {
duke@435	83	_base = noreg;
duke@435	84	_index = noreg;
duke@435	85	_scale = no_scale;
duke@435	86	_disp = disp;
duke@435	87	switch (rtype) {
duke@435	88	case relocInfo::external_word_type:
duke@435	89	_rspec = external_word_Relocation::spec(loc);
duke@435	90	break;
duke@435	91	case relocInfo::internal_word_type:
duke@435	92	_rspec = internal_word_Relocation::spec(loc);
duke@435	93	break;
duke@435	94	case relocInfo::runtime_call_type:
duke@435	95	// HMM
duke@435	96	_rspec = runtime_call_Relocation::spec();
duke@435	97	break;
duke@435	98	case relocInfo::none:
duke@435	99	break;
duke@435	100	default:
duke@435	101	ShouldNotReachHere();
duke@435	102	}
duke@435	103	}
duke@435	104
duke@435	105	// Convert the raw encoding form into the form expected by the constructor for
duke@435	106	// Address. An index of 4 (rsp) corresponds to having no index, so convert
duke@435	107	// that to noreg for the Address constructor.
duke@435	108	Address Address::make_raw(int base, int index, int scale, int disp) {
duke@435	109	bool valid_index = index != rsp->encoding();
duke@435	110	if (valid_index) {
duke@435	111	Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
duke@435	112	return madr;
duke@435	113	} else {
duke@435	114	Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
duke@435	115	return madr;
duke@435	116	}
duke@435	117	}
duke@435	118
duke@435	119
duke@435	120	// Implementation of Assembler
duke@435	121	int AbstractAssembler::code_fill_byte() {
duke@435	122	return (u_char)'\xF4'; // hlt
duke@435	123	}
duke@435	124
duke@435	125	// This should only be used by 64bit instructions that can use rip-relative
duke@435	126	// it cannot be used by instructions that want an immediate value.
duke@435	127
duke@435	128	bool Assembler::reachable(AddressLiteral adr) {
duke@435	129	int64_t disp;
duke@435	130	// None will force a 64bit literal to the code stream. Likely a placeholder
duke@435	131	// for something that will be patched later and we need to certain it will
duke@435	132	// always be reachable.
duke@435	133	if (adr.reloc() == relocInfo::none) {
duke@435	134	return false;
duke@435	135	}
duke@435	136	if (adr.reloc() == relocInfo::internal_word_type) {
duke@435	137	// This should be rip relative and easily reachable.
duke@435	138	return true;
duke@435	139	}
duke@435	140	if (adr.reloc() != relocInfo::external_word_type &&
duke@435	141	adr.reloc() != relocInfo::runtime_call_type ) {
duke@435	142	return false;
duke@435	143	}
duke@435	144
duke@435	145	// Stress the correction code
duke@435	146	if (ForceUnreachable) {
duke@435	147	// Must be runtimecall reloc, see if it is in the codecache
duke@435	148	// Flipping stuff in the codecache to be unreachable causes issues
duke@435	149	// with things like inline caches where the additional instructions
duke@435	150	// are not handled.
duke@435	151	if (CodeCache::find_blob(adr._target) == NULL) {
duke@435	152	return false;
duke@435	153	}
duke@435	154	}
duke@435	155	// For external_word_type/runtime_call_type if it is reachable from where we
duke@435	156	// are now (possibly a temp buffer) and where we might end up
duke@435	157	// anywhere in the codeCache then we are always reachable.
duke@435	158	// This would have to change if we ever save/restore shared code
duke@435	159	// to be more pessimistic.
duke@435	160
duke@435	161	disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
duke@435	162	if (!is_simm32(disp)) return false;
duke@435	163	disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
duke@435	164	if (!is_simm32(disp)) return false;
duke@435	165
duke@435	166	disp = (int64_t)adr._target - ((int64_t)_code_pos + sizeof(int));
duke@435	167
duke@435	168	// Because rip relative is a disp + address_of_next_instruction and we
duke@435	169	// don't know the value of address_of_next_instruction we apply a fudge factor
duke@435	170	// to make sure we will be ok no matter the size of the instruction we get placed into.
duke@435	171	// We don't have to fudge the checks above here because they are already worst case.
duke@435	172
duke@435	173	// 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
duke@435	174	// + 4 because better safe than sorry.
duke@435	175	const int fudge = 12 + 4;
duke@435	176	if (disp < 0) {
duke@435	177	disp -= fudge;
duke@435	178	} else {
duke@435	179	disp += fudge;
duke@435	180	}
duke@435	181	return is_simm32(disp);
duke@435	182	}
duke@435	183
duke@435	184
duke@435	185	// make this go away eventually
duke@435	186	void Assembler::emit_data(jint data,
duke@435	187	relocInfo::relocType rtype,
duke@435	188	int format) {
duke@435	189	if (rtype == relocInfo::none) {
duke@435	190	emit_long(data);
duke@435	191	} else {
duke@435	192	emit_data(data, Relocation::spec_simple(rtype), format);
duke@435	193	}
duke@435	194	}
duke@435	195
duke@435	196	void Assembler::emit_data(jint data,
duke@435	197	RelocationHolder const& rspec,
duke@435	198	int format) {
duke@435	199	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	200	assert(imm64_operand != format, "must not be imm64");
duke@435	201	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	202	if (rspec.type() != relocInfo::none) {
duke@435	203	#ifdef ASSERT
duke@435	204	check_relocation(rspec, format);
duke@435	205	#endif
duke@435	206	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	207	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	208
duke@435	209	// hack. call32 is too wide for mask so use disp32
duke@435	210	if (format == call32_operand)
duke@435	211	code_section()->relocate(inst_mark(), rspec, disp32_operand);
duke@435	212	else
duke@435	213	code_section()->relocate(inst_mark(), rspec, format);
duke@435	214	}
duke@435	215	emit_long(data);
duke@435	216	}
duke@435	217
duke@435	218	void Assembler::emit_data64(jlong data,
duke@435	219	relocInfo::relocType rtype,
duke@435	220	int format) {
duke@435	221	if (rtype == relocInfo::none) {
duke@435	222	emit_long64(data);
duke@435	223	} else {
duke@435	224	emit_data64(data, Relocation::spec_simple(rtype), format);
duke@435	225	}
duke@435	226	}
duke@435	227
duke@435	228	void Assembler::emit_data64(jlong data,
duke@435	229	RelocationHolder const& rspec,
duke@435	230	int format) {
duke@435	231	assert(imm64_operand == 0, "default format must be imm64 in this file");
duke@435	232	assert(imm64_operand == format, "must be imm64");
duke@435	233	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	234	// Do not use AbstractAssembler::relocate, which is not intended for
duke@435	235	// embedded words. Instead, relocate to the enclosing instruction.
duke@435	236	code_section()->relocate(inst_mark(), rspec, format);
duke@435	237	#ifdef ASSERT
duke@435	238	check_relocation(rspec, format);
duke@435	239	#endif
duke@435	240	emit_long64(data);
duke@435	241	}
duke@435	242
duke@435	243	void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
duke@435	244	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	245	assert(isByte(imm8), "not a byte");
duke@435	246	assert((op1 & 0x01) == 0, "should be 8bit operation");
duke@435	247	int dstenc = dst->encoding();
duke@435	248	if (dstenc >= 8) {
duke@435	249	dstenc -= 8;
duke@435	250	}
duke@435	251	emit_byte(op1);
duke@435	252	emit_byte(op2 | dstenc);
duke@435	253	emit_byte(imm8);
duke@435	254	}
duke@435	255
duke@435	256	void Assembler::emit_arith(int op1, int op2, Register dst, int imm32) {
duke@435	257	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	258	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	259	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	260	int dstenc = dst->encoding();
duke@435	261	if (dstenc >= 8) {
duke@435	262	dstenc -= 8;
duke@435	263	}
duke@435	264	if (is8bit(imm32)) {
duke@435	265	emit_byte(op1 | 0x02); // set sign bit
duke@435	266	emit_byte(op2 | dstenc);
duke@435	267	emit_byte(imm32 & 0xFF);
duke@435	268	} else {
duke@435	269	emit_byte(op1);
duke@435	270	emit_byte(op2 | dstenc);
duke@435	271	emit_long(imm32);
duke@435	272	}
duke@435	273	}
duke@435	274
duke@435	275	// immediate-to-memory forms
duke@435	276	void Assembler::emit_arith_operand(int op1,
duke@435	277	Register rm, Address adr,
duke@435	278	int imm32) {
duke@435	279	assert((op1 & 0x01) == 1, "should be 32bit operation");
duke@435	280	assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
duke@435	281	if (is8bit(imm32)) {
duke@435	282	emit_byte(op1 | 0x02); // set sign bit
duke@435	283	emit_operand(rm, adr, 1);
duke@435	284	emit_byte(imm32 & 0xFF);
duke@435	285	} else {
duke@435	286	emit_byte(op1);
duke@435	287	emit_operand(rm, adr, 4);
duke@435	288	emit_long(imm32);
duke@435	289	}
duke@435	290	}
duke@435	291
duke@435	292
duke@435	293	void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
duke@435	294	assert(isByte(op1) && isByte(op2), "wrong opcode");
duke@435	295	int dstenc = dst->encoding();
duke@435	296	int srcenc = src->encoding();
duke@435	297	if (dstenc >= 8) {
duke@435	298	dstenc -= 8;
duke@435	299	}
duke@435	300	if (srcenc >= 8) {
duke@435	301	srcenc -= 8;
duke@435	302	}
duke@435	303	emit_byte(op1);
duke@435	304	emit_byte(op2 | dstenc << 3 | srcenc);
duke@435	305	}
duke@435	306
duke@435	307	void Assembler::emit_operand(Register reg, Register base, Register index,
duke@435	308	Address::ScaleFactor scale, int disp,
duke@435	309	RelocationHolder const& rspec,
duke@435	310	int rip_relative_correction) {
duke@435	311	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	312	int regenc = reg->encoding();
duke@435	313	if (regenc >= 8) {
duke@435	314	regenc -= 8;
duke@435	315	}
duke@435	316	if (base->is_valid()) {
duke@435	317	if (index->is_valid()) {
duke@435	318	assert(scale != Address::no_scale, "inconsistent address");
duke@435	319	int indexenc = index->encoding();
duke@435	320	if (indexenc >= 8) {
duke@435	321	indexenc -= 8;
duke@435	322	}
duke@435	323	int baseenc = base->encoding();
duke@435	324	if (baseenc >= 8) {
duke@435	325	baseenc -= 8;
duke@435	326	}
duke@435	327	// [base + index*scale + disp]
duke@435	328	if (disp == 0 && rtype == relocInfo::none &&
duke@435	329	base != rbp && base != r13) {
duke@435	330	// [base + index*scale]
duke@435	331	// [00 reg 100][ss index base]
duke@435	332	assert(index != rsp, "illegal addressing mode");
duke@435	333	emit_byte(0x04 | regenc << 3);
duke@435	334	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	335	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	336	// [base + index*scale + imm8]
duke@435	337	// [01 reg 100][ss index base] imm8
duke@435	338	assert(index != rsp, "illegal addressing mode");
duke@435	339	emit_byte(0x44 | regenc << 3);
duke@435	340	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	341	emit_byte(disp & 0xFF);
duke@435	342	} else {
duke@435	343	// [base + index*scale + disp32]
duke@435	344	// [10 reg 100][ss index base] disp32
duke@435	345	assert(index != rsp, "illegal addressing mode");
duke@435	346	emit_byte(0x84 | regenc << 3);
duke@435	347	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	348	emit_data(disp, rspec, disp32_operand);
duke@435	349	}
duke@435	350	} else if (base == rsp || base == r12) {
duke@435	351	// [rsp + disp]
duke@435	352	if (disp == 0 && rtype == relocInfo::none) {
duke@435	353	// [rsp]
duke@435	354	// [00 reg 100][00 100 100]
duke@435	355	emit_byte(0x04 | regenc << 3);
duke@435	356	emit_byte(0x24);
duke@435	357	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	358	// [rsp + imm8]
duke@435	359	// [01 reg 100][00 100 100] disp8
duke@435	360	emit_byte(0x44 | regenc << 3);
duke@435	361	emit_byte(0x24);
duke@435	362	emit_byte(disp & 0xFF);
duke@435	363	} else {
duke@435	364	// [rsp + imm32]
duke@435	365	// [10 reg 100][00 100 100] disp32
duke@435	366	emit_byte(0x84 | regenc << 3);
duke@435	367	emit_byte(0x24);
duke@435	368	emit_data(disp, rspec, disp32_operand);
duke@435	369	}
duke@435	370	} else {
duke@435	371	// [base + disp]
duke@435	372	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	373	int baseenc = base->encoding();
duke@435	374	if (baseenc >= 8) {
duke@435	375	baseenc -= 8;
duke@435	376	}
duke@435	377	if (disp == 0 && rtype == relocInfo::none &&
duke@435	378	base != rbp && base != r13) {
duke@435	379	// [base]
duke@435	380	// [00 reg base]
duke@435	381	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	382	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	383	// [base + disp8]
duke@435	384	// [01 reg base] disp8
duke@435	385	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	386	emit_byte(disp & 0xFF);
duke@435	387	} else {
duke@435	388	// [base + disp32]
duke@435	389	// [10 reg base] disp32
duke@435	390	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	391	emit_data(disp, rspec, disp32_operand);
duke@435	392	}
duke@435	393	}
duke@435	394	} else {
duke@435	395	if (index->is_valid()) {
duke@435	396	assert(scale != Address::no_scale, "inconsistent address");
duke@435	397	int indexenc = index->encoding();
duke@435	398	if (indexenc >= 8) {
duke@435	399	indexenc -= 8;
duke@435	400	}
duke@435	401	// [index*scale + disp]
duke@435	402	// [00 reg 100][ss index 101] disp32
duke@435	403	assert(index != rsp, "illegal addressing mode");
duke@435	404	emit_byte(0x04 | regenc << 3);
duke@435	405	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	406	emit_data(disp, rspec, disp32_operand);
duke@435	407	#ifdef _LP64
duke@435	408	} else if (rtype != relocInfo::none ) {
duke@435	409	// [disp] RIP-RELATIVE
duke@435	410	// [00 000 101] disp32
duke@435	411
duke@435	412	emit_byte(0x05 | regenc << 3);
duke@435	413	// Note that the RIP-rel. correction applies to the generated
duke@435	414	// disp field, but _not_ to the target address in the rspec.
duke@435	415
duke@435	416	// disp was created by converting the target address minus the pc
duke@435	417	// at the start of the instruction. That needs more correction here.
duke@435	418	// intptr_t disp = target - next_ip;
duke@435	419	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	420	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	421	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	422	assert(is_simm32(adjusted),
duke@435	423	"must be 32bit offset (RIP relative address)");
duke@435	424	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	425
duke@435	426	#endif // _LP64
duke@435	427	} else {
duke@435	428	// [disp] ABSOLUTE
duke@435	429	// [00 reg 100][00 100 101] disp32
duke@435	430	emit_byte(0x04 | regenc << 3);
duke@435	431	emit_byte(0x25);
duke@435	432	emit_data(disp, rspec, disp32_operand);
duke@435	433	}
duke@435	434	}
duke@435	435	}
duke@435	436
duke@435	437	void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
duke@435	438	Address::ScaleFactor scale, int disp,
duke@435	439	RelocationHolder const& rspec,
duke@435	440	int rip_relative_correction) {
duke@435	441	relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
duke@435	442	int regenc = reg->encoding();
duke@435	443	if (regenc >= 8) {
duke@435	444	regenc -= 8;
duke@435	445	}
duke@435	446	if (base->is_valid()) {
duke@435	447	if (index->is_valid()) {
duke@435	448	assert(scale != Address::no_scale, "inconsistent address");
duke@435	449	int indexenc = index->encoding();
duke@435	450	if (indexenc >= 8) {
duke@435	451	indexenc -= 8;
duke@435	452	}
duke@435	453	int baseenc = base->encoding();
duke@435	454	if (baseenc >= 8) {
duke@435	455	baseenc -= 8;
duke@435	456	}
duke@435	457	// [base + index*scale + disp]
duke@435	458	if (disp == 0 && rtype == relocInfo::none &&
duke@435	459	base != rbp && base != r13) {
duke@435	460	// [base + index*scale]
duke@435	461	// [00 reg 100][ss index base]
duke@435	462	assert(index != rsp, "illegal addressing mode");
duke@435	463	emit_byte(0x04 | regenc << 3);
duke@435	464	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	465	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	466	// [base + index*scale + disp8]
duke@435	467	// [01 reg 100][ss index base] disp8
duke@435	468	assert(index != rsp, "illegal addressing mode");
duke@435	469	emit_byte(0x44 | regenc << 3);
duke@435	470	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	471	emit_byte(disp & 0xFF);
duke@435	472	} else {
duke@435	473	// [base + index*scale + disp32]
duke@435	474	// [10 reg 100][ss index base] disp32
duke@435	475	assert(index != rsp, "illegal addressing mode");
duke@435	476	emit_byte(0x84 | regenc << 3);
duke@435	477	emit_byte(scale << 6 | indexenc << 3 | baseenc);
duke@435	478	emit_data(disp, rspec, disp32_operand);
duke@435	479	}
duke@435	480	} else if (base == rsp || base == r12) {
duke@435	481	// [rsp + disp]
duke@435	482	if (disp == 0 && rtype == relocInfo::none) {
duke@435	483	// [rsp]
duke@435	484	// [00 reg 100][00 100 100]
duke@435	485	emit_byte(0x04 | regenc << 3);
duke@435	486	emit_byte(0x24);
duke@435	487	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	488	// [rsp + imm8]
duke@435	489	// [01 reg 100][00 100 100] disp8
duke@435	490	emit_byte(0x44 | regenc << 3);
duke@435	491	emit_byte(0x24);
duke@435	492	emit_byte(disp & 0xFF);
duke@435	493	} else {
duke@435	494	// [rsp + imm32]
duke@435	495	// [10 reg 100][00 100 100] disp32
duke@435	496	emit_byte(0x84 | regenc << 3);
duke@435	497	emit_byte(0x24);
duke@435	498	emit_data(disp, rspec, disp32_operand);
duke@435	499	}
duke@435	500	} else {
duke@435	501	// [base + disp]
duke@435	502	assert(base != rsp && base != r12, "illegal addressing mode");
duke@435	503	int baseenc = base->encoding();
duke@435	504	if (baseenc >= 8) {
duke@435	505	baseenc -= 8;
duke@435	506	}
duke@435	507	if (disp == 0 && rtype == relocInfo::none &&
duke@435	508	base != rbp && base != r13) {
duke@435	509	// [base]
duke@435	510	// [00 reg base]
duke@435	511	emit_byte(0x00 | regenc << 3 | baseenc);
duke@435	512	} else if (is8bit(disp) && rtype == relocInfo::none) {
duke@435	513	// [base + imm8]
duke@435	514	// [01 reg base] disp8
duke@435	515	emit_byte(0x40 | regenc << 3 | baseenc);
duke@435	516	emit_byte(disp & 0xFF);
duke@435	517	} else {
duke@435	518	// [base + imm32]
duke@435	519	// [10 reg base] disp32
duke@435	520	emit_byte(0x80 | regenc << 3 | baseenc);
duke@435	521	emit_data(disp, rspec, disp32_operand);
duke@435	522	}
duke@435	523	}
duke@435	524	} else {
duke@435	525	if (index->is_valid()) {
duke@435	526	assert(scale != Address::no_scale, "inconsistent address");
duke@435	527	int indexenc = index->encoding();
duke@435	528	if (indexenc >= 8) {
duke@435	529	indexenc -= 8;
duke@435	530	}
duke@435	531	// [index*scale + disp]
duke@435	532	// [00 reg 100][ss index 101] disp32
duke@435	533	assert(index != rsp, "illegal addressing mode");
duke@435	534	emit_byte(0x04 | regenc << 3);
duke@435	535	emit_byte(scale << 6 | indexenc << 3 | 0x05);
duke@435	536	emit_data(disp, rspec, disp32_operand);
duke@435	537	#ifdef _LP64
duke@435	538	} else if ( rtype != relocInfo::none ) {
duke@435	539	// [disp] RIP-RELATIVE
duke@435	540	// [00 reg 101] disp32
duke@435	541	emit_byte(0x05 | regenc << 3);
duke@435	542	// Note that the RIP-rel. correction applies to the generated
duke@435	543	// disp field, but _not_ to the target address in the rspec.
duke@435	544
duke@435	545	// disp was created by converting the target address minus the pc
duke@435	546	// at the start of the instruction. That needs more correction here.
duke@435	547	// intptr_t disp = target - next_ip;
duke@435	548
duke@435	549	assert(inst_mark() != NULL, "must be inside InstructionMark");
duke@435	550	address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
duke@435	551
duke@435	552	int64_t adjusted = (int64_t) disp - (next_ip - inst_mark());
duke@435	553	assert(is_simm32(adjusted),
duke@435	554	"must be 32bit offset (RIP relative address)");
duke@435	555	emit_data((int) adjusted, rspec, disp32_operand);
duke@435	556	#endif // _LP64
duke@435	557	} else {
duke@435	558	// [disp] ABSOLUTE
duke@435	559	// [00 reg 100][00 100 101] disp32
duke@435	560	emit_byte(0x04 | regenc << 3);
duke@435	561	emit_byte(0x25);
duke@435	562	emit_data(disp, rspec, disp32_operand);
duke@435	563	}
duke@435	564	}
duke@435	565	}
duke@435	566
duke@435	567	// Secret local extension to Assembler::WhichOperand:
duke@435	568	#define end_pc_operand (_WhichOperand_limit)
duke@435	569
duke@435	570	address Assembler::locate_operand(address inst, WhichOperand which) {
duke@435	571	// Decode the given instruction, and return the address of
duke@435	572	// an embedded 32-bit operand word.
duke@435	573
duke@435	574	// If "which" is disp32_operand, selects the displacement portion
duke@435	575	// of an effective address specifier.
duke@435	576	// If "which" is imm64_operand, selects the trailing immediate constant.
duke@435	577	// If "which" is call32_operand, selects the displacement of a call or jump.
duke@435	578	// Caller is responsible for ensuring that there is such an operand,
duke@435	579	// and that it is 32/64 bits wide.
duke@435	580
duke@435	581	// If "which" is end_pc_operand, find the end of the instruction.
duke@435	582
duke@435	583	address ip = inst;
duke@435	584	bool is_64bit = false;
duke@435	585
duke@435	586	debug_only(bool has_disp32 = false);
duke@435	587	int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
duke@435	588
duke@435	589	again_after_prefix:
duke@435	590	switch (0xFF & *ip++) {
duke@435	591
duke@435	592	// These convenience macros generate groups of "case" labels for the switch.
duke@435	593	#define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
duke@435	594	#define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
duke@435	595	case (x)+4: case (x)+5: case (x)+6: case (x)+7
duke@435	596	#define REP16(x) REP8((x)+0): \
duke@435	597	case REP8((x)+8)
duke@435	598
duke@435	599	case CS_segment:
duke@435	600	case SS_segment:
duke@435	601	case DS_segment:
duke@435	602	case ES_segment:
duke@435	603	case FS_segment:
duke@435	604	case GS_segment:
duke@435	605	assert(0, "shouldn't have that prefix");
duke@435	606	assert(ip == inst + 1 || ip == inst + 2, "only two prefixes allowed");
duke@435	607	goto again_after_prefix;
duke@435	608
duke@435	609	case 0x67:
duke@435	610	case REX:
duke@435	611	case REX_B:
duke@435	612	case REX_X:
duke@435	613	case REX_XB:
duke@435	614	case REX_R:
duke@435	615	case REX_RB:
duke@435	616	case REX_RX:
duke@435	617	case REX_RXB:
duke@435	618	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	619	goto again_after_prefix;
duke@435	620
duke@435	621	case REX_W:
duke@435	622	case REX_WB:
duke@435	623	case REX_WX:
duke@435	624	case REX_WXB:
duke@435	625	case REX_WR:
duke@435	626	case REX_WRB:
duke@435	627	case REX_WRX:
duke@435	628	case REX_WRXB:
duke@435	629	is_64bit = true;
duke@435	630	// assert(ip == inst + 1, "only one prefix allowed");
duke@435	631	goto again_after_prefix;
duke@435	632
duke@435	633	case 0xFF: // pushq a; decl a; incl a; call a; jmp a
duke@435	634	case 0x88: // movb a, r
duke@435	635	case 0x89: // movl a, r
duke@435	636	case 0x8A: // movb r, a
duke@435	637	case 0x8B: // movl r, a
duke@435	638	case 0x8F: // popl a
duke@435	639	debug_only(has_disp32 = true);
duke@435	640	break;
duke@435	641
duke@435	642	case 0x68: // pushq #32
duke@435	643	if (which == end_pc_operand) {
duke@435	644	return ip + 4;
duke@435	645	}
duke@435	646	assert(0, "pushq has no disp32 or imm64");
duke@435	647	ShouldNotReachHere();
duke@435	648
duke@435	649	case 0x66: // movw ... (size prefix)
duke@435	650	again_after_size_prefix2:
duke@435	651	switch (0xFF & *ip++) {
duke@435	652	case REX:
duke@435	653	case REX_B:
duke@435	654	case REX_X:
duke@435	655	case REX_XB:
duke@435	656	case REX_R:
duke@435	657	case REX_RB:
duke@435	658	case REX_RX:
duke@435	659	case REX_RXB:
duke@435	660	case REX_W:
duke@435	661	case REX_WB:
duke@435	662	case REX_WX:
duke@435	663	case REX_WXB:
duke@435	664	case REX_WR:
duke@435	665	case REX_WRB:
duke@435	666	case REX_WRX:
duke@435	667	case REX_WRXB:
duke@435	668	goto again_after_size_prefix2;
duke@435	669	case 0x8B: // movw r, a
duke@435	670	case 0x89: // movw a, r
duke@435	671	break;
duke@435	672	case 0xC7: // movw a, #16
duke@435	673	tail_size = 2; // the imm16
duke@435	674	break;
duke@435	675	case 0x0F: // several SSE/SSE2 variants
duke@435	676	ip--; // reparse the 0x0F
duke@435	677	goto again_after_prefix;
duke@435	678	default:
duke@435	679	ShouldNotReachHere();
duke@435	680	}
duke@435	681	break;
duke@435	682
duke@435	683	case REP8(0xB8): // movl/q r, #32/#64(oop?)
duke@435	684	if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
duke@435	685	assert((which == call32_operand || which == imm64_operand) && is_64bit, "");
duke@435	686	return ip;
duke@435	687
duke@435	688	case 0x69: // imul r, a, #32
duke@435	689	case 0xC7: // movl a, #32(oop?)
duke@435	690	tail_size = 4;
duke@435	691	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	692	break;
duke@435	693
duke@435	694	case 0x0F: // movx..., etc.
duke@435	695	switch (0xFF & *ip++) {
duke@435	696	case 0x12: // movlps
duke@435	697	case 0x28: // movaps
duke@435	698	case 0x2E: // ucomiss
duke@435	699	case 0x2F: // comiss
duke@435	700	case 0x54: // andps
duke@435	701	case 0x57: // xorps
duke@435	702	case 0x6E: // movd
duke@435	703	case 0x7E: // movd
duke@435	704	case 0xAE: // ldmxcsr a
duke@435	705	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	706	break;
duke@435	707	case 0xAD: // shrd r, a, %cl
duke@435	708	case 0xAF: // imul r, a
duke@435	709	case 0xBE: // movsbl r, a
duke@435	710	case 0xBF: // movswl r, a
duke@435	711	case 0xB6: // movzbl r, a
duke@435	712	case 0xB7: // movzwl r, a
duke@435	713	case REP16(0x40): // cmovl cc, r, a
duke@435	714	case 0xB0: // cmpxchgb
duke@435	715	case 0xB1: // cmpxchg
duke@435	716	case 0xC1: // xaddl
duke@435	717	case 0xC7: // cmpxchg8
duke@435	718	case REP16(0x90): // setcc a
duke@435	719	debug_only(has_disp32 = true);
duke@435	720	// fall out of the switch to decode the address
duke@435	721	break;
duke@435	722	case 0xAC: // shrd r, a, #8
duke@435	723	debug_only(has_disp32 = true);
duke@435	724	tail_size = 1; // the imm8
duke@435	725	break;
duke@435	726	case REP16(0x80): // jcc rdisp32
duke@435	727	if (which == end_pc_operand) return ip + 4;
duke@435	728	assert(which == call32_operand, "jcc has no disp32 or imm64");
duke@435	729	return ip;
duke@435	730	default:
duke@435	731	ShouldNotReachHere();
duke@435	732	}
duke@435	733	break;
duke@435	734
duke@435	735	case 0x81: // addl a, #32; addl r, #32
duke@435	736	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	737	tail_size = 4;
duke@435	738	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	739	break;
duke@435	740
duke@435	741	case 0x83: // addl a, #8; addl r, #8
duke@435	742	// also: orl, adcl, sbbl, andl, subl, xorl, cmpl
duke@435	743	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	744	tail_size = 1;
duke@435	745	break;
duke@435	746
duke@435	747	case 0x9B:
duke@435	748	switch (0xFF & *ip++) {
duke@435	749	case 0xD9: // fnstcw a
duke@435	750	debug_only(has_disp32 = true);
duke@435	751	break;
duke@435	752	default:
duke@435	753	ShouldNotReachHere();
duke@435	754	}
duke@435	755	break;
duke@435	756
duke@435	757	case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
duke@435	758	case REP4(0x10): // adc...
duke@435	759	case REP4(0x20): // and...
duke@435	760	case REP4(0x30): // xor...
duke@435	761	case REP4(0x08): // or...
duke@435	762	case REP4(0x18): // sbb...
duke@435	763	case REP4(0x28): // sub...
duke@435	764	case 0xF7: // mull a
duke@435	765	case 0x87: // xchg r, a
duke@435	766	debug_only(has_disp32 = true);
duke@435	767	break;
duke@435	768	case REP4(0x38): // cmp...
duke@435	769	case 0x8D: // lea r, a
duke@435	770	case 0x85: // test r, a
duke@435	771	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	772	break;
duke@435	773
duke@435	774	case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
duke@435	775	case 0xC6: // movb a, #8
duke@435	776	case 0x80: // cmpb a, #8
duke@435	777	case 0x6B: // imul r, a, #8
duke@435	778	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	779	tail_size = 1; // the imm8
duke@435	780	break;
duke@435	781
duke@435	782	case 0xE8: // call rdisp32
duke@435	783	case 0xE9: // jmp rdisp32
duke@435	784	if (which == end_pc_operand) return ip + 4;
duke@435	785	assert(which == call32_operand, "call has no disp32 or imm32");
duke@435	786	return ip;
duke@435	787
duke@435	788	case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
duke@435	789	case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
duke@435	790	case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
duke@435	791	case 0xDD: // fld_d a; fst_d a; fstp_d a
duke@435	792	case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
duke@435	793	case 0xDF: // fild_d a; fistp_d a
duke@435	794	case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
duke@435	795	case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
duke@435	796	case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
duke@435	797	debug_only(has_disp32 = true);
duke@435	798	break;
duke@435	799
duke@435	800	case 0xF3: // For SSE
duke@435	801	case 0xF2: // For SSE2
duke@435	802	switch (0xFF & *ip++) {
duke@435	803	case REX:
duke@435	804	case REX_B:
duke@435	805	case REX_X:
duke@435	806	case REX_XB:
duke@435	807	case REX_R:
duke@435	808	case REX_RB:
duke@435	809	case REX_RX:
duke@435	810	case REX_RXB:
duke@435	811	case REX_W:
duke@435	812	case REX_WB:
duke@435	813	case REX_WX:
duke@435	814	case REX_WXB:
duke@435	815	case REX_WR:
duke@435	816	case REX_WRB:
duke@435	817	case REX_WRX:
duke@435	818	case REX_WRXB:
duke@435	819	ip++;
duke@435	820	default:
duke@435	821	ip++;
duke@435	822	}
duke@435	823	debug_only(has_disp32 = true); // has both kinds of operands!
duke@435	824	break;
duke@435	825
duke@435	826	default:
duke@435	827	ShouldNotReachHere();
duke@435	828
duke@435	829	#undef REP8
duke@435	830	#undef REP16
duke@435	831	}
duke@435	832
duke@435	833	assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
duke@435	834	assert(which != imm64_operand, "instruction is not a movq reg, imm64");
duke@435	835	assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
duke@435	836
duke@435	837	// parse the output of emit_operand
duke@435	838	int op2 = 0xFF & *ip++;
duke@435	839	int base = op2 & 0x07;
duke@435	840	int op3 = -1;
duke@435	841	const int b100 = 4;
duke@435	842	const int b101 = 5;
duke@435	843	if (base == b100 && (op2 >> 6) != 3) {
duke@435	844	op3 = 0xFF & *ip++;
duke@435	845	base = op3 & 0x07; // refetch the base
duke@435	846	}
duke@435	847	// now ip points at the disp (if any)
duke@435	848
duke@435	849	switch (op2 >> 6) {
duke@435	850	case 0:
duke@435	851	// [00 reg 100][ss index base]
duke@435	852	// [00 reg 100][00 100 esp]
duke@435	853	// [00 reg base]
duke@435	854	// [00 reg 100][ss index 101][disp32]
duke@435	855	// [00 reg 101] [disp32]
duke@435	856
duke@435	857	if (base == b101) {
duke@435	858	if (which == disp32_operand)
duke@435	859	return ip; // caller wants the disp32
duke@435	860	ip += 4; // skip the disp32
duke@435	861	}
duke@435	862	break;
duke@435	863
duke@435	864	case 1:
duke@435	865	// [01 reg 100][ss index base][disp8]
duke@435	866	// [01 reg 100][00 100 esp][disp8]
duke@435	867	// [01 reg base] [disp8]
duke@435	868	ip += 1; // skip the disp8
duke@435	869	break;
duke@435	870
duke@435	871	case 2:
duke@435	872	// [10 reg 100][ss index base][disp32]
duke@435	873	// [10 reg 100][00 100 esp][disp32]
duke@435	874	// [10 reg base] [disp32]
duke@435	875	if (which == disp32_operand)
duke@435	876	return ip; // caller wants the disp32
duke@435	877	ip += 4; // skip the disp32
duke@435	878	break;
duke@435	879
duke@435	880	case 3:
duke@435	881	// [11 reg base] (not a memory addressing mode)
duke@435	882	break;
duke@435	883	}
duke@435	884
duke@435	885	if (which == end_pc_operand) {
duke@435	886	return ip + tail_size;
duke@435	887	}
duke@435	888
duke@435	889	assert(0, "fix locate_operand");
duke@435	890	return ip;
duke@435	891	}
duke@435	892
duke@435	893	address Assembler::locate_next_instruction(address inst) {
duke@435	894	// Secretly share code with locate_operand:
duke@435	895	return locate_operand(inst, end_pc_operand);
duke@435	896	}
duke@435	897
duke@435	898	#ifdef ASSERT
duke@435	899	void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
duke@435	900	address inst = inst_mark();
duke@435	901	assert(inst != NULL && inst < pc(),
duke@435	902	"must point to beginning of instruction");
duke@435	903	address opnd;
duke@435	904
duke@435	905	Relocation* r = rspec.reloc();
duke@435	906	if (r->type() == relocInfo::none) {
duke@435	907	return;
duke@435	908	} else if (r->is_call() || format == call32_operand) {
duke@435	909	opnd = locate_operand(inst, call32_operand);
duke@435	910	} else if (r->is_data()) {
duke@435	911	assert(format == imm64_operand || format == disp32_operand, "format ok");
duke@435	912	opnd = locate_operand(inst, (WhichOperand) format);
duke@435	913	} else {
duke@435	914	assert(format == 0, "cannot specify a format");
duke@435	915	return;
duke@435	916	}
duke@435	917	assert(opnd == pc(), "must put operand where relocs can find it");
duke@435	918	}
duke@435	919	#endif
duke@435	920
duke@435	921	int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
duke@435	922	if (reg_enc >= 8) {
duke@435	923	prefix(REX_B);
duke@435	924	reg_enc -= 8;
duke@435	925	} else if (byteinst && reg_enc >= 4) {
duke@435	926	prefix(REX);
duke@435	927	}
duke@435	928	return reg_enc;
duke@435	929	}
duke@435	930
duke@435	931	int Assembler::prefixq_and_encode(int reg_enc) {
duke@435	932	if (reg_enc < 8) {
duke@435	933	prefix(REX_W);
duke@435	934	} else {
duke@435	935	prefix(REX_WB);
duke@435	936	reg_enc -= 8;
duke@435	937	}
duke@435	938	return reg_enc;
duke@435	939	}
duke@435	940
duke@435	941	int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
duke@435	942	if (dst_enc < 8) {
duke@435	943	if (src_enc >= 8) {
duke@435	944	prefix(REX_B);
duke@435	945	src_enc -= 8;
duke@435	946	} else if (byteinst && src_enc >= 4) {
duke@435	947	prefix(REX);
duke@435	948	}
duke@435	949	} else {
duke@435	950	if (src_enc < 8) {
duke@435	951	prefix(REX_R);
duke@435	952	} else {
duke@435	953	prefix(REX_RB);
duke@435	954	src_enc -= 8;
duke@435	955	}
duke@435	956	dst_enc -= 8;
duke@435	957	}
duke@435	958	return dst_enc << 3 | src_enc;
duke@435	959	}
duke@435	960
duke@435	961	int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
duke@435	962	if (dst_enc < 8) {
duke@435	963	if (src_enc < 8) {
duke@435	964	prefix(REX_W);
duke@435	965	} else {
duke@435	966	prefix(REX_WB);
duke@435	967	src_enc -= 8;
duke@435	968	}
duke@435	969	} else {
duke@435	970	if (src_enc < 8) {
duke@435	971	prefix(REX_WR);
duke@435	972	} else {
duke@435	973	prefix(REX_WRB);
duke@435	974	src_enc -= 8;
duke@435	975	}
duke@435	976	dst_enc -= 8;
duke@435	977	}
duke@435	978	return dst_enc << 3 | src_enc;
duke@435	979	}
duke@435	980
duke@435	981	void Assembler::prefix(Register reg) {
duke@435	982	if (reg->encoding() >= 8) {
duke@435	983	prefix(REX_B);
duke@435	984	}
duke@435	985	}
duke@435	986
duke@435	987	void Assembler::prefix(Address adr) {
duke@435	988	if (adr.base_needs_rex()) {
duke@435	989	if (adr.index_needs_rex()) {
duke@435	990	prefix(REX_XB);
duke@435	991	} else {
duke@435	992	prefix(REX_B);
duke@435	993	}
duke@435	994	} else {
duke@435	995	if (adr.index_needs_rex()) {
duke@435	996	prefix(REX_X);
duke@435	997	}
duke@435	998	}
duke@435	999	}
duke@435	1000
duke@435	1001	void Assembler::prefixq(Address adr) {
duke@435	1002	if (adr.base_needs_rex()) {
duke@435	1003	if (adr.index_needs_rex()) {
duke@435	1004	prefix(REX_WXB);
duke@435	1005	} else {
duke@435	1006	prefix(REX_WB);
duke@435	1007	}
duke@435	1008	} else {
duke@435	1009	if (adr.index_needs_rex()) {
duke@435	1010	prefix(REX_WX);
duke@435	1011	} else {
duke@435	1012	prefix(REX_W);
duke@435	1013	}
duke@435	1014	}
duke@435	1015	}
duke@435	1016
duke@435	1017
duke@435	1018	void Assembler::prefix(Address adr, Register reg, bool byteinst) {
duke@435	1019	if (reg->encoding() < 8) {
duke@435	1020	if (adr.base_needs_rex()) {
duke@435	1021	if (adr.index_needs_rex()) {
duke@435	1022	prefix(REX_XB);
duke@435	1023	} else {
duke@435	1024	prefix(REX_B);
duke@435	1025	}
duke@435	1026	} else {
duke@435	1027	if (adr.index_needs_rex()) {
duke@435	1028	prefix(REX_X);
duke@435	1029	} else if (reg->encoding() >= 4 ) {
duke@435	1030	prefix(REX);
duke@435	1031	}
duke@435	1032	}
duke@435	1033	} else {
duke@435	1034	if (adr.base_needs_rex()) {
duke@435	1035	if (adr.index_needs_rex()) {
duke@435	1036	prefix(REX_RXB);
duke@435	1037	} else {
duke@435	1038	prefix(REX_RB);
duke@435	1039	}
duke@435	1040	} else {
duke@435	1041	if (adr.index_needs_rex()) {
duke@435	1042	prefix(REX_RX);
duke@435	1043	} else {
duke@435	1044	prefix(REX_R);
duke@435	1045	}
duke@435	1046	}
duke@435	1047	}
duke@435	1048	}
duke@435	1049
duke@435	1050	void Assembler::prefixq(Address adr, Register src) {
duke@435	1051	if (src->encoding() < 8) {
duke@435	1052	if (adr.base_needs_rex()) {
duke@435	1053	if (adr.index_needs_rex()) {
duke@435	1054	prefix(REX_WXB);
duke@435	1055	} else {
duke@435	1056	prefix(REX_WB);
duke@435	1057	}
duke@435	1058	} else {
duke@435	1059	if (adr.index_needs_rex()) {
duke@435	1060	prefix(REX_WX);
duke@435	1061	} else {
duke@435	1062	prefix(REX_W);
duke@435	1063	}
duke@435	1064	}
duke@435	1065	} else {
duke@435	1066	if (adr.base_needs_rex()) {
duke@435	1067	if (adr.index_needs_rex()) {
duke@435	1068	prefix(REX_WRXB);
duke@435	1069	} else {
duke@435	1070	prefix(REX_WRB);
duke@435	1071	}
duke@435	1072	} else {
duke@435	1073	if (adr.index_needs_rex()) {
duke@435	1074	prefix(REX_WRX);
duke@435	1075	} else {
duke@435	1076	prefix(REX_WR);
duke@435	1077	}
duke@435	1078	}
duke@435	1079	}
duke@435	1080	}
duke@435	1081
duke@435	1082	void Assembler::prefix(Address adr, XMMRegister reg) {
duke@435	1083	if (reg->encoding() < 8) {
duke@435	1084	if (adr.base_needs_rex()) {
duke@435	1085	if (adr.index_needs_rex()) {
duke@435	1086	prefix(REX_XB);
duke@435	1087	} else {
duke@435	1088	prefix(REX_B);
duke@435	1089	}
duke@435	1090	} else {
duke@435	1091	if (adr.index_needs_rex()) {
duke@435	1092	prefix(REX_X);
duke@435	1093	}
duke@435	1094	}
duke@435	1095	} else {
duke@435	1096	if (adr.base_needs_rex()) {
duke@435	1097	if (adr.index_needs_rex()) {
duke@435	1098	prefix(REX_RXB);
duke@435	1099	} else {
duke@435	1100	prefix(REX_RB);
duke@435	1101	}
duke@435	1102	} else {
duke@435	1103	if (adr.index_needs_rex()) {
duke@435	1104	prefix(REX_RX);
duke@435	1105	} else {
duke@435	1106	prefix(REX_R);
duke@435	1107	}
duke@435	1108	}
duke@435	1109	}
duke@435	1110	}
duke@435	1111
duke@435	1112	void Assembler::emit_operand(Register reg, Address adr,
duke@435	1113	int rip_relative_correction) {
duke@435	1114	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1115	adr._rspec,
duke@435	1116	rip_relative_correction);
duke@435	1117	}
duke@435	1118
duke@435	1119	void Assembler::emit_operand(XMMRegister reg, Address adr,
duke@435	1120	int rip_relative_correction) {
duke@435	1121	emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
duke@435	1122	adr._rspec,
duke@435	1123	rip_relative_correction);
duke@435	1124	}
duke@435	1125
duke@435	1126	void Assembler::emit_farith(int b1, int b2, int i) {
duke@435	1127	assert(isByte(b1) && isByte(b2), "wrong opcode");
duke@435	1128	assert(0 <= i && i < 8, "illegal stack offset");
duke@435	1129	emit_byte(b1);
duke@435	1130	emit_byte(b2 + i);
duke@435	1131	}
duke@435	1132
duke@435	1133	// pushad is invalid, use this instead.
duke@435	1134	// NOTE: Kills flags!!
duke@435	1135	void Assembler::pushaq() {
duke@435	1136	// we have to store original rsp. ABI says that 128 bytes
duke@435	1137	// below rsp are local scratch.
duke@435	1138	movq(Address(rsp, -5 * wordSize), rsp);
duke@435	1139
duke@435	1140	subq(rsp, 16 * wordSize);
duke@435	1141
duke@435	1142	movq(Address(rsp, 15 * wordSize), rax);
duke@435	1143	movq(Address(rsp, 14 * wordSize), rcx);
duke@435	1144	movq(Address(rsp, 13 * wordSize), rdx);
duke@435	1145	movq(Address(rsp, 12 * wordSize), rbx);
duke@435	1146	// skip rsp
duke@435	1147	movq(Address(rsp, 10 * wordSize), rbp);
duke@435	1148	movq(Address(rsp, 9 * wordSize), rsi);
duke@435	1149	movq(Address(rsp, 8 * wordSize), rdi);
duke@435	1150	movq(Address(rsp, 7 * wordSize), r8);
duke@435	1151	movq(Address(rsp, 6 * wordSize), r9);
duke@435	1152	movq(Address(rsp, 5 * wordSize), r10);
duke@435	1153	movq(Address(rsp, 4 * wordSize), r11);
duke@435	1154	movq(Address(rsp, 3 * wordSize), r12);
duke@435	1155	movq(Address(rsp, 2 * wordSize), r13);
duke@435	1156	movq(Address(rsp, wordSize), r14);
duke@435	1157	movq(Address(rsp, 0), r15);
duke@435	1158	}
duke@435	1159
duke@435	1160	// popad is invalid, use this instead
duke@435	1161	// NOTE: Kills flags!!
duke@435	1162	void Assembler::popaq() {
duke@435	1163	movq(r15, Address(rsp, 0));
duke@435	1164	movq(r14, Address(rsp, wordSize));
duke@435	1165	movq(r13, Address(rsp, 2 * wordSize));
duke@435	1166	movq(r12, Address(rsp, 3 * wordSize));
duke@435	1167	movq(r11, Address(rsp, 4 * wordSize));
duke@435	1168	movq(r10, Address(rsp, 5 * wordSize));
duke@435	1169	movq(r9, Address(rsp, 6 * wordSize));
duke@435	1170	movq(r8, Address(rsp, 7 * wordSize));
duke@435	1171	movq(rdi, Address(rsp, 8 * wordSize));
duke@435	1172	movq(rsi, Address(rsp, 9 * wordSize));
duke@435	1173	movq(rbp, Address(rsp, 10 * wordSize));
duke@435	1174	// skip rsp
duke@435	1175	movq(rbx, Address(rsp, 12 * wordSize));
duke@435	1176	movq(rdx, Address(rsp, 13 * wordSize));
duke@435	1177	movq(rcx, Address(rsp, 14 * wordSize));
duke@435	1178	movq(rax, Address(rsp, 15 * wordSize));
duke@435	1179
duke@435	1180	addq(rsp, 16 * wordSize);
duke@435	1181	}
duke@435	1182
duke@435	1183	void Assembler::pushfq() {
duke@435	1184	emit_byte(0x9C);
duke@435	1185	}
duke@435	1186
duke@435	1187	void Assembler::popfq() {
duke@435	1188	emit_byte(0x9D);
duke@435	1189	}
duke@435	1190
duke@435	1191	void Assembler::pushq(int imm32) {
duke@435	1192	emit_byte(0x68);
duke@435	1193	emit_long(imm32);
duke@435	1194	}
duke@435	1195
duke@435	1196	void Assembler::pushq(Register src) {
duke@435	1197	int encode = prefix_and_encode(src->encoding());
duke@435	1198
duke@435	1199	emit_byte(0x50 | encode);
duke@435	1200	}
duke@435	1201
duke@435	1202	void Assembler::pushq(Address src) {
duke@435	1203	InstructionMark im(this);
duke@435	1204	prefix(src);
duke@435	1205	emit_byte(0xFF);
duke@435	1206	emit_operand(rsi, src);
duke@435	1207	}
duke@435	1208
duke@435	1209	void Assembler::popq(Register dst) {
duke@435	1210	int encode = prefix_and_encode(dst->encoding());
duke@435	1211	emit_byte(0x58 | encode);
duke@435	1212	}
duke@435	1213
duke@435	1214	void Assembler::popq(Address dst) {
duke@435	1215	InstructionMark im(this);
duke@435	1216	prefix(dst);
duke@435	1217	emit_byte(0x8F);
duke@435	1218	emit_operand(rax, dst);
duke@435	1219	}
duke@435	1220
duke@435	1221	void Assembler::prefix(Prefix p) {
duke@435	1222	a_byte(p);
duke@435	1223	}
duke@435	1224
duke@435	1225	void Assembler::movb(Register dst, Address src) {
duke@435	1226	InstructionMark im(this);
duke@435	1227	prefix(src, dst, true);
duke@435	1228	emit_byte(0x8A);
duke@435	1229	emit_operand(dst, src);
duke@435	1230	}
duke@435	1231
duke@435	1232	void Assembler::movb(Address dst, int imm8) {
duke@435	1233	InstructionMark im(this);
duke@435	1234	prefix(dst);
duke@435	1235	emit_byte(0xC6);
duke@435	1236	emit_operand(rax, dst, 1);
duke@435	1237	emit_byte(imm8);
duke@435	1238	}
duke@435	1239
duke@435	1240	void Assembler::movb(Address dst, Register src) {
duke@435	1241	InstructionMark im(this);
duke@435	1242	prefix(dst, src, true);
duke@435	1243	emit_byte(0x88);
duke@435	1244	emit_operand(src, dst);
duke@435	1245	}
duke@435	1246
duke@435	1247	void Assembler::movw(Address dst, int imm16) {
duke@435	1248	InstructionMark im(this);
duke@435	1249	emit_byte(0x66); // switch to 16-bit mode
duke@435	1250	prefix(dst);
duke@435	1251	emit_byte(0xC7);
duke@435	1252	emit_operand(rax, dst, 2);
duke@435	1253	emit_word(imm16);
duke@435	1254	}
duke@435	1255
duke@435	1256	void Assembler::movw(Register dst, Address src) {
duke@435	1257	InstructionMark im(this);
duke@435	1258	emit_byte(0x66);
duke@435	1259	prefix(src, dst);
duke@435	1260	emit_byte(0x8B);
duke@435	1261	emit_operand(dst, src);
duke@435	1262	}
duke@435	1263
duke@435	1264	void Assembler::movw(Address dst, Register src) {
duke@435	1265	InstructionMark im(this);
duke@435	1266	emit_byte(0x66);
duke@435	1267	prefix(dst, src);
duke@435	1268	emit_byte(0x89);
duke@435	1269	emit_operand(src, dst);
duke@435	1270	}
duke@435	1271
duke@435	1272	// Uses zero extension.
duke@435	1273	void Assembler::movl(Register dst, int imm32) {
duke@435	1274	int encode = prefix_and_encode(dst->encoding());
duke@435	1275	emit_byte(0xB8 | encode);
duke@435	1276	emit_long(imm32);
duke@435	1277	}
duke@435	1278
duke@435	1279	void Assembler::movl(Register dst, Register src) {
duke@435	1280	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1281	emit_byte(0x8B);
duke@435	1282	emit_byte(0xC0 | encode);
duke@435	1283	}
duke@435	1284
duke@435	1285	void Assembler::movl(Register dst, Address src) {
duke@435	1286	InstructionMark im(this);
duke@435	1287	prefix(src, dst);
duke@435	1288	emit_byte(0x8B);
duke@435	1289	emit_operand(dst, src);
duke@435	1290	}
duke@435	1291
duke@435	1292	void Assembler::movl(Address dst, int imm32) {
duke@435	1293	InstructionMark im(this);
duke@435	1294	prefix(dst);
duke@435	1295	emit_byte(0xC7);
duke@435	1296	emit_operand(rax, dst, 4);
duke@435	1297	emit_long(imm32);
duke@435	1298	}
duke@435	1299
duke@435	1300	void Assembler::movl(Address dst, Register src) {
duke@435	1301	InstructionMark im(this);
duke@435	1302	prefix(dst, src);
duke@435	1303	emit_byte(0x89);
duke@435	1304	emit_operand(src, dst);
duke@435	1305	}
duke@435	1306
duke@435	1307	void Assembler::mov64(Register dst, int64_t imm64) {
duke@435	1308	InstructionMark im(this);
duke@435	1309	int encode = prefixq_and_encode(dst->encoding());
duke@435	1310	emit_byte(0xB8 | encode);
duke@435	1311	emit_long64(imm64);
duke@435	1312	}
duke@435	1313
duke@435	1314	void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
duke@435	1315	InstructionMark im(this);
duke@435	1316	int encode = prefixq_and_encode(dst->encoding());
duke@435	1317	emit_byte(0xB8 | encode);
duke@435	1318	emit_data64(imm64, rspec);
duke@435	1319	}
duke@435	1320
duke@435	1321	void Assembler::movq(Register dst, Register src) {
duke@435	1322	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1323	emit_byte(0x8B);
duke@435	1324	emit_byte(0xC0 | encode);
duke@435	1325	}
duke@435	1326
duke@435	1327	void Assembler::movq(Register dst, Address src) {
duke@435	1328	InstructionMark im(this);
duke@435	1329	prefixq(src, dst);
duke@435	1330	emit_byte(0x8B);
duke@435	1331	emit_operand(dst, src);
duke@435	1332	}
duke@435	1333
duke@435	1334	void Assembler::mov64(Address dst, int64_t imm32) {
duke@435	1335	assert(is_simm32(imm32), "lost bits");
duke@435	1336	InstructionMark im(this);
duke@435	1337	prefixq(dst);
duke@435	1338	emit_byte(0xC7);
duke@435	1339	emit_operand(rax, dst, 4);
duke@435	1340	emit_long(imm32);
duke@435	1341	}
duke@435	1342
duke@435	1343	void Assembler::movq(Address dst, Register src) {
duke@435	1344	InstructionMark im(this);
duke@435	1345	prefixq(dst, src);
duke@435	1346	emit_byte(0x89);
duke@435	1347	emit_operand(src, dst);
duke@435	1348	}
duke@435	1349
duke@435	1350	void Assembler::movsbl(Register dst, Address src) {
duke@435	1351	InstructionMark im(this);
duke@435	1352	prefix(src, dst);
duke@435	1353	emit_byte(0x0F);
duke@435	1354	emit_byte(0xBE);
duke@435	1355	emit_operand(dst, src);
duke@435	1356	}
duke@435	1357
duke@435	1358	void Assembler::movsbl(Register dst, Register src) {
duke@435	1359	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1360	emit_byte(0x0F);
duke@435	1361	emit_byte(0xBE);
duke@435	1362	emit_byte(0xC0 | encode);
duke@435	1363	}
duke@435	1364
duke@435	1365	void Assembler::movswl(Register dst, Address src) {
duke@435	1366	InstructionMark im(this);
duke@435	1367	prefix(src, dst);
duke@435	1368	emit_byte(0x0F);
duke@435	1369	emit_byte(0xBF);
duke@435	1370	emit_operand(dst, src);
duke@435	1371	}
duke@435	1372
duke@435	1373	void Assembler::movswl(Register dst, Register src) {
duke@435	1374	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1375	emit_byte(0x0F);
duke@435	1376	emit_byte(0xBF);
duke@435	1377	emit_byte(0xC0 | encode);
duke@435	1378	}
duke@435	1379
duke@435	1380	void Assembler::movslq(Register dst, Address src) {
duke@435	1381	InstructionMark im(this);
duke@435	1382	prefixq(src, dst);
duke@435	1383	emit_byte(0x63);
duke@435	1384	emit_operand(dst, src);
duke@435	1385	}
duke@435	1386
duke@435	1387	void Assembler::movslq(Register dst, Register src) {
duke@435	1388	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1389	emit_byte(0x63);
duke@435	1390	emit_byte(0xC0 | encode);
duke@435	1391	}
duke@435	1392
duke@435	1393	void Assembler::movzbl(Register dst, Address src) {
duke@435	1394	InstructionMark im(this);
duke@435	1395	prefix(src, dst);
duke@435	1396	emit_byte(0x0F);
duke@435	1397	emit_byte(0xB6);
duke@435	1398	emit_operand(dst, src);
duke@435	1399	}
duke@435	1400
duke@435	1401	void Assembler::movzbl(Register dst, Register src) {
duke@435	1402	int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
duke@435	1403	emit_byte(0x0F);
duke@435	1404	emit_byte(0xB6);
duke@435	1405	emit_byte(0xC0 | encode);
duke@435	1406	}
duke@435	1407
duke@435	1408	void Assembler::movzwl(Register dst, Address src) {
duke@435	1409	InstructionMark im(this);
duke@435	1410	prefix(src, dst);
duke@435	1411	emit_byte(0x0F);
duke@435	1412	emit_byte(0xB7);
duke@435	1413	emit_operand(dst, src);
duke@435	1414	}
duke@435	1415
duke@435	1416	void Assembler::movzwl(Register dst, Register src) {
duke@435	1417	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1418	emit_byte(0x0F);
duke@435	1419	emit_byte(0xB7);
duke@435	1420	emit_byte(0xC0 | encode);
duke@435	1421	}
duke@435	1422
duke@435	1423	void Assembler::movss(XMMRegister dst, XMMRegister src) {
duke@435	1424	emit_byte(0xF3);
duke@435	1425	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1426	emit_byte(0x0F);
duke@435	1427	emit_byte(0x10);
duke@435	1428	emit_byte(0xC0 | encode);
duke@435	1429	}
duke@435	1430
duke@435	1431	void Assembler::movss(XMMRegister dst, Address src) {
duke@435	1432	InstructionMark im(this);
duke@435	1433	emit_byte(0xF3);
duke@435	1434	prefix(src, dst);
duke@435	1435	emit_byte(0x0F);
duke@435	1436	emit_byte(0x10);
duke@435	1437	emit_operand(dst, src);
duke@435	1438	}
duke@435	1439
duke@435	1440	void Assembler::movss(Address dst, XMMRegister src) {
duke@435	1441	InstructionMark im(this);
duke@435	1442	emit_byte(0xF3);
duke@435	1443	prefix(dst, src);
duke@435	1444	emit_byte(0x0F);
duke@435	1445	emit_byte(0x11);
duke@435	1446	emit_operand(src, dst);
duke@435	1447	}
duke@435	1448
duke@435	1449	void Assembler::movsd(XMMRegister dst, XMMRegister src) {
duke@435	1450	emit_byte(0xF2);
duke@435	1451	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1452	emit_byte(0x0F);
duke@435	1453	emit_byte(0x10);
duke@435	1454	emit_byte(0xC0 | encode);
duke@435	1455	}
duke@435	1456
duke@435	1457	void Assembler::movsd(XMMRegister dst, Address src) {
duke@435	1458	InstructionMark im(this);
duke@435	1459	emit_byte(0xF2);
duke@435	1460	prefix(src, dst);
duke@435	1461	emit_byte(0x0F);
duke@435	1462	emit_byte(0x10);
duke@435	1463	emit_operand(dst, src);
duke@435	1464	}
duke@435	1465
duke@435	1466	void Assembler::movsd(Address dst, XMMRegister src) {
duke@435	1467	InstructionMark im(this);
duke@435	1468	emit_byte(0xF2);
duke@435	1469	prefix(dst, src);
duke@435	1470	emit_byte(0x0F);
duke@435	1471	emit_byte(0x11);
duke@435	1472	emit_operand(src, dst);
duke@435	1473	}
duke@435	1474
duke@435	1475	// New cpus require to use movsd and movss to avoid partial register stall
duke@435	1476	// when loading from memory. But for old Opteron use movlpd instead of movsd.
duke@435	1477	// The selection is done in MacroAssembler::movdbl() and movflt().
duke@435	1478	void Assembler::movlpd(XMMRegister dst, Address src) {
duke@435	1479	InstructionMark im(this);
duke@435	1480	emit_byte(0x66);
duke@435	1481	prefix(src, dst);
duke@435	1482	emit_byte(0x0F);
duke@435	1483	emit_byte(0x12);
duke@435	1484	emit_operand(dst, src);
duke@435	1485	}
duke@435	1486
duke@435	1487	void Assembler::movapd(XMMRegister dst, XMMRegister src) {
duke@435	1488	int dstenc = dst->encoding();
duke@435	1489	int srcenc = src->encoding();
duke@435	1490	emit_byte(0x66);
duke@435	1491	if (dstenc < 8) {
duke@435	1492	if (srcenc >= 8) {
duke@435	1493	prefix(REX_B);
duke@435	1494	srcenc -= 8;
duke@435	1495	}
duke@435	1496	} else {
duke@435	1497	if (srcenc < 8) {
duke@435	1498	prefix(REX_R);
duke@435	1499	} else {
duke@435	1500	prefix(REX_RB);
duke@435	1501	srcenc -= 8;
duke@435	1502	}
duke@435	1503	dstenc -= 8;
duke@435	1504	}
duke@435	1505	emit_byte(0x0F);
duke@435	1506	emit_byte(0x28);
duke@435	1507	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1508	}
duke@435	1509
duke@435	1510	void Assembler::movaps(XMMRegister dst, XMMRegister src) {
duke@435	1511	int dstenc = dst->encoding();
duke@435	1512	int srcenc = src->encoding();
duke@435	1513	if (dstenc < 8) {
duke@435	1514	if (srcenc >= 8) {
duke@435	1515	prefix(REX_B);
duke@435	1516	srcenc -= 8;
duke@435	1517	}
duke@435	1518	} else {
duke@435	1519	if (srcenc < 8) {
duke@435	1520	prefix(REX_R);
duke@435	1521	} else {
duke@435	1522	prefix(REX_RB);
duke@435	1523	srcenc -= 8;
duke@435	1524	}
duke@435	1525	dstenc -= 8;
duke@435	1526	}
duke@435	1527	emit_byte(0x0F);
duke@435	1528	emit_byte(0x28);
duke@435	1529	emit_byte(0xC0 | dstenc << 3 | srcenc);
duke@435	1530	}
duke@435	1531
duke@435	1532	void Assembler::movdl(XMMRegister dst, Register src) {
duke@435	1533	emit_byte(0x66);
duke@435	1534	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1535	emit_byte(0x0F);
duke@435	1536	emit_byte(0x6E);
duke@435	1537	emit_byte(0xC0 | encode);
duke@435	1538	}
duke@435	1539
duke@435	1540	void Assembler::movdl(Register dst, XMMRegister src) {
duke@435	1541	emit_byte(0x66);
duke@435	1542	// swap src/dst to get correct prefix
duke@435	1543	int encode = prefix_and_encode(src->encoding(), dst->encoding());
duke@435	1544	emit_byte(0x0F);
duke@435	1545	emit_byte(0x7E);
duke@435	1546	emit_byte(0xC0 | encode);
duke@435	1547	}
duke@435	1548
duke@435	1549	void Assembler::movdq(XMMRegister dst, Register src) {
duke@435	1550	emit_byte(0x66);
duke@435	1551	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1552	emit_byte(0x0F);
duke@435	1553	emit_byte(0x6E);
duke@435	1554	emit_byte(0xC0 | encode);
duke@435	1555	}
duke@435	1556
duke@435	1557	void Assembler::movdq(Register dst, XMMRegister src) {
duke@435	1558	emit_byte(0x66);
duke@435	1559	// swap src/dst to get correct prefix
duke@435	1560	int encode = prefixq_and_encode(src->encoding(), dst->encoding());
duke@435	1561	emit_byte(0x0F);
duke@435	1562	emit_byte(0x7E);
duke@435	1563	emit_byte(0xC0 | encode);
duke@435	1564	}
duke@435	1565
duke@435	1566	void Assembler::pxor(XMMRegister dst, Address src) {
duke@435	1567	InstructionMark im(this);
duke@435	1568	emit_byte(0x66);
duke@435	1569	prefix(src, dst);
duke@435	1570	emit_byte(0x0F);
duke@435	1571	emit_byte(0xEF);
duke@435	1572	emit_operand(dst, src);
duke@435	1573	}
duke@435	1574
duke@435	1575	void Assembler::pxor(XMMRegister dst, XMMRegister src) {
duke@435	1576	InstructionMark im(this);
duke@435	1577	emit_byte(0x66);
duke@435	1578	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1579	emit_byte(0x0F);
duke@435	1580	emit_byte(0xEF);
duke@435	1581	emit_byte(0xC0 | encode);
duke@435	1582	}
duke@435	1583
duke@435	1584	void Assembler::movdqa(XMMRegister dst, Address src) {
duke@435	1585	InstructionMark im(this);
duke@435	1586	emit_byte(0x66);
duke@435	1587	prefix(src, dst);
duke@435	1588	emit_byte(0x0F);
duke@435	1589	emit_byte(0x6F);
duke@435	1590	emit_operand(dst, src);
duke@435	1591	}
duke@435	1592
duke@435	1593	void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
duke@435	1594	emit_byte(0x66);
duke@435	1595	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1596	emit_byte(0x0F);
duke@435	1597	emit_byte(0x6F);
duke@435	1598	emit_byte(0xC0 | encode);
duke@435	1599	}
duke@435	1600
duke@435	1601	void Assembler::movdqa(Address dst, XMMRegister src) {
duke@435	1602	InstructionMark im(this);
duke@435	1603	emit_byte(0x66);
duke@435	1604	prefix(dst, src);
duke@435	1605	emit_byte(0x0F);
duke@435	1606	emit_byte(0x7F);
duke@435	1607	emit_operand(src, dst);
duke@435	1608	}
duke@435	1609
duke@435	1610	void Assembler::movq(XMMRegister dst, Address src) {
duke@435	1611	InstructionMark im(this);
duke@435	1612	emit_byte(0xF3);
duke@435	1613	prefix(src, dst);
duke@435	1614	emit_byte(0x0F);
duke@435	1615	emit_byte(0x7E);
duke@435	1616	emit_operand(dst, src);
duke@435	1617	}
duke@435	1618
duke@435	1619	void Assembler::movq(Address dst, XMMRegister src) {
duke@435	1620	InstructionMark im(this);
duke@435	1621	emit_byte(0x66);
duke@435	1622	prefix(dst, src);
duke@435	1623	emit_byte(0x0F);
duke@435	1624	emit_byte(0xD6);
duke@435	1625	emit_operand(src, dst);
duke@435	1626	}
duke@435	1627
duke@435	1628	void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1629	assert(isByte(mode), "invalid value");
duke@435	1630	emit_byte(0x66);
duke@435	1631	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1632	emit_byte(0x0F);
duke@435	1633	emit_byte(0x70);
duke@435	1634	emit_byte(0xC0 | encode);
duke@435	1635	emit_byte(mode & 0xFF);
duke@435	1636	}
duke@435	1637
duke@435	1638	void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
duke@435	1639	assert(isByte(mode), "invalid value");
duke@435	1640	InstructionMark im(this);
duke@435	1641	emit_byte(0x66);
duke@435	1642	emit_byte(0x0F);
duke@435	1643	emit_byte(0x70);
duke@435	1644	emit_operand(dst, src);
duke@435	1645	emit_byte(mode & 0xFF);
duke@435	1646	}
duke@435	1647
duke@435	1648	void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
duke@435	1649	assert(isByte(mode), "invalid value");
duke@435	1650	emit_byte(0xF2);
duke@435	1651	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1652	emit_byte(0x0F);
duke@435	1653	emit_byte(0x70);
duke@435	1654	emit_byte(0xC0 | encode);
duke@435	1655	emit_byte(mode & 0xFF);
duke@435	1656	}
duke@435	1657
duke@435	1658	void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
duke@435	1659	assert(isByte(mode), "invalid value");
duke@435	1660	InstructionMark im(this);
duke@435	1661	emit_byte(0xF2);
duke@435	1662	emit_byte(0x0F);
duke@435	1663	emit_byte(0x70);
duke@435	1664	emit_operand(dst, src);
duke@435	1665	emit_byte(mode & 0xFF);
duke@435	1666	}
duke@435	1667
duke@435	1668	void Assembler::cmovl(Condition cc, Register dst, Register src) {
duke@435	1669	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1670	emit_byte(0x0F);
duke@435	1671	emit_byte(0x40 | cc);
duke@435	1672	emit_byte(0xC0 | encode);
duke@435	1673	}
duke@435	1674
duke@435	1675	void Assembler::cmovl(Condition cc, Register dst, Address src) {
duke@435	1676	InstructionMark im(this);
duke@435	1677	prefix(src, dst);
duke@435	1678	emit_byte(0x0F);
duke@435	1679	emit_byte(0x40 | cc);
duke@435	1680	emit_operand(dst, src);
duke@435	1681	}
duke@435	1682
duke@435	1683	void Assembler::cmovq(Condition cc, Register dst, Register src) {
duke@435	1684	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1685	emit_byte(0x0F);
duke@435	1686	emit_byte(0x40 | cc);
duke@435	1687	emit_byte(0xC0 | encode);
duke@435	1688	}
duke@435	1689
duke@435	1690	void Assembler::cmovq(Condition cc, Register dst, Address src) {
duke@435	1691	InstructionMark im(this);
duke@435	1692	prefixq(src, dst);
duke@435	1693	emit_byte(0x0F);
duke@435	1694	emit_byte(0x40 | cc);
duke@435	1695	emit_operand(dst, src);
duke@435	1696	}
duke@435	1697
duke@435	1698	void Assembler::prefetch_prefix(Address src) {
duke@435	1699	prefix(src);
duke@435	1700	emit_byte(0x0F);
duke@435	1701	}
duke@435	1702
duke@435	1703	void Assembler::prefetcht0(Address src) {
duke@435	1704	InstructionMark im(this);
duke@435	1705	prefetch_prefix(src);
duke@435	1706	emit_byte(0x18);
duke@435	1707	emit_operand(rcx, src); // 1, src
duke@435	1708	}
duke@435	1709
duke@435	1710	void Assembler::prefetcht1(Address src) {
duke@435	1711	InstructionMark im(this);
duke@435	1712	prefetch_prefix(src);
duke@435	1713	emit_byte(0x18);
duke@435	1714	emit_operand(rdx, src); // 2, src
duke@435	1715	}
duke@435	1716
duke@435	1717	void Assembler::prefetcht2(Address src) {
duke@435	1718	InstructionMark im(this);
duke@435	1719	prefetch_prefix(src);
duke@435	1720	emit_byte(0x18);
duke@435	1721	emit_operand(rbx, src); // 3, src
duke@435	1722	}
duke@435	1723
duke@435	1724	void Assembler::prefetchnta(Address src) {
duke@435	1725	InstructionMark im(this);
duke@435	1726	prefetch_prefix(src);
duke@435	1727	emit_byte(0x18);
duke@435	1728	emit_operand(rax, src); // 0, src
duke@435	1729	}
duke@435	1730
duke@435	1731	void Assembler::prefetchw(Address src) {
duke@435	1732	InstructionMark im(this);
duke@435	1733	prefetch_prefix(src);
duke@435	1734	emit_byte(0x0D);
duke@435	1735	emit_operand(rcx, src); // 1, src
duke@435	1736	}
duke@435	1737
duke@435	1738	void Assembler::adcl(Register dst, int imm32) {
duke@435	1739	prefix(dst);
duke@435	1740	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1741	}
duke@435	1742
duke@435	1743	void Assembler::adcl(Register dst, Address src) {
duke@435	1744	InstructionMark im(this);
duke@435	1745	prefix(src, dst);
duke@435	1746	emit_byte(0x13);
duke@435	1747	emit_operand(dst, src);
duke@435	1748	}
duke@435	1749
duke@435	1750	void Assembler::adcl(Register dst, Register src) {
duke@435	1751	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1752	emit_arith(0x13, 0xC0, dst, src);
duke@435	1753	}
duke@435	1754
duke@435	1755	void Assembler::adcq(Register dst, int imm32) {
duke@435	1756	(void) prefixq_and_encode(dst->encoding());
duke@435	1757	emit_arith(0x81, 0xD0, dst, imm32);
duke@435	1758	}
duke@435	1759
duke@435	1760	void Assembler::adcq(Register dst, Address src) {
duke@435	1761	InstructionMark im(this);
duke@435	1762	prefixq(src, dst);
duke@435	1763	emit_byte(0x13);
duke@435	1764	emit_operand(dst, src);
duke@435	1765	}
duke@435	1766
duke@435	1767	void Assembler::adcq(Register dst, Register src) {
duke@435	1768	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1769	emit_arith(0x13, 0xC0, dst, src);
duke@435	1770	}
duke@435	1771
duke@435	1772	void Assembler::addl(Address dst, int imm32) {
duke@435	1773	InstructionMark im(this);
duke@435	1774	prefix(dst);
duke@435	1775	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1776	}
duke@435	1777
duke@435	1778	void Assembler::addl(Address dst, Register src) {
duke@435	1779	InstructionMark im(this);
duke@435	1780	prefix(dst, src);
duke@435	1781	emit_byte(0x01);
duke@435	1782	emit_operand(src, dst);
duke@435	1783	}
duke@435	1784
duke@435	1785	void Assembler::addl(Register dst, int imm32) {
duke@435	1786	prefix(dst);
duke@435	1787	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1788	}
duke@435	1789
duke@435	1790	void Assembler::addl(Register dst, Address src) {
duke@435	1791	InstructionMark im(this);
duke@435	1792	prefix(src, dst);
duke@435	1793	emit_byte(0x03);
duke@435	1794	emit_operand(dst, src);
duke@435	1795	}
duke@435	1796
duke@435	1797	void Assembler::addl(Register dst, Register src) {
duke@435	1798	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1799	emit_arith(0x03, 0xC0, dst, src);
duke@435	1800	}
duke@435	1801
duke@435	1802	void Assembler::addq(Address dst, int imm32) {
duke@435	1803	InstructionMark im(this);
duke@435	1804	prefixq(dst);
duke@435	1805	emit_arith_operand(0x81, rax, dst,imm32);
duke@435	1806	}
duke@435	1807
duke@435	1808	void Assembler::addq(Address dst, Register src) {
duke@435	1809	InstructionMark im(this);
duke@435	1810	prefixq(dst, src);
duke@435	1811	emit_byte(0x01);
duke@435	1812	emit_operand(src, dst);
duke@435	1813	}
duke@435	1814
duke@435	1815	void Assembler::addq(Register dst, int imm32) {
duke@435	1816	(void) prefixq_and_encode(dst->encoding());
duke@435	1817	emit_arith(0x81, 0xC0, dst, imm32);
duke@435	1818	}
duke@435	1819
duke@435	1820	void Assembler::addq(Register dst, Address src) {
duke@435	1821	InstructionMark im(this);
duke@435	1822	prefixq(src, dst);
duke@435	1823	emit_byte(0x03);
duke@435	1824	emit_operand(dst, src);
duke@435	1825	}
duke@435	1826
duke@435	1827	void Assembler::addq(Register dst, Register src) {
duke@435	1828	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1829	emit_arith(0x03, 0xC0, dst, src);
duke@435	1830	}
duke@435	1831
duke@435	1832	void Assembler::andl(Register dst, int imm32) {
duke@435	1833	prefix(dst);
duke@435	1834	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1835	}
duke@435	1836
duke@435	1837	void Assembler::andl(Register dst, Address src) {
duke@435	1838	InstructionMark im(this);
duke@435	1839	prefix(src, dst);
duke@435	1840	emit_byte(0x23);
duke@435	1841	emit_operand(dst, src);
duke@435	1842	}
duke@435	1843
duke@435	1844	void Assembler::andl(Register dst, Register src) {
duke@435	1845	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1846	emit_arith(0x23, 0xC0, dst, src);
duke@435	1847	}
duke@435	1848
duke@435	1849	void Assembler::andq(Register dst, int imm32) {
duke@435	1850	(void) prefixq_and_encode(dst->encoding());
duke@435	1851	emit_arith(0x81, 0xE0, dst, imm32);
duke@435	1852	}
duke@435	1853
duke@435	1854	void Assembler::andq(Register dst, Address src) {
duke@435	1855	InstructionMark im(this);
duke@435	1856	prefixq(src, dst);
duke@435	1857	emit_byte(0x23);
duke@435	1858	emit_operand(dst, src);
duke@435	1859	}
duke@435	1860
duke@435	1861	void Assembler::andq(Register dst, Register src) {
duke@435	1862	(int) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1863	emit_arith(0x23, 0xC0, dst, src);
duke@435	1864	}
duke@435	1865
duke@435	1866	void Assembler::cmpb(Address dst, int imm8) {
duke@435	1867	InstructionMark im(this);
duke@435	1868	prefix(dst);
duke@435	1869	emit_byte(0x80);
duke@435	1870	emit_operand(rdi, dst, 1);
duke@435	1871	emit_byte(imm8);
duke@435	1872	}
duke@435	1873
duke@435	1874	void Assembler::cmpl(Address dst, int imm32) {
duke@435	1875	InstructionMark im(this);
duke@435	1876	prefix(dst);
duke@435	1877	emit_byte(0x81);
duke@435	1878	emit_operand(rdi, dst, 4);
duke@435	1879	emit_long(imm32);
duke@435	1880	}
duke@435	1881
duke@435	1882	void Assembler::cmpl(Register dst, int imm32) {
duke@435	1883	prefix(dst);
duke@435	1884	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1885	}
duke@435	1886
duke@435	1887	void Assembler::cmpl(Register dst, Register src) {
duke@435	1888	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1889	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1890	}
duke@435	1891
duke@435	1892	void Assembler::cmpl(Register dst, Address src) {
duke@435	1893	InstructionMark im(this);
duke@435	1894	prefix(src, dst);
duke@435	1895	emit_byte(0x3B);
duke@435	1896	emit_operand(dst, src);
duke@435	1897	}
duke@435	1898
duke@435	1899	void Assembler::cmpq(Address dst, int imm32) {
duke@435	1900	InstructionMark im(this);
duke@435	1901	prefixq(dst);
duke@435	1902	emit_byte(0x81);
duke@435	1903	emit_operand(rdi, dst, 4);
duke@435	1904	emit_long(imm32);
duke@435	1905	}
duke@435	1906
duke@435	1907	void Assembler::cmpq(Register dst, int imm32) {
duke@435	1908	(void) prefixq_and_encode(dst->encoding());
duke@435	1909	emit_arith(0x81, 0xF8, dst, imm32);
duke@435	1910	}
duke@435	1911
duke@435	1912	void Assembler::cmpq(Address dst, Register src) {
duke@435	1913	prefixq(dst, src);
duke@435	1914	emit_byte(0x3B);
duke@435	1915	emit_operand(src, dst);
duke@435	1916	}
duke@435	1917
duke@435	1918	void Assembler::cmpq(Register dst, Register src) {
duke@435	1919	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	1920	emit_arith(0x3B, 0xC0, dst, src);
duke@435	1921	}
duke@435	1922
duke@435	1923	void Assembler::cmpq(Register dst, Address src) {
duke@435	1924	InstructionMark im(this);
duke@435	1925	prefixq(src, dst);
duke@435	1926	emit_byte(0x3B);
duke@435	1927	emit_operand(dst, src);
duke@435	1928	}
duke@435	1929
duke@435	1930	void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
duke@435	1931	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1932	emit_byte(0x0F);
duke@435	1933	emit_byte(0x2E);
duke@435	1934	emit_byte(0xC0 | encode);
duke@435	1935	}
duke@435	1936
duke@435	1937	void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
duke@435	1938	emit_byte(0x66);
duke@435	1939	ucomiss(dst, src);
duke@435	1940	}
duke@435	1941
duke@435	1942	void Assembler::decl(Register dst) {
duke@435	1943	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1944	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1945	int encode = prefix_and_encode(dst->encoding());
duke@435	1946	emit_byte(0xFF);
duke@435	1947	emit_byte(0xC8 | encode);
duke@435	1948	}
duke@435	1949
duke@435	1950	void Assembler::decl(Address dst) {
duke@435	1951	// Don't use it directly. Use MacroAssembler::decrementl() instead.
duke@435	1952	InstructionMark im(this);
duke@435	1953	prefix(dst);
duke@435	1954	emit_byte(0xFF);
duke@435	1955	emit_operand(rcx, dst);
duke@435	1956	}
duke@435	1957
duke@435	1958	void Assembler::decq(Register dst) {
duke@435	1959	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1960	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	1961	int encode = prefixq_and_encode(dst->encoding());
duke@435	1962	emit_byte(0xFF);
duke@435	1963	emit_byte(0xC8 | encode);
duke@435	1964	}
duke@435	1965
duke@435	1966	void Assembler::decq(Address dst) {
duke@435	1967	// Don't use it directly. Use MacroAssembler::decrementq() instead.
duke@435	1968	InstructionMark im(this);
duke@435	1969	prefixq(dst);
duke@435	1970	emit_byte(0xFF);
duke@435	1971	emit_operand(rcx, dst);
duke@435	1972	}
duke@435	1973
duke@435	1974	void Assembler::idivl(Register src) {
duke@435	1975	int encode = prefix_and_encode(src->encoding());
duke@435	1976	emit_byte(0xF7);
duke@435	1977	emit_byte(0xF8 | encode);
duke@435	1978	}
duke@435	1979
duke@435	1980	void Assembler::idivq(Register src) {
duke@435	1981	int encode = prefixq_and_encode(src->encoding());
duke@435	1982	emit_byte(0xF7);
duke@435	1983	emit_byte(0xF8 | encode);
duke@435	1984	}
duke@435	1985
duke@435	1986	void Assembler::cdql() {
duke@435	1987	emit_byte(0x99);
duke@435	1988	}
duke@435	1989
duke@435	1990	void Assembler::cdqq() {
duke@435	1991	prefix(REX_W);
duke@435	1992	emit_byte(0x99);
duke@435	1993	}
duke@435	1994
duke@435	1995	void Assembler::imull(Register dst, Register src) {
duke@435	1996	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	1997	emit_byte(0x0F);
duke@435	1998	emit_byte(0xAF);
duke@435	1999	emit_byte(0xC0 | encode);
duke@435	2000	}
duke@435	2001
duke@435	2002	void Assembler::imull(Register dst, Register src, int value) {
duke@435	2003	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2004	if (is8bit(value)) {
duke@435	2005	emit_byte(0x6B);
duke@435	2006	emit_byte(0xC0 | encode);
duke@435	2007	emit_byte(value);
duke@435	2008	} else {
duke@435	2009	emit_byte(0x69);
duke@435	2010	emit_byte(0xC0 | encode);
duke@435	2011	emit_long(value);
duke@435	2012	}
duke@435	2013	}
duke@435	2014
duke@435	2015	void Assembler::imulq(Register dst, Register src) {
duke@435	2016	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2017	emit_byte(0x0F);
duke@435	2018	emit_byte(0xAF);
duke@435	2019	emit_byte(0xC0 | encode);
duke@435	2020	}
duke@435	2021
duke@435	2022	void Assembler::imulq(Register dst, Register src, int value) {
duke@435	2023	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2024	if (is8bit(value)) {
duke@435	2025	emit_byte(0x6B);
duke@435	2026	emit_byte(0xC0 | encode);
duke@435	2027	emit_byte(value);
duke@435	2028	} else {
duke@435	2029	emit_byte(0x69);
duke@435	2030	emit_byte(0xC0 | encode);
duke@435	2031	emit_long(value);
duke@435	2032	}
duke@435	2033	}
duke@435	2034
duke@435	2035	void Assembler::incl(Register dst) {
duke@435	2036	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2037	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2038	int encode = prefix_and_encode(dst->encoding());
duke@435	2039	emit_byte(0xFF);
duke@435	2040	emit_byte(0xC0 | encode);
duke@435	2041	}
duke@435	2042
duke@435	2043	void Assembler::incl(Address dst) {
duke@435	2044	// Don't use it directly. Use MacroAssembler::incrementl() instead.
duke@435	2045	InstructionMark im(this);
duke@435	2046	prefix(dst);
duke@435	2047	emit_byte(0xFF);
duke@435	2048	emit_operand(rax, dst);
duke@435	2049	}
duke@435	2050
duke@435	2051	void Assembler::incq(Register dst) {
duke@435	2052	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2053	// Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
duke@435	2054	int encode = prefixq_and_encode(dst->encoding());
duke@435	2055	emit_byte(0xFF);
duke@435	2056	emit_byte(0xC0 | encode);
duke@435	2057	}
duke@435	2058
duke@435	2059	void Assembler::incq(Address dst) {
duke@435	2060	// Don't use it directly. Use MacroAssembler::incrementq() instead.
duke@435	2061	InstructionMark im(this);
duke@435	2062	prefixq(dst);
duke@435	2063	emit_byte(0xFF);
duke@435	2064	emit_operand(rax, dst);
duke@435	2065	}
duke@435	2066
duke@435	2067	void Assembler::leal(Register dst, Address src) {
duke@435	2068	InstructionMark im(this);
duke@435	2069	emit_byte(0x67); // addr32
duke@435	2070	prefix(src, dst);
duke@435	2071	emit_byte(0x8D);
duke@435	2072	emit_operand(dst, src);
duke@435	2073	}
duke@435	2074
duke@435	2075	void Assembler::leaq(Register dst, Address src) {
duke@435	2076	InstructionMark im(this);
duke@435	2077	prefixq(src, dst);
duke@435	2078	emit_byte(0x8D);
duke@435	2079	emit_operand(dst, src);
duke@435	2080	}
duke@435	2081
duke@435	2082	void Assembler::mull(Address src) {
duke@435	2083	InstructionMark im(this);
duke@435	2084	// was missing
duke@435	2085	prefix(src);
duke@435	2086	emit_byte(0xF7);
duke@435	2087	emit_operand(rsp, src);
duke@435	2088	}
duke@435	2089
duke@435	2090	void Assembler::mull(Register src) {
duke@435	2091	// was missing
duke@435	2092	int encode = prefix_and_encode(src->encoding());
duke@435	2093	emit_byte(0xF7);
duke@435	2094	emit_byte(0xE0 | encode);
duke@435	2095	}
duke@435	2096
duke@435	2097	void Assembler::negl(Register dst) {
duke@435	2098	int encode = prefix_and_encode(dst->encoding());
duke@435	2099	emit_byte(0xF7);
duke@435	2100	emit_byte(0xD8 | encode);
duke@435	2101	}
duke@435	2102
duke@435	2103	void Assembler::negq(Register dst) {
duke@435	2104	int encode = prefixq_and_encode(dst->encoding());
duke@435	2105	emit_byte(0xF7);
duke@435	2106	emit_byte(0xD8 | encode);
duke@435	2107	}
duke@435	2108
duke@435	2109	void Assembler::notl(Register dst) {
duke@435	2110	int encode = prefix_and_encode(dst->encoding());
duke@435	2111	emit_byte(0xF7);
duke@435	2112	emit_byte(0xD0 | encode);
duke@435	2113	}
duke@435	2114
duke@435	2115	void Assembler::notq(Register dst) {
duke@435	2116	int encode = prefixq_and_encode(dst->encoding());
duke@435	2117	emit_byte(0xF7);
duke@435	2118	emit_byte(0xD0 | encode);
duke@435	2119	}
duke@435	2120
duke@435	2121	void Assembler::orl(Address dst, int imm32) {
duke@435	2122	InstructionMark im(this);
duke@435	2123	prefix(dst);
duke@435	2124	emit_byte(0x81);
duke@435	2125	emit_operand(rcx, dst, 4);
duke@435	2126	emit_long(imm32);
duke@435	2127	}
duke@435	2128
duke@435	2129	void Assembler::orl(Register dst, int imm32) {
duke@435	2130	prefix(dst);
duke@435	2131	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2132	}
duke@435	2133
duke@435	2134	void Assembler::orl(Register dst, Address src) {
duke@435	2135	InstructionMark im(this);
duke@435	2136	prefix(src, dst);
duke@435	2137	emit_byte(0x0B);
duke@435	2138	emit_operand(dst, src);
duke@435	2139	}
duke@435	2140
duke@435	2141	void Assembler::orl(Register dst, Register src) {
duke@435	2142	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2143	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2144	}
duke@435	2145
duke@435	2146	void Assembler::orq(Address dst, int imm32) {
duke@435	2147	InstructionMark im(this);
duke@435	2148	prefixq(dst);
duke@435	2149	emit_byte(0x81);
duke@435	2150	emit_operand(rcx, dst, 4);
duke@435	2151	emit_long(imm32);
duke@435	2152	}
duke@435	2153
duke@435	2154	void Assembler::orq(Register dst, int imm32) {
duke@435	2155	(void) prefixq_and_encode(dst->encoding());
duke@435	2156	emit_arith(0x81, 0xC8, dst, imm32);
duke@435	2157	}
duke@435	2158
duke@435	2159	void Assembler::orq(Register dst, Address src) {
duke@435	2160	InstructionMark im(this);
duke@435	2161	prefixq(src, dst);
duke@435	2162	emit_byte(0x0B);
duke@435	2163	emit_operand(dst, src);
duke@435	2164	}
duke@435	2165
duke@435	2166	void Assembler::orq(Register dst, Register src) {
duke@435	2167	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2168	emit_arith(0x0B, 0xC0, dst, src);
duke@435	2169	}
duke@435	2170
duke@435	2171	void Assembler::rcll(Register dst, int imm8) {
duke@435	2172	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2173	int encode = prefix_and_encode(dst->encoding());
duke@435	2174	if (imm8 == 1) {
duke@435	2175	emit_byte(0xD1);
duke@435	2176	emit_byte(0xD0 | encode);
duke@435	2177	} else {
duke@435	2178	emit_byte(0xC1);
duke@435	2179	emit_byte(0xD0 | encode);
duke@435	2180	emit_byte(imm8);
duke@435	2181	}
duke@435	2182	}
duke@435	2183
duke@435	2184	void Assembler::rclq(Register dst, int imm8) {
duke@435	2185	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2186	int encode = prefixq_and_encode(dst->encoding());
duke@435	2187	if (imm8 == 1) {
duke@435	2188	emit_byte(0xD1);
duke@435	2189	emit_byte(0xD0 | encode);
duke@435	2190	} else {
duke@435	2191	emit_byte(0xC1);
duke@435	2192	emit_byte(0xD0 | encode);
duke@435	2193	emit_byte(imm8);
duke@435	2194	}
duke@435	2195	}
duke@435	2196
duke@435	2197	void Assembler::sarl(Register dst, int imm8) {
duke@435	2198	int encode = prefix_and_encode(dst->encoding());
duke@435	2199	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2200	if (imm8 == 1) {
duke@435	2201	emit_byte(0xD1);
duke@435	2202	emit_byte(0xF8 | encode);
duke@435	2203	} else {
duke@435	2204	emit_byte(0xC1);
duke@435	2205	emit_byte(0xF8 | encode);
duke@435	2206	emit_byte(imm8);
duke@435	2207	}
duke@435	2208	}
duke@435	2209
duke@435	2210	void Assembler::sarl(Register dst) {
duke@435	2211	int encode = prefix_and_encode(dst->encoding());
duke@435	2212	emit_byte(0xD3);
duke@435	2213	emit_byte(0xF8 | encode);
duke@435	2214	}
duke@435	2215
duke@435	2216	void Assembler::sarq(Register dst, int imm8) {
duke@435	2217	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2218	int encode = prefixq_and_encode(dst->encoding());
duke@435	2219	if (imm8 == 1) {
duke@435	2220	emit_byte(0xD1);
duke@435	2221	emit_byte(0xF8 | encode);
duke@435	2222	} else {
duke@435	2223	emit_byte(0xC1);
duke@435	2224	emit_byte(0xF8 | encode);
duke@435	2225	emit_byte(imm8);
duke@435	2226	}
duke@435	2227	}
duke@435	2228
duke@435	2229	void Assembler::sarq(Register dst) {
duke@435	2230	int encode = prefixq_and_encode(dst->encoding());
duke@435	2231	emit_byte(0xD3);
duke@435	2232	emit_byte(0xF8 | encode);
duke@435	2233	}
duke@435	2234
duke@435	2235	void Assembler::sbbl(Address dst, int imm32) {
duke@435	2236	InstructionMark im(this);
duke@435	2237	prefix(dst);
duke@435	2238	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2239	}
duke@435	2240
duke@435	2241	void Assembler::sbbl(Register dst, int imm32) {
duke@435	2242	prefix(dst);
duke@435	2243	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2244	}
duke@435	2245
duke@435	2246	void Assembler::sbbl(Register dst, Address src) {
duke@435	2247	InstructionMark im(this);
duke@435	2248	prefix(src, dst);
duke@435	2249	emit_byte(0x1B);
duke@435	2250	emit_operand(dst, src);
duke@435	2251	}
duke@435	2252
duke@435	2253	void Assembler::sbbl(Register dst, Register src) {
duke@435	2254	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2255	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2256	}
duke@435	2257
duke@435	2258	void Assembler::sbbq(Address dst, int imm32) {
duke@435	2259	InstructionMark im(this);
duke@435	2260	prefixq(dst);
duke@435	2261	emit_arith_operand(0x81, rbx, dst, imm32);
duke@435	2262	}
duke@435	2263
duke@435	2264	void Assembler::sbbq(Register dst, int imm32) {
duke@435	2265	(void) prefixq_and_encode(dst->encoding());
duke@435	2266	emit_arith(0x81, 0xD8, dst, imm32);
duke@435	2267	}
duke@435	2268
duke@435	2269	void Assembler::sbbq(Register dst, Address src) {
duke@435	2270	InstructionMark im(this);
duke@435	2271	prefixq(src, dst);
duke@435	2272	emit_byte(0x1B);
duke@435	2273	emit_operand(dst, src);
duke@435	2274	}
duke@435	2275
duke@435	2276	void Assembler::sbbq(Register dst, Register src) {
duke@435	2277	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2278	emit_arith(0x1B, 0xC0, dst, src);
duke@435	2279	}
duke@435	2280
duke@435	2281	void Assembler::shll(Register dst, int imm8) {
duke@435	2282	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2283	int encode = prefix_and_encode(dst->encoding());
duke@435	2284	if (imm8 == 1 ) {
duke@435	2285	emit_byte(0xD1);
duke@435	2286	emit_byte(0xE0 | encode);
duke@435	2287	} else {
duke@435	2288	emit_byte(0xC1);
duke@435	2289	emit_byte(0xE0 | encode);
duke@435	2290	emit_byte(imm8);
duke@435	2291	}
duke@435	2292	}
duke@435	2293
duke@435	2294	void Assembler::shll(Register dst) {
duke@435	2295	int encode = prefix_and_encode(dst->encoding());
duke@435	2296	emit_byte(0xD3);
duke@435	2297	emit_byte(0xE0 | encode);
duke@435	2298	}
duke@435	2299
duke@435	2300	void Assembler::shlq(Register dst, int imm8) {
duke@435	2301	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2302	int encode = prefixq_and_encode(dst->encoding());
duke@435	2303	if (imm8 == 1) {
duke@435	2304	emit_byte(0xD1);
duke@435	2305	emit_byte(0xE0 | encode);
duke@435	2306	} else {
duke@435	2307	emit_byte(0xC1);
duke@435	2308	emit_byte(0xE0 | encode);
duke@435	2309	emit_byte(imm8);
duke@435	2310	}
duke@435	2311	}
duke@435	2312
duke@435	2313	void Assembler::shlq(Register dst) {
duke@435	2314	int encode = prefixq_and_encode(dst->encoding());
duke@435	2315	emit_byte(0xD3);
duke@435	2316	emit_byte(0xE0 | encode);
duke@435	2317	}
duke@435	2318
duke@435	2319	void Assembler::shrl(Register dst, int imm8) {
duke@435	2320	assert(isShiftCount(imm8), "illegal shift count");
duke@435	2321	int encode = prefix_and_encode(dst->encoding());
duke@435	2322	emit_byte(0xC1);
duke@435	2323	emit_byte(0xE8 | encode);
duke@435	2324	emit_byte(imm8);
duke@435	2325	}
duke@435	2326
duke@435	2327	void Assembler::shrl(Register dst) {
duke@435	2328	int encode = prefix_and_encode(dst->encoding());
duke@435	2329	emit_byte(0xD3);
duke@435	2330	emit_byte(0xE8 | encode);
duke@435	2331	}
duke@435	2332
duke@435	2333	void Assembler::shrq(Register dst, int imm8) {
duke@435	2334	assert(isShiftCount(imm8 >> 1), "illegal shift count");
duke@435	2335	int encode = prefixq_and_encode(dst->encoding());
duke@435	2336	emit_byte(0xC1);
duke@435	2337	emit_byte(0xE8 | encode);
duke@435	2338	emit_byte(imm8);
duke@435	2339	}
duke@435	2340
duke@435	2341	void Assembler::shrq(Register dst) {
duke@435	2342	int encode = prefixq_and_encode(dst->encoding());
duke@435	2343	emit_byte(0xD3);
duke@435	2344	emit_byte(0xE8 | encode);
duke@435	2345	}
duke@435	2346
duke@435	2347	void Assembler::subl(Address dst, int imm32) {
duke@435	2348	InstructionMark im(this);
duke@435	2349	prefix(dst);
duke@435	2350	if (is8bit(imm32)) {
duke@435	2351	emit_byte(0x83);
duke@435	2352	emit_operand(rbp, dst, 1);
duke@435	2353	emit_byte(imm32 & 0xFF);
duke@435	2354	} else {
duke@435	2355	emit_byte(0x81);
duke@435	2356	emit_operand(rbp, dst, 4);
duke@435	2357	emit_long(imm32);
duke@435	2358	}
duke@435	2359	}
duke@435	2360
duke@435	2361	void Assembler::subl(Register dst, int imm32) {
duke@435	2362	prefix(dst);
duke@435	2363	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2364	}
duke@435	2365
duke@435	2366	void Assembler::subl(Address dst, Register src) {
duke@435	2367	InstructionMark im(this);
duke@435	2368	prefix(dst, src);
duke@435	2369	emit_byte(0x29);
duke@435	2370	emit_operand(src, dst);
duke@435	2371	}
duke@435	2372
duke@435	2373	void Assembler::subl(Register dst, Address src) {
duke@435	2374	InstructionMark im(this);
duke@435	2375	prefix(src, dst);
duke@435	2376	emit_byte(0x2B);
duke@435	2377	emit_operand(dst, src);
duke@435	2378	}
duke@435	2379
duke@435	2380	void Assembler::subl(Register dst, Register src) {
duke@435	2381	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2382	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2383	}
duke@435	2384
duke@435	2385	void Assembler::subq(Address dst, int imm32) {
duke@435	2386	InstructionMark im(this);
duke@435	2387	prefixq(dst);
duke@435	2388	if (is8bit(imm32)) {
duke@435	2389	emit_byte(0x83);
duke@435	2390	emit_operand(rbp, dst, 1);
duke@435	2391	emit_byte(imm32 & 0xFF);
duke@435	2392	} else {
duke@435	2393	emit_byte(0x81);
duke@435	2394	emit_operand(rbp, dst, 4);
duke@435	2395	emit_long(imm32);
duke@435	2396	}
duke@435	2397	}
duke@435	2398
duke@435	2399	void Assembler::subq(Register dst, int imm32) {
duke@435	2400	(void) prefixq_and_encode(dst->encoding());
duke@435	2401	emit_arith(0x81, 0xE8, dst, imm32);
duke@435	2402	}
duke@435	2403
duke@435	2404	void Assembler::subq(Address dst, Register src) {
duke@435	2405	InstructionMark im(this);
duke@435	2406	prefixq(dst, src);
duke@435	2407	emit_byte(0x29);
duke@435	2408	emit_operand(src, dst);
duke@435	2409	}
duke@435	2410
duke@435	2411	void Assembler::subq(Register dst, Address src) {
duke@435	2412	InstructionMark im(this);
duke@435	2413	prefixq(src, dst);
duke@435	2414	emit_byte(0x2B);
duke@435	2415	emit_operand(dst, src);
duke@435	2416	}
duke@435	2417
duke@435	2418	void Assembler::subq(Register dst, Register src) {
duke@435	2419	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2420	emit_arith(0x2B, 0xC0, dst, src);
duke@435	2421	}
duke@435	2422
duke@435	2423	void Assembler::testb(Register dst, int imm8) {
duke@435	2424	(void) prefix_and_encode(dst->encoding(), true);
duke@435	2425	emit_arith_b(0xF6, 0xC0, dst, imm8);
duke@435	2426	}
duke@435	2427
duke@435	2428	void Assembler::testl(Register dst, int imm32) {
duke@435	2429	// not using emit_arith because test
duke@435	2430	// doesn't support sign-extension of
duke@435	2431	// 8bit operands
duke@435	2432	int encode = dst->encoding();
duke@435	2433	if (encode == 0) {
duke@435	2434	emit_byte(0xA9);
duke@435	2435	} else {
duke@435	2436	encode = prefix_and_encode(encode);
duke@435	2437	emit_byte(0xF7);
duke@435	2438	emit_byte(0xC0 | encode);
duke@435	2439	}
duke@435	2440	emit_long(imm32);
duke@435	2441	}
duke@435	2442
duke@435	2443	void Assembler::testl(Register dst, Register src) {
duke@435	2444	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2445	emit_arith(0x85, 0xC0, dst, src);
duke@435	2446	}
duke@435	2447
duke@435	2448	void Assembler::testq(Register dst, int imm32) {
duke@435	2449	// not using emit_arith because test
duke@435	2450	// doesn't support sign-extension of
duke@435	2451	// 8bit operands
duke@435	2452	int encode = dst->encoding();
duke@435	2453	if (encode == 0) {
duke@435	2454	prefix(REX_W);
duke@435	2455	emit_byte(0xA9);
duke@435	2456	} else {
duke@435	2457	encode = prefixq_and_encode(encode);
duke@435	2458	emit_byte(0xF7);
duke@435	2459	emit_byte(0xC0 | encode);
duke@435	2460	}
duke@435	2461	emit_long(imm32);
duke@435	2462	}
duke@435	2463
duke@435	2464	void Assembler::testq(Register dst, Register src) {
duke@435	2465	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2466	emit_arith(0x85, 0xC0, dst, src);
duke@435	2467	}
duke@435	2468
duke@435	2469	void Assembler::xaddl(Address dst, Register src) {
duke@435	2470	InstructionMark im(this);
duke@435	2471	prefix(dst, src);
duke@435	2472	emit_byte(0x0F);
duke@435	2473	emit_byte(0xC1);
duke@435	2474	emit_operand(src, dst);
duke@435	2475	}
duke@435	2476
duke@435	2477	void Assembler::xaddq(Address dst, Register src) {
duke@435	2478	InstructionMark im(this);
duke@435	2479	prefixq(dst, src);
duke@435	2480	emit_byte(0x0F);
duke@435	2481	emit_byte(0xC1);
duke@435	2482	emit_operand(src, dst);
duke@435	2483	}
duke@435	2484
duke@435	2485	void Assembler::xorl(Register dst, int imm32) {
duke@435	2486	prefix(dst);
duke@435	2487	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2488	}
duke@435	2489
duke@435	2490	void Assembler::xorl(Register dst, Register src) {
duke@435	2491	(void) prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2492	emit_arith(0x33, 0xC0, dst, src);
duke@435	2493	}
duke@435	2494
duke@435	2495	void Assembler::xorl(Register dst, Address src) {
duke@435	2496	InstructionMark im(this);
duke@435	2497	prefix(src, dst);
duke@435	2498	emit_byte(0x33);
duke@435	2499	emit_operand(dst, src);
duke@435	2500	}
duke@435	2501
duke@435	2502	void Assembler::xorq(Register dst, int imm32) {
duke@435	2503	(void) prefixq_and_encode(dst->encoding());
duke@435	2504	emit_arith(0x81, 0xF0, dst, imm32);
duke@435	2505	}
duke@435	2506
duke@435	2507	void Assembler::xorq(Register dst, Register src) {
duke@435	2508	(void) prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2509	emit_arith(0x33, 0xC0, dst, src);
duke@435	2510	}
duke@435	2511
duke@435	2512	void Assembler::xorq(Register dst, Address src) {
duke@435	2513	InstructionMark im(this);
duke@435	2514	prefixq(src, dst);
duke@435	2515	emit_byte(0x33);
duke@435	2516	emit_operand(dst, src);
duke@435	2517	}
duke@435	2518
duke@435	2519	void Assembler::bswapl(Register reg) {
duke@435	2520	int encode = prefix_and_encode(reg->encoding());
duke@435	2521	emit_byte(0x0F);
duke@435	2522	emit_byte(0xC8 | encode);
duke@435	2523	}
duke@435	2524
duke@435	2525	void Assembler::bswapq(Register reg) {
duke@435	2526	int encode = prefixq_and_encode(reg->encoding());
duke@435	2527	emit_byte(0x0F);
duke@435	2528	emit_byte(0xC8 | encode);
duke@435	2529	}
duke@435	2530
duke@435	2531	void Assembler::lock() {
duke@435	2532	emit_byte(0xF0);
duke@435	2533	}
duke@435	2534
duke@435	2535	void Assembler::xchgl(Register dst, Address src) {
duke@435	2536	InstructionMark im(this);
duke@435	2537	prefix(src, dst);
duke@435	2538	emit_byte(0x87);
duke@435	2539	emit_operand(dst, src);
duke@435	2540	}
duke@435	2541
duke@435	2542	void Assembler::xchgl(Register dst, Register src) {
duke@435	2543	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	2544	emit_byte(0x87);
duke@435	2545	emit_byte(0xc0 | encode);
duke@435	2546	}
duke@435	2547
duke@435	2548	void Assembler::xchgq(Register dst, Address src) {
duke@435	2549	InstructionMark im(this);
duke@435	2550	prefixq(src, dst);
duke@435	2551	emit_byte(0x87);
duke@435	2552	emit_operand(dst, src);
duke@435	2553	}
duke@435	2554
duke@435	2555	void Assembler::xchgq(Register dst, Register src) {
duke@435	2556	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	2557	emit_byte(0x87);
duke@435	2558	emit_byte(0xc0 | encode);
duke@435	2559	}
duke@435	2560
duke@435	2561	void Assembler::cmpxchgl(Register reg, Address adr) {
duke@435	2562	InstructionMark im(this);
duke@435	2563	prefix(adr, reg);
duke@435	2564	emit_byte(0x0F);
duke@435	2565	emit_byte(0xB1);
duke@435	2566	emit_operand(reg, adr);
duke@435	2567	}
duke@435	2568
duke@435	2569	void Assembler::cmpxchgq(Register reg, Address adr) {
duke@435	2570	InstructionMark im(this);
duke@435	2571	prefixq(adr, reg);
duke@435	2572	emit_byte(0x0F);
duke@435	2573	emit_byte(0xB1);
duke@435	2574	emit_operand(reg, adr);
duke@435	2575	}
duke@435	2576
duke@435	2577	void Assembler::hlt() {
duke@435	2578	emit_byte(0xF4);
duke@435	2579	}
duke@435	2580
duke@435	2581
duke@435	2582	void Assembler::addr_nop_4() {
duke@435	2583	// 4 bytes: NOP DWORD PTR [EAX+0]
duke@435	2584	emit_byte(0x0F);
duke@435	2585	emit_byte(0x1F);
duke@435	2586	emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
duke@435	2587	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2588	}
duke@435	2589
duke@435	2590	void Assembler::addr_nop_5() {
duke@435	2591	// 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
duke@435	2592	emit_byte(0x0F);
duke@435	2593	emit_byte(0x1F);
duke@435	2594	emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
duke@435	2595	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2596	emit_byte(0); // 8-bits offset (1 byte)
duke@435	2597	}
duke@435	2598
duke@435	2599	void Assembler::addr_nop_7() {
duke@435	2600	// 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
duke@435	2601	emit_byte(0x0F);
duke@435	2602	emit_byte(0x1F);
duke@435	2603	emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
duke@435	2604	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2605	}
duke@435	2606
duke@435	2607	void Assembler::addr_nop_8() {
duke@435	2608	// 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
duke@435	2609	emit_byte(0x0F);
duke@435	2610	emit_byte(0x1F);
duke@435	2611	emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
duke@435	2612	emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
duke@435	2613	emit_long(0); // 32-bits offset (4 bytes)
duke@435	2614	}
duke@435	2615
duke@435	2616	void Assembler::nop(int i) {
duke@435	2617	assert(i > 0, " ");
duke@435	2618	if (UseAddressNop && VM_Version::is_intel()) {
duke@435	2619	//
duke@435	2620	// Using multi-bytes nops "0x0F 0x1F [address]" for Intel
duke@435	2621	// 1: 0x90
duke@435	2622	// 2: 0x66 0x90
duke@435	2623	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2624	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2625	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2626	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2627	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2628	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2629	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2630	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2631	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2632
duke@435	2633	// The rest coding is Intel specific - don't use consecutive address nops
duke@435	2634
duke@435	2635	// 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2636	// 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2637	// 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2638	// 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
duke@435	2639
duke@435	2640	while(i >= 15) {
duke@435	2641	// For Intel don't generate consecutive addess nops (mix with regular nops)
duke@435	2642	i -= 15;
duke@435	2643	emit_byte(0x66); // size prefix
duke@435	2644	emit_byte(0x66); // size prefix
duke@435	2645	emit_byte(0x66); // size prefix
duke@435	2646	addr_nop_8();
duke@435	2647	emit_byte(0x66); // size prefix
duke@435	2648	emit_byte(0x66); // size prefix
duke@435	2649	emit_byte(0x66); // size prefix
duke@435	2650	emit_byte(0x90); // nop
duke@435	2651	}
duke@435	2652	switch (i) {
duke@435	2653	case 14:
duke@435	2654	emit_byte(0x66); // size prefix
duke@435	2655	case 13:
duke@435	2656	emit_byte(0x66); // size prefix
duke@435	2657	case 12:
duke@435	2658	addr_nop_8();
duke@435	2659	emit_byte(0x66); // size prefix
duke@435	2660	emit_byte(0x66); // size prefix
duke@435	2661	emit_byte(0x66); // size prefix
duke@435	2662	emit_byte(0x90); // nop
duke@435	2663	break;
duke@435	2664	case 11:
duke@435	2665	emit_byte(0x66); // size prefix
duke@435	2666	case 10:
duke@435	2667	emit_byte(0x66); // size prefix
duke@435	2668	case 9:
duke@435	2669	emit_byte(0x66); // size prefix
duke@435	2670	case 8:
duke@435	2671	addr_nop_8();
duke@435	2672	break;
duke@435	2673	case 7:
duke@435	2674	addr_nop_7();
duke@435	2675	break;
duke@435	2676	case 6:
duke@435	2677	emit_byte(0x66); // size prefix
duke@435	2678	case 5:
duke@435	2679	addr_nop_5();
duke@435	2680	break;
duke@435	2681	case 4:
duke@435	2682	addr_nop_4();
duke@435	2683	break;
duke@435	2684	case 3:
duke@435	2685	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2686	emit_byte(0x66); // size prefix
duke@435	2687	case 2:
duke@435	2688	emit_byte(0x66); // size prefix
duke@435	2689	case 1:
duke@435	2690	emit_byte(0x90); // nop
duke@435	2691	break;
duke@435	2692	default:
duke@435	2693	assert(i == 0, " ");
duke@435	2694	}
duke@435	2695	return;
duke@435	2696	}
duke@435	2697	if (UseAddressNop && VM_Version::is_amd()) {
duke@435	2698	//
duke@435	2699	// Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
duke@435	2700	// 1: 0x90
duke@435	2701	// 2: 0x66 0x90
duke@435	2702	// 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
duke@435	2703	// 4: 0x0F 0x1F 0x40 0x00
duke@435	2704	// 5: 0x0F 0x1F 0x44 0x00 0x00
duke@435	2705	// 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2706	// 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2707	// 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2708	// 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2709	// 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2710	// 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2711
duke@435	2712	// The rest coding is AMD specific - use consecutive address nops
duke@435	2713
duke@435	2714	// 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2715	// 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
duke@435	2716	// 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2717	// 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
duke@435	2718	// 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
duke@435	2719	// Size prefixes (0x66) are added for larger sizes
duke@435	2720
duke@435	2721	while(i >= 22) {
duke@435	2722	i -= 11;
duke@435	2723	emit_byte(0x66); // size prefix
duke@435	2724	emit_byte(0x66); // size prefix
duke@435	2725	emit_byte(0x66); // size prefix
duke@435	2726	addr_nop_8();
duke@435	2727	}
duke@435	2728	// Generate first nop for size between 21-12
duke@435	2729	switch (i) {
duke@435	2730	case 21:
duke@435	2731	i -= 1;
duke@435	2732	emit_byte(0x66); // size prefix
duke@435	2733	case 20:
duke@435	2734	case 19:
duke@435	2735	i -= 1;
duke@435	2736	emit_byte(0x66); // size prefix
duke@435	2737	case 18:
duke@435	2738	case 17:
duke@435	2739	i -= 1;
duke@435	2740	emit_byte(0x66); // size prefix
duke@435	2741	case 16:
duke@435	2742	case 15:
duke@435	2743	i -= 8;
duke@435	2744	addr_nop_8();
duke@435	2745	break;
duke@435	2746	case 14:
duke@435	2747	case 13:
duke@435	2748	i -= 7;
duke@435	2749	addr_nop_7();
duke@435	2750	break;
duke@435	2751	case 12:
duke@435	2752	i -= 6;
duke@435	2753	emit_byte(0x66); // size prefix
duke@435	2754	addr_nop_5();
duke@435	2755	break;
duke@435	2756	default:
duke@435	2757	assert(i < 12, " ");
duke@435	2758	}
duke@435	2759
duke@435	2760	// Generate second nop for size between 11-1
duke@435	2761	switch (i) {
duke@435	2762	case 11:
duke@435	2763	emit_byte(0x66); // size prefix
duke@435	2764	case 10:
duke@435	2765	emit_byte(0x66); // size prefix
duke@435	2766	case 9:
duke@435	2767	emit_byte(0x66); // size prefix
duke@435	2768	case 8:
duke@435	2769	addr_nop_8();
duke@435	2770	break;
duke@435	2771	case 7:
duke@435	2772	addr_nop_7();
duke@435	2773	break;
duke@435	2774	case 6:
duke@435	2775	emit_byte(0x66); // size prefix
duke@435	2776	case 5:
duke@435	2777	addr_nop_5();
duke@435	2778	break;
duke@435	2779	case 4:
duke@435	2780	addr_nop_4();
duke@435	2781	break;
duke@435	2782	case 3:
duke@435	2783	// Don't use "0x0F 0x1F 0x00" - need patching safe padding
duke@435	2784	emit_byte(0x66); // size prefix
duke@435	2785	case 2:
duke@435	2786	emit_byte(0x66); // size prefix
duke@435	2787	case 1:
duke@435	2788	emit_byte(0x90); // nop
duke@435	2789	break;
duke@435	2790	default:
duke@435	2791	assert(i == 0, " ");
duke@435	2792	}
duke@435	2793	return;
duke@435	2794	}
duke@435	2795
duke@435	2796	// Using nops with size prefixes "0x66 0x90".
duke@435	2797	// From AMD Optimization Guide:
duke@435	2798	// 1: 0x90
duke@435	2799	// 2: 0x66 0x90
duke@435	2800	// 3: 0x66 0x66 0x90
duke@435	2801	// 4: 0x66 0x66 0x66 0x90
duke@435	2802	// 5: 0x66 0x66 0x90 0x66 0x90
duke@435	2803	// 6: 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2804	// 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2805	// 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
duke@435	2806	// 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2807	// 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
duke@435	2808	//
duke@435	2809	while(i > 12) {
duke@435	2810	i -= 4;
duke@435	2811	emit_byte(0x66); // size prefix
duke@435	2812	emit_byte(0x66);
duke@435	2813	emit_byte(0x66);
duke@435	2814	emit_byte(0x90); // nop
duke@435	2815	}
duke@435	2816	// 1 - 12 nops
duke@435	2817	if(i > 8) {
duke@435	2818	if(i > 9) {
duke@435	2819	i -= 1;
duke@435	2820	emit_byte(0x66);
duke@435	2821	}
duke@435	2822	i -= 3;
duke@435	2823	emit_byte(0x66);
duke@435	2824	emit_byte(0x66);
duke@435	2825	emit_byte(0x90);
duke@435	2826	}
duke@435	2827	// 1 - 8 nops
duke@435	2828	if(i > 4) {
duke@435	2829	if(i > 6) {
duke@435	2830	i -= 1;
duke@435	2831	emit_byte(0x66);
duke@435	2832	}
duke@435	2833	i -= 3;
duke@435	2834	emit_byte(0x66);
duke@435	2835	emit_byte(0x66);
duke@435	2836	emit_byte(0x90);
duke@435	2837	}
duke@435	2838	switch (i) {
duke@435	2839	case 4:
duke@435	2840	emit_byte(0x66);
duke@435	2841	case 3:
duke@435	2842	emit_byte(0x66);
duke@435	2843	case 2:
duke@435	2844	emit_byte(0x66);
duke@435	2845	case 1:
duke@435	2846	emit_byte(0x90);
duke@435	2847	break;
duke@435	2848	default:
duke@435	2849	assert(i == 0, " ");
duke@435	2850	}
duke@435	2851	}
duke@435	2852
duke@435	2853	void Assembler::ret(int imm16) {
duke@435	2854	if (imm16 == 0) {
duke@435	2855	emit_byte(0xC3);
duke@435	2856	} else {
duke@435	2857	emit_byte(0xC2);
duke@435	2858	emit_word(imm16);
duke@435	2859	}
duke@435	2860	}
duke@435	2861
duke@435	2862	// copies a single word from [esi] to [edi]
duke@435	2863	void Assembler::smovl() {
duke@435	2864	emit_byte(0xA5);
duke@435	2865	}
duke@435	2866
duke@435	2867	// copies data from [rsi] to [rdi] using rcx words (m32)
duke@435	2868	void Assembler::rep_movl() {
duke@435	2869	// REP
duke@435	2870	emit_byte(0xF3);
duke@435	2871	// MOVSL
duke@435	2872	emit_byte(0xA5);
duke@435	2873	}
duke@435	2874
duke@435	2875	// copies data from [rsi] to [rdi] using rcx double words (m64)
duke@435	2876	void Assembler::rep_movq() {
duke@435	2877	// REP
duke@435	2878	emit_byte(0xF3);
duke@435	2879	// MOVSQ
duke@435	2880	prefix(REX_W);
duke@435	2881	emit_byte(0xA5);
duke@435	2882	}
duke@435	2883
duke@435	2884	// sets rcx double words (m64) with rax value at [rdi]
duke@435	2885	void Assembler::rep_set() {
duke@435	2886	// REP
duke@435	2887	emit_byte(0xF3);
duke@435	2888	// STOSQ
duke@435	2889	prefix(REX_W);
duke@435	2890	emit_byte(0xAB);
duke@435	2891	}
duke@435	2892
duke@435	2893	// scans rcx double words (m64) at [rdi] for occurance of rax
duke@435	2894	void Assembler::repne_scan() {
duke@435	2895	// REPNE/REPNZ
duke@435	2896	emit_byte(0xF2);
duke@435	2897	// SCASQ
duke@435	2898	prefix(REX_W);
duke@435	2899	emit_byte(0xAF);
duke@435	2900	}
duke@435	2901
duke@435	2902	void Assembler::setb(Condition cc, Register dst) {
duke@435	2903	assert(0 <= cc && cc < 16, "illegal cc");
duke@435	2904	int encode = prefix_and_encode(dst->encoding(), true);
duke@435	2905	emit_byte(0x0F);
duke@435	2906	emit_byte(0x90 | cc);
duke@435	2907	emit_byte(0xC0 | encode);
duke@435	2908	}
duke@435	2909
duke@435	2910	void Assembler::clflush(Address adr) {
duke@435	2911	prefix(adr);
duke@435	2912	emit_byte(0x0F);
duke@435	2913	emit_byte(0xAE);
duke@435	2914	emit_operand(rdi, adr);
duke@435	2915	}
duke@435	2916
duke@435	2917	void Assembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	2918	if (L.is_bound()) {
duke@435	2919	const int long_size = 5;
duke@435	2920	int offs = (int)( target(L) - pc() );
duke@435	2921	assert(offs <= 0, "assembler error");
duke@435	2922	InstructionMark im(this);
duke@435	2923	// 1110 1000 #32-bit disp
duke@435	2924	emit_byte(0xE8);
duke@435	2925	emit_data(offs - long_size, rtype, disp32_operand);
duke@435	2926	} else {
duke@435	2927	InstructionMark im(this);
duke@435	2928	// 1110 1000 #32-bit disp
duke@435	2929	L.add_patch_at(code(), locator());
duke@435	2930
duke@435	2931	emit_byte(0xE8);
duke@435	2932	emit_data(int(0), rtype, disp32_operand);
duke@435	2933	}
duke@435	2934	}
duke@435	2935
duke@435	2936	void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
duke@435	2937	assert(entry != NULL, "call most probably wrong");
duke@435	2938	InstructionMark im(this);
duke@435	2939	emit_byte(0xE8);
duke@435	2940	intptr_t disp = entry - (_code_pos + sizeof(int32_t));
duke@435	2941	assert(is_simm32(disp), "must be 32bit offset (call2)");
duke@435	2942	// Technically, should use call32_operand, but this format is
duke@435	2943	// implied by the fact that we're emitting a call instruction.
duke@435	2944	emit_data((int) disp, rspec, disp32_operand);
duke@435	2945	}
duke@435	2946
duke@435	2947
duke@435	2948	void Assembler::call(Register dst) {
duke@435	2949	// This was originally using a 32bit register encoding
duke@435	2950	// and surely we want 64bit!
duke@435	2951	// this is a 32bit encoding but in 64bit mode the default
duke@435	2952	// operand size is 64bit so there is no need for the
duke@435	2953	// wide prefix. So prefix only happens if we use the
duke@435	2954	// new registers. Much like push/pop.
duke@435	2955	int encode = prefixq_and_encode(dst->encoding());
duke@435	2956	emit_byte(0xFF);
duke@435	2957	emit_byte(0xD0 | encode);
duke@435	2958	}
duke@435	2959
duke@435	2960	void Assembler::call(Address adr) {
duke@435	2961	InstructionMark im(this);
duke@435	2962	prefix(adr);
duke@435	2963	emit_byte(0xFF);
duke@435	2964	emit_operand(rdx, adr);
duke@435	2965	}
duke@435	2966
duke@435	2967	void Assembler::jmp(Register reg) {
duke@435	2968	int encode = prefix_and_encode(reg->encoding());
duke@435	2969	emit_byte(0xFF);
duke@435	2970	emit_byte(0xE0 | encode);
duke@435	2971	}
duke@435	2972
duke@435	2973	void Assembler::jmp(Address adr) {
duke@435	2974	InstructionMark im(this);
duke@435	2975	prefix(adr);
duke@435	2976	emit_byte(0xFF);
duke@435	2977	emit_operand(rsp, adr);
duke@435	2978	}
duke@435	2979
duke@435	2980	void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
duke@435	2981	InstructionMark im(this);
duke@435	2982	emit_byte(0xE9);
duke@435	2983	assert(dest != NULL, "must have a target");
duke@435	2984	intptr_t disp = dest - (_code_pos + sizeof(int32_t));
duke@435	2985	assert(is_simm32(disp), "must be 32bit offset (jmp)");
duke@435	2986	emit_data(disp, rspec.reloc(), call32_operand);
duke@435	2987	}
duke@435	2988
duke@435	2989	void Assembler::jmp(Label& L, relocInfo::relocType rtype) {
duke@435	2990	if (L.is_bound()) {
duke@435	2991	address entry = target(L);
duke@435	2992	assert(entry != NULL, "jmp most probably wrong");
duke@435	2993	InstructionMark im(this);
duke@435	2994	const int short_size = 2;
duke@435	2995	const int long_size = 5;
duke@435	2996	intptr_t offs = entry - _code_pos;
duke@435	2997	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	2998	emit_byte(0xEB);
duke@435	2999	emit_byte((offs - short_size) & 0xFF);
duke@435	3000	} else {
duke@435	3001	emit_byte(0xE9);
duke@435	3002	emit_long(offs - long_size);
duke@435	3003	}
duke@435	3004	} else {
duke@435	3005	// By default, forward jumps are always 32-bit displacements, since
duke@435	3006	// we can't yet know where the label will be bound. If you're sure that
duke@435	3007	// the forward jump will not run beyond 256 bytes, use jmpb to
duke@435	3008	// force an 8-bit displacement.
duke@435	3009	InstructionMark im(this);
duke@435	3010	relocate(rtype);
duke@435	3011	L.add_patch_at(code(), locator());
duke@435	3012	emit_byte(0xE9);
duke@435	3013	emit_long(0);
duke@435	3014	}
duke@435	3015	}
duke@435	3016
duke@435	3017	void Assembler::jmpb(Label& L) {
duke@435	3018	if (L.is_bound()) {
duke@435	3019	const int short_size = 2;
duke@435	3020	address entry = target(L);
duke@435	3021	assert(is8bit((entry - _code_pos) + short_size),
duke@435	3022	"Dispacement too large for a short jmp");
duke@435	3023	assert(entry != NULL, "jmp most probably wrong");
duke@435	3024	intptr_t offs = entry - _code_pos;
duke@435	3025	emit_byte(0xEB);
duke@435	3026	emit_byte((offs - short_size) & 0xFF);
duke@435	3027	} else {
duke@435	3028	InstructionMark im(this);
duke@435	3029	L.add_patch_at(code(), locator());
duke@435	3030	emit_byte(0xEB);
duke@435	3031	emit_byte(0);
duke@435	3032	}
duke@435	3033	}
duke@435	3034
duke@435	3035	void Assembler::jcc(Condition cc, Label& L, relocInfo::relocType rtype) {
duke@435	3036	InstructionMark im(this);
duke@435	3037	relocate(rtype);
duke@435	3038	assert((0 <= cc) && (cc < 16), "illegal cc");
duke@435	3039	if (L.is_bound()) {
duke@435	3040	address dst = target(L);
duke@435	3041	assert(dst != NULL, "jcc most probably wrong");
duke@435	3042
duke@435	3043	const int short_size = 2;
duke@435	3044	const int long_size = 6;
duke@435	3045	intptr_t offs = (intptr_t)dst - (intptr_t)_code_pos;
duke@435	3046	if (rtype == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3047	// 0111 tttn #8-bit disp
duke@435	3048	emit_byte(0x70 | cc);
duke@435	3049	emit_byte((offs - short_size) & 0xFF);
duke@435	3050	} else {
duke@435	3051	// 0000 1111 1000 tttn #32-bit disp
duke@435	3052	assert(is_simm32(offs - long_size),
duke@435	3053	"must be 32bit offset (call4)");
duke@435	3054	emit_byte(0x0F);
duke@435	3055	emit_byte(0x80 | cc);
duke@435	3056	emit_long(offs - long_size);
duke@435	3057	}
duke@435	3058	} else {
duke@435	3059	// Note: could eliminate cond. jumps to this jump if condition
duke@435	3060	// is the same however, seems to be rather unlikely case.
duke@435	3061	// Note: use jccb() if label to be bound is very close to get
duke@435	3062	// an 8-bit displacement
duke@435	3063	L.add_patch_at(code(), locator());
duke@435	3064	emit_byte(0x0F);
duke@435	3065	emit_byte(0x80 | cc);
duke@435	3066	emit_long(0);
duke@435	3067	}
duke@435	3068	}
duke@435	3069
duke@435	3070	void Assembler::jccb(Condition cc, Label& L) {
duke@435	3071	if (L.is_bound()) {
duke@435	3072	const int short_size = 2;
duke@435	3073	const int long_size = 6;
duke@435	3074	address entry = target(L);
duke@435	3075	assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),
duke@435	3076	"Dispacement too large for a short jmp");
duke@435	3077	intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;
duke@435	3078	// 0111 tttn #8-bit disp
duke@435	3079	emit_byte(0x70 | cc);
duke@435	3080	emit_byte((offs - short_size) & 0xFF);
duke@435	3081	} else {
duke@435	3082	InstructionMark im(this);
duke@435	3083	L.add_patch_at(code(), locator());
duke@435	3084	emit_byte(0x70 | cc);
duke@435	3085	emit_byte(0);
duke@435	3086	}
duke@435	3087	}
duke@435	3088
duke@435	3089	// FP instructions
duke@435	3090
duke@435	3091	void Assembler::fxsave(Address dst) {
duke@435	3092	prefixq(dst);
duke@435	3093	emit_byte(0x0F);
duke@435	3094	emit_byte(0xAE);
duke@435	3095	emit_operand(as_Register(0), dst);
duke@435	3096	}
duke@435	3097
duke@435	3098	void Assembler::fxrstor(Address src) {
duke@435	3099	prefixq(src);
duke@435	3100	emit_byte(0x0F);
duke@435	3101	emit_byte(0xAE);
duke@435	3102	emit_operand(as_Register(1), src);
duke@435	3103	}
duke@435	3104
duke@435	3105	void Assembler::ldmxcsr(Address src) {
duke@435	3106	InstructionMark im(this);
duke@435	3107	prefix(src);
duke@435	3108	emit_byte(0x0F);
duke@435	3109	emit_byte(0xAE);
duke@435	3110	emit_operand(as_Register(2), src);
duke@435	3111	}
duke@435	3112
duke@435	3113	void Assembler::stmxcsr(Address dst) {
duke@435	3114	InstructionMark im(this);
duke@435	3115	prefix(dst);
duke@435	3116	emit_byte(0x0F);
duke@435	3117	emit_byte(0xAE);
duke@435	3118	emit_operand(as_Register(3), dst);
duke@435	3119	}
duke@435	3120
duke@435	3121	void Assembler::addss(XMMRegister dst, XMMRegister src) {
duke@435	3122	emit_byte(0xF3);
duke@435	3123	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3124	emit_byte(0x0F);
duke@435	3125	emit_byte(0x58);
duke@435	3126	emit_byte(0xC0 | encode);
duke@435	3127	}
duke@435	3128
duke@435	3129	void Assembler::addss(XMMRegister dst, Address src) {
duke@435	3130	InstructionMark im(this);
duke@435	3131	emit_byte(0xF3);
duke@435	3132	prefix(src, dst);
duke@435	3133	emit_byte(0x0F);
duke@435	3134	emit_byte(0x58);
duke@435	3135	emit_operand(dst, src);
duke@435	3136	}
duke@435	3137
duke@435	3138	void Assembler::subss(XMMRegister dst, XMMRegister src) {
duke@435	3139	emit_byte(0xF3);
duke@435	3140	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3141	emit_byte(0x0F);
duke@435	3142	emit_byte(0x5C);
duke@435	3143	emit_byte(0xC0 | encode);
duke@435	3144	}
duke@435	3145
duke@435	3146	void Assembler::subss(XMMRegister dst, Address src) {
duke@435	3147	InstructionMark im(this);
duke@435	3148	emit_byte(0xF3);
duke@435	3149	prefix(src, dst);
duke@435	3150	emit_byte(0x0F);
duke@435	3151	emit_byte(0x5C);
duke@435	3152	emit_operand(dst, src);
duke@435	3153	}
duke@435	3154
duke@435	3155	void Assembler::mulss(XMMRegister dst, XMMRegister src) {
duke@435	3156	emit_byte(0xF3);
duke@435	3157	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3158	emit_byte(0x0F);
duke@435	3159	emit_byte(0x59);
duke@435	3160	emit_byte(0xC0 | encode);
duke@435	3161	}
duke@435	3162
duke@435	3163	void Assembler::mulss(XMMRegister dst, Address src) {
duke@435	3164	InstructionMark im(this);
duke@435	3165	emit_byte(0xF3);
duke@435	3166	prefix(src, dst);
duke@435	3167	emit_byte(0x0F);
duke@435	3168	emit_byte(0x59);
duke@435	3169	emit_operand(dst, src);
duke@435	3170	}
duke@435	3171
duke@435	3172	void Assembler::divss(XMMRegister dst, XMMRegister src) {
duke@435	3173	emit_byte(0xF3);
duke@435	3174	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3175	emit_byte(0x0F);
duke@435	3176	emit_byte(0x5E);
duke@435	3177	emit_byte(0xC0 | encode);
duke@435	3178	}
duke@435	3179
duke@435	3180	void Assembler::divss(XMMRegister dst, Address src) {
duke@435	3181	InstructionMark im(this);
duke@435	3182	emit_byte(0xF3);
duke@435	3183	prefix(src, dst);
duke@435	3184	emit_byte(0x0F);
duke@435	3185	emit_byte(0x5E);
duke@435	3186	emit_operand(dst, src);
duke@435	3187	}
duke@435	3188
duke@435	3189	void Assembler::addsd(XMMRegister dst, XMMRegister src) {
duke@435	3190	emit_byte(0xF2);
duke@435	3191	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3192	emit_byte(0x0F);
duke@435	3193	emit_byte(0x58);
duke@435	3194	emit_byte(0xC0 | encode);
duke@435	3195	}
duke@435	3196
duke@435	3197	void Assembler::addsd(XMMRegister dst, Address src) {
duke@435	3198	InstructionMark im(this);
duke@435	3199	emit_byte(0xF2);
duke@435	3200	prefix(src, dst);
duke@435	3201	emit_byte(0x0F);
duke@435	3202	emit_byte(0x58);
duke@435	3203	emit_operand(dst, src);
duke@435	3204	}
duke@435	3205
duke@435	3206	void Assembler::subsd(XMMRegister dst, XMMRegister src) {
duke@435	3207	emit_byte(0xF2);
duke@435	3208	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3209	emit_byte(0x0F);
duke@435	3210	emit_byte(0x5C);
duke@435	3211	emit_byte(0xC0 | encode);
duke@435	3212	}
duke@435	3213
duke@435	3214	void Assembler::subsd(XMMRegister dst, Address src) {
duke@435	3215	InstructionMark im(this);
duke@435	3216	emit_byte(0xF2);
duke@435	3217	prefix(src, dst);
duke@435	3218	emit_byte(0x0F);
duke@435	3219	emit_byte(0x5C);
duke@435	3220	emit_operand(dst, src);
duke@435	3221	}
duke@435	3222
duke@435	3223	void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
duke@435	3224	emit_byte(0xF2);
duke@435	3225	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3226	emit_byte(0x0F);
duke@435	3227	emit_byte(0x59);
duke@435	3228	emit_byte(0xC0 | encode);
duke@435	3229	}
duke@435	3230
duke@435	3231	void Assembler::mulsd(XMMRegister dst, Address src) {
duke@435	3232	InstructionMark im(this);
duke@435	3233	emit_byte(0xF2);
duke@435	3234	prefix(src, dst);
duke@435	3235	emit_byte(0x0F);
duke@435	3236	emit_byte(0x59);
duke@435	3237	emit_operand(dst, src);
duke@435	3238	}
duke@435	3239
duke@435	3240	void Assembler::divsd(XMMRegister dst, XMMRegister src) {
duke@435	3241	emit_byte(0xF2);
duke@435	3242	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3243	emit_byte(0x0F);
duke@435	3244	emit_byte(0x5E);
duke@435	3245	emit_byte(0xC0 | encode);
duke@435	3246	}
duke@435	3247
duke@435	3248	void Assembler::divsd(XMMRegister dst, Address src) {
duke@435	3249	InstructionMark im(this);
duke@435	3250	emit_byte(0xF2);
duke@435	3251	prefix(src, dst);
duke@435	3252	emit_byte(0x0F);
duke@435	3253	emit_byte(0x5E);
duke@435	3254	emit_operand(dst, src);
duke@435	3255	}
duke@435	3256
duke@435	3257	void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
duke@435	3258	emit_byte(0xF2);
duke@435	3259	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3260	emit_byte(0x0F);
duke@435	3261	emit_byte(0x51);
duke@435	3262	emit_byte(0xC0 | encode);
duke@435	3263	}
duke@435	3264
duke@435	3265	void Assembler::sqrtsd(XMMRegister dst, Address src) {
duke@435	3266	InstructionMark im(this);
duke@435	3267	emit_byte(0xF2);
duke@435	3268	prefix(src, dst);
duke@435	3269	emit_byte(0x0F);
duke@435	3270	emit_byte(0x51);
duke@435	3271	emit_operand(dst, src);
duke@435	3272	}
duke@435	3273
duke@435	3274	void Assembler::xorps(XMMRegister dst, XMMRegister src) {
duke@435	3275	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3276	emit_byte(0x0F);
duke@435	3277	emit_byte(0x57);
duke@435	3278	emit_byte(0xC0 | encode);
duke@435	3279	}
duke@435	3280
duke@435	3281	void Assembler::xorps(XMMRegister dst, Address src) {
duke@435	3282	InstructionMark im(this);
duke@435	3283	prefix(src, dst);
duke@435	3284	emit_byte(0x0F);
duke@435	3285	emit_byte(0x57);
duke@435	3286	emit_operand(dst, src);
duke@435	3287	}
duke@435	3288
duke@435	3289	void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
duke@435	3290	emit_byte(0x66);
duke@435	3291	xorps(dst, src);
duke@435	3292	}
duke@435	3293
duke@435	3294	void Assembler::xorpd(XMMRegister dst, Address src) {
duke@435	3295	InstructionMark im(this);
duke@435	3296	emit_byte(0x66);
duke@435	3297	prefix(src, dst);
duke@435	3298	emit_byte(0x0F);
duke@435	3299	emit_byte(0x57);
duke@435	3300	emit_operand(dst, src);
duke@435	3301	}
duke@435	3302
duke@435	3303	void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
duke@435	3304	emit_byte(0xF3);
duke@435	3305	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3306	emit_byte(0x0F);
duke@435	3307	emit_byte(0x2A);
duke@435	3308	emit_byte(0xC0 | encode);
duke@435	3309	}
duke@435	3310
duke@435	3311	void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
duke@435	3312	emit_byte(0xF3);
duke@435	3313	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3314	emit_byte(0x0F);
duke@435	3315	emit_byte(0x2A);
duke@435	3316	emit_byte(0xC0 | encode);
duke@435	3317	}
duke@435	3318
duke@435	3319	void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
duke@435	3320	emit_byte(0xF2);
duke@435	3321	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3322	emit_byte(0x0F);
duke@435	3323	emit_byte(0x2A);
duke@435	3324	emit_byte(0xC0 | encode);
duke@435	3325	}
duke@435	3326
duke@435	3327	void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
duke@435	3328	emit_byte(0xF2);
duke@435	3329	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3330	emit_byte(0x0F);
duke@435	3331	emit_byte(0x2A);
duke@435	3332	emit_byte(0xC0 | encode);
duke@435	3333	}
duke@435	3334
duke@435	3335	void Assembler::cvttss2sil(Register dst, XMMRegister src) {
duke@435	3336	emit_byte(0xF3);
duke@435	3337	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3338	emit_byte(0x0F);
duke@435	3339	emit_byte(0x2C);
duke@435	3340	emit_byte(0xC0 | encode);
duke@435	3341	}
duke@435	3342
duke@435	3343	void Assembler::cvttss2siq(Register dst, XMMRegister src) {
duke@435	3344	emit_byte(0xF3);
duke@435	3345	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3346	emit_byte(0x0F);
duke@435	3347	emit_byte(0x2C);
duke@435	3348	emit_byte(0xC0 | encode);
duke@435	3349	}
duke@435	3350
duke@435	3351	void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
duke@435	3352	emit_byte(0xF2);
duke@435	3353	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3354	emit_byte(0x0F);
duke@435	3355	emit_byte(0x2C);
duke@435	3356	emit_byte(0xC0 | encode);
duke@435	3357	}
duke@435	3358
duke@435	3359	void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
duke@435	3360	emit_byte(0xF2);
duke@435	3361	int encode = prefixq_and_encode(dst->encoding(), src->encoding());
duke@435	3362	emit_byte(0x0F);
duke@435	3363	emit_byte(0x2C);
duke@435	3364	emit_byte(0xC0 | encode);
duke@435	3365	}
duke@435	3366
duke@435	3367	void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
duke@435	3368	emit_byte(0xF3);
duke@435	3369	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3370	emit_byte(0x0F);
duke@435	3371	emit_byte(0x5A);
duke@435	3372	emit_byte(0xC0 | encode);
duke@435	3373	}
duke@435	3374
duke@435	3375	void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
duke@435	3376	emit_byte(0xF2);
duke@435	3377	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3378	emit_byte(0x0F);
duke@435	3379	emit_byte(0x5A);
duke@435	3380	emit_byte(0xC0 | encode);
duke@435	3381	}
duke@435	3382
duke@435	3383	void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
duke@435	3384	emit_byte(0x66);
duke@435	3385	int encode = prefix_and_encode(dst->encoding(), src->encoding());
duke@435	3386	emit_byte(0x0F);
duke@435	3387	emit_byte(0x60);
duke@435	3388	emit_byte(0xC0 | encode);
duke@435	3389	}
duke@435	3390
duke@435	3391	// Implementation of MacroAssembler
duke@435	3392
duke@435	3393	// On 32 bit it returns a vanilla displacement on 64 bit is a rip relative displacement
duke@435	3394	Address MacroAssembler::as_Address(AddressLiteral adr) {
duke@435	3395	assert(!adr.is_lval(), "must be rval");
duke@435	3396	assert(reachable(adr), "must be");
duke@435	3397	return Address((int)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
duke@435	3398	}
duke@435	3399
duke@435	3400	Address MacroAssembler::as_Address(ArrayAddress adr) {
duke@435	3401	#ifdef _LP64
duke@435	3402	AddressLiteral base = adr.base();
duke@435	3403	lea(rscratch1, base);
duke@435	3404	Address index = adr.index();
duke@435	3405	assert(index._disp == 0, "must not have disp"); // maybe it can?
duke@435	3406	Address array(rscratch1, index._index, index._scale, index._disp);
duke@435	3407	return array;
duke@435	3408	#else
duke@435	3409	return Address::make_array(adr);
duke@435	3410	#endif // _LP64
duke@435	3411
duke@435	3412	}
duke@435	3413
duke@435	3414	void MacroAssembler::fat_nop() {
duke@435	3415	// A 5 byte nop that is safe for patching (see patch_verified_entry)
duke@435	3416	// Recommened sequence from 'Software Optimization Guide for the AMD
duke@435	3417	// Hammer Processor'
duke@435	3418	emit_byte(0x66);
duke@435	3419	emit_byte(0x66);
duke@435	3420	emit_byte(0x90);
duke@435	3421	emit_byte(0x66);
duke@435	3422	emit_byte(0x90);
duke@435	3423	}
duke@435	3424
duke@435	3425	static Assembler::Condition reverse[] = {
duke@435	3426	Assembler::noOverflow /* overflow = 0x0 */ ,
duke@435	3427	Assembler::overflow /* noOverflow = 0x1 */ ,
duke@435	3428	Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
duke@435	3429	Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
duke@435	3430	Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
duke@435	3431	Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ ,
duke@435	3432	Assembler::above /* belowEqual = 0x6 */ ,
duke@435	3433	Assembler::belowEqual /* above = 0x7 */ ,
duke@435	3434	Assembler::positive /* negative = 0x8 */ ,
duke@435	3435	Assembler::negative /* positive = 0x9 */ ,
duke@435	3436	Assembler::noParity /* parity = 0xa */ ,
duke@435	3437	Assembler::parity /* noParity = 0xb */ ,
duke@435	3438	Assembler::greaterEqual /* less = 0xc */ ,
duke@435	3439	Assembler::less /* greaterEqual = 0xd */ ,
duke@435	3440	Assembler::greater /* lessEqual = 0xe */ ,
duke@435	3441	Assembler::lessEqual /* greater = 0xf, */
duke@435	3442
duke@435	3443	};
duke@435	3444
duke@435	3445	// 32bit can do a case table jump in one instruction but we no longer allow the base
duke@435	3446	// to be installed in the Address class
duke@435	3447	void MacroAssembler::jump(ArrayAddress entry) {
duke@435	3448	#ifdef _LP64
duke@435	3449	lea(rscratch1, entry.base());
duke@435	3450	Address dispatch = entry.index();
duke@435	3451	assert(dispatch._base == noreg, "must be");
duke@435	3452	dispatch._base = rscratch1;
duke@435	3453	jmp(dispatch);
duke@435	3454	#else
duke@435	3455	jmp(as_Address(entry));
duke@435	3456	#endif // _LP64
duke@435	3457	}
duke@435	3458
duke@435	3459	void MacroAssembler::jump(AddressLiteral dst) {
duke@435	3460	if (reachable(dst)) {
duke@435	3461	jmp_literal(dst.target(), dst.rspec());
duke@435	3462	} else {
duke@435	3463	lea(rscratch1, dst);
duke@435	3464	jmp(rscratch1);
duke@435	3465	}
duke@435	3466	}
duke@435	3467
duke@435	3468	void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
duke@435	3469	if (reachable(dst)) {
duke@435	3470	InstructionMark im(this);
duke@435	3471	relocate(dst.reloc());
duke@435	3472	const int short_size = 2;
duke@435	3473	const int long_size = 6;
duke@435	3474	int offs = (intptr_t)dst.target() - ((intptr_t)_code_pos);
duke@435	3475	if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
duke@435	3476	// 0111 tttn #8-bit disp
duke@435	3477	emit_byte(0x70 | cc);
duke@435	3478	emit_byte((offs - short_size) & 0xFF);
duke@435	3479	} else {
duke@435	3480	// 0000 1111 1000 tttn #32-bit disp
duke@435	3481	emit_byte(0x0F);
duke@435	3482	emit_byte(0x80 | cc);
duke@435	3483	emit_long(offs - long_size);
duke@435	3484	}
duke@435	3485	} else {
duke@435	3486	#ifdef ASSERT
duke@435	3487	warning("reversing conditional branch");
duke@435	3488	#endif /* ASSERT */
duke@435	3489	Label skip;
duke@435	3490	jccb(reverse[cc], skip);
duke@435	3491	lea(rscratch1, dst);
duke@435	3492	Assembler::jmp(rscratch1);
duke@435	3493	bind(skip);
duke@435	3494	}
duke@435	3495	}
duke@435	3496
duke@435	3497	// Wouldn't need if AddressLiteral version had new name
duke@435	3498	void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
duke@435	3499	Assembler::call(L, rtype);
duke@435	3500	}
duke@435	3501
duke@435	3502	// Wouldn't need if AddressLiteral version had new name
duke@435	3503	void MacroAssembler::call(Register entry) {
duke@435	3504	Assembler::call(entry);
duke@435	3505	}
duke@435	3506
duke@435	3507	void MacroAssembler::call(AddressLiteral entry) {
duke@435	3508	if (reachable(entry)) {
duke@435	3509	Assembler::call_literal(entry.target(), entry.rspec());
duke@435	3510	} else {
duke@435	3511	lea(rscratch1, entry);
duke@435	3512	Assembler::call(rscratch1);
duke@435	3513	}
duke@435	3514	}
duke@435	3515
duke@435	3516	void MacroAssembler::cmp8(AddressLiteral src1, int8_t src2) {
duke@435	3517	if (reachable(src1)) {
duke@435	3518	cmpb(as_Address(src1), src2);
duke@435	3519	} else {
duke@435	3520	lea(rscratch1, src1);
duke@435	3521	cmpb(Address(rscratch1, 0), src2);
duke@435	3522	}
duke@435	3523	}
duke@435	3524
duke@435	3525	void MacroAssembler::cmp32(AddressLiteral src1, int32_t src2) {
duke@435	3526	if (reachable(src1)) {
duke@435	3527	cmpl(as_Address(src1), src2);
duke@435	3528	} else {
duke@435	3529	lea(rscratch1, src1);
duke@435	3530	cmpl(Address(rscratch1, 0), src2);
duke@435	3531	}
duke@435	3532	}
duke@435	3533
duke@435	3534	void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
duke@435	3535	if (reachable(src2)) {
duke@435	3536	cmpl(src1, as_Address(src2));
duke@435	3537	} else {
duke@435	3538	lea(rscratch1, src2);
duke@435	3539	cmpl(src1, Address(rscratch1, 0));
duke@435	3540	}
duke@435	3541	}
duke@435	3542
duke@435	3543	void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
duke@435	3544	#ifdef _LP64
duke@435	3545	if (src2.is_lval()) {
duke@435	3546	movptr(rscratch1, src2);
duke@435	3547	Assembler::cmpq(src1, rscratch1);
duke@435	3548	} else if (reachable(src2)) {
duke@435	3549	cmpq(src1, as_Address(src2));
duke@435	3550	} else {
duke@435	3551	lea(rscratch1, src2);
duke@435	3552	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3553	}
duke@435	3554	#else
duke@435	3555	if (src2.is_lval()) {
duke@435	3556	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3557	} else {
duke@435	3558	cmpl(src1, as_Address(src2));
duke@435	3559	}
duke@435	3560	#endif // _LP64
duke@435	3561	}
duke@435	3562
duke@435	3563	void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
duke@435	3564	assert(src2.is_lval(), "not a mem-mem compare");
duke@435	3565	#ifdef _LP64
duke@435	3566	// moves src2's literal address
duke@435	3567	movptr(rscratch1, src2);
duke@435	3568	Assembler::cmpq(src1, rscratch1);
duke@435	3569	#else
duke@435	3570	cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
duke@435	3571	#endif // _LP64
duke@435	3572	}
duke@435	3573
duke@435	3574	void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
duke@435	3575	assert(!src2.is_lval(), "should use cmpptr");
duke@435	3576
duke@435	3577	if (reachable(src2)) {
duke@435	3578	#ifdef _LP64
duke@435	3579	cmpq(src1, as_Address(src2));
duke@435	3580	#else
duke@435	3581	ShouldNotReachHere();
duke@435	3582	#endif // _LP64
duke@435	3583	} else {
duke@435	3584	lea(rscratch1, src2);
duke@435	3585	Assembler::cmpq(src1, Address(rscratch1, 0));
duke@435	3586	}
duke@435	3587	}
duke@435	3588
duke@435	3589	void MacroAssembler::cmpxchgptr(Register reg, AddressLiteral adr) {
duke@435	3590	if (reachable(adr)) {
duke@435	3591	#ifdef _LP64
duke@435	3592	cmpxchgq(reg, as_Address(adr));
duke@435	3593	#else
duke@435	3594	cmpxchgl(reg, as_Address(adr));
duke@435	3595	#endif // _LP64
duke@435	3596	} else {
duke@435	3597	lea(rscratch1, adr);
duke@435	3598	cmpxchgq(reg, Address(rscratch1, 0));
duke@435	3599	}
duke@435	3600	}
duke@435	3601
duke@435	3602	void MacroAssembler::incrementl(AddressLiteral dst) {
duke@435	3603	if (reachable(dst)) {
duke@435	3604	incrementl(as_Address(dst));
duke@435	3605	} else {
duke@435	3606	lea(rscratch1, dst);
duke@435	3607	incrementl(Address(rscratch1, 0));
duke@435	3608	}
duke@435	3609	}
duke@435	3610
duke@435	3611	void MacroAssembler::incrementl(ArrayAddress dst) {
duke@435	3612	incrementl(as_Address(dst));
duke@435	3613	}
duke@435	3614
duke@435	3615	void MacroAssembler::lea(Register dst, Address src) {
duke@435	3616	#ifdef _LP64
duke@435	3617	leaq(dst, src);
duke@435	3618	#else
duke@435	3619	leal(dst, src);
duke@435	3620	#endif // _LP64
duke@435	3621	}
duke@435	3622
duke@435	3623	void MacroAssembler::lea(Register dst, AddressLiteral src) {
duke@435	3624	#ifdef _LP64
duke@435	3625	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3626	#else
duke@435	3627	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3628	#endif // _LP64
duke@435	3629	}
duke@435	3630
duke@435	3631	void MacroAssembler::mov32(AddressLiteral dst, Register src) {
duke@435	3632	if (reachable(dst)) {
duke@435	3633	movl(as_Address(dst), src);
duke@435	3634	} else {
duke@435	3635	lea(rscratch1, dst);
duke@435	3636	movl(Address(rscratch1, 0), src);
duke@435	3637	}
duke@435	3638	}
duke@435	3639
duke@435	3640	void MacroAssembler::mov32(Register dst, AddressLiteral src) {
duke@435	3641	if (reachable(src)) {
duke@435	3642	movl(dst, as_Address(src));
duke@435	3643	} else {
duke@435	3644	lea(rscratch1, src);
duke@435	3645	movl(dst, Address(rscratch1, 0));
duke@435	3646	}
duke@435	3647	}
duke@435	3648
duke@435	3649	void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
duke@435	3650	if (reachable(src)) {
duke@435	3651	if (UseXmmLoadAndClearUpper) {
duke@435	3652	movsd (dst, as_Address(src));
duke@435	3653	} else {
duke@435	3654	movlpd(dst, as_Address(src));
duke@435	3655	}
duke@435	3656	} else {
duke@435	3657	lea(rscratch1, src);
duke@435	3658	if (UseXmmLoadAndClearUpper) {
duke@435	3659	movsd (dst, Address(rscratch1, 0));
duke@435	3660	} else {
duke@435	3661	movlpd(dst, Address(rscratch1, 0));
duke@435	3662	}
duke@435	3663	}
duke@435	3664	}
duke@435	3665
duke@435	3666	void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
duke@435	3667	if (reachable(src)) {
duke@435	3668	movss(dst, as_Address(src));
duke@435	3669	} else {
duke@435	3670	lea(rscratch1, src);
duke@435	3671	movss(dst, Address(rscratch1, 0));
duke@435	3672	}
duke@435	3673	}
duke@435	3674
duke@435	3675	void MacroAssembler::movoop(Register dst, jobject obj) {
duke@435	3676	mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3677	}
duke@435	3678
duke@435	3679	void MacroAssembler::movoop(Address dst, jobject obj) {
duke@435	3680	mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3681	movq(dst, rscratch1);
duke@435	3682	}
duke@435	3683
duke@435	3684	void MacroAssembler::movptr(Register dst, AddressLiteral src) {
duke@435	3685	#ifdef _LP64
duke@435	3686	if (src.is_lval()) {
duke@435	3687	mov_literal64(dst, (intptr_t)src.target(), src.rspec());
duke@435	3688	} else {
duke@435	3689	if (reachable(src)) {
duke@435	3690	movq(dst, as_Address(src));
duke@435	3691	} else {
duke@435	3692	lea(rscratch1, src);
duke@435	3693	movq(dst, Address(rscratch1,0));
duke@435	3694	}
duke@435	3695	}
duke@435	3696	#else
duke@435	3697	if (src.is_lval()) {
duke@435	3698	mov_literal32(dst, (intptr_t)src.target(), src.rspec());
duke@435	3699	} else {
duke@435	3700	movl(dst, as_Address(src));
duke@435	3701	}
duke@435	3702	#endif // LP64
duke@435	3703	}
duke@435	3704
duke@435	3705	void MacroAssembler::movptr(ArrayAddress dst, Register src) {
duke@435	3706	#ifdef _LP64
duke@435	3707	movq(as_Address(dst), src);
duke@435	3708	#else
duke@435	3709	movl(as_Address(dst), src);
duke@435	3710	#endif // _LP64
duke@435	3711	}
duke@435	3712
duke@435	3713	void MacroAssembler::pushoop(jobject obj) {
duke@435	3714	#ifdef _LP64
duke@435	3715	movoop(rscratch1, obj);
duke@435	3716	pushq(rscratch1);
duke@435	3717	#else
duke@435	3718	push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
duke@435	3719	#endif // _LP64
duke@435	3720	}
duke@435	3721
duke@435	3722	void MacroAssembler::pushptr(AddressLiteral src) {
duke@435	3723	#ifdef _LP64
duke@435	3724	lea(rscratch1, src);
duke@435	3725	if (src.is_lval()) {
duke@435	3726	pushq(rscratch1);
duke@435	3727	} else {
duke@435	3728	pushq(Address(rscratch1, 0));
duke@435	3729	}
duke@435	3730	#else
duke@435	3731	if (src.is_lval()) {
duke@435	3732	push_literal((int32_t)src.target(), src.rspec());
duke@435	3733	else {
duke@435	3734	pushl(as_Address(src));
duke@435	3735	}
duke@435	3736	#endif // _LP64
duke@435	3737	}
duke@435	3738
duke@435	3739	void MacroAssembler::ldmxcsr(AddressLiteral src) {
duke@435	3740	if (reachable(src)) {
duke@435	3741	Assembler::ldmxcsr(as_Address(src));
duke@435	3742	} else {
duke@435	3743	lea(rscratch1, src);
duke@435	3744	Assembler::ldmxcsr(Address(rscratch1, 0));
duke@435	3745	}
duke@435	3746	}
duke@435	3747
duke@435	3748	void MacroAssembler::movlpd(XMMRegister dst, AddressLiteral src) {
duke@435	3749	if (reachable(src)) {
duke@435	3750	movlpd(dst, as_Address(src));
duke@435	3751	} else {
duke@435	3752	lea(rscratch1, src);
duke@435	3753	movlpd(dst, Address(rscratch1, 0));
duke@435	3754	}
duke@435	3755	}
duke@435	3756
duke@435	3757	void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
duke@435	3758	if (reachable(src)) {
duke@435	3759	movss(dst, as_Address(src));
duke@435	3760	} else {
duke@435	3761	lea(rscratch1, src);
duke@435	3762	movss(dst, Address(rscratch1, 0));
duke@435	3763	}
duke@435	3764	}
duke@435	3765	void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
duke@435	3766	if (reachable(src)) {
duke@435	3767	xorpd(dst, as_Address(src));
duke@435	3768	} else {
duke@435	3769	lea(rscratch1, src);
duke@435	3770	xorpd(dst, Address(rscratch1, 0));
duke@435	3771	}
duke@435	3772	}
duke@435	3773
duke@435	3774	void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
duke@435	3775	if (reachable(src)) {
duke@435	3776	xorps(dst, as_Address(src));
duke@435	3777	} else {
duke@435	3778	lea(rscratch1, src);
duke@435	3779	xorps(dst, Address(rscratch1, 0));
duke@435	3780	}
duke@435	3781	}
duke@435	3782
duke@435	3783	void MacroAssembler::null_check(Register reg, int offset) {
duke@435	3784	if (needs_explicit_null_check(offset)) {
duke@435	3785	// provoke OS NULL exception if reg = NULL by
duke@435	3786	// accessing M[reg] w/o changing any (non-CC) registers
duke@435	3787	cmpq(rax, Address(reg, 0));
duke@435	3788	// Note: should probably use testl(rax, Address(reg, 0));
duke@435	3789	// may be shorter code (however, this version of
duke@435	3790	// testl needs to be implemented first)
duke@435	3791	} else {
duke@435	3792	// nothing to do, (later) access of M[reg + offset]
duke@435	3793	// will provoke OS NULL exception if reg = NULL
duke@435	3794	}
duke@435	3795	}
duke@435	3796
duke@435	3797	int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
duke@435	3798	int off = offset();
duke@435	3799	movzbl(dst, src);
duke@435	3800	return off;
duke@435	3801	}
duke@435	3802
duke@435	3803	int MacroAssembler::load_unsigned_word(Register dst, Address src) {
duke@435	3804	int off = offset();
duke@435	3805	movzwl(dst, src);
duke@435	3806	return off;
duke@435	3807	}
duke@435	3808
duke@435	3809	int MacroAssembler::load_signed_byte(Register dst, Address src) {
duke@435	3810	int off = offset();
duke@435	3811	movsbl(dst, src);
duke@435	3812	return off;
duke@435	3813	}
duke@435	3814
duke@435	3815	int MacroAssembler::load_signed_word(Register dst, Address src) {
duke@435	3816	int off = offset();
duke@435	3817	movswl(dst, src);
duke@435	3818	return off;
duke@435	3819	}
duke@435	3820
duke@435	3821	void MacroAssembler::incrementl(Register reg, int value) {
duke@435	3822	if (value == min_jint) { addl(reg, value); return; }
duke@435	3823	if (value < 0) { decrementl(reg, -value); return; }
duke@435	3824	if (value == 0) { ; return; }
duke@435	3825	if (value == 1 && UseIncDec) { incl(reg) ; return; }
duke@435	3826	/* else */ { addl(reg, value) ; return; }
duke@435	3827	}
duke@435	3828
duke@435	3829	void MacroAssembler::decrementl(Register reg, int value) {
duke@435	3830	if (value == min_jint) { subl(reg, value); return; }
duke@435	3831	if (value < 0) { incrementl(reg, -value); return; }
duke@435	3832	if (value == 0) { ; return; }
duke@435	3833	if (value == 1 && UseIncDec) { decl(reg) ; return; }
duke@435	3834	/* else */ { subl(reg, value) ; return; }
duke@435	3835	}
duke@435	3836
duke@435	3837	void MacroAssembler::incrementq(Register reg, int value) {
duke@435	3838	if (value == min_jint) { addq(reg, value); return; }
duke@435	3839	if (value < 0) { decrementq(reg, -value); return; }
duke@435	3840	if (value == 0) { ; return; }
duke@435	3841	if (value == 1 && UseIncDec) { incq(reg) ; return; }
duke@435	3842	/* else */ { addq(reg, value) ; return; }
duke@435	3843	}
duke@435	3844
duke@435	3845	void MacroAssembler::decrementq(Register reg, int value) {
duke@435	3846	if (value == min_jint) { subq(reg, value); return; }
duke@435	3847	if (value < 0) { incrementq(reg, -value); return; }
duke@435	3848	if (value == 0) { ; return; }
duke@435	3849	if (value == 1 && UseIncDec) { decq(reg) ; return; }
duke@435	3850	/* else */ { subq(reg, value) ; return; }
duke@435	3851	}
duke@435	3852
duke@435	3853	void MacroAssembler::incrementl(Address dst, int value) {
duke@435	3854	if (value == min_jint) { addl(dst, value); return; }
duke@435	3855	if (value < 0) { decrementl(dst, -value); return; }
duke@435	3856	if (value == 0) { ; return; }
duke@435	3857	if (value == 1 && UseIncDec) { incl(dst) ; return; }
duke@435	3858	/* else */ { addl(dst, value) ; return; }
duke@435	3859	}
duke@435	3860
duke@435	3861	void MacroAssembler::decrementl(Address dst, int value) {
duke@435	3862	if (value == min_jint) { subl(dst, value); return; }
duke@435	3863	if (value < 0) { incrementl(dst, -value); return; }
duke@435	3864	if (value == 0) { ; return; }
duke@435	3865	if (value == 1 && UseIncDec) { decl(dst) ; return; }
duke@435	3866	/* else */ { subl(dst, value) ; return; }
duke@435	3867	}
duke@435	3868
duke@435	3869	void MacroAssembler::incrementq(Address dst, int value) {
duke@435	3870	if (value == min_jint) { addq(dst, value); return; }
duke@435	3871	if (value < 0) { decrementq(dst, -value); return; }
duke@435	3872	if (value == 0) { ; return; }
duke@435	3873	if (value == 1 && UseIncDec) { incq(dst) ; return; }
duke@435	3874	/* else */ { addq(dst, value) ; return; }
duke@435	3875	}
duke@435	3876
duke@435	3877	void MacroAssembler::decrementq(Address dst, int value) {
duke@435	3878	if (value == min_jint) { subq(dst, value); return; }
duke@435	3879	if (value < 0) { incrementq(dst, -value); return; }
duke@435	3880	if (value == 0) { ; return; }
duke@435	3881	if (value == 1 && UseIncDec) { decq(dst) ; return; }
duke@435	3882	/* else */ { subq(dst, value) ; return; }
duke@435	3883	}
duke@435	3884
duke@435	3885	void MacroAssembler::align(int modulus) {
duke@435	3886	if (offset() % modulus != 0) {
duke@435	3887	nop(modulus - (offset() % modulus));
duke@435	3888	}
duke@435	3889	}
duke@435	3890
duke@435	3891	void MacroAssembler::enter() {
duke@435	3892	pushq(rbp);
duke@435	3893	movq(rbp, rsp);
duke@435	3894	}
duke@435	3895
duke@435	3896	void MacroAssembler::leave() {
duke@435	3897	emit_byte(0xC9); // LEAVE
duke@435	3898	}
duke@435	3899
duke@435	3900	// C++ bool manipulation
duke@435	3901
duke@435	3902	void MacroAssembler::movbool(Register dst, Address src) {
duke@435	3903	if(sizeof(bool) == 1)
duke@435	3904	movb(dst, src);
duke@435	3905	else if(sizeof(bool) == 2)
duke@435	3906	movw(dst, src);
duke@435	3907	else if(sizeof(bool) == 4)
duke@435	3908	movl(dst, src);
duke@435	3909	else {
duke@435	3910	// unsupported
duke@435	3911	ShouldNotReachHere();
duke@435	3912	}
duke@435	3913	}
duke@435	3914
duke@435	3915	void MacroAssembler::movbool(Address dst, bool boolconst) {
duke@435	3916	if(sizeof(bool) == 1)
duke@435	3917	movb(dst, (int) boolconst);
duke@435	3918	else if(sizeof(bool) == 2)
duke@435	3919	movw(dst, (int) boolconst);
duke@435	3920	else if(sizeof(bool) == 4)
duke@435	3921	movl(dst, (int) boolconst);
duke@435	3922	else {
duke@435	3923	// unsupported
duke@435	3924	ShouldNotReachHere();
duke@435	3925	}
duke@435	3926	}
duke@435	3927
duke@435	3928	void MacroAssembler::movbool(Address dst, Register src) {
duke@435	3929	if(sizeof(bool) == 1)
duke@435	3930	movb(dst, src);
duke@435	3931	else if(sizeof(bool) == 2)
duke@435	3932	movw(dst, src);
duke@435	3933	else if(sizeof(bool) == 4)
duke@435	3934	movl(dst, src);
duke@435	3935	else {
duke@435	3936	// unsupported
duke@435	3937	ShouldNotReachHere();
duke@435	3938	}
duke@435	3939	}
duke@435	3940
duke@435	3941	void MacroAssembler::testbool(Register dst) {
duke@435	3942	if(sizeof(bool) == 1)
duke@435	3943	testb(dst, (int) 0xff);
duke@435	3944	else if(sizeof(bool) == 2) {
duke@435	3945	// need testw impl
duke@435	3946	ShouldNotReachHere();
duke@435	3947	} else if(sizeof(bool) == 4)
duke@435	3948	testl(dst, dst);
duke@435	3949	else {
duke@435	3950	// unsupported
duke@435	3951	ShouldNotReachHere();
duke@435	3952	}
duke@435	3953	}
duke@435	3954
duke@435	3955	void MacroAssembler::set_last_Java_frame(Register last_java_sp,
duke@435	3956	Register last_java_fp,
duke@435	3957	address last_java_pc) {
duke@435	3958	// determine last_java_sp register
duke@435	3959	if (!last_java_sp->is_valid()) {
duke@435	3960	last_java_sp = rsp;
duke@435	3961	}
duke@435	3962
duke@435	3963	// last_java_fp is optional
duke@435	3964	if (last_java_fp->is_valid()) {
duke@435	3965	movq(Address(r15_thread, JavaThread::last_Java_fp_offset()),
duke@435	3966	last_java_fp);
duke@435	3967	}
duke@435	3968
duke@435	3969	// last_java_pc is optional
duke@435	3970	if (last_java_pc != NULL) {
duke@435	3971	Address java_pc(r15_thread,
duke@435	3972	JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
duke@435	3973	lea(rscratch1, InternalAddress(last_java_pc));
duke@435	3974	movq(java_pc, rscratch1);
duke@435	3975	}
duke@435	3976
duke@435	3977	movq(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
duke@435	3978	}
duke@435	3979
duke@435	3980	void MacroAssembler::reset_last_Java_frame(bool clear_fp,
duke@435	3981	bool clear_pc) {
duke@435	3982	// we must set sp to zero to clear frame
duke@435	3983	movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
duke@435	3984	// must clear fp, so that compiled frames are not confused; it is
duke@435	3985	// possible that we need it only for debugging
duke@435	3986	if (clear_fp) {
duke@435	3987	movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
duke@435	3988	}
duke@435	3989
duke@435	3990	if (clear_pc) {
duke@435	3991	movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
duke@435	3992	}
duke@435	3993	}
duke@435	3994
duke@435	3995
duke@435	3996	// Implementation of call_VM versions
duke@435	3997
duke@435	3998	void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
duke@435	3999	Label L, E;
duke@435	4000
duke@435	4001	#ifdef _WIN64
duke@435	4002	// Windows always allocates space for it's register args
duke@435	4003	assert(num_args <= 4, "only register arguments supported");
duke@435	4004	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4005	#endif
duke@435	4006
duke@435	4007	// Align stack if necessary
duke@435	4008	testl(rsp, 15);
duke@435	4009	jcc(Assembler::zero, L);
duke@435	4010
duke@435	4011	subq(rsp, 8);
duke@435	4012	{
duke@435	4013	call(RuntimeAddress(entry_point));
duke@435	4014	}
duke@435	4015	addq(rsp, 8);
duke@435	4016	jmp(E);
duke@435	4017
duke@435	4018	bind(L);
duke@435	4019	{
duke@435	4020	call(RuntimeAddress(entry_point));
duke@435	4021	}
duke@435	4022
duke@435	4023	bind(E);
duke@435	4024
duke@435	4025	#ifdef _WIN64
duke@435	4026	// restore stack pointer
duke@435	4027	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4028	#endif
duke@435	4029
duke@435	4030	}
duke@435	4031
duke@435	4032
duke@435	4033	void MacroAssembler::call_VM_base(Register oop_result,
duke@435	4034	Register java_thread,
duke@435	4035	Register last_java_sp,
duke@435	4036	address entry_point,
duke@435	4037	int num_args,
duke@435	4038	bool check_exceptions) {
duke@435	4039	// determine last_java_sp register
duke@435	4040	if (!last_java_sp->is_valid()) {
duke@435	4041	last_java_sp = rsp;
duke@435	4042	}
duke@435	4043
duke@435	4044	// debugging support
duke@435	4045	assert(num_args >= 0, "cannot have negative number of arguments");
duke@435	4046	assert(r15_thread != oop_result,
duke@435	4047	"cannot use the same register for java_thread & oop_result");
duke@435	4048	assert(r15_thread != last_java_sp,
duke@435	4049	"cannot use the same register for java_thread & last_java_sp");
duke@435	4050
duke@435	4051	// set last Java frame before call
duke@435	4052
duke@435	4053	// This sets last_Java_fp which is only needed from interpreted frames
duke@435	4054	// and should really be done only from the interp_masm version before
duke@435	4055	// calling the underlying call_VM. That doesn't happen yet so we set
duke@435	4056	// last_Java_fp here even though some callers don't need it and
duke@435	4057	// also clear it below.
duke@435	4058	set_last_Java_frame(last_java_sp, rbp, NULL);
duke@435	4059
duke@435	4060	{
duke@435	4061	Label L, E;
duke@435	4062
duke@435	4063	// Align stack if necessary
duke@435	4064	#ifdef _WIN64
duke@435	4065	assert(num_args <= 4, "only register arguments supported");
duke@435	4066	// Windows always allocates space for it's register args
duke@435	4067	subq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4068	#endif
duke@435	4069	testl(rsp, 15);
duke@435	4070	jcc(Assembler::zero, L);
duke@435	4071
duke@435	4072	subq(rsp, 8);
duke@435	4073	{
duke@435	4074	call(RuntimeAddress(entry_point));
duke@435	4075	}
duke@435	4076	addq(rsp, 8);
duke@435	4077	jmp(E);
duke@435	4078
duke@435	4079
duke@435	4080	bind(L);
duke@435	4081	{
duke@435	4082	call(RuntimeAddress(entry_point));
duke@435	4083	}
duke@435	4084
duke@435	4085	bind(E);
duke@435	4086
duke@435	4087	#ifdef _WIN64
duke@435	4088	// restore stack pointer
duke@435	4089	addq(rsp, frame::arg_reg_save_area_bytes);
duke@435	4090	#endif
duke@435	4091	}
duke@435	4092
duke@435	4093	#ifdef ASSERT
duke@435	4094	pushq(rax);
duke@435	4095	{
duke@435	4096	Label L;
duke@435	4097	get_thread(rax);
duke@435	4098	cmpq(r15_thread, rax);
duke@435	4099	jcc(Assembler::equal, L);
duke@435	4100	stop("MacroAssembler::call_VM_base: register not callee saved?");
duke@435	4101	bind(L);
duke@435	4102	}
duke@435	4103	popq(rax);
duke@435	4104	#endif
duke@435	4105
duke@435	4106	// reset last Java frame
duke@435	4107	// This really shouldn't have to clear fp set note above at the
duke@435	4108	// call to set_last_Java_frame
duke@435	4109	reset_last_Java_frame(true, false);
duke@435	4110
duke@435	4111	check_and_handle_popframe(noreg);
duke@435	4112	check_and_handle_earlyret(noreg);
duke@435	4113
duke@435	4114	if (check_exceptions) {
duke@435	4115	cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL);
duke@435	4116	// This used to conditionally jump to forward_exception however it is
duke@435	4117	// possible if we relocate that the branch will not reach. So we must jump
duke@435	4118	// around so we can always reach
duke@435	4119	Label ok;
duke@435	4120	jcc(Assembler::equal, ok);
duke@435	4121	jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
duke@435	4122	bind(ok);
duke@435	4123	}
duke@435	4124
duke@435	4125	// get oop result if there is one and reset the value in the thread
duke@435	4126	if (oop_result->is_valid()) {
duke@435	4127	movq(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
duke@435	4128	movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
duke@435	4129	verify_oop(oop_result);
duke@435	4130	}
duke@435	4131	}
duke@435	4132
duke@435	4133	void MacroAssembler::check_and_handle_popframe(Register java_thread) {}
duke@435	4134	void MacroAssembler::check_and_handle_earlyret(Register java_thread) {}
duke@435	4135
duke@435	4136	void MacroAssembler::call_VM_helper(Register oop_result,
duke@435	4137	address entry_point,
duke@435	4138	int num_args,
duke@435	4139	bool check_exceptions) {
duke@435	4140	// Java thread becomes first argument of C function
duke@435	4141	movq(c_rarg0, r15_thread);
duke@435	4142
duke@435	4143	// We've pushed one address, correct last_Java_sp
duke@435	4144	leaq(rax, Address(rsp, wordSize));
duke@435	4145
duke@435	4146	call_VM_base(oop_result, noreg, rax, entry_point, num_args,
duke@435	4147	check_exceptions);
duke@435	4148	}
duke@435	4149
duke@435	4150
duke@435	4151	void MacroAssembler::call_VM(Register oop_result,
duke@435	4152	address entry_point,
duke@435	4153	bool check_exceptions) {
duke@435	4154	Label C, E;
duke@435	4155	Assembler::call(C, relocInfo::none);
duke@435	4156	jmp(E);
duke@435	4157
duke@435	4158	bind(C);
duke@435	4159	call_VM_helper(oop_result, entry_point, 0, check_exceptions);
duke@435	4160	ret(0);
duke@435	4161
duke@435	4162	bind(E);
duke@435	4163	}
duke@435	4164
duke@435	4165
duke@435	4166	void MacroAssembler::call_VM(Register oop_result,
duke@435	4167	address entry_point,
duke@435	4168	Register arg_1,
duke@435	4169	bool check_exceptions) {
duke@435	4170	assert(rax != arg_1, "smashed argument");
duke@435	4171	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4172
duke@435	4173	Label C, E;
duke@435	4174	Assembler::call(C, relocInfo::none);
duke@435	4175	jmp(E);
duke@435	4176
duke@435	4177	bind(C);
duke@435	4178	// c_rarg0 is reserved for thread
duke@435	4179	if (c_rarg1 != arg_1) {
duke@435	4180	movq(c_rarg1, arg_1);
duke@435	4181	}
duke@435	4182	call_VM_helper(oop_result, entry_point, 1, check_exceptions);
duke@435	4183	ret(0);
duke@435	4184
duke@435	4185	bind(E);
duke@435	4186	}
duke@435	4187
duke@435	4188	void MacroAssembler::call_VM(Register oop_result,
duke@435	4189	address entry_point,
duke@435	4190	Register arg_1,
duke@435	4191	Register arg_2,
duke@435	4192	bool check_exceptions) {
duke@435	4193	assert(rax != arg_1, "smashed argument");
duke@435	4194	assert(rax != arg_2, "smashed argument");
duke@435	4195	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4196	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4197	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4198	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4199
duke@435	4200	Label C, E;
duke@435	4201	Assembler::call(C, relocInfo::none);
duke@435	4202	jmp(E);
duke@435	4203
duke@435	4204	bind(C);
duke@435	4205	// c_rarg0 is reserved for thread
duke@435	4206	if (c_rarg1 != arg_1) {
duke@435	4207	movq(c_rarg1, arg_1);
duke@435	4208	}
duke@435	4209	if (c_rarg2 != arg_2) {
duke@435	4210	movq(c_rarg2, arg_2);
duke@435	4211	}
duke@435	4212	call_VM_helper(oop_result, entry_point, 2, check_exceptions);
duke@435	4213	ret(0);
duke@435	4214
duke@435	4215	bind(E);
duke@435	4216	}
duke@435	4217
duke@435	4218
duke@435	4219	void MacroAssembler::call_VM(Register oop_result,
duke@435	4220	address entry_point,
duke@435	4221	Register arg_1,
duke@435	4222	Register arg_2,
duke@435	4223	Register arg_3,
duke@435	4224	bool check_exceptions) {
duke@435	4225	assert(rax != arg_1, "smashed argument");
duke@435	4226	assert(rax != arg_2, "smashed argument");
duke@435	4227	assert(rax != arg_3, "smashed argument");
duke@435	4228	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4229	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4230	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4231	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4232	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4233	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4234	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4235	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4236	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4237
duke@435	4238	Label C, E;
duke@435	4239	Assembler::call(C, relocInfo::none);
duke@435	4240	jmp(E);
duke@435	4241
duke@435	4242	bind(C);
duke@435	4243	// c_rarg0 is reserved for thread
duke@435	4244	if (c_rarg1 != arg_1) {
duke@435	4245	movq(c_rarg1, arg_1);
duke@435	4246	}
duke@435	4247	if (c_rarg2 != arg_2) {
duke@435	4248	movq(c_rarg2, arg_2);
duke@435	4249	}
duke@435	4250	if (c_rarg3 != arg_3) {
duke@435	4251	movq(c_rarg3, arg_3);
duke@435	4252	}
duke@435	4253	call_VM_helper(oop_result, entry_point, 3, check_exceptions);
duke@435	4254	ret(0);
duke@435	4255
duke@435	4256	bind(E);
duke@435	4257	}
duke@435	4258
duke@435	4259	void MacroAssembler::call_VM(Register oop_result,
duke@435	4260	Register last_java_sp,
duke@435	4261	address entry_point,
duke@435	4262	int num_args,
duke@435	4263	bool check_exceptions) {
duke@435	4264	call_VM_base(oop_result, noreg, last_java_sp, entry_point, num_args,
duke@435	4265	check_exceptions);
duke@435	4266	}
duke@435	4267
duke@435	4268	void MacroAssembler::call_VM(Register oop_result,
duke@435	4269	Register last_java_sp,
duke@435	4270	address entry_point,
duke@435	4271	Register arg_1,
duke@435	4272	bool check_exceptions) {
duke@435	4273	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4274	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4275	// c_rarg0 is reserved for thread
duke@435	4276	if (c_rarg1 != arg_1) {
duke@435	4277	movq(c_rarg1, arg_1);
duke@435	4278	}
duke@435	4279	call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
duke@435	4280	}
duke@435	4281
duke@435	4282	void MacroAssembler::call_VM(Register oop_result,
duke@435	4283	Register last_java_sp,
duke@435	4284	address entry_point,
duke@435	4285	Register arg_1,
duke@435	4286	Register arg_2,
duke@435	4287	bool check_exceptions) {
duke@435	4288	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4289	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4290	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4291	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4292	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4293	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4294	// c_rarg0 is reserved for thread
duke@435	4295	if (c_rarg1 != arg_1) {
duke@435	4296	movq(c_rarg1, arg_1);
duke@435	4297	}
duke@435	4298	if (c_rarg2 != arg_2) {
duke@435	4299	movq(c_rarg2, arg_2);
duke@435	4300	}
duke@435	4301	call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
duke@435	4302	}
duke@435	4303
duke@435	4304
duke@435	4305	void MacroAssembler::call_VM(Register oop_result,
duke@435	4306	Register last_java_sp,
duke@435	4307	address entry_point,
duke@435	4308	Register arg_1,
duke@435	4309	Register arg_2,
duke@435	4310	Register arg_3,
duke@435	4311	bool check_exceptions) {
duke@435	4312	assert(c_rarg0 != arg_1, "smashed argument");
duke@435	4313	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4314	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4315	assert(c_rarg1 != arg_2, "smashed argument");
duke@435	4316	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4317	assert(c_rarg1 != last_java_sp, "smashed argument");
duke@435	4318	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4319	assert(c_rarg2 != arg_3, "smashed argument");
duke@435	4320	assert(c_rarg2 != last_java_sp, "smashed argument");
duke@435	4321	assert(c_rarg3 != arg_1, "smashed argument");
duke@435	4322	assert(c_rarg3 != arg_2, "smashed argument");
duke@435	4323	assert(c_rarg3 != last_java_sp, "smashed argument");
duke@435	4324	// c_rarg0 is reserved for thread
duke@435	4325	if (c_rarg1 != arg_1) {
duke@435	4326	movq(c_rarg1, arg_1);
duke@435	4327	}
duke@435	4328	if (c_rarg2 != arg_2) {
duke@435	4329	movq(c_rarg2, arg_2);
duke@435	4330	}
duke@435	4331	if (c_rarg3 != arg_3) {
duke@435	4332	movq(c_rarg2, arg_3);
duke@435	4333	}
duke@435	4334	call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
duke@435	4335	}
duke@435	4336
duke@435	4337	void MacroAssembler::call_VM_leaf(address entry_point, int num_args) {
duke@435	4338	call_VM_leaf_base(entry_point, num_args);
duke@435	4339	}
duke@435	4340
duke@435	4341	void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
duke@435	4342	if (c_rarg0 != arg_1) {
duke@435	4343	movq(c_rarg0, arg_1);
duke@435	4344	}
duke@435	4345	call_VM_leaf(entry_point, 1);
duke@435	4346	}
duke@435	4347
duke@435	4348	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4349	Register arg_1,
duke@435	4350	Register arg_2) {
duke@435	4351	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4352	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4353	if (c_rarg0 != arg_1) {
duke@435	4354	movq(c_rarg0, arg_1);
duke@435	4355	}
duke@435	4356	if (c_rarg1 != arg_2) {
duke@435	4357	movq(c_rarg1, arg_2);
duke@435	4358	}
duke@435	4359	call_VM_leaf(entry_point, 2);
duke@435	4360	}
duke@435	4361
duke@435	4362	void MacroAssembler::call_VM_leaf(address entry_point,
duke@435	4363	Register arg_1,
duke@435	4364	Register arg_2,
duke@435	4365	Register arg_3) {
duke@435	4366	assert(c_rarg0 != arg_2, "smashed argument");
duke@435	4367	assert(c_rarg0 != arg_3, "smashed argument");
duke@435	4368	assert(c_rarg1 != arg_1, "smashed argument");
duke@435	4369	assert(c_rarg1 != arg_3, "smashed argument");
duke@435	4370	assert(c_rarg2 != arg_1, "smashed argument");
duke@435	4371	assert(c_rarg2 != arg_2, "smashed argument");
duke@435	4372	if (c_rarg0 != arg_1) {
duke@435	4373	movq(c_rarg0, arg_1);
duke@435	4374	}
duke@435	4375	if (c_rarg1 != arg_2) {
duke@435	4376	movq(c_rarg1, arg_2);
duke@435	4377	}
duke@435	4378	if (c_rarg2 != arg_3) {
duke@435	4379	movq(c_rarg2, arg_3);
duke@435	4380	}
duke@435	4381	call_VM_leaf(entry_point, 3);
duke@435	4382	}
duke@435	4383
duke@435	4384
duke@435	4385	// Calls to C land
duke@435	4386	//
duke@435	4387	// When entering C land, the rbp & rsp of the last Java frame have to
duke@435	4388	// be recorded in the (thread-local) JavaThread object. When leaving C
duke@435	4389	// land, the last Java fp has to be reset to 0. This is required to
duke@435	4390	// allow proper stack traversal.
duke@435	4391	void MacroAssembler::store_check(Register obj) {
duke@435	4392	// Does a store check for the oop in register obj. The content of
duke@435	4393	// register obj is destroyed afterwards.
duke@435	4394	store_check_part_1(obj);
duke@435	4395	store_check_part_2(obj);
duke@435	4396	}
duke@435	4397
duke@435	4398	void MacroAssembler::store_check(Register obj, Address dst) {
duke@435	4399	store_check(obj);
duke@435	4400	}
duke@435	4401
duke@435	4402	// split the store check operation so that other instructions can be
duke@435	4403	// scheduled inbetween
duke@435	4404	void MacroAssembler::store_check_part_1(Register obj) {
duke@435	4405	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4406	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4407	shrq(obj, CardTableModRefBS::card_shift);
duke@435	4408	}
duke@435	4409
duke@435	4410	void MacroAssembler::store_check_part_2(Register obj) {
duke@435	4411	BarrierSet* bs = Universe::heap()->barrier_set();
duke@435	4412	assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
duke@435	4413	CardTableModRefBS* ct = (CardTableModRefBS*)bs;
duke@435	4414	assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
duke@435	4415	ExternalAddress cardtable((address)ct->byte_map_base);
duke@435	4416	Address index(noreg, obj, Address::times_1);
duke@435	4417	movb(as_Address(ArrayAddress(cardtable, index)), 0);
duke@435	4418	}
duke@435	4419
duke@435	4420	void MacroAssembler::c2bool(Register x) {
duke@435	4421	// implements x == 0 ? 0 : 1
duke@435	4422	// note: must only look at least-significant byte of x
duke@435	4423	// since C-style booleans are stored in one byte
duke@435	4424	// only! (was bug)
duke@435	4425	andl(x, 0xFF);
duke@435	4426	setb(Assembler::notZero, x);
duke@435	4427	}
duke@435	4428
duke@435	4429	int MacroAssembler::corrected_idivl(Register reg) {
duke@435	4430	// Full implementation of Java idiv and irem; checks for special
duke@435	4431	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4432	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4433	// for implicit exceptions.
duke@435	4434	//
duke@435	4435	// normal case special case
duke@435	4436	//
duke@435	4437	// input : eax: dividend min_int
duke@435	4438	// reg: divisor (may not be eax/edx) -1
duke@435	4439	//
duke@435	4440	// output: eax: quotient (= eax idiv reg) min_int
duke@435	4441	// edx: remainder (= eax irem reg) 0
duke@435	4442	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4443	const int min_int = 0x80000000;
duke@435	4444	Label normal_case, special_case;
duke@435	4445
duke@435	4446	// check for special case
duke@435	4447	cmpl(rax, min_int);
duke@435	4448	jcc(Assembler::notEqual, normal_case);
duke@435	4449	xorl(rdx, rdx); // prepare edx for possible special case (where
duke@435	4450	// remainder = 0)
duke@435	4451	cmpl(reg, -1);
duke@435	4452	jcc(Assembler::equal, special_case);
duke@435	4453
duke@435	4454	// handle normal case
duke@435	4455	bind(normal_case);
duke@435	4456	cdql();
duke@435	4457	int idivl_offset = offset();
duke@435	4458	idivl(reg);
duke@435	4459
duke@435	4460	// normal and special case exit
duke@435	4461	bind(special_case);
duke@435	4462
duke@435	4463	return idivl_offset;
duke@435	4464	}
duke@435	4465
duke@435	4466	int MacroAssembler::corrected_idivq(Register reg) {
duke@435	4467	// Full implementation of Java ldiv and lrem; checks for special
duke@435	4468	// case as described in JVM spec., p.243 & p.271. The function
duke@435	4469	// returns the (pc) offset of the idivl instruction - may be needed
duke@435	4470	// for implicit exceptions.
duke@435	4471	//
duke@435	4472	// normal case special case
duke@435	4473	//
duke@435	4474	// input : rax: dividend min_long
duke@435	4475	// reg: divisor (may not be eax/edx) -1
duke@435	4476	//
duke@435	4477	// output: rax: quotient (= rax idiv reg) min_long
duke@435	4478	// rdx: remainder (= rax irem reg) 0
duke@435	4479	assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
duke@435	4480	static const int64_t min_long = 0x8000000000000000;
duke@435	4481	Label normal_case, special_case;
duke@435	4482
duke@435	4483	// check for special case
duke@435	4484	cmp64(rax, ExternalAddress((address) &min_long));
duke@435	4485	jcc(Assembler::notEqual, normal_case);
duke@435	4486	xorl(rdx, rdx); // prepare rdx for possible special case (where
duke@435	4487	// remainder = 0)
duke@435	4488	cmpq(reg, -1);
duke@435	4489	jcc(Assembler::equal, special_case);
duke@435	4490
duke@435	4491	// handle normal case
duke@435	4492	bind(normal_case);
duke@435	4493	cdqq();
duke@435	4494	int idivq_offset = offset();
duke@435	4495	idivq(reg);
duke@435	4496
duke@435	4497	// normal and special case exit
duke@435	4498	bind(special_case);
duke@435	4499
duke@435	4500	return idivq_offset;
duke@435	4501	}
duke@435	4502
duke@435	4503	void MacroAssembler::push_IU_state() {
duke@435	4504	pushfq(); // Push flags first because pushaq kills them
duke@435	4505	subq(rsp, 8); // Make sure rsp stays 16-byte aligned
duke@435	4506	pushaq();
duke@435	4507	}
duke@435	4508
duke@435	4509	void MacroAssembler::pop_IU_state() {
duke@435	4510	popaq();
duke@435	4511	addq(rsp, 8);
duke@435	4512	popfq();
duke@435	4513	}
duke@435	4514
duke@435	4515	void MacroAssembler::push_FPU_state() {
duke@435	4516	subq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4517	fxsave(Address(rsp, 0));
duke@435	4518	}
duke@435	4519
duke@435	4520	void MacroAssembler::pop_FPU_state() {
duke@435	4521	fxrstor(Address(rsp, 0));
duke@435	4522	addq(rsp, FPUStateSizeInWords * wordSize);
duke@435	4523	}
duke@435	4524
duke@435	4525	// Save Integer and Float state
duke@435	4526	// Warning: Stack must be 16 byte aligned
duke@435	4527	void MacroAssembler::push_CPU_state() {
duke@435	4528	push_IU_state();
duke@435	4529	push_FPU_state();
duke@435	4530	}
duke@435	4531
duke@435	4532	void MacroAssembler::pop_CPU_state() {
duke@435	4533	pop_FPU_state();
duke@435	4534	pop_IU_state();
duke@435	4535	}
duke@435	4536
duke@435	4537	void MacroAssembler::sign_extend_short(Register reg) {
duke@435	4538	movswl(reg, reg);
duke@435	4539	}
duke@435	4540
duke@435	4541	void MacroAssembler::sign_extend_byte(Register reg) {
duke@435	4542	movsbl(reg, reg);
duke@435	4543	}
duke@435	4544
duke@435	4545	void MacroAssembler::division_with_shift(Register reg, int shift_value) {
duke@435	4546	assert (shift_value > 0, "illegal shift value");
duke@435	4547	Label _is_positive;
duke@435	4548	testl (reg, reg);
duke@435	4549	jcc (Assembler::positive, _is_positive);
duke@435	4550	int offset = (1 << shift_value) - 1 ;
duke@435	4551
duke@435	4552	if (offset == 1) {
duke@435	4553	incrementl(reg);
duke@435	4554	} else {
duke@435	4555	addl(reg, offset);
duke@435	4556	}
duke@435	4557
duke@435	4558	bind (_is_positive);
duke@435	4559	sarl(reg, shift_value);
duke@435	4560	}
duke@435	4561
duke@435	4562	void MacroAssembler::round_to_l(Register reg, int modulus) {
duke@435	4563	addl(reg, modulus - 1);
duke@435	4564	andl(reg, -modulus);
duke@435	4565	}
duke@435	4566
duke@435	4567	void MacroAssembler::round_to_q(Register reg, int modulus) {
duke@435	4568	addq(reg, modulus - 1);
duke@435	4569	andq(reg, -modulus);
duke@435	4570	}
duke@435	4571
duke@435	4572	void MacroAssembler::verify_oop(Register reg, const char* s) {
duke@435	4573	if (!VerifyOops) {
duke@435	4574	return;
duke@435	4575	}
duke@435	4576
duke@435	4577	// Pass register number to verify_oop_subroutine
duke@435	4578	char* b = new char[strlen(s) + 50];
duke@435	4579	sprintf(b, "verify_oop: %s: %s", reg->name(), s);
duke@435	4580
duke@435	4581	pushq(rax); // save rax, restored by receiver
duke@435	4582
duke@435	4583	// pass args on stack, only touch rax
duke@435	4584	pushq(reg);
duke@435	4585
duke@435	4586	// avoid using pushptr, as it modifies scratch registers
duke@435	4587	// and our contract is not to modify anything
duke@435	4588	ExternalAddress buffer((address)b);
duke@435	4589	movptr(rax, buffer.addr());
duke@435	4590	pushq(rax);
duke@435	4591
duke@435	4592	// call indirectly to solve generation ordering problem
duke@435	4593	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4594	call(rax); // no alignment requirement
duke@435	4595	// everything popped by receiver
duke@435	4596	}
duke@435	4597
duke@435	4598	void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
duke@435	4599	if (!VerifyOops) return;
duke@435	4600	// Pass register number to verify_oop_subroutine
duke@435	4601	char* b = new char[strlen(s) + 50];
duke@435	4602	sprintf(b, "verify_oop_addr: %s", s);
duke@435	4603	pushq(rax); // save rax
duke@435	4604	movq(addr, rax);
duke@435	4605	pushq(rax); // pass register argument
duke@435	4606
duke@435	4607
duke@435	4608	// avoid using pushptr, as it modifies scratch registers
duke@435	4609	// and our contract is not to modify anything
duke@435	4610	ExternalAddress buffer((address)b);
duke@435	4611	movptr(rax, buffer.addr());
duke@435	4612	pushq(rax);
duke@435	4613
duke@435	4614	// call indirectly to solve generation ordering problem
duke@435	4615	movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
duke@435	4616	call(rax); // no alignment requirement
duke@435	4617	// everything popped by receiver
duke@435	4618	}
duke@435	4619
duke@435	4620
duke@435	4621	void MacroAssembler::stop(const char* msg) {
duke@435	4622	address rip = pc();
duke@435	4623	pushaq(); // get regs on stack
duke@435	4624	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4625	lea(c_rarg1, InternalAddress(rip));
duke@435	4626	movq(c_rarg2, rsp); // pass pointer to regs array
duke@435	4627	andq(rsp, -16); // align stack as required by ABI
duke@435	4628	call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
duke@435	4629	hlt();
duke@435	4630	}
duke@435	4631
duke@435	4632	void MacroAssembler::warn(const char* msg) {
duke@435	4633	pushq(r12);
duke@435	4634	movq(r12, rsp);
duke@435	4635	andq(rsp, -16); // align stack as required by push_CPU_state and call
duke@435	4636
duke@435	4637	push_CPU_state(); // keeps alignment at 16 bytes
duke@435	4638	lea(c_rarg0, ExternalAddress((address) msg));
duke@435	4639	call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
duke@435	4640	pop_CPU_state();
duke@435	4641
duke@435	4642	movq(rsp, r12);
duke@435	4643	popq(r12);
duke@435	4644	}
duke@435	4645
duke@435	4646	void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
duke@435	4647	// In order to get locks to work, we need to fake a in_VM state
duke@435	4648	if (ShowMessageBoxOnError ) {
duke@435	4649	JavaThread* thread = JavaThread::current();
duke@435	4650	JavaThreadState saved_state = thread->thread_state();
duke@435	4651	thread->set_thread_state(_thread_in_vm);
duke@435	4652	ttyLocker ttyl;
duke@435	4653	#ifndef PRODUCT
duke@435	4654	if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
duke@435	4655	BytecodeCounter::print();
duke@435	4656	}
duke@435	4657	#endif
duke@435	4658	// To see where a verify_oop failed, get $ebx+40/X for this frame.
duke@435	4659	// XXX correct this offset for amd64
duke@435	4660	// This is the value of eip which points to where verify_oop will return.
duke@435	4661	if (os::message_box(msg, "Execution stopped, print registers?")) {
duke@435	4662	tty->print_cr("rip = 0x%016lx", pc);
duke@435	4663	tty->print_cr("rax = 0x%016lx", regs[15]);
duke@435	4664	tty->print_cr("rbx = 0x%016lx", regs[12]);
duke@435	4665	tty->print_cr("rcx = 0x%016lx", regs[14]);
duke@435	4666	tty->print_cr("rdx = 0x%016lx", regs[13]);
duke@435	4667	tty->print_cr("rdi = 0x%016lx", regs[8]);
duke@435	4668	tty->print_cr("rsi = 0x%016lx", regs[9]);
duke@435	4669	tty->print_cr("rbp = 0x%016lx", regs[10]);
duke@435	4670	tty->print_cr("rsp = 0x%016lx", regs[11]);
duke@435	4671	tty->print_cr("r8 = 0x%016lx", regs[7]);
duke@435	4672	tty->print_cr("r9 = 0x%016lx", regs[6]);
duke@435	4673	tty->print_cr("r10 = 0x%016lx", regs[5]);
duke@435	4674	tty->print_cr("r11 = 0x%016lx", regs[4]);
duke@435	4675	tty->print_cr("r12 = 0x%016lx", regs[3]);
duke@435	4676	tty->print_cr("r13 = 0x%016lx", regs[2]);
duke@435	4677	tty->print_cr("r14 = 0x%016lx", regs[1]);
duke@435	4678	tty->print_cr("r15 = 0x%016lx", regs[0]);
duke@435	4679	BREAKPOINT;
duke@435	4680	}
duke@435	4681	ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
duke@435	4682	} else {
duke@435	4683	::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
duke@435	4684	msg);
duke@435	4685	}
duke@435	4686	}
duke@435	4687
duke@435	4688	void MacroAssembler::os_breakpoint() {
duke@435	4689	// instead of directly emitting a breakpoint, call os:breakpoint for
duke@435	4690	// better debugability
duke@435	4691	// This shouldn't need alignment, it's an empty function
duke@435	4692	call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
duke@435	4693	}
duke@435	4694
duke@435	4695	// Write serialization page so VM thread can do a pseudo remote membar.
duke@435	4696	// We use the current thread pointer to calculate a thread specific
duke@435	4697	// offset to write to within the page. This minimizes bus traffic
duke@435	4698	// due to cache line collision.
duke@435	4699	void MacroAssembler::serialize_memory(Register thread,
duke@435	4700	Register tmp) {
duke@435	4701
duke@435	4702	movl(tmp, thread);
duke@435	4703	shrl(tmp, os::get_serialize_page_shift_count());
duke@435	4704	andl(tmp, (os::vm_page_size() - sizeof(int)));
duke@435	4705
duke@435	4706	Address index(noreg, tmp, Address::times_1);
duke@435	4707	ExternalAddress page(os::get_memory_serialize_page());
duke@435	4708
duke@435	4709	movptr(ArrayAddress(page, index), tmp);
duke@435	4710	}
duke@435	4711
duke@435	4712	void MacroAssembler::verify_tlab() {
duke@435	4713	#ifdef ASSERT
duke@435	4714	if (UseTLAB) {
duke@435	4715	Label next, ok;
duke@435	4716	Register t1 = rsi;
duke@435	4717
duke@435	4718	pushq(t1);
duke@435	4719
duke@435	4720	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4721	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_start_offset())));
duke@435	4722	jcc(Assembler::aboveEqual, next);
duke@435	4723	stop("assert(top >= start)");
duke@435	4724	should_not_reach_here();
duke@435	4725
duke@435	4726	bind(next);
duke@435	4727	movq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4728	cmpq(t1, Address(r15_thread, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4729	jcc(Assembler::aboveEqual, ok);
duke@435	4730	stop("assert(top <= end)");
duke@435	4731	should_not_reach_here();
duke@435	4732
duke@435	4733	bind(ok);
duke@435	4734
duke@435	4735	popq(t1);
duke@435	4736	}
duke@435	4737	#endif
duke@435	4738	}
duke@435	4739
duke@435	4740	// Defines obj, preserves var_size_in_bytes
duke@435	4741	void MacroAssembler::eden_allocate(Register obj,
duke@435	4742	Register var_size_in_bytes,
duke@435	4743	int con_size_in_bytes,
duke@435	4744	Register t1,
duke@435	4745	Label& slow_case) {
duke@435	4746	assert(obj == rax, "obj must be in rax for cmpxchg");
duke@435	4747	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4748	Register end = t1;
duke@435	4749	Label retry;
duke@435	4750	bind(retry);
duke@435	4751	ExternalAddress heap_top((address) Universe::heap()->top_addr());
duke@435	4752	movptr(obj, heap_top);
duke@435	4753	if (var_size_in_bytes == noreg) {
duke@435	4754	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4755	} else {
duke@435	4756	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4757	}
duke@435	4758	// if end < obj then we wrapped around => object too long => slow case
duke@435	4759	cmpq(end, obj);
duke@435	4760	jcc(Assembler::below, slow_case);
duke@435	4761	cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
duke@435	4762
duke@435	4763	jcc(Assembler::above, slow_case);
duke@435	4764	// Compare obj with the top addr, and if still equal, store the new
duke@435	4765	// top addr in end at the address of the top addr pointer. Sets ZF
duke@435	4766	// if was equal, and clears it otherwise. Use lock prefix for
duke@435	4767	// atomicity on MPs.
duke@435	4768	if (os::is_MP()) {
duke@435	4769	lock();
duke@435	4770	}
duke@435	4771	cmpxchgptr(end, heap_top);
duke@435	4772	// if someone beat us on the allocation, try again, otherwise continue
duke@435	4773	jcc(Assembler::notEqual, retry);
duke@435	4774	}
duke@435	4775
duke@435	4776	// Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
duke@435	4777	void MacroAssembler::tlab_allocate(Register obj,
duke@435	4778	Register var_size_in_bytes,
duke@435	4779	int con_size_in_bytes,
duke@435	4780	Register t1,
duke@435	4781	Register t2,
duke@435	4782	Label& slow_case) {
duke@435	4783	assert_different_registers(obj, t1, t2);
duke@435	4784	assert_different_registers(obj, var_size_in_bytes, t1);
duke@435	4785	Register end = t2;
duke@435	4786
duke@435	4787	verify_tlab();
duke@435	4788
duke@435	4789	movq(obj, Address(r15_thread, JavaThread::tlab_top_offset()));
duke@435	4790	if (var_size_in_bytes == noreg) {
duke@435	4791	leaq(end, Address(obj, con_size_in_bytes));
duke@435	4792	} else {
duke@435	4793	leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
duke@435	4794	}
duke@435	4795	cmpq(end, Address(r15_thread, JavaThread::tlab_end_offset()));
duke@435	4796	jcc(Assembler::above, slow_case);
duke@435	4797
duke@435	4798	// update the tlab top pointer
duke@435	4799	movq(Address(r15_thread, JavaThread::tlab_top_offset()), end);
duke@435	4800
duke@435	4801	// recover var_size_in_bytes if necessary
duke@435	4802	if (var_size_in_bytes == end) {
duke@435	4803	subq(var_size_in_bytes, obj);
duke@435	4804	}
duke@435	4805	verify_tlab();
duke@435	4806	}
duke@435	4807
duke@435	4808	// Preserves rbx and rdx.
duke@435	4809	void MacroAssembler::tlab_refill(Label& retry,
duke@435	4810	Label& try_eden,
duke@435	4811	Label& slow_case) {
duke@435	4812	Register top = rax;
duke@435	4813	Register t1 = rcx;
duke@435	4814	Register t2 = rsi;
duke@435	4815	Register t3 = r10;
duke@435	4816	Register thread_reg = r15_thread;
duke@435	4817	assert_different_registers(top, thread_reg, t1, t2, t3,
duke@435	4818	/* preserve: */ rbx, rdx);
duke@435	4819	Label do_refill, discard_tlab;
duke@435	4820
duke@435	4821	if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
duke@435	4822	// No allocation in the shared eden.
duke@435	4823	jmp(slow_case);
duke@435	4824	}
duke@435	4825
duke@435	4826	movq(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
duke@435	4827	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
duke@435	4828
duke@435	4829	// calculate amount of free space
duke@435	4830	subq(t1, top);
duke@435	4831	shrq(t1, LogHeapWordSize);
duke@435	4832
duke@435	4833	// Retain tlab and allocate object in shared space if
duke@435	4834	// the amount free in the tlab is too large to discard.
duke@435	4835	cmpq(t1, Address(thread_reg, // size_t
duke@435	4836	in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
duke@435	4837	jcc(Assembler::lessEqual, discard_tlab);
duke@435	4838
duke@435	4839	// Retain
duke@435	4840	mov64(t2, ThreadLocalAllocBuffer::refill_waste_limit_increment());
duke@435	4841	addq(Address(thread_reg, // size_t
duke@435	4842	in_bytes(JavaThread::tlab_refill_waste_limit_offset())),
duke@435	4843	t2);
duke@435	4844	if (TLABStats) {
duke@435	4845	// increment number of slow_allocations
duke@435	4846	addl(Address(thread_reg, // unsigned int
duke@435	4847	in_bytes(JavaThread::tlab_slow_allocations_offset())),
duke@435	4848	1);
duke@435	4849	}
duke@435	4850	jmp(try_eden);
duke@435	4851
duke@435	4852	bind(discard_tlab);
duke@435	4853	if (TLABStats) {
duke@435	4854	// increment number of refills
duke@435	4855	addl(Address(thread_reg, // unsigned int
duke@435	4856	in_bytes(JavaThread::tlab_number_of_refills_offset())),
duke@435	4857	1);
duke@435	4858	// accumulate wastage -- t1 is amount free in tlab
duke@435	4859	addl(Address(thread_reg, // unsigned int
duke@435	4860	in_bytes(JavaThread::tlab_fast_refill_waste_offset())),
duke@435	4861	t1);
duke@435	4862	}
duke@435	4863
duke@435	4864	// if tlab is currently allocated (top or end != null) then
duke@435	4865	// fill [top, end + alignment_reserve) with array object
duke@435	4866	testq(top, top);
duke@435	4867	jcc(Assembler::zero, do_refill);
duke@435	4868
duke@435	4869	// set up the mark word
duke@435	4870	mov64(t3, (int64_t) markOopDesc::prototype()->copy_set_hash(0x2));
duke@435	4871	movq(Address(top, oopDesc::mark_offset_in_bytes()), t3);
duke@435	4872	// set the length to the remaining space
duke@435	4873	subq(t1, typeArrayOopDesc::header_size(T_INT));
duke@435	4874	addq(t1, (int)ThreadLocalAllocBuffer::alignment_reserve());
duke@435	4875	shlq(t1, log2_intptr(HeapWordSize / sizeof(jint)));
duke@435	4876	movq(Address(top, arrayOopDesc::length_offset_in_bytes()), t1);
duke@435	4877	// set klass to intArrayKlass
duke@435	4878	movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr()));
duke@435	4879	movq(Address(top, oopDesc::klass_offset_in_bytes()), t1);
duke@435	4880
duke@435	4881	// refill the tlab with an eden allocation
duke@435	4882	bind(do_refill);
duke@435	4883	movq(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4884	shlq(t1, LogHeapWordSize);
duke@435	4885	// add object_size ??
duke@435	4886	eden_allocate(top, t1, 0, t2, slow_case);
duke@435	4887
duke@435	4888	// Check that t1 was preserved in eden_allocate.
duke@435	4889	#ifdef ASSERT
duke@435	4890	if (UseTLAB) {
duke@435	4891	Label ok;
duke@435	4892	Register tsize = rsi;
duke@435	4893	assert_different_registers(tsize, thread_reg, t1);
duke@435	4894	pushq(tsize);
duke@435	4895	movq(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
duke@435	4896	shlq(tsize, LogHeapWordSize);
duke@435	4897	cmpq(t1, tsize);
duke@435	4898	jcc(Assembler::equal, ok);
duke@435	4899	stop("assert(t1 != tlab size)");
duke@435	4900	should_not_reach_here();
duke@435	4901
duke@435	4902	bind(ok);
duke@435	4903	popq(tsize);
duke@435	4904	}
duke@435	4905	#endif
duke@435	4906	movq(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
duke@435	4907	movq(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
duke@435	4908	addq(top, t1);
duke@435	4909	subq(top, (int)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
duke@435	4910	movq(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
duke@435	4911	verify_tlab();
duke@435	4912	jmp(retry);
duke@435	4913	}
duke@435	4914
duke@435	4915
duke@435	4916	int MacroAssembler::biased_locking_enter(Register lock_reg, Register obj_reg, Register swap_reg, Register tmp_reg,
duke@435	4917	bool swap_reg_contains_mark,
duke@435	4918	Label& done, Label* slow_case,
duke@435	4919	BiasedLockingCounters* counters) {
duke@435	4920	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	4921	assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
duke@435	4922	assert(tmp_reg != noreg, "tmp_reg must be supplied");
duke@435	4923	assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
duke@435	4924	assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
duke@435	4925	Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes());
duke@435	4926	Address klass_addr (obj_reg, oopDesc::klass_offset_in_bytes());
duke@435	4927	Address saved_mark_addr(lock_reg, 0);
duke@435	4928
duke@435	4929	if (PrintBiasedLockingStatistics && counters == NULL)
duke@435	4930	counters = BiasedLocking::counters();
duke@435	4931
duke@435	4932	// Biased locking
duke@435	4933	// See whether the lock is currently biased toward our thread and
duke@435	4934	// whether the epoch is still valid
duke@435	4935	// Note that the runtime guarantees sufficient alignment of JavaThread
duke@435	4936	// pointers to allow age to be placed into low bits
duke@435	4937	// First check to see whether biasing is even enabled for this object
duke@435	4938	Label cas_label;
duke@435	4939	int null_check_offset = -1;
duke@435	4940	if (!swap_reg_contains_mark) {
duke@435	4941	null_check_offset = offset();
duke@435	4942	movq(swap_reg, mark_addr);
duke@435	4943	}
duke@435	4944	movq(tmp_reg, swap_reg);
duke@435	4945	andq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	4946	cmpq(tmp_reg, markOopDesc::biased_lock_pattern);
duke@435	4947	jcc(Assembler::notEqual, cas_label);
duke@435	4948	// The bias pattern is present in the object's header. Need to check
duke@435	4949	// whether the bias owner and the epoch are both still current.
duke@435	4950	movq(tmp_reg, klass_addr);
duke@435	4951	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	4952	orq(tmp_reg, r15_thread);
duke@435	4953	xorq(tmp_reg, swap_reg);
duke@435	4954	andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place));
duke@435	4955	if (counters != NULL) {
duke@435	4956	cond_inc32(Assembler::zero,
duke@435	4957	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	4958	}
duke@435	4959	jcc(Assembler::equal, done);
duke@435	4960
duke@435	4961	Label try_revoke_bias;
duke@435	4962	Label try_rebias;
duke@435	4963
duke@435	4964	// At this point we know that the header has the bias pattern and
duke@435	4965	// that we are not the bias owner in the current epoch. We need to
duke@435	4966	// figure out more details about the state of the header in order to
duke@435	4967	// know what operations can be legally performed on the object's
duke@435	4968	// header.
duke@435	4969
duke@435	4970	// If the low three bits in the xor result aren't clear, that means
duke@435	4971	// the prototype header is no longer biased and we have to revoke
duke@435	4972	// the bias on this object.
duke@435	4973	testq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	4974	jcc(Assembler::notZero, try_revoke_bias);
duke@435	4975
duke@435	4976	// Biasing is still enabled for this data type. See whether the
duke@435	4977	// epoch of the current bias is still valid, meaning that the epoch
duke@435	4978	// bits of the mark word are equal to the epoch bits of the
duke@435	4979	// prototype header. (Note that the prototype header's epoch bits
duke@435	4980	// only change at a safepoint.) If not, attempt to rebias the object
duke@435	4981	// toward the current thread. Note that we must be absolutely sure
duke@435	4982	// that the current epoch is invalid in order to do this because
duke@435	4983	// otherwise the manipulations it performs on the mark word are
duke@435	4984	// illegal.
duke@435	4985	testq(tmp_reg, markOopDesc::epoch_mask_in_place);
duke@435	4986	jcc(Assembler::notZero, try_rebias);
duke@435	4987
duke@435	4988	// The epoch of the current bias is still valid but we know nothing
duke@435	4989	// about the owner; it might be set or it might be clear. Try to
duke@435	4990	// acquire the bias of the object using an atomic operation. If this
duke@435	4991	// fails we will go in to the runtime to revoke the object's bias.
duke@435	4992	// Note that we first construct the presumed unbiased header so we
duke@435	4993	// don't accidentally blow away another thread's valid bias.
duke@435	4994	andq(swap_reg,
duke@435	4995	markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
duke@435	4996	movq(tmp_reg, swap_reg);
duke@435	4997	orq(tmp_reg, r15_thread);
duke@435	4998	if (os::is_MP()) {
duke@435	4999	lock();
duke@435	5000	}
duke@435	5001	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5002	// If the biasing toward our thread failed, this means that
duke@435	5003	// another thread succeeded in biasing it toward itself and we
duke@435	5004	// need to revoke that bias. The revocation will occur in the
duke@435	5005	// interpreter runtime in the slow case.
duke@435	5006	if (counters != NULL) {
duke@435	5007	cond_inc32(Assembler::zero,
duke@435	5008	ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
duke@435	5009	}
duke@435	5010	if (slow_case != NULL) {
duke@435	5011	jcc(Assembler::notZero, *slow_case);
duke@435	5012	}
duke@435	5013	jmp(done);
duke@435	5014
duke@435	5015	bind(try_rebias);
duke@435	5016	// At this point we know the epoch has expired, meaning that the
duke@435	5017	// current "bias owner", if any, is actually invalid. Under these
duke@435	5018	// circumstances _only_, we are allowed to use the current header's
duke@435	5019	// value as the comparison value when doing the cas to acquire the
duke@435	5020	// bias in the current epoch. In other words, we allow transfer of
duke@435	5021	// the bias from one thread to another directly in this situation.
duke@435	5022	//
duke@435	5023	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5024	// bits in this situation. Should attempt to preserve them.
duke@435	5025	movq(tmp_reg, klass_addr);
duke@435	5026	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5027	orq(tmp_reg, r15_thread);
duke@435	5028	if (os::is_MP()) {
duke@435	5029	lock();
duke@435	5030	}
duke@435	5031	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5032	// If the biasing toward our thread failed, then another thread
duke@435	5033	// succeeded in biasing it toward itself and we need to revoke that
duke@435	5034	// bias. The revocation will occur in the runtime in the slow case.
duke@435	5035	if (counters != NULL) {
duke@435	5036	cond_inc32(Assembler::zero,
duke@435	5037	ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
duke@435	5038	}
duke@435	5039	if (slow_case != NULL) {
duke@435	5040	jcc(Assembler::notZero, *slow_case);
duke@435	5041	}
duke@435	5042	jmp(done);
duke@435	5043
duke@435	5044	bind(try_revoke_bias);
duke@435	5045	// The prototype mark in the klass doesn't have the bias bit set any
duke@435	5046	// more, indicating that objects of this data type are not supposed
duke@435	5047	// to be biased any more. We are going to try to reset the mark of
duke@435	5048	// this object to the prototype value and fall through to the
duke@435	5049	// CAS-based locking scheme. Note that if our CAS fails, it means
duke@435	5050	// that another thread raced us for the privilege of revoking the
duke@435	5051	// bias of this particular object, so it's okay to continue in the
duke@435	5052	// normal locking code.
duke@435	5053	//
duke@435	5054	// FIXME: due to a lack of registers we currently blow away the age
duke@435	5055	// bits in this situation. Should attempt to preserve them.
duke@435	5056	movq(tmp_reg, klass_addr);
duke@435	5057	movq(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
duke@435	5058	if (os::is_MP()) {
duke@435	5059	lock();
duke@435	5060	}
duke@435	5061	cmpxchgq(tmp_reg, Address(obj_reg, 0));
duke@435	5062	// Fall through to the normal CAS-based lock, because no matter what
duke@435	5063	// the result of the above CAS, some thread must have succeeded in
duke@435	5064	// removing the bias bit from the object's header.
duke@435	5065	if (counters != NULL) {
duke@435	5066	cond_inc32(Assembler::zero,
duke@435	5067	ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
duke@435	5068	}
duke@435	5069
duke@435	5070	bind(cas_label);
duke@435	5071
duke@435	5072	return null_check_offset;
duke@435	5073	}
duke@435	5074
duke@435	5075
duke@435	5076	void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
duke@435	5077	assert(UseBiasedLocking, "why call this otherwise?");
duke@435	5078
duke@435	5079	// Check for biased locking unlock case, which is a no-op
duke@435	5080	// Note: we do not have to check the thread ID for two reasons.
duke@435	5081	// First, the interpreter checks for IllegalMonitorStateException at
duke@435	5082	// a higher level. Second, if the bias was revoked while we held the
duke@435	5083	// lock, the object could not be rebiased toward another thread, so
duke@435	5084	// the bias bit would be clear.
duke@435	5085	movq(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
duke@435	5086	andq(temp_reg, markOopDesc::biased_lock_mask_in_place);
duke@435	5087	cmpq(temp_reg, markOopDesc::biased_lock_pattern);
duke@435	5088	jcc(Assembler::equal, done);
duke@435	5089	}
duke@435	5090
duke@435	5091
duke@435	5092	Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
duke@435	5093	switch (cond) {
duke@435	5094	// Note some conditions are synonyms for others
duke@435	5095	case Assembler::zero: return Assembler::notZero;
duke@435	5096	case Assembler::notZero: return Assembler::zero;
duke@435	5097	case Assembler::less: return Assembler::greaterEqual;
duke@435	5098	case Assembler::lessEqual: return Assembler::greater;
duke@435	5099	case Assembler::greater: return Assembler::lessEqual;
duke@435	5100	case Assembler::greaterEqual: return Assembler::less;
duke@435	5101	case Assembler::below: return Assembler::aboveEqual;
duke@435	5102	case Assembler::belowEqual: return Assembler::above;
duke@435	5103	case Assembler::above: return Assembler::belowEqual;
duke@435	5104	case Assembler::aboveEqual: return Assembler::below;
duke@435	5105	case Assembler::overflow: return Assembler::noOverflow;
duke@435	5106	case Assembler::noOverflow: return Assembler::overflow;
duke@435	5107	case Assembler::negative: return Assembler::positive;
duke@435	5108	case Assembler::positive: return Assembler::negative;
duke@435	5109	case Assembler::parity: return Assembler::noParity;
duke@435	5110	case Assembler::noParity: return Assembler::parity;
duke@435	5111	}
duke@435	5112	ShouldNotReachHere(); return Assembler::overflow;
duke@435	5113	}
duke@435	5114
duke@435	5115
duke@435	5116	void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
duke@435	5117	Condition negated_cond = negate_condition(cond);
duke@435	5118	Label L;
duke@435	5119	jcc(negated_cond, L);
duke@435	5120	atomic_incl(counter_addr);
duke@435	5121	bind(L);
duke@435	5122	}
duke@435	5123
duke@435	5124	void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
duke@435	5125	pushfq();
duke@435	5126	if (os::is_MP())
duke@435	5127	lock();
duke@435	5128	incrementl(counter_addr);
duke@435	5129	popfq();
duke@435	5130	}
duke@435	5131
duke@435	5132	SkipIfEqual::SkipIfEqual(
duke@435	5133	MacroAssembler* masm, const bool* flag_addr, bool value) {
duke@435	5134	_masm = masm;
duke@435	5135	_masm->cmp8(ExternalAddress((address)flag_addr), value);
duke@435	5136	_masm->jcc(Assembler::equal, _label);
duke@435	5137	}
duke@435	5138
duke@435	5139	SkipIfEqual::~SkipIfEqual() {
duke@435	5140	_masm->bind(_label);
duke@435	5141	}
duke@435	5142
duke@435	5143	void MacroAssembler::bang_stack_size(Register size, Register tmp) {
duke@435	5144	movq(tmp, rsp);
duke@435	5145	// Bang stack for total size given plus shadow page size.
duke@435	5146	// Bang one page at a time because large size can bang beyond yellow and
duke@435	5147	// red zones.
duke@435	5148	Label loop;
duke@435	5149	bind(loop);
duke@435	5150	movl(Address(tmp, (-os::vm_page_size())), size );
duke@435	5151	subq(tmp, os::vm_page_size());
duke@435	5152	subl(size, os::vm_page_size());
duke@435	5153	jcc(Assembler::greater, loop);
duke@435	5154
duke@435	5155	// Bang down shadow pages too.
duke@435	5156	// The -1 because we already subtracted 1 page.
duke@435	5157	for (int i = 0; i< StackShadowPages-1; i++) {
duke@435	5158	movq(Address(tmp, (-i*os::vm_page_size())), size );
duke@435	5159	}
duke@435	5160	}