Wed, 19 Dec 2012 15:40:35 -0800
8004835: Improve AES intrinsics on x86
Summary: Enable AES intrinsics on non-AVX cpus, group together aes instructions in crypto stubs.
Reviewed-by: roland, twisti
1 /*
2 * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "asm/assembler.inline.hpp"
28 #include "gc_interface/collectedHeap.inline.hpp"
29 #include "interpreter/interpreter.hpp"
30 #include "memory/cardTableModRefBS.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "prims/methodHandles.hpp"
33 #include "runtime/biasedLocking.hpp"
34 #include "runtime/interfaceSupport.hpp"
35 #include "runtime/objectMonitor.hpp"
36 #include "runtime/os.hpp"
37 #include "runtime/sharedRuntime.hpp"
38 #include "runtime/stubRoutines.hpp"
39 #ifndef SERIALGC
40 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
41 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
42 #include "gc_implementation/g1/heapRegion.hpp"
43 #endif
45 #ifdef PRODUCT
46 #define BLOCK_COMMENT(str) /* nothing */
47 #define STOP(error) stop(error)
48 #else
49 #define BLOCK_COMMENT(str) block_comment(str)
50 #define STOP(error) block_comment(error); stop(error)
51 #endif
53 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
54 // Implementation of AddressLiteral
56 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
57 _is_lval = false;
58 _target = target;
59 switch (rtype) {
60 case relocInfo::oop_type:
61 case relocInfo::metadata_type:
62 // Oops are a special case. Normally they would be their own section
63 // but in cases like icBuffer they are literals in the code stream that
64 // we don't have a section for. We use none so that we get a literal address
65 // which is always patchable.
66 break;
67 case relocInfo::external_word_type:
68 _rspec = external_word_Relocation::spec(target);
69 break;
70 case relocInfo::internal_word_type:
71 _rspec = internal_word_Relocation::spec(target);
72 break;
73 case relocInfo::opt_virtual_call_type:
74 _rspec = opt_virtual_call_Relocation::spec();
75 break;
76 case relocInfo::static_call_type:
77 _rspec = static_call_Relocation::spec();
78 break;
79 case relocInfo::runtime_call_type:
80 _rspec = runtime_call_Relocation::spec();
81 break;
82 case relocInfo::poll_type:
83 case relocInfo::poll_return_type:
84 _rspec = Relocation::spec_simple(rtype);
85 break;
86 case relocInfo::none:
87 break;
88 default:
89 ShouldNotReachHere();
90 break;
91 }
92 }
94 // Implementation of Address
96 #ifdef _LP64
98 Address Address::make_array(ArrayAddress adr) {
99 // Not implementable on 64bit machines
100 // Should have been handled higher up the call chain.
101 ShouldNotReachHere();
102 return Address();
103 }
105 // exceedingly dangerous constructor
106 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
107 _base = noreg;
108 _index = noreg;
109 _scale = no_scale;
110 _disp = disp;
111 switch (rtype) {
112 case relocInfo::external_word_type:
113 _rspec = external_word_Relocation::spec(loc);
114 break;
115 case relocInfo::internal_word_type:
116 _rspec = internal_word_Relocation::spec(loc);
117 break;
118 case relocInfo::runtime_call_type:
119 // HMM
120 _rspec = runtime_call_Relocation::spec();
121 break;
122 case relocInfo::poll_type:
123 case relocInfo::poll_return_type:
124 _rspec = Relocation::spec_simple(rtype);
125 break;
126 case relocInfo::none:
127 break;
128 default:
129 ShouldNotReachHere();
130 }
131 }
132 #else // LP64
134 Address Address::make_array(ArrayAddress adr) {
135 AddressLiteral base = adr.base();
136 Address index = adr.index();
137 assert(index._disp == 0, "must not have disp"); // maybe it can?
138 Address array(index._base, index._index, index._scale, (intptr_t) base.target());
139 array._rspec = base._rspec;
140 return array;
141 }
143 // exceedingly dangerous constructor
144 Address::Address(address loc, RelocationHolder spec) {
145 _base = noreg;
146 _index = noreg;
147 _scale = no_scale;
148 _disp = (intptr_t) loc;
149 _rspec = spec;
150 }
152 #endif // _LP64
156 // Convert the raw encoding form into the form expected by the constructor for
157 // Address. An index of 4 (rsp) corresponds to having no index, so convert
158 // that to noreg for the Address constructor.
159 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
160 RelocationHolder rspec;
161 if (disp_reloc != relocInfo::none) {
162 rspec = Relocation::spec_simple(disp_reloc);
163 }
164 bool valid_index = index != rsp->encoding();
165 if (valid_index) {
166 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
167 madr._rspec = rspec;
168 return madr;
169 } else {
170 Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
171 madr._rspec = rspec;
172 return madr;
173 }
174 }
176 // Implementation of Assembler
178 int AbstractAssembler::code_fill_byte() {
179 return (u_char)'\xF4'; // hlt
180 }
182 // make this go away someday
183 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
184 if (rtype == relocInfo::none)
185 emit_long(data);
186 else emit_data(data, Relocation::spec_simple(rtype), format);
187 }
189 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
190 assert(imm_operand == 0, "default format must be immediate in this file");
191 assert(inst_mark() != NULL, "must be inside InstructionMark");
192 if (rspec.type() != relocInfo::none) {
193 #ifdef ASSERT
194 check_relocation(rspec, format);
195 #endif
196 // Do not use AbstractAssembler::relocate, which is not intended for
197 // embedded words. Instead, relocate to the enclosing instruction.
199 // hack. call32 is too wide for mask so use disp32
200 if (format == call32_operand)
201 code_section()->relocate(inst_mark(), rspec, disp32_operand);
202 else
203 code_section()->relocate(inst_mark(), rspec, format);
204 }
205 emit_long(data);
206 }
208 static int encode(Register r) {
209 int enc = r->encoding();
210 if (enc >= 8) {
211 enc -= 8;
212 }
213 return enc;
214 }
216 static int encode(XMMRegister r) {
217 int enc = r->encoding();
218 if (enc >= 8) {
219 enc -= 8;
220 }
221 return enc;
222 }
224 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
225 assert(dst->has_byte_register(), "must have byte register");
226 assert(isByte(op1) && isByte(op2), "wrong opcode");
227 assert(isByte(imm8), "not a byte");
228 assert((op1 & 0x01) == 0, "should be 8bit operation");
229 emit_byte(op1);
230 emit_byte(op2 | encode(dst));
231 emit_byte(imm8);
232 }
235 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
236 assert(isByte(op1) && isByte(op2), "wrong opcode");
237 assert((op1 & 0x01) == 1, "should be 32bit operation");
238 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
239 if (is8bit(imm32)) {
240 emit_byte(op1 | 0x02); // set sign bit
241 emit_byte(op2 | encode(dst));
242 emit_byte(imm32 & 0xFF);
243 } else {
244 emit_byte(op1);
245 emit_byte(op2 | encode(dst));
246 emit_long(imm32);
247 }
248 }
250 // Force generation of a 4 byte immediate value even if it fits into 8bit
251 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
252 assert(isByte(op1) && isByte(op2), "wrong opcode");
253 assert((op1 & 0x01) == 1, "should be 32bit operation");
254 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
255 emit_byte(op1);
256 emit_byte(op2 | encode(dst));
257 emit_long(imm32);
258 }
260 // immediate-to-memory forms
261 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
262 assert((op1 & 0x01) == 1, "should be 32bit operation");
263 assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
264 if (is8bit(imm32)) {
265 emit_byte(op1 | 0x02); // set sign bit
266 emit_operand(rm, adr, 1);
267 emit_byte(imm32 & 0xFF);
268 } else {
269 emit_byte(op1);
270 emit_operand(rm, adr, 4);
271 emit_long(imm32);
272 }
273 }
276 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
277 assert(isByte(op1) && isByte(op2), "wrong opcode");
278 emit_byte(op1);
279 emit_byte(op2 | encode(dst) << 3 | encode(src));
280 }
283 void Assembler::emit_operand(Register reg, Register base, Register index,
284 Address::ScaleFactor scale, int disp,
285 RelocationHolder const& rspec,
286 int rip_relative_correction) {
287 relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
289 // Encode the registers as needed in the fields they are used in
291 int regenc = encode(reg) << 3;
292 int indexenc = index->is_valid() ? encode(index) << 3 : 0;
293 int baseenc = base->is_valid() ? encode(base) : 0;
295 if (base->is_valid()) {
296 if (index->is_valid()) {
297 assert(scale != Address::no_scale, "inconsistent address");
298 // [base + index*scale + disp]
299 if (disp == 0 && rtype == relocInfo::none &&
300 base != rbp LP64_ONLY(&& base != r13)) {
301 // [base + index*scale]
302 // [00 reg 100][ss index base]
303 assert(index != rsp, "illegal addressing mode");
304 emit_byte(0x04 | regenc);
305 emit_byte(scale << 6 | indexenc | baseenc);
306 } else if (is8bit(disp) && rtype == relocInfo::none) {
307 // [base + index*scale + imm8]
308 // [01 reg 100][ss index base] imm8
309 assert(index != rsp, "illegal addressing mode");
310 emit_byte(0x44 | regenc);
311 emit_byte(scale << 6 | indexenc | baseenc);
312 emit_byte(disp & 0xFF);
313 } else {
314 // [base + index*scale + disp32]
315 // [10 reg 100][ss index base] disp32
316 assert(index != rsp, "illegal addressing mode");
317 emit_byte(0x84 | regenc);
318 emit_byte(scale << 6 | indexenc | baseenc);
319 emit_data(disp, rspec, disp32_operand);
320 }
321 } else if (base == rsp LP64_ONLY(|| base == r12)) {
322 // [rsp + disp]
323 if (disp == 0 && rtype == relocInfo::none) {
324 // [rsp]
325 // [00 reg 100][00 100 100]
326 emit_byte(0x04 | regenc);
327 emit_byte(0x24);
328 } else if (is8bit(disp) && rtype == relocInfo::none) {
329 // [rsp + imm8]
330 // [01 reg 100][00 100 100] disp8
331 emit_byte(0x44 | regenc);
332 emit_byte(0x24);
333 emit_byte(disp & 0xFF);
334 } else {
335 // [rsp + imm32]
336 // [10 reg 100][00 100 100] disp32
337 emit_byte(0x84 | regenc);
338 emit_byte(0x24);
339 emit_data(disp, rspec, disp32_operand);
340 }
341 } else {
342 // [base + disp]
343 assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
344 if (disp == 0 && rtype == relocInfo::none &&
345 base != rbp LP64_ONLY(&& base != r13)) {
346 // [base]
347 // [00 reg base]
348 emit_byte(0x00 | regenc | baseenc);
349 } else if (is8bit(disp) && rtype == relocInfo::none) {
350 // [base + disp8]
351 // [01 reg base] disp8
352 emit_byte(0x40 | regenc | baseenc);
353 emit_byte(disp & 0xFF);
354 } else {
355 // [base + disp32]
356 // [10 reg base] disp32
357 emit_byte(0x80 | regenc | baseenc);
358 emit_data(disp, rspec, disp32_operand);
359 }
360 }
361 } else {
362 if (index->is_valid()) {
363 assert(scale != Address::no_scale, "inconsistent address");
364 // [index*scale + disp]
365 // [00 reg 100][ss index 101] disp32
366 assert(index != rsp, "illegal addressing mode");
367 emit_byte(0x04 | regenc);
368 emit_byte(scale << 6 | indexenc | 0x05);
369 emit_data(disp, rspec, disp32_operand);
370 } else if (rtype != relocInfo::none ) {
371 // [disp] (64bit) RIP-RELATIVE (32bit) abs
372 // [00 000 101] disp32
374 emit_byte(0x05 | regenc);
375 // Note that the RIP-rel. correction applies to the generated
376 // disp field, but _not_ to the target address in the rspec.
378 // disp was created by converting the target address minus the pc
379 // at the start of the instruction. That needs more correction here.
380 // intptr_t disp = target - next_ip;
381 assert(inst_mark() != NULL, "must be inside InstructionMark");
382 address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
383 int64_t adjusted = disp;
384 // Do rip-rel adjustment for 64bit
385 LP64_ONLY(adjusted -= (next_ip - inst_mark()));
386 assert(is_simm32(adjusted),
387 "must be 32bit offset (RIP relative address)");
388 emit_data((int32_t) adjusted, rspec, disp32_operand);
390 } else {
391 // 32bit never did this, did everything as the rip-rel/disp code above
392 // [disp] ABSOLUTE
393 // [00 reg 100][00 100 101] disp32
394 emit_byte(0x04 | regenc);
395 emit_byte(0x25);
396 emit_data(disp, rspec, disp32_operand);
397 }
398 }
399 }
401 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
402 Address::ScaleFactor scale, int disp,
403 RelocationHolder const& rspec) {
404 emit_operand((Register)reg, base, index, scale, disp, rspec);
405 }
407 // Secret local extension to Assembler::WhichOperand:
408 #define end_pc_operand (_WhichOperand_limit)
410 address Assembler::locate_operand(address inst, WhichOperand which) {
411 // Decode the given instruction, and return the address of
412 // an embedded 32-bit operand word.
414 // If "which" is disp32_operand, selects the displacement portion
415 // of an effective address specifier.
416 // If "which" is imm64_operand, selects the trailing immediate constant.
417 // If "which" is call32_operand, selects the displacement of a call or jump.
418 // Caller is responsible for ensuring that there is such an operand,
419 // and that it is 32/64 bits wide.
421 // If "which" is end_pc_operand, find the end of the instruction.
423 address ip = inst;
424 bool is_64bit = false;
426 debug_only(bool has_disp32 = false);
427 int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
429 again_after_prefix:
430 switch (0xFF & *ip++) {
432 // These convenience macros generate groups of "case" labels for the switch.
433 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
434 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
435 case (x)+4: case (x)+5: case (x)+6: case (x)+7
436 #define REP16(x) REP8((x)+0): \
437 case REP8((x)+8)
439 case CS_segment:
440 case SS_segment:
441 case DS_segment:
442 case ES_segment:
443 case FS_segment:
444 case GS_segment:
445 // Seems dubious
446 LP64_ONLY(assert(false, "shouldn't have that prefix"));
447 assert(ip == inst+1, "only one prefix allowed");
448 goto again_after_prefix;
450 case 0x67:
451 case REX:
452 case REX_B:
453 case REX_X:
454 case REX_XB:
455 case REX_R:
456 case REX_RB:
457 case REX_RX:
458 case REX_RXB:
459 NOT_LP64(assert(false, "64bit prefixes"));
460 goto again_after_prefix;
462 case REX_W:
463 case REX_WB:
464 case REX_WX:
465 case REX_WXB:
466 case REX_WR:
467 case REX_WRB:
468 case REX_WRX:
469 case REX_WRXB:
470 NOT_LP64(assert(false, "64bit prefixes"));
471 is_64bit = true;
472 goto again_after_prefix;
474 case 0xFF: // pushq a; decl a; incl a; call a; jmp a
475 case 0x88: // movb a, r
476 case 0x89: // movl a, r
477 case 0x8A: // movb r, a
478 case 0x8B: // movl r, a
479 case 0x8F: // popl a
480 debug_only(has_disp32 = true);
481 break;
483 case 0x68: // pushq #32
484 if (which == end_pc_operand) {
485 return ip + 4;
486 }
487 assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
488 return ip; // not produced by emit_operand
490 case 0x66: // movw ... (size prefix)
491 again_after_size_prefix2:
492 switch (0xFF & *ip++) {
493 case REX:
494 case REX_B:
495 case REX_X:
496 case REX_XB:
497 case REX_R:
498 case REX_RB:
499 case REX_RX:
500 case REX_RXB:
501 case REX_W:
502 case REX_WB:
503 case REX_WX:
504 case REX_WXB:
505 case REX_WR:
506 case REX_WRB:
507 case REX_WRX:
508 case REX_WRXB:
509 NOT_LP64(assert(false, "64bit prefix found"));
510 goto again_after_size_prefix2;
511 case 0x8B: // movw r, a
512 case 0x89: // movw a, r
513 debug_only(has_disp32 = true);
514 break;
515 case 0xC7: // movw a, #16
516 debug_only(has_disp32 = true);
517 tail_size = 2; // the imm16
518 break;
519 case 0x0F: // several SSE/SSE2 variants
520 ip--; // reparse the 0x0F
521 goto again_after_prefix;
522 default:
523 ShouldNotReachHere();
524 }
525 break;
527 case REP8(0xB8): // movl/q r, #32/#64(oop?)
528 if (which == end_pc_operand) return ip + (is_64bit ? 8 : 4);
529 // these asserts are somewhat nonsensical
530 #ifndef _LP64
531 assert(which == imm_operand || which == disp32_operand,
532 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
533 #else
534 assert((which == call32_operand || which == imm_operand) && is_64bit ||
535 which == narrow_oop_operand && !is_64bit,
536 err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
537 #endif // _LP64
538 return ip;
540 case 0x69: // imul r, a, #32
541 case 0xC7: // movl a, #32(oop?)
542 tail_size = 4;
543 debug_only(has_disp32 = true); // has both kinds of operands!
544 break;
546 case 0x0F: // movx..., etc.
547 switch (0xFF & *ip++) {
548 case 0x3A: // pcmpestri
549 tail_size = 1;
550 case 0x38: // ptest, pmovzxbw
551 ip++; // skip opcode
552 debug_only(has_disp32 = true); // has both kinds of operands!
553 break;
555 case 0x70: // pshufd r, r/a, #8
556 debug_only(has_disp32 = true); // has both kinds of operands!
557 case 0x73: // psrldq r, #8
558 tail_size = 1;
559 break;
561 case 0x12: // movlps
562 case 0x28: // movaps
563 case 0x2E: // ucomiss
564 case 0x2F: // comiss
565 case 0x54: // andps
566 case 0x55: // andnps
567 case 0x56: // orps
568 case 0x57: // xorps
569 case 0x6E: // movd
570 case 0x7E: // movd
571 case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
572 debug_only(has_disp32 = true);
573 break;
575 case 0xAD: // shrd r, a, %cl
576 case 0xAF: // imul r, a
577 case 0xBE: // movsbl r, a (movsxb)
578 case 0xBF: // movswl r, a (movsxw)
579 case 0xB6: // movzbl r, a (movzxb)
580 case 0xB7: // movzwl r, a (movzxw)
581 case REP16(0x40): // cmovl cc, r, a
582 case 0xB0: // cmpxchgb
583 case 0xB1: // cmpxchg
584 case 0xC1: // xaddl
585 case 0xC7: // cmpxchg8
586 case REP16(0x90): // setcc a
587 debug_only(has_disp32 = true);
588 // fall out of the switch to decode the address
589 break;
591 case 0xC4: // pinsrw r, a, #8
592 debug_only(has_disp32 = true);
593 case 0xC5: // pextrw r, r, #8
594 tail_size = 1; // the imm8
595 break;
597 case 0xAC: // shrd r, a, #8
598 debug_only(has_disp32 = true);
599 tail_size = 1; // the imm8
600 break;
602 case REP16(0x80): // jcc rdisp32
603 if (which == end_pc_operand) return ip + 4;
604 assert(which == call32_operand, "jcc has no disp32 or imm");
605 return ip;
606 default:
607 ShouldNotReachHere();
608 }
609 break;
611 case 0x81: // addl a, #32; addl r, #32
612 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
613 // on 32bit in the case of cmpl, the imm might be an oop
614 tail_size = 4;
615 debug_only(has_disp32 = true); // has both kinds of operands!
616 break;
618 case 0x83: // addl a, #8; addl r, #8
619 // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
620 debug_only(has_disp32 = true); // has both kinds of operands!
621 tail_size = 1;
622 break;
624 case 0x9B:
625 switch (0xFF & *ip++) {
626 case 0xD9: // fnstcw a
627 debug_only(has_disp32 = true);
628 break;
629 default:
630 ShouldNotReachHere();
631 }
632 break;
634 case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
635 case REP4(0x10): // adc...
636 case REP4(0x20): // and...
637 case REP4(0x30): // xor...
638 case REP4(0x08): // or...
639 case REP4(0x18): // sbb...
640 case REP4(0x28): // sub...
641 case 0xF7: // mull a
642 case 0x8D: // lea r, a
643 case 0x87: // xchg r, a
644 case REP4(0x38): // cmp...
645 case 0x85: // test r, a
646 debug_only(has_disp32 = true); // has both kinds of operands!
647 break;
649 case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
650 case 0xC6: // movb a, #8
651 case 0x80: // cmpb a, #8
652 case 0x6B: // imul r, a, #8
653 debug_only(has_disp32 = true); // has both kinds of operands!
654 tail_size = 1; // the imm8
655 break;
657 case 0xC4: // VEX_3bytes
658 case 0xC5: // VEX_2bytes
659 assert((UseAVX > 0), "shouldn't have VEX prefix");
660 assert(ip == inst+1, "no prefixes allowed");
661 // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
662 // but they have prefix 0x0F and processed when 0x0F processed above.
663 //
664 // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
665 // instructions (these instructions are not supported in 64-bit mode).
666 // To distinguish them bits [7:6] are set in the VEX second byte since
667 // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
668 // those VEX bits REX and vvvv bits are inverted.
669 //
670 // Fortunately C2 doesn't generate these instructions so we don't need
671 // to check for them in product version.
673 // Check second byte
674 NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
676 // First byte
677 if ((0xFF & *inst) == VEX_3bytes) {
678 ip++; // third byte
679 is_64bit = ((VEX_W & *ip) == VEX_W);
680 }
681 ip++; // opcode
682 // To find the end of instruction (which == end_pc_operand).
683 switch (0xFF & *ip) {
684 case 0x61: // pcmpestri r, r/a, #8
685 case 0x70: // pshufd r, r/a, #8
686 case 0x73: // psrldq r, #8
687 tail_size = 1; // the imm8
688 break;
689 default:
690 break;
691 }
692 ip++; // skip opcode
693 debug_only(has_disp32 = true); // has both kinds of operands!
694 break;
696 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
697 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
698 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
699 case 0xDD: // fld_d a; fst_d a; fstp_d a
700 case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
701 case 0xDF: // fild_d a; fistp_d a
702 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
703 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
704 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
705 debug_only(has_disp32 = true);
706 break;
708 case 0xE8: // call rdisp32
709 case 0xE9: // jmp rdisp32
710 if (which == end_pc_operand) return ip + 4;
711 assert(which == call32_operand, "call has no disp32 or imm");
712 return ip;
714 case 0xF0: // Lock
715 assert(os::is_MP(), "only on MP");
716 goto again_after_prefix;
718 case 0xF3: // For SSE
719 case 0xF2: // For SSE2
720 switch (0xFF & *ip++) {
721 case REX:
722 case REX_B:
723 case REX_X:
724 case REX_XB:
725 case REX_R:
726 case REX_RB:
727 case REX_RX:
728 case REX_RXB:
729 case REX_W:
730 case REX_WB:
731 case REX_WX:
732 case REX_WXB:
733 case REX_WR:
734 case REX_WRB:
735 case REX_WRX:
736 case REX_WRXB:
737 NOT_LP64(assert(false, "found 64bit prefix"));
738 ip++;
739 default:
740 ip++;
741 }
742 debug_only(has_disp32 = true); // has both kinds of operands!
743 break;
745 default:
746 ShouldNotReachHere();
748 #undef REP8
749 #undef REP16
750 }
752 assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
753 #ifdef _LP64
754 assert(which != imm_operand, "instruction is not a movq reg, imm64");
755 #else
756 // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
757 assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
758 #endif // LP64
759 assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
761 // parse the output of emit_operand
762 int op2 = 0xFF & *ip++;
763 int base = op2 & 0x07;
764 int op3 = -1;
765 const int b100 = 4;
766 const int b101 = 5;
767 if (base == b100 && (op2 >> 6) != 3) {
768 op3 = 0xFF & *ip++;
769 base = op3 & 0x07; // refetch the base
770 }
771 // now ip points at the disp (if any)
773 switch (op2 >> 6) {
774 case 0:
775 // [00 reg 100][ss index base]
776 // [00 reg 100][00 100 esp]
777 // [00 reg base]
778 // [00 reg 100][ss index 101][disp32]
779 // [00 reg 101] [disp32]
781 if (base == b101) {
782 if (which == disp32_operand)
783 return ip; // caller wants the disp32
784 ip += 4; // skip the disp32
785 }
786 break;
788 case 1:
789 // [01 reg 100][ss index base][disp8]
790 // [01 reg 100][00 100 esp][disp8]
791 // [01 reg base] [disp8]
792 ip += 1; // skip the disp8
793 break;
795 case 2:
796 // [10 reg 100][ss index base][disp32]
797 // [10 reg 100][00 100 esp][disp32]
798 // [10 reg base] [disp32]
799 if (which == disp32_operand)
800 return ip; // caller wants the disp32
801 ip += 4; // skip the disp32
802 break;
804 case 3:
805 // [11 reg base] (not a memory addressing mode)
806 break;
807 }
809 if (which == end_pc_operand) {
810 return ip + tail_size;
811 }
813 #ifdef _LP64
814 assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
815 #else
816 assert(which == imm_operand, "instruction has only an imm field");
817 #endif // LP64
818 return ip;
819 }
821 address Assembler::locate_next_instruction(address inst) {
822 // Secretly share code with locate_operand:
823 return locate_operand(inst, end_pc_operand);
824 }
827 #ifdef ASSERT
828 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
829 address inst = inst_mark();
830 assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
831 address opnd;
833 Relocation* r = rspec.reloc();
834 if (r->type() == relocInfo::none) {
835 return;
836 } else if (r->is_call() || format == call32_operand) {
837 // assert(format == imm32_operand, "cannot specify a nonzero format");
838 opnd = locate_operand(inst, call32_operand);
839 } else if (r->is_data()) {
840 assert(format == imm_operand || format == disp32_operand
841 LP64_ONLY(|| format == narrow_oop_operand), "format ok");
842 opnd = locate_operand(inst, (WhichOperand)format);
843 } else {
844 assert(format == imm_operand, "cannot specify a format");
845 return;
846 }
847 assert(opnd == pc(), "must put operand where relocs can find it");
848 }
849 #endif // ASSERT
851 void Assembler::emit_operand32(Register reg, Address adr) {
852 assert(reg->encoding() < 8, "no extended registers");
853 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
854 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
855 adr._rspec);
856 }
858 void Assembler::emit_operand(Register reg, Address adr,
859 int rip_relative_correction) {
860 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
861 adr._rspec,
862 rip_relative_correction);
863 }
865 void Assembler::emit_operand(XMMRegister reg, Address adr) {
866 emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
867 adr._rspec);
868 }
870 // MMX operations
871 void Assembler::emit_operand(MMXRegister reg, Address adr) {
872 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
873 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
874 }
876 // work around gcc (3.2.1-7a) bug
877 void Assembler::emit_operand(Address adr, MMXRegister reg) {
878 assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
879 emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
880 }
883 void Assembler::emit_farith(int b1, int b2, int i) {
884 assert(isByte(b1) && isByte(b2), "wrong opcode");
885 assert(0 <= i && i < 8, "illegal stack offset");
886 emit_byte(b1);
887 emit_byte(b2 + i);
888 }
891 // Now the Assembler instructions (identical for 32/64 bits)
893 void Assembler::adcl(Address dst, int32_t imm32) {
894 InstructionMark im(this);
895 prefix(dst);
896 emit_arith_operand(0x81, rdx, dst, imm32);
897 }
899 void Assembler::adcl(Address dst, Register src) {
900 InstructionMark im(this);
901 prefix(dst, src);
902 emit_byte(0x11);
903 emit_operand(src, dst);
904 }
906 void Assembler::adcl(Register dst, int32_t imm32) {
907 prefix(dst);
908 emit_arith(0x81, 0xD0, dst, imm32);
909 }
911 void Assembler::adcl(Register dst, Address src) {
912 InstructionMark im(this);
913 prefix(src, dst);
914 emit_byte(0x13);
915 emit_operand(dst, src);
916 }
918 void Assembler::adcl(Register dst, Register src) {
919 (void) prefix_and_encode(dst->encoding(), src->encoding());
920 emit_arith(0x13, 0xC0, dst, src);
921 }
923 void Assembler::addl(Address dst, int32_t imm32) {
924 InstructionMark im(this);
925 prefix(dst);
926 emit_arith_operand(0x81, rax, dst, imm32);
927 }
929 void Assembler::addl(Address dst, Register src) {
930 InstructionMark im(this);
931 prefix(dst, src);
932 emit_byte(0x01);
933 emit_operand(src, dst);
934 }
936 void Assembler::addl(Register dst, int32_t imm32) {
937 prefix(dst);
938 emit_arith(0x81, 0xC0, dst, imm32);
939 }
941 void Assembler::addl(Register dst, Address src) {
942 InstructionMark im(this);
943 prefix(src, dst);
944 emit_byte(0x03);
945 emit_operand(dst, src);
946 }
948 void Assembler::addl(Register dst, Register src) {
949 (void) prefix_and_encode(dst->encoding(), src->encoding());
950 emit_arith(0x03, 0xC0, dst, src);
951 }
953 void Assembler::addr_nop_4() {
954 assert(UseAddressNop, "no CPU support");
955 // 4 bytes: NOP DWORD PTR [EAX+0]
956 emit_byte(0x0F);
957 emit_byte(0x1F);
958 emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
959 emit_byte(0); // 8-bits offset (1 byte)
960 }
962 void Assembler::addr_nop_5() {
963 assert(UseAddressNop, "no CPU support");
964 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
965 emit_byte(0x0F);
966 emit_byte(0x1F);
967 emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
968 emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
969 emit_byte(0); // 8-bits offset (1 byte)
970 }
972 void Assembler::addr_nop_7() {
973 assert(UseAddressNop, "no CPU support");
974 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
975 emit_byte(0x0F);
976 emit_byte(0x1F);
977 emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
978 emit_long(0); // 32-bits offset (4 bytes)
979 }
981 void Assembler::addr_nop_8() {
982 assert(UseAddressNop, "no CPU support");
983 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
984 emit_byte(0x0F);
985 emit_byte(0x1F);
986 emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
987 emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
988 emit_long(0); // 32-bits offset (4 bytes)
989 }
991 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
992 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
993 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
994 }
996 void Assembler::addsd(XMMRegister dst, Address src) {
997 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
998 emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
999 }
1001 void Assembler::addss(XMMRegister dst, XMMRegister src) {
1002 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1003 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1004 }
1006 void Assembler::addss(XMMRegister dst, Address src) {
1007 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1008 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1009 }
1011 void Assembler::aesdec(XMMRegister dst, Address src) {
1012 assert(VM_Version::supports_aes(), "");
1013 InstructionMark im(this);
1014 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1015 emit_byte(0xde);
1016 emit_operand(dst, src);
1017 }
1019 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1020 assert(VM_Version::supports_aes(), "");
1021 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1022 emit_byte(0xde);
1023 emit_byte(0xC0 | encode);
1024 }
1026 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1027 assert(VM_Version::supports_aes(), "");
1028 InstructionMark im(this);
1029 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1030 emit_byte(0xdf);
1031 emit_operand(dst, src);
1032 }
1034 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1035 assert(VM_Version::supports_aes(), "");
1036 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1037 emit_byte(0xdf);
1038 emit_byte(0xC0 | encode);
1039 }
1041 void Assembler::aesenc(XMMRegister dst, Address src) {
1042 assert(VM_Version::supports_aes(), "");
1043 InstructionMark im(this);
1044 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1045 emit_byte(0xdc);
1046 emit_operand(dst, src);
1047 }
1049 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1050 assert(VM_Version::supports_aes(), "");
1051 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1052 emit_byte(0xdc);
1053 emit_byte(0xC0 | encode);
1054 }
1056 void Assembler::aesenclast(XMMRegister dst, Address src) {
1057 assert(VM_Version::supports_aes(), "");
1058 InstructionMark im(this);
1059 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1060 emit_byte(0xdd);
1061 emit_operand(dst, src);
1062 }
1064 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1065 assert(VM_Version::supports_aes(), "");
1066 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1067 emit_byte(0xdd);
1068 emit_byte(0xC0 | encode);
1069 }
1072 void Assembler::andl(Address dst, int32_t imm32) {
1073 InstructionMark im(this);
1074 prefix(dst);
1075 emit_byte(0x81);
1076 emit_operand(rsp, dst, 4);
1077 emit_long(imm32);
1078 }
1080 void Assembler::andl(Register dst, int32_t imm32) {
1081 prefix(dst);
1082 emit_arith(0x81, 0xE0, dst, imm32);
1083 }
1085 void Assembler::andl(Register dst, Address src) {
1086 InstructionMark im(this);
1087 prefix(src, dst);
1088 emit_byte(0x23);
1089 emit_operand(dst, src);
1090 }
1092 void Assembler::andl(Register dst, Register src) {
1093 (void) prefix_and_encode(dst->encoding(), src->encoding());
1094 emit_arith(0x23, 0xC0, dst, src);
1095 }
1097 void Assembler::bsfl(Register dst, Register src) {
1098 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1099 emit_byte(0x0F);
1100 emit_byte(0xBC);
1101 emit_byte(0xC0 | encode);
1102 }
1104 void Assembler::bsrl(Register dst, Register src) {
1105 assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
1106 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1107 emit_byte(0x0F);
1108 emit_byte(0xBD);
1109 emit_byte(0xC0 | encode);
1110 }
1112 void Assembler::bswapl(Register reg) { // bswap
1113 int encode = prefix_and_encode(reg->encoding());
1114 emit_byte(0x0F);
1115 emit_byte(0xC8 | encode);
1116 }
1118 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1119 // suspect disp32 is always good
1120 int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1122 if (L.is_bound()) {
1123 const int long_size = 5;
1124 int offs = (int)( target(L) - pc() );
1125 assert(offs <= 0, "assembler error");
1126 InstructionMark im(this);
1127 // 1110 1000 #32-bit disp
1128 emit_byte(0xE8);
1129 emit_data(offs - long_size, rtype, operand);
1130 } else {
1131 InstructionMark im(this);
1132 // 1110 1000 #32-bit disp
1133 L.add_patch_at(code(), locator());
1135 emit_byte(0xE8);
1136 emit_data(int(0), rtype, operand);
1137 }
1138 }
1140 void Assembler::call(Register dst) {
1141 int encode = prefix_and_encode(dst->encoding());
1142 emit_byte(0xFF);
1143 emit_byte(0xD0 | encode);
1144 }
1147 void Assembler::call(Address adr) {
1148 InstructionMark im(this);
1149 prefix(adr);
1150 emit_byte(0xFF);
1151 emit_operand(rdx, adr);
1152 }
1154 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1155 assert(entry != NULL, "call most probably wrong");
1156 InstructionMark im(this);
1157 emit_byte(0xE8);
1158 intptr_t disp = entry - (pc() + sizeof(int32_t));
1159 assert(is_simm32(disp), "must be 32bit offset (call2)");
1160 // Technically, should use call32_operand, but this format is
1161 // implied by the fact that we're emitting a call instruction.
1163 int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1164 emit_data((int) disp, rspec, operand);
1165 }
1167 void Assembler::cdql() {
1168 emit_byte(0x99);
1169 }
1171 void Assembler::cld() {
1172 emit_byte(0xfc);
1173 }
1175 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1176 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1177 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1178 emit_byte(0x0F);
1179 emit_byte(0x40 | cc);
1180 emit_byte(0xC0 | encode);
1181 }
1184 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1185 NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1186 prefix(src, dst);
1187 emit_byte(0x0F);
1188 emit_byte(0x40 | cc);
1189 emit_operand(dst, src);
1190 }
1192 void Assembler::cmpb(Address dst, int imm8) {
1193 InstructionMark im(this);
1194 prefix(dst);
1195 emit_byte(0x80);
1196 emit_operand(rdi, dst, 1);
1197 emit_byte(imm8);
1198 }
1200 void Assembler::cmpl(Address dst, int32_t imm32) {
1201 InstructionMark im(this);
1202 prefix(dst);
1203 emit_byte(0x81);
1204 emit_operand(rdi, dst, 4);
1205 emit_long(imm32);
1206 }
1208 void Assembler::cmpl(Register dst, int32_t imm32) {
1209 prefix(dst);
1210 emit_arith(0x81, 0xF8, dst, imm32);
1211 }
1213 void Assembler::cmpl(Register dst, Register src) {
1214 (void) prefix_and_encode(dst->encoding(), src->encoding());
1215 emit_arith(0x3B, 0xC0, dst, src);
1216 }
1219 void Assembler::cmpl(Register dst, Address src) {
1220 InstructionMark im(this);
1221 prefix(src, dst);
1222 emit_byte(0x3B);
1223 emit_operand(dst, src);
1224 }
1226 void Assembler::cmpw(Address dst, int imm16) {
1227 InstructionMark im(this);
1228 assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1229 emit_byte(0x66);
1230 emit_byte(0x81);
1231 emit_operand(rdi, dst, 2);
1232 emit_int16(imm16);
1233 }
1235 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1236 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1237 // The ZF is set if the compared values were equal, and cleared otherwise.
1238 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1239 InstructionMark im(this);
1240 prefix(adr, reg);
1241 emit_byte(0x0F);
1242 emit_byte(0xB1);
1243 emit_operand(reg, adr);
1244 }
1246 void Assembler::comisd(XMMRegister dst, Address src) {
1247 // NOTE: dbx seems to decode this as comiss even though the
1248 // 0x66 is there. Strangly ucomisd comes out correct
1249 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1250 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1251 }
1253 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1254 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1255 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1256 }
1258 void Assembler::comiss(XMMRegister dst, Address src) {
1259 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1260 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1261 }
1263 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1264 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1265 emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1266 }
1268 void Assembler::cpuid() {
1269 emit_byte(0x0F);
1270 emit_byte(0xA2);
1271 }
1273 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1274 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1275 emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3);
1276 }
1278 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1279 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1280 emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE);
1281 }
1283 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1284 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1285 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1286 }
1288 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1289 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1290 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1291 }
1293 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1294 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1295 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
1296 emit_byte(0x2A);
1297 emit_byte(0xC0 | encode);
1298 }
1300 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1301 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1302 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1303 }
1305 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1306 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1307 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
1308 emit_byte(0x2A);
1309 emit_byte(0xC0 | encode);
1310 }
1312 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1313 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1314 emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3);
1315 }
1317 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1318 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1319 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1320 }
1322 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1323 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1324 emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1325 }
1328 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1329 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1330 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
1331 emit_byte(0x2C);
1332 emit_byte(0xC0 | encode);
1333 }
1335 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1336 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1337 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
1338 emit_byte(0x2C);
1339 emit_byte(0xC0 | encode);
1340 }
1342 void Assembler::decl(Address dst) {
1343 // Don't use it directly. Use MacroAssembler::decrement() instead.
1344 InstructionMark im(this);
1345 prefix(dst);
1346 emit_byte(0xFF);
1347 emit_operand(rcx, dst);
1348 }
1350 void Assembler::divsd(XMMRegister dst, Address src) {
1351 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1352 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1353 }
1355 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1356 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1357 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1358 }
1360 void Assembler::divss(XMMRegister dst, Address src) {
1361 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1362 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1363 }
1365 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1366 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1367 emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1368 }
1370 void Assembler::emms() {
1371 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1372 emit_byte(0x0F);
1373 emit_byte(0x77);
1374 }
1376 void Assembler::hlt() {
1377 emit_byte(0xF4);
1378 }
1380 void Assembler::idivl(Register src) {
1381 int encode = prefix_and_encode(src->encoding());
1382 emit_byte(0xF7);
1383 emit_byte(0xF8 | encode);
1384 }
1386 void Assembler::divl(Register src) { // Unsigned
1387 int encode = prefix_and_encode(src->encoding());
1388 emit_byte(0xF7);
1389 emit_byte(0xF0 | encode);
1390 }
1392 void Assembler::imull(Register dst, Register src) {
1393 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1394 emit_byte(0x0F);
1395 emit_byte(0xAF);
1396 emit_byte(0xC0 | encode);
1397 }
1400 void Assembler::imull(Register dst, Register src, int value) {
1401 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1402 if (is8bit(value)) {
1403 emit_byte(0x6B);
1404 emit_byte(0xC0 | encode);
1405 emit_byte(value & 0xFF);
1406 } else {
1407 emit_byte(0x69);
1408 emit_byte(0xC0 | encode);
1409 emit_long(value);
1410 }
1411 }
1413 void Assembler::incl(Address dst) {
1414 // Don't use it directly. Use MacroAssembler::increment() instead.
1415 InstructionMark im(this);
1416 prefix(dst);
1417 emit_byte(0xFF);
1418 emit_operand(rax, dst);
1419 }
1421 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1422 InstructionMark im(this);
1423 assert((0 <= cc) && (cc < 16), "illegal cc");
1424 if (L.is_bound()) {
1425 address dst = target(L);
1426 assert(dst != NULL, "jcc most probably wrong");
1428 const int short_size = 2;
1429 const int long_size = 6;
1430 intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1431 if (maybe_short && is8bit(offs - short_size)) {
1432 // 0111 tttn #8-bit disp
1433 emit_byte(0x70 | cc);
1434 emit_byte((offs - short_size) & 0xFF);
1435 } else {
1436 // 0000 1111 1000 tttn #32-bit disp
1437 assert(is_simm32(offs - long_size),
1438 "must be 32bit offset (call4)");
1439 emit_byte(0x0F);
1440 emit_byte(0x80 | cc);
1441 emit_long(offs - long_size);
1442 }
1443 } else {
1444 // Note: could eliminate cond. jumps to this jump if condition
1445 // is the same however, seems to be rather unlikely case.
1446 // Note: use jccb() if label to be bound is very close to get
1447 // an 8-bit displacement
1448 L.add_patch_at(code(), locator());
1449 emit_byte(0x0F);
1450 emit_byte(0x80 | cc);
1451 emit_long(0);
1452 }
1453 }
1455 void Assembler::jccb(Condition cc, Label& L) {
1456 if (L.is_bound()) {
1457 const int short_size = 2;
1458 address entry = target(L);
1459 #ifdef ASSERT
1460 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1461 intptr_t delta = short_branch_delta();
1462 if (delta != 0) {
1463 dist += (dist < 0 ? (-delta) :delta);
1464 }
1465 assert(is8bit(dist), "Dispacement too large for a short jmp");
1466 #endif
1467 intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1468 // 0111 tttn #8-bit disp
1469 emit_byte(0x70 | cc);
1470 emit_byte((offs - short_size) & 0xFF);
1471 } else {
1472 InstructionMark im(this);
1473 L.add_patch_at(code(), locator());
1474 emit_byte(0x70 | cc);
1475 emit_byte(0);
1476 }
1477 }
1479 void Assembler::jmp(Address adr) {
1480 InstructionMark im(this);
1481 prefix(adr);
1482 emit_byte(0xFF);
1483 emit_operand(rsp, adr);
1484 }
1486 void Assembler::jmp(Label& L, bool maybe_short) {
1487 if (L.is_bound()) {
1488 address entry = target(L);
1489 assert(entry != NULL, "jmp most probably wrong");
1490 InstructionMark im(this);
1491 const int short_size = 2;
1492 const int long_size = 5;
1493 intptr_t offs = entry - pc();
1494 if (maybe_short && is8bit(offs - short_size)) {
1495 emit_byte(0xEB);
1496 emit_byte((offs - short_size) & 0xFF);
1497 } else {
1498 emit_byte(0xE9);
1499 emit_long(offs - long_size);
1500 }
1501 } else {
1502 // By default, forward jumps are always 32-bit displacements, since
1503 // we can't yet know where the label will be bound. If you're sure that
1504 // the forward jump will not run beyond 256 bytes, use jmpb to
1505 // force an 8-bit displacement.
1506 InstructionMark im(this);
1507 L.add_patch_at(code(), locator());
1508 emit_byte(0xE9);
1509 emit_long(0);
1510 }
1511 }
1513 void Assembler::jmp(Register entry) {
1514 int encode = prefix_and_encode(entry->encoding());
1515 emit_byte(0xFF);
1516 emit_byte(0xE0 | encode);
1517 }
1519 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
1520 InstructionMark im(this);
1521 emit_byte(0xE9);
1522 assert(dest != NULL, "must have a target");
1523 intptr_t disp = dest - (pc() + sizeof(int32_t));
1524 assert(is_simm32(disp), "must be 32bit offset (jmp)");
1525 emit_data(disp, rspec.reloc(), call32_operand);
1526 }
1528 void Assembler::jmpb(Label& L) {
1529 if (L.is_bound()) {
1530 const int short_size = 2;
1531 address entry = target(L);
1532 assert(entry != NULL, "jmp most probably wrong");
1533 #ifdef ASSERT
1534 intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1535 intptr_t delta = short_branch_delta();
1536 if (delta != 0) {
1537 dist += (dist < 0 ? (-delta) :delta);
1538 }
1539 assert(is8bit(dist), "Dispacement too large for a short jmp");
1540 #endif
1541 intptr_t offs = entry - pc();
1542 emit_byte(0xEB);
1543 emit_byte((offs - short_size) & 0xFF);
1544 } else {
1545 InstructionMark im(this);
1546 L.add_patch_at(code(), locator());
1547 emit_byte(0xEB);
1548 emit_byte(0);
1549 }
1550 }
1552 void Assembler::ldmxcsr( Address src) {
1553 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1554 InstructionMark im(this);
1555 prefix(src);
1556 emit_byte(0x0F);
1557 emit_byte(0xAE);
1558 emit_operand(as_Register(2), src);
1559 }
1561 void Assembler::leal(Register dst, Address src) {
1562 InstructionMark im(this);
1563 #ifdef _LP64
1564 emit_byte(0x67); // addr32
1565 prefix(src, dst);
1566 #endif // LP64
1567 emit_byte(0x8D);
1568 emit_operand(dst, src);
1569 }
1571 void Assembler::lfence() {
1572 emit_byte(0x0F);
1573 emit_byte(0xAE);
1574 emit_byte(0xE8);
1575 }
1577 void Assembler::lock() {
1578 emit_byte(0xF0);
1579 }
1581 void Assembler::lzcntl(Register dst, Register src) {
1582 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
1583 emit_byte(0xF3);
1584 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1585 emit_byte(0x0F);
1586 emit_byte(0xBD);
1587 emit_byte(0xC0 | encode);
1588 }
1590 // Emit mfence instruction
1591 void Assembler::mfence() {
1592 NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
1593 emit_byte( 0x0F );
1594 emit_byte( 0xAE );
1595 emit_byte( 0xF0 );
1596 }
1598 void Assembler::mov(Register dst, Register src) {
1599 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
1600 }
1602 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
1603 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1604 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
1605 }
1607 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
1608 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1609 emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
1610 }
1612 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
1613 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1614 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE);
1615 emit_byte(0x16);
1616 emit_byte(0xC0 | encode);
1617 }
1619 void Assembler::movb(Register dst, Address src) {
1620 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
1621 InstructionMark im(this);
1622 prefix(src, dst, true);
1623 emit_byte(0x8A);
1624 emit_operand(dst, src);
1625 }
1628 void Assembler::movb(Address dst, int imm8) {
1629 InstructionMark im(this);
1630 prefix(dst);
1631 emit_byte(0xC6);
1632 emit_operand(rax, dst, 1);
1633 emit_byte(imm8);
1634 }
1637 void Assembler::movb(Address dst, Register src) {
1638 assert(src->has_byte_register(), "must have byte register");
1639 InstructionMark im(this);
1640 prefix(dst, src, true);
1641 emit_byte(0x88);
1642 emit_operand(src, dst);
1643 }
1645 void Assembler::movdl(XMMRegister dst, Register src) {
1646 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1647 int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
1648 emit_byte(0x6E);
1649 emit_byte(0xC0 | encode);
1650 }
1652 void Assembler::movdl(Register dst, XMMRegister src) {
1653 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1654 // swap src/dst to get correct prefix
1655 int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);
1656 emit_byte(0x7E);
1657 emit_byte(0xC0 | encode);
1658 }
1660 void Assembler::movdl(XMMRegister dst, Address src) {
1661 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1662 InstructionMark im(this);
1663 simd_prefix(dst, src, VEX_SIMD_66);
1664 emit_byte(0x6E);
1665 emit_operand(dst, src);
1666 }
1668 void Assembler::movdl(Address dst, XMMRegister src) {
1669 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1670 InstructionMark im(this);
1671 simd_prefix(dst, src, VEX_SIMD_66);
1672 emit_byte(0x7E);
1673 emit_operand(src, dst);
1674 }
1676 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
1677 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1678 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
1679 }
1681 void Assembler::movdqu(XMMRegister dst, Address src) {
1682 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1683 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1684 }
1686 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
1687 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1688 emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1689 }
1691 void Assembler::movdqu(Address dst, XMMRegister src) {
1692 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1693 InstructionMark im(this);
1694 simd_prefix(dst, src, VEX_SIMD_F3);
1695 emit_byte(0x7F);
1696 emit_operand(src, dst);
1697 }
1699 // Move Unaligned 256bit Vector
1700 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
1701 assert(UseAVX, "");
1702 bool vector256 = true;
1703 int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1704 emit_byte(0x6F);
1705 emit_byte(0xC0 | encode);
1706 }
1708 void Assembler::vmovdqu(XMMRegister dst, Address src) {
1709 assert(UseAVX, "");
1710 InstructionMark im(this);
1711 bool vector256 = true;
1712 vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1713 emit_byte(0x6F);
1714 emit_operand(dst, src);
1715 }
1717 void Assembler::vmovdqu(Address dst, XMMRegister src) {
1718 assert(UseAVX, "");
1719 InstructionMark im(this);
1720 bool vector256 = true;
1721 // swap src<->dst for encoding
1722 assert(src != xnoreg, "sanity");
1723 vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector256);
1724 emit_byte(0x7F);
1725 emit_operand(src, dst);
1726 }
1728 // Uses zero extension on 64bit
1730 void Assembler::movl(Register dst, int32_t imm32) {
1731 int encode = prefix_and_encode(dst->encoding());
1732 emit_byte(0xB8 | encode);
1733 emit_long(imm32);
1734 }
1736 void Assembler::movl(Register dst, Register src) {
1737 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1738 emit_byte(0x8B);
1739 emit_byte(0xC0 | encode);
1740 }
1742 void Assembler::movl(Register dst, Address src) {
1743 InstructionMark im(this);
1744 prefix(src, dst);
1745 emit_byte(0x8B);
1746 emit_operand(dst, src);
1747 }
1749 void Assembler::movl(Address dst, int32_t imm32) {
1750 InstructionMark im(this);
1751 prefix(dst);
1752 emit_byte(0xC7);
1753 emit_operand(rax, dst, 4);
1754 emit_long(imm32);
1755 }
1757 void Assembler::movl(Address dst, Register src) {
1758 InstructionMark im(this);
1759 prefix(dst, src);
1760 emit_byte(0x89);
1761 emit_operand(src, dst);
1762 }
1764 // New cpus require to use movsd and movss to avoid partial register stall
1765 // when loading from memory. But for old Opteron use movlpd instead of movsd.
1766 // The selection is done in MacroAssembler::movdbl() and movflt().
1767 void Assembler::movlpd(XMMRegister dst, Address src) {
1768 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1769 emit_simd_arith(0x12, dst, src, VEX_SIMD_66);
1770 }
1772 void Assembler::movq( MMXRegister dst, Address src ) {
1773 assert( VM_Version::supports_mmx(), "" );
1774 emit_byte(0x0F);
1775 emit_byte(0x6F);
1776 emit_operand(dst, src);
1777 }
1779 void Assembler::movq( Address dst, MMXRegister src ) {
1780 assert( VM_Version::supports_mmx(), "" );
1781 emit_byte(0x0F);
1782 emit_byte(0x7F);
1783 // workaround gcc (3.2.1-7a) bug
1784 // In that version of gcc with only an emit_operand(MMX, Address)
1785 // gcc will tail jump and try and reverse the parameters completely
1786 // obliterating dst in the process. By having a version available
1787 // that doesn't need to swap the args at the tail jump the bug is
1788 // avoided.
1789 emit_operand(dst, src);
1790 }
1792 void Assembler::movq(XMMRegister dst, Address src) {
1793 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1794 InstructionMark im(this);
1795 simd_prefix(dst, src, VEX_SIMD_F3);
1796 emit_byte(0x7E);
1797 emit_operand(dst, src);
1798 }
1800 void Assembler::movq(Address dst, XMMRegister src) {
1801 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1802 InstructionMark im(this);
1803 simd_prefix(dst, src, VEX_SIMD_66);
1804 emit_byte(0xD6);
1805 emit_operand(src, dst);
1806 }
1808 void Assembler::movsbl(Register dst, Address src) { // movsxb
1809 InstructionMark im(this);
1810 prefix(src, dst);
1811 emit_byte(0x0F);
1812 emit_byte(0xBE);
1813 emit_operand(dst, src);
1814 }
1816 void Assembler::movsbl(Register dst, Register src) { // movsxb
1817 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1818 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1819 emit_byte(0x0F);
1820 emit_byte(0xBE);
1821 emit_byte(0xC0 | encode);
1822 }
1824 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
1825 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1826 emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
1827 }
1829 void Assembler::movsd(XMMRegister dst, Address src) {
1830 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1831 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
1832 }
1834 void Assembler::movsd(Address dst, XMMRegister src) {
1835 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1836 InstructionMark im(this);
1837 simd_prefix(dst, src, VEX_SIMD_F2);
1838 emit_byte(0x11);
1839 emit_operand(src, dst);
1840 }
1842 void Assembler::movss(XMMRegister dst, XMMRegister src) {
1843 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1844 emit_simd_arith(0x10, dst, src, VEX_SIMD_F3);
1845 }
1847 void Assembler::movss(XMMRegister dst, Address src) {
1848 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1849 emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3);
1850 }
1852 void Assembler::movss(Address dst, XMMRegister src) {
1853 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1854 InstructionMark im(this);
1855 simd_prefix(dst, src, VEX_SIMD_F3);
1856 emit_byte(0x11);
1857 emit_operand(src, dst);
1858 }
1860 void Assembler::movswl(Register dst, Address src) { // movsxw
1861 InstructionMark im(this);
1862 prefix(src, dst);
1863 emit_byte(0x0F);
1864 emit_byte(0xBF);
1865 emit_operand(dst, src);
1866 }
1868 void Assembler::movswl(Register dst, Register src) { // movsxw
1869 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1870 emit_byte(0x0F);
1871 emit_byte(0xBF);
1872 emit_byte(0xC0 | encode);
1873 }
1875 void Assembler::movw(Address dst, int imm16) {
1876 InstructionMark im(this);
1878 emit_byte(0x66); // switch to 16-bit mode
1879 prefix(dst);
1880 emit_byte(0xC7);
1881 emit_operand(rax, dst, 2);
1882 emit_int16(imm16);
1883 }
1885 void Assembler::movw(Register dst, Address src) {
1886 InstructionMark im(this);
1887 emit_byte(0x66);
1888 prefix(src, dst);
1889 emit_byte(0x8B);
1890 emit_operand(dst, src);
1891 }
1893 void Assembler::movw(Address dst, Register src) {
1894 InstructionMark im(this);
1895 emit_byte(0x66);
1896 prefix(dst, src);
1897 emit_byte(0x89);
1898 emit_operand(src, dst);
1899 }
1901 void Assembler::movzbl(Register dst, Address src) { // movzxb
1902 InstructionMark im(this);
1903 prefix(src, dst);
1904 emit_byte(0x0F);
1905 emit_byte(0xB6);
1906 emit_operand(dst, src);
1907 }
1909 void Assembler::movzbl(Register dst, Register src) { // movzxb
1910 NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1911 int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1912 emit_byte(0x0F);
1913 emit_byte(0xB6);
1914 emit_byte(0xC0 | encode);
1915 }
1917 void Assembler::movzwl(Register dst, Address src) { // movzxw
1918 InstructionMark im(this);
1919 prefix(src, dst);
1920 emit_byte(0x0F);
1921 emit_byte(0xB7);
1922 emit_operand(dst, src);
1923 }
1925 void Assembler::movzwl(Register dst, Register src) { // movzxw
1926 int encode = prefix_and_encode(dst->encoding(), src->encoding());
1927 emit_byte(0x0F);
1928 emit_byte(0xB7);
1929 emit_byte(0xC0 | encode);
1930 }
1932 void Assembler::mull(Address src) {
1933 InstructionMark im(this);
1934 prefix(src);
1935 emit_byte(0xF7);
1936 emit_operand(rsp, src);
1937 }
1939 void Assembler::mull(Register src) {
1940 int encode = prefix_and_encode(src->encoding());
1941 emit_byte(0xF7);
1942 emit_byte(0xE0 | encode);
1943 }
1945 void Assembler::mulsd(XMMRegister dst, Address src) {
1946 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1947 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
1948 }
1950 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
1951 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1952 emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
1953 }
1955 void Assembler::mulss(XMMRegister dst, Address src) {
1956 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1957 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
1958 }
1960 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
1961 NOT_LP64(assert(VM_Version::supports_sse(), ""));
1962 emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
1963 }
1965 void Assembler::negl(Register dst) {
1966 int encode = prefix_and_encode(dst->encoding());
1967 emit_byte(0xF7);
1968 emit_byte(0xD8 | encode);
1969 }
1971 void Assembler::nop(int i) {
1972 #ifdef ASSERT
1973 assert(i > 0, " ");
1974 // The fancy nops aren't currently recognized by debuggers making it a
1975 // pain to disassemble code while debugging. If asserts are on clearly
1976 // speed is not an issue so simply use the single byte traditional nop
1977 // to do alignment.
1979 for (; i > 0 ; i--) emit_byte(0x90);
1980 return;
1982 #endif // ASSERT
1984 if (UseAddressNop && VM_Version::is_intel()) {
1985 //
1986 // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
1987 // 1: 0x90
1988 // 2: 0x66 0x90
1989 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
1990 // 4: 0x0F 0x1F 0x40 0x00
1991 // 5: 0x0F 0x1F 0x44 0x00 0x00
1992 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
1993 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
1994 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1995 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1996 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1997 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1999 // The rest coding is Intel specific - don't use consecutive address nops
2001 // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2002 // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2003 // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2004 // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2006 while(i >= 15) {
2007 // For Intel don't generate consecutive addess nops (mix with regular nops)
2008 i -= 15;
2009 emit_byte(0x66); // size prefix
2010 emit_byte(0x66); // size prefix
2011 emit_byte(0x66); // size prefix
2012 addr_nop_8();
2013 emit_byte(0x66); // size prefix
2014 emit_byte(0x66); // size prefix
2015 emit_byte(0x66); // size prefix
2016 emit_byte(0x90); // nop
2017 }
2018 switch (i) {
2019 case 14:
2020 emit_byte(0x66); // size prefix
2021 case 13:
2022 emit_byte(0x66); // size prefix
2023 case 12:
2024 addr_nop_8();
2025 emit_byte(0x66); // size prefix
2026 emit_byte(0x66); // size prefix
2027 emit_byte(0x66); // size prefix
2028 emit_byte(0x90); // nop
2029 break;
2030 case 11:
2031 emit_byte(0x66); // size prefix
2032 case 10:
2033 emit_byte(0x66); // size prefix
2034 case 9:
2035 emit_byte(0x66); // size prefix
2036 case 8:
2037 addr_nop_8();
2038 break;
2039 case 7:
2040 addr_nop_7();
2041 break;
2042 case 6:
2043 emit_byte(0x66); // size prefix
2044 case 5:
2045 addr_nop_5();
2046 break;
2047 case 4:
2048 addr_nop_4();
2049 break;
2050 case 3:
2051 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2052 emit_byte(0x66); // size prefix
2053 case 2:
2054 emit_byte(0x66); // size prefix
2055 case 1:
2056 emit_byte(0x90); // nop
2057 break;
2058 default:
2059 assert(i == 0, " ");
2060 }
2061 return;
2062 }
2063 if (UseAddressNop && VM_Version::is_amd()) {
2064 //
2065 // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2066 // 1: 0x90
2067 // 2: 0x66 0x90
2068 // 3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2069 // 4: 0x0F 0x1F 0x40 0x00
2070 // 5: 0x0F 0x1F 0x44 0x00 0x00
2071 // 6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2072 // 7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2073 // 8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2074 // 9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2075 // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2076 // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2078 // The rest coding is AMD specific - use consecutive address nops
2080 // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2081 // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2082 // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2083 // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2084 // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2085 // Size prefixes (0x66) are added for larger sizes
2087 while(i >= 22) {
2088 i -= 11;
2089 emit_byte(0x66); // size prefix
2090 emit_byte(0x66); // size prefix
2091 emit_byte(0x66); // size prefix
2092 addr_nop_8();
2093 }
2094 // Generate first nop for size between 21-12
2095 switch (i) {
2096 case 21:
2097 i -= 1;
2098 emit_byte(0x66); // size prefix
2099 case 20:
2100 case 19:
2101 i -= 1;
2102 emit_byte(0x66); // size prefix
2103 case 18:
2104 case 17:
2105 i -= 1;
2106 emit_byte(0x66); // size prefix
2107 case 16:
2108 case 15:
2109 i -= 8;
2110 addr_nop_8();
2111 break;
2112 case 14:
2113 case 13:
2114 i -= 7;
2115 addr_nop_7();
2116 break;
2117 case 12:
2118 i -= 6;
2119 emit_byte(0x66); // size prefix
2120 addr_nop_5();
2121 break;
2122 default:
2123 assert(i < 12, " ");
2124 }
2126 // Generate second nop for size between 11-1
2127 switch (i) {
2128 case 11:
2129 emit_byte(0x66); // size prefix
2130 case 10:
2131 emit_byte(0x66); // size prefix
2132 case 9:
2133 emit_byte(0x66); // size prefix
2134 case 8:
2135 addr_nop_8();
2136 break;
2137 case 7:
2138 addr_nop_7();
2139 break;
2140 case 6:
2141 emit_byte(0x66); // size prefix
2142 case 5:
2143 addr_nop_5();
2144 break;
2145 case 4:
2146 addr_nop_4();
2147 break;
2148 case 3:
2149 // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2150 emit_byte(0x66); // size prefix
2151 case 2:
2152 emit_byte(0x66); // size prefix
2153 case 1:
2154 emit_byte(0x90); // nop
2155 break;
2156 default:
2157 assert(i == 0, " ");
2158 }
2159 return;
2160 }
2162 // Using nops with size prefixes "0x66 0x90".
2163 // From AMD Optimization Guide:
2164 // 1: 0x90
2165 // 2: 0x66 0x90
2166 // 3: 0x66 0x66 0x90
2167 // 4: 0x66 0x66 0x66 0x90
2168 // 5: 0x66 0x66 0x90 0x66 0x90
2169 // 6: 0x66 0x66 0x90 0x66 0x66 0x90
2170 // 7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2171 // 8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2172 // 9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2173 // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2174 //
2175 while(i > 12) {
2176 i -= 4;
2177 emit_byte(0x66); // size prefix
2178 emit_byte(0x66);
2179 emit_byte(0x66);
2180 emit_byte(0x90); // nop
2181 }
2182 // 1 - 12 nops
2183 if(i > 8) {
2184 if(i > 9) {
2185 i -= 1;
2186 emit_byte(0x66);
2187 }
2188 i -= 3;
2189 emit_byte(0x66);
2190 emit_byte(0x66);
2191 emit_byte(0x90);
2192 }
2193 // 1 - 8 nops
2194 if(i > 4) {
2195 if(i > 6) {
2196 i -= 1;
2197 emit_byte(0x66);
2198 }
2199 i -= 3;
2200 emit_byte(0x66);
2201 emit_byte(0x66);
2202 emit_byte(0x90);
2203 }
2204 switch (i) {
2205 case 4:
2206 emit_byte(0x66);
2207 case 3:
2208 emit_byte(0x66);
2209 case 2:
2210 emit_byte(0x66);
2211 case 1:
2212 emit_byte(0x90);
2213 break;
2214 default:
2215 assert(i == 0, " ");
2216 }
2217 }
2219 void Assembler::notl(Register dst) {
2220 int encode = prefix_and_encode(dst->encoding());
2221 emit_byte(0xF7);
2222 emit_byte(0xD0 | encode );
2223 }
2225 void Assembler::orl(Address dst, int32_t imm32) {
2226 InstructionMark im(this);
2227 prefix(dst);
2228 emit_arith_operand(0x81, rcx, dst, imm32);
2229 }
2231 void Assembler::orl(Register dst, int32_t imm32) {
2232 prefix(dst);
2233 emit_arith(0x81, 0xC8, dst, imm32);
2234 }
2236 void Assembler::orl(Register dst, Address src) {
2237 InstructionMark im(this);
2238 prefix(src, dst);
2239 emit_byte(0x0B);
2240 emit_operand(dst, src);
2241 }
2243 void Assembler::orl(Register dst, Register src) {
2244 (void) prefix_and_encode(dst->encoding(), src->encoding());
2245 emit_arith(0x0B, 0xC0, dst, src);
2246 }
2248 void Assembler::packuswb(XMMRegister dst, Address src) {
2249 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2250 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2251 emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2252 }
2254 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2255 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2256 emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2257 }
2259 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2260 assert(VM_Version::supports_sse4_2(), "");
2261 InstructionMark im(this);
2262 simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2263 emit_byte(0x61);
2264 emit_operand(dst, src);
2265 emit_byte(imm8);
2266 }
2268 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2269 assert(VM_Version::supports_sse4_2(), "");
2270 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2271 emit_byte(0x61);
2272 emit_byte(0xC0 | encode);
2273 emit_byte(imm8);
2274 }
2276 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
2277 assert(VM_Version::supports_sse4_1(), "");
2278 InstructionMark im(this);
2279 simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2280 emit_byte(0x30);
2281 emit_operand(dst, src);
2282 }
2284 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2285 assert(VM_Version::supports_sse4_1(), "");
2286 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2287 emit_byte(0x30);
2288 emit_byte(0xC0 | encode);
2289 }
2291 // generic
2292 void Assembler::pop(Register dst) {
2293 int encode = prefix_and_encode(dst->encoding());
2294 emit_byte(0x58 | encode);
2295 }
2297 void Assembler::popcntl(Register dst, Address src) {
2298 assert(VM_Version::supports_popcnt(), "must support");
2299 InstructionMark im(this);
2300 emit_byte(0xF3);
2301 prefix(src, dst);
2302 emit_byte(0x0F);
2303 emit_byte(0xB8);
2304 emit_operand(dst, src);
2305 }
2307 void Assembler::popcntl(Register dst, Register src) {
2308 assert(VM_Version::supports_popcnt(), "must support");
2309 emit_byte(0xF3);
2310 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2311 emit_byte(0x0F);
2312 emit_byte(0xB8);
2313 emit_byte(0xC0 | encode);
2314 }
2316 void Assembler::popf() {
2317 emit_byte(0x9D);
2318 }
2320 #ifndef _LP64 // no 32bit push/pop on amd64
2321 void Assembler::popl(Address dst) {
2322 // NOTE: this will adjust stack by 8byte on 64bits
2323 InstructionMark im(this);
2324 prefix(dst);
2325 emit_byte(0x8F);
2326 emit_operand(rax, dst);
2327 }
2328 #endif
2330 void Assembler::prefetch_prefix(Address src) {
2331 prefix(src);
2332 emit_byte(0x0F);
2333 }
2335 void Assembler::prefetchnta(Address src) {
2336 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2337 InstructionMark im(this);
2338 prefetch_prefix(src);
2339 emit_byte(0x18);
2340 emit_operand(rax, src); // 0, src
2341 }
2343 void Assembler::prefetchr(Address src) {
2344 assert(VM_Version::supports_3dnow_prefetch(), "must support");
2345 InstructionMark im(this);
2346 prefetch_prefix(src);
2347 emit_byte(0x0D);
2348 emit_operand(rax, src); // 0, src
2349 }
2351 void Assembler::prefetcht0(Address src) {
2352 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2353 InstructionMark im(this);
2354 prefetch_prefix(src);
2355 emit_byte(0x18);
2356 emit_operand(rcx, src); // 1, src
2357 }
2359 void Assembler::prefetcht1(Address src) {
2360 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2361 InstructionMark im(this);
2362 prefetch_prefix(src);
2363 emit_byte(0x18);
2364 emit_operand(rdx, src); // 2, src
2365 }
2367 void Assembler::prefetcht2(Address src) {
2368 NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2369 InstructionMark im(this);
2370 prefetch_prefix(src);
2371 emit_byte(0x18);
2372 emit_operand(rbx, src); // 3, src
2373 }
2375 void Assembler::prefetchw(Address src) {
2376 assert(VM_Version::supports_3dnow_prefetch(), "must support");
2377 InstructionMark im(this);
2378 prefetch_prefix(src);
2379 emit_byte(0x0D);
2380 emit_operand(rcx, src); // 1, src
2381 }
2383 void Assembler::prefix(Prefix p) {
2384 a_byte(p);
2385 }
2387 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
2388 assert(VM_Version::supports_ssse3(), "");
2389 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2390 emit_byte(0x00);
2391 emit_byte(0xC0 | encode);
2392 }
2394 void Assembler::pshufb(XMMRegister dst, Address src) {
2395 assert(VM_Version::supports_ssse3(), "");
2396 InstructionMark im(this);
2397 simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2398 emit_byte(0x00);
2399 emit_operand(dst, src);
2400 }
2402 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
2403 assert(isByte(mode), "invalid value");
2404 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2405 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
2406 emit_byte(mode & 0xFF);
2408 }
2410 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
2411 assert(isByte(mode), "invalid value");
2412 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2413 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2414 InstructionMark im(this);
2415 simd_prefix(dst, src, VEX_SIMD_66);
2416 emit_byte(0x70);
2417 emit_operand(dst, src);
2418 emit_byte(mode & 0xFF);
2419 }
2421 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
2422 assert(isByte(mode), "invalid value");
2423 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2424 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2);
2425 emit_byte(mode & 0xFF);
2426 }
2428 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
2429 assert(isByte(mode), "invalid value");
2430 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2431 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2432 InstructionMark im(this);
2433 simd_prefix(dst, src, VEX_SIMD_F2);
2434 emit_byte(0x70);
2435 emit_operand(dst, src);
2436 emit_byte(mode & 0xFF);
2437 }
2439 void Assembler::psrldq(XMMRegister dst, int shift) {
2440 // Shift 128 bit value in xmm register by number of bytes.
2441 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2442 int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);
2443 emit_byte(0x73);
2444 emit_byte(0xC0 | encode);
2445 emit_byte(shift);
2446 }
2448 void Assembler::ptest(XMMRegister dst, Address src) {
2449 assert(VM_Version::supports_sse4_1(), "");
2450 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2451 InstructionMark im(this);
2452 simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2453 emit_byte(0x17);
2454 emit_operand(dst, src);
2455 }
2457 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
2458 assert(VM_Version::supports_sse4_1(), "");
2459 int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2460 emit_byte(0x17);
2461 emit_byte(0xC0 | encode);
2462 }
2464 void Assembler::punpcklbw(XMMRegister dst, Address src) {
2465 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2466 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2467 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2468 }
2470 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
2471 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2472 emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2473 }
2475 void Assembler::punpckldq(XMMRegister dst, Address src) {
2476 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2477 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2478 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2479 }
2481 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
2482 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2483 emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2484 }
2486 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
2487 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2488 emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
2489 }
2491 void Assembler::push(int32_t imm32) {
2492 // in 64bits we push 64bits onto the stack but only
2493 // take a 32bit immediate
2494 emit_byte(0x68);
2495 emit_long(imm32);
2496 }
2498 void Assembler::push(Register src) {
2499 int encode = prefix_and_encode(src->encoding());
2501 emit_byte(0x50 | encode);
2502 }
2504 void Assembler::pushf() {
2505 emit_byte(0x9C);
2506 }
2508 #ifndef _LP64 // no 32bit push/pop on amd64
2509 void Assembler::pushl(Address src) {
2510 // Note this will push 64bit on 64bit
2511 InstructionMark im(this);
2512 prefix(src);
2513 emit_byte(0xFF);
2514 emit_operand(rsi, src);
2515 }
2516 #endif
2518 void Assembler::rcll(Register dst, int imm8) {
2519 assert(isShiftCount(imm8), "illegal shift count");
2520 int encode = prefix_and_encode(dst->encoding());
2521 if (imm8 == 1) {
2522 emit_byte(0xD1);
2523 emit_byte(0xD0 | encode);
2524 } else {
2525 emit_byte(0xC1);
2526 emit_byte(0xD0 | encode);
2527 emit_byte(imm8);
2528 }
2529 }
2531 // copies data from [esi] to [edi] using rcx pointer sized words
2532 // generic
2533 void Assembler::rep_mov() {
2534 emit_byte(0xF3);
2535 // MOVSQ
2536 LP64_ONLY(prefix(REX_W));
2537 emit_byte(0xA5);
2538 }
2540 // sets rcx pointer sized words with rax, value at [edi]
2541 // generic
2542 void Assembler::rep_set() { // rep_set
2543 emit_byte(0xF3);
2544 // STOSQ
2545 LP64_ONLY(prefix(REX_W));
2546 emit_byte(0xAB);
2547 }
2549 // scans rcx pointer sized words at [edi] for occurance of rax,
2550 // generic
2551 void Assembler::repne_scan() { // repne_scan
2552 emit_byte(0xF2);
2553 // SCASQ
2554 LP64_ONLY(prefix(REX_W));
2555 emit_byte(0xAF);
2556 }
2558 #ifdef _LP64
2559 // scans rcx 4 byte words at [edi] for occurance of rax,
2560 // generic
2561 void Assembler::repne_scanl() { // repne_scan
2562 emit_byte(0xF2);
2563 // SCASL
2564 emit_byte(0xAF);
2565 }
2566 #endif
2568 void Assembler::ret(int imm16) {
2569 if (imm16 == 0) {
2570 emit_byte(0xC3);
2571 } else {
2572 emit_byte(0xC2);
2573 emit_int16(imm16);
2574 }
2575 }
2577 void Assembler::sahf() {
2578 #ifdef _LP64
2579 // Not supported in 64bit mode
2580 ShouldNotReachHere();
2581 #endif
2582 emit_byte(0x9E);
2583 }
2585 void Assembler::sarl(Register dst, int imm8) {
2586 int encode = prefix_and_encode(dst->encoding());
2587 assert(isShiftCount(imm8), "illegal shift count");
2588 if (imm8 == 1) {
2589 emit_byte(0xD1);
2590 emit_byte(0xF8 | encode);
2591 } else {
2592 emit_byte(0xC1);
2593 emit_byte(0xF8 | encode);
2594 emit_byte(imm8);
2595 }
2596 }
2598 void Assembler::sarl(Register dst) {
2599 int encode = prefix_and_encode(dst->encoding());
2600 emit_byte(0xD3);
2601 emit_byte(0xF8 | encode);
2602 }
2604 void Assembler::sbbl(Address dst, int32_t imm32) {
2605 InstructionMark im(this);
2606 prefix(dst);
2607 emit_arith_operand(0x81, rbx, dst, imm32);
2608 }
2610 void Assembler::sbbl(Register dst, int32_t imm32) {
2611 prefix(dst);
2612 emit_arith(0x81, 0xD8, dst, imm32);
2613 }
2616 void Assembler::sbbl(Register dst, Address src) {
2617 InstructionMark im(this);
2618 prefix(src, dst);
2619 emit_byte(0x1B);
2620 emit_operand(dst, src);
2621 }
2623 void Assembler::sbbl(Register dst, Register src) {
2624 (void) prefix_and_encode(dst->encoding(), src->encoding());
2625 emit_arith(0x1B, 0xC0, dst, src);
2626 }
2628 void Assembler::setb(Condition cc, Register dst) {
2629 assert(0 <= cc && cc < 16, "illegal cc");
2630 int encode = prefix_and_encode(dst->encoding(), true);
2631 emit_byte(0x0F);
2632 emit_byte(0x90 | cc);
2633 emit_byte(0xC0 | encode);
2634 }
2636 void Assembler::shll(Register dst, int imm8) {
2637 assert(isShiftCount(imm8), "illegal shift count");
2638 int encode = prefix_and_encode(dst->encoding());
2639 if (imm8 == 1 ) {
2640 emit_byte(0xD1);
2641 emit_byte(0xE0 | encode);
2642 } else {
2643 emit_byte(0xC1);
2644 emit_byte(0xE0 | encode);
2645 emit_byte(imm8);
2646 }
2647 }
2649 void Assembler::shll(Register dst) {
2650 int encode = prefix_and_encode(dst->encoding());
2651 emit_byte(0xD3);
2652 emit_byte(0xE0 | encode);
2653 }
2655 void Assembler::shrl(Register dst, int imm8) {
2656 assert(isShiftCount(imm8), "illegal shift count");
2657 int encode = prefix_and_encode(dst->encoding());
2658 emit_byte(0xC1);
2659 emit_byte(0xE8 | encode);
2660 emit_byte(imm8);
2661 }
2663 void Assembler::shrl(Register dst) {
2664 int encode = prefix_and_encode(dst->encoding());
2665 emit_byte(0xD3);
2666 emit_byte(0xE8 | encode);
2667 }
2669 // copies a single word from [esi] to [edi]
2670 void Assembler::smovl() {
2671 emit_byte(0xA5);
2672 }
2674 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
2675 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2676 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2677 }
2679 void Assembler::sqrtsd(XMMRegister dst, Address src) {
2680 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2681 emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2682 }
2684 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
2685 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2686 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2687 }
2689 void Assembler::std() {
2690 emit_byte(0xfd);
2691 }
2693 void Assembler::sqrtss(XMMRegister dst, Address src) {
2694 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2695 emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2696 }
2698 void Assembler::stmxcsr( Address dst) {
2699 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2700 InstructionMark im(this);
2701 prefix(dst);
2702 emit_byte(0x0F);
2703 emit_byte(0xAE);
2704 emit_operand(as_Register(3), dst);
2705 }
2707 void Assembler::subl(Address dst, int32_t imm32) {
2708 InstructionMark im(this);
2709 prefix(dst);
2710 emit_arith_operand(0x81, rbp, dst, imm32);
2711 }
2713 void Assembler::subl(Address dst, Register src) {
2714 InstructionMark im(this);
2715 prefix(dst, src);
2716 emit_byte(0x29);
2717 emit_operand(src, dst);
2718 }
2720 void Assembler::subl(Register dst, int32_t imm32) {
2721 prefix(dst);
2722 emit_arith(0x81, 0xE8, dst, imm32);
2723 }
2725 // Force generation of a 4 byte immediate value even if it fits into 8bit
2726 void Assembler::subl_imm32(Register dst, int32_t imm32) {
2727 prefix(dst);
2728 emit_arith_imm32(0x81, 0xE8, dst, imm32);
2729 }
2731 void Assembler::subl(Register dst, Address src) {
2732 InstructionMark im(this);
2733 prefix(src, dst);
2734 emit_byte(0x2B);
2735 emit_operand(dst, src);
2736 }
2738 void Assembler::subl(Register dst, Register src) {
2739 (void) prefix_and_encode(dst->encoding(), src->encoding());
2740 emit_arith(0x2B, 0xC0, dst, src);
2741 }
2743 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
2744 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2745 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2746 }
2748 void Assembler::subsd(XMMRegister dst, Address src) {
2749 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2750 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2751 }
2753 void Assembler::subss(XMMRegister dst, XMMRegister src) {
2754 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2755 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2756 }
2758 void Assembler::subss(XMMRegister dst, Address src) {
2759 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2760 emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2761 }
2763 void Assembler::testb(Register dst, int imm8) {
2764 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2765 (void) prefix_and_encode(dst->encoding(), true);
2766 emit_arith_b(0xF6, 0xC0, dst, imm8);
2767 }
2769 void Assembler::testl(Register dst, int32_t imm32) {
2770 // not using emit_arith because test
2771 // doesn't support sign-extension of
2772 // 8bit operands
2773 int encode = dst->encoding();
2774 if (encode == 0) {
2775 emit_byte(0xA9);
2776 } else {
2777 encode = prefix_and_encode(encode);
2778 emit_byte(0xF7);
2779 emit_byte(0xC0 | encode);
2780 }
2781 emit_long(imm32);
2782 }
2784 void Assembler::testl(Register dst, Register src) {
2785 (void) prefix_and_encode(dst->encoding(), src->encoding());
2786 emit_arith(0x85, 0xC0, dst, src);
2787 }
2789 void Assembler::testl(Register dst, Address src) {
2790 InstructionMark im(this);
2791 prefix(src, dst);
2792 emit_byte(0x85);
2793 emit_operand(dst, src);
2794 }
2796 void Assembler::ucomisd(XMMRegister dst, Address src) {
2797 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2798 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2799 }
2801 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
2802 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2803 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2804 }
2806 void Assembler::ucomiss(XMMRegister dst, Address src) {
2807 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2808 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2809 }
2811 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2812 NOT_LP64(assert(VM_Version::supports_sse(), ""));
2813 emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2814 }
2817 void Assembler::xaddl(Address dst, Register src) {
2818 InstructionMark im(this);
2819 prefix(dst, src);
2820 emit_byte(0x0F);
2821 emit_byte(0xC1);
2822 emit_operand(src, dst);
2823 }
2825 void Assembler::xchgl(Register dst, Address src) { // xchg
2826 InstructionMark im(this);
2827 prefix(src, dst);
2828 emit_byte(0x87);
2829 emit_operand(dst, src);
2830 }
2832 void Assembler::xchgl(Register dst, Register src) {
2833 int encode = prefix_and_encode(dst->encoding(), src->encoding());
2834 emit_byte(0x87);
2835 emit_byte(0xc0 | encode);
2836 }
2838 void Assembler::xgetbv() {
2839 emit_byte(0x0F);
2840 emit_byte(0x01);
2841 emit_byte(0xD0);
2842 }
2844 void Assembler::xorl(Register dst, int32_t imm32) {
2845 prefix(dst);
2846 emit_arith(0x81, 0xF0, dst, imm32);
2847 }
2849 void Assembler::xorl(Register dst, Address src) {
2850 InstructionMark im(this);
2851 prefix(src, dst);
2852 emit_byte(0x33);
2853 emit_operand(dst, src);
2854 }
2856 void Assembler::xorl(Register dst, Register src) {
2857 (void) prefix_and_encode(dst->encoding(), src->encoding());
2858 emit_arith(0x33, 0xC0, dst, src);
2859 }
2862 // AVX 3-operands scalar float-point arithmetic instructions
2864 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
2865 assert(VM_Version::supports_avx(), "");
2866 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2867 }
2869 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2870 assert(VM_Version::supports_avx(), "");
2871 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2872 }
2874 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
2875 assert(VM_Version::supports_avx(), "");
2876 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2877 }
2879 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2880 assert(VM_Version::supports_avx(), "");
2881 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2882 }
2884 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
2885 assert(VM_Version::supports_avx(), "");
2886 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2887 }
2889 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2890 assert(VM_Version::supports_avx(), "");
2891 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2892 }
2894 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
2895 assert(VM_Version::supports_avx(), "");
2896 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2897 }
2899 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2900 assert(VM_Version::supports_avx(), "");
2901 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2902 }
2904 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
2905 assert(VM_Version::supports_avx(), "");
2906 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2907 }
2909 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2910 assert(VM_Version::supports_avx(), "");
2911 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2912 }
2914 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
2915 assert(VM_Version::supports_avx(), "");
2916 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2917 }
2919 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2920 assert(VM_Version::supports_avx(), "");
2921 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2922 }
2924 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
2925 assert(VM_Version::supports_avx(), "");
2926 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2927 }
2929 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2930 assert(VM_Version::supports_avx(), "");
2931 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2932 }
2934 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
2935 assert(VM_Version::supports_avx(), "");
2936 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2937 }
2939 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2940 assert(VM_Version::supports_avx(), "");
2941 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2942 }
2944 //====================VECTOR ARITHMETIC=====================================
2946 // Float-point vector arithmetic
2948 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
2949 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2950 emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
2951 }
2953 void Assembler::addps(XMMRegister dst, XMMRegister src) {
2954 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2955 emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
2956 }
2958 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2959 assert(VM_Version::supports_avx(), "");
2960 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
2961 }
2963 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2964 assert(VM_Version::supports_avx(), "");
2965 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
2966 }
2968 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
2969 assert(VM_Version::supports_avx(), "");
2970 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
2971 }
2973 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
2974 assert(VM_Version::supports_avx(), "");
2975 emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
2976 }
2978 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
2979 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2980 emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
2981 }
2983 void Assembler::subps(XMMRegister dst, XMMRegister src) {
2984 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2985 emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
2986 }
2988 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2989 assert(VM_Version::supports_avx(), "");
2990 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
2991 }
2993 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2994 assert(VM_Version::supports_avx(), "");
2995 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
2996 }
2998 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
2999 assert(VM_Version::supports_avx(), "");
3000 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
3001 }
3003 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3004 assert(VM_Version::supports_avx(), "");
3005 emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
3006 }
3008 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
3009 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3010 emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
3011 }
3013 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
3014 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3015 emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
3016 }
3018 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3019 assert(VM_Version::supports_avx(), "");
3020 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3021 }
3023 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3024 assert(VM_Version::supports_avx(), "");
3025 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3026 }
3028 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3029 assert(VM_Version::supports_avx(), "");
3030 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3031 }
3033 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3034 assert(VM_Version::supports_avx(), "");
3035 emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3036 }
3038 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
3039 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3040 emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
3041 }
3043 void Assembler::divps(XMMRegister dst, XMMRegister src) {
3044 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3045 emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
3046 }
3048 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3049 assert(VM_Version::supports_avx(), "");
3050 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3051 }
3053 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3054 assert(VM_Version::supports_avx(), "");
3055 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3056 }
3058 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3059 assert(VM_Version::supports_avx(), "");
3060 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3061 }
3063 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3064 assert(VM_Version::supports_avx(), "");
3065 emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3066 }
3068 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3069 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3070 emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3071 }
3073 void Assembler::andps(XMMRegister dst, XMMRegister src) {
3074 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3075 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3076 }
3078 void Assembler::andps(XMMRegister dst, Address src) {
3079 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3080 emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3081 }
3083 void Assembler::andpd(XMMRegister dst, Address src) {
3084 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3085 emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3086 }
3088 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3089 assert(VM_Version::supports_avx(), "");
3090 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3091 }
3093 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3094 assert(VM_Version::supports_avx(), "");
3095 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3096 }
3098 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3099 assert(VM_Version::supports_avx(), "");
3100 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3101 }
3103 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3104 assert(VM_Version::supports_avx(), "");
3105 emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3106 }
3108 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3109 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3110 emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3111 }
3113 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
3114 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3115 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3116 }
3118 void Assembler::xorpd(XMMRegister dst, Address src) {
3119 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3120 emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3121 }
3123 void Assembler::xorps(XMMRegister dst, Address src) {
3124 NOT_LP64(assert(VM_Version::supports_sse(), ""));
3125 emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3126 }
3128 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3129 assert(VM_Version::supports_avx(), "");
3130 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3131 }
3133 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3134 assert(VM_Version::supports_avx(), "");
3135 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3136 }
3138 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3139 assert(VM_Version::supports_avx(), "");
3140 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3141 }
3143 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3144 assert(VM_Version::supports_avx(), "");
3145 emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3146 }
3149 // Integer vector arithmetic
3150 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
3151 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3152 emit_simd_arith(0xFC, dst, src, VEX_SIMD_66);
3153 }
3155 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
3156 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3157 emit_simd_arith(0xFD, dst, src, VEX_SIMD_66);
3158 }
3160 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
3161 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3162 emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
3163 }
3165 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
3166 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3167 emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
3168 }
3170 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3171 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3172 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3173 }
3175 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3176 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3177 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3178 }
3180 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3181 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3182 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3183 }
3185 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3186 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3187 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3188 }
3190 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3191 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3192 emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3193 }
3195 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3196 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3197 emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3198 }
3200 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3201 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3202 emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3203 }
3205 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3206 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3207 emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3208 }
3210 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
3211 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3212 emit_simd_arith(0xF8, dst, src, VEX_SIMD_66);
3213 }
3215 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
3216 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3217 emit_simd_arith(0xF9, dst, src, VEX_SIMD_66);
3218 }
3220 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
3221 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3222 emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
3223 }
3225 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
3226 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3227 emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
3228 }
3230 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3231 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3232 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3233 }
3235 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3236 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3237 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3238 }
3240 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3241 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3242 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3243 }
3245 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3246 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3247 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3248 }
3250 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3251 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3252 emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3253 }
3255 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3256 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3257 emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3258 }
3260 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3261 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3262 emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3263 }
3265 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3266 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3267 emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3268 }
3270 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
3271 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3272 emit_simd_arith(0xD5, dst, src, VEX_SIMD_66);
3273 }
3275 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
3276 assert(VM_Version::supports_sse4_1(), "");
3277 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
3278 emit_byte(0x40);
3279 emit_byte(0xC0 | encode);
3280 }
3282 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3283 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3284 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3285 }
3287 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3288 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3289 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
3290 emit_byte(0x40);
3291 emit_byte(0xC0 | encode);
3292 }
3294 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3295 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3296 emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3297 }
3299 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3300 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3301 InstructionMark im(this);
3302 int dst_enc = dst->encoding();
3303 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3304 vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
3305 emit_byte(0x40);
3306 emit_operand(dst, src);
3307 }
3309 // Shift packed integers left by specified number of bits.
3310 void Assembler::psllw(XMMRegister dst, int shift) {
3311 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3312 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3313 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3314 emit_byte(0x71);
3315 emit_byte(0xC0 | encode);
3316 emit_byte(shift & 0xFF);
3317 }
3319 void Assembler::pslld(XMMRegister dst, int shift) {
3320 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3321 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3322 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3323 emit_byte(0x72);
3324 emit_byte(0xC0 | encode);
3325 emit_byte(shift & 0xFF);
3326 }
3328 void Assembler::psllq(XMMRegister dst, int shift) {
3329 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3330 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3331 int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3332 emit_byte(0x73);
3333 emit_byte(0xC0 | encode);
3334 emit_byte(shift & 0xFF);
3335 }
3337 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
3338 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3339 emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66);
3340 }
3342 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
3343 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3344 emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
3345 }
3347 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
3348 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3349 emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
3350 }
3352 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3353 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3354 // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3355 emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector256);
3356 emit_byte(shift & 0xFF);
3357 }
3359 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3360 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3361 // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3362 emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector256);
3363 emit_byte(shift & 0xFF);
3364 }
3366 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3367 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3368 // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3369 emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector256);
3370 emit_byte(shift & 0xFF);
3371 }
3373 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3374 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3375 emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector256);
3376 }
3378 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3379 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3380 emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector256);
3381 }
3383 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3384 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3385 emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector256);
3386 }
3388 // Shift packed integers logically right by specified number of bits.
3389 void Assembler::psrlw(XMMRegister dst, int shift) {
3390 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3391 // XMM2 is for /2 encoding: 66 0F 71 /2 ib
3392 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3393 emit_byte(0x71);
3394 emit_byte(0xC0 | encode);
3395 emit_byte(shift & 0xFF);
3396 }
3398 void Assembler::psrld(XMMRegister dst, int shift) {
3399 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3400 // XMM2 is for /2 encoding: 66 0F 72 /2 ib
3401 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3402 emit_byte(0x72);
3403 emit_byte(0xC0 | encode);
3404 emit_byte(shift & 0xFF);
3405 }
3407 void Assembler::psrlq(XMMRegister dst, int shift) {
3408 // Do not confuse it with psrldq SSE2 instruction which
3409 // shifts 128 bit value in xmm register by number of bytes.
3410 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3411 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3412 int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3413 emit_byte(0x73);
3414 emit_byte(0xC0 | encode);
3415 emit_byte(shift & 0xFF);
3416 }
3418 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
3419 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3420 emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66);
3421 }
3423 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
3424 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3425 emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
3426 }
3428 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
3429 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3430 emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
3431 }
3433 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3434 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3435 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3436 emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector256);
3437 emit_byte(shift & 0xFF);
3438 }
3440 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3441 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3442 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3443 emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector256);
3444 emit_byte(shift & 0xFF);
3445 }
3447 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3448 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3449 // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3450 emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector256);
3451 emit_byte(shift & 0xFF);
3452 }
3454 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3455 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3456 emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector256);
3457 }
3459 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3460 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3461 emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector256);
3462 }
3464 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3465 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3466 emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector256);
3467 }
3469 // Shift packed integers arithmetically right by specified number of bits.
3470 void Assembler::psraw(XMMRegister dst, int shift) {
3471 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3472 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3473 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3474 emit_byte(0x71);
3475 emit_byte(0xC0 | encode);
3476 emit_byte(shift & 0xFF);
3477 }
3479 void Assembler::psrad(XMMRegister dst, int shift) {
3480 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3481 // XMM4 is for /4 encoding: 66 0F 72 /4 ib
3482 int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3483 emit_byte(0x72);
3484 emit_byte(0xC0 | encode);
3485 emit_byte(shift & 0xFF);
3486 }
3488 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
3489 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3490 emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66);
3491 }
3493 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
3494 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3495 emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
3496 }
3498 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3499 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3500 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3501 emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector256);
3502 emit_byte(shift & 0xFF);
3503 }
3505 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3506 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3507 // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3508 emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector256);
3509 emit_byte(shift & 0xFF);
3510 }
3512 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3513 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3514 emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector256);
3515 }
3517 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3518 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3519 emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector256);
3520 }
3523 // AND packed integers
3524 void Assembler::pand(XMMRegister dst, XMMRegister src) {
3525 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3526 emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
3527 }
3529 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3530 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3531 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3532 }
3534 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3535 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3536 emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3537 }
3539 void Assembler::por(XMMRegister dst, XMMRegister src) {
3540 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3541 emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
3542 }
3544 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3545 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3546 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3547 }
3549 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3550 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3551 emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3552 }
3554 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
3555 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3556 emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
3557 }
3559 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3560 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3561 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3562 }
3564 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3565 assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3566 emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3567 }
3570 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3571 assert(VM_Version::supports_avx(), "");
3572 bool vector256 = true;
3573 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3574 emit_byte(0x18);
3575 emit_byte(0xC0 | encode);
3576 // 0x00 - insert into lower 128 bits
3577 // 0x01 - insert into upper 128 bits
3578 emit_byte(0x01);
3579 }
3581 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
3582 assert(VM_Version::supports_avx(), "");
3583 InstructionMark im(this);
3584 bool vector256 = true;
3585 assert(dst != xnoreg, "sanity");
3586 int dst_enc = dst->encoding();
3587 // swap src<->dst for encoding
3588 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3589 emit_byte(0x18);
3590 emit_operand(dst, src);
3591 // 0x01 - insert into upper 128 bits
3592 emit_byte(0x01);
3593 }
3595 void Assembler::vextractf128h(Address dst, XMMRegister src) {
3596 assert(VM_Version::supports_avx(), "");
3597 InstructionMark im(this);
3598 bool vector256 = true;
3599 assert(src != xnoreg, "sanity");
3600 int src_enc = src->encoding();
3601 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3602 emit_byte(0x19);
3603 emit_operand(src, dst);
3604 // 0x01 - extract from upper 128 bits
3605 emit_byte(0x01);
3606 }
3608 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3609 assert(VM_Version::supports_avx2(), "");
3610 bool vector256 = true;
3611 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3612 emit_byte(0x38);
3613 emit_byte(0xC0 | encode);
3614 // 0x00 - insert into lower 128 bits
3615 // 0x01 - insert into upper 128 bits
3616 emit_byte(0x01);
3617 }
3619 void Assembler::vinserti128h(XMMRegister dst, Address src) {
3620 assert(VM_Version::supports_avx2(), "");
3621 InstructionMark im(this);
3622 bool vector256 = true;
3623 assert(dst != xnoreg, "sanity");
3624 int dst_enc = dst->encoding();
3625 // swap src<->dst for encoding
3626 vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3627 emit_byte(0x38);
3628 emit_operand(dst, src);
3629 // 0x01 - insert into upper 128 bits
3630 emit_byte(0x01);
3631 }
3633 void Assembler::vextracti128h(Address dst, XMMRegister src) {
3634 assert(VM_Version::supports_avx2(), "");
3635 InstructionMark im(this);
3636 bool vector256 = true;
3637 assert(src != xnoreg, "sanity");
3638 int src_enc = src->encoding();
3639 vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3640 emit_byte(0x39);
3641 emit_operand(src, dst);
3642 // 0x01 - extract from upper 128 bits
3643 emit_byte(0x01);
3644 }
3646 void Assembler::vzeroupper() {
3647 assert(VM_Version::supports_avx(), "");
3648 (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
3649 emit_byte(0x77);
3650 }
3653 #ifndef _LP64
3654 // 32bit only pieces of the assembler
3656 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
3657 // NO PREFIX AS NEVER 64BIT
3658 InstructionMark im(this);
3659 emit_byte(0x81);
3660 emit_byte(0xF8 | src1->encoding());
3661 emit_data(imm32, rspec, 0);
3662 }
3664 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
3665 // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
3666 InstructionMark im(this);
3667 emit_byte(0x81);
3668 emit_operand(rdi, src1);
3669 emit_data(imm32, rspec, 0);
3670 }
3672 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
3673 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
3674 // into rdx:rax. The ZF is set if the compared values were equal, and cleared otherwise.
3675 void Assembler::cmpxchg8(Address adr) {
3676 InstructionMark im(this);
3677 emit_byte(0x0F);
3678 emit_byte(0xc7);
3679 emit_operand(rcx, adr);
3680 }
3682 void Assembler::decl(Register dst) {
3683 // Don't use it directly. Use MacroAssembler::decrementl() instead.
3684 emit_byte(0x48 | dst->encoding());
3685 }
3687 #endif // _LP64
3689 // 64bit typically doesn't use the x87 but needs to for the trig funcs
3691 void Assembler::fabs() {
3692 emit_byte(0xD9);
3693 emit_byte(0xE1);
3694 }
3696 void Assembler::fadd(int i) {
3697 emit_farith(0xD8, 0xC0, i);
3698 }
3700 void Assembler::fadd_d(Address src) {
3701 InstructionMark im(this);
3702 emit_byte(0xDC);
3703 emit_operand32(rax, src);
3704 }
3706 void Assembler::fadd_s(Address src) {
3707 InstructionMark im(this);
3708 emit_byte(0xD8);
3709 emit_operand32(rax, src);
3710 }
3712 void Assembler::fadda(int i) {
3713 emit_farith(0xDC, 0xC0, i);
3714 }
3716 void Assembler::faddp(int i) {
3717 emit_farith(0xDE, 0xC0, i);
3718 }
3720 void Assembler::fchs() {
3721 emit_byte(0xD9);
3722 emit_byte(0xE0);
3723 }
3725 void Assembler::fcom(int i) {
3726 emit_farith(0xD8, 0xD0, i);
3727 }
3729 void Assembler::fcomp(int i) {
3730 emit_farith(0xD8, 0xD8, i);
3731 }
3733 void Assembler::fcomp_d(Address src) {
3734 InstructionMark im(this);
3735 emit_byte(0xDC);
3736 emit_operand32(rbx, src);
3737 }
3739 void Assembler::fcomp_s(Address src) {
3740 InstructionMark im(this);
3741 emit_byte(0xD8);
3742 emit_operand32(rbx, src);
3743 }
3745 void Assembler::fcompp() {
3746 emit_byte(0xDE);
3747 emit_byte(0xD9);
3748 }
3750 void Assembler::fcos() {
3751 emit_byte(0xD9);
3752 emit_byte(0xFF);
3753 }
3755 void Assembler::fdecstp() {
3756 emit_byte(0xD9);
3757 emit_byte(0xF6);
3758 }
3760 void Assembler::fdiv(int i) {
3761 emit_farith(0xD8, 0xF0, i);
3762 }
3764 void Assembler::fdiv_d(Address src) {
3765 InstructionMark im(this);
3766 emit_byte(0xDC);
3767 emit_operand32(rsi, src);
3768 }
3770 void Assembler::fdiv_s(Address src) {
3771 InstructionMark im(this);
3772 emit_byte(0xD8);
3773 emit_operand32(rsi, src);
3774 }
3776 void Assembler::fdiva(int i) {
3777 emit_farith(0xDC, 0xF8, i);
3778 }
3780 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
3781 // is erroneous for some of the floating-point instructions below.
3783 void Assembler::fdivp(int i) {
3784 emit_farith(0xDE, 0xF8, i); // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
3785 }
3787 void Assembler::fdivr(int i) {
3788 emit_farith(0xD8, 0xF8, i);
3789 }
3791 void Assembler::fdivr_d(Address src) {
3792 InstructionMark im(this);
3793 emit_byte(0xDC);
3794 emit_operand32(rdi, src);
3795 }
3797 void Assembler::fdivr_s(Address src) {
3798 InstructionMark im(this);
3799 emit_byte(0xD8);
3800 emit_operand32(rdi, src);
3801 }
3803 void Assembler::fdivra(int i) {
3804 emit_farith(0xDC, 0xF0, i);
3805 }
3807 void Assembler::fdivrp(int i) {
3808 emit_farith(0xDE, 0xF0, i); // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
3809 }
3811 void Assembler::ffree(int i) {
3812 emit_farith(0xDD, 0xC0, i);
3813 }
3815 void Assembler::fild_d(Address adr) {
3816 InstructionMark im(this);
3817 emit_byte(0xDF);
3818 emit_operand32(rbp, adr);
3819 }
3821 void Assembler::fild_s(Address adr) {
3822 InstructionMark im(this);
3823 emit_byte(0xDB);
3824 emit_operand32(rax, adr);
3825 }
3827 void Assembler::fincstp() {
3828 emit_byte(0xD9);
3829 emit_byte(0xF7);
3830 }
3832 void Assembler::finit() {
3833 emit_byte(0x9B);
3834 emit_byte(0xDB);
3835 emit_byte(0xE3);
3836 }
3838 void Assembler::fist_s(Address adr) {
3839 InstructionMark im(this);
3840 emit_byte(0xDB);
3841 emit_operand32(rdx, adr);
3842 }
3844 void Assembler::fistp_d(Address adr) {
3845 InstructionMark im(this);
3846 emit_byte(0xDF);
3847 emit_operand32(rdi, adr);
3848 }
3850 void Assembler::fistp_s(Address adr) {
3851 InstructionMark im(this);
3852 emit_byte(0xDB);
3853 emit_operand32(rbx, adr);
3854 }
3856 void Assembler::fld1() {
3857 emit_byte(0xD9);
3858 emit_byte(0xE8);
3859 }
3861 void Assembler::fld_d(Address adr) {
3862 InstructionMark im(this);
3863 emit_byte(0xDD);
3864 emit_operand32(rax, adr);
3865 }
3867 void Assembler::fld_s(Address adr) {
3868 InstructionMark im(this);
3869 emit_byte(0xD9);
3870 emit_operand32(rax, adr);
3871 }
3874 void Assembler::fld_s(int index) {
3875 emit_farith(0xD9, 0xC0, index);
3876 }
3878 void Assembler::fld_x(Address adr) {
3879 InstructionMark im(this);
3880 emit_byte(0xDB);
3881 emit_operand32(rbp, adr);
3882 }
3884 void Assembler::fldcw(Address src) {
3885 InstructionMark im(this);
3886 emit_byte(0xd9);
3887 emit_operand32(rbp, src);
3888 }
3890 void Assembler::fldenv(Address src) {
3891 InstructionMark im(this);
3892 emit_byte(0xD9);
3893 emit_operand32(rsp, src);
3894 }
3896 void Assembler::fldlg2() {
3897 emit_byte(0xD9);
3898 emit_byte(0xEC);
3899 }
3901 void Assembler::fldln2() {
3902 emit_byte(0xD9);
3903 emit_byte(0xED);
3904 }
3906 void Assembler::fldz() {
3907 emit_byte(0xD9);
3908 emit_byte(0xEE);
3909 }
3911 void Assembler::flog() {
3912 fldln2();
3913 fxch();
3914 fyl2x();
3915 }
3917 void Assembler::flog10() {
3918 fldlg2();
3919 fxch();
3920 fyl2x();
3921 }
3923 void Assembler::fmul(int i) {
3924 emit_farith(0xD8, 0xC8, i);
3925 }
3927 void Assembler::fmul_d(Address src) {
3928 InstructionMark im(this);
3929 emit_byte(0xDC);
3930 emit_operand32(rcx, src);
3931 }
3933 void Assembler::fmul_s(Address src) {
3934 InstructionMark im(this);
3935 emit_byte(0xD8);
3936 emit_operand32(rcx, src);
3937 }
3939 void Assembler::fmula(int i) {
3940 emit_farith(0xDC, 0xC8, i);
3941 }
3943 void Assembler::fmulp(int i) {
3944 emit_farith(0xDE, 0xC8, i);
3945 }
3947 void Assembler::fnsave(Address dst) {
3948 InstructionMark im(this);
3949 emit_byte(0xDD);
3950 emit_operand32(rsi, dst);
3951 }
3953 void Assembler::fnstcw(Address src) {
3954 InstructionMark im(this);
3955 emit_byte(0x9B);
3956 emit_byte(0xD9);
3957 emit_operand32(rdi, src);
3958 }
3960 void Assembler::fnstsw_ax() {
3961 emit_byte(0xdF);
3962 emit_byte(0xE0);
3963 }
3965 void Assembler::fprem() {
3966 emit_byte(0xD9);
3967 emit_byte(0xF8);
3968 }
3970 void Assembler::fprem1() {
3971 emit_byte(0xD9);
3972 emit_byte(0xF5);
3973 }
3975 void Assembler::frstor(Address src) {
3976 InstructionMark im(this);
3977 emit_byte(0xDD);
3978 emit_operand32(rsp, src);
3979 }
3981 void Assembler::fsin() {
3982 emit_byte(0xD9);
3983 emit_byte(0xFE);
3984 }
3986 void Assembler::fsqrt() {
3987 emit_byte(0xD9);
3988 emit_byte(0xFA);
3989 }
3991 void Assembler::fst_d(Address adr) {
3992 InstructionMark im(this);
3993 emit_byte(0xDD);
3994 emit_operand32(rdx, adr);
3995 }
3997 void Assembler::fst_s(Address adr) {
3998 InstructionMark im(this);
3999 emit_byte(0xD9);
4000 emit_operand32(rdx, adr);
4001 }
4003 void Assembler::fstp_d(Address adr) {
4004 InstructionMark im(this);
4005 emit_byte(0xDD);
4006 emit_operand32(rbx, adr);
4007 }
4009 void Assembler::fstp_d(int index) {
4010 emit_farith(0xDD, 0xD8, index);
4011 }
4013 void Assembler::fstp_s(Address adr) {
4014 InstructionMark im(this);
4015 emit_byte(0xD9);
4016 emit_operand32(rbx, adr);
4017 }
4019 void Assembler::fstp_x(Address adr) {
4020 InstructionMark im(this);
4021 emit_byte(0xDB);
4022 emit_operand32(rdi, adr);
4023 }
4025 void Assembler::fsub(int i) {
4026 emit_farith(0xD8, 0xE0, i);
4027 }
4029 void Assembler::fsub_d(Address src) {
4030 InstructionMark im(this);
4031 emit_byte(0xDC);
4032 emit_operand32(rsp, src);
4033 }
4035 void Assembler::fsub_s(Address src) {
4036 InstructionMark im(this);
4037 emit_byte(0xD8);
4038 emit_operand32(rsp, src);
4039 }
4041 void Assembler::fsuba(int i) {
4042 emit_farith(0xDC, 0xE8, i);
4043 }
4045 void Assembler::fsubp(int i) {
4046 emit_farith(0xDE, 0xE8, i); // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
4047 }
4049 void Assembler::fsubr(int i) {
4050 emit_farith(0xD8, 0xE8, i);
4051 }
4053 void Assembler::fsubr_d(Address src) {
4054 InstructionMark im(this);
4055 emit_byte(0xDC);
4056 emit_operand32(rbp, src);
4057 }
4059 void Assembler::fsubr_s(Address src) {
4060 InstructionMark im(this);
4061 emit_byte(0xD8);
4062 emit_operand32(rbp, src);
4063 }
4065 void Assembler::fsubra(int i) {
4066 emit_farith(0xDC, 0xE0, i);
4067 }
4069 void Assembler::fsubrp(int i) {
4070 emit_farith(0xDE, 0xE0, i); // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
4071 }
4073 void Assembler::ftan() {
4074 emit_byte(0xD9);
4075 emit_byte(0xF2);
4076 emit_byte(0xDD);
4077 emit_byte(0xD8);
4078 }
4080 void Assembler::ftst() {
4081 emit_byte(0xD9);
4082 emit_byte(0xE4);
4083 }
4085 void Assembler::fucomi(int i) {
4086 // make sure the instruction is supported (introduced for P6, together with cmov)
4087 guarantee(VM_Version::supports_cmov(), "illegal instruction");
4088 emit_farith(0xDB, 0xE8, i);
4089 }
4091 void Assembler::fucomip(int i) {
4092 // make sure the instruction is supported (introduced for P6, together with cmov)
4093 guarantee(VM_Version::supports_cmov(), "illegal instruction");
4094 emit_farith(0xDF, 0xE8, i);
4095 }
4097 void Assembler::fwait() {
4098 emit_byte(0x9B);
4099 }
4101 void Assembler::fxch(int i) {
4102 emit_farith(0xD9, 0xC8, i);
4103 }
4105 void Assembler::fyl2x() {
4106 emit_byte(0xD9);
4107 emit_byte(0xF1);
4108 }
4110 void Assembler::frndint() {
4111 emit_byte(0xD9);
4112 emit_byte(0xFC);
4113 }
4115 void Assembler::f2xm1() {
4116 emit_byte(0xD9);
4117 emit_byte(0xF0);
4118 }
4120 void Assembler::fldl2e() {
4121 emit_byte(0xD9);
4122 emit_byte(0xEA);
4123 }
4125 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
4126 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
4127 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
4128 static int simd_opc[4] = { 0, 0, 0x38, 0x3A };
4130 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
4131 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4132 if (pre > 0) {
4133 emit_byte(simd_pre[pre]);
4134 }
4135 if (rex_w) {
4136 prefixq(adr, xreg);
4137 } else {
4138 prefix(adr, xreg);
4139 }
4140 if (opc > 0) {
4141 emit_byte(0x0F);
4142 int opc2 = simd_opc[opc];
4143 if (opc2 > 0) {
4144 emit_byte(opc2);
4145 }
4146 }
4147 }
4149 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4150 if (pre > 0) {
4151 emit_byte(simd_pre[pre]);
4152 }
4153 int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
4154 prefix_and_encode(dst_enc, src_enc);
4155 if (opc > 0) {
4156 emit_byte(0x0F);
4157 int opc2 = simd_opc[opc];
4158 if (opc2 > 0) {
4159 emit_byte(opc2);
4160 }
4161 }
4162 return encode;
4163 }
4166 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool vector256) {
4167 if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
4168 prefix(VEX_3bytes);
4170 int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
4171 byte1 = (~byte1) & 0xE0;
4172 byte1 |= opc;
4173 a_byte(byte1);
4175 int byte2 = ((~nds_enc) & 0xf) << 3;
4176 byte2 |= (vex_w ? VEX_W : 0) | (vector256 ? 4 : 0) | pre;
4177 emit_byte(byte2);
4178 } else {
4179 prefix(VEX_2bytes);
4181 int byte1 = vex_r ? VEX_R : 0;
4182 byte1 = (~byte1) & 0x80;
4183 byte1 |= ((~nds_enc) & 0xf) << 3;
4184 byte1 |= (vector256 ? 4 : 0) | pre;
4185 emit_byte(byte1);
4186 }
4187 }
4189 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256){
4190 bool vex_r = (xreg_enc >= 8);
4191 bool vex_b = adr.base_needs_rex();
4192 bool vex_x = adr.index_needs_rex();
4193 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4194 }
4196 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256) {
4197 bool vex_r = (dst_enc >= 8);
4198 bool vex_b = (src_enc >= 8);
4199 bool vex_x = false;
4200 vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4201 return (((dst_enc & 7) << 3) | (src_enc & 7));
4202 }
4205 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4206 if (UseAVX > 0) {
4207 int xreg_enc = xreg->encoding();
4208 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4209 vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector256);
4210 } else {
4211 assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
4212 rex_prefix(adr, xreg, pre, opc, rex_w);
4213 }
4214 }
4216 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4217 int dst_enc = dst->encoding();
4218 int src_enc = src->encoding();
4219 if (UseAVX > 0) {
4220 int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4221 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
4222 } else {
4223 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
4224 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
4225 }
4226 }
4228 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4229 InstructionMark im(this);
4230 simd_prefix(dst, dst, src, pre);
4231 emit_byte(opcode);
4232 emit_operand(dst, src);
4233 }
4235 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4236 int encode = simd_prefix_and_encode(dst, dst, src, pre);
4237 emit_byte(opcode);
4238 emit_byte(0xC0 | encode);
4239 }
4241 // Versions with no second source register (non-destructive source).
4242 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4243 InstructionMark im(this);
4244 simd_prefix(dst, xnoreg, src, pre);
4245 emit_byte(opcode);
4246 emit_operand(dst, src);
4247 }
4249 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4250 int encode = simd_prefix_and_encode(dst, xnoreg, src, pre);
4251 emit_byte(opcode);
4252 emit_byte(0xC0 | encode);
4253 }
4255 // 3-operands AVX instructions
4256 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4257 Address src, VexSimdPrefix pre, bool vector256) {
4258 InstructionMark im(this);
4259 vex_prefix(dst, nds, src, pre, vector256);
4260 emit_byte(opcode);
4261 emit_operand(dst, src);
4262 }
4264 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4265 XMMRegister src, VexSimdPrefix pre, bool vector256) {
4266 int encode = vex_prefix_and_encode(dst, nds, src, pre, vector256);
4267 emit_byte(opcode);
4268 emit_byte(0xC0 | encode);
4269 }
4271 #ifndef _LP64
4273 void Assembler::incl(Register dst) {
4274 // Don't use it directly. Use MacroAssembler::incrementl() instead.
4275 emit_byte(0x40 | dst->encoding());
4276 }
4278 void Assembler::lea(Register dst, Address src) {
4279 leal(dst, src);
4280 }
4282 void Assembler::mov_literal32(Address dst, int32_t imm32, RelocationHolder const& rspec) {
4283 InstructionMark im(this);
4284 emit_byte(0xC7);
4285 emit_operand(rax, dst);
4286 emit_data((int)imm32, rspec, 0);
4287 }
4289 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
4290 InstructionMark im(this);
4291 int encode = prefix_and_encode(dst->encoding());
4292 emit_byte(0xB8 | encode);
4293 emit_data((int)imm32, rspec, 0);
4294 }
4296 void Assembler::popa() { // 32bit
4297 emit_byte(0x61);
4298 }
4300 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
4301 InstructionMark im(this);
4302 emit_byte(0x68);
4303 emit_data(imm32, rspec, 0);
4304 }
4306 void Assembler::pusha() { // 32bit
4307 emit_byte(0x60);
4308 }
4310 void Assembler::set_byte_if_not_zero(Register dst) {
4311 emit_byte(0x0F);
4312 emit_byte(0x95);
4313 emit_byte(0xE0 | dst->encoding());
4314 }
4316 void Assembler::shldl(Register dst, Register src) {
4317 emit_byte(0x0F);
4318 emit_byte(0xA5);
4319 emit_byte(0xC0 | src->encoding() << 3 | dst->encoding());
4320 }
4322 void Assembler::shrdl(Register dst, Register src) {
4323 emit_byte(0x0F);
4324 emit_byte(0xAD);
4325 emit_byte(0xC0 | src->encoding() << 3 | dst->encoding());
4326 }
4328 #else // LP64
4330 void Assembler::set_byte_if_not_zero(Register dst) {
4331 int enc = prefix_and_encode(dst->encoding(), true);
4332 emit_byte(0x0F);
4333 emit_byte(0x95);
4334 emit_byte(0xE0 | enc);
4335 }
4337 // 64bit only pieces of the assembler
4338 // This should only be used by 64bit instructions that can use rip-relative
4339 // it cannot be used by instructions that want an immediate value.
4341 bool Assembler::reachable(AddressLiteral adr) {
4342 int64_t disp;
4343 // None will force a 64bit literal to the code stream. Likely a placeholder
4344 // for something that will be patched later and we need to certain it will
4345 // always be reachable.
4346 if (adr.reloc() == relocInfo::none) {
4347 return false;
4348 }
4349 if (adr.reloc() == relocInfo::internal_word_type) {
4350 // This should be rip relative and easily reachable.
4351 return true;
4352 }
4353 if (adr.reloc() == relocInfo::virtual_call_type ||
4354 adr.reloc() == relocInfo::opt_virtual_call_type ||
4355 adr.reloc() == relocInfo::static_call_type ||
4356 adr.reloc() == relocInfo::static_stub_type ) {
4357 // This should be rip relative within the code cache and easily
4358 // reachable until we get huge code caches. (At which point
4359 // ic code is going to have issues).
4360 return true;
4361 }
4362 if (adr.reloc() != relocInfo::external_word_type &&
4363 adr.reloc() != relocInfo::poll_return_type && // these are really external_word but need special
4364 adr.reloc() != relocInfo::poll_type && // relocs to identify them
4365 adr.reloc() != relocInfo::runtime_call_type ) {
4366 return false;
4367 }
4369 // Stress the correction code
4370 if (ForceUnreachable) {
4371 // Must be runtimecall reloc, see if it is in the codecache
4372 // Flipping stuff in the codecache to be unreachable causes issues
4373 // with things like inline caches where the additional instructions
4374 // are not handled.
4375 if (CodeCache::find_blob(adr._target) == NULL) {
4376 return false;
4377 }
4378 }
4379 // For external_word_type/runtime_call_type if it is reachable from where we
4380 // are now (possibly a temp buffer) and where we might end up
4381 // anywhere in the codeCache then we are always reachable.
4382 // This would have to change if we ever save/restore shared code
4383 // to be more pessimistic.
4384 disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
4385 if (!is_simm32(disp)) return false;
4386 disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
4387 if (!is_simm32(disp)) return false;
4389 disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
4391 // Because rip relative is a disp + address_of_next_instruction and we
4392 // don't know the value of address_of_next_instruction we apply a fudge factor
4393 // to make sure we will be ok no matter the size of the instruction we get placed into.
4394 // We don't have to fudge the checks above here because they are already worst case.
4396 // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
4397 // + 4 because better safe than sorry.
4398 const int fudge = 12 + 4;
4399 if (disp < 0) {
4400 disp -= fudge;
4401 } else {
4402 disp += fudge;
4403 }
4404 return is_simm32(disp);
4405 }
4407 // Check if the polling page is not reachable from the code cache using rip-relative
4408 // addressing.
4409 bool Assembler::is_polling_page_far() {
4410 intptr_t addr = (intptr_t)os::get_polling_page();
4411 return ForceUnreachable ||
4412 !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
4413 !is_simm32(addr - (intptr_t)CodeCache::high_bound());
4414 }
4416 void Assembler::emit_data64(jlong data,
4417 relocInfo::relocType rtype,
4418 int format) {
4419 if (rtype == relocInfo::none) {
4420 emit_int64(data);
4421 } else {
4422 emit_data64(data, Relocation::spec_simple(rtype), format);
4423 }
4424 }
4426 void Assembler::emit_data64(jlong data,
4427 RelocationHolder const& rspec,
4428 int format) {
4429 assert(imm_operand == 0, "default format must be immediate in this file");
4430 assert(imm_operand == format, "must be immediate");
4431 assert(inst_mark() != NULL, "must be inside InstructionMark");
4432 // Do not use AbstractAssembler::relocate, which is not intended for
4433 // embedded words. Instead, relocate to the enclosing instruction.
4434 code_section()->relocate(inst_mark(), rspec, format);
4435 #ifdef ASSERT
4436 check_relocation(rspec, format);
4437 #endif
4438 emit_int64(data);
4439 }
4441 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
4442 if (reg_enc >= 8) {
4443 prefix(REX_B);
4444 reg_enc -= 8;
4445 } else if (byteinst && reg_enc >= 4) {
4446 prefix(REX);
4447 }
4448 return reg_enc;
4449 }
4451 int Assembler::prefixq_and_encode(int reg_enc) {
4452 if (reg_enc < 8) {
4453 prefix(REX_W);
4454 } else {
4455 prefix(REX_WB);
4456 reg_enc -= 8;
4457 }
4458 return reg_enc;
4459 }
4461 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
4462 if (dst_enc < 8) {
4463 if (src_enc >= 8) {
4464 prefix(REX_B);
4465 src_enc -= 8;
4466 } else if (byteinst && src_enc >= 4) {
4467 prefix(REX);
4468 }
4469 } else {
4470 if (src_enc < 8) {
4471 prefix(REX_R);
4472 } else {
4473 prefix(REX_RB);
4474 src_enc -= 8;
4475 }
4476 dst_enc -= 8;
4477 }
4478 return dst_enc << 3 | src_enc;
4479 }
4481 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
4482 if (dst_enc < 8) {
4483 if (src_enc < 8) {
4484 prefix(REX_W);
4485 } else {
4486 prefix(REX_WB);
4487 src_enc -= 8;
4488 }
4489 } else {
4490 if (src_enc < 8) {
4491 prefix(REX_WR);
4492 } else {
4493 prefix(REX_WRB);
4494 src_enc -= 8;
4495 }
4496 dst_enc -= 8;
4497 }
4498 return dst_enc << 3 | src_enc;
4499 }
4501 void Assembler::prefix(Register reg) {
4502 if (reg->encoding() >= 8) {
4503 prefix(REX_B);
4504 }
4505 }
4507 void Assembler::prefix(Address adr) {
4508 if (adr.base_needs_rex()) {
4509 if (adr.index_needs_rex()) {
4510 prefix(REX_XB);
4511 } else {
4512 prefix(REX_B);
4513 }
4514 } else {
4515 if (adr.index_needs_rex()) {
4516 prefix(REX_X);
4517 }
4518 }
4519 }
4521 void Assembler::prefixq(Address adr) {
4522 if (adr.base_needs_rex()) {
4523 if (adr.index_needs_rex()) {
4524 prefix(REX_WXB);
4525 } else {
4526 prefix(REX_WB);
4527 }
4528 } else {
4529 if (adr.index_needs_rex()) {
4530 prefix(REX_WX);
4531 } else {
4532 prefix(REX_W);
4533 }
4534 }
4535 }
4538 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
4539 if (reg->encoding() < 8) {
4540 if (adr.base_needs_rex()) {
4541 if (adr.index_needs_rex()) {
4542 prefix(REX_XB);
4543 } else {
4544 prefix(REX_B);
4545 }
4546 } else {
4547 if (adr.index_needs_rex()) {
4548 prefix(REX_X);
4549 } else if (byteinst && reg->encoding() >= 4 ) {
4550 prefix(REX);
4551 }
4552 }
4553 } else {
4554 if (adr.base_needs_rex()) {
4555 if (adr.index_needs_rex()) {
4556 prefix(REX_RXB);
4557 } else {
4558 prefix(REX_RB);
4559 }
4560 } else {
4561 if (adr.index_needs_rex()) {
4562 prefix(REX_RX);
4563 } else {
4564 prefix(REX_R);
4565 }
4566 }
4567 }
4568 }
4570 void Assembler::prefixq(Address adr, Register src) {
4571 if (src->encoding() < 8) {
4572 if (adr.base_needs_rex()) {
4573 if (adr.index_needs_rex()) {
4574 prefix(REX_WXB);
4575 } else {
4576 prefix(REX_WB);
4577 }
4578 } else {
4579 if (adr.index_needs_rex()) {
4580 prefix(REX_WX);
4581 } else {
4582 prefix(REX_W);
4583 }
4584 }
4585 } else {
4586 if (adr.base_needs_rex()) {
4587 if (adr.index_needs_rex()) {
4588 prefix(REX_WRXB);
4589 } else {
4590 prefix(REX_WRB);
4591 }
4592 } else {
4593 if (adr.index_needs_rex()) {
4594 prefix(REX_WRX);
4595 } else {
4596 prefix(REX_WR);
4597 }
4598 }
4599 }
4600 }
4602 void Assembler::prefix(Address adr, XMMRegister reg) {
4603 if (reg->encoding() < 8) {
4604 if (adr.base_needs_rex()) {
4605 if (adr.index_needs_rex()) {
4606 prefix(REX_XB);
4607 } else {
4608 prefix(REX_B);
4609 }
4610 } else {
4611 if (adr.index_needs_rex()) {
4612 prefix(REX_X);
4613 }
4614 }
4615 } else {
4616 if (adr.base_needs_rex()) {
4617 if (adr.index_needs_rex()) {
4618 prefix(REX_RXB);
4619 } else {
4620 prefix(REX_RB);
4621 }
4622 } else {
4623 if (adr.index_needs_rex()) {
4624 prefix(REX_RX);
4625 } else {
4626 prefix(REX_R);
4627 }
4628 }
4629 }
4630 }
4632 void Assembler::prefixq(Address adr, XMMRegister src) {
4633 if (src->encoding() < 8) {
4634 if (adr.base_needs_rex()) {
4635 if (adr.index_needs_rex()) {
4636 prefix(REX_WXB);
4637 } else {
4638 prefix(REX_WB);
4639 }
4640 } else {
4641 if (adr.index_needs_rex()) {
4642 prefix(REX_WX);
4643 } else {
4644 prefix(REX_W);
4645 }
4646 }
4647 } else {
4648 if (adr.base_needs_rex()) {
4649 if (adr.index_needs_rex()) {
4650 prefix(REX_WRXB);
4651 } else {
4652 prefix(REX_WRB);
4653 }
4654 } else {
4655 if (adr.index_needs_rex()) {
4656 prefix(REX_WRX);
4657 } else {
4658 prefix(REX_WR);
4659 }
4660 }
4661 }
4662 }
4664 void Assembler::adcq(Register dst, int32_t imm32) {
4665 (void) prefixq_and_encode(dst->encoding());
4666 emit_arith(0x81, 0xD0, dst, imm32);
4667 }
4669 void Assembler::adcq(Register dst, Address src) {
4670 InstructionMark im(this);
4671 prefixq(src, dst);
4672 emit_byte(0x13);
4673 emit_operand(dst, src);
4674 }
4676 void Assembler::adcq(Register dst, Register src) {
4677 (int) prefixq_and_encode(dst->encoding(), src->encoding());
4678 emit_arith(0x13, 0xC0, dst, src);
4679 }
4681 void Assembler::addq(Address dst, int32_t imm32) {
4682 InstructionMark im(this);
4683 prefixq(dst);
4684 emit_arith_operand(0x81, rax, dst,imm32);
4685 }
4687 void Assembler::addq(Address dst, Register src) {
4688 InstructionMark im(this);
4689 prefixq(dst, src);
4690 emit_byte(0x01);
4691 emit_operand(src, dst);
4692 }
4694 void Assembler::addq(Register dst, int32_t imm32) {
4695 (void) prefixq_and_encode(dst->encoding());
4696 emit_arith(0x81, 0xC0, dst, imm32);
4697 }
4699 void Assembler::addq(Register dst, Address src) {
4700 InstructionMark im(this);
4701 prefixq(src, dst);
4702 emit_byte(0x03);
4703 emit_operand(dst, src);
4704 }
4706 void Assembler::addq(Register dst, Register src) {
4707 (void) prefixq_and_encode(dst->encoding(), src->encoding());
4708 emit_arith(0x03, 0xC0, dst, src);
4709 }
4711 void Assembler::andq(Address dst, int32_t imm32) {
4712 InstructionMark im(this);
4713 prefixq(dst);
4714 emit_byte(0x81);
4715 emit_operand(rsp, dst, 4);
4716 emit_long(imm32);
4717 }
4719 void Assembler::andq(Register dst, int32_t imm32) {
4720 (void) prefixq_and_encode(dst->encoding());
4721 emit_arith(0x81, 0xE0, dst, imm32);
4722 }
4724 void Assembler::andq(Register dst, Address src) {
4725 InstructionMark im(this);
4726 prefixq(src, dst);
4727 emit_byte(0x23);
4728 emit_operand(dst, src);
4729 }
4731 void Assembler::andq(Register dst, Register src) {
4732 (int) prefixq_and_encode(dst->encoding(), src->encoding());
4733 emit_arith(0x23, 0xC0, dst, src);
4734 }
4736 void Assembler::bsfq(Register dst, Register src) {
4737 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4738 emit_byte(0x0F);
4739 emit_byte(0xBC);
4740 emit_byte(0xC0 | encode);
4741 }
4743 void Assembler::bsrq(Register dst, Register src) {
4744 assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
4745 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4746 emit_byte(0x0F);
4747 emit_byte(0xBD);
4748 emit_byte(0xC0 | encode);
4749 }
4751 void Assembler::bswapq(Register reg) {
4752 int encode = prefixq_and_encode(reg->encoding());
4753 emit_byte(0x0F);
4754 emit_byte(0xC8 | encode);
4755 }
4757 void Assembler::cdqq() {
4758 prefix(REX_W);
4759 emit_byte(0x99);
4760 }
4762 void Assembler::clflush(Address adr) {
4763 prefix(adr);
4764 emit_byte(0x0F);
4765 emit_byte(0xAE);
4766 emit_operand(rdi, adr);
4767 }
4769 void Assembler::cmovq(Condition cc, Register dst, Register src) {
4770 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4771 emit_byte(0x0F);
4772 emit_byte(0x40 | cc);
4773 emit_byte(0xC0 | encode);
4774 }
4776 void Assembler::cmovq(Condition cc, Register dst, Address src) {
4777 InstructionMark im(this);
4778 prefixq(src, dst);
4779 emit_byte(0x0F);
4780 emit_byte(0x40 | cc);
4781 emit_operand(dst, src);
4782 }
4784 void Assembler::cmpq(Address dst, int32_t imm32) {
4785 InstructionMark im(this);
4786 prefixq(dst);
4787 emit_byte(0x81);
4788 emit_operand(rdi, dst, 4);
4789 emit_long(imm32);
4790 }
4792 void Assembler::cmpq(Register dst, int32_t imm32) {
4793 (void) prefixq_and_encode(dst->encoding());
4794 emit_arith(0x81, 0xF8, dst, imm32);
4795 }
4797 void Assembler::cmpq(Address dst, Register src) {
4798 InstructionMark im(this);
4799 prefixq(dst, src);
4800 emit_byte(0x3B);
4801 emit_operand(src, dst);
4802 }
4804 void Assembler::cmpq(Register dst, Register src) {
4805 (void) prefixq_and_encode(dst->encoding(), src->encoding());
4806 emit_arith(0x3B, 0xC0, dst, src);
4807 }
4809 void Assembler::cmpq(Register dst, Address src) {
4810 InstructionMark im(this);
4811 prefixq(src, dst);
4812 emit_byte(0x3B);
4813 emit_operand(dst, src);
4814 }
4816 void Assembler::cmpxchgq(Register reg, Address adr) {
4817 InstructionMark im(this);
4818 prefixq(adr, reg);
4819 emit_byte(0x0F);
4820 emit_byte(0xB1);
4821 emit_operand(reg, adr);
4822 }
4824 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
4825 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4826 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2);
4827 emit_byte(0x2A);
4828 emit_byte(0xC0 | encode);
4829 }
4831 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
4832 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4833 InstructionMark im(this);
4834 simd_prefix_q(dst, dst, src, VEX_SIMD_F2);
4835 emit_byte(0x2A);
4836 emit_operand(dst, src);
4837 }
4839 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
4840 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4841 int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3);
4842 emit_byte(0x2A);
4843 emit_byte(0xC0 | encode);
4844 }
4846 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
4847 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4848 InstructionMark im(this);
4849 simd_prefix_q(dst, dst, src, VEX_SIMD_F3);
4850 emit_byte(0x2A);
4851 emit_operand(dst, src);
4852 }
4854 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
4855 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4856 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2);
4857 emit_byte(0x2C);
4858 emit_byte(0xC0 | encode);
4859 }
4861 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
4862 NOT_LP64(assert(VM_Version::supports_sse(), ""));
4863 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3);
4864 emit_byte(0x2C);
4865 emit_byte(0xC0 | encode);
4866 }
4868 void Assembler::decl(Register dst) {
4869 // Don't use it directly. Use MacroAssembler::decrementl() instead.
4870 // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
4871 int encode = prefix_and_encode(dst->encoding());
4872 emit_byte(0xFF);
4873 emit_byte(0xC8 | encode);
4874 }
4876 void Assembler::decq(Register dst) {
4877 // Don't use it directly. Use MacroAssembler::decrementq() instead.
4878 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
4879 int encode = prefixq_and_encode(dst->encoding());
4880 emit_byte(0xFF);
4881 emit_byte(0xC8 | encode);
4882 }
4884 void Assembler::decq(Address dst) {
4885 // Don't use it directly. Use MacroAssembler::decrementq() instead.
4886 InstructionMark im(this);
4887 prefixq(dst);
4888 emit_byte(0xFF);
4889 emit_operand(rcx, dst);
4890 }
4892 void Assembler::fxrstor(Address src) {
4893 prefixq(src);
4894 emit_byte(0x0F);
4895 emit_byte(0xAE);
4896 emit_operand(as_Register(1), src);
4897 }
4899 void Assembler::fxsave(Address dst) {
4900 prefixq(dst);
4901 emit_byte(0x0F);
4902 emit_byte(0xAE);
4903 emit_operand(as_Register(0), dst);
4904 }
4906 void Assembler::idivq(Register src) {
4907 int encode = prefixq_and_encode(src->encoding());
4908 emit_byte(0xF7);
4909 emit_byte(0xF8 | encode);
4910 }
4912 void Assembler::imulq(Register dst, Register src) {
4913 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4914 emit_byte(0x0F);
4915 emit_byte(0xAF);
4916 emit_byte(0xC0 | encode);
4917 }
4919 void Assembler::imulq(Register dst, Register src, int value) {
4920 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4921 if (is8bit(value)) {
4922 emit_byte(0x6B);
4923 emit_byte(0xC0 | encode);
4924 emit_byte(value & 0xFF);
4925 } else {
4926 emit_byte(0x69);
4927 emit_byte(0xC0 | encode);
4928 emit_long(value);
4929 }
4930 }
4932 void Assembler::incl(Register dst) {
4933 // Don't use it directly. Use MacroAssembler::incrementl() instead.
4934 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
4935 int encode = prefix_and_encode(dst->encoding());
4936 emit_byte(0xFF);
4937 emit_byte(0xC0 | encode);
4938 }
4940 void Assembler::incq(Register dst) {
4941 // Don't use it directly. Use MacroAssembler::incrementq() instead.
4942 // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
4943 int encode = prefixq_and_encode(dst->encoding());
4944 emit_byte(0xFF);
4945 emit_byte(0xC0 | encode);
4946 }
4948 void Assembler::incq(Address dst) {
4949 // Don't use it directly. Use MacroAssembler::incrementq() instead.
4950 InstructionMark im(this);
4951 prefixq(dst);
4952 emit_byte(0xFF);
4953 emit_operand(rax, dst);
4954 }
4956 void Assembler::lea(Register dst, Address src) {
4957 leaq(dst, src);
4958 }
4960 void Assembler::leaq(Register dst, Address src) {
4961 InstructionMark im(this);
4962 prefixq(src, dst);
4963 emit_byte(0x8D);
4964 emit_operand(dst, src);
4965 }
4967 void Assembler::mov64(Register dst, int64_t imm64) {
4968 InstructionMark im(this);
4969 int encode = prefixq_and_encode(dst->encoding());
4970 emit_byte(0xB8 | encode);
4971 emit_int64(imm64);
4972 }
4974 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
4975 InstructionMark im(this);
4976 int encode = prefixq_and_encode(dst->encoding());
4977 emit_byte(0xB8 | encode);
4978 emit_data64(imm64, rspec);
4979 }
4981 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
4982 InstructionMark im(this);
4983 int encode = prefix_and_encode(dst->encoding());
4984 emit_byte(0xB8 | encode);
4985 emit_data((int)imm32, rspec, narrow_oop_operand);
4986 }
4988 void Assembler::mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec) {
4989 InstructionMark im(this);
4990 prefix(dst);
4991 emit_byte(0xC7);
4992 emit_operand(rax, dst, 4);
4993 emit_data((int)imm32, rspec, narrow_oop_operand);
4994 }
4996 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
4997 InstructionMark im(this);
4998 int encode = prefix_and_encode(src1->encoding());
4999 emit_byte(0x81);
5000 emit_byte(0xF8 | encode);
5001 emit_data((int)imm32, rspec, narrow_oop_operand);
5002 }
5004 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5005 InstructionMark im(this);
5006 prefix(src1);
5007 emit_byte(0x81);
5008 emit_operand(rax, src1, 4);
5009 emit_data((int)imm32, rspec, narrow_oop_operand);
5010 }
5012 void Assembler::lzcntq(Register dst, Register src) {
5013 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
5014 emit_byte(0xF3);
5015 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5016 emit_byte(0x0F);
5017 emit_byte(0xBD);
5018 emit_byte(0xC0 | encode);
5019 }
5021 void Assembler::movdq(XMMRegister dst, Register src) {
5022 // table D-1 says MMX/SSE2
5023 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5024 int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66);
5025 emit_byte(0x6E);
5026 emit_byte(0xC0 | encode);
5027 }
5029 void Assembler::movdq(Register dst, XMMRegister src) {
5030 // table D-1 says MMX/SSE2
5031 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5032 // swap src/dst to get correct prefix
5033 int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66);
5034 emit_byte(0x7E);
5035 emit_byte(0xC0 | encode);
5036 }
5038 void Assembler::movq(Register dst, Register src) {
5039 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5040 emit_byte(0x8B);
5041 emit_byte(0xC0 | encode);
5042 }
5044 void Assembler::movq(Register dst, Address src) {
5045 InstructionMark im(this);
5046 prefixq(src, dst);
5047 emit_byte(0x8B);
5048 emit_operand(dst, src);
5049 }
5051 void Assembler::movq(Address dst, Register src) {
5052 InstructionMark im(this);
5053 prefixq(dst, src);
5054 emit_byte(0x89);
5055 emit_operand(src, dst);
5056 }
5058 void Assembler::movsbq(Register dst, Address src) {
5059 InstructionMark im(this);
5060 prefixq(src, dst);
5061 emit_byte(0x0F);
5062 emit_byte(0xBE);
5063 emit_operand(dst, src);
5064 }
5066 void Assembler::movsbq(Register dst, Register src) {
5067 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5068 emit_byte(0x0F);
5069 emit_byte(0xBE);
5070 emit_byte(0xC0 | encode);
5071 }
5073 void Assembler::movslq(Register dst, int32_t imm32) {
5074 // dbx shows movslq(rcx, 3) as movq $0x0000000049000000,(%rbx)
5075 // and movslq(r8, 3); as movl $0x0000000048000000,(%rbx)
5076 // as a result we shouldn't use until tested at runtime...
5077 ShouldNotReachHere();
5078 InstructionMark im(this);
5079 int encode = prefixq_and_encode(dst->encoding());
5080 emit_byte(0xC7 | encode);
5081 emit_long(imm32);
5082 }
5084 void Assembler::movslq(Address dst, int32_t imm32) {
5085 assert(is_simm32(imm32), "lost bits");
5086 InstructionMark im(this);
5087 prefixq(dst);
5088 emit_byte(0xC7);
5089 emit_operand(rax, dst, 4);
5090 emit_long(imm32);
5091 }
5093 void Assembler::movslq(Register dst, Address src) {
5094 InstructionMark im(this);
5095 prefixq(src, dst);
5096 emit_byte(0x63);
5097 emit_operand(dst, src);
5098 }
5100 void Assembler::movslq(Register dst, Register src) {
5101 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5102 emit_byte(0x63);
5103 emit_byte(0xC0 | encode);
5104 }
5106 void Assembler::movswq(Register dst, Address src) {
5107 InstructionMark im(this);
5108 prefixq(src, dst);
5109 emit_byte(0x0F);
5110 emit_byte(0xBF);
5111 emit_operand(dst, src);
5112 }
5114 void Assembler::movswq(Register dst, Register src) {
5115 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5116 emit_byte(0x0F);
5117 emit_byte(0xBF);
5118 emit_byte(0xC0 | encode);
5119 }
5121 void Assembler::movzbq(Register dst, Address src) {
5122 InstructionMark im(this);
5123 prefixq(src, dst);
5124 emit_byte(0x0F);
5125 emit_byte(0xB6);
5126 emit_operand(dst, src);
5127 }
5129 void Assembler::movzbq(Register dst, Register src) {
5130 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5131 emit_byte(0x0F);
5132 emit_byte(0xB6);
5133 emit_byte(0xC0 | encode);
5134 }
5136 void Assembler::movzwq(Register dst, Address src) {
5137 InstructionMark im(this);
5138 prefixq(src, dst);
5139 emit_byte(0x0F);
5140 emit_byte(0xB7);
5141 emit_operand(dst, src);
5142 }
5144 void Assembler::movzwq(Register dst, Register src) {
5145 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5146 emit_byte(0x0F);
5147 emit_byte(0xB7);
5148 emit_byte(0xC0 | encode);
5149 }
5151 void Assembler::negq(Register dst) {
5152 int encode = prefixq_and_encode(dst->encoding());
5153 emit_byte(0xF7);
5154 emit_byte(0xD8 | encode);
5155 }
5157 void Assembler::notq(Register dst) {
5158 int encode = prefixq_and_encode(dst->encoding());
5159 emit_byte(0xF7);
5160 emit_byte(0xD0 | encode);
5161 }
5163 void Assembler::orq(Address dst, int32_t imm32) {
5164 InstructionMark im(this);
5165 prefixq(dst);
5166 emit_byte(0x81);
5167 emit_operand(rcx, dst, 4);
5168 emit_long(imm32);
5169 }
5171 void Assembler::orq(Register dst, int32_t imm32) {
5172 (void) prefixq_and_encode(dst->encoding());
5173 emit_arith(0x81, 0xC8, dst, imm32);
5174 }
5176 void Assembler::orq(Register dst, Address src) {
5177 InstructionMark im(this);
5178 prefixq(src, dst);
5179 emit_byte(0x0B);
5180 emit_operand(dst, src);
5181 }
5183 void Assembler::orq(Register dst, Register src) {
5184 (void) prefixq_and_encode(dst->encoding(), src->encoding());
5185 emit_arith(0x0B, 0xC0, dst, src);
5186 }
5188 void Assembler::popa() { // 64bit
5189 movq(r15, Address(rsp, 0));
5190 movq(r14, Address(rsp, wordSize));
5191 movq(r13, Address(rsp, 2 * wordSize));
5192 movq(r12, Address(rsp, 3 * wordSize));
5193 movq(r11, Address(rsp, 4 * wordSize));
5194 movq(r10, Address(rsp, 5 * wordSize));
5195 movq(r9, Address(rsp, 6 * wordSize));
5196 movq(r8, Address(rsp, 7 * wordSize));
5197 movq(rdi, Address(rsp, 8 * wordSize));
5198 movq(rsi, Address(rsp, 9 * wordSize));
5199 movq(rbp, Address(rsp, 10 * wordSize));
5200 // skip rsp
5201 movq(rbx, Address(rsp, 12 * wordSize));
5202 movq(rdx, Address(rsp, 13 * wordSize));
5203 movq(rcx, Address(rsp, 14 * wordSize));
5204 movq(rax, Address(rsp, 15 * wordSize));
5206 addq(rsp, 16 * wordSize);
5207 }
5209 void Assembler::popcntq(Register dst, Address src) {
5210 assert(VM_Version::supports_popcnt(), "must support");
5211 InstructionMark im(this);
5212 emit_byte(0xF3);
5213 prefixq(src, dst);
5214 emit_byte(0x0F);
5215 emit_byte(0xB8);
5216 emit_operand(dst, src);
5217 }
5219 void Assembler::popcntq(Register dst, Register src) {
5220 assert(VM_Version::supports_popcnt(), "must support");
5221 emit_byte(0xF3);
5222 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5223 emit_byte(0x0F);
5224 emit_byte(0xB8);
5225 emit_byte(0xC0 | encode);
5226 }
5228 void Assembler::popq(Address dst) {
5229 InstructionMark im(this);
5230 prefixq(dst);
5231 emit_byte(0x8F);
5232 emit_operand(rax, dst);
5233 }
5235 void Assembler::pusha() { // 64bit
5236 // we have to store original rsp. ABI says that 128 bytes
5237 // below rsp are local scratch.
5238 movq(Address(rsp, -5 * wordSize), rsp);
5240 subq(rsp, 16 * wordSize);
5242 movq(Address(rsp, 15 * wordSize), rax);
5243 movq(Address(rsp, 14 * wordSize), rcx);
5244 movq(Address(rsp, 13 * wordSize), rdx);
5245 movq(Address(rsp, 12 * wordSize), rbx);
5246 // skip rsp
5247 movq(Address(rsp, 10 * wordSize), rbp);
5248 movq(Address(rsp, 9 * wordSize), rsi);
5249 movq(Address(rsp, 8 * wordSize), rdi);
5250 movq(Address(rsp, 7 * wordSize), r8);
5251 movq(Address(rsp, 6 * wordSize), r9);
5252 movq(Address(rsp, 5 * wordSize), r10);
5253 movq(Address(rsp, 4 * wordSize), r11);
5254 movq(Address(rsp, 3 * wordSize), r12);
5255 movq(Address(rsp, 2 * wordSize), r13);
5256 movq(Address(rsp, wordSize), r14);
5257 movq(Address(rsp, 0), r15);
5258 }
5260 void Assembler::pushq(Address src) {
5261 InstructionMark im(this);
5262 prefixq(src);
5263 emit_byte(0xFF);
5264 emit_operand(rsi, src);
5265 }
5267 void Assembler::rclq(Register dst, int imm8) {
5268 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5269 int encode = prefixq_and_encode(dst->encoding());
5270 if (imm8 == 1) {
5271 emit_byte(0xD1);
5272 emit_byte(0xD0 | encode);
5273 } else {
5274 emit_byte(0xC1);
5275 emit_byte(0xD0 | encode);
5276 emit_byte(imm8);
5277 }
5278 }
5279 void Assembler::sarq(Register dst, int imm8) {
5280 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5281 int encode = prefixq_and_encode(dst->encoding());
5282 if (imm8 == 1) {
5283 emit_byte(0xD1);
5284 emit_byte(0xF8 | encode);
5285 } else {
5286 emit_byte(0xC1);
5287 emit_byte(0xF8 | encode);
5288 emit_byte(imm8);
5289 }
5290 }
5292 void Assembler::sarq(Register dst) {
5293 int encode = prefixq_and_encode(dst->encoding());
5294 emit_byte(0xD3);
5295 emit_byte(0xF8 | encode);
5296 }
5298 void Assembler::sbbq(Address dst, int32_t imm32) {
5299 InstructionMark im(this);
5300 prefixq(dst);
5301 emit_arith_operand(0x81, rbx, dst, imm32);
5302 }
5304 void Assembler::sbbq(Register dst, int32_t imm32) {
5305 (void) prefixq_and_encode(dst->encoding());
5306 emit_arith(0x81, 0xD8, dst, imm32);
5307 }
5309 void Assembler::sbbq(Register dst, Address src) {
5310 InstructionMark im(this);
5311 prefixq(src, dst);
5312 emit_byte(0x1B);
5313 emit_operand(dst, src);
5314 }
5316 void Assembler::sbbq(Register dst, Register src) {
5317 (void) prefixq_and_encode(dst->encoding(), src->encoding());
5318 emit_arith(0x1B, 0xC0, dst, src);
5319 }
5321 void Assembler::shlq(Register dst, int imm8) {
5322 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5323 int encode = prefixq_and_encode(dst->encoding());
5324 if (imm8 == 1) {
5325 emit_byte(0xD1);
5326 emit_byte(0xE0 | encode);
5327 } else {
5328 emit_byte(0xC1);
5329 emit_byte(0xE0 | encode);
5330 emit_byte(imm8);
5331 }
5332 }
5334 void Assembler::shlq(Register dst) {
5335 int encode = prefixq_and_encode(dst->encoding());
5336 emit_byte(0xD3);
5337 emit_byte(0xE0 | encode);
5338 }
5340 void Assembler::shrq(Register dst, int imm8) {
5341 assert(isShiftCount(imm8 >> 1), "illegal shift count");
5342 int encode = prefixq_and_encode(dst->encoding());
5343 emit_byte(0xC1);
5344 emit_byte(0xE8 | encode);
5345 emit_byte(imm8);
5346 }
5348 void Assembler::shrq(Register dst) {
5349 int encode = prefixq_and_encode(dst->encoding());
5350 emit_byte(0xD3);
5351 emit_byte(0xE8 | encode);
5352 }
5354 void Assembler::subq(Address dst, int32_t imm32) {
5355 InstructionMark im(this);
5356 prefixq(dst);
5357 emit_arith_operand(0x81, rbp, dst, imm32);
5358 }
5360 void Assembler::subq(Address dst, Register src) {
5361 InstructionMark im(this);
5362 prefixq(dst, src);
5363 emit_byte(0x29);
5364 emit_operand(src, dst);
5365 }
5367 void Assembler::subq(Register dst, int32_t imm32) {
5368 (void) prefixq_and_encode(dst->encoding());
5369 emit_arith(0x81, 0xE8, dst, imm32);
5370 }
5372 // Force generation of a 4 byte immediate value even if it fits into 8bit
5373 void Assembler::subq_imm32(Register dst, int32_t imm32) {
5374 (void) prefixq_and_encode(dst->encoding());
5375 emit_arith_imm32(0x81, 0xE8, dst, imm32);
5376 }
5378 void Assembler::subq(Register dst, Address src) {
5379 InstructionMark im(this);
5380 prefixq(src, dst);
5381 emit_byte(0x2B);
5382 emit_operand(dst, src);
5383 }
5385 void Assembler::subq(Register dst, Register src) {
5386 (void) prefixq_and_encode(dst->encoding(), src->encoding());
5387 emit_arith(0x2B, 0xC0, dst, src);
5388 }
5390 void Assembler::testq(Register dst, int32_t imm32) {
5391 // not using emit_arith because test
5392 // doesn't support sign-extension of
5393 // 8bit operands
5394 int encode = dst->encoding();
5395 if (encode == 0) {
5396 prefix(REX_W);
5397 emit_byte(0xA9);
5398 } else {
5399 encode = prefixq_and_encode(encode);
5400 emit_byte(0xF7);
5401 emit_byte(0xC0 | encode);
5402 }
5403 emit_long(imm32);
5404 }
5406 void Assembler::testq(Register dst, Register src) {
5407 (void) prefixq_and_encode(dst->encoding(), src->encoding());
5408 emit_arith(0x85, 0xC0, dst, src);
5409 }
5411 void Assembler::xaddq(Address dst, Register src) {
5412 InstructionMark im(this);
5413 prefixq(dst, src);
5414 emit_byte(0x0F);
5415 emit_byte(0xC1);
5416 emit_operand(src, dst);
5417 }
5419 void Assembler::xchgq(Register dst, Address src) {
5420 InstructionMark im(this);
5421 prefixq(src, dst);
5422 emit_byte(0x87);
5423 emit_operand(dst, src);
5424 }
5426 void Assembler::xchgq(Register dst, Register src) {
5427 int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5428 emit_byte(0x87);
5429 emit_byte(0xc0 | encode);
5430 }
5432 void Assembler::xorq(Register dst, Register src) {
5433 (void) prefixq_and_encode(dst->encoding(), src->encoding());
5434 emit_arith(0x33, 0xC0, dst, src);
5435 }
5437 void Assembler::xorq(Register dst, Address src) {
5438 InstructionMark im(this);
5439 prefixq(src, dst);
5440 emit_byte(0x33);
5441 emit_operand(dst, src);
5442 }
5444 #endif // !LP64