Wed, 23 Jan 2013 13:02:39 -0500
8005915: Unify SERIALGC and INCLUDE_ALTERNATE_GCS
Summary: Rename INCLUDE_ALTERNATE_GCS to INCLUDE_ALL_GCS and replace SERIALGC with INCLUDE_ALL_GCS.
Reviewed-by: coleenp, stefank
twisti@4318 | 1 | /* |
twisti@4318 | 2 | * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved. |
twisti@4318 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
twisti@4318 | 4 | * |
twisti@4318 | 5 | * This code is free software; you can redistribute it and/or modify it |
twisti@4318 | 6 | * under the terms of the GNU General Public License version 2 only, as |
twisti@4318 | 7 | * published by the Free Software Foundation. |
twisti@4318 | 8 | * |
twisti@4318 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
twisti@4318 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
twisti@4318 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
twisti@4318 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
twisti@4318 | 13 | * accompanied this code). |
twisti@4318 | 14 | * |
twisti@4318 | 15 | * You should have received a copy of the GNU General Public License version |
twisti@4318 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
twisti@4318 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
twisti@4318 | 18 | * |
twisti@4318 | 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
twisti@4318 | 20 | * or visit www.oracle.com if you need additional information or have any |
twisti@4318 | 21 | * questions. |
twisti@4318 | 22 | * |
twisti@4318 | 23 | */ |
twisti@4318 | 24 | |
twisti@4318 | 25 | #include "precompiled.hpp" |
twisti@4318 | 26 | #include "asm/assembler.hpp" |
twisti@4318 | 27 | #include "asm/assembler.inline.hpp" |
twisti@4318 | 28 | #include "compiler/disassembler.hpp" |
twisti@4318 | 29 | #include "gc_interface/collectedHeap.inline.hpp" |
twisti@4318 | 30 | #include "interpreter/interpreter.hpp" |
twisti@4318 | 31 | #include "memory/cardTableModRefBS.hpp" |
twisti@4318 | 32 | #include "memory/resourceArea.hpp" |
twisti@4318 | 33 | #include "prims/methodHandles.hpp" |
twisti@4318 | 34 | #include "runtime/biasedLocking.hpp" |
twisti@4318 | 35 | #include "runtime/interfaceSupport.hpp" |
twisti@4318 | 36 | #include "runtime/objectMonitor.hpp" |
twisti@4318 | 37 | #include "runtime/os.hpp" |
twisti@4318 | 38 | #include "runtime/sharedRuntime.hpp" |
twisti@4318 | 39 | #include "runtime/stubRoutines.hpp" |
jprovino@4542 | 40 | #include "utilities/macros.hpp" |
jprovino@4542 | 41 | #if INCLUDE_ALL_GCS |
twisti@4318 | 42 | #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" |
twisti@4318 | 43 | #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" |
twisti@4318 | 44 | #include "gc_implementation/g1/heapRegion.hpp" |
jprovino@4542 | 45 | #endif // INCLUDE_ALL_GCS |
twisti@4318 | 46 | |
twisti@4318 | 47 | #ifdef PRODUCT |
twisti@4318 | 48 | #define BLOCK_COMMENT(str) /* nothing */ |
twisti@4318 | 49 | #define STOP(error) stop(error) |
twisti@4318 | 50 | #else |
twisti@4318 | 51 | #define BLOCK_COMMENT(str) block_comment(str) |
twisti@4318 | 52 | #define STOP(error) block_comment(error); stop(error) |
twisti@4318 | 53 | #endif |
twisti@4318 | 54 | |
twisti@4318 | 55 | #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") |
twisti@4318 | 56 | |
twisti@4318 | 57 | |
twisti@4323 | 58 | #ifdef ASSERT |
twisti@4323 | 59 | bool AbstractAssembler::pd_check_instruction_mark() { return true; } |
twisti@4323 | 60 | #endif |
twisti@4323 | 61 | |
twisti@4318 | 62 | static Assembler::Condition reverse[] = { |
twisti@4318 | 63 | Assembler::noOverflow /* overflow = 0x0 */ , |
twisti@4318 | 64 | Assembler::overflow /* noOverflow = 0x1 */ , |
twisti@4318 | 65 | Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ , |
twisti@4318 | 66 | Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ , |
twisti@4318 | 67 | Assembler::notZero /* zero = 0x4, equal = 0x4 */ , |
twisti@4318 | 68 | Assembler::zero /* notZero = 0x5, notEqual = 0x5 */ , |
twisti@4318 | 69 | Assembler::above /* belowEqual = 0x6 */ , |
twisti@4318 | 70 | Assembler::belowEqual /* above = 0x7 */ , |
twisti@4318 | 71 | Assembler::positive /* negative = 0x8 */ , |
twisti@4318 | 72 | Assembler::negative /* positive = 0x9 */ , |
twisti@4318 | 73 | Assembler::noParity /* parity = 0xa */ , |
twisti@4318 | 74 | Assembler::parity /* noParity = 0xb */ , |
twisti@4318 | 75 | Assembler::greaterEqual /* less = 0xc */ , |
twisti@4318 | 76 | Assembler::less /* greaterEqual = 0xd */ , |
twisti@4318 | 77 | Assembler::greater /* lessEqual = 0xe */ , |
twisti@4318 | 78 | Assembler::lessEqual /* greater = 0xf, */ |
twisti@4318 | 79 | |
twisti@4318 | 80 | }; |
twisti@4318 | 81 | |
twisti@4318 | 82 | |
twisti@4318 | 83 | // Implementation of MacroAssembler |
twisti@4318 | 84 | |
twisti@4318 | 85 | // First all the versions that have distinct versions depending on 32/64 bit |
twisti@4318 | 86 | // Unless the difference is trivial (1 line or so). |
twisti@4318 | 87 | |
twisti@4318 | 88 | #ifndef _LP64 |
twisti@4318 | 89 | |
twisti@4318 | 90 | // 32bit versions |
twisti@4318 | 91 | |
twisti@4318 | 92 | Address MacroAssembler::as_Address(AddressLiteral adr) { |
twisti@4318 | 93 | return Address(adr.target(), adr.rspec()); |
twisti@4318 | 94 | } |
twisti@4318 | 95 | |
twisti@4318 | 96 | Address MacroAssembler::as_Address(ArrayAddress adr) { |
twisti@4318 | 97 | return Address::make_array(adr); |
twisti@4318 | 98 | } |
twisti@4318 | 99 | |
twisti@4318 | 100 | int MacroAssembler::biased_locking_enter(Register lock_reg, |
twisti@4318 | 101 | Register obj_reg, |
twisti@4318 | 102 | Register swap_reg, |
twisti@4318 | 103 | Register tmp_reg, |
twisti@4318 | 104 | bool swap_reg_contains_mark, |
twisti@4318 | 105 | Label& done, |
twisti@4318 | 106 | Label* slow_case, |
twisti@4318 | 107 | BiasedLockingCounters* counters) { |
twisti@4318 | 108 | assert(UseBiasedLocking, "why call this otherwise?"); |
twisti@4318 | 109 | assert(swap_reg == rax, "swap_reg must be rax, for cmpxchg"); |
twisti@4318 | 110 | assert_different_registers(lock_reg, obj_reg, swap_reg); |
twisti@4318 | 111 | |
twisti@4318 | 112 | if (PrintBiasedLockingStatistics && counters == NULL) |
twisti@4318 | 113 | counters = BiasedLocking::counters(); |
twisti@4318 | 114 | |
twisti@4318 | 115 | bool need_tmp_reg = false; |
twisti@4318 | 116 | if (tmp_reg == noreg) { |
twisti@4318 | 117 | need_tmp_reg = true; |
twisti@4318 | 118 | tmp_reg = lock_reg; |
twisti@4318 | 119 | } else { |
twisti@4318 | 120 | assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg); |
twisti@4318 | 121 | } |
twisti@4318 | 122 | assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout"); |
twisti@4318 | 123 | Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes()); |
twisti@4318 | 124 | Address klass_addr (obj_reg, oopDesc::klass_offset_in_bytes()); |
twisti@4318 | 125 | Address saved_mark_addr(lock_reg, 0); |
twisti@4318 | 126 | |
twisti@4318 | 127 | // Biased locking |
twisti@4318 | 128 | // See whether the lock is currently biased toward our thread and |
twisti@4318 | 129 | // whether the epoch is still valid |
twisti@4318 | 130 | // Note that the runtime guarantees sufficient alignment of JavaThread |
twisti@4318 | 131 | // pointers to allow age to be placed into low bits |
twisti@4318 | 132 | // First check to see whether biasing is even enabled for this object |
twisti@4318 | 133 | Label cas_label; |
twisti@4318 | 134 | int null_check_offset = -1; |
twisti@4318 | 135 | if (!swap_reg_contains_mark) { |
twisti@4318 | 136 | null_check_offset = offset(); |
twisti@4318 | 137 | movl(swap_reg, mark_addr); |
twisti@4318 | 138 | } |
twisti@4318 | 139 | if (need_tmp_reg) { |
twisti@4318 | 140 | push(tmp_reg); |
twisti@4318 | 141 | } |
twisti@4318 | 142 | movl(tmp_reg, swap_reg); |
twisti@4318 | 143 | andl(tmp_reg, markOopDesc::biased_lock_mask_in_place); |
twisti@4318 | 144 | cmpl(tmp_reg, markOopDesc::biased_lock_pattern); |
twisti@4318 | 145 | if (need_tmp_reg) { |
twisti@4318 | 146 | pop(tmp_reg); |
twisti@4318 | 147 | } |
twisti@4318 | 148 | jcc(Assembler::notEqual, cas_label); |
twisti@4318 | 149 | // The bias pattern is present in the object's header. Need to check |
twisti@4318 | 150 | // whether the bias owner and the epoch are both still current. |
twisti@4318 | 151 | // Note that because there is no current thread register on x86 we |
twisti@4318 | 152 | // need to store off the mark word we read out of the object to |
twisti@4318 | 153 | // avoid reloading it and needing to recheck invariants below. This |
twisti@4318 | 154 | // store is unfortunate but it makes the overall code shorter and |
twisti@4318 | 155 | // simpler. |
twisti@4318 | 156 | movl(saved_mark_addr, swap_reg); |
twisti@4318 | 157 | if (need_tmp_reg) { |
twisti@4318 | 158 | push(tmp_reg); |
twisti@4318 | 159 | } |
twisti@4318 | 160 | get_thread(tmp_reg); |
twisti@4318 | 161 | xorl(swap_reg, tmp_reg); |
twisti@4318 | 162 | if (swap_reg_contains_mark) { |
twisti@4318 | 163 | null_check_offset = offset(); |
twisti@4318 | 164 | } |
twisti@4318 | 165 | movl(tmp_reg, klass_addr); |
twisti@4318 | 166 | xorl(swap_reg, Address(tmp_reg, Klass::prototype_header_offset())); |
twisti@4318 | 167 | andl(swap_reg, ~((int) markOopDesc::age_mask_in_place)); |
twisti@4318 | 168 | if (need_tmp_reg) { |
twisti@4318 | 169 | pop(tmp_reg); |
twisti@4318 | 170 | } |
twisti@4318 | 171 | if (counters != NULL) { |
twisti@4318 | 172 | cond_inc32(Assembler::zero, |
twisti@4318 | 173 | ExternalAddress((address)counters->biased_lock_entry_count_addr())); |
twisti@4318 | 174 | } |
twisti@4318 | 175 | jcc(Assembler::equal, done); |
twisti@4318 | 176 | |
twisti@4318 | 177 | Label try_revoke_bias; |
twisti@4318 | 178 | Label try_rebias; |
twisti@4318 | 179 | |
twisti@4318 | 180 | // At this point we know that the header has the bias pattern and |
twisti@4318 | 181 | // that we are not the bias owner in the current epoch. We need to |
twisti@4318 | 182 | // figure out more details about the state of the header in order to |
twisti@4318 | 183 | // know what operations can be legally performed on the object's |
twisti@4318 | 184 | // header. |
twisti@4318 | 185 | |
twisti@4318 | 186 | // If the low three bits in the xor result aren't clear, that means |
twisti@4318 | 187 | // the prototype header is no longer biased and we have to revoke |
twisti@4318 | 188 | // the bias on this object. |
twisti@4318 | 189 | testl(swap_reg, markOopDesc::biased_lock_mask_in_place); |
twisti@4318 | 190 | jcc(Assembler::notZero, try_revoke_bias); |
twisti@4318 | 191 | |
twisti@4318 | 192 | // Biasing is still enabled for this data type. See whether the |
twisti@4318 | 193 | // epoch of the current bias is still valid, meaning that the epoch |
twisti@4318 | 194 | // bits of the mark word are equal to the epoch bits of the |
twisti@4318 | 195 | // prototype header. (Note that the prototype header's epoch bits |
twisti@4318 | 196 | // only change at a safepoint.) If not, attempt to rebias the object |
twisti@4318 | 197 | // toward the current thread. Note that we must be absolutely sure |
twisti@4318 | 198 | // that the current epoch is invalid in order to do this because |
twisti@4318 | 199 | // otherwise the manipulations it performs on the mark word are |
twisti@4318 | 200 | // illegal. |
twisti@4318 | 201 | testl(swap_reg, markOopDesc::epoch_mask_in_place); |
twisti@4318 | 202 | jcc(Assembler::notZero, try_rebias); |
twisti@4318 | 203 | |
twisti@4318 | 204 | // The epoch of the current bias is still valid but we know nothing |
twisti@4318 | 205 | // about the owner; it might be set or it might be clear. Try to |
twisti@4318 | 206 | // acquire the bias of the object using an atomic operation. If this |
twisti@4318 | 207 | // fails we will go in to the runtime to revoke the object's bias. |
twisti@4318 | 208 | // Note that we first construct the presumed unbiased header so we |
twisti@4318 | 209 | // don't accidentally blow away another thread's valid bias. |
twisti@4318 | 210 | movl(swap_reg, saved_mark_addr); |
twisti@4318 | 211 | andl(swap_reg, |
twisti@4318 | 212 | markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place); |
twisti@4318 | 213 | if (need_tmp_reg) { |
twisti@4318 | 214 | push(tmp_reg); |
twisti@4318 | 215 | } |
twisti@4318 | 216 | get_thread(tmp_reg); |
twisti@4318 | 217 | orl(tmp_reg, swap_reg); |
twisti@4318 | 218 | if (os::is_MP()) { |
twisti@4318 | 219 | lock(); |
twisti@4318 | 220 | } |
twisti@4318 | 221 | cmpxchgptr(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 222 | if (need_tmp_reg) { |
twisti@4318 | 223 | pop(tmp_reg); |
twisti@4318 | 224 | } |
twisti@4318 | 225 | // If the biasing toward our thread failed, this means that |
twisti@4318 | 226 | // another thread succeeded in biasing it toward itself and we |
twisti@4318 | 227 | // need to revoke that bias. The revocation will occur in the |
twisti@4318 | 228 | // interpreter runtime in the slow case. |
twisti@4318 | 229 | if (counters != NULL) { |
twisti@4318 | 230 | cond_inc32(Assembler::zero, |
twisti@4318 | 231 | ExternalAddress((address)counters->anonymously_biased_lock_entry_count_addr())); |
twisti@4318 | 232 | } |
twisti@4318 | 233 | if (slow_case != NULL) { |
twisti@4318 | 234 | jcc(Assembler::notZero, *slow_case); |
twisti@4318 | 235 | } |
twisti@4318 | 236 | jmp(done); |
twisti@4318 | 237 | |
twisti@4318 | 238 | bind(try_rebias); |
twisti@4318 | 239 | // At this point we know the epoch has expired, meaning that the |
twisti@4318 | 240 | // current "bias owner", if any, is actually invalid. Under these |
twisti@4318 | 241 | // circumstances _only_, we are allowed to use the current header's |
twisti@4318 | 242 | // value as the comparison value when doing the cas to acquire the |
twisti@4318 | 243 | // bias in the current epoch. In other words, we allow transfer of |
twisti@4318 | 244 | // the bias from one thread to another directly in this situation. |
twisti@4318 | 245 | // |
twisti@4318 | 246 | // FIXME: due to a lack of registers we currently blow away the age |
twisti@4318 | 247 | // bits in this situation. Should attempt to preserve them. |
twisti@4318 | 248 | if (need_tmp_reg) { |
twisti@4318 | 249 | push(tmp_reg); |
twisti@4318 | 250 | } |
twisti@4318 | 251 | get_thread(tmp_reg); |
twisti@4318 | 252 | movl(swap_reg, klass_addr); |
twisti@4318 | 253 | orl(tmp_reg, Address(swap_reg, Klass::prototype_header_offset())); |
twisti@4318 | 254 | movl(swap_reg, saved_mark_addr); |
twisti@4318 | 255 | if (os::is_MP()) { |
twisti@4318 | 256 | lock(); |
twisti@4318 | 257 | } |
twisti@4318 | 258 | cmpxchgptr(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 259 | if (need_tmp_reg) { |
twisti@4318 | 260 | pop(tmp_reg); |
twisti@4318 | 261 | } |
twisti@4318 | 262 | // If the biasing toward our thread failed, then another thread |
twisti@4318 | 263 | // succeeded in biasing it toward itself and we need to revoke that |
twisti@4318 | 264 | // bias. The revocation will occur in the runtime in the slow case. |
twisti@4318 | 265 | if (counters != NULL) { |
twisti@4318 | 266 | cond_inc32(Assembler::zero, |
twisti@4318 | 267 | ExternalAddress((address)counters->rebiased_lock_entry_count_addr())); |
twisti@4318 | 268 | } |
twisti@4318 | 269 | if (slow_case != NULL) { |
twisti@4318 | 270 | jcc(Assembler::notZero, *slow_case); |
twisti@4318 | 271 | } |
twisti@4318 | 272 | jmp(done); |
twisti@4318 | 273 | |
twisti@4318 | 274 | bind(try_revoke_bias); |
twisti@4318 | 275 | // The prototype mark in the klass doesn't have the bias bit set any |
twisti@4318 | 276 | // more, indicating that objects of this data type are not supposed |
twisti@4318 | 277 | // to be biased any more. We are going to try to reset the mark of |
twisti@4318 | 278 | // this object to the prototype value and fall through to the |
twisti@4318 | 279 | // CAS-based locking scheme. Note that if our CAS fails, it means |
twisti@4318 | 280 | // that another thread raced us for the privilege of revoking the |
twisti@4318 | 281 | // bias of this particular object, so it's okay to continue in the |
twisti@4318 | 282 | // normal locking code. |
twisti@4318 | 283 | // |
twisti@4318 | 284 | // FIXME: due to a lack of registers we currently blow away the age |
twisti@4318 | 285 | // bits in this situation. Should attempt to preserve them. |
twisti@4318 | 286 | movl(swap_reg, saved_mark_addr); |
twisti@4318 | 287 | if (need_tmp_reg) { |
twisti@4318 | 288 | push(tmp_reg); |
twisti@4318 | 289 | } |
twisti@4318 | 290 | movl(tmp_reg, klass_addr); |
twisti@4318 | 291 | movl(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset())); |
twisti@4318 | 292 | if (os::is_MP()) { |
twisti@4318 | 293 | lock(); |
twisti@4318 | 294 | } |
twisti@4318 | 295 | cmpxchgptr(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 296 | if (need_tmp_reg) { |
twisti@4318 | 297 | pop(tmp_reg); |
twisti@4318 | 298 | } |
twisti@4318 | 299 | // Fall through to the normal CAS-based lock, because no matter what |
twisti@4318 | 300 | // the result of the above CAS, some thread must have succeeded in |
twisti@4318 | 301 | // removing the bias bit from the object's header. |
twisti@4318 | 302 | if (counters != NULL) { |
twisti@4318 | 303 | cond_inc32(Assembler::zero, |
twisti@4318 | 304 | ExternalAddress((address)counters->revoked_lock_entry_count_addr())); |
twisti@4318 | 305 | } |
twisti@4318 | 306 | |
twisti@4318 | 307 | bind(cas_label); |
twisti@4318 | 308 | |
twisti@4318 | 309 | return null_check_offset; |
twisti@4318 | 310 | } |
twisti@4318 | 311 | void MacroAssembler::call_VM_leaf_base(address entry_point, |
twisti@4318 | 312 | int number_of_arguments) { |
twisti@4318 | 313 | call(RuntimeAddress(entry_point)); |
twisti@4318 | 314 | increment(rsp, number_of_arguments * wordSize); |
twisti@4318 | 315 | } |
twisti@4318 | 316 | |
twisti@4318 | 317 | void MacroAssembler::cmpklass(Address src1, Metadata* obj) { |
twisti@4318 | 318 | cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 319 | } |
twisti@4318 | 320 | |
twisti@4318 | 321 | void MacroAssembler::cmpklass(Register src1, Metadata* obj) { |
twisti@4318 | 322 | cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 323 | } |
twisti@4318 | 324 | |
twisti@4318 | 325 | void MacroAssembler::cmpoop(Address src1, jobject obj) { |
twisti@4318 | 326 | cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 327 | } |
twisti@4318 | 328 | |
twisti@4318 | 329 | void MacroAssembler::cmpoop(Register src1, jobject obj) { |
twisti@4318 | 330 | cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 331 | } |
twisti@4318 | 332 | |
twisti@4318 | 333 | void MacroAssembler::extend_sign(Register hi, Register lo) { |
twisti@4318 | 334 | // According to Intel Doc. AP-526, "Integer Divide", p.18. |
twisti@4318 | 335 | if (VM_Version::is_P6() && hi == rdx && lo == rax) { |
twisti@4318 | 336 | cdql(); |
twisti@4318 | 337 | } else { |
twisti@4318 | 338 | movl(hi, lo); |
twisti@4318 | 339 | sarl(hi, 31); |
twisti@4318 | 340 | } |
twisti@4318 | 341 | } |
twisti@4318 | 342 | |
twisti@4318 | 343 | void MacroAssembler::jC2(Register tmp, Label& L) { |
twisti@4318 | 344 | // set parity bit if FPU flag C2 is set (via rax) |
twisti@4318 | 345 | save_rax(tmp); |
twisti@4318 | 346 | fwait(); fnstsw_ax(); |
twisti@4318 | 347 | sahf(); |
twisti@4318 | 348 | restore_rax(tmp); |
twisti@4318 | 349 | // branch |
twisti@4318 | 350 | jcc(Assembler::parity, L); |
twisti@4318 | 351 | } |
twisti@4318 | 352 | |
twisti@4318 | 353 | void MacroAssembler::jnC2(Register tmp, Label& L) { |
twisti@4318 | 354 | // set parity bit if FPU flag C2 is set (via rax) |
twisti@4318 | 355 | save_rax(tmp); |
twisti@4318 | 356 | fwait(); fnstsw_ax(); |
twisti@4318 | 357 | sahf(); |
twisti@4318 | 358 | restore_rax(tmp); |
twisti@4318 | 359 | // branch |
twisti@4318 | 360 | jcc(Assembler::noParity, L); |
twisti@4318 | 361 | } |
twisti@4318 | 362 | |
twisti@4318 | 363 | // 32bit can do a case table jump in one instruction but we no longer allow the base |
twisti@4318 | 364 | // to be installed in the Address class |
twisti@4318 | 365 | void MacroAssembler::jump(ArrayAddress entry) { |
twisti@4318 | 366 | jmp(as_Address(entry)); |
twisti@4318 | 367 | } |
twisti@4318 | 368 | |
twisti@4318 | 369 | // Note: y_lo will be destroyed |
twisti@4318 | 370 | void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
twisti@4318 | 371 | // Long compare for Java (semantics as described in JVM spec.) |
twisti@4318 | 372 | Label high, low, done; |
twisti@4318 | 373 | |
twisti@4318 | 374 | cmpl(x_hi, y_hi); |
twisti@4318 | 375 | jcc(Assembler::less, low); |
twisti@4318 | 376 | jcc(Assembler::greater, high); |
twisti@4318 | 377 | // x_hi is the return register |
twisti@4318 | 378 | xorl(x_hi, x_hi); |
twisti@4318 | 379 | cmpl(x_lo, y_lo); |
twisti@4318 | 380 | jcc(Assembler::below, low); |
twisti@4318 | 381 | jcc(Assembler::equal, done); |
twisti@4318 | 382 | |
twisti@4318 | 383 | bind(high); |
twisti@4318 | 384 | xorl(x_hi, x_hi); |
twisti@4318 | 385 | increment(x_hi); |
twisti@4318 | 386 | jmp(done); |
twisti@4318 | 387 | |
twisti@4318 | 388 | bind(low); |
twisti@4318 | 389 | xorl(x_hi, x_hi); |
twisti@4318 | 390 | decrementl(x_hi); |
twisti@4318 | 391 | |
twisti@4318 | 392 | bind(done); |
twisti@4318 | 393 | } |
twisti@4318 | 394 | |
twisti@4318 | 395 | void MacroAssembler::lea(Register dst, AddressLiteral src) { |
twisti@4318 | 396 | mov_literal32(dst, (int32_t)src.target(), src.rspec()); |
twisti@4318 | 397 | } |
twisti@4318 | 398 | |
twisti@4318 | 399 | void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
twisti@4318 | 400 | // leal(dst, as_Address(adr)); |
twisti@4318 | 401 | // see note in movl as to why we must use a move |
twisti@4318 | 402 | mov_literal32(dst, (int32_t) adr.target(), adr.rspec()); |
twisti@4318 | 403 | } |
twisti@4318 | 404 | |
twisti@4318 | 405 | void MacroAssembler::leave() { |
twisti@4318 | 406 | mov(rsp, rbp); |
twisti@4318 | 407 | pop(rbp); |
twisti@4318 | 408 | } |
twisti@4318 | 409 | |
twisti@4318 | 410 | void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) { |
twisti@4318 | 411 | // Multiplication of two Java long values stored on the stack |
twisti@4318 | 412 | // as illustrated below. Result is in rdx:rax. |
twisti@4318 | 413 | // |
twisti@4318 | 414 | // rsp ---> [ ?? ] \ \ |
twisti@4318 | 415 | // .... | y_rsp_offset | |
twisti@4318 | 416 | // [ y_lo ] / (in bytes) | x_rsp_offset |
twisti@4318 | 417 | // [ y_hi ] | (in bytes) |
twisti@4318 | 418 | // .... | |
twisti@4318 | 419 | // [ x_lo ] / |
twisti@4318 | 420 | // [ x_hi ] |
twisti@4318 | 421 | // .... |
twisti@4318 | 422 | // |
twisti@4318 | 423 | // Basic idea: lo(result) = lo(x_lo * y_lo) |
twisti@4318 | 424 | // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi) |
twisti@4318 | 425 | Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset); |
twisti@4318 | 426 | Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset); |
twisti@4318 | 427 | Label quick; |
twisti@4318 | 428 | // load x_hi, y_hi and check if quick |
twisti@4318 | 429 | // multiplication is possible |
twisti@4318 | 430 | movl(rbx, x_hi); |
twisti@4318 | 431 | movl(rcx, y_hi); |
twisti@4318 | 432 | movl(rax, rbx); |
twisti@4318 | 433 | orl(rbx, rcx); // rbx, = 0 <=> x_hi = 0 and y_hi = 0 |
twisti@4318 | 434 | jcc(Assembler::zero, quick); // if rbx, = 0 do quick multiply |
twisti@4318 | 435 | // do full multiplication |
twisti@4318 | 436 | // 1st step |
twisti@4318 | 437 | mull(y_lo); // x_hi * y_lo |
twisti@4318 | 438 | movl(rbx, rax); // save lo(x_hi * y_lo) in rbx, |
twisti@4318 | 439 | // 2nd step |
twisti@4318 | 440 | movl(rax, x_lo); |
twisti@4318 | 441 | mull(rcx); // x_lo * y_hi |
twisti@4318 | 442 | addl(rbx, rax); // add lo(x_lo * y_hi) to rbx, |
twisti@4318 | 443 | // 3rd step |
twisti@4318 | 444 | bind(quick); // note: rbx, = 0 if quick multiply! |
twisti@4318 | 445 | movl(rax, x_lo); |
twisti@4318 | 446 | mull(y_lo); // x_lo * y_lo |
twisti@4318 | 447 | addl(rdx, rbx); // correct hi(x_lo * y_lo) |
twisti@4318 | 448 | } |
twisti@4318 | 449 | |
twisti@4318 | 450 | void MacroAssembler::lneg(Register hi, Register lo) { |
twisti@4318 | 451 | negl(lo); |
twisti@4318 | 452 | adcl(hi, 0); |
twisti@4318 | 453 | negl(hi); |
twisti@4318 | 454 | } |
twisti@4318 | 455 | |
twisti@4318 | 456 | void MacroAssembler::lshl(Register hi, Register lo) { |
twisti@4318 | 457 | // Java shift left long support (semantics as described in JVM spec., p.305) |
twisti@4318 | 458 | // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n)) |
twisti@4318 | 459 | // shift value is in rcx ! |
twisti@4318 | 460 | assert(hi != rcx, "must not use rcx"); |
twisti@4318 | 461 | assert(lo != rcx, "must not use rcx"); |
twisti@4318 | 462 | const Register s = rcx; // shift count |
twisti@4318 | 463 | const int n = BitsPerWord; |
twisti@4318 | 464 | Label L; |
twisti@4318 | 465 | andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
twisti@4318 | 466 | cmpl(s, n); // if (s < n) |
twisti@4318 | 467 | jcc(Assembler::less, L); // else (s >= n) |
twisti@4318 | 468 | movl(hi, lo); // x := x << n |
twisti@4318 | 469 | xorl(lo, lo); |
twisti@4318 | 470 | // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
twisti@4318 | 471 | bind(L); // s (mod n) < n |
twisti@4318 | 472 | shldl(hi, lo); // x := x << s |
twisti@4318 | 473 | shll(lo); |
twisti@4318 | 474 | } |
twisti@4318 | 475 | |
twisti@4318 | 476 | |
twisti@4318 | 477 | void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) { |
twisti@4318 | 478 | // Java shift right long support (semantics as described in JVM spec., p.306 & p.310) |
twisti@4318 | 479 | // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n)) |
twisti@4318 | 480 | assert(hi != rcx, "must not use rcx"); |
twisti@4318 | 481 | assert(lo != rcx, "must not use rcx"); |
twisti@4318 | 482 | const Register s = rcx; // shift count |
twisti@4318 | 483 | const int n = BitsPerWord; |
twisti@4318 | 484 | Label L; |
twisti@4318 | 485 | andl(s, 0x3f); // s := s & 0x3f (s < 0x40) |
twisti@4318 | 486 | cmpl(s, n); // if (s < n) |
twisti@4318 | 487 | jcc(Assembler::less, L); // else (s >= n) |
twisti@4318 | 488 | movl(lo, hi); // x := x >> n |
twisti@4318 | 489 | if (sign_extension) sarl(hi, 31); |
twisti@4318 | 490 | else xorl(hi, hi); |
twisti@4318 | 491 | // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n! |
twisti@4318 | 492 | bind(L); // s (mod n) < n |
twisti@4318 | 493 | shrdl(lo, hi); // x := x >> s |
twisti@4318 | 494 | if (sign_extension) sarl(hi); |
twisti@4318 | 495 | else shrl(hi); |
twisti@4318 | 496 | } |
twisti@4318 | 497 | |
twisti@4318 | 498 | void MacroAssembler::movoop(Register dst, jobject obj) { |
twisti@4318 | 499 | mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 500 | } |
twisti@4318 | 501 | |
twisti@4318 | 502 | void MacroAssembler::movoop(Address dst, jobject obj) { |
twisti@4318 | 503 | mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 504 | } |
twisti@4318 | 505 | |
twisti@4318 | 506 | void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
twisti@4318 | 507 | mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 508 | } |
twisti@4318 | 509 | |
twisti@4318 | 510 | void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
twisti@4318 | 511 | mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 512 | } |
twisti@4318 | 513 | |
twisti@4318 | 514 | void MacroAssembler::movptr(Register dst, AddressLiteral src) { |
twisti@4318 | 515 | if (src.is_lval()) { |
twisti@4318 | 516 | mov_literal32(dst, (intptr_t)src.target(), src.rspec()); |
twisti@4318 | 517 | } else { |
twisti@4318 | 518 | movl(dst, as_Address(src)); |
twisti@4318 | 519 | } |
twisti@4318 | 520 | } |
twisti@4318 | 521 | |
twisti@4318 | 522 | void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
twisti@4318 | 523 | movl(as_Address(dst), src); |
twisti@4318 | 524 | } |
twisti@4318 | 525 | |
twisti@4318 | 526 | void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
twisti@4318 | 527 | movl(dst, as_Address(src)); |
twisti@4318 | 528 | } |
twisti@4318 | 529 | |
twisti@4318 | 530 | // src should NEVER be a real pointer. Use AddressLiteral for true pointers |
twisti@4318 | 531 | void MacroAssembler::movptr(Address dst, intptr_t src) { |
twisti@4318 | 532 | movl(dst, src); |
twisti@4318 | 533 | } |
twisti@4318 | 534 | |
twisti@4318 | 535 | |
twisti@4318 | 536 | void MacroAssembler::pop_callee_saved_registers() { |
twisti@4318 | 537 | pop(rcx); |
twisti@4318 | 538 | pop(rdx); |
twisti@4318 | 539 | pop(rdi); |
twisti@4318 | 540 | pop(rsi); |
twisti@4318 | 541 | } |
twisti@4318 | 542 | |
twisti@4318 | 543 | void MacroAssembler::pop_fTOS() { |
twisti@4318 | 544 | fld_d(Address(rsp, 0)); |
twisti@4318 | 545 | addl(rsp, 2 * wordSize); |
twisti@4318 | 546 | } |
twisti@4318 | 547 | |
twisti@4318 | 548 | void MacroAssembler::push_callee_saved_registers() { |
twisti@4318 | 549 | push(rsi); |
twisti@4318 | 550 | push(rdi); |
twisti@4318 | 551 | push(rdx); |
twisti@4318 | 552 | push(rcx); |
twisti@4318 | 553 | } |
twisti@4318 | 554 | |
twisti@4318 | 555 | void MacroAssembler::push_fTOS() { |
twisti@4318 | 556 | subl(rsp, 2 * wordSize); |
twisti@4318 | 557 | fstp_d(Address(rsp, 0)); |
twisti@4318 | 558 | } |
twisti@4318 | 559 | |
twisti@4318 | 560 | |
twisti@4318 | 561 | void MacroAssembler::pushoop(jobject obj) { |
twisti@4318 | 562 | push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 563 | } |
twisti@4318 | 564 | |
twisti@4318 | 565 | void MacroAssembler::pushklass(Metadata* obj) { |
twisti@4318 | 566 | push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 567 | } |
twisti@4318 | 568 | |
twisti@4318 | 569 | void MacroAssembler::pushptr(AddressLiteral src) { |
twisti@4318 | 570 | if (src.is_lval()) { |
twisti@4318 | 571 | push_literal32((int32_t)src.target(), src.rspec()); |
twisti@4318 | 572 | } else { |
twisti@4318 | 573 | pushl(as_Address(src)); |
twisti@4318 | 574 | } |
twisti@4318 | 575 | } |
twisti@4318 | 576 | |
twisti@4318 | 577 | void MacroAssembler::set_word_if_not_zero(Register dst) { |
twisti@4318 | 578 | xorl(dst, dst); |
twisti@4318 | 579 | set_byte_if_not_zero(dst); |
twisti@4318 | 580 | } |
twisti@4318 | 581 | |
twisti@4318 | 582 | static void pass_arg0(MacroAssembler* masm, Register arg) { |
twisti@4318 | 583 | masm->push(arg); |
twisti@4318 | 584 | } |
twisti@4318 | 585 | |
twisti@4318 | 586 | static void pass_arg1(MacroAssembler* masm, Register arg) { |
twisti@4318 | 587 | masm->push(arg); |
twisti@4318 | 588 | } |
twisti@4318 | 589 | |
twisti@4318 | 590 | static void pass_arg2(MacroAssembler* masm, Register arg) { |
twisti@4318 | 591 | masm->push(arg); |
twisti@4318 | 592 | } |
twisti@4318 | 593 | |
twisti@4318 | 594 | static void pass_arg3(MacroAssembler* masm, Register arg) { |
twisti@4318 | 595 | masm->push(arg); |
twisti@4318 | 596 | } |
twisti@4318 | 597 | |
twisti@4318 | 598 | #ifndef PRODUCT |
twisti@4318 | 599 | extern "C" void findpc(intptr_t x); |
twisti@4318 | 600 | #endif |
twisti@4318 | 601 | |
twisti@4318 | 602 | void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) { |
twisti@4318 | 603 | // In order to get locks to work, we need to fake a in_VM state |
twisti@4318 | 604 | JavaThread* thread = JavaThread::current(); |
twisti@4318 | 605 | JavaThreadState saved_state = thread->thread_state(); |
twisti@4318 | 606 | thread->set_thread_state(_thread_in_vm); |
twisti@4318 | 607 | if (ShowMessageBoxOnError) { |
twisti@4318 | 608 | JavaThread* thread = JavaThread::current(); |
twisti@4318 | 609 | JavaThreadState saved_state = thread->thread_state(); |
twisti@4318 | 610 | thread->set_thread_state(_thread_in_vm); |
twisti@4318 | 611 | if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
twisti@4318 | 612 | ttyLocker ttyl; |
twisti@4318 | 613 | BytecodeCounter::print(); |
twisti@4318 | 614 | } |
twisti@4318 | 615 | // To see where a verify_oop failed, get $ebx+40/X for this frame. |
twisti@4318 | 616 | // This is the value of eip which points to where verify_oop will return. |
twisti@4318 | 617 | if (os::message_box(msg, "Execution stopped, print registers?")) { |
twisti@4318 | 618 | print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip); |
twisti@4318 | 619 | BREAKPOINT; |
twisti@4318 | 620 | } |
twisti@4318 | 621 | } else { |
twisti@4318 | 622 | ttyLocker ttyl; |
twisti@4318 | 623 | ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg); |
twisti@4318 | 624 | } |
twisti@4318 | 625 | // Don't assert holding the ttyLock |
twisti@4318 | 626 | assert(false, err_msg("DEBUG MESSAGE: %s", msg)); |
twisti@4318 | 627 | ThreadStateTransition::transition(thread, _thread_in_vm, saved_state); |
twisti@4318 | 628 | } |
twisti@4318 | 629 | |
twisti@4318 | 630 | void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) { |
twisti@4318 | 631 | ttyLocker ttyl; |
twisti@4318 | 632 | FlagSetting fs(Debugging, true); |
twisti@4318 | 633 | tty->print_cr("eip = 0x%08x", eip); |
twisti@4318 | 634 | #ifndef PRODUCT |
twisti@4318 | 635 | if ((WizardMode || Verbose) && PrintMiscellaneous) { |
twisti@4318 | 636 | tty->cr(); |
twisti@4318 | 637 | findpc(eip); |
twisti@4318 | 638 | tty->cr(); |
twisti@4318 | 639 | } |
twisti@4318 | 640 | #endif |
twisti@4318 | 641 | #define PRINT_REG(rax) \ |
twisti@4318 | 642 | { tty->print("%s = ", #rax); os::print_location(tty, rax); } |
twisti@4318 | 643 | PRINT_REG(rax); |
twisti@4318 | 644 | PRINT_REG(rbx); |
twisti@4318 | 645 | PRINT_REG(rcx); |
twisti@4318 | 646 | PRINT_REG(rdx); |
twisti@4318 | 647 | PRINT_REG(rdi); |
twisti@4318 | 648 | PRINT_REG(rsi); |
twisti@4318 | 649 | PRINT_REG(rbp); |
twisti@4318 | 650 | PRINT_REG(rsp); |
twisti@4318 | 651 | #undef PRINT_REG |
twisti@4318 | 652 | // Print some words near top of staack. |
twisti@4318 | 653 | int* dump_sp = (int*) rsp; |
twisti@4318 | 654 | for (int col1 = 0; col1 < 8; col1++) { |
twisti@4318 | 655 | tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
twisti@4318 | 656 | os::print_location(tty, *dump_sp++); |
twisti@4318 | 657 | } |
twisti@4318 | 658 | for (int row = 0; row < 16; row++) { |
twisti@4318 | 659 | tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp); |
twisti@4318 | 660 | for (int col = 0; col < 8; col++) { |
twisti@4318 | 661 | tty->print(" 0x%08x", *dump_sp++); |
twisti@4318 | 662 | } |
twisti@4318 | 663 | tty->cr(); |
twisti@4318 | 664 | } |
twisti@4318 | 665 | // Print some instructions around pc: |
twisti@4318 | 666 | Disassembler::decode((address)eip-64, (address)eip); |
twisti@4318 | 667 | tty->print_cr("--------"); |
twisti@4318 | 668 | Disassembler::decode((address)eip, (address)eip+32); |
twisti@4318 | 669 | } |
twisti@4318 | 670 | |
twisti@4318 | 671 | void MacroAssembler::stop(const char* msg) { |
twisti@4318 | 672 | ExternalAddress message((address)msg); |
twisti@4318 | 673 | // push address of message |
twisti@4318 | 674 | pushptr(message.addr()); |
twisti@4318 | 675 | { Label L; call(L, relocInfo::none); bind(L); } // push eip |
twisti@4318 | 676 | pusha(); // push registers |
twisti@4318 | 677 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32))); |
twisti@4318 | 678 | hlt(); |
twisti@4318 | 679 | } |
twisti@4318 | 680 | |
twisti@4318 | 681 | void MacroAssembler::warn(const char* msg) { |
twisti@4318 | 682 | push_CPU_state(); |
twisti@4318 | 683 | |
twisti@4318 | 684 | ExternalAddress message((address) msg); |
twisti@4318 | 685 | // push address of message |
twisti@4318 | 686 | pushptr(message.addr()); |
twisti@4318 | 687 | |
twisti@4318 | 688 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning))); |
twisti@4318 | 689 | addl(rsp, wordSize); // discard argument |
twisti@4318 | 690 | pop_CPU_state(); |
twisti@4318 | 691 | } |
twisti@4318 | 692 | |
twisti@4318 | 693 | void MacroAssembler::print_state() { |
twisti@4318 | 694 | { Label L; call(L, relocInfo::none); bind(L); } // push eip |
twisti@4318 | 695 | pusha(); // push registers |
twisti@4318 | 696 | |
twisti@4318 | 697 | push_CPU_state(); |
twisti@4318 | 698 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32))); |
twisti@4318 | 699 | pop_CPU_state(); |
twisti@4318 | 700 | |
twisti@4318 | 701 | popa(); |
twisti@4318 | 702 | addl(rsp, wordSize); |
twisti@4318 | 703 | } |
twisti@4318 | 704 | |
twisti@4318 | 705 | #else // _LP64 |
twisti@4318 | 706 | |
twisti@4318 | 707 | // 64 bit versions |
twisti@4318 | 708 | |
twisti@4318 | 709 | Address MacroAssembler::as_Address(AddressLiteral adr) { |
twisti@4318 | 710 | // amd64 always does this as a pc-rel |
twisti@4318 | 711 | // we can be absolute or disp based on the instruction type |
twisti@4318 | 712 | // jmp/call are displacements others are absolute |
twisti@4318 | 713 | assert(!adr.is_lval(), "must be rval"); |
twisti@4318 | 714 | assert(reachable(adr), "must be"); |
twisti@4318 | 715 | return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc()); |
twisti@4318 | 716 | |
twisti@4318 | 717 | } |
twisti@4318 | 718 | |
twisti@4318 | 719 | Address MacroAssembler::as_Address(ArrayAddress adr) { |
twisti@4318 | 720 | AddressLiteral base = adr.base(); |
twisti@4318 | 721 | lea(rscratch1, base); |
twisti@4318 | 722 | Address index = adr.index(); |
twisti@4318 | 723 | assert(index._disp == 0, "must not have disp"); // maybe it can? |
twisti@4318 | 724 | Address array(rscratch1, index._index, index._scale, index._disp); |
twisti@4318 | 725 | return array; |
twisti@4318 | 726 | } |
twisti@4318 | 727 | |
twisti@4318 | 728 | int MacroAssembler::biased_locking_enter(Register lock_reg, |
twisti@4318 | 729 | Register obj_reg, |
twisti@4318 | 730 | Register swap_reg, |
twisti@4318 | 731 | Register tmp_reg, |
twisti@4318 | 732 | bool swap_reg_contains_mark, |
twisti@4318 | 733 | Label& done, |
twisti@4318 | 734 | Label* slow_case, |
twisti@4318 | 735 | BiasedLockingCounters* counters) { |
twisti@4318 | 736 | assert(UseBiasedLocking, "why call this otherwise?"); |
twisti@4318 | 737 | assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq"); |
twisti@4318 | 738 | assert(tmp_reg != noreg, "tmp_reg must be supplied"); |
twisti@4318 | 739 | assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg); |
twisti@4318 | 740 | assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout"); |
twisti@4318 | 741 | Address mark_addr (obj_reg, oopDesc::mark_offset_in_bytes()); |
twisti@4318 | 742 | Address saved_mark_addr(lock_reg, 0); |
twisti@4318 | 743 | |
twisti@4318 | 744 | if (PrintBiasedLockingStatistics && counters == NULL) |
twisti@4318 | 745 | counters = BiasedLocking::counters(); |
twisti@4318 | 746 | |
twisti@4318 | 747 | // Biased locking |
twisti@4318 | 748 | // See whether the lock is currently biased toward our thread and |
twisti@4318 | 749 | // whether the epoch is still valid |
twisti@4318 | 750 | // Note that the runtime guarantees sufficient alignment of JavaThread |
twisti@4318 | 751 | // pointers to allow age to be placed into low bits |
twisti@4318 | 752 | // First check to see whether biasing is even enabled for this object |
twisti@4318 | 753 | Label cas_label; |
twisti@4318 | 754 | int null_check_offset = -1; |
twisti@4318 | 755 | if (!swap_reg_contains_mark) { |
twisti@4318 | 756 | null_check_offset = offset(); |
twisti@4318 | 757 | movq(swap_reg, mark_addr); |
twisti@4318 | 758 | } |
twisti@4318 | 759 | movq(tmp_reg, swap_reg); |
twisti@4318 | 760 | andq(tmp_reg, markOopDesc::biased_lock_mask_in_place); |
twisti@4318 | 761 | cmpq(tmp_reg, markOopDesc::biased_lock_pattern); |
twisti@4318 | 762 | jcc(Assembler::notEqual, cas_label); |
twisti@4318 | 763 | // The bias pattern is present in the object's header. Need to check |
twisti@4318 | 764 | // whether the bias owner and the epoch are both still current. |
twisti@4318 | 765 | load_prototype_header(tmp_reg, obj_reg); |
twisti@4318 | 766 | orq(tmp_reg, r15_thread); |
twisti@4318 | 767 | xorq(tmp_reg, swap_reg); |
twisti@4318 | 768 | andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place)); |
twisti@4318 | 769 | if (counters != NULL) { |
twisti@4318 | 770 | cond_inc32(Assembler::zero, |
twisti@4318 | 771 | ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr())); |
twisti@4318 | 772 | } |
twisti@4318 | 773 | jcc(Assembler::equal, done); |
twisti@4318 | 774 | |
twisti@4318 | 775 | Label try_revoke_bias; |
twisti@4318 | 776 | Label try_rebias; |
twisti@4318 | 777 | |
twisti@4318 | 778 | // At this point we know that the header has the bias pattern and |
twisti@4318 | 779 | // that we are not the bias owner in the current epoch. We need to |
twisti@4318 | 780 | // figure out more details about the state of the header in order to |
twisti@4318 | 781 | // know what operations can be legally performed on the object's |
twisti@4318 | 782 | // header. |
twisti@4318 | 783 | |
twisti@4318 | 784 | // If the low three bits in the xor result aren't clear, that means |
twisti@4318 | 785 | // the prototype header is no longer biased and we have to revoke |
twisti@4318 | 786 | // the bias on this object. |
twisti@4318 | 787 | testq(tmp_reg, markOopDesc::biased_lock_mask_in_place); |
twisti@4318 | 788 | jcc(Assembler::notZero, try_revoke_bias); |
twisti@4318 | 789 | |
twisti@4318 | 790 | // Biasing is still enabled for this data type. See whether the |
twisti@4318 | 791 | // epoch of the current bias is still valid, meaning that the epoch |
twisti@4318 | 792 | // bits of the mark word are equal to the epoch bits of the |
twisti@4318 | 793 | // prototype header. (Note that the prototype header's epoch bits |
twisti@4318 | 794 | // only change at a safepoint.) If not, attempt to rebias the object |
twisti@4318 | 795 | // toward the current thread. Note that we must be absolutely sure |
twisti@4318 | 796 | // that the current epoch is invalid in order to do this because |
twisti@4318 | 797 | // otherwise the manipulations it performs on the mark word are |
twisti@4318 | 798 | // illegal. |
twisti@4318 | 799 | testq(tmp_reg, markOopDesc::epoch_mask_in_place); |
twisti@4318 | 800 | jcc(Assembler::notZero, try_rebias); |
twisti@4318 | 801 | |
twisti@4318 | 802 | // The epoch of the current bias is still valid but we know nothing |
twisti@4318 | 803 | // about the owner; it might be set or it might be clear. Try to |
twisti@4318 | 804 | // acquire the bias of the object using an atomic operation. If this |
twisti@4318 | 805 | // fails we will go in to the runtime to revoke the object's bias. |
twisti@4318 | 806 | // Note that we first construct the presumed unbiased header so we |
twisti@4318 | 807 | // don't accidentally blow away another thread's valid bias. |
twisti@4318 | 808 | andq(swap_reg, |
twisti@4318 | 809 | markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place); |
twisti@4318 | 810 | movq(tmp_reg, swap_reg); |
twisti@4318 | 811 | orq(tmp_reg, r15_thread); |
twisti@4318 | 812 | if (os::is_MP()) { |
twisti@4318 | 813 | lock(); |
twisti@4318 | 814 | } |
twisti@4318 | 815 | cmpxchgq(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 816 | // If the biasing toward our thread failed, this means that |
twisti@4318 | 817 | // another thread succeeded in biasing it toward itself and we |
twisti@4318 | 818 | // need to revoke that bias. The revocation will occur in the |
twisti@4318 | 819 | // interpreter runtime in the slow case. |
twisti@4318 | 820 | if (counters != NULL) { |
twisti@4318 | 821 | cond_inc32(Assembler::zero, |
twisti@4318 | 822 | ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr())); |
twisti@4318 | 823 | } |
twisti@4318 | 824 | if (slow_case != NULL) { |
twisti@4318 | 825 | jcc(Assembler::notZero, *slow_case); |
twisti@4318 | 826 | } |
twisti@4318 | 827 | jmp(done); |
twisti@4318 | 828 | |
twisti@4318 | 829 | bind(try_rebias); |
twisti@4318 | 830 | // At this point we know the epoch has expired, meaning that the |
twisti@4318 | 831 | // current "bias owner", if any, is actually invalid. Under these |
twisti@4318 | 832 | // circumstances _only_, we are allowed to use the current header's |
twisti@4318 | 833 | // value as the comparison value when doing the cas to acquire the |
twisti@4318 | 834 | // bias in the current epoch. In other words, we allow transfer of |
twisti@4318 | 835 | // the bias from one thread to another directly in this situation. |
twisti@4318 | 836 | // |
twisti@4318 | 837 | // FIXME: due to a lack of registers we currently blow away the age |
twisti@4318 | 838 | // bits in this situation. Should attempt to preserve them. |
twisti@4318 | 839 | load_prototype_header(tmp_reg, obj_reg); |
twisti@4318 | 840 | orq(tmp_reg, r15_thread); |
twisti@4318 | 841 | if (os::is_MP()) { |
twisti@4318 | 842 | lock(); |
twisti@4318 | 843 | } |
twisti@4318 | 844 | cmpxchgq(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 845 | // If the biasing toward our thread failed, then another thread |
twisti@4318 | 846 | // succeeded in biasing it toward itself and we need to revoke that |
twisti@4318 | 847 | // bias. The revocation will occur in the runtime in the slow case. |
twisti@4318 | 848 | if (counters != NULL) { |
twisti@4318 | 849 | cond_inc32(Assembler::zero, |
twisti@4318 | 850 | ExternalAddress((address) counters->rebiased_lock_entry_count_addr())); |
twisti@4318 | 851 | } |
twisti@4318 | 852 | if (slow_case != NULL) { |
twisti@4318 | 853 | jcc(Assembler::notZero, *slow_case); |
twisti@4318 | 854 | } |
twisti@4318 | 855 | jmp(done); |
twisti@4318 | 856 | |
twisti@4318 | 857 | bind(try_revoke_bias); |
twisti@4318 | 858 | // The prototype mark in the klass doesn't have the bias bit set any |
twisti@4318 | 859 | // more, indicating that objects of this data type are not supposed |
twisti@4318 | 860 | // to be biased any more. We are going to try to reset the mark of |
twisti@4318 | 861 | // this object to the prototype value and fall through to the |
twisti@4318 | 862 | // CAS-based locking scheme. Note that if our CAS fails, it means |
twisti@4318 | 863 | // that another thread raced us for the privilege of revoking the |
twisti@4318 | 864 | // bias of this particular object, so it's okay to continue in the |
twisti@4318 | 865 | // normal locking code. |
twisti@4318 | 866 | // |
twisti@4318 | 867 | // FIXME: due to a lack of registers we currently blow away the age |
twisti@4318 | 868 | // bits in this situation. Should attempt to preserve them. |
twisti@4318 | 869 | load_prototype_header(tmp_reg, obj_reg); |
twisti@4318 | 870 | if (os::is_MP()) { |
twisti@4318 | 871 | lock(); |
twisti@4318 | 872 | } |
twisti@4318 | 873 | cmpxchgq(tmp_reg, Address(obj_reg, 0)); |
twisti@4318 | 874 | // Fall through to the normal CAS-based lock, because no matter what |
twisti@4318 | 875 | // the result of the above CAS, some thread must have succeeded in |
twisti@4318 | 876 | // removing the bias bit from the object's header. |
twisti@4318 | 877 | if (counters != NULL) { |
twisti@4318 | 878 | cond_inc32(Assembler::zero, |
twisti@4318 | 879 | ExternalAddress((address) counters->revoked_lock_entry_count_addr())); |
twisti@4318 | 880 | } |
twisti@4318 | 881 | |
twisti@4318 | 882 | bind(cas_label); |
twisti@4318 | 883 | |
twisti@4318 | 884 | return null_check_offset; |
twisti@4318 | 885 | } |
twisti@4318 | 886 | |
twisti@4318 | 887 | void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) { |
twisti@4318 | 888 | Label L, E; |
twisti@4318 | 889 | |
twisti@4318 | 890 | #ifdef _WIN64 |
twisti@4318 | 891 | // Windows always allocates space for it's register args |
twisti@4318 | 892 | assert(num_args <= 4, "only register arguments supported"); |
twisti@4318 | 893 | subq(rsp, frame::arg_reg_save_area_bytes); |
twisti@4318 | 894 | #endif |
twisti@4318 | 895 | |
twisti@4318 | 896 | // Align stack if necessary |
twisti@4318 | 897 | testl(rsp, 15); |
twisti@4318 | 898 | jcc(Assembler::zero, L); |
twisti@4318 | 899 | |
twisti@4318 | 900 | subq(rsp, 8); |
twisti@4318 | 901 | { |
twisti@4318 | 902 | call(RuntimeAddress(entry_point)); |
twisti@4318 | 903 | } |
twisti@4318 | 904 | addq(rsp, 8); |
twisti@4318 | 905 | jmp(E); |
twisti@4318 | 906 | |
twisti@4318 | 907 | bind(L); |
twisti@4318 | 908 | { |
twisti@4318 | 909 | call(RuntimeAddress(entry_point)); |
twisti@4318 | 910 | } |
twisti@4318 | 911 | |
twisti@4318 | 912 | bind(E); |
twisti@4318 | 913 | |
twisti@4318 | 914 | #ifdef _WIN64 |
twisti@4318 | 915 | // restore stack pointer |
twisti@4318 | 916 | addq(rsp, frame::arg_reg_save_area_bytes); |
twisti@4318 | 917 | #endif |
twisti@4318 | 918 | |
twisti@4318 | 919 | } |
twisti@4318 | 920 | |
twisti@4318 | 921 | void MacroAssembler::cmp64(Register src1, AddressLiteral src2) { |
twisti@4318 | 922 | assert(!src2.is_lval(), "should use cmpptr"); |
twisti@4318 | 923 | |
twisti@4318 | 924 | if (reachable(src2)) { |
twisti@4318 | 925 | cmpq(src1, as_Address(src2)); |
twisti@4318 | 926 | } else { |
twisti@4318 | 927 | lea(rscratch1, src2); |
twisti@4318 | 928 | Assembler::cmpq(src1, Address(rscratch1, 0)); |
twisti@4318 | 929 | } |
twisti@4318 | 930 | } |
twisti@4318 | 931 | |
twisti@4318 | 932 | int MacroAssembler::corrected_idivq(Register reg) { |
twisti@4318 | 933 | // Full implementation of Java ldiv and lrem; checks for special |
twisti@4318 | 934 | // case as described in JVM spec., p.243 & p.271. The function |
twisti@4318 | 935 | // returns the (pc) offset of the idivl instruction - may be needed |
twisti@4318 | 936 | // for implicit exceptions. |
twisti@4318 | 937 | // |
twisti@4318 | 938 | // normal case special case |
twisti@4318 | 939 | // |
twisti@4318 | 940 | // input : rax: dividend min_long |
twisti@4318 | 941 | // reg: divisor (may not be eax/edx) -1 |
twisti@4318 | 942 | // |
twisti@4318 | 943 | // output: rax: quotient (= rax idiv reg) min_long |
twisti@4318 | 944 | // rdx: remainder (= rax irem reg) 0 |
twisti@4318 | 945 | assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register"); |
twisti@4318 | 946 | static const int64_t min_long = 0x8000000000000000; |
twisti@4318 | 947 | Label normal_case, special_case; |
twisti@4318 | 948 | |
twisti@4318 | 949 | // check for special case |
twisti@4318 | 950 | cmp64(rax, ExternalAddress((address) &min_long)); |
twisti@4318 | 951 | jcc(Assembler::notEqual, normal_case); |
twisti@4318 | 952 | xorl(rdx, rdx); // prepare rdx for possible special case (where |
twisti@4318 | 953 | // remainder = 0) |
twisti@4318 | 954 | cmpq(reg, -1); |
twisti@4318 | 955 | jcc(Assembler::equal, special_case); |
twisti@4318 | 956 | |
twisti@4318 | 957 | // handle normal case |
twisti@4318 | 958 | bind(normal_case); |
twisti@4318 | 959 | cdqq(); |
twisti@4318 | 960 | int idivq_offset = offset(); |
twisti@4318 | 961 | idivq(reg); |
twisti@4318 | 962 | |
twisti@4318 | 963 | // normal and special case exit |
twisti@4318 | 964 | bind(special_case); |
twisti@4318 | 965 | |
twisti@4318 | 966 | return idivq_offset; |
twisti@4318 | 967 | } |
twisti@4318 | 968 | |
twisti@4318 | 969 | void MacroAssembler::decrementq(Register reg, int value) { |
twisti@4318 | 970 | if (value == min_jint) { subq(reg, value); return; } |
twisti@4318 | 971 | if (value < 0) { incrementq(reg, -value); return; } |
twisti@4318 | 972 | if (value == 0) { ; return; } |
twisti@4318 | 973 | if (value == 1 && UseIncDec) { decq(reg) ; return; } |
twisti@4318 | 974 | /* else */ { subq(reg, value) ; return; } |
twisti@4318 | 975 | } |
twisti@4318 | 976 | |
twisti@4318 | 977 | void MacroAssembler::decrementq(Address dst, int value) { |
twisti@4318 | 978 | if (value == min_jint) { subq(dst, value); return; } |
twisti@4318 | 979 | if (value < 0) { incrementq(dst, -value); return; } |
twisti@4318 | 980 | if (value == 0) { ; return; } |
twisti@4318 | 981 | if (value == 1 && UseIncDec) { decq(dst) ; return; } |
twisti@4318 | 982 | /* else */ { subq(dst, value) ; return; } |
twisti@4318 | 983 | } |
twisti@4318 | 984 | |
twisti@4318 | 985 | void MacroAssembler::incrementq(Register reg, int value) { |
twisti@4318 | 986 | if (value == min_jint) { addq(reg, value); return; } |
twisti@4318 | 987 | if (value < 0) { decrementq(reg, -value); return; } |
twisti@4318 | 988 | if (value == 0) { ; return; } |
twisti@4318 | 989 | if (value == 1 && UseIncDec) { incq(reg) ; return; } |
twisti@4318 | 990 | /* else */ { addq(reg, value) ; return; } |
twisti@4318 | 991 | } |
twisti@4318 | 992 | |
twisti@4318 | 993 | void MacroAssembler::incrementq(Address dst, int value) { |
twisti@4318 | 994 | if (value == min_jint) { addq(dst, value); return; } |
twisti@4318 | 995 | if (value < 0) { decrementq(dst, -value); return; } |
twisti@4318 | 996 | if (value == 0) { ; return; } |
twisti@4318 | 997 | if (value == 1 && UseIncDec) { incq(dst) ; return; } |
twisti@4318 | 998 | /* else */ { addq(dst, value) ; return; } |
twisti@4318 | 999 | } |
twisti@4318 | 1000 | |
twisti@4318 | 1001 | // 32bit can do a case table jump in one instruction but we no longer allow the base |
twisti@4318 | 1002 | // to be installed in the Address class |
twisti@4318 | 1003 | void MacroAssembler::jump(ArrayAddress entry) { |
twisti@4318 | 1004 | lea(rscratch1, entry.base()); |
twisti@4318 | 1005 | Address dispatch = entry.index(); |
twisti@4318 | 1006 | assert(dispatch._base == noreg, "must be"); |
twisti@4318 | 1007 | dispatch._base = rscratch1; |
twisti@4318 | 1008 | jmp(dispatch); |
twisti@4318 | 1009 | } |
twisti@4318 | 1010 | |
twisti@4318 | 1011 | void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) { |
twisti@4318 | 1012 | ShouldNotReachHere(); // 64bit doesn't use two regs |
twisti@4318 | 1013 | cmpq(x_lo, y_lo); |
twisti@4318 | 1014 | } |
twisti@4318 | 1015 | |
twisti@4318 | 1016 | void MacroAssembler::lea(Register dst, AddressLiteral src) { |
twisti@4318 | 1017 | mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
twisti@4318 | 1018 | } |
twisti@4318 | 1019 | |
twisti@4318 | 1020 | void MacroAssembler::lea(Address dst, AddressLiteral adr) { |
twisti@4318 | 1021 | mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec()); |
twisti@4318 | 1022 | movptr(dst, rscratch1); |
twisti@4318 | 1023 | } |
twisti@4318 | 1024 | |
twisti@4318 | 1025 | void MacroAssembler::leave() { |
twisti@4318 | 1026 | // %%% is this really better? Why not on 32bit too? |
twisti@4366 | 1027 | emit_int8((unsigned char)0xC9); // LEAVE |
twisti@4318 | 1028 | } |
twisti@4318 | 1029 | |
twisti@4318 | 1030 | void MacroAssembler::lneg(Register hi, Register lo) { |
twisti@4318 | 1031 | ShouldNotReachHere(); // 64bit doesn't use two regs |
twisti@4318 | 1032 | negq(lo); |
twisti@4318 | 1033 | } |
twisti@4318 | 1034 | |
twisti@4318 | 1035 | void MacroAssembler::movoop(Register dst, jobject obj) { |
twisti@4318 | 1036 | mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 1037 | } |
twisti@4318 | 1038 | |
twisti@4318 | 1039 | void MacroAssembler::movoop(Address dst, jobject obj) { |
twisti@4318 | 1040 | mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate()); |
twisti@4318 | 1041 | movq(dst, rscratch1); |
twisti@4318 | 1042 | } |
twisti@4318 | 1043 | |
twisti@4318 | 1044 | void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { |
twisti@4318 | 1045 | mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 1046 | } |
twisti@4318 | 1047 | |
twisti@4318 | 1048 | void MacroAssembler::mov_metadata(Address dst, Metadata* obj) { |
twisti@4318 | 1049 | mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate()); |
twisti@4318 | 1050 | movq(dst, rscratch1); |
twisti@4318 | 1051 | } |
twisti@4318 | 1052 | |
twisti@4318 | 1053 | void MacroAssembler::movptr(Register dst, AddressLiteral src) { |
twisti@4318 | 1054 | if (src.is_lval()) { |
twisti@4318 | 1055 | mov_literal64(dst, (intptr_t)src.target(), src.rspec()); |
twisti@4318 | 1056 | } else { |
twisti@4318 | 1057 | if (reachable(src)) { |
twisti@4318 | 1058 | movq(dst, as_Address(src)); |
twisti@4318 | 1059 | } else { |
twisti@4318 | 1060 | lea(rscratch1, src); |
twisti@4318 | 1061 | movq(dst, Address(rscratch1,0)); |
twisti@4318 | 1062 | } |
twisti@4318 | 1063 | } |
twisti@4318 | 1064 | } |
twisti@4318 | 1065 | |
twisti@4318 | 1066 | void MacroAssembler::movptr(ArrayAddress dst, Register src) { |
twisti@4318 | 1067 | movq(as_Address(dst), src); |
twisti@4318 | 1068 | } |
twisti@4318 | 1069 | |
twisti@4318 | 1070 | void MacroAssembler::movptr(Register dst, ArrayAddress src) { |
twisti@4318 | 1071 | movq(dst, as_Address(src)); |
twisti@4318 | 1072 | } |
twisti@4318 | 1073 | |
twisti@4318 | 1074 | // src should NEVER be a real pointer. Use AddressLiteral for true pointers |
twisti@4318 | 1075 | void MacroAssembler::movptr(Address dst, intptr_t src) { |
twisti@4318 | 1076 | mov64(rscratch1, src); |
twisti@4318 | 1077 | movq(dst, rscratch1); |
twisti@4318 | 1078 | } |
twisti@4318 | 1079 | |
twisti@4318 | 1080 | // These are mostly for initializing NULL |
twisti@4318 | 1081 | void MacroAssembler::movptr(Address dst, int32_t src) { |
twisti@4318 | 1082 | movslq(dst, src); |
twisti@4318 | 1083 | } |
twisti@4318 | 1084 | |
twisti@4318 | 1085 | void MacroAssembler::movptr(Register dst, int32_t src) { |
twisti@4318 | 1086 | mov64(dst, (intptr_t)src); |
twisti@4318 | 1087 | } |
twisti@4318 | 1088 | |
twisti@4318 | 1089 | void MacroAssembler::pushoop(jobject obj) { |
twisti@4318 | 1090 | movoop(rscratch1, obj); |
twisti@4318 | 1091 | push(rscratch1); |
twisti@4318 | 1092 | } |
twisti@4318 | 1093 | |
twisti@4318 | 1094 | void MacroAssembler::pushklass(Metadata* obj) { |
twisti@4318 | 1095 | mov_metadata(rscratch1, obj); |
twisti@4318 | 1096 | push(rscratch1); |
twisti@4318 | 1097 | } |
twisti@4318 | 1098 | |
twisti@4318 | 1099 | void MacroAssembler::pushptr(AddressLiteral src) { |
twisti@4318 | 1100 | lea(rscratch1, src); |
twisti@4318 | 1101 | if (src.is_lval()) { |
twisti@4318 | 1102 | push(rscratch1); |
twisti@4318 | 1103 | } else { |
twisti@4318 | 1104 | pushq(Address(rscratch1, 0)); |
twisti@4318 | 1105 | } |
twisti@4318 | 1106 | } |
twisti@4318 | 1107 | |
twisti@4318 | 1108 | void MacroAssembler::reset_last_Java_frame(bool clear_fp, |
twisti@4318 | 1109 | bool clear_pc) { |
twisti@4318 | 1110 | // we must set sp to zero to clear frame |
twisti@4318 | 1111 | movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
twisti@4318 | 1112 | // must clear fp, so that compiled frames are not confused; it is |
twisti@4318 | 1113 | // possible that we need it only for debugging |
twisti@4318 | 1114 | if (clear_fp) { |
twisti@4318 | 1115 | movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
twisti@4318 | 1116 | } |
twisti@4318 | 1117 | |
twisti@4318 | 1118 | if (clear_pc) { |
twisti@4318 | 1119 | movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
twisti@4318 | 1120 | } |
twisti@4318 | 1121 | } |
twisti@4318 | 1122 | |
twisti@4318 | 1123 | void MacroAssembler::set_last_Java_frame(Register last_java_sp, |
twisti@4318 | 1124 | Register last_java_fp, |
twisti@4318 | 1125 | address last_java_pc) { |
twisti@4318 | 1126 | // determine last_java_sp register |
twisti@4318 | 1127 | if (!last_java_sp->is_valid()) { |
twisti@4318 | 1128 | last_java_sp = rsp; |
twisti@4318 | 1129 | } |
twisti@4318 | 1130 | |
twisti@4318 | 1131 | // last_java_fp is optional |
twisti@4318 | 1132 | if (last_java_fp->is_valid()) { |
twisti@4318 | 1133 | movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), |
twisti@4318 | 1134 | last_java_fp); |
twisti@4318 | 1135 | } |
twisti@4318 | 1136 | |
twisti@4318 | 1137 | // last_java_pc is optional |
twisti@4318 | 1138 | if (last_java_pc != NULL) { |
twisti@4318 | 1139 | Address java_pc(r15_thread, |
twisti@4318 | 1140 | JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()); |
twisti@4318 | 1141 | lea(rscratch1, InternalAddress(last_java_pc)); |
twisti@4318 | 1142 | movptr(java_pc, rscratch1); |
twisti@4318 | 1143 | } |
twisti@4318 | 1144 | |
twisti@4318 | 1145 | movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
twisti@4318 | 1146 | } |
twisti@4318 | 1147 | |
twisti@4318 | 1148 | static void pass_arg0(MacroAssembler* masm, Register arg) { |
twisti@4318 | 1149 | if (c_rarg0 != arg ) { |
twisti@4318 | 1150 | masm->mov(c_rarg0, arg); |
twisti@4318 | 1151 | } |
twisti@4318 | 1152 | } |
twisti@4318 | 1153 | |
twisti@4318 | 1154 | static void pass_arg1(MacroAssembler* masm, Register arg) { |
twisti@4318 | 1155 | if (c_rarg1 != arg ) { |
twisti@4318 | 1156 | masm->mov(c_rarg1, arg); |
twisti@4318 | 1157 | } |
twisti@4318 | 1158 | } |
twisti@4318 | 1159 | |
twisti@4318 | 1160 | static void pass_arg2(MacroAssembler* masm, Register arg) { |
twisti@4318 | 1161 | if (c_rarg2 != arg ) { |
twisti@4318 | 1162 | masm->mov(c_rarg2, arg); |
twisti@4318 | 1163 | } |
twisti@4318 | 1164 | } |
twisti@4318 | 1165 | |
twisti@4318 | 1166 | static void pass_arg3(MacroAssembler* masm, Register arg) { |
twisti@4318 | 1167 | if (c_rarg3 != arg ) { |
twisti@4318 | 1168 | masm->mov(c_rarg3, arg); |
twisti@4318 | 1169 | } |
twisti@4318 | 1170 | } |
twisti@4318 | 1171 | |
twisti@4318 | 1172 | void MacroAssembler::stop(const char* msg) { |
twisti@4318 | 1173 | address rip = pc(); |
twisti@4318 | 1174 | pusha(); // get regs on stack |
twisti@4318 | 1175 | lea(c_rarg0, ExternalAddress((address) msg)); |
twisti@4318 | 1176 | lea(c_rarg1, InternalAddress(rip)); |
twisti@4318 | 1177 | movq(c_rarg2, rsp); // pass pointer to regs array |
twisti@4318 | 1178 | andq(rsp, -16); // align stack as required by ABI |
twisti@4318 | 1179 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); |
twisti@4318 | 1180 | hlt(); |
twisti@4318 | 1181 | } |
twisti@4318 | 1182 | |
twisti@4318 | 1183 | void MacroAssembler::warn(const char* msg) { |
twisti@4318 | 1184 | push(rbp); |
twisti@4318 | 1185 | movq(rbp, rsp); |
twisti@4318 | 1186 | andq(rsp, -16); // align stack as required by push_CPU_state and call |
twisti@4318 | 1187 | push_CPU_state(); // keeps alignment at 16 bytes |
twisti@4318 | 1188 | lea(c_rarg0, ExternalAddress((address) msg)); |
twisti@4318 | 1189 | call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0); |
twisti@4318 | 1190 | pop_CPU_state(); |
twisti@4318 | 1191 | mov(rsp, rbp); |
twisti@4318 | 1192 | pop(rbp); |
twisti@4318 | 1193 | } |
twisti@4318 | 1194 | |
twisti@4318 | 1195 | void MacroAssembler::print_state() { |
twisti@4318 | 1196 | address rip = pc(); |
twisti@4318 | 1197 | pusha(); // get regs on stack |
twisti@4318 | 1198 | push(rbp); |
twisti@4318 | 1199 | movq(rbp, rsp); |
twisti@4318 | 1200 | andq(rsp, -16); // align stack as required by push_CPU_state and call |
twisti@4318 | 1201 | push_CPU_state(); // keeps alignment at 16 bytes |
twisti@4318 | 1202 | |
twisti@4318 | 1203 | lea(c_rarg0, InternalAddress(rip)); |
twisti@4318 | 1204 | lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array |
twisti@4318 | 1205 | call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1); |
twisti@4318 | 1206 | |
twisti@4318 | 1207 | pop_CPU_state(); |
twisti@4318 | 1208 | mov(rsp, rbp); |
twisti@4318 | 1209 | pop(rbp); |
twisti@4318 | 1210 | popa(); |
twisti@4318 | 1211 | } |
twisti@4318 | 1212 | |
twisti@4318 | 1213 | #ifndef PRODUCT |
twisti@4318 | 1214 | extern "C" void findpc(intptr_t x); |
twisti@4318 | 1215 | #endif |
twisti@4318 | 1216 | |
twisti@4318 | 1217 | void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) { |
twisti@4318 | 1218 | // In order to get locks to work, we need to fake a in_VM state |
twisti@4318 | 1219 | if (ShowMessageBoxOnError) { |
twisti@4318 | 1220 | JavaThread* thread = JavaThread::current(); |
twisti@4318 | 1221 | JavaThreadState saved_state = thread->thread_state(); |
twisti@4318 | 1222 | thread->set_thread_state(_thread_in_vm); |
twisti@4318 | 1223 | #ifndef PRODUCT |
twisti@4318 | 1224 | if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { |
twisti@4318 | 1225 | ttyLocker ttyl; |
twisti@4318 | 1226 | BytecodeCounter::print(); |
twisti@4318 | 1227 | } |
twisti@4318 | 1228 | #endif |
twisti@4318 | 1229 | // To see where a verify_oop failed, get $ebx+40/X for this frame. |
twisti@4318 | 1230 | // XXX correct this offset for amd64 |
twisti@4318 | 1231 | // This is the value of eip which points to where verify_oop will return. |
twisti@4318 | 1232 | if (os::message_box(msg, "Execution stopped, print registers?")) { |
twisti@4318 | 1233 | print_state64(pc, regs); |
twisti@4318 | 1234 | BREAKPOINT; |
twisti@4318 | 1235 | assert(false, "start up GDB"); |
twisti@4318 | 1236 | } |
twisti@4318 | 1237 | ThreadStateTransition::transition(thread, _thread_in_vm, saved_state); |
twisti@4318 | 1238 | } else { |
twisti@4318 | 1239 | ttyLocker ttyl; |
twisti@4318 | 1240 | ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", |
twisti@4318 | 1241 | msg); |
twisti@4318 | 1242 | assert(false, err_msg("DEBUG MESSAGE: %s", msg)); |
twisti@4318 | 1243 | } |
twisti@4318 | 1244 | } |
twisti@4318 | 1245 | |
twisti@4318 | 1246 | void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) { |
twisti@4318 | 1247 | ttyLocker ttyl; |
twisti@4318 | 1248 | FlagSetting fs(Debugging, true); |
twisti@4318 | 1249 | tty->print_cr("rip = 0x%016lx", pc); |
twisti@4318 | 1250 | #ifndef PRODUCT |
twisti@4318 | 1251 | tty->cr(); |
twisti@4318 | 1252 | findpc(pc); |
twisti@4318 | 1253 | tty->cr(); |
twisti@4318 | 1254 | #endif |
twisti@4318 | 1255 | #define PRINT_REG(rax, value) \ |
twisti@4318 | 1256 | { tty->print("%s = ", #rax); os::print_location(tty, value); } |
twisti@4318 | 1257 | PRINT_REG(rax, regs[15]); |
twisti@4318 | 1258 | PRINT_REG(rbx, regs[12]); |
twisti@4318 | 1259 | PRINT_REG(rcx, regs[14]); |
twisti@4318 | 1260 | PRINT_REG(rdx, regs[13]); |
twisti@4318 | 1261 | PRINT_REG(rdi, regs[8]); |
twisti@4318 | 1262 | PRINT_REG(rsi, regs[9]); |
twisti@4318 | 1263 | PRINT_REG(rbp, regs[10]); |
twisti@4318 | 1264 | PRINT_REG(rsp, regs[11]); |
twisti@4318 | 1265 | PRINT_REG(r8 , regs[7]); |
twisti@4318 | 1266 | PRINT_REG(r9 , regs[6]); |
twisti@4318 | 1267 | PRINT_REG(r10, regs[5]); |
twisti@4318 | 1268 | PRINT_REG(r11, regs[4]); |
twisti@4318 | 1269 | PRINT_REG(r12, regs[3]); |
twisti@4318 | 1270 | PRINT_REG(r13, regs[2]); |
twisti@4318 | 1271 | PRINT_REG(r14, regs[1]); |
twisti@4318 | 1272 | PRINT_REG(r15, regs[0]); |
twisti@4318 | 1273 | #undef PRINT_REG |
twisti@4318 | 1274 | // Print some words near top of staack. |
twisti@4318 | 1275 | int64_t* rsp = (int64_t*) regs[11]; |
twisti@4318 | 1276 | int64_t* dump_sp = rsp; |
twisti@4318 | 1277 | for (int col1 = 0; col1 < 8; col1++) { |
twisti@4318 | 1278 | tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp); |
twisti@4318 | 1279 | os::print_location(tty, *dump_sp++); |
twisti@4318 | 1280 | } |
twisti@4318 | 1281 | for (int row = 0; row < 25; row++) { |
twisti@4318 | 1282 | tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp); |
twisti@4318 | 1283 | for (int col = 0; col < 4; col++) { |
twisti@4318 | 1284 | tty->print(" 0x%016lx", *dump_sp++); |
twisti@4318 | 1285 | } |
twisti@4318 | 1286 | tty->cr(); |
twisti@4318 | 1287 | } |
twisti@4318 | 1288 | // Print some instructions around pc: |
twisti@4318 | 1289 | Disassembler::decode((address)pc-64, (address)pc); |
twisti@4318 | 1290 | tty->print_cr("--------"); |
twisti@4318 | 1291 | Disassembler::decode((address)pc, (address)pc+32); |
twisti@4318 | 1292 | } |
twisti@4318 | 1293 | |
twisti@4318 | 1294 | #endif // _LP64 |
twisti@4318 | 1295 | |
twisti@4318 | 1296 | // Now versions that are common to 32/64 bit |
twisti@4318 | 1297 | |
twisti@4318 | 1298 | void MacroAssembler::addptr(Register dst, int32_t imm32) { |
twisti@4318 | 1299 | LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32)); |
twisti@4318 | 1300 | } |
twisti@4318 | 1301 | |
twisti@4318 | 1302 | void MacroAssembler::addptr(Register dst, Register src) { |
twisti@4318 | 1303 | LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
twisti@4318 | 1304 | } |
twisti@4318 | 1305 | |
twisti@4318 | 1306 | void MacroAssembler::addptr(Address dst, Register src) { |
twisti@4318 | 1307 | LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src)); |
twisti@4318 | 1308 | } |
twisti@4318 | 1309 | |
twisti@4318 | 1310 | void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1311 | if (reachable(src)) { |
twisti@4318 | 1312 | Assembler::addsd(dst, as_Address(src)); |
twisti@4318 | 1313 | } else { |
twisti@4318 | 1314 | lea(rscratch1, src); |
twisti@4318 | 1315 | Assembler::addsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 1316 | } |
twisti@4318 | 1317 | } |
twisti@4318 | 1318 | |
twisti@4318 | 1319 | void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1320 | if (reachable(src)) { |
twisti@4318 | 1321 | addss(dst, as_Address(src)); |
twisti@4318 | 1322 | } else { |
twisti@4318 | 1323 | lea(rscratch1, src); |
twisti@4318 | 1324 | addss(dst, Address(rscratch1, 0)); |
twisti@4318 | 1325 | } |
twisti@4318 | 1326 | } |
twisti@4318 | 1327 | |
twisti@4318 | 1328 | void MacroAssembler::align(int modulus) { |
twisti@4318 | 1329 | if (offset() % modulus != 0) { |
twisti@4318 | 1330 | nop(modulus - (offset() % modulus)); |
twisti@4318 | 1331 | } |
twisti@4318 | 1332 | } |
twisti@4318 | 1333 | |
twisti@4318 | 1334 | void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1335 | // Used in sign-masking with aligned address. |
twisti@4318 | 1336 | assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
twisti@4318 | 1337 | if (reachable(src)) { |
twisti@4318 | 1338 | Assembler::andpd(dst, as_Address(src)); |
twisti@4318 | 1339 | } else { |
twisti@4318 | 1340 | lea(rscratch1, src); |
twisti@4318 | 1341 | Assembler::andpd(dst, Address(rscratch1, 0)); |
twisti@4318 | 1342 | } |
twisti@4318 | 1343 | } |
twisti@4318 | 1344 | |
twisti@4318 | 1345 | void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1346 | // Used in sign-masking with aligned address. |
twisti@4318 | 1347 | assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
twisti@4318 | 1348 | if (reachable(src)) { |
twisti@4318 | 1349 | Assembler::andps(dst, as_Address(src)); |
twisti@4318 | 1350 | } else { |
twisti@4318 | 1351 | lea(rscratch1, src); |
twisti@4318 | 1352 | Assembler::andps(dst, Address(rscratch1, 0)); |
twisti@4318 | 1353 | } |
twisti@4318 | 1354 | } |
twisti@4318 | 1355 | |
twisti@4318 | 1356 | void MacroAssembler::andptr(Register dst, int32_t imm32) { |
twisti@4318 | 1357 | LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); |
twisti@4318 | 1358 | } |
twisti@4318 | 1359 | |
twisti@4318 | 1360 | void MacroAssembler::atomic_incl(AddressLiteral counter_addr) { |
twisti@4318 | 1361 | pushf(); |
twisti@4318 | 1362 | if (os::is_MP()) |
twisti@4318 | 1363 | lock(); |
twisti@4318 | 1364 | incrementl(counter_addr); |
twisti@4318 | 1365 | popf(); |
twisti@4318 | 1366 | } |
twisti@4318 | 1367 | |
twisti@4318 | 1368 | // Writes to stack successive pages until offset reached to check for |
twisti@4318 | 1369 | // stack overflow + shadow pages. This clobbers tmp. |
twisti@4318 | 1370 | void MacroAssembler::bang_stack_size(Register size, Register tmp) { |
twisti@4318 | 1371 | movptr(tmp, rsp); |
twisti@4318 | 1372 | // Bang stack for total size given plus shadow page size. |
twisti@4318 | 1373 | // Bang one page at a time because large size can bang beyond yellow and |
twisti@4318 | 1374 | // red zones. |
twisti@4318 | 1375 | Label loop; |
twisti@4318 | 1376 | bind(loop); |
twisti@4318 | 1377 | movl(Address(tmp, (-os::vm_page_size())), size ); |
twisti@4318 | 1378 | subptr(tmp, os::vm_page_size()); |
twisti@4318 | 1379 | subl(size, os::vm_page_size()); |
twisti@4318 | 1380 | jcc(Assembler::greater, loop); |
twisti@4318 | 1381 | |
twisti@4318 | 1382 | // Bang down shadow pages too. |
twisti@4318 | 1383 | // The -1 because we already subtracted 1 page. |
twisti@4318 | 1384 | for (int i = 0; i< StackShadowPages-1; i++) { |
twisti@4318 | 1385 | // this could be any sized move but this is can be a debugging crumb |
twisti@4318 | 1386 | // so the bigger the better. |
twisti@4318 | 1387 | movptr(Address(tmp, (-i*os::vm_page_size())), size ); |
twisti@4318 | 1388 | } |
twisti@4318 | 1389 | } |
twisti@4318 | 1390 | |
twisti@4318 | 1391 | void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) { |
twisti@4318 | 1392 | assert(UseBiasedLocking, "why call this otherwise?"); |
twisti@4318 | 1393 | |
twisti@4318 | 1394 | // Check for biased locking unlock case, which is a no-op |
twisti@4318 | 1395 | // Note: we do not have to check the thread ID for two reasons. |
twisti@4318 | 1396 | // First, the interpreter checks for IllegalMonitorStateException at |
twisti@4318 | 1397 | // a higher level. Second, if the bias was revoked while we held the |
twisti@4318 | 1398 | // lock, the object could not be rebiased toward another thread, so |
twisti@4318 | 1399 | // the bias bit would be clear. |
twisti@4318 | 1400 | movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes())); |
twisti@4318 | 1401 | andptr(temp_reg, markOopDesc::biased_lock_mask_in_place); |
twisti@4318 | 1402 | cmpptr(temp_reg, markOopDesc::biased_lock_pattern); |
twisti@4318 | 1403 | jcc(Assembler::equal, done); |
twisti@4318 | 1404 | } |
twisti@4318 | 1405 | |
twisti@4318 | 1406 | void MacroAssembler::c2bool(Register x) { |
twisti@4318 | 1407 | // implements x == 0 ? 0 : 1 |
twisti@4318 | 1408 | // note: must only look at least-significant byte of x |
twisti@4318 | 1409 | // since C-style booleans are stored in one byte |
twisti@4318 | 1410 | // only! (was bug) |
twisti@4318 | 1411 | andl(x, 0xFF); |
twisti@4318 | 1412 | setb(Assembler::notZero, x); |
twisti@4318 | 1413 | } |
twisti@4318 | 1414 | |
twisti@4318 | 1415 | // Wouldn't need if AddressLiteral version had new name |
twisti@4318 | 1416 | void MacroAssembler::call(Label& L, relocInfo::relocType rtype) { |
twisti@4318 | 1417 | Assembler::call(L, rtype); |
twisti@4318 | 1418 | } |
twisti@4318 | 1419 | |
twisti@4318 | 1420 | void MacroAssembler::call(Register entry) { |
twisti@4318 | 1421 | Assembler::call(entry); |
twisti@4318 | 1422 | } |
twisti@4318 | 1423 | |
twisti@4318 | 1424 | void MacroAssembler::call(AddressLiteral entry) { |
twisti@4318 | 1425 | if (reachable(entry)) { |
twisti@4318 | 1426 | Assembler::call_literal(entry.target(), entry.rspec()); |
twisti@4318 | 1427 | } else { |
twisti@4318 | 1428 | lea(rscratch1, entry); |
twisti@4318 | 1429 | Assembler::call(rscratch1); |
twisti@4318 | 1430 | } |
twisti@4318 | 1431 | } |
twisti@4318 | 1432 | |
twisti@4318 | 1433 | void MacroAssembler::ic_call(address entry) { |
twisti@4318 | 1434 | RelocationHolder rh = virtual_call_Relocation::spec(pc()); |
twisti@4318 | 1435 | movptr(rax, (intptr_t)Universe::non_oop_word()); |
twisti@4318 | 1436 | call(AddressLiteral(entry, rh)); |
twisti@4318 | 1437 | } |
twisti@4318 | 1438 | |
twisti@4318 | 1439 | // Implementation of call_VM versions |
twisti@4318 | 1440 | |
twisti@4318 | 1441 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1442 | address entry_point, |
twisti@4318 | 1443 | bool check_exceptions) { |
twisti@4318 | 1444 | Label C, E; |
twisti@4318 | 1445 | call(C, relocInfo::none); |
twisti@4318 | 1446 | jmp(E); |
twisti@4318 | 1447 | |
twisti@4318 | 1448 | bind(C); |
twisti@4318 | 1449 | call_VM_helper(oop_result, entry_point, 0, check_exceptions); |
twisti@4318 | 1450 | ret(0); |
twisti@4318 | 1451 | |
twisti@4318 | 1452 | bind(E); |
twisti@4318 | 1453 | } |
twisti@4318 | 1454 | |
twisti@4318 | 1455 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1456 | address entry_point, |
twisti@4318 | 1457 | Register arg_1, |
twisti@4318 | 1458 | bool check_exceptions) { |
twisti@4318 | 1459 | Label C, E; |
twisti@4318 | 1460 | call(C, relocInfo::none); |
twisti@4318 | 1461 | jmp(E); |
twisti@4318 | 1462 | |
twisti@4318 | 1463 | bind(C); |
twisti@4318 | 1464 | pass_arg1(this, arg_1); |
twisti@4318 | 1465 | call_VM_helper(oop_result, entry_point, 1, check_exceptions); |
twisti@4318 | 1466 | ret(0); |
twisti@4318 | 1467 | |
twisti@4318 | 1468 | bind(E); |
twisti@4318 | 1469 | } |
twisti@4318 | 1470 | |
twisti@4318 | 1471 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1472 | address entry_point, |
twisti@4318 | 1473 | Register arg_1, |
twisti@4318 | 1474 | Register arg_2, |
twisti@4318 | 1475 | bool check_exceptions) { |
twisti@4318 | 1476 | Label C, E; |
twisti@4318 | 1477 | call(C, relocInfo::none); |
twisti@4318 | 1478 | jmp(E); |
twisti@4318 | 1479 | |
twisti@4318 | 1480 | bind(C); |
twisti@4318 | 1481 | |
twisti@4318 | 1482 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1483 | |
twisti@4318 | 1484 | pass_arg2(this, arg_2); |
twisti@4318 | 1485 | pass_arg1(this, arg_1); |
twisti@4318 | 1486 | call_VM_helper(oop_result, entry_point, 2, check_exceptions); |
twisti@4318 | 1487 | ret(0); |
twisti@4318 | 1488 | |
twisti@4318 | 1489 | bind(E); |
twisti@4318 | 1490 | } |
twisti@4318 | 1491 | |
twisti@4318 | 1492 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1493 | address entry_point, |
twisti@4318 | 1494 | Register arg_1, |
twisti@4318 | 1495 | Register arg_2, |
twisti@4318 | 1496 | Register arg_3, |
twisti@4318 | 1497 | bool check_exceptions) { |
twisti@4318 | 1498 | Label C, E; |
twisti@4318 | 1499 | call(C, relocInfo::none); |
twisti@4318 | 1500 | jmp(E); |
twisti@4318 | 1501 | |
twisti@4318 | 1502 | bind(C); |
twisti@4318 | 1503 | |
twisti@4318 | 1504 | LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
twisti@4318 | 1505 | LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
twisti@4318 | 1506 | pass_arg3(this, arg_3); |
twisti@4318 | 1507 | |
twisti@4318 | 1508 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1509 | pass_arg2(this, arg_2); |
twisti@4318 | 1510 | |
twisti@4318 | 1511 | pass_arg1(this, arg_1); |
twisti@4318 | 1512 | call_VM_helper(oop_result, entry_point, 3, check_exceptions); |
twisti@4318 | 1513 | ret(0); |
twisti@4318 | 1514 | |
twisti@4318 | 1515 | bind(E); |
twisti@4318 | 1516 | } |
twisti@4318 | 1517 | |
twisti@4318 | 1518 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1519 | Register last_java_sp, |
twisti@4318 | 1520 | address entry_point, |
twisti@4318 | 1521 | int number_of_arguments, |
twisti@4318 | 1522 | bool check_exceptions) { |
twisti@4318 | 1523 | Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
twisti@4318 | 1524 | call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
twisti@4318 | 1525 | } |
twisti@4318 | 1526 | |
twisti@4318 | 1527 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1528 | Register last_java_sp, |
twisti@4318 | 1529 | address entry_point, |
twisti@4318 | 1530 | Register arg_1, |
twisti@4318 | 1531 | bool check_exceptions) { |
twisti@4318 | 1532 | pass_arg1(this, arg_1); |
twisti@4318 | 1533 | call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
twisti@4318 | 1534 | } |
twisti@4318 | 1535 | |
twisti@4318 | 1536 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1537 | Register last_java_sp, |
twisti@4318 | 1538 | address entry_point, |
twisti@4318 | 1539 | Register arg_1, |
twisti@4318 | 1540 | Register arg_2, |
twisti@4318 | 1541 | bool check_exceptions) { |
twisti@4318 | 1542 | |
twisti@4318 | 1543 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1544 | pass_arg2(this, arg_2); |
twisti@4318 | 1545 | pass_arg1(this, arg_1); |
twisti@4318 | 1546 | call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
twisti@4318 | 1547 | } |
twisti@4318 | 1548 | |
twisti@4318 | 1549 | void MacroAssembler::call_VM(Register oop_result, |
twisti@4318 | 1550 | Register last_java_sp, |
twisti@4318 | 1551 | address entry_point, |
twisti@4318 | 1552 | Register arg_1, |
twisti@4318 | 1553 | Register arg_2, |
twisti@4318 | 1554 | Register arg_3, |
twisti@4318 | 1555 | bool check_exceptions) { |
twisti@4318 | 1556 | LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
twisti@4318 | 1557 | LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
twisti@4318 | 1558 | pass_arg3(this, arg_3); |
twisti@4318 | 1559 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1560 | pass_arg2(this, arg_2); |
twisti@4318 | 1561 | pass_arg1(this, arg_1); |
twisti@4318 | 1562 | call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
twisti@4318 | 1563 | } |
twisti@4318 | 1564 | |
twisti@4318 | 1565 | void MacroAssembler::super_call_VM(Register oop_result, |
twisti@4318 | 1566 | Register last_java_sp, |
twisti@4318 | 1567 | address entry_point, |
twisti@4318 | 1568 | int number_of_arguments, |
twisti@4318 | 1569 | bool check_exceptions) { |
twisti@4318 | 1570 | Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg); |
twisti@4318 | 1571 | MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions); |
twisti@4318 | 1572 | } |
twisti@4318 | 1573 | |
twisti@4318 | 1574 | void MacroAssembler::super_call_VM(Register oop_result, |
twisti@4318 | 1575 | Register last_java_sp, |
twisti@4318 | 1576 | address entry_point, |
twisti@4318 | 1577 | Register arg_1, |
twisti@4318 | 1578 | bool check_exceptions) { |
twisti@4318 | 1579 | pass_arg1(this, arg_1); |
twisti@4318 | 1580 | super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); |
twisti@4318 | 1581 | } |
twisti@4318 | 1582 | |
twisti@4318 | 1583 | void MacroAssembler::super_call_VM(Register oop_result, |
twisti@4318 | 1584 | Register last_java_sp, |
twisti@4318 | 1585 | address entry_point, |
twisti@4318 | 1586 | Register arg_1, |
twisti@4318 | 1587 | Register arg_2, |
twisti@4318 | 1588 | bool check_exceptions) { |
twisti@4318 | 1589 | |
twisti@4318 | 1590 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1591 | pass_arg2(this, arg_2); |
twisti@4318 | 1592 | pass_arg1(this, arg_1); |
twisti@4318 | 1593 | super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); |
twisti@4318 | 1594 | } |
twisti@4318 | 1595 | |
twisti@4318 | 1596 | void MacroAssembler::super_call_VM(Register oop_result, |
twisti@4318 | 1597 | Register last_java_sp, |
twisti@4318 | 1598 | address entry_point, |
twisti@4318 | 1599 | Register arg_1, |
twisti@4318 | 1600 | Register arg_2, |
twisti@4318 | 1601 | Register arg_3, |
twisti@4318 | 1602 | bool check_exceptions) { |
twisti@4318 | 1603 | LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
twisti@4318 | 1604 | LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
twisti@4318 | 1605 | pass_arg3(this, arg_3); |
twisti@4318 | 1606 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1607 | pass_arg2(this, arg_2); |
twisti@4318 | 1608 | pass_arg1(this, arg_1); |
twisti@4318 | 1609 | super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); |
twisti@4318 | 1610 | } |
twisti@4318 | 1611 | |
twisti@4318 | 1612 | void MacroAssembler::call_VM_base(Register oop_result, |
twisti@4318 | 1613 | Register java_thread, |
twisti@4318 | 1614 | Register last_java_sp, |
twisti@4318 | 1615 | address entry_point, |
twisti@4318 | 1616 | int number_of_arguments, |
twisti@4318 | 1617 | bool check_exceptions) { |
twisti@4318 | 1618 | // determine java_thread register |
twisti@4318 | 1619 | if (!java_thread->is_valid()) { |
twisti@4318 | 1620 | #ifdef _LP64 |
twisti@4318 | 1621 | java_thread = r15_thread; |
twisti@4318 | 1622 | #else |
twisti@4318 | 1623 | java_thread = rdi; |
twisti@4318 | 1624 | get_thread(java_thread); |
twisti@4318 | 1625 | #endif // LP64 |
twisti@4318 | 1626 | } |
twisti@4318 | 1627 | // determine last_java_sp register |
twisti@4318 | 1628 | if (!last_java_sp->is_valid()) { |
twisti@4318 | 1629 | last_java_sp = rsp; |
twisti@4318 | 1630 | } |
twisti@4318 | 1631 | // debugging support |
twisti@4318 | 1632 | assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); |
twisti@4318 | 1633 | LP64_ONLY(assert(java_thread == r15_thread, "unexpected register")); |
twisti@4318 | 1634 | #ifdef ASSERT |
twisti@4318 | 1635 | // TraceBytecodes does not use r12 but saves it over the call, so don't verify |
twisti@4318 | 1636 | // r12 is the heapbase. |
twisti@4318 | 1637 | LP64_ONLY(if ((UseCompressedOops || UseCompressedKlassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");) |
twisti@4318 | 1638 | #endif // ASSERT |
twisti@4318 | 1639 | |
twisti@4318 | 1640 | assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); |
twisti@4318 | 1641 | assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); |
twisti@4318 | 1642 | |
twisti@4318 | 1643 | // push java thread (becomes first argument of C function) |
twisti@4318 | 1644 | |
twisti@4318 | 1645 | NOT_LP64(push(java_thread); number_of_arguments++); |
twisti@4318 | 1646 | LP64_ONLY(mov(c_rarg0, r15_thread)); |
twisti@4318 | 1647 | |
twisti@4318 | 1648 | // set last Java frame before call |
twisti@4318 | 1649 | assert(last_java_sp != rbp, "can't use ebp/rbp"); |
twisti@4318 | 1650 | |
twisti@4318 | 1651 | // Only interpreter should have to set fp |
twisti@4318 | 1652 | set_last_Java_frame(java_thread, last_java_sp, rbp, NULL); |
twisti@4318 | 1653 | |
twisti@4318 | 1654 | // do the call, remove parameters |
twisti@4318 | 1655 | MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments); |
twisti@4318 | 1656 | |
twisti@4318 | 1657 | // restore the thread (cannot use the pushed argument since arguments |
twisti@4318 | 1658 | // may be overwritten by C code generated by an optimizing compiler); |
twisti@4318 | 1659 | // however can use the register value directly if it is callee saved. |
twisti@4318 | 1660 | if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) { |
twisti@4318 | 1661 | // rdi & rsi (also r15) are callee saved -> nothing to do |
twisti@4318 | 1662 | #ifdef ASSERT |
twisti@4318 | 1663 | guarantee(java_thread != rax, "change this code"); |
twisti@4318 | 1664 | push(rax); |
twisti@4318 | 1665 | { Label L; |
twisti@4318 | 1666 | get_thread(rax); |
twisti@4318 | 1667 | cmpptr(java_thread, rax); |
twisti@4318 | 1668 | jcc(Assembler::equal, L); |
twisti@4318 | 1669 | STOP("MacroAssembler::call_VM_base: rdi not callee saved?"); |
twisti@4318 | 1670 | bind(L); |
twisti@4318 | 1671 | } |
twisti@4318 | 1672 | pop(rax); |
twisti@4318 | 1673 | #endif |
twisti@4318 | 1674 | } else { |
twisti@4318 | 1675 | get_thread(java_thread); |
twisti@4318 | 1676 | } |
twisti@4318 | 1677 | // reset last Java frame |
twisti@4318 | 1678 | // Only interpreter should have to clear fp |
twisti@4318 | 1679 | reset_last_Java_frame(java_thread, true, false); |
twisti@4318 | 1680 | |
twisti@4318 | 1681 | #ifndef CC_INTERP |
twisti@4318 | 1682 | // C++ interp handles this in the interpreter |
twisti@4318 | 1683 | check_and_handle_popframe(java_thread); |
twisti@4318 | 1684 | check_and_handle_earlyret(java_thread); |
twisti@4318 | 1685 | #endif /* CC_INTERP */ |
twisti@4318 | 1686 | |
twisti@4318 | 1687 | if (check_exceptions) { |
twisti@4318 | 1688 | // check for pending exceptions (java_thread is set upon return) |
twisti@4318 | 1689 | cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD); |
twisti@4318 | 1690 | #ifndef _LP64 |
twisti@4318 | 1691 | jump_cc(Assembler::notEqual, |
twisti@4318 | 1692 | RuntimeAddress(StubRoutines::forward_exception_entry())); |
twisti@4318 | 1693 | #else |
twisti@4318 | 1694 | // This used to conditionally jump to forward_exception however it is |
twisti@4318 | 1695 | // possible if we relocate that the branch will not reach. So we must jump |
twisti@4318 | 1696 | // around so we can always reach |
twisti@4318 | 1697 | |
twisti@4318 | 1698 | Label ok; |
twisti@4318 | 1699 | jcc(Assembler::equal, ok); |
twisti@4318 | 1700 | jump(RuntimeAddress(StubRoutines::forward_exception_entry())); |
twisti@4318 | 1701 | bind(ok); |
twisti@4318 | 1702 | #endif // LP64 |
twisti@4318 | 1703 | } |
twisti@4318 | 1704 | |
twisti@4318 | 1705 | // get oop result if there is one and reset the value in the thread |
twisti@4318 | 1706 | if (oop_result->is_valid()) { |
twisti@4318 | 1707 | get_vm_result(oop_result, java_thread); |
twisti@4318 | 1708 | } |
twisti@4318 | 1709 | } |
twisti@4318 | 1710 | |
twisti@4318 | 1711 | void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { |
twisti@4318 | 1712 | |
twisti@4318 | 1713 | // Calculate the value for last_Java_sp |
twisti@4318 | 1714 | // somewhat subtle. call_VM does an intermediate call |
twisti@4318 | 1715 | // which places a return address on the stack just under the |
twisti@4318 | 1716 | // stack pointer as the user finsihed with it. This allows |
twisti@4318 | 1717 | // use to retrieve last_Java_pc from last_Java_sp[-1]. |
twisti@4318 | 1718 | // On 32bit we then have to push additional args on the stack to accomplish |
twisti@4318 | 1719 | // the actual requested call. On 64bit call_VM only can use register args |
twisti@4318 | 1720 | // so the only extra space is the return address that call_VM created. |
twisti@4318 | 1721 | // This hopefully explains the calculations here. |
twisti@4318 | 1722 | |
twisti@4318 | 1723 | #ifdef _LP64 |
twisti@4318 | 1724 | // We've pushed one address, correct last_Java_sp |
twisti@4318 | 1725 | lea(rax, Address(rsp, wordSize)); |
twisti@4318 | 1726 | #else |
twisti@4318 | 1727 | lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize)); |
twisti@4318 | 1728 | #endif // LP64 |
twisti@4318 | 1729 | |
twisti@4318 | 1730 | call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions); |
twisti@4318 | 1731 | |
twisti@4318 | 1732 | } |
twisti@4318 | 1733 | |
twisti@4318 | 1734 | void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { |
twisti@4318 | 1735 | call_VM_leaf_base(entry_point, number_of_arguments); |
twisti@4318 | 1736 | } |
twisti@4318 | 1737 | |
twisti@4318 | 1738 | void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { |
twisti@4318 | 1739 | pass_arg0(this, arg_0); |
twisti@4318 | 1740 | call_VM_leaf(entry_point, 1); |
twisti@4318 | 1741 | } |
twisti@4318 | 1742 | |
twisti@4318 | 1743 | void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
twisti@4318 | 1744 | |
twisti@4318 | 1745 | LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
twisti@4318 | 1746 | pass_arg1(this, arg_1); |
twisti@4318 | 1747 | pass_arg0(this, arg_0); |
twisti@4318 | 1748 | call_VM_leaf(entry_point, 2); |
twisti@4318 | 1749 | } |
twisti@4318 | 1750 | |
twisti@4318 | 1751 | void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
twisti@4318 | 1752 | LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
twisti@4318 | 1753 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1754 | pass_arg2(this, arg_2); |
twisti@4318 | 1755 | LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
twisti@4318 | 1756 | pass_arg1(this, arg_1); |
twisti@4318 | 1757 | pass_arg0(this, arg_0); |
twisti@4318 | 1758 | call_VM_leaf(entry_point, 3); |
twisti@4318 | 1759 | } |
twisti@4318 | 1760 | |
twisti@4318 | 1761 | void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { |
twisti@4318 | 1762 | pass_arg0(this, arg_0); |
twisti@4318 | 1763 | MacroAssembler::call_VM_leaf_base(entry_point, 1); |
twisti@4318 | 1764 | } |
twisti@4318 | 1765 | |
twisti@4318 | 1766 | void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { |
twisti@4318 | 1767 | |
twisti@4318 | 1768 | LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
twisti@4318 | 1769 | pass_arg1(this, arg_1); |
twisti@4318 | 1770 | pass_arg0(this, arg_0); |
twisti@4318 | 1771 | MacroAssembler::call_VM_leaf_base(entry_point, 2); |
twisti@4318 | 1772 | } |
twisti@4318 | 1773 | |
twisti@4318 | 1774 | void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { |
twisti@4318 | 1775 | LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
twisti@4318 | 1776 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1777 | pass_arg2(this, arg_2); |
twisti@4318 | 1778 | LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
twisti@4318 | 1779 | pass_arg1(this, arg_1); |
twisti@4318 | 1780 | pass_arg0(this, arg_0); |
twisti@4318 | 1781 | MacroAssembler::call_VM_leaf_base(entry_point, 3); |
twisti@4318 | 1782 | } |
twisti@4318 | 1783 | |
twisti@4318 | 1784 | void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { |
twisti@4318 | 1785 | LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg")); |
twisti@4318 | 1786 | LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg")); |
twisti@4318 | 1787 | LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg")); |
twisti@4318 | 1788 | pass_arg3(this, arg_3); |
twisti@4318 | 1789 | LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg")); |
twisti@4318 | 1790 | LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg")); |
twisti@4318 | 1791 | pass_arg2(this, arg_2); |
twisti@4318 | 1792 | LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg")); |
twisti@4318 | 1793 | pass_arg1(this, arg_1); |
twisti@4318 | 1794 | pass_arg0(this, arg_0); |
twisti@4318 | 1795 | MacroAssembler::call_VM_leaf_base(entry_point, 4); |
twisti@4318 | 1796 | } |
twisti@4318 | 1797 | |
twisti@4318 | 1798 | void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { |
twisti@4318 | 1799 | movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset())); |
twisti@4318 | 1800 | movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD); |
twisti@4318 | 1801 | verify_oop(oop_result, "broken oop in call_VM_base"); |
twisti@4318 | 1802 | } |
twisti@4318 | 1803 | |
twisti@4318 | 1804 | void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { |
twisti@4318 | 1805 | movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); |
twisti@4318 | 1806 | movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD); |
twisti@4318 | 1807 | } |
twisti@4318 | 1808 | |
twisti@4318 | 1809 | void MacroAssembler::check_and_handle_earlyret(Register java_thread) { |
twisti@4318 | 1810 | } |
twisti@4318 | 1811 | |
twisti@4318 | 1812 | void MacroAssembler::check_and_handle_popframe(Register java_thread) { |
twisti@4318 | 1813 | } |
twisti@4318 | 1814 | |
twisti@4318 | 1815 | void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) { |
twisti@4318 | 1816 | if (reachable(src1)) { |
twisti@4318 | 1817 | cmpl(as_Address(src1), imm); |
twisti@4318 | 1818 | } else { |
twisti@4318 | 1819 | lea(rscratch1, src1); |
twisti@4318 | 1820 | cmpl(Address(rscratch1, 0), imm); |
twisti@4318 | 1821 | } |
twisti@4318 | 1822 | } |
twisti@4318 | 1823 | |
twisti@4318 | 1824 | void MacroAssembler::cmp32(Register src1, AddressLiteral src2) { |
twisti@4318 | 1825 | assert(!src2.is_lval(), "use cmpptr"); |
twisti@4318 | 1826 | if (reachable(src2)) { |
twisti@4318 | 1827 | cmpl(src1, as_Address(src2)); |
twisti@4318 | 1828 | } else { |
twisti@4318 | 1829 | lea(rscratch1, src2); |
twisti@4318 | 1830 | cmpl(src1, Address(rscratch1, 0)); |
twisti@4318 | 1831 | } |
twisti@4318 | 1832 | } |
twisti@4318 | 1833 | |
twisti@4318 | 1834 | void MacroAssembler::cmp32(Register src1, int32_t imm) { |
twisti@4318 | 1835 | Assembler::cmpl(src1, imm); |
twisti@4318 | 1836 | } |
twisti@4318 | 1837 | |
twisti@4318 | 1838 | void MacroAssembler::cmp32(Register src1, Address src2) { |
twisti@4318 | 1839 | Assembler::cmpl(src1, src2); |
twisti@4318 | 1840 | } |
twisti@4318 | 1841 | |
twisti@4318 | 1842 | void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
twisti@4318 | 1843 | ucomisd(opr1, opr2); |
twisti@4318 | 1844 | |
twisti@4318 | 1845 | Label L; |
twisti@4318 | 1846 | if (unordered_is_less) { |
twisti@4318 | 1847 | movl(dst, -1); |
twisti@4318 | 1848 | jcc(Assembler::parity, L); |
twisti@4318 | 1849 | jcc(Assembler::below , L); |
twisti@4318 | 1850 | movl(dst, 0); |
twisti@4318 | 1851 | jcc(Assembler::equal , L); |
twisti@4318 | 1852 | increment(dst); |
twisti@4318 | 1853 | } else { // unordered is greater |
twisti@4318 | 1854 | movl(dst, 1); |
twisti@4318 | 1855 | jcc(Assembler::parity, L); |
twisti@4318 | 1856 | jcc(Assembler::above , L); |
twisti@4318 | 1857 | movl(dst, 0); |
twisti@4318 | 1858 | jcc(Assembler::equal , L); |
twisti@4318 | 1859 | decrementl(dst); |
twisti@4318 | 1860 | } |
twisti@4318 | 1861 | bind(L); |
twisti@4318 | 1862 | } |
twisti@4318 | 1863 | |
twisti@4318 | 1864 | void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) { |
twisti@4318 | 1865 | ucomiss(opr1, opr2); |
twisti@4318 | 1866 | |
twisti@4318 | 1867 | Label L; |
twisti@4318 | 1868 | if (unordered_is_less) { |
twisti@4318 | 1869 | movl(dst, -1); |
twisti@4318 | 1870 | jcc(Assembler::parity, L); |
twisti@4318 | 1871 | jcc(Assembler::below , L); |
twisti@4318 | 1872 | movl(dst, 0); |
twisti@4318 | 1873 | jcc(Assembler::equal , L); |
twisti@4318 | 1874 | increment(dst); |
twisti@4318 | 1875 | } else { // unordered is greater |
twisti@4318 | 1876 | movl(dst, 1); |
twisti@4318 | 1877 | jcc(Assembler::parity, L); |
twisti@4318 | 1878 | jcc(Assembler::above , L); |
twisti@4318 | 1879 | movl(dst, 0); |
twisti@4318 | 1880 | jcc(Assembler::equal , L); |
twisti@4318 | 1881 | decrementl(dst); |
twisti@4318 | 1882 | } |
twisti@4318 | 1883 | bind(L); |
twisti@4318 | 1884 | } |
twisti@4318 | 1885 | |
twisti@4318 | 1886 | |
twisti@4318 | 1887 | void MacroAssembler::cmp8(AddressLiteral src1, int imm) { |
twisti@4318 | 1888 | if (reachable(src1)) { |
twisti@4318 | 1889 | cmpb(as_Address(src1), imm); |
twisti@4318 | 1890 | } else { |
twisti@4318 | 1891 | lea(rscratch1, src1); |
twisti@4318 | 1892 | cmpb(Address(rscratch1, 0), imm); |
twisti@4318 | 1893 | } |
twisti@4318 | 1894 | } |
twisti@4318 | 1895 | |
twisti@4318 | 1896 | void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) { |
twisti@4318 | 1897 | #ifdef _LP64 |
twisti@4318 | 1898 | if (src2.is_lval()) { |
twisti@4318 | 1899 | movptr(rscratch1, src2); |
twisti@4318 | 1900 | Assembler::cmpq(src1, rscratch1); |
twisti@4318 | 1901 | } else if (reachable(src2)) { |
twisti@4318 | 1902 | cmpq(src1, as_Address(src2)); |
twisti@4318 | 1903 | } else { |
twisti@4318 | 1904 | lea(rscratch1, src2); |
twisti@4318 | 1905 | Assembler::cmpq(src1, Address(rscratch1, 0)); |
twisti@4318 | 1906 | } |
twisti@4318 | 1907 | #else |
twisti@4318 | 1908 | if (src2.is_lval()) { |
twisti@4318 | 1909 | cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
twisti@4318 | 1910 | } else { |
twisti@4318 | 1911 | cmpl(src1, as_Address(src2)); |
twisti@4318 | 1912 | } |
twisti@4318 | 1913 | #endif // _LP64 |
twisti@4318 | 1914 | } |
twisti@4318 | 1915 | |
twisti@4318 | 1916 | void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) { |
twisti@4318 | 1917 | assert(src2.is_lval(), "not a mem-mem compare"); |
twisti@4318 | 1918 | #ifdef _LP64 |
twisti@4318 | 1919 | // moves src2's literal address |
twisti@4318 | 1920 | movptr(rscratch1, src2); |
twisti@4318 | 1921 | Assembler::cmpq(src1, rscratch1); |
twisti@4318 | 1922 | #else |
twisti@4318 | 1923 | cmp_literal32(src1, (int32_t) src2.target(), src2.rspec()); |
twisti@4318 | 1924 | #endif // _LP64 |
twisti@4318 | 1925 | } |
twisti@4318 | 1926 | |
twisti@4318 | 1927 | void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) { |
twisti@4318 | 1928 | if (reachable(adr)) { |
twisti@4318 | 1929 | if (os::is_MP()) |
twisti@4318 | 1930 | lock(); |
twisti@4318 | 1931 | cmpxchgptr(reg, as_Address(adr)); |
twisti@4318 | 1932 | } else { |
twisti@4318 | 1933 | lea(rscratch1, adr); |
twisti@4318 | 1934 | if (os::is_MP()) |
twisti@4318 | 1935 | lock(); |
twisti@4318 | 1936 | cmpxchgptr(reg, Address(rscratch1, 0)); |
twisti@4318 | 1937 | } |
twisti@4318 | 1938 | } |
twisti@4318 | 1939 | |
twisti@4318 | 1940 | void MacroAssembler::cmpxchgptr(Register reg, Address adr) { |
twisti@4318 | 1941 | LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr)); |
twisti@4318 | 1942 | } |
twisti@4318 | 1943 | |
twisti@4318 | 1944 | void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1945 | if (reachable(src)) { |
twisti@4318 | 1946 | Assembler::comisd(dst, as_Address(src)); |
twisti@4318 | 1947 | } else { |
twisti@4318 | 1948 | lea(rscratch1, src); |
twisti@4318 | 1949 | Assembler::comisd(dst, Address(rscratch1, 0)); |
twisti@4318 | 1950 | } |
twisti@4318 | 1951 | } |
twisti@4318 | 1952 | |
twisti@4318 | 1953 | void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 1954 | if (reachable(src)) { |
twisti@4318 | 1955 | Assembler::comiss(dst, as_Address(src)); |
twisti@4318 | 1956 | } else { |
twisti@4318 | 1957 | lea(rscratch1, src); |
twisti@4318 | 1958 | Assembler::comiss(dst, Address(rscratch1, 0)); |
twisti@4318 | 1959 | } |
twisti@4318 | 1960 | } |
twisti@4318 | 1961 | |
twisti@4318 | 1962 | |
twisti@4318 | 1963 | void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) { |
twisti@4318 | 1964 | Condition negated_cond = negate_condition(cond); |
twisti@4318 | 1965 | Label L; |
twisti@4318 | 1966 | jcc(negated_cond, L); |
twisti@4318 | 1967 | atomic_incl(counter_addr); |
twisti@4318 | 1968 | bind(L); |
twisti@4318 | 1969 | } |
twisti@4318 | 1970 | |
twisti@4318 | 1971 | int MacroAssembler::corrected_idivl(Register reg) { |
twisti@4318 | 1972 | // Full implementation of Java idiv and irem; checks for |
twisti@4318 | 1973 | // special case as described in JVM spec., p.243 & p.271. |
twisti@4318 | 1974 | // The function returns the (pc) offset of the idivl |
twisti@4318 | 1975 | // instruction - may be needed for implicit exceptions. |
twisti@4318 | 1976 | // |
twisti@4318 | 1977 | // normal case special case |
twisti@4318 | 1978 | // |
twisti@4318 | 1979 | // input : rax,: dividend min_int |
twisti@4318 | 1980 | // reg: divisor (may not be rax,/rdx) -1 |
twisti@4318 | 1981 | // |
twisti@4318 | 1982 | // output: rax,: quotient (= rax, idiv reg) min_int |
twisti@4318 | 1983 | // rdx: remainder (= rax, irem reg) 0 |
twisti@4318 | 1984 | assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register"); |
twisti@4318 | 1985 | const int min_int = 0x80000000; |
twisti@4318 | 1986 | Label normal_case, special_case; |
twisti@4318 | 1987 | |
twisti@4318 | 1988 | // check for special case |
twisti@4318 | 1989 | cmpl(rax, min_int); |
twisti@4318 | 1990 | jcc(Assembler::notEqual, normal_case); |
twisti@4318 | 1991 | xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0) |
twisti@4318 | 1992 | cmpl(reg, -1); |
twisti@4318 | 1993 | jcc(Assembler::equal, special_case); |
twisti@4318 | 1994 | |
twisti@4318 | 1995 | // handle normal case |
twisti@4318 | 1996 | bind(normal_case); |
twisti@4318 | 1997 | cdql(); |
twisti@4318 | 1998 | int idivl_offset = offset(); |
twisti@4318 | 1999 | idivl(reg); |
twisti@4318 | 2000 | |
twisti@4318 | 2001 | // normal and special case exit |
twisti@4318 | 2002 | bind(special_case); |
twisti@4318 | 2003 | |
twisti@4318 | 2004 | return idivl_offset; |
twisti@4318 | 2005 | } |
twisti@4318 | 2006 | |
twisti@4318 | 2007 | |
twisti@4318 | 2008 | |
twisti@4318 | 2009 | void MacroAssembler::decrementl(Register reg, int value) { |
twisti@4318 | 2010 | if (value == min_jint) {subl(reg, value) ; return; } |
twisti@4318 | 2011 | if (value < 0) { incrementl(reg, -value); return; } |
twisti@4318 | 2012 | if (value == 0) { ; return; } |
twisti@4318 | 2013 | if (value == 1 && UseIncDec) { decl(reg) ; return; } |
twisti@4318 | 2014 | /* else */ { subl(reg, value) ; return; } |
twisti@4318 | 2015 | } |
twisti@4318 | 2016 | |
twisti@4318 | 2017 | void MacroAssembler::decrementl(Address dst, int value) { |
twisti@4318 | 2018 | if (value == min_jint) {subl(dst, value) ; return; } |
twisti@4318 | 2019 | if (value < 0) { incrementl(dst, -value); return; } |
twisti@4318 | 2020 | if (value == 0) { ; return; } |
twisti@4318 | 2021 | if (value == 1 && UseIncDec) { decl(dst) ; return; } |
twisti@4318 | 2022 | /* else */ { subl(dst, value) ; return; } |
twisti@4318 | 2023 | } |
twisti@4318 | 2024 | |
twisti@4318 | 2025 | void MacroAssembler::division_with_shift (Register reg, int shift_value) { |
twisti@4318 | 2026 | assert (shift_value > 0, "illegal shift value"); |
twisti@4318 | 2027 | Label _is_positive; |
twisti@4318 | 2028 | testl (reg, reg); |
twisti@4318 | 2029 | jcc (Assembler::positive, _is_positive); |
twisti@4318 | 2030 | int offset = (1 << shift_value) - 1 ; |
twisti@4318 | 2031 | |
twisti@4318 | 2032 | if (offset == 1) { |
twisti@4318 | 2033 | incrementl(reg); |
twisti@4318 | 2034 | } else { |
twisti@4318 | 2035 | addl(reg, offset); |
twisti@4318 | 2036 | } |
twisti@4318 | 2037 | |
twisti@4318 | 2038 | bind (_is_positive); |
twisti@4318 | 2039 | sarl(reg, shift_value); |
twisti@4318 | 2040 | } |
twisti@4318 | 2041 | |
twisti@4318 | 2042 | void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2043 | if (reachable(src)) { |
twisti@4318 | 2044 | Assembler::divsd(dst, as_Address(src)); |
twisti@4318 | 2045 | } else { |
twisti@4318 | 2046 | lea(rscratch1, src); |
twisti@4318 | 2047 | Assembler::divsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 2048 | } |
twisti@4318 | 2049 | } |
twisti@4318 | 2050 | |
twisti@4318 | 2051 | void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2052 | if (reachable(src)) { |
twisti@4318 | 2053 | Assembler::divss(dst, as_Address(src)); |
twisti@4318 | 2054 | } else { |
twisti@4318 | 2055 | lea(rscratch1, src); |
twisti@4318 | 2056 | Assembler::divss(dst, Address(rscratch1, 0)); |
twisti@4318 | 2057 | } |
twisti@4318 | 2058 | } |
twisti@4318 | 2059 | |
twisti@4318 | 2060 | // !defined(COMPILER2) is because of stupid core builds |
twisti@4318 | 2061 | #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) |
twisti@4318 | 2062 | void MacroAssembler::empty_FPU_stack() { |
twisti@4318 | 2063 | if (VM_Version::supports_mmx()) { |
twisti@4318 | 2064 | emms(); |
twisti@4318 | 2065 | } else { |
twisti@4318 | 2066 | for (int i = 8; i-- > 0; ) ffree(i); |
twisti@4318 | 2067 | } |
twisti@4318 | 2068 | } |
twisti@4318 | 2069 | #endif // !LP64 || C1 || !C2 |
twisti@4318 | 2070 | |
twisti@4318 | 2071 | |
twisti@4318 | 2072 | // Defines obj, preserves var_size_in_bytes |
twisti@4318 | 2073 | void MacroAssembler::eden_allocate(Register obj, |
twisti@4318 | 2074 | Register var_size_in_bytes, |
twisti@4318 | 2075 | int con_size_in_bytes, |
twisti@4318 | 2076 | Register t1, |
twisti@4318 | 2077 | Label& slow_case) { |
twisti@4318 | 2078 | assert(obj == rax, "obj must be in rax, for cmpxchg"); |
twisti@4318 | 2079 | assert_different_registers(obj, var_size_in_bytes, t1); |
twisti@4318 | 2080 | if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) { |
twisti@4318 | 2081 | jmp(slow_case); |
twisti@4318 | 2082 | } else { |
twisti@4318 | 2083 | Register end = t1; |
twisti@4318 | 2084 | Label retry; |
twisti@4318 | 2085 | bind(retry); |
twisti@4318 | 2086 | ExternalAddress heap_top((address) Universe::heap()->top_addr()); |
twisti@4318 | 2087 | movptr(obj, heap_top); |
twisti@4318 | 2088 | if (var_size_in_bytes == noreg) { |
twisti@4318 | 2089 | lea(end, Address(obj, con_size_in_bytes)); |
twisti@4318 | 2090 | } else { |
twisti@4318 | 2091 | lea(end, Address(obj, var_size_in_bytes, Address::times_1)); |
twisti@4318 | 2092 | } |
twisti@4318 | 2093 | // if end < obj then we wrapped around => object too long => slow case |
twisti@4318 | 2094 | cmpptr(end, obj); |
twisti@4318 | 2095 | jcc(Assembler::below, slow_case); |
twisti@4318 | 2096 | cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr())); |
twisti@4318 | 2097 | jcc(Assembler::above, slow_case); |
twisti@4318 | 2098 | // Compare obj with the top addr, and if still equal, store the new top addr in |
twisti@4318 | 2099 | // end at the address of the top addr pointer. Sets ZF if was equal, and clears |
twisti@4318 | 2100 | // it otherwise. Use lock prefix for atomicity on MPs. |
twisti@4318 | 2101 | locked_cmpxchgptr(end, heap_top); |
twisti@4318 | 2102 | jcc(Assembler::notEqual, retry); |
twisti@4318 | 2103 | } |
twisti@4318 | 2104 | } |
twisti@4318 | 2105 | |
twisti@4318 | 2106 | void MacroAssembler::enter() { |
twisti@4318 | 2107 | push(rbp); |
twisti@4318 | 2108 | mov(rbp, rsp); |
twisti@4318 | 2109 | } |
twisti@4318 | 2110 | |
twisti@4318 | 2111 | // A 5 byte nop that is safe for patching (see patch_verified_entry) |
twisti@4318 | 2112 | void MacroAssembler::fat_nop() { |
twisti@4318 | 2113 | if (UseAddressNop) { |
twisti@4318 | 2114 | addr_nop_5(); |
twisti@4318 | 2115 | } else { |
twisti@4366 | 2116 | emit_int8(0x26); // es: |
twisti@4366 | 2117 | emit_int8(0x2e); // cs: |
twisti@4366 | 2118 | emit_int8(0x64); // fs: |
twisti@4366 | 2119 | emit_int8(0x65); // gs: |
twisti@4366 | 2120 | emit_int8((unsigned char)0x90); |
twisti@4318 | 2121 | } |
twisti@4318 | 2122 | } |
twisti@4318 | 2123 | |
twisti@4318 | 2124 | void MacroAssembler::fcmp(Register tmp) { |
twisti@4318 | 2125 | fcmp(tmp, 1, true, true); |
twisti@4318 | 2126 | } |
twisti@4318 | 2127 | |
twisti@4318 | 2128 | void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) { |
twisti@4318 | 2129 | assert(!pop_right || pop_left, "usage error"); |
twisti@4318 | 2130 | if (VM_Version::supports_cmov()) { |
twisti@4318 | 2131 | assert(tmp == noreg, "unneeded temp"); |
twisti@4318 | 2132 | if (pop_left) { |
twisti@4318 | 2133 | fucomip(index); |
twisti@4318 | 2134 | } else { |
twisti@4318 | 2135 | fucomi(index); |
twisti@4318 | 2136 | } |
twisti@4318 | 2137 | if (pop_right) { |
twisti@4318 | 2138 | fpop(); |
twisti@4318 | 2139 | } |
twisti@4318 | 2140 | } else { |
twisti@4318 | 2141 | assert(tmp != noreg, "need temp"); |
twisti@4318 | 2142 | if (pop_left) { |
twisti@4318 | 2143 | if (pop_right) { |
twisti@4318 | 2144 | fcompp(); |
twisti@4318 | 2145 | } else { |
twisti@4318 | 2146 | fcomp(index); |
twisti@4318 | 2147 | } |
twisti@4318 | 2148 | } else { |
twisti@4318 | 2149 | fcom(index); |
twisti@4318 | 2150 | } |
twisti@4318 | 2151 | // convert FPU condition into eflags condition via rax, |
twisti@4318 | 2152 | save_rax(tmp); |
twisti@4318 | 2153 | fwait(); fnstsw_ax(); |
twisti@4318 | 2154 | sahf(); |
twisti@4318 | 2155 | restore_rax(tmp); |
twisti@4318 | 2156 | } |
twisti@4318 | 2157 | // condition codes set as follows: |
twisti@4318 | 2158 | // |
twisti@4318 | 2159 | // CF (corresponds to C0) if x < y |
twisti@4318 | 2160 | // PF (corresponds to C2) if unordered |
twisti@4318 | 2161 | // ZF (corresponds to C3) if x = y |
twisti@4318 | 2162 | } |
twisti@4318 | 2163 | |
twisti@4318 | 2164 | void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) { |
twisti@4318 | 2165 | fcmp2int(dst, unordered_is_less, 1, true, true); |
twisti@4318 | 2166 | } |
twisti@4318 | 2167 | |
twisti@4318 | 2168 | void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) { |
twisti@4318 | 2169 | fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right); |
twisti@4318 | 2170 | Label L; |
twisti@4318 | 2171 | if (unordered_is_less) { |
twisti@4318 | 2172 | movl(dst, -1); |
twisti@4318 | 2173 | jcc(Assembler::parity, L); |
twisti@4318 | 2174 | jcc(Assembler::below , L); |
twisti@4318 | 2175 | movl(dst, 0); |
twisti@4318 | 2176 | jcc(Assembler::equal , L); |
twisti@4318 | 2177 | increment(dst); |
twisti@4318 | 2178 | } else { // unordered is greater |
twisti@4318 | 2179 | movl(dst, 1); |
twisti@4318 | 2180 | jcc(Assembler::parity, L); |
twisti@4318 | 2181 | jcc(Assembler::above , L); |
twisti@4318 | 2182 | movl(dst, 0); |
twisti@4318 | 2183 | jcc(Assembler::equal , L); |
twisti@4318 | 2184 | decrementl(dst); |
twisti@4318 | 2185 | } |
twisti@4318 | 2186 | bind(L); |
twisti@4318 | 2187 | } |
twisti@4318 | 2188 | |
twisti@4318 | 2189 | void MacroAssembler::fld_d(AddressLiteral src) { |
twisti@4318 | 2190 | fld_d(as_Address(src)); |
twisti@4318 | 2191 | } |
twisti@4318 | 2192 | |
twisti@4318 | 2193 | void MacroAssembler::fld_s(AddressLiteral src) { |
twisti@4318 | 2194 | fld_s(as_Address(src)); |
twisti@4318 | 2195 | } |
twisti@4318 | 2196 | |
twisti@4318 | 2197 | void MacroAssembler::fld_x(AddressLiteral src) { |
twisti@4318 | 2198 | Assembler::fld_x(as_Address(src)); |
twisti@4318 | 2199 | } |
twisti@4318 | 2200 | |
twisti@4318 | 2201 | void MacroAssembler::fldcw(AddressLiteral src) { |
twisti@4318 | 2202 | Assembler::fldcw(as_Address(src)); |
twisti@4318 | 2203 | } |
twisti@4318 | 2204 | |
twisti@4318 | 2205 | void MacroAssembler::pow_exp_core_encoding() { |
twisti@4318 | 2206 | // kills rax, rcx, rdx |
twisti@4318 | 2207 | subptr(rsp,sizeof(jdouble)); |
twisti@4318 | 2208 | // computes 2^X. Stack: X ... |
twisti@4318 | 2209 | // f2xm1 computes 2^X-1 but only operates on -1<=X<=1. Get int(X) and |
twisti@4318 | 2210 | // keep it on the thread's stack to compute 2^int(X) later |
twisti@4318 | 2211 | // then compute 2^(X-int(X)) as (2^(X-int(X)-1+1) |
twisti@4318 | 2212 | // final result is obtained with: 2^X = 2^int(X) * 2^(X-int(X)) |
twisti@4318 | 2213 | fld_s(0); // Stack: X X ... |
twisti@4318 | 2214 | frndint(); // Stack: int(X) X ... |
twisti@4318 | 2215 | fsuba(1); // Stack: int(X) X-int(X) ... |
twisti@4318 | 2216 | fistp_s(Address(rsp,0)); // move int(X) as integer to thread's stack. Stack: X-int(X) ... |
twisti@4318 | 2217 | f2xm1(); // Stack: 2^(X-int(X))-1 ... |
twisti@4318 | 2218 | fld1(); // Stack: 1 2^(X-int(X))-1 ... |
twisti@4318 | 2219 | faddp(1); // Stack: 2^(X-int(X)) |
twisti@4318 | 2220 | // computes 2^(int(X)): add exponent bias (1023) to int(X), then |
twisti@4318 | 2221 | // shift int(X)+1023 to exponent position. |
twisti@4318 | 2222 | // Exponent is limited to 11 bits if int(X)+1023 does not fit in 11 |
twisti@4318 | 2223 | // bits, set result to NaN. 0x000 and 0x7FF are reserved exponent |
twisti@4318 | 2224 | // values so detect them and set result to NaN. |
twisti@4318 | 2225 | movl(rax,Address(rsp,0)); |
twisti@4318 | 2226 | movl(rcx, -2048); // 11 bit mask and valid NaN binary encoding |
twisti@4318 | 2227 | addl(rax, 1023); |
twisti@4318 | 2228 | movl(rdx,rax); |
twisti@4318 | 2229 | shll(rax,20); |
twisti@4318 | 2230 | // Check that 0 < int(X)+1023 < 2047. Otherwise set rax to NaN. |
twisti@4318 | 2231 | addl(rdx,1); |
twisti@4318 | 2232 | // Check that 1 < int(X)+1023+1 < 2048 |
twisti@4318 | 2233 | // in 3 steps: |
twisti@4318 | 2234 | // 1- (int(X)+1023+1)&-2048 == 0 => 0 <= int(X)+1023+1 < 2048 |
twisti@4318 | 2235 | // 2- (int(X)+1023+1)&-2048 != 0 |
twisti@4318 | 2236 | // 3- (int(X)+1023+1)&-2048 != 1 |
twisti@4318 | 2237 | // Do 2- first because addl just updated the flags. |
twisti@4318 | 2238 | cmov32(Assembler::equal,rax,rcx); |
twisti@4318 | 2239 | cmpl(rdx,1); |
twisti@4318 | 2240 | cmov32(Assembler::equal,rax,rcx); |
twisti@4318 | 2241 | testl(rdx,rcx); |
twisti@4318 | 2242 | cmov32(Assembler::notEqual,rax,rcx); |
twisti@4318 | 2243 | movl(Address(rsp,4),rax); |
twisti@4318 | 2244 | movl(Address(rsp,0),0); |
twisti@4318 | 2245 | fmul_d(Address(rsp,0)); // Stack: 2^X ... |
twisti@4318 | 2246 | addptr(rsp,sizeof(jdouble)); |
twisti@4318 | 2247 | } |
twisti@4318 | 2248 | |
twisti@4318 | 2249 | void MacroAssembler::increase_precision() { |
twisti@4318 | 2250 | subptr(rsp, BytesPerWord); |
twisti@4318 | 2251 | fnstcw(Address(rsp, 0)); |
twisti@4318 | 2252 | movl(rax, Address(rsp, 0)); |
twisti@4318 | 2253 | orl(rax, 0x300); |
twisti@4318 | 2254 | push(rax); |
twisti@4318 | 2255 | fldcw(Address(rsp, 0)); |
twisti@4318 | 2256 | pop(rax); |
twisti@4318 | 2257 | } |
twisti@4318 | 2258 | |
twisti@4318 | 2259 | void MacroAssembler::restore_precision() { |
twisti@4318 | 2260 | fldcw(Address(rsp, 0)); |
twisti@4318 | 2261 | addptr(rsp, BytesPerWord); |
twisti@4318 | 2262 | } |
twisti@4318 | 2263 | |
twisti@4318 | 2264 | void MacroAssembler::fast_pow() { |
twisti@4318 | 2265 | // computes X^Y = 2^(Y * log2(X)) |
twisti@4318 | 2266 | // if fast computation is not possible, result is NaN. Requires |
twisti@4318 | 2267 | // fallback from user of this macro. |
twisti@4318 | 2268 | // increase precision for intermediate steps of the computation |
twisti@4318 | 2269 | increase_precision(); |
twisti@4318 | 2270 | fyl2x(); // Stack: (Y*log2(X)) ... |
twisti@4318 | 2271 | pow_exp_core_encoding(); // Stack: exp(X) ... |
twisti@4318 | 2272 | restore_precision(); |
twisti@4318 | 2273 | } |
twisti@4318 | 2274 | |
twisti@4318 | 2275 | void MacroAssembler::fast_exp() { |
twisti@4318 | 2276 | // computes exp(X) = 2^(X * log2(e)) |
twisti@4318 | 2277 | // if fast computation is not possible, result is NaN. Requires |
twisti@4318 | 2278 | // fallback from user of this macro. |
twisti@4318 | 2279 | // increase precision for intermediate steps of the computation |
twisti@4318 | 2280 | increase_precision(); |
twisti@4318 | 2281 | fldl2e(); // Stack: log2(e) X ... |
twisti@4318 | 2282 | fmulp(1); // Stack: (X*log2(e)) ... |
twisti@4318 | 2283 | pow_exp_core_encoding(); // Stack: exp(X) ... |
twisti@4318 | 2284 | restore_precision(); |
twisti@4318 | 2285 | } |
twisti@4318 | 2286 | |
twisti@4318 | 2287 | void MacroAssembler::pow_or_exp(bool is_exp, int num_fpu_regs_in_use) { |
twisti@4318 | 2288 | // kills rax, rcx, rdx |
twisti@4318 | 2289 | // pow and exp needs 2 extra registers on the fpu stack. |
twisti@4318 | 2290 | Label slow_case, done; |
twisti@4318 | 2291 | Register tmp = noreg; |
twisti@4318 | 2292 | if (!VM_Version::supports_cmov()) { |
twisti@4318 | 2293 | // fcmp needs a temporary so preserve rdx, |
twisti@4318 | 2294 | tmp = rdx; |
twisti@4318 | 2295 | } |
twisti@4318 | 2296 | Register tmp2 = rax; |
twisti@4318 | 2297 | Register tmp3 = rcx; |
twisti@4318 | 2298 | |
twisti@4318 | 2299 | if (is_exp) { |
twisti@4318 | 2300 | // Stack: X |
twisti@4318 | 2301 | fld_s(0); // duplicate argument for runtime call. Stack: X X |
twisti@4318 | 2302 | fast_exp(); // Stack: exp(X) X |
twisti@4318 | 2303 | fcmp(tmp, 0, false, false); // Stack: exp(X) X |
twisti@4318 | 2304 | // exp(X) not equal to itself: exp(X) is NaN go to slow case. |
twisti@4318 | 2305 | jcc(Assembler::parity, slow_case); |
twisti@4318 | 2306 | // get rid of duplicate argument. Stack: exp(X) |
twisti@4318 | 2307 | if (num_fpu_regs_in_use > 0) { |
twisti@4318 | 2308 | fxch(); |
twisti@4318 | 2309 | fpop(); |
twisti@4318 | 2310 | } else { |
twisti@4318 | 2311 | ffree(1); |
twisti@4318 | 2312 | } |
twisti@4318 | 2313 | jmp(done); |
twisti@4318 | 2314 | } else { |
twisti@4318 | 2315 | // Stack: X Y |
twisti@4318 | 2316 | Label x_negative, y_odd; |
twisti@4318 | 2317 | |
twisti@4318 | 2318 | fldz(); // Stack: 0 X Y |
twisti@4318 | 2319 | fcmp(tmp, 1, true, false); // Stack: X Y |
twisti@4318 | 2320 | jcc(Assembler::above, x_negative); |
twisti@4318 | 2321 | |
twisti@4318 | 2322 | // X >= 0 |
twisti@4318 | 2323 | |
twisti@4318 | 2324 | fld_s(1); // duplicate arguments for runtime call. Stack: Y X Y |
twisti@4318 | 2325 | fld_s(1); // Stack: X Y X Y |
twisti@4318 | 2326 | fast_pow(); // Stack: X^Y X Y |
twisti@4318 | 2327 | fcmp(tmp, 0, false, false); // Stack: X^Y X Y |
twisti@4318 | 2328 | // X^Y not equal to itself: X^Y is NaN go to slow case. |
twisti@4318 | 2329 | jcc(Assembler::parity, slow_case); |
twisti@4318 | 2330 | // get rid of duplicate arguments. Stack: X^Y |
twisti@4318 | 2331 | if (num_fpu_regs_in_use > 0) { |
twisti@4318 | 2332 | fxch(); fpop(); |
twisti@4318 | 2333 | fxch(); fpop(); |
twisti@4318 | 2334 | } else { |
twisti@4318 | 2335 | ffree(2); |
twisti@4318 | 2336 | ffree(1); |
twisti@4318 | 2337 | } |
twisti@4318 | 2338 | jmp(done); |
twisti@4318 | 2339 | |
twisti@4318 | 2340 | // X <= 0 |
twisti@4318 | 2341 | bind(x_negative); |
twisti@4318 | 2342 | |
twisti@4318 | 2343 | fld_s(1); // Stack: Y X Y |
twisti@4318 | 2344 | frndint(); // Stack: int(Y) X Y |
twisti@4318 | 2345 | fcmp(tmp, 2, false, false); // Stack: int(Y) X Y |
twisti@4318 | 2346 | jcc(Assembler::notEqual, slow_case); |
twisti@4318 | 2347 | |
twisti@4318 | 2348 | subptr(rsp, 8); |
twisti@4318 | 2349 | |
twisti@4318 | 2350 | // For X^Y, when X < 0, Y has to be an integer and the final |
twisti@4318 | 2351 | // result depends on whether it's odd or even. We just checked |
twisti@4318 | 2352 | // that int(Y) == Y. We move int(Y) to gp registers as a 64 bit |
twisti@4318 | 2353 | // integer to test its parity. If int(Y) is huge and doesn't fit |
twisti@4318 | 2354 | // in the 64 bit integer range, the integer indefinite value will |
twisti@4318 | 2355 | // end up in the gp registers. Huge numbers are all even, the |
twisti@4318 | 2356 | // integer indefinite number is even so it's fine. |
twisti@4318 | 2357 | |
twisti@4318 | 2358 | #ifdef ASSERT |
twisti@4318 | 2359 | // Let's check we don't end up with an integer indefinite number |
twisti@4318 | 2360 | // when not expected. First test for huge numbers: check whether |
twisti@4318 | 2361 | // int(Y)+1 == int(Y) which is true for very large numbers and |
twisti@4318 | 2362 | // those are all even. A 64 bit integer is guaranteed to not |
twisti@4318 | 2363 | // overflow for numbers where y+1 != y (when precision is set to |
twisti@4318 | 2364 | // double precision). |
twisti@4318 | 2365 | Label y_not_huge; |
twisti@4318 | 2366 | |
twisti@4318 | 2367 | fld1(); // Stack: 1 int(Y) X Y |
twisti@4318 | 2368 | fadd(1); // Stack: 1+int(Y) int(Y) X Y |
twisti@4318 | 2369 | |
twisti@4318 | 2370 | #ifdef _LP64 |
twisti@4318 | 2371 | // trip to memory to force the precision down from double extended |
twisti@4318 | 2372 | // precision |
twisti@4318 | 2373 | fstp_d(Address(rsp, 0)); |
twisti@4318 | 2374 | fld_d(Address(rsp, 0)); |
twisti@4318 | 2375 | #endif |
twisti@4318 | 2376 | |
twisti@4318 | 2377 | fcmp(tmp, 1, true, false); // Stack: int(Y) X Y |
twisti@4318 | 2378 | #endif |
twisti@4318 | 2379 | |
twisti@4318 | 2380 | // move int(Y) as 64 bit integer to thread's stack |
twisti@4318 | 2381 | fistp_d(Address(rsp,0)); // Stack: X Y |
twisti@4318 | 2382 | |
twisti@4318 | 2383 | #ifdef ASSERT |
twisti@4318 | 2384 | jcc(Assembler::notEqual, y_not_huge); |
twisti@4318 | 2385 | |
twisti@4318 | 2386 | // Y is huge so we know it's even. It may not fit in a 64 bit |
twisti@4318 | 2387 | // integer and we don't want the debug code below to see the |
twisti@4318 | 2388 | // integer indefinite value so overwrite int(Y) on the thread's |
twisti@4318 | 2389 | // stack with 0. |
twisti@4318 | 2390 | movl(Address(rsp, 0), 0); |
twisti@4318 | 2391 | movl(Address(rsp, 4), 0); |
twisti@4318 | 2392 | |
twisti@4318 | 2393 | bind(y_not_huge); |
twisti@4318 | 2394 | #endif |
twisti@4318 | 2395 | |
twisti@4318 | 2396 | fld_s(1); // duplicate arguments for runtime call. Stack: Y X Y |
twisti@4318 | 2397 | fld_s(1); // Stack: X Y X Y |
twisti@4318 | 2398 | fabs(); // Stack: abs(X) Y X Y |
twisti@4318 | 2399 | fast_pow(); // Stack: abs(X)^Y X Y |
twisti@4318 | 2400 | fcmp(tmp, 0, false, false); // Stack: abs(X)^Y X Y |
twisti@4318 | 2401 | // abs(X)^Y not equal to itself: abs(X)^Y is NaN go to slow case. |
twisti@4318 | 2402 | |
twisti@4318 | 2403 | pop(tmp2); |
twisti@4318 | 2404 | NOT_LP64(pop(tmp3)); |
twisti@4318 | 2405 | jcc(Assembler::parity, slow_case); |
twisti@4318 | 2406 | |
twisti@4318 | 2407 | #ifdef ASSERT |
twisti@4318 | 2408 | // Check that int(Y) is not integer indefinite value (int |
twisti@4318 | 2409 | // overflow). Shouldn't happen because for values that would |
twisti@4318 | 2410 | // overflow, 1+int(Y)==Y which was tested earlier. |
twisti@4318 | 2411 | #ifndef _LP64 |
twisti@4318 | 2412 | { |
twisti@4318 | 2413 | Label integer; |
twisti@4318 | 2414 | testl(tmp2, tmp2); |
twisti@4318 | 2415 | jcc(Assembler::notZero, integer); |
twisti@4318 | 2416 | cmpl(tmp3, 0x80000000); |
twisti@4318 | 2417 | jcc(Assembler::notZero, integer); |
twisti@4318 | 2418 | STOP("integer indefinite value shouldn't be seen here"); |
twisti@4318 | 2419 | bind(integer); |
twisti@4318 | 2420 | } |
twisti@4318 | 2421 | #else |
twisti@4318 | 2422 | { |
twisti@4318 | 2423 | Label integer; |
twisti@4318 | 2424 | mov(tmp3, tmp2); // preserve tmp2 for parity check below |
twisti@4318 | 2425 | shlq(tmp3, 1); |
twisti@4318 | 2426 | jcc(Assembler::carryClear, integer); |
twisti@4318 | 2427 | jcc(Assembler::notZero, integer); |
twisti@4318 | 2428 | STOP("integer indefinite value shouldn't be seen here"); |
twisti@4318 | 2429 | bind(integer); |
twisti@4318 | 2430 | } |
twisti@4318 | 2431 | #endif |
twisti@4318 | 2432 | #endif |
twisti@4318 | 2433 | |
twisti@4318 | 2434 | // get rid of duplicate arguments. Stack: X^Y |
twisti@4318 | 2435 | if (num_fpu_regs_in_use > 0) { |
twisti@4318 | 2436 | fxch(); fpop(); |
twisti@4318 | 2437 | fxch(); fpop(); |
twisti@4318 | 2438 | } else { |
twisti@4318 | 2439 | ffree(2); |
twisti@4318 | 2440 | ffree(1); |
twisti@4318 | 2441 | } |
twisti@4318 | 2442 | |
twisti@4318 | 2443 | testl(tmp2, 1); |
twisti@4318 | 2444 | jcc(Assembler::zero, done); // X <= 0, Y even: X^Y = abs(X)^Y |
twisti@4318 | 2445 | // X <= 0, Y even: X^Y = -abs(X)^Y |
twisti@4318 | 2446 | |
twisti@4318 | 2447 | fchs(); // Stack: -abs(X)^Y Y |
twisti@4318 | 2448 | jmp(done); |
twisti@4318 | 2449 | } |
twisti@4318 | 2450 | |
twisti@4318 | 2451 | // slow case: runtime call |
twisti@4318 | 2452 | bind(slow_case); |
twisti@4318 | 2453 | |
twisti@4318 | 2454 | fpop(); // pop incorrect result or int(Y) |
twisti@4318 | 2455 | |
twisti@4318 | 2456 | fp_runtime_fallback(is_exp ? CAST_FROM_FN_PTR(address, SharedRuntime::dexp) : CAST_FROM_FN_PTR(address, SharedRuntime::dpow), |
twisti@4318 | 2457 | is_exp ? 1 : 2, num_fpu_regs_in_use); |
twisti@4318 | 2458 | |
twisti@4318 | 2459 | // Come here with result in F-TOS |
twisti@4318 | 2460 | bind(done); |
twisti@4318 | 2461 | } |
twisti@4318 | 2462 | |
twisti@4318 | 2463 | void MacroAssembler::fpop() { |
twisti@4318 | 2464 | ffree(); |
twisti@4318 | 2465 | fincstp(); |
twisti@4318 | 2466 | } |
twisti@4318 | 2467 | |
twisti@4318 | 2468 | void MacroAssembler::fremr(Register tmp) { |
twisti@4318 | 2469 | save_rax(tmp); |
twisti@4318 | 2470 | { Label L; |
twisti@4318 | 2471 | bind(L); |
twisti@4318 | 2472 | fprem(); |
twisti@4318 | 2473 | fwait(); fnstsw_ax(); |
twisti@4318 | 2474 | #ifdef _LP64 |
twisti@4318 | 2475 | testl(rax, 0x400); |
twisti@4318 | 2476 | jcc(Assembler::notEqual, L); |
twisti@4318 | 2477 | #else |
twisti@4318 | 2478 | sahf(); |
twisti@4318 | 2479 | jcc(Assembler::parity, L); |
twisti@4318 | 2480 | #endif // _LP64 |
twisti@4318 | 2481 | } |
twisti@4318 | 2482 | restore_rax(tmp); |
twisti@4318 | 2483 | // Result is in ST0. |
twisti@4318 | 2484 | // Note: fxch & fpop to get rid of ST1 |
twisti@4318 | 2485 | // (otherwise FPU stack could overflow eventually) |
twisti@4318 | 2486 | fxch(1); |
twisti@4318 | 2487 | fpop(); |
twisti@4318 | 2488 | } |
twisti@4318 | 2489 | |
twisti@4318 | 2490 | |
twisti@4318 | 2491 | void MacroAssembler::incrementl(AddressLiteral dst) { |
twisti@4318 | 2492 | if (reachable(dst)) { |
twisti@4318 | 2493 | incrementl(as_Address(dst)); |
twisti@4318 | 2494 | } else { |
twisti@4318 | 2495 | lea(rscratch1, dst); |
twisti@4318 | 2496 | incrementl(Address(rscratch1, 0)); |
twisti@4318 | 2497 | } |
twisti@4318 | 2498 | } |
twisti@4318 | 2499 | |
twisti@4318 | 2500 | void MacroAssembler::incrementl(ArrayAddress dst) { |
twisti@4318 | 2501 | incrementl(as_Address(dst)); |
twisti@4318 | 2502 | } |
twisti@4318 | 2503 | |
twisti@4318 | 2504 | void MacroAssembler::incrementl(Register reg, int value) { |
twisti@4318 | 2505 | if (value == min_jint) {addl(reg, value) ; return; } |
twisti@4318 | 2506 | if (value < 0) { decrementl(reg, -value); return; } |
twisti@4318 | 2507 | if (value == 0) { ; return; } |
twisti@4318 | 2508 | if (value == 1 && UseIncDec) { incl(reg) ; return; } |
twisti@4318 | 2509 | /* else */ { addl(reg, value) ; return; } |
twisti@4318 | 2510 | } |
twisti@4318 | 2511 | |
twisti@4318 | 2512 | void MacroAssembler::incrementl(Address dst, int value) { |
twisti@4318 | 2513 | if (value == min_jint) {addl(dst, value) ; return; } |
twisti@4318 | 2514 | if (value < 0) { decrementl(dst, -value); return; } |
twisti@4318 | 2515 | if (value == 0) { ; return; } |
twisti@4318 | 2516 | if (value == 1 && UseIncDec) { incl(dst) ; return; } |
twisti@4318 | 2517 | /* else */ { addl(dst, value) ; return; } |
twisti@4318 | 2518 | } |
twisti@4318 | 2519 | |
twisti@4318 | 2520 | void MacroAssembler::jump(AddressLiteral dst) { |
twisti@4318 | 2521 | if (reachable(dst)) { |
twisti@4318 | 2522 | jmp_literal(dst.target(), dst.rspec()); |
twisti@4318 | 2523 | } else { |
twisti@4318 | 2524 | lea(rscratch1, dst); |
twisti@4318 | 2525 | jmp(rscratch1); |
twisti@4318 | 2526 | } |
twisti@4318 | 2527 | } |
twisti@4318 | 2528 | |
twisti@4318 | 2529 | void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) { |
twisti@4318 | 2530 | if (reachable(dst)) { |
twisti@4318 | 2531 | InstructionMark im(this); |
twisti@4318 | 2532 | relocate(dst.reloc()); |
twisti@4318 | 2533 | const int short_size = 2; |
twisti@4318 | 2534 | const int long_size = 6; |
twisti@4318 | 2535 | int offs = (intptr_t)dst.target() - ((intptr_t)pc()); |
twisti@4318 | 2536 | if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) { |
twisti@4318 | 2537 | // 0111 tttn #8-bit disp |
twisti@4366 | 2538 | emit_int8(0x70 | cc); |
twisti@4366 | 2539 | emit_int8((offs - short_size) & 0xFF); |
twisti@4318 | 2540 | } else { |
twisti@4318 | 2541 | // 0000 1111 1000 tttn #32-bit disp |
twisti@4366 | 2542 | emit_int8(0x0F); |
twisti@4366 | 2543 | emit_int8((unsigned char)(0x80 | cc)); |
twisti@4412 | 2544 | emit_int32(offs - long_size); |
twisti@4318 | 2545 | } |
twisti@4318 | 2546 | } else { |
twisti@4318 | 2547 | #ifdef ASSERT |
twisti@4318 | 2548 | warning("reversing conditional branch"); |
twisti@4318 | 2549 | #endif /* ASSERT */ |
twisti@4318 | 2550 | Label skip; |
twisti@4318 | 2551 | jccb(reverse[cc], skip); |
twisti@4318 | 2552 | lea(rscratch1, dst); |
twisti@4318 | 2553 | Assembler::jmp(rscratch1); |
twisti@4318 | 2554 | bind(skip); |
twisti@4318 | 2555 | } |
twisti@4318 | 2556 | } |
twisti@4318 | 2557 | |
twisti@4318 | 2558 | void MacroAssembler::ldmxcsr(AddressLiteral src) { |
twisti@4318 | 2559 | if (reachable(src)) { |
twisti@4318 | 2560 | Assembler::ldmxcsr(as_Address(src)); |
twisti@4318 | 2561 | } else { |
twisti@4318 | 2562 | lea(rscratch1, src); |
twisti@4318 | 2563 | Assembler::ldmxcsr(Address(rscratch1, 0)); |
twisti@4318 | 2564 | } |
twisti@4318 | 2565 | } |
twisti@4318 | 2566 | |
twisti@4318 | 2567 | int MacroAssembler::load_signed_byte(Register dst, Address src) { |
twisti@4318 | 2568 | int off; |
twisti@4318 | 2569 | if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
twisti@4318 | 2570 | off = offset(); |
twisti@4318 | 2571 | movsbl(dst, src); // movsxb |
twisti@4318 | 2572 | } else { |
twisti@4318 | 2573 | off = load_unsigned_byte(dst, src); |
twisti@4318 | 2574 | shll(dst, 24); |
twisti@4318 | 2575 | sarl(dst, 24); |
twisti@4318 | 2576 | } |
twisti@4318 | 2577 | return off; |
twisti@4318 | 2578 | } |
twisti@4318 | 2579 | |
twisti@4318 | 2580 | // Note: load_signed_short used to be called load_signed_word. |
twisti@4318 | 2581 | // Although the 'w' in x86 opcodes refers to the term "word" in the assembler |
twisti@4318 | 2582 | // manual, which means 16 bits, that usage is found nowhere in HotSpot code. |
twisti@4318 | 2583 | // The term "word" in HotSpot means a 32- or 64-bit machine word. |
twisti@4318 | 2584 | int MacroAssembler::load_signed_short(Register dst, Address src) { |
twisti@4318 | 2585 | int off; |
twisti@4318 | 2586 | if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
twisti@4318 | 2587 | // This is dubious to me since it seems safe to do a signed 16 => 64 bit |
twisti@4318 | 2588 | // version but this is what 64bit has always done. This seems to imply |
twisti@4318 | 2589 | // that users are only using 32bits worth. |
twisti@4318 | 2590 | off = offset(); |
twisti@4318 | 2591 | movswl(dst, src); // movsxw |
twisti@4318 | 2592 | } else { |
twisti@4318 | 2593 | off = load_unsigned_short(dst, src); |
twisti@4318 | 2594 | shll(dst, 16); |
twisti@4318 | 2595 | sarl(dst, 16); |
twisti@4318 | 2596 | } |
twisti@4318 | 2597 | return off; |
twisti@4318 | 2598 | } |
twisti@4318 | 2599 | |
twisti@4318 | 2600 | int MacroAssembler::load_unsigned_byte(Register dst, Address src) { |
twisti@4318 | 2601 | // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
twisti@4318 | 2602 | // and "3.9 Partial Register Penalties", p. 22). |
twisti@4318 | 2603 | int off; |
twisti@4318 | 2604 | if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) { |
twisti@4318 | 2605 | off = offset(); |
twisti@4318 | 2606 | movzbl(dst, src); // movzxb |
twisti@4318 | 2607 | } else { |
twisti@4318 | 2608 | xorl(dst, dst); |
twisti@4318 | 2609 | off = offset(); |
twisti@4318 | 2610 | movb(dst, src); |
twisti@4318 | 2611 | } |
twisti@4318 | 2612 | return off; |
twisti@4318 | 2613 | } |
twisti@4318 | 2614 | |
twisti@4318 | 2615 | // Note: load_unsigned_short used to be called load_unsigned_word. |
twisti@4318 | 2616 | int MacroAssembler::load_unsigned_short(Register dst, Address src) { |
twisti@4318 | 2617 | // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16, |
twisti@4318 | 2618 | // and "3.9 Partial Register Penalties", p. 22). |
twisti@4318 | 2619 | int off; |
twisti@4318 | 2620 | if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) { |
twisti@4318 | 2621 | off = offset(); |
twisti@4318 | 2622 | movzwl(dst, src); // movzxw |
twisti@4318 | 2623 | } else { |
twisti@4318 | 2624 | xorl(dst, dst); |
twisti@4318 | 2625 | off = offset(); |
twisti@4318 | 2626 | movw(dst, src); |
twisti@4318 | 2627 | } |
twisti@4318 | 2628 | return off; |
twisti@4318 | 2629 | } |
twisti@4318 | 2630 | |
twisti@4318 | 2631 | void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) { |
twisti@4318 | 2632 | switch (size_in_bytes) { |
twisti@4318 | 2633 | #ifndef _LP64 |
twisti@4318 | 2634 | case 8: |
twisti@4318 | 2635 | assert(dst2 != noreg, "second dest register required"); |
twisti@4318 | 2636 | movl(dst, src); |
twisti@4318 | 2637 | movl(dst2, src.plus_disp(BytesPerInt)); |
twisti@4318 | 2638 | break; |
twisti@4318 | 2639 | #else |
twisti@4318 | 2640 | case 8: movq(dst, src); break; |
twisti@4318 | 2641 | #endif |
twisti@4318 | 2642 | case 4: movl(dst, src); break; |
twisti@4318 | 2643 | case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; |
twisti@4318 | 2644 | case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; |
twisti@4318 | 2645 | default: ShouldNotReachHere(); |
twisti@4318 | 2646 | } |
twisti@4318 | 2647 | } |
twisti@4318 | 2648 | |
twisti@4318 | 2649 | void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) { |
twisti@4318 | 2650 | switch (size_in_bytes) { |
twisti@4318 | 2651 | #ifndef _LP64 |
twisti@4318 | 2652 | case 8: |
twisti@4318 | 2653 | assert(src2 != noreg, "second source register required"); |
twisti@4318 | 2654 | movl(dst, src); |
twisti@4318 | 2655 | movl(dst.plus_disp(BytesPerInt), src2); |
twisti@4318 | 2656 | break; |
twisti@4318 | 2657 | #else |
twisti@4318 | 2658 | case 8: movq(dst, src); break; |
twisti@4318 | 2659 | #endif |
twisti@4318 | 2660 | case 4: movl(dst, src); break; |
twisti@4318 | 2661 | case 2: movw(dst, src); break; |
twisti@4318 | 2662 | case 1: movb(dst, src); break; |
twisti@4318 | 2663 | default: ShouldNotReachHere(); |
twisti@4318 | 2664 | } |
twisti@4318 | 2665 | } |
twisti@4318 | 2666 | |
twisti@4318 | 2667 | void MacroAssembler::mov32(AddressLiteral dst, Register src) { |
twisti@4318 | 2668 | if (reachable(dst)) { |
twisti@4318 | 2669 | movl(as_Address(dst), src); |
twisti@4318 | 2670 | } else { |
twisti@4318 | 2671 | lea(rscratch1, dst); |
twisti@4318 | 2672 | movl(Address(rscratch1, 0), src); |
twisti@4318 | 2673 | } |
twisti@4318 | 2674 | } |
twisti@4318 | 2675 | |
twisti@4318 | 2676 | void MacroAssembler::mov32(Register dst, AddressLiteral src) { |
twisti@4318 | 2677 | if (reachable(src)) { |
twisti@4318 | 2678 | movl(dst, as_Address(src)); |
twisti@4318 | 2679 | } else { |
twisti@4318 | 2680 | lea(rscratch1, src); |
twisti@4318 | 2681 | movl(dst, Address(rscratch1, 0)); |
twisti@4318 | 2682 | } |
twisti@4318 | 2683 | } |
twisti@4318 | 2684 | |
twisti@4318 | 2685 | // C++ bool manipulation |
twisti@4318 | 2686 | |
twisti@4318 | 2687 | void MacroAssembler::movbool(Register dst, Address src) { |
twisti@4318 | 2688 | if(sizeof(bool) == 1) |
twisti@4318 | 2689 | movb(dst, src); |
twisti@4318 | 2690 | else if(sizeof(bool) == 2) |
twisti@4318 | 2691 | movw(dst, src); |
twisti@4318 | 2692 | else if(sizeof(bool) == 4) |
twisti@4318 | 2693 | movl(dst, src); |
twisti@4318 | 2694 | else |
twisti@4318 | 2695 | // unsupported |
twisti@4318 | 2696 | ShouldNotReachHere(); |
twisti@4318 | 2697 | } |
twisti@4318 | 2698 | |
twisti@4318 | 2699 | void MacroAssembler::movbool(Address dst, bool boolconst) { |
twisti@4318 | 2700 | if(sizeof(bool) == 1) |
twisti@4318 | 2701 | movb(dst, (int) boolconst); |
twisti@4318 | 2702 | else if(sizeof(bool) == 2) |
twisti@4318 | 2703 | movw(dst, (int) boolconst); |
twisti@4318 | 2704 | else if(sizeof(bool) == 4) |
twisti@4318 | 2705 | movl(dst, (int) boolconst); |
twisti@4318 | 2706 | else |
twisti@4318 | 2707 | // unsupported |
twisti@4318 | 2708 | ShouldNotReachHere(); |
twisti@4318 | 2709 | } |
twisti@4318 | 2710 | |
twisti@4318 | 2711 | void MacroAssembler::movbool(Address dst, Register src) { |
twisti@4318 | 2712 | if(sizeof(bool) == 1) |
twisti@4318 | 2713 | movb(dst, src); |
twisti@4318 | 2714 | else if(sizeof(bool) == 2) |
twisti@4318 | 2715 | movw(dst, src); |
twisti@4318 | 2716 | else if(sizeof(bool) == 4) |
twisti@4318 | 2717 | movl(dst, src); |
twisti@4318 | 2718 | else |
twisti@4318 | 2719 | // unsupported |
twisti@4318 | 2720 | ShouldNotReachHere(); |
twisti@4318 | 2721 | } |
twisti@4318 | 2722 | |
twisti@4318 | 2723 | void MacroAssembler::movbyte(ArrayAddress dst, int src) { |
twisti@4318 | 2724 | movb(as_Address(dst), src); |
twisti@4318 | 2725 | } |
twisti@4318 | 2726 | |
twisti@4318 | 2727 | void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2728 | if (reachable(src)) { |
twisti@4318 | 2729 | movdl(dst, as_Address(src)); |
twisti@4318 | 2730 | } else { |
twisti@4318 | 2731 | lea(rscratch1, src); |
twisti@4318 | 2732 | movdl(dst, Address(rscratch1, 0)); |
twisti@4318 | 2733 | } |
twisti@4318 | 2734 | } |
twisti@4318 | 2735 | |
twisti@4318 | 2736 | void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2737 | if (reachable(src)) { |
twisti@4318 | 2738 | movq(dst, as_Address(src)); |
twisti@4318 | 2739 | } else { |
twisti@4318 | 2740 | lea(rscratch1, src); |
twisti@4318 | 2741 | movq(dst, Address(rscratch1, 0)); |
twisti@4318 | 2742 | } |
twisti@4318 | 2743 | } |
twisti@4318 | 2744 | |
twisti@4318 | 2745 | void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2746 | if (reachable(src)) { |
twisti@4318 | 2747 | if (UseXmmLoadAndClearUpper) { |
twisti@4318 | 2748 | movsd (dst, as_Address(src)); |
twisti@4318 | 2749 | } else { |
twisti@4318 | 2750 | movlpd(dst, as_Address(src)); |
twisti@4318 | 2751 | } |
twisti@4318 | 2752 | } else { |
twisti@4318 | 2753 | lea(rscratch1, src); |
twisti@4318 | 2754 | if (UseXmmLoadAndClearUpper) { |
twisti@4318 | 2755 | movsd (dst, Address(rscratch1, 0)); |
twisti@4318 | 2756 | } else { |
twisti@4318 | 2757 | movlpd(dst, Address(rscratch1, 0)); |
twisti@4318 | 2758 | } |
twisti@4318 | 2759 | } |
twisti@4318 | 2760 | } |
twisti@4318 | 2761 | |
twisti@4318 | 2762 | void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2763 | if (reachable(src)) { |
twisti@4318 | 2764 | movss(dst, as_Address(src)); |
twisti@4318 | 2765 | } else { |
twisti@4318 | 2766 | lea(rscratch1, src); |
twisti@4318 | 2767 | movss(dst, Address(rscratch1, 0)); |
twisti@4318 | 2768 | } |
twisti@4318 | 2769 | } |
twisti@4318 | 2770 | |
twisti@4318 | 2771 | void MacroAssembler::movptr(Register dst, Register src) { |
twisti@4318 | 2772 | LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
twisti@4318 | 2773 | } |
twisti@4318 | 2774 | |
twisti@4318 | 2775 | void MacroAssembler::movptr(Register dst, Address src) { |
twisti@4318 | 2776 | LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
twisti@4318 | 2777 | } |
twisti@4318 | 2778 | |
twisti@4318 | 2779 | // src should NEVER be a real pointer. Use AddressLiteral for true pointers |
twisti@4318 | 2780 | void MacroAssembler::movptr(Register dst, intptr_t src) { |
twisti@4318 | 2781 | LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src)); |
twisti@4318 | 2782 | } |
twisti@4318 | 2783 | |
twisti@4318 | 2784 | void MacroAssembler::movptr(Address dst, Register src) { |
twisti@4318 | 2785 | LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src)); |
twisti@4318 | 2786 | } |
twisti@4318 | 2787 | |
twisti@4318 | 2788 | void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2789 | if (reachable(src)) { |
twisti@4318 | 2790 | Assembler::movdqu(dst, as_Address(src)); |
twisti@4318 | 2791 | } else { |
twisti@4318 | 2792 | lea(rscratch1, src); |
twisti@4318 | 2793 | Assembler::movdqu(dst, Address(rscratch1, 0)); |
twisti@4318 | 2794 | } |
twisti@4318 | 2795 | } |
twisti@4318 | 2796 | |
twisti@4318 | 2797 | void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2798 | if (reachable(src)) { |
twisti@4318 | 2799 | Assembler::movsd(dst, as_Address(src)); |
twisti@4318 | 2800 | } else { |
twisti@4318 | 2801 | lea(rscratch1, src); |
twisti@4318 | 2802 | Assembler::movsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 2803 | } |
twisti@4318 | 2804 | } |
twisti@4318 | 2805 | |
twisti@4318 | 2806 | void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2807 | if (reachable(src)) { |
twisti@4318 | 2808 | Assembler::movss(dst, as_Address(src)); |
twisti@4318 | 2809 | } else { |
twisti@4318 | 2810 | lea(rscratch1, src); |
twisti@4318 | 2811 | Assembler::movss(dst, Address(rscratch1, 0)); |
twisti@4318 | 2812 | } |
twisti@4318 | 2813 | } |
twisti@4318 | 2814 | |
twisti@4318 | 2815 | void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2816 | if (reachable(src)) { |
twisti@4318 | 2817 | Assembler::mulsd(dst, as_Address(src)); |
twisti@4318 | 2818 | } else { |
twisti@4318 | 2819 | lea(rscratch1, src); |
twisti@4318 | 2820 | Assembler::mulsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 2821 | } |
twisti@4318 | 2822 | } |
twisti@4318 | 2823 | |
twisti@4318 | 2824 | void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 2825 | if (reachable(src)) { |
twisti@4318 | 2826 | Assembler::mulss(dst, as_Address(src)); |
twisti@4318 | 2827 | } else { |
twisti@4318 | 2828 | lea(rscratch1, src); |
twisti@4318 | 2829 | Assembler::mulss(dst, Address(rscratch1, 0)); |
twisti@4318 | 2830 | } |
twisti@4318 | 2831 | } |
twisti@4318 | 2832 | |
twisti@4318 | 2833 | void MacroAssembler::null_check(Register reg, int offset) { |
twisti@4318 | 2834 | if (needs_explicit_null_check(offset)) { |
twisti@4318 | 2835 | // provoke OS NULL exception if reg = NULL by |
twisti@4318 | 2836 | // accessing M[reg] w/o changing any (non-CC) registers |
twisti@4318 | 2837 | // NOTE: cmpl is plenty here to provoke a segv |
twisti@4318 | 2838 | cmpptr(rax, Address(reg, 0)); |
twisti@4318 | 2839 | // Note: should probably use testl(rax, Address(reg, 0)); |
twisti@4318 | 2840 | // may be shorter code (however, this version of |
twisti@4318 | 2841 | // testl needs to be implemented first) |
twisti@4318 | 2842 | } else { |
twisti@4318 | 2843 | // nothing to do, (later) access of M[reg + offset] |
twisti@4318 | 2844 | // will provoke OS NULL exception if reg = NULL |
twisti@4318 | 2845 | } |
twisti@4318 | 2846 | } |
twisti@4318 | 2847 | |
twisti@4318 | 2848 | void MacroAssembler::os_breakpoint() { |
twisti@4318 | 2849 | // instead of directly emitting a breakpoint, call os:breakpoint for better debugability |
twisti@4318 | 2850 | // (e.g., MSVC can't call ps() otherwise) |
twisti@4318 | 2851 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint))); |
twisti@4318 | 2852 | } |
twisti@4318 | 2853 | |
twisti@4318 | 2854 | void MacroAssembler::pop_CPU_state() { |
twisti@4318 | 2855 | pop_FPU_state(); |
twisti@4318 | 2856 | pop_IU_state(); |
twisti@4318 | 2857 | } |
twisti@4318 | 2858 | |
twisti@4318 | 2859 | void MacroAssembler::pop_FPU_state() { |
twisti@4318 | 2860 | NOT_LP64(frstor(Address(rsp, 0));) |
twisti@4318 | 2861 | LP64_ONLY(fxrstor(Address(rsp, 0));) |
twisti@4318 | 2862 | addptr(rsp, FPUStateSizeInWords * wordSize); |
twisti@4318 | 2863 | } |
twisti@4318 | 2864 | |
twisti@4318 | 2865 | void MacroAssembler::pop_IU_state() { |
twisti@4318 | 2866 | popa(); |
twisti@4318 | 2867 | LP64_ONLY(addq(rsp, 8)); |
twisti@4318 | 2868 | popf(); |
twisti@4318 | 2869 | } |
twisti@4318 | 2870 | |
twisti@4318 | 2871 | // Save Integer and Float state |
twisti@4318 | 2872 | // Warning: Stack must be 16 byte aligned (64bit) |
twisti@4318 | 2873 | void MacroAssembler::push_CPU_state() { |
twisti@4318 | 2874 | push_IU_state(); |
twisti@4318 | 2875 | push_FPU_state(); |
twisti@4318 | 2876 | } |
twisti@4318 | 2877 | |
twisti@4318 | 2878 | void MacroAssembler::push_FPU_state() { |
twisti@4318 | 2879 | subptr(rsp, FPUStateSizeInWords * wordSize); |
twisti@4318 | 2880 | #ifndef _LP64 |
twisti@4318 | 2881 | fnsave(Address(rsp, 0)); |
twisti@4318 | 2882 | fwait(); |
twisti@4318 | 2883 | #else |
twisti@4318 | 2884 | fxsave(Address(rsp, 0)); |
twisti@4318 | 2885 | #endif // LP64 |
twisti@4318 | 2886 | } |
twisti@4318 | 2887 | |
twisti@4318 | 2888 | void MacroAssembler::push_IU_state() { |
twisti@4318 | 2889 | // Push flags first because pusha kills them |
twisti@4318 | 2890 | pushf(); |
twisti@4318 | 2891 | // Make sure rsp stays 16-byte aligned |
twisti@4318 | 2892 | LP64_ONLY(subq(rsp, 8)); |
twisti@4318 | 2893 | pusha(); |
twisti@4318 | 2894 | } |
twisti@4318 | 2895 | |
twisti@4318 | 2896 | void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp, bool clear_pc) { |
twisti@4318 | 2897 | // determine java_thread register |
twisti@4318 | 2898 | if (!java_thread->is_valid()) { |
twisti@4318 | 2899 | java_thread = rdi; |
twisti@4318 | 2900 | get_thread(java_thread); |
twisti@4318 | 2901 | } |
twisti@4318 | 2902 | // we must set sp to zero to clear frame |
twisti@4318 | 2903 | movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD); |
twisti@4318 | 2904 | if (clear_fp) { |
twisti@4318 | 2905 | movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD); |
twisti@4318 | 2906 | } |
twisti@4318 | 2907 | |
twisti@4318 | 2908 | if (clear_pc) |
twisti@4318 | 2909 | movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD); |
twisti@4318 | 2910 | |
twisti@4318 | 2911 | } |
twisti@4318 | 2912 | |
twisti@4318 | 2913 | void MacroAssembler::restore_rax(Register tmp) { |
twisti@4318 | 2914 | if (tmp == noreg) pop(rax); |
twisti@4318 | 2915 | else if (tmp != rax) mov(rax, tmp); |
twisti@4318 | 2916 | } |
twisti@4318 | 2917 | |
twisti@4318 | 2918 | void MacroAssembler::round_to(Register reg, int modulus) { |
twisti@4318 | 2919 | addptr(reg, modulus - 1); |
twisti@4318 | 2920 | andptr(reg, -modulus); |
twisti@4318 | 2921 | } |
twisti@4318 | 2922 | |
twisti@4318 | 2923 | void MacroAssembler::save_rax(Register tmp) { |
twisti@4318 | 2924 | if (tmp == noreg) push(rax); |
twisti@4318 | 2925 | else if (tmp != rax) mov(tmp, rax); |
twisti@4318 | 2926 | } |
twisti@4318 | 2927 | |
twisti@4318 | 2928 | // Write serialization page so VM thread can do a pseudo remote membar. |
twisti@4318 | 2929 | // We use the current thread pointer to calculate a thread specific |
twisti@4318 | 2930 | // offset to write to within the page. This minimizes bus traffic |
twisti@4318 | 2931 | // due to cache line collision. |
twisti@4318 | 2932 | void MacroAssembler::serialize_memory(Register thread, Register tmp) { |
twisti@4318 | 2933 | movl(tmp, thread); |
twisti@4318 | 2934 | shrl(tmp, os::get_serialize_page_shift_count()); |
twisti@4318 | 2935 | andl(tmp, (os::vm_page_size() - sizeof(int))); |
twisti@4318 | 2936 | |
twisti@4318 | 2937 | Address index(noreg, tmp, Address::times_1); |
twisti@4318 | 2938 | ExternalAddress page(os::get_memory_serialize_page()); |
twisti@4318 | 2939 | |
twisti@4318 | 2940 | // Size of store must match masking code above |
twisti@4318 | 2941 | movl(as_Address(ArrayAddress(page, index)), tmp); |
twisti@4318 | 2942 | } |
twisti@4318 | 2943 | |
twisti@4318 | 2944 | // Calls to C land |
twisti@4318 | 2945 | // |
twisti@4318 | 2946 | // When entering C land, the rbp, & rsp of the last Java frame have to be recorded |
twisti@4318 | 2947 | // in the (thread-local) JavaThread object. When leaving C land, the last Java fp |
twisti@4318 | 2948 | // has to be reset to 0. This is required to allow proper stack traversal. |
twisti@4318 | 2949 | void MacroAssembler::set_last_Java_frame(Register java_thread, |
twisti@4318 | 2950 | Register last_java_sp, |
twisti@4318 | 2951 | Register last_java_fp, |
twisti@4318 | 2952 | address last_java_pc) { |
twisti@4318 | 2953 | // determine java_thread register |
twisti@4318 | 2954 | if (!java_thread->is_valid()) { |
twisti@4318 | 2955 | java_thread = rdi; |
twisti@4318 | 2956 | get_thread(java_thread); |
twisti@4318 | 2957 | } |
twisti@4318 | 2958 | // determine last_java_sp register |
twisti@4318 | 2959 | if (!last_java_sp->is_valid()) { |
twisti@4318 | 2960 | last_java_sp = rsp; |
twisti@4318 | 2961 | } |
twisti@4318 | 2962 | |
twisti@4318 | 2963 | // last_java_fp is optional |
twisti@4318 | 2964 | |
twisti@4318 | 2965 | if (last_java_fp->is_valid()) { |
twisti@4318 | 2966 | movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp); |
twisti@4318 | 2967 | } |
twisti@4318 | 2968 | |
twisti@4318 | 2969 | // last_java_pc is optional |
twisti@4318 | 2970 | |
twisti@4318 | 2971 | if (last_java_pc != NULL) { |
twisti@4318 | 2972 | lea(Address(java_thread, |
twisti@4318 | 2973 | JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()), |
twisti@4318 | 2974 | InternalAddress(last_java_pc)); |
twisti@4318 | 2975 | |
twisti@4318 | 2976 | } |
twisti@4318 | 2977 | movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp); |
twisti@4318 | 2978 | } |
twisti@4318 | 2979 | |
twisti@4318 | 2980 | void MacroAssembler::shlptr(Register dst, int imm8) { |
twisti@4318 | 2981 | LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8)); |
twisti@4318 | 2982 | } |
twisti@4318 | 2983 | |
twisti@4318 | 2984 | void MacroAssembler::shrptr(Register dst, int imm8) { |
twisti@4318 | 2985 | LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8)); |
twisti@4318 | 2986 | } |
twisti@4318 | 2987 | |
twisti@4318 | 2988 | void MacroAssembler::sign_extend_byte(Register reg) { |
twisti@4318 | 2989 | if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) { |
twisti@4318 | 2990 | movsbl(reg, reg); // movsxb |
twisti@4318 | 2991 | } else { |
twisti@4318 | 2992 | shll(reg, 24); |
twisti@4318 | 2993 | sarl(reg, 24); |
twisti@4318 | 2994 | } |
twisti@4318 | 2995 | } |
twisti@4318 | 2996 | |
twisti@4318 | 2997 | void MacroAssembler::sign_extend_short(Register reg) { |
twisti@4318 | 2998 | if (LP64_ONLY(true ||) VM_Version::is_P6()) { |
twisti@4318 | 2999 | movswl(reg, reg); // movsxw |
twisti@4318 | 3000 | } else { |
twisti@4318 | 3001 | shll(reg, 16); |
twisti@4318 | 3002 | sarl(reg, 16); |
twisti@4318 | 3003 | } |
twisti@4318 | 3004 | } |
twisti@4318 | 3005 | |
twisti@4318 | 3006 | void MacroAssembler::testl(Register dst, AddressLiteral src) { |
twisti@4318 | 3007 | assert(reachable(src), "Address should be reachable"); |
twisti@4318 | 3008 | testl(dst, as_Address(src)); |
twisti@4318 | 3009 | } |
twisti@4318 | 3010 | |
twisti@4318 | 3011 | void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3012 | if (reachable(src)) { |
twisti@4318 | 3013 | Assembler::sqrtsd(dst, as_Address(src)); |
twisti@4318 | 3014 | } else { |
twisti@4318 | 3015 | lea(rscratch1, src); |
twisti@4318 | 3016 | Assembler::sqrtsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 3017 | } |
twisti@4318 | 3018 | } |
twisti@4318 | 3019 | |
twisti@4318 | 3020 | void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3021 | if (reachable(src)) { |
twisti@4318 | 3022 | Assembler::sqrtss(dst, as_Address(src)); |
twisti@4318 | 3023 | } else { |
twisti@4318 | 3024 | lea(rscratch1, src); |
twisti@4318 | 3025 | Assembler::sqrtss(dst, Address(rscratch1, 0)); |
twisti@4318 | 3026 | } |
twisti@4318 | 3027 | } |
twisti@4318 | 3028 | |
twisti@4318 | 3029 | void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3030 | if (reachable(src)) { |
twisti@4318 | 3031 | Assembler::subsd(dst, as_Address(src)); |
twisti@4318 | 3032 | } else { |
twisti@4318 | 3033 | lea(rscratch1, src); |
twisti@4318 | 3034 | Assembler::subsd(dst, Address(rscratch1, 0)); |
twisti@4318 | 3035 | } |
twisti@4318 | 3036 | } |
twisti@4318 | 3037 | |
twisti@4318 | 3038 | void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3039 | if (reachable(src)) { |
twisti@4318 | 3040 | Assembler::subss(dst, as_Address(src)); |
twisti@4318 | 3041 | } else { |
twisti@4318 | 3042 | lea(rscratch1, src); |
twisti@4318 | 3043 | Assembler::subss(dst, Address(rscratch1, 0)); |
twisti@4318 | 3044 | } |
twisti@4318 | 3045 | } |
twisti@4318 | 3046 | |
twisti@4318 | 3047 | void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3048 | if (reachable(src)) { |
twisti@4318 | 3049 | Assembler::ucomisd(dst, as_Address(src)); |
twisti@4318 | 3050 | } else { |
twisti@4318 | 3051 | lea(rscratch1, src); |
twisti@4318 | 3052 | Assembler::ucomisd(dst, Address(rscratch1, 0)); |
twisti@4318 | 3053 | } |
twisti@4318 | 3054 | } |
twisti@4318 | 3055 | |
twisti@4318 | 3056 | void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3057 | if (reachable(src)) { |
twisti@4318 | 3058 | Assembler::ucomiss(dst, as_Address(src)); |
twisti@4318 | 3059 | } else { |
twisti@4318 | 3060 | lea(rscratch1, src); |
twisti@4318 | 3061 | Assembler::ucomiss(dst, Address(rscratch1, 0)); |
twisti@4318 | 3062 | } |
twisti@4318 | 3063 | } |
twisti@4318 | 3064 | |
twisti@4318 | 3065 | void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3066 | // Used in sign-bit flipping with aligned address. |
twisti@4318 | 3067 | assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
twisti@4318 | 3068 | if (reachable(src)) { |
twisti@4318 | 3069 | Assembler::xorpd(dst, as_Address(src)); |
twisti@4318 | 3070 | } else { |
twisti@4318 | 3071 | lea(rscratch1, src); |
twisti@4318 | 3072 | Assembler::xorpd(dst, Address(rscratch1, 0)); |
twisti@4318 | 3073 | } |
twisti@4318 | 3074 | } |
twisti@4318 | 3075 | |
twisti@4318 | 3076 | void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3077 | // Used in sign-bit flipping with aligned address. |
twisti@4318 | 3078 | assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes"); |
twisti@4318 | 3079 | if (reachable(src)) { |
twisti@4318 | 3080 | Assembler::xorps(dst, as_Address(src)); |
twisti@4318 | 3081 | } else { |
twisti@4318 | 3082 | lea(rscratch1, src); |
twisti@4318 | 3083 | Assembler::xorps(dst, Address(rscratch1, 0)); |
twisti@4318 | 3084 | } |
twisti@4318 | 3085 | } |
twisti@4318 | 3086 | |
twisti@4318 | 3087 | void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) { |
twisti@4318 | 3088 | // Used in sign-bit flipping with aligned address. |
kvn@4363 | 3089 | bool aligned_adr = (((intptr_t)src.target() & 15) == 0); |
kvn@4363 | 3090 | assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes"); |
twisti@4318 | 3091 | if (reachable(src)) { |
twisti@4318 | 3092 | Assembler::pshufb(dst, as_Address(src)); |
twisti@4318 | 3093 | } else { |
twisti@4318 | 3094 | lea(rscratch1, src); |
twisti@4318 | 3095 | Assembler::pshufb(dst, Address(rscratch1, 0)); |
twisti@4318 | 3096 | } |
twisti@4318 | 3097 | } |
twisti@4318 | 3098 | |
twisti@4318 | 3099 | // AVX 3-operands instructions |
twisti@4318 | 3100 | |
twisti@4318 | 3101 | void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3102 | if (reachable(src)) { |
twisti@4318 | 3103 | vaddsd(dst, nds, as_Address(src)); |
twisti@4318 | 3104 | } else { |
twisti@4318 | 3105 | lea(rscratch1, src); |
twisti@4318 | 3106 | vaddsd(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3107 | } |
twisti@4318 | 3108 | } |
twisti@4318 | 3109 | |
twisti@4318 | 3110 | void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3111 | if (reachable(src)) { |
twisti@4318 | 3112 | vaddss(dst, nds, as_Address(src)); |
twisti@4318 | 3113 | } else { |
twisti@4318 | 3114 | lea(rscratch1, src); |
twisti@4318 | 3115 | vaddss(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3116 | } |
twisti@4318 | 3117 | } |
twisti@4318 | 3118 | |
twisti@4318 | 3119 | void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) { |
twisti@4318 | 3120 | if (reachable(src)) { |
twisti@4318 | 3121 | vandpd(dst, nds, as_Address(src), vector256); |
twisti@4318 | 3122 | } else { |
twisti@4318 | 3123 | lea(rscratch1, src); |
twisti@4318 | 3124 | vandpd(dst, nds, Address(rscratch1, 0), vector256); |
twisti@4318 | 3125 | } |
twisti@4318 | 3126 | } |
twisti@4318 | 3127 | |
twisti@4318 | 3128 | void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) { |
twisti@4318 | 3129 | if (reachable(src)) { |
twisti@4318 | 3130 | vandps(dst, nds, as_Address(src), vector256); |
twisti@4318 | 3131 | } else { |
twisti@4318 | 3132 | lea(rscratch1, src); |
twisti@4318 | 3133 | vandps(dst, nds, Address(rscratch1, 0), vector256); |
twisti@4318 | 3134 | } |
twisti@4318 | 3135 | } |
twisti@4318 | 3136 | |
twisti@4318 | 3137 | void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3138 | if (reachable(src)) { |
twisti@4318 | 3139 | vdivsd(dst, nds, as_Address(src)); |
twisti@4318 | 3140 | } else { |
twisti@4318 | 3141 | lea(rscratch1, src); |
twisti@4318 | 3142 | vdivsd(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3143 | } |
twisti@4318 | 3144 | } |
twisti@4318 | 3145 | |
twisti@4318 | 3146 | void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3147 | if (reachable(src)) { |
twisti@4318 | 3148 | vdivss(dst, nds, as_Address(src)); |
twisti@4318 | 3149 | } else { |
twisti@4318 | 3150 | lea(rscratch1, src); |
twisti@4318 | 3151 | vdivss(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3152 | } |
twisti@4318 | 3153 | } |
twisti@4318 | 3154 | |
twisti@4318 | 3155 | void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3156 | if (reachable(src)) { |
twisti@4318 | 3157 | vmulsd(dst, nds, as_Address(src)); |
twisti@4318 | 3158 | } else { |
twisti@4318 | 3159 | lea(rscratch1, src); |
twisti@4318 | 3160 | vmulsd(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3161 | } |
twisti@4318 | 3162 | } |
twisti@4318 | 3163 | |
twisti@4318 | 3164 | void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3165 | if (reachable(src)) { |
twisti@4318 | 3166 | vmulss(dst, nds, as_Address(src)); |
twisti@4318 | 3167 | } else { |
twisti@4318 | 3168 | lea(rscratch1, src); |
twisti@4318 | 3169 | vmulss(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3170 | } |
twisti@4318 | 3171 | } |
twisti@4318 | 3172 | |
twisti@4318 | 3173 | void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3174 | if (reachable(src)) { |
twisti@4318 | 3175 | vsubsd(dst, nds, as_Address(src)); |
twisti@4318 | 3176 | } else { |
twisti@4318 | 3177 | lea(rscratch1, src); |
twisti@4318 | 3178 | vsubsd(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3179 | } |
twisti@4318 | 3180 | } |
twisti@4318 | 3181 | |
twisti@4318 | 3182 | void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) { |
twisti@4318 | 3183 | if (reachable(src)) { |
twisti@4318 | 3184 | vsubss(dst, nds, as_Address(src)); |
twisti@4318 | 3185 | } else { |
twisti@4318 | 3186 | lea(rscratch1, src); |
twisti@4318 | 3187 | vsubss(dst, nds, Address(rscratch1, 0)); |
twisti@4318 | 3188 | } |
twisti@4318 | 3189 | } |
twisti@4318 | 3190 | |
twisti@4318 | 3191 | void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) { |
twisti@4318 | 3192 | if (reachable(src)) { |
twisti@4318 | 3193 | vxorpd(dst, nds, as_Address(src), vector256); |
twisti@4318 | 3194 | } else { |
twisti@4318 | 3195 | lea(rscratch1, src); |
twisti@4318 | 3196 | vxorpd(dst, nds, Address(rscratch1, 0), vector256); |
twisti@4318 | 3197 | } |
twisti@4318 | 3198 | } |
twisti@4318 | 3199 | |
twisti@4318 | 3200 | void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) { |
twisti@4318 | 3201 | if (reachable(src)) { |
twisti@4318 | 3202 | vxorps(dst, nds, as_Address(src), vector256); |
twisti@4318 | 3203 | } else { |
twisti@4318 | 3204 | lea(rscratch1, src); |
twisti@4318 | 3205 | vxorps(dst, nds, Address(rscratch1, 0), vector256); |
twisti@4318 | 3206 | } |
twisti@4318 | 3207 | } |
twisti@4318 | 3208 | |
twisti@4318 | 3209 | |
twisti@4318 | 3210 | ////////////////////////////////////////////////////////////////////////////////// |
jprovino@4542 | 3211 | #if INCLUDE_ALL_GCS |
twisti@4318 | 3212 | |
twisti@4318 | 3213 | void MacroAssembler::g1_write_barrier_pre(Register obj, |
twisti@4318 | 3214 | Register pre_val, |
twisti@4318 | 3215 | Register thread, |
twisti@4318 | 3216 | Register tmp, |
twisti@4318 | 3217 | bool tosca_live, |
twisti@4318 | 3218 | bool expand_call) { |
twisti@4318 | 3219 | |
twisti@4318 | 3220 | // If expand_call is true then we expand the call_VM_leaf macro |
twisti@4318 | 3221 | // directly to skip generating the check by |
twisti@4318 | 3222 | // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. |
twisti@4318 | 3223 | |
twisti@4318 | 3224 | #ifdef _LP64 |
twisti@4318 | 3225 | assert(thread == r15_thread, "must be"); |
twisti@4318 | 3226 | #endif // _LP64 |
twisti@4318 | 3227 | |
twisti@4318 | 3228 | Label done; |
twisti@4318 | 3229 | Label runtime; |
twisti@4318 | 3230 | |
twisti@4318 | 3231 | assert(pre_val != noreg, "check this code"); |
twisti@4318 | 3232 | |
twisti@4318 | 3233 | if (obj != noreg) { |
twisti@4318 | 3234 | assert_different_registers(obj, pre_val, tmp); |
twisti@4318 | 3235 | assert(pre_val != rax, "check this code"); |
twisti@4318 | 3236 | } |
twisti@4318 | 3237 | |
twisti@4318 | 3238 | Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
twisti@4318 | 3239 | PtrQueue::byte_offset_of_active())); |
twisti@4318 | 3240 | Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
twisti@4318 | 3241 | PtrQueue::byte_offset_of_index())); |
twisti@4318 | 3242 | Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + |
twisti@4318 | 3243 | PtrQueue::byte_offset_of_buf())); |
twisti@4318 | 3244 | |
twisti@4318 | 3245 | |
twisti@4318 | 3246 | // Is marking active? |
twisti@4318 | 3247 | if (in_bytes(PtrQueue::byte_width_of_active()) == 4) { |
twisti@4318 | 3248 | cmpl(in_progress, 0); |
twisti@4318 | 3249 | } else { |
twisti@4318 | 3250 | assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption"); |
twisti@4318 | 3251 | cmpb(in_progress, 0); |
twisti@4318 | 3252 | } |
twisti@4318 | 3253 | jcc(Assembler::equal, done); |
twisti@4318 | 3254 | |
twisti@4318 | 3255 | // Do we need to load the previous value? |
twisti@4318 | 3256 | if (obj != noreg) { |
twisti@4318 | 3257 | load_heap_oop(pre_val, Address(obj, 0)); |
twisti@4318 | 3258 | } |
twisti@4318 | 3259 | |
twisti@4318 | 3260 | // Is the previous value null? |
twisti@4318 | 3261 | cmpptr(pre_val, (int32_t) NULL_WORD); |
twisti@4318 | 3262 | jcc(Assembler::equal, done); |
twisti@4318 | 3263 | |
twisti@4318 | 3264 | // Can we store original value in the thread's buffer? |
twisti@4318 | 3265 | // Is index == 0? |
twisti@4318 | 3266 | // (The index field is typed as size_t.) |
twisti@4318 | 3267 | |
twisti@4318 | 3268 | movptr(tmp, index); // tmp := *index_adr |
twisti@4318 | 3269 | cmpptr(tmp, 0); // tmp == 0? |
twisti@4318 | 3270 | jcc(Assembler::equal, runtime); // If yes, goto runtime |
twisti@4318 | 3271 | |
twisti@4318 | 3272 | subptr(tmp, wordSize); // tmp := tmp - wordSize |
twisti@4318 | 3273 | movptr(index, tmp); // *index_adr := tmp |
twisti@4318 | 3274 | addptr(tmp, buffer); // tmp := tmp + *buffer_adr |
twisti@4318 | 3275 | |
twisti@4318 | 3276 | // Record the previous value |
twisti@4318 | 3277 | movptr(Address(tmp, 0), pre_val); |
twisti@4318 | 3278 | jmp(done); |
twisti@4318 | 3279 | |
twisti@4318 | 3280 | bind(runtime); |
twisti@4318 | 3281 | // save the live input values |
twisti@4318 | 3282 | if(tosca_live) push(rax); |
twisti@4318 | 3283 | |
twisti@4318 | 3284 | if (obj != noreg && obj != rax) |
twisti@4318 | 3285 | push(obj); |
twisti@4318 | 3286 | |
twisti@4318 | 3287 | if (pre_val != rax) |
twisti@4318 | 3288 | push(pre_val); |
twisti@4318 | 3289 | |
twisti@4318 | 3290 | // Calling the runtime using the regular call_VM_leaf mechanism generates |
twisti@4318 | 3291 | // code (generated by InterpreterMacroAssember::call_VM_leaf_base) |
twisti@4318 | 3292 | // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL. |
twisti@4318 | 3293 | // |
twisti@4318 | 3294 | // If we care generating the pre-barrier without a frame (e.g. in the |
twisti@4318 | 3295 | // intrinsified Reference.get() routine) then ebp might be pointing to |
twisti@4318 | 3296 | // the caller frame and so this check will most likely fail at runtime. |
twisti@4318 | 3297 | // |
twisti@4318 | 3298 | // Expanding the call directly bypasses the generation of the check. |
twisti@4318 | 3299 | // So when we do not have have a full interpreter frame on the stack |
twisti@4318 | 3300 | // expand_call should be passed true. |
twisti@4318 | 3301 | |
twisti@4318 | 3302 | NOT_LP64( push(thread); ) |
twisti@4318 | 3303 | |
twisti@4318 | 3304 | if (expand_call) { |
twisti@4318 | 3305 | LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); ) |
twisti@4318 | 3306 | pass_arg1(this, thread); |
twisti@4318 | 3307 | pass_arg0(this, pre_val); |
twisti@4318 | 3308 | MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2); |
twisti@4318 | 3309 | } else { |
twisti@4318 | 3310 | call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); |
twisti@4318 | 3311 | } |
twisti@4318 | 3312 | |
twisti@4318 | 3313 | NOT_LP64( pop(thread); ) |
twisti@4318 | 3314 | |
twisti@4318 | 3315 | // save the live input values |
twisti@4318 | 3316 | if (pre_val != rax) |
twisti@4318 | 3317 | pop(pre_val); |
twisti@4318 | 3318 | |
twisti@4318 | 3319 | if (obj != noreg && obj != rax) |
twisti@4318 | 3320 | pop(obj); |
twisti@4318 | 3321 | |
twisti@4318 | 3322 | if(tosca_live) pop(rax); |
twisti@4318 | 3323 | |
twisti@4318 | 3324 | bind(done); |
twisti@4318 | 3325 | } |
twisti@4318 | 3326 | |
twisti@4318 | 3327 | void MacroAssembler::g1_write_barrier_post(Register store_addr, |
twisti@4318 | 3328 | Register new_val, |
twisti@4318 | 3329 | Register thread, |
twisti@4318 | 3330 | Register tmp, |
twisti@4318 | 3331 | Register tmp2) { |
twisti@4318 | 3332 | #ifdef _LP64 |
twisti@4318 | 3333 | assert(thread == r15_thread, "must be"); |
twisti@4318 | 3334 | #endif // _LP64 |
twisti@4318 | 3335 | |
twisti@4318 | 3336 | Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + |
twisti@4318 | 3337 | PtrQueue::byte_offset_of_index())); |
twisti@4318 | 3338 | Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + |
twisti@4318 | 3339 | PtrQueue::byte_offset_of_buf())); |
twisti@4318 | 3340 | |
twisti@4318 | 3341 | BarrierSet* bs = Universe::heap()->barrier_set(); |
twisti@4318 | 3342 | CardTableModRefBS* ct = (CardTableModRefBS*)bs; |
twisti@4318 | 3343 | Label done; |
twisti@4318 | 3344 | Label runtime; |
twisti@4318 | 3345 | |
twisti@4318 | 3346 | // Does store cross heap regions? |
twisti@4318 | 3347 | |
twisti@4318 | 3348 | movptr(tmp, store_addr); |
twisti@4318 | 3349 | xorptr(tmp, new_val); |
twisti@4318 | 3350 | shrptr(tmp, HeapRegion::LogOfHRGrainBytes); |
twisti@4318 | 3351 | jcc(Assembler::equal, done); |
twisti@4318 | 3352 | |
twisti@4318 | 3353 | // crosses regions, storing NULL? |
twisti@4318 | 3354 | |
twisti@4318 | 3355 | cmpptr(new_val, (int32_t) NULL_WORD); |
twisti@4318 | 3356 | jcc(Assembler::equal, done); |
twisti@4318 | 3357 | |
twisti@4318 | 3358 | // storing region crossing non-NULL, is card already dirty? |
twisti@4318 | 3359 | |
twisti@4318 | 3360 | ExternalAddress cardtable((address) ct->byte_map_base); |
twisti@4318 | 3361 | assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
twisti@4318 | 3362 | #ifdef _LP64 |
twisti@4318 | 3363 | const Register card_addr = tmp; |
twisti@4318 | 3364 | |
twisti@4318 | 3365 | movq(card_addr, store_addr); |
twisti@4318 | 3366 | shrq(card_addr, CardTableModRefBS::card_shift); |
twisti@4318 | 3367 | |
twisti@4318 | 3368 | lea(tmp2, cardtable); |
twisti@4318 | 3369 | |
twisti@4318 | 3370 | // get the address of the card |
twisti@4318 | 3371 | addq(card_addr, tmp2); |
twisti@4318 | 3372 | #else |
twisti@4318 | 3373 | const Register card_index = tmp; |
twisti@4318 | 3374 | |
twisti@4318 | 3375 | movl(card_index, store_addr); |
twisti@4318 | 3376 | shrl(card_index, CardTableModRefBS::card_shift); |
twisti@4318 | 3377 | |
twisti@4318 | 3378 | Address index(noreg, card_index, Address::times_1); |
twisti@4318 | 3379 | const Register card_addr = tmp; |
twisti@4318 | 3380 | lea(card_addr, as_Address(ArrayAddress(cardtable, index))); |
twisti@4318 | 3381 | #endif |
twisti@4318 | 3382 | cmpb(Address(card_addr, 0), 0); |
twisti@4318 | 3383 | jcc(Assembler::equal, done); |
twisti@4318 | 3384 | |
twisti@4318 | 3385 | // storing a region crossing, non-NULL oop, card is clean. |
twisti@4318 | 3386 | // dirty card and log. |
twisti@4318 | 3387 | |
twisti@4318 | 3388 | movb(Address(card_addr, 0), 0); |
twisti@4318 | 3389 | |
twisti@4318 | 3390 | cmpl(queue_index, 0); |
twisti@4318 | 3391 | jcc(Assembler::equal, runtime); |
twisti@4318 | 3392 | subl(queue_index, wordSize); |
twisti@4318 | 3393 | movptr(tmp2, buffer); |
twisti@4318 | 3394 | #ifdef _LP64 |
twisti@4318 | 3395 | movslq(rscratch1, queue_index); |
twisti@4318 | 3396 | addq(tmp2, rscratch1); |
twisti@4318 | 3397 | movq(Address(tmp2, 0), card_addr); |
twisti@4318 | 3398 | #else |
twisti@4318 | 3399 | addl(tmp2, queue_index); |
twisti@4318 | 3400 | movl(Address(tmp2, 0), card_index); |
twisti@4318 | 3401 | #endif |
twisti@4318 | 3402 | jmp(done); |
twisti@4318 | 3403 | |
twisti@4318 | 3404 | bind(runtime); |
twisti@4318 | 3405 | // save the live input values |
twisti@4318 | 3406 | push(store_addr); |
twisti@4318 | 3407 | push(new_val); |
twisti@4318 | 3408 | #ifdef _LP64 |
twisti@4318 | 3409 | call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread); |
twisti@4318 | 3410 | #else |
twisti@4318 | 3411 | push(thread); |
twisti@4318 | 3412 | call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); |
twisti@4318 | 3413 | pop(thread); |
twisti@4318 | 3414 | #endif |
twisti@4318 | 3415 | pop(new_val); |
twisti@4318 | 3416 | pop(store_addr); |
twisti@4318 | 3417 | |
twisti@4318 | 3418 | bind(done); |
twisti@4318 | 3419 | } |
twisti@4318 | 3420 | |
jprovino@4542 | 3421 | #endif // INCLUDE_ALL_GCS |
twisti@4318 | 3422 | ////////////////////////////////////////////////////////////////////////////////// |
twisti@4318 | 3423 | |
twisti@4318 | 3424 | |
twisti@4318 | 3425 | void MacroAssembler::store_check(Register obj) { |
twisti@4318 | 3426 | // Does a store check for the oop in register obj. The content of |
twisti@4318 | 3427 | // register obj is destroyed afterwards. |
twisti@4318 | 3428 | store_check_part_1(obj); |
twisti@4318 | 3429 | store_check_part_2(obj); |
twisti@4318 | 3430 | } |
twisti@4318 | 3431 | |
twisti@4318 | 3432 | void MacroAssembler::store_check(Register obj, Address dst) { |
twisti@4318 | 3433 | store_check(obj); |
twisti@4318 | 3434 | } |
twisti@4318 | 3435 | |
twisti@4318 | 3436 | |
twisti@4318 | 3437 | // split the store check operation so that other instructions can be scheduled inbetween |
twisti@4318 | 3438 | void MacroAssembler::store_check_part_1(Register obj) { |
twisti@4318 | 3439 | BarrierSet* bs = Universe::heap()->barrier_set(); |
twisti@4318 | 3440 | assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); |
twisti@4318 | 3441 | shrptr(obj, CardTableModRefBS::card_shift); |
twisti@4318 | 3442 | } |
twisti@4318 | 3443 | |
twisti@4318 | 3444 | void MacroAssembler::store_check_part_2(Register obj) { |
twisti@4318 | 3445 | BarrierSet* bs = Universe::heap()->barrier_set(); |
twisti@4318 | 3446 | assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind"); |
twisti@4318 | 3447 | CardTableModRefBS* ct = (CardTableModRefBS*)bs; |
twisti@4318 | 3448 | assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
twisti@4318 | 3449 | |
twisti@4318 | 3450 | // The calculation for byte_map_base is as follows: |
twisti@4318 | 3451 | // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); |
twisti@4318 | 3452 | // So this essentially converts an address to a displacement and |
twisti@4318 | 3453 | // it will never need to be relocated. On 64bit however the value may be too |
twisti@4318 | 3454 | // large for a 32bit displacement |
twisti@4318 | 3455 | |
twisti@4318 | 3456 | intptr_t disp = (intptr_t) ct->byte_map_base; |
twisti@4318 | 3457 | if (is_simm32(disp)) { |
twisti@4318 | 3458 | Address cardtable(noreg, obj, Address::times_1, disp); |
twisti@4318 | 3459 | movb(cardtable, 0); |
twisti@4318 | 3460 | } else { |
twisti@4318 | 3461 | // By doing it as an ExternalAddress disp could be converted to a rip-relative |
twisti@4318 | 3462 | // displacement and done in a single instruction given favorable mapping and |
twisti@4318 | 3463 | // a smarter version of as_Address. Worst case it is two instructions which |
twisti@4318 | 3464 | // is no worse off then loading disp into a register and doing as a simple |
twisti@4318 | 3465 | // Address() as above. |
twisti@4318 | 3466 | // We can't do as ExternalAddress as the only style since if disp == 0 we'll |
twisti@4318 | 3467 | // assert since NULL isn't acceptable in a reloci (see 6644928). In any case |
twisti@4318 | 3468 | // in some cases we'll get a single instruction version. |
twisti@4318 | 3469 | |
twisti@4318 | 3470 | ExternalAddress cardtable((address)disp); |
twisti@4318 | 3471 | Address index(noreg, obj, Address::times_1); |
twisti@4318 | 3472 | movb(as_Address(ArrayAddress(cardtable, index)), 0); |
twisti@4318 | 3473 | } |
twisti@4318 | 3474 | } |
twisti@4318 | 3475 | |
twisti@4318 | 3476 | void MacroAssembler::subptr(Register dst, int32_t imm32) { |
twisti@4318 | 3477 | LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32)); |
twisti@4318 | 3478 | } |
twisti@4318 | 3479 | |
twisti@4318 | 3480 | // Force generation of a 4 byte immediate value even if it fits into 8bit |
twisti@4318 | 3481 | void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) { |
twisti@4318 | 3482 | LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32)); |
twisti@4318 | 3483 | } |
twisti@4318 | 3484 | |
twisti@4318 | 3485 | void MacroAssembler::subptr(Register dst, Register src) { |
twisti@4318 | 3486 | LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src)); |
twisti@4318 | 3487 | } |
twisti@4318 | 3488 | |
twisti@4318 | 3489 | // C++ bool manipulation |
twisti@4318 | 3490 | void MacroAssembler::testbool(Register dst) { |
twisti@4318 | 3491 | if(sizeof(bool) == 1) |
twisti@4318 | 3492 | testb(dst, 0xff); |
twisti@4318 | 3493 | else if(sizeof(bool) == 2) { |
twisti@4318 | 3494 | // testw implementation needed for two byte bools |
twisti@4318 | 3495 | ShouldNotReachHere(); |
twisti@4318 | 3496 | } else if(sizeof(bool) == 4) |
twisti@4318 | 3497 | testl(dst, dst); |
twisti@4318 | 3498 | else |
twisti@4318 | 3499 | // unsupported |
twisti@4318 | 3500 | ShouldNotReachHere(); |
twisti@4318 | 3501 | } |
twisti@4318 | 3502 | |
twisti@4318 | 3503 | void MacroAssembler::testptr(Register dst, Register src) { |
twisti@4318 | 3504 | LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src)); |
twisti@4318 | 3505 | } |
twisti@4318 | 3506 | |
twisti@4318 | 3507 | // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes. |
twisti@4318 | 3508 | void MacroAssembler::tlab_allocate(Register obj, |
twisti@4318 | 3509 | Register var_size_in_bytes, |
twisti@4318 | 3510 | int con_size_in_bytes, |
twisti@4318 | 3511 | Register t1, |
twisti@4318 | 3512 | Register t2, |
twisti@4318 | 3513 | Label& slow_case) { |
twisti@4318 | 3514 | assert_different_registers(obj, t1, t2); |
twisti@4318 | 3515 | assert_different_registers(obj, var_size_in_bytes, t1); |
twisti@4318 | 3516 | Register end = t2; |
twisti@4318 | 3517 | Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread); |
twisti@4318 | 3518 | |
twisti@4318 | 3519 | verify_tlab(); |
twisti@4318 | 3520 | |
twisti@4318 | 3521 | NOT_LP64(get_thread(thread)); |
twisti@4318 | 3522 | |
twisti@4318 | 3523 | movptr(obj, Address(thread, JavaThread::tlab_top_offset())); |
twisti@4318 | 3524 | if (var_size_in_bytes == noreg) { |
twisti@4318 | 3525 | lea(end, Address(obj, con_size_in_bytes)); |
twisti@4318 | 3526 | } else { |
twisti@4318 | 3527 | lea(end, Address(obj, var_size_in_bytes, Address::times_1)); |
twisti@4318 | 3528 | } |
twisti@4318 | 3529 | cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); |
twisti@4318 | 3530 | jcc(Assembler::above, slow_case); |
twisti@4318 | 3531 | |
twisti@4318 | 3532 | // update the tlab top pointer |
twisti@4318 | 3533 | movptr(Address(thread, JavaThread::tlab_top_offset()), end); |
twisti@4318 | 3534 | |
twisti@4318 | 3535 | // recover var_size_in_bytes if necessary |
twisti@4318 | 3536 | if (var_size_in_bytes == end) { |
twisti@4318 | 3537 | subptr(var_size_in_bytes, obj); |
twisti@4318 | 3538 | } |
twisti@4318 | 3539 | verify_tlab(); |
twisti@4318 | 3540 | } |
twisti@4318 | 3541 | |
twisti@4318 | 3542 | // Preserves rbx, and rdx. |
twisti@4318 | 3543 | Register MacroAssembler::tlab_refill(Label& retry, |
twisti@4318 | 3544 | Label& try_eden, |
twisti@4318 | 3545 | Label& slow_case) { |
twisti@4318 | 3546 | Register top = rax; |
twisti@4318 | 3547 | Register t1 = rcx; |
twisti@4318 | 3548 | Register t2 = rsi; |
twisti@4318 | 3549 | Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread); |
twisti@4318 | 3550 | assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx); |
twisti@4318 | 3551 | Label do_refill, discard_tlab; |
twisti@4318 | 3552 | |
twisti@4318 | 3553 | if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) { |
twisti@4318 | 3554 | // No allocation in the shared eden. |
twisti@4318 | 3555 | jmp(slow_case); |
twisti@4318 | 3556 | } |
twisti@4318 | 3557 | |
twisti@4318 | 3558 | NOT_LP64(get_thread(thread_reg)); |
twisti@4318 | 3559 | |
twisti@4318 | 3560 | movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
twisti@4318 | 3561 | movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); |
twisti@4318 | 3562 | |
twisti@4318 | 3563 | // calculate amount of free space |
twisti@4318 | 3564 | subptr(t1, top); |
twisti@4318 | 3565 | shrptr(t1, LogHeapWordSize); |
twisti@4318 | 3566 | |
twisti@4318 | 3567 | // Retain tlab and allocate object in shared space if |
twisti@4318 | 3568 | // the amount free in the tlab is too large to discard. |
twisti@4318 | 3569 | cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset()))); |
twisti@4318 | 3570 | jcc(Assembler::lessEqual, discard_tlab); |
twisti@4318 | 3571 | |
twisti@4318 | 3572 | // Retain |
twisti@4318 | 3573 | // %%% yuck as movptr... |
twisti@4318 | 3574 | movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment()); |
twisti@4318 | 3575 | addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2); |
twisti@4318 | 3576 | if (TLABStats) { |
twisti@4318 | 3577 | // increment number of slow_allocations |
twisti@4318 | 3578 | addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1); |
twisti@4318 | 3579 | } |
twisti@4318 | 3580 | jmp(try_eden); |
twisti@4318 | 3581 | |
twisti@4318 | 3582 | bind(discard_tlab); |
twisti@4318 | 3583 | if (TLABStats) { |
twisti@4318 | 3584 | // increment number of refills |
twisti@4318 | 3585 | addl(Address(thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset())), 1); |
twisti@4318 | 3586 | // accumulate wastage -- t1 is amount free in tlab |
twisti@4318 | 3587 | addl(Address(thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1); |
twisti@4318 | 3588 | } |
twisti@4318 | 3589 | |
twisti@4318 | 3590 | // if tlab is currently allocated (top or end != null) then |
twisti@4318 | 3591 | // fill [top, end + alignment_reserve) with array object |
twisti@4318 | 3592 | testptr(top, top); |
twisti@4318 | 3593 | jcc(Assembler::zero, do_refill); |
twisti@4318 | 3594 | |
twisti@4318 | 3595 | // set up the mark word |
twisti@4318 | 3596 | movptr(Address(top, oopDesc::mark_offset_in_bytes()), (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2)); |
twisti@4318 | 3597 | // set the length to the remaining space |
twisti@4318 | 3598 | subptr(t1, typeArrayOopDesc::header_size(T_INT)); |
twisti@4318 | 3599 | addptr(t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve()); |
twisti@4318 | 3600 | shlptr(t1, log2_intptr(HeapWordSize/sizeof(jint))); |
twisti@4318 | 3601 | movl(Address(top, arrayOopDesc::length_offset_in_bytes()), t1); |
twisti@4318 | 3602 | // set klass to intArrayKlass |
twisti@4318 | 3603 | // dubious reloc why not an oop reloc? |
twisti@4318 | 3604 | movptr(t1, ExternalAddress((address)Universe::intArrayKlassObj_addr())); |
twisti@4318 | 3605 | // store klass last. concurrent gcs assumes klass length is valid if |
twisti@4318 | 3606 | // klass field is not null. |
twisti@4318 | 3607 | store_klass(top, t1); |
twisti@4318 | 3608 | |
twisti@4318 | 3609 | movptr(t1, top); |
twisti@4318 | 3610 | subptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
twisti@4318 | 3611 | incr_allocated_bytes(thread_reg, t1, 0); |
twisti@4318 | 3612 | |
twisti@4318 | 3613 | // refill the tlab with an eden allocation |
twisti@4318 | 3614 | bind(do_refill); |
twisti@4318 | 3615 | movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset()))); |
twisti@4318 | 3616 | shlptr(t1, LogHeapWordSize); |
twisti@4318 | 3617 | // allocate new tlab, address returned in top |
twisti@4318 | 3618 | eden_allocate(top, t1, 0, t2, slow_case); |
twisti@4318 | 3619 | |
twisti@4318 | 3620 | // Check that t1 was preserved in eden_allocate. |
twisti@4318 | 3621 | #ifdef ASSERT |
twisti@4318 | 3622 | if (UseTLAB) { |
twisti@4318 | 3623 | Label ok; |
twisti@4318 | 3624 | Register tsize = rsi; |
twisti@4318 | 3625 | assert_different_registers(tsize, thread_reg, t1); |
twisti@4318 | 3626 | push(tsize); |
twisti@4318 | 3627 | movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset()))); |
twisti@4318 | 3628 | shlptr(tsize, LogHeapWordSize); |
twisti@4318 | 3629 | cmpptr(t1, tsize); |
twisti@4318 | 3630 | jcc(Assembler::equal, ok); |
twisti@4318 | 3631 | STOP("assert(t1 != tlab size)"); |
twisti@4318 | 3632 | should_not_reach_here(); |
twisti@4318 | 3633 | |
twisti@4318 | 3634 | bind(ok); |
twisti@4318 | 3635 | pop(tsize); |
twisti@4318 | 3636 | } |
twisti@4318 | 3637 | #endif |
twisti@4318 | 3638 | movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top); |
twisti@4318 | 3639 | movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top); |
twisti@4318 | 3640 | addptr(top, t1); |
twisti@4318 | 3641 | subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes()); |
twisti@4318 | 3642 | movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top); |
twisti@4318 | 3643 | verify_tlab(); |
twisti@4318 | 3644 | jmp(retry); |
twisti@4318 | 3645 | |
twisti@4318 | 3646 | return thread_reg; // for use by caller |
twisti@4318 | 3647 | } |
twisti@4318 | 3648 | |
twisti@4318 | 3649 | void MacroAssembler::incr_allocated_bytes(Register thread, |
twisti@4318 | 3650 | Register var_size_in_bytes, |
twisti@4318 | 3651 | int con_size_in_bytes, |
twisti@4318 | 3652 | Register t1) { |
twisti@4318 | 3653 | if (!thread->is_valid()) { |
twisti@4318 | 3654 | #ifdef _LP64 |
twisti@4318 | 3655 | thread = r15_thread; |
twisti@4318 | 3656 | #else |
twisti@4318 | 3657 | assert(t1->is_valid(), "need temp reg"); |
twisti@4318 | 3658 | thread = t1; |
twisti@4318 | 3659 | get_thread(thread); |
twisti@4318 | 3660 | #endif |
twisti@4318 | 3661 | } |
twisti@4318 | 3662 | |
twisti@4318 | 3663 | #ifdef _LP64 |
twisti@4318 | 3664 | if (var_size_in_bytes->is_valid()) { |
twisti@4318 | 3665 | addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); |
twisti@4318 | 3666 | } else { |
twisti@4318 | 3667 | addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); |
twisti@4318 | 3668 | } |
twisti@4318 | 3669 | #else |
twisti@4318 | 3670 | if (var_size_in_bytes->is_valid()) { |
twisti@4318 | 3671 | addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); |
twisti@4318 | 3672 | } else { |
twisti@4318 | 3673 | addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); |
twisti@4318 | 3674 | } |
twisti@4318 | 3675 | adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); |
twisti@4318 | 3676 | #endif |
twisti@4318 | 3677 | } |
twisti@4318 | 3678 | |
twisti@4318 | 3679 | void MacroAssembler::fp_runtime_fallback(address runtime_entry, int nb_args, int num_fpu_regs_in_use) { |
twisti@4318 | 3680 | pusha(); |
twisti@4318 | 3681 | |
twisti@4318 | 3682 | // if we are coming from c1, xmm registers may be live |
twisti@4318 | 3683 | int off = 0; |
twisti@4318 | 3684 | if (UseSSE == 1) { |
twisti@4318 | 3685 | subptr(rsp, sizeof(jdouble)*8); |
twisti@4318 | 3686 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm0); |
twisti@4318 | 3687 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm1); |
twisti@4318 | 3688 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm2); |
twisti@4318 | 3689 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm3); |
twisti@4318 | 3690 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm4); |
twisti@4318 | 3691 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm5); |
twisti@4318 | 3692 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm6); |
twisti@4318 | 3693 | movflt(Address(rsp,off++*sizeof(jdouble)),xmm7); |
twisti@4318 | 3694 | } else if (UseSSE >= 2) { |
twisti@4318 | 3695 | #ifdef COMPILER2 |
twisti@4318 | 3696 | if (MaxVectorSize > 16) { |
twisti@4318 | 3697 | assert(UseAVX > 0, "256bit vectors are supported only with AVX"); |
twisti@4318 | 3698 | // Save upper half of YMM registes |
twisti@4318 | 3699 | subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8)); |
twisti@4318 | 3700 | vextractf128h(Address(rsp, 0),xmm0); |
twisti@4318 | 3701 | vextractf128h(Address(rsp, 16),xmm1); |
twisti@4318 | 3702 | vextractf128h(Address(rsp, 32),xmm2); |
twisti@4318 | 3703 | vextractf128h(Address(rsp, 48),xmm3); |
twisti@4318 | 3704 | vextractf128h(Address(rsp, 64),xmm4); |
twisti@4318 | 3705 | vextractf128h(Address(rsp, 80),xmm5); |
twisti@4318 | 3706 | vextractf128h(Address(rsp, 96),xmm6); |
twisti@4318 | 3707 | vextractf128h(Address(rsp,112),xmm7); |
twisti@4318 | 3708 | #ifdef _LP64 |
twisti@4318 | 3709 | vextractf128h(Address(rsp,128),xmm8); |
twisti@4318 | 3710 | vextractf128h(Address(rsp,144),xmm9); |
twisti@4318 | 3711 | vextractf128h(Address(rsp,160),xmm10); |
twisti@4318 | 3712 | vextractf128h(Address(rsp,176),xmm11); |
twisti@4318 | 3713 | vextractf128h(Address(rsp,192),xmm12); |
twisti@4318 | 3714 | vextractf128h(Address(rsp,208),xmm13); |
twisti@4318 | 3715 | vextractf128h(Address(rsp,224),xmm14); |
twisti@4318 | 3716 | vextractf128h(Address(rsp,240),xmm15); |
twisti@4318 | 3717 | #endif |
twisti@4318 | 3718 | } |
twisti@4318 | 3719 | #endif |
twisti@4318 | 3720 | // Save whole 128bit (16 bytes) XMM regiters |
twisti@4318 | 3721 | subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8)); |
twisti@4318 | 3722 | movdqu(Address(rsp,off++*16),xmm0); |
twisti@4318 | 3723 | movdqu(Address(rsp,off++*16),xmm1); |
twisti@4318 | 3724 | movdqu(Address(rsp,off++*16),xmm2); |
twisti@4318 | 3725 | movdqu(Address(rsp,off++*16),xmm3); |
twisti@4318 | 3726 | movdqu(Address(rsp,off++*16),xmm4); |
twisti@4318 | 3727 | movdqu(Address(rsp,off++*16),xmm5); |
twisti@4318 | 3728 | movdqu(Address(rsp,off++*16),xmm6); |
twisti@4318 | 3729 | movdqu(Address(rsp,off++*16),xmm7); |
twisti@4318 | 3730 | #ifdef _LP64 |
twisti@4318 | 3731 | movdqu(Address(rsp,off++*16),xmm8); |
twisti@4318 | 3732 | movdqu(Address(rsp,off++*16),xmm9); |
twisti@4318 | 3733 | movdqu(Address(rsp,off++*16),xmm10); |
twisti@4318 | 3734 | movdqu(Address(rsp,off++*16),xmm11); |
twisti@4318 | 3735 | movdqu(Address(rsp,off++*16),xmm12); |
twisti@4318 | 3736 | movdqu(Address(rsp,off++*16),xmm13); |
twisti@4318 | 3737 | movdqu(Address(rsp,off++*16),xmm14); |
twisti@4318 | 3738 | movdqu(Address(rsp,off++*16),xmm15); |
twisti@4318 | 3739 | #endif |
twisti@4318 | 3740 | } |
twisti@4318 | 3741 | |
twisti@4318 | 3742 | // Preserve registers across runtime call |
twisti@4318 | 3743 | int incoming_argument_and_return_value_offset = -1; |
twisti@4318 | 3744 | if (num_fpu_regs_in_use > 1) { |
twisti@4318 | 3745 | // Must preserve all other FPU regs (could alternatively convert |
twisti@4318 | 3746 | // SharedRuntime::dsin, dcos etc. into assembly routines known not to trash |
twisti@4318 | 3747 | // FPU state, but can not trust C compiler) |
twisti@4318 | 3748 | NEEDS_CLEANUP; |
twisti@4318 | 3749 | // NOTE that in this case we also push the incoming argument(s) to |
twisti@4318 | 3750 | // the stack and restore it later; we also use this stack slot to |
twisti@4318 | 3751 | // hold the return value from dsin, dcos etc. |
twisti@4318 | 3752 | for (int i = 0; i < num_fpu_regs_in_use; i++) { |
twisti@4318 | 3753 | subptr(rsp, sizeof(jdouble)); |
twisti@4318 | 3754 | fstp_d(Address(rsp, 0)); |
twisti@4318 | 3755 | } |
twisti@4318 | 3756 | incoming_argument_and_return_value_offset = sizeof(jdouble)*(num_fpu_regs_in_use-1); |
twisti@4318 | 3757 | for (int i = nb_args-1; i >= 0; i--) { |
twisti@4318 | 3758 | fld_d(Address(rsp, incoming_argument_and_return_value_offset-i*sizeof(jdouble))); |
twisti@4318 | 3759 | } |
twisti@4318 | 3760 | } |
twisti@4318 | 3761 | |
twisti@4318 | 3762 | subptr(rsp, nb_args*sizeof(jdouble)); |
twisti@4318 | 3763 | for (int i = 0; i < nb_args; i++) { |
twisti@4318 | 3764 | fstp_d(Address(rsp, i*sizeof(jdouble))); |
twisti@4318 | 3765 | } |
twisti@4318 | 3766 | |
twisti@4318 | 3767 | #ifdef _LP64 |
twisti@4318 | 3768 | if (nb_args > 0) { |
twisti@4318 | 3769 | movdbl(xmm0, Address(rsp, 0)); |
twisti@4318 | 3770 | } |
twisti@4318 | 3771 | if (nb_args > 1) { |
twisti@4318 | 3772 | movdbl(xmm1, Address(rsp, sizeof(jdouble))); |
twisti@4318 | 3773 | } |
twisti@4318 | 3774 | assert(nb_args <= 2, "unsupported number of args"); |
twisti@4318 | 3775 | #endif // _LP64 |
twisti@4318 | 3776 | |
twisti@4318 | 3777 | // NOTE: we must not use call_VM_leaf here because that requires a |
twisti@4318 | 3778 | // complete interpreter frame in debug mode -- same bug as 4387334 |
twisti@4318 | 3779 | // MacroAssembler::call_VM_leaf_base is perfectly safe and will |
twisti@4318 | 3780 | // do proper 64bit abi |
twisti@4318 | 3781 | |
twisti@4318 | 3782 | NEEDS_CLEANUP; |
twisti@4318 | 3783 | // Need to add stack banging before this runtime call if it needs to |
twisti@4318 | 3784 | // be taken; however, there is no generic stack banging routine at |
twisti@4318 | 3785 | // the MacroAssembler level |
twisti@4318 | 3786 | |
twisti@4318 | 3787 | MacroAssembler::call_VM_leaf_base(runtime_entry, 0); |
twisti@4318 | 3788 | |
twisti@4318 | 3789 | #ifdef _LP64 |
twisti@4318 | 3790 | movsd(Address(rsp, 0), xmm0); |
twisti@4318 | 3791 | fld_d(Address(rsp, 0)); |
twisti@4318 | 3792 | #endif // _LP64 |
twisti@4318 | 3793 | addptr(rsp, sizeof(jdouble) * nb_args); |
twisti@4318 | 3794 | if (num_fpu_regs_in_use > 1) { |
twisti@4318 | 3795 | // Must save return value to stack and then restore entire FPU |
twisti@4318 | 3796 | // stack except incoming arguments |
twisti@4318 | 3797 | fstp_d(Address(rsp, incoming_argument_and_return_value_offset)); |
twisti@4318 | 3798 | for (int i = 0; i < num_fpu_regs_in_use - nb_args; i++) { |
twisti@4318 | 3799 | fld_d(Address(rsp, 0)); |
twisti@4318 | 3800 | addptr(rsp, sizeof(jdouble)); |
twisti@4318 | 3801 | } |
twisti@4318 | 3802 | fld_d(Address(rsp, (nb_args-1)*sizeof(jdouble))); |
twisti@4318 | 3803 | addptr(rsp, sizeof(jdouble) * nb_args); |
twisti@4318 | 3804 | } |
twisti@4318 | 3805 | |
twisti@4318 | 3806 | off = 0; |
twisti@4318 | 3807 | if (UseSSE == 1) { |
twisti@4318 | 3808 | movflt(xmm0, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3809 | movflt(xmm1, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3810 | movflt(xmm2, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3811 | movflt(xmm3, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3812 | movflt(xmm4, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3813 | movflt(xmm5, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3814 | movflt(xmm6, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3815 | movflt(xmm7, Address(rsp,off++*sizeof(jdouble))); |
twisti@4318 | 3816 | addptr(rsp, sizeof(jdouble)*8); |
twisti@4318 | 3817 | } else if (UseSSE >= 2) { |
twisti@4318 | 3818 | // Restore whole 128bit (16 bytes) XMM regiters |
twisti@4318 | 3819 | movdqu(xmm0, Address(rsp,off++*16)); |
twisti@4318 | 3820 | movdqu(xmm1, Address(rsp,off++*16)); |
twisti@4318 | 3821 | movdqu(xmm2, Address(rsp,off++*16)); |
twisti@4318 | 3822 | movdqu(xmm3, Address(rsp,off++*16)); |
twisti@4318 | 3823 | movdqu(xmm4, Address(rsp,off++*16)); |
twisti@4318 | 3824 | movdqu(xmm5, Address(rsp,off++*16)); |
twisti@4318 | 3825 | movdqu(xmm6, Address(rsp,off++*16)); |
twisti@4318 | 3826 | movdqu(xmm7, Address(rsp,off++*16)); |
twisti@4318 | 3827 | #ifdef _LP64 |
twisti@4318 | 3828 | movdqu(xmm8, Address(rsp,off++*16)); |
twisti@4318 | 3829 | movdqu(xmm9, Address(rsp,off++*16)); |
twisti@4318 | 3830 | movdqu(xmm10, Address(rsp,off++*16)); |
twisti@4318 | 3831 | movdqu(xmm11, Address(rsp,off++*16)); |
twisti@4318 | 3832 | movdqu(xmm12, Address(rsp,off++*16)); |
twisti@4318 | 3833 | movdqu(xmm13, Address(rsp,off++*16)); |
twisti@4318 | 3834 | movdqu(xmm14, Address(rsp,off++*16)); |
twisti@4318 | 3835 | movdqu(xmm15, Address(rsp,off++*16)); |
twisti@4318 | 3836 | #endif |
twisti@4318 | 3837 | addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8)); |
twisti@4318 | 3838 | #ifdef COMPILER2 |
twisti@4318 | 3839 | if (MaxVectorSize > 16) { |
twisti@4318 | 3840 | // Restore upper half of YMM registes. |
twisti@4318 | 3841 | vinsertf128h(xmm0, Address(rsp, 0)); |
twisti@4318 | 3842 | vinsertf128h(xmm1, Address(rsp, 16)); |
twisti@4318 | 3843 | vinsertf128h(xmm2, Address(rsp, 32)); |
twisti@4318 | 3844 | vinsertf128h(xmm3, Address(rsp, 48)); |
twisti@4318 | 3845 | vinsertf128h(xmm4, Address(rsp, 64)); |
twisti@4318 | 3846 | vinsertf128h(xmm5, Address(rsp, 80)); |
twisti@4318 | 3847 | vinsertf128h(xmm6, Address(rsp, 96)); |
twisti@4318 | 3848 | vinsertf128h(xmm7, Address(rsp,112)); |
twisti@4318 | 3849 | #ifdef _LP64 |
twisti@4318 | 3850 | vinsertf128h(xmm8, Address(rsp,128)); |
twisti@4318 | 3851 | vinsertf128h(xmm9, Address(rsp,144)); |
twisti@4318 | 3852 | vinsertf128h(xmm10, Address(rsp,160)); |
twisti@4318 | 3853 | vinsertf128h(xmm11, Address(rsp,176)); |
twisti@4318 | 3854 | vinsertf128h(xmm12, Address(rsp,192)); |
twisti@4318 | 3855 | vinsertf128h(xmm13, Address(rsp,208)); |
twisti@4318 | 3856 | vinsertf128h(xmm14, Address(rsp,224)); |
twisti@4318 | 3857 | vinsertf128h(xmm15, Address(rsp,240)); |
twisti@4318 | 3858 | #endif |
twisti@4318 | 3859 | addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8)); |
twisti@4318 | 3860 | } |
twisti@4318 | 3861 | #endif |
twisti@4318 | 3862 | } |
twisti@4318 | 3863 | popa(); |
twisti@4318 | 3864 | } |
twisti@4318 | 3865 | |
twisti@4318 | 3866 | static const double pi_4 = 0.7853981633974483; |
twisti@4318 | 3867 | |
twisti@4318 | 3868 | void MacroAssembler::trigfunc(char trig, int num_fpu_regs_in_use) { |
twisti@4318 | 3869 | // A hand-coded argument reduction for values in fabs(pi/4, pi/2) |
twisti@4318 | 3870 | // was attempted in this code; unfortunately it appears that the |
twisti@4318 | 3871 | // switch to 80-bit precision and back causes this to be |
twisti@4318 | 3872 | // unprofitable compared with simply performing a runtime call if |
twisti@4318 | 3873 | // the argument is out of the (-pi/4, pi/4) range. |
twisti@4318 | 3874 | |
twisti@4318 | 3875 | Register tmp = noreg; |
twisti@4318 | 3876 | if (!VM_Version::supports_cmov()) { |
twisti@4318 | 3877 | // fcmp needs a temporary so preserve rbx, |
twisti@4318 | 3878 | tmp = rbx; |
twisti@4318 | 3879 | push(tmp); |
twisti@4318 | 3880 | } |
twisti@4318 | 3881 | |
twisti@4318 | 3882 | Label slow_case, done; |
twisti@4318 | 3883 | |
twisti@4318 | 3884 | ExternalAddress pi4_adr = (address)&pi_4; |
twisti@4318 | 3885 | if (reachable(pi4_adr)) { |
twisti@4318 | 3886 | // x ?<= pi/4 |
twisti@4318 | 3887 | fld_d(pi4_adr); |
twisti@4318 | 3888 | fld_s(1); // Stack: X PI/4 X |
twisti@4318 | 3889 | fabs(); // Stack: |X| PI/4 X |
twisti@4318 | 3890 | fcmp(tmp); |
twisti@4318 | 3891 | jcc(Assembler::above, slow_case); |
twisti@4318 | 3892 | |
twisti@4318 | 3893 | // fastest case: -pi/4 <= x <= pi/4 |
twisti@4318 | 3894 | switch(trig) { |
twisti@4318 | 3895 | case 's': |
twisti@4318 | 3896 | fsin(); |
twisti@4318 | 3897 | break; |
twisti@4318 | 3898 | case 'c': |
twisti@4318 | 3899 | fcos(); |
twisti@4318 | 3900 | break; |
twisti@4318 | 3901 | case 't': |
twisti@4318 | 3902 | ftan(); |
twisti@4318 | 3903 | break; |
twisti@4318 | 3904 | default: |
twisti@4318 | 3905 | assert(false, "bad intrinsic"); |
twisti@4318 | 3906 | break; |
twisti@4318 | 3907 | } |
twisti@4318 | 3908 | jmp(done); |
twisti@4318 | 3909 | } |
twisti@4318 | 3910 | |
twisti@4318 | 3911 | // slow case: runtime call |
twisti@4318 | 3912 | bind(slow_case); |
twisti@4318 | 3913 | |
twisti@4318 | 3914 | switch(trig) { |
twisti@4318 | 3915 | case 's': |
twisti@4318 | 3916 | { |
twisti@4318 | 3917 | fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), 1, num_fpu_regs_in_use); |
twisti@4318 | 3918 | } |
twisti@4318 | 3919 | break; |
twisti@4318 | 3920 | case 'c': |
twisti@4318 | 3921 | { |
twisti@4318 | 3922 | fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), 1, num_fpu_regs_in_use); |
twisti@4318 | 3923 | } |
twisti@4318 | 3924 | break; |
twisti@4318 | 3925 | case 't': |
twisti@4318 | 3926 | { |
twisti@4318 | 3927 | fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), 1, num_fpu_regs_in_use); |
twisti@4318 | 3928 | } |
twisti@4318 | 3929 | break; |
twisti@4318 | 3930 | default: |
twisti@4318 | 3931 | assert(false, "bad intrinsic"); |
twisti@4318 | 3932 | break; |
twisti@4318 | 3933 | } |
twisti@4318 | 3934 | |
twisti@4318 | 3935 | // Come here with result in F-TOS |
twisti@4318 | 3936 | bind(done); |
twisti@4318 | 3937 | |
twisti@4318 | 3938 | if (tmp != noreg) { |
twisti@4318 | 3939 | pop(tmp); |
twisti@4318 | 3940 | } |
twisti@4318 | 3941 | } |
twisti@4318 | 3942 | |
twisti@4318 | 3943 | |
twisti@4318 | 3944 | // Look up the method for a megamorphic invokeinterface call. |
twisti@4318 | 3945 | // The target method is determined by <intf_klass, itable_index>. |
twisti@4318 | 3946 | // The receiver klass is in recv_klass. |
twisti@4318 | 3947 | // On success, the result will be in method_result, and execution falls through. |
twisti@4318 | 3948 | // On failure, execution transfers to the given label. |
twisti@4318 | 3949 | void MacroAssembler::lookup_interface_method(Register recv_klass, |
twisti@4318 | 3950 | Register intf_klass, |
twisti@4318 | 3951 | RegisterOrConstant itable_index, |
twisti@4318 | 3952 | Register method_result, |
twisti@4318 | 3953 | Register scan_temp, |
twisti@4318 | 3954 | Label& L_no_such_interface) { |
twisti@4318 | 3955 | assert_different_registers(recv_klass, intf_klass, method_result, scan_temp); |
twisti@4318 | 3956 | assert(itable_index.is_constant() || itable_index.as_register() == method_result, |
twisti@4318 | 3957 | "caller must use same register for non-constant itable index as for method"); |
twisti@4318 | 3958 | |
twisti@4318 | 3959 | // Compute start of first itableOffsetEntry (which is at the end of the vtable) |
twisti@4318 | 3960 | int vtable_base = InstanceKlass::vtable_start_offset() * wordSize; |
twisti@4318 | 3961 | int itentry_off = itableMethodEntry::method_offset_in_bytes(); |
twisti@4318 | 3962 | int scan_step = itableOffsetEntry::size() * wordSize; |
twisti@4318 | 3963 | int vte_size = vtableEntry::size() * wordSize; |
twisti@4318 | 3964 | Address::ScaleFactor times_vte_scale = Address::times_ptr; |
twisti@4318 | 3965 | assert(vte_size == wordSize, "else adjust times_vte_scale"); |
twisti@4318 | 3966 | |
twisti@4318 | 3967 | movl(scan_temp, Address(recv_klass, InstanceKlass::vtable_length_offset() * wordSize)); |
twisti@4318 | 3968 | |
twisti@4318 | 3969 | // %%% Could store the aligned, prescaled offset in the klassoop. |
twisti@4318 | 3970 | lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base)); |
twisti@4318 | 3971 | if (HeapWordsPerLong > 1) { |
twisti@4318 | 3972 | // Round up to align_object_offset boundary |
twisti@4318 | 3973 | // see code for InstanceKlass::start_of_itable! |
twisti@4318 | 3974 | round_to(scan_temp, BytesPerLong); |
twisti@4318 | 3975 | } |
twisti@4318 | 3976 | |
twisti@4318 | 3977 | // Adjust recv_klass by scaled itable_index, so we can free itable_index. |
twisti@4318 | 3978 | assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); |
twisti@4318 | 3979 | lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off)); |
twisti@4318 | 3980 | |
twisti@4318 | 3981 | // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) { |
twisti@4318 | 3982 | // if (scan->interface() == intf) { |
twisti@4318 | 3983 | // result = (klass + scan->offset() + itable_index); |
twisti@4318 | 3984 | // } |
twisti@4318 | 3985 | // } |
twisti@4318 | 3986 | Label search, found_method; |
twisti@4318 | 3987 | |
twisti@4318 | 3988 | for (int peel = 1; peel >= 0; peel--) { |
twisti@4318 | 3989 | movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes())); |
twisti@4318 | 3990 | cmpptr(intf_klass, method_result); |
twisti@4318 | 3991 | |
twisti@4318 | 3992 | if (peel) { |
twisti@4318 | 3993 | jccb(Assembler::equal, found_method); |
twisti@4318 | 3994 | } else { |
twisti@4318 | 3995 | jccb(Assembler::notEqual, search); |
twisti@4318 | 3996 | // (invert the test to fall through to found_method...) |
twisti@4318 | 3997 | } |
twisti@4318 | 3998 | |
twisti@4318 | 3999 | if (!peel) break; |
twisti@4318 | 4000 | |
twisti@4318 | 4001 | bind(search); |
twisti@4318 | 4002 | |
twisti@4318 | 4003 | // Check that the previous entry is non-null. A null entry means that |
twisti@4318 | 4004 | // the receiver class doesn't implement the interface, and wasn't the |
twisti@4318 | 4005 | // same as when the caller was compiled. |
twisti@4318 | 4006 | testptr(method_result, method_result); |
twisti@4318 | 4007 | jcc(Assembler::zero, L_no_such_interface); |
twisti@4318 | 4008 | addptr(scan_temp, scan_step); |
twisti@4318 | 4009 | } |
twisti@4318 | 4010 | |
twisti@4318 | 4011 | bind(found_method); |
twisti@4318 | 4012 | |
twisti@4318 | 4013 | // Got a hit. |
twisti@4318 | 4014 | movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes())); |
twisti@4318 | 4015 | movptr(method_result, Address(recv_klass, scan_temp, Address::times_1)); |
twisti@4318 | 4016 | } |
twisti@4318 | 4017 | |
twisti@4318 | 4018 | |
twisti@4318 | 4019 | // virtual method calling |
twisti@4318 | 4020 | void MacroAssembler::lookup_virtual_method(Register recv_klass, |
twisti@4318 | 4021 | RegisterOrConstant vtable_index, |
twisti@4318 | 4022 | Register method_result) { |
twisti@4318 | 4023 | const int base = InstanceKlass::vtable_start_offset() * wordSize; |
twisti@4318 | 4024 | assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below"); |
twisti@4318 | 4025 | Address vtable_entry_addr(recv_klass, |
twisti@4318 | 4026 | vtable_index, Address::times_ptr, |
twisti@4318 | 4027 | base + vtableEntry::method_offset_in_bytes()); |
twisti@4318 | 4028 | movptr(method_result, vtable_entry_addr); |
twisti@4318 | 4029 | } |
twisti@4318 | 4030 | |
twisti@4318 | 4031 | |
twisti@4318 | 4032 | void MacroAssembler::check_klass_subtype(Register sub_klass, |
twisti@4318 | 4033 | Register super_klass, |
twisti@4318 | 4034 | Register temp_reg, |
twisti@4318 | 4035 | Label& L_success) { |
twisti@4318 | 4036 | Label L_failure; |
twisti@4318 | 4037 | check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg, &L_success, &L_failure, NULL); |
twisti@4318 | 4038 | check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL); |
twisti@4318 | 4039 | bind(L_failure); |
twisti@4318 | 4040 | } |
twisti@4318 | 4041 | |
twisti@4318 | 4042 | |
twisti@4318 | 4043 | void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, |
twisti@4318 | 4044 | Register super_klass, |
twisti@4318 | 4045 | Register temp_reg, |
twisti@4318 | 4046 | Label* L_success, |
twisti@4318 | 4047 | Label* L_failure, |
twisti@4318 | 4048 | Label* L_slow_path, |
twisti@4318 | 4049 | RegisterOrConstant super_check_offset) { |
twisti@4318 | 4050 | assert_different_registers(sub_klass, super_klass, temp_reg); |
twisti@4318 | 4051 | bool must_load_sco = (super_check_offset.constant_or_zero() == -1); |
twisti@4318 | 4052 | if (super_check_offset.is_register()) { |
twisti@4318 | 4053 | assert_different_registers(sub_klass, super_klass, |
twisti@4318 | 4054 | super_check_offset.as_register()); |
twisti@4318 | 4055 | } else if (must_load_sco) { |
twisti@4318 | 4056 | assert(temp_reg != noreg, "supply either a temp or a register offset"); |
twisti@4318 | 4057 | } |
twisti@4318 | 4058 | |
twisti@4318 | 4059 | Label L_fallthrough; |
twisti@4318 | 4060 | int label_nulls = 0; |
twisti@4318 | 4061 | if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
twisti@4318 | 4062 | if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
twisti@4318 | 4063 | if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; } |
twisti@4318 | 4064 | assert(label_nulls <= 1, "at most one NULL in the batch"); |
twisti@4318 | 4065 | |
twisti@4318 | 4066 | int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
twisti@4318 | 4067 | int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
twisti@4318 | 4068 | Address super_check_offset_addr(super_klass, sco_offset); |
twisti@4318 | 4069 | |
twisti@4318 | 4070 | // Hacked jcc, which "knows" that L_fallthrough, at least, is in |
twisti@4318 | 4071 | // range of a jccb. If this routine grows larger, reconsider at |
twisti@4318 | 4072 | // least some of these. |
twisti@4318 | 4073 | #define local_jcc(assembler_cond, label) \ |
twisti@4318 | 4074 | if (&(label) == &L_fallthrough) jccb(assembler_cond, label); \ |
twisti@4318 | 4075 | else jcc( assembler_cond, label) /*omit semi*/ |
twisti@4318 | 4076 | |
twisti@4318 | 4077 | // Hacked jmp, which may only be used just before L_fallthrough. |
twisti@4318 | 4078 | #define final_jmp(label) \ |
twisti@4318 | 4079 | if (&(label) == &L_fallthrough) { /*do nothing*/ } \ |
twisti@4318 | 4080 | else jmp(label) /*omit semi*/ |
twisti@4318 | 4081 | |
twisti@4318 | 4082 | // If the pointers are equal, we are done (e.g., String[] elements). |
twisti@4318 | 4083 | // This self-check enables sharing of secondary supertype arrays among |
twisti@4318 | 4084 | // non-primary types such as array-of-interface. Otherwise, each such |
twisti@4318 | 4085 | // type would need its own customized SSA. |
twisti@4318 | 4086 | // We move this check to the front of the fast path because many |
twisti@4318 | 4087 | // type checks are in fact trivially successful in this manner, |
twisti@4318 | 4088 | // so we get a nicely predicted branch right at the start of the check. |
twisti@4318 | 4089 | cmpptr(sub_klass, super_klass); |
twisti@4318 | 4090 | local_jcc(Assembler::equal, *L_success); |
twisti@4318 | 4091 | |
twisti@4318 | 4092 | // Check the supertype display: |
twisti@4318 | 4093 | if (must_load_sco) { |
twisti@4318 | 4094 | // Positive movl does right thing on LP64. |
twisti@4318 | 4095 | movl(temp_reg, super_check_offset_addr); |
twisti@4318 | 4096 | super_check_offset = RegisterOrConstant(temp_reg); |
twisti@4318 | 4097 | } |
twisti@4318 | 4098 | Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); |
twisti@4318 | 4099 | cmpptr(super_klass, super_check_addr); // load displayed supertype |
twisti@4318 | 4100 | |
twisti@4318 | 4101 | // This check has worked decisively for primary supers. |
twisti@4318 | 4102 | // Secondary supers are sought in the super_cache ('super_cache_addr'). |
twisti@4318 | 4103 | // (Secondary supers are interfaces and very deeply nested subtypes.) |
twisti@4318 | 4104 | // This works in the same check above because of a tricky aliasing |
twisti@4318 | 4105 | // between the super_cache and the primary super display elements. |
twisti@4318 | 4106 | // (The 'super_check_addr' can address either, as the case requires.) |
twisti@4318 | 4107 | // Note that the cache is updated below if it does not help us find |
twisti@4318 | 4108 | // what we need immediately. |
twisti@4318 | 4109 | // So if it was a primary super, we can just fail immediately. |
twisti@4318 | 4110 | // Otherwise, it's the slow path for us (no success at this point). |
twisti@4318 | 4111 | |
twisti@4318 | 4112 | if (super_check_offset.is_register()) { |
twisti@4318 | 4113 | local_jcc(Assembler::equal, *L_success); |
twisti@4318 | 4114 | cmpl(super_check_offset.as_register(), sc_offset); |
twisti@4318 | 4115 | if (L_failure == &L_fallthrough) { |
twisti@4318 | 4116 | local_jcc(Assembler::equal, *L_slow_path); |
twisti@4318 | 4117 | } else { |
twisti@4318 | 4118 | local_jcc(Assembler::notEqual, *L_failure); |
twisti@4318 | 4119 | final_jmp(*L_slow_path); |
twisti@4318 | 4120 | } |
twisti@4318 | 4121 | } else if (super_check_offset.as_constant() == sc_offset) { |
twisti@4318 | 4122 | // Need a slow path; fast failure is impossible. |
twisti@4318 | 4123 | if (L_slow_path == &L_fallthrough) { |
twisti@4318 | 4124 | local_jcc(Assembler::equal, *L_success); |
twisti@4318 | 4125 | } else { |
twisti@4318 | 4126 | local_jcc(Assembler::notEqual, *L_slow_path); |
twisti@4318 | 4127 | final_jmp(*L_success); |
twisti@4318 | 4128 | } |
twisti@4318 | 4129 | } else { |
twisti@4318 | 4130 | // No slow path; it's a fast decision. |
twisti@4318 | 4131 | if (L_failure == &L_fallthrough) { |
twisti@4318 | 4132 | local_jcc(Assembler::equal, *L_success); |
twisti@4318 | 4133 | } else { |
twisti@4318 | 4134 | local_jcc(Assembler::notEqual, *L_failure); |
twisti@4318 | 4135 | final_jmp(*L_success); |
twisti@4318 | 4136 | } |
twisti@4318 | 4137 | } |
twisti@4318 | 4138 | |
twisti@4318 | 4139 | bind(L_fallthrough); |
twisti@4318 | 4140 | |
twisti@4318 | 4141 | #undef local_jcc |
twisti@4318 | 4142 | #undef final_jmp |
twisti@4318 | 4143 | } |
twisti@4318 | 4144 | |
twisti@4318 | 4145 | |
twisti@4318 | 4146 | void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, |
twisti@4318 | 4147 | Register super_klass, |
twisti@4318 | 4148 | Register temp_reg, |
twisti@4318 | 4149 | Register temp2_reg, |
twisti@4318 | 4150 | Label* L_success, |
twisti@4318 | 4151 | Label* L_failure, |
twisti@4318 | 4152 | bool set_cond_codes) { |
twisti@4318 | 4153 | assert_different_registers(sub_klass, super_klass, temp_reg); |
twisti@4318 | 4154 | if (temp2_reg != noreg) |
twisti@4318 | 4155 | assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg); |
twisti@4318 | 4156 | #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg) |
twisti@4318 | 4157 | |
twisti@4318 | 4158 | Label L_fallthrough; |
twisti@4318 | 4159 | int label_nulls = 0; |
twisti@4318 | 4160 | if (L_success == NULL) { L_success = &L_fallthrough; label_nulls++; } |
twisti@4318 | 4161 | if (L_failure == NULL) { L_failure = &L_fallthrough; label_nulls++; } |
twisti@4318 | 4162 | assert(label_nulls <= 1, "at most one NULL in the batch"); |
twisti@4318 | 4163 | |
twisti@4318 | 4164 | // a couple of useful fields in sub_klass: |
twisti@4318 | 4165 | int ss_offset = in_bytes(Klass::secondary_supers_offset()); |
twisti@4318 | 4166 | int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); |
twisti@4318 | 4167 | Address secondary_supers_addr(sub_klass, ss_offset); |
twisti@4318 | 4168 | Address super_cache_addr( sub_klass, sc_offset); |
twisti@4318 | 4169 | |
twisti@4318 | 4170 | // Do a linear scan of the secondary super-klass chain. |
twisti@4318 | 4171 | // This code is rarely used, so simplicity is a virtue here. |
twisti@4318 | 4172 | // The repne_scan instruction uses fixed registers, which we must spill. |
twisti@4318 | 4173 | // Don't worry too much about pre-existing connections with the input regs. |
twisti@4318 | 4174 | |
twisti@4318 | 4175 | assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super) |
twisti@4318 | 4176 | assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter) |
twisti@4318 | 4177 | |
twisti@4318 | 4178 | // Get super_klass value into rax (even if it was in rdi or rcx). |
twisti@4318 | 4179 | bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false; |
twisti@4318 | 4180 | if (super_klass != rax || UseCompressedOops) { |
twisti@4318 | 4181 | if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; } |
twisti@4318 | 4182 | mov(rax, super_klass); |
twisti@4318 | 4183 | } |
twisti@4318 | 4184 | if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; } |
twisti@4318 | 4185 | if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; } |
twisti@4318 | 4186 | |
twisti@4318 | 4187 | #ifndef PRODUCT |
twisti@4318 | 4188 | int* pst_counter = &SharedRuntime::_partial_subtype_ctr; |
twisti@4318 | 4189 | ExternalAddress pst_counter_addr((address) pst_counter); |
twisti@4318 | 4190 | NOT_LP64( incrementl(pst_counter_addr) ); |
twisti@4318 | 4191 | LP64_ONLY( lea(rcx, pst_counter_addr) ); |
twisti@4318 | 4192 | LP64_ONLY( incrementl(Address(rcx, 0)) ); |
twisti@4318 | 4193 | #endif //PRODUCT |
twisti@4318 | 4194 | |
twisti@4318 | 4195 | // We will consult the secondary-super array. |
twisti@4318 | 4196 | movptr(rdi, secondary_supers_addr); |
twisti@4318 | 4197 | // Load the array length. (Positive movl does right thing on LP64.) |
twisti@4318 | 4198 | movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes())); |
twisti@4318 | 4199 | // Skip to start of data. |
twisti@4318 | 4200 | addptr(rdi, Array<Klass*>::base_offset_in_bytes()); |
twisti@4318 | 4201 | |
twisti@4318 | 4202 | // Scan RCX words at [RDI] for an occurrence of RAX. |
twisti@4318 | 4203 | // Set NZ/Z based on last compare. |
twisti@4318 | 4204 | // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does |
twisti@4318 | 4205 | // not change flags (only scas instruction which is repeated sets flags). |
twisti@4318 | 4206 | // Set Z = 0 (not equal) before 'repne' to indicate that class was not found. |
twisti@4318 | 4207 | |
twisti@4318 | 4208 | testptr(rax,rax); // Set Z = 0 |
twisti@4318 | 4209 | repne_scan(); |
twisti@4318 | 4210 | |
twisti@4318 | 4211 | // Unspill the temp. registers: |
twisti@4318 | 4212 | if (pushed_rdi) pop(rdi); |
twisti@4318 | 4213 | if (pushed_rcx) pop(rcx); |
twisti@4318 | 4214 | if (pushed_rax) pop(rax); |
twisti@4318 | 4215 | |
twisti@4318 | 4216 | if (set_cond_codes) { |
twisti@4318 | 4217 | // Special hack for the AD files: rdi is guaranteed non-zero. |
twisti@4318 | 4218 | assert(!pushed_rdi, "rdi must be left non-NULL"); |
twisti@4318 | 4219 | // Also, the condition codes are properly set Z/NZ on succeed/failure. |
twisti@4318 | 4220 | } |
twisti@4318 | 4221 | |
twisti@4318 | 4222 | if (L_failure == &L_fallthrough) |
twisti@4318 | 4223 | jccb(Assembler::notEqual, *L_failure); |
twisti@4318 | 4224 | else jcc(Assembler::notEqual, *L_failure); |
twisti@4318 | 4225 | |
twisti@4318 | 4226 | // Success. Cache the super we found and proceed in triumph. |
twisti@4318 | 4227 | movptr(super_cache_addr, super_klass); |
twisti@4318 | 4228 | |
twisti@4318 | 4229 | if (L_success != &L_fallthrough) { |
twisti@4318 | 4230 | jmp(*L_success); |
twisti@4318 | 4231 | } |
twisti@4318 | 4232 | |
twisti@4318 | 4233 | #undef IS_A_TEMP |
twisti@4318 | 4234 | |
twisti@4318 | 4235 | bind(L_fallthrough); |
twisti@4318 | 4236 | } |
twisti@4318 | 4237 | |
twisti@4318 | 4238 | |
twisti@4318 | 4239 | void MacroAssembler::cmov32(Condition cc, Register dst, Address src) { |
twisti@4318 | 4240 | if (VM_Version::supports_cmov()) { |
twisti@4318 | 4241 | cmovl(cc, dst, src); |
twisti@4318 | 4242 | } else { |
twisti@4318 | 4243 | Label L; |
twisti@4318 | 4244 | jccb(negate_condition(cc), L); |
twisti@4318 | 4245 | movl(dst, src); |
twisti@4318 | 4246 | bind(L); |
twisti@4318 | 4247 | } |
twisti@4318 | 4248 | } |
twisti@4318 | 4249 | |
twisti@4318 | 4250 | void MacroAssembler::cmov32(Condition cc, Register dst, Register src) { |
twisti@4318 | 4251 | if (VM_Version::supports_cmov()) { |
twisti@4318 | 4252 | cmovl(cc, dst, src); |
twisti@4318 | 4253 | } else { |
twisti@4318 | 4254 | Label L; |
twisti@4318 | 4255 | jccb(negate_condition(cc), L); |
twisti@4318 | 4256 | movl(dst, src); |
twisti@4318 | 4257 | bind(L); |
twisti@4318 | 4258 | } |
twisti@4318 | 4259 | } |
twisti@4318 | 4260 | |
twisti@4318 | 4261 | void MacroAssembler::verify_oop(Register reg, const char* s) { |
twisti@4318 | 4262 | if (!VerifyOops) return; |
twisti@4318 | 4263 | |
twisti@4318 | 4264 | // Pass register number to verify_oop_subroutine |
twisti@4318 | 4265 | char* b = new char[strlen(s) + 50]; |
twisti@4318 | 4266 | sprintf(b, "verify_oop: %s: %s", reg->name(), s); |
twisti@4318 | 4267 | BLOCK_COMMENT("verify_oop {"); |
twisti@4318 | 4268 | #ifdef _LP64 |
twisti@4318 | 4269 | push(rscratch1); // save r10, trashed by movptr() |
twisti@4318 | 4270 | #endif |
twisti@4318 | 4271 | push(rax); // save rax, |
twisti@4318 | 4272 | push(reg); // pass register argument |
twisti@4318 | 4273 | ExternalAddress buffer((address) b); |
twisti@4318 | 4274 | // avoid using pushptr, as it modifies scratch registers |
twisti@4318 | 4275 | // and our contract is not to modify anything |
twisti@4318 | 4276 | movptr(rax, buffer.addr()); |
twisti@4318 | 4277 | push(rax); |
twisti@4318 | 4278 | // call indirectly to solve generation ordering problem |
twisti@4318 | 4279 | movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
twisti@4318 | 4280 | call(rax); |
twisti@4318 | 4281 | // Caller pops the arguments (oop, message) and restores rax, r10 |
twisti@4318 | 4282 | BLOCK_COMMENT("} verify_oop"); |
twisti@4318 | 4283 | } |
twisti@4318 | 4284 | |
twisti@4318 | 4285 | |
twisti@4318 | 4286 | RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr, |
twisti@4318 | 4287 | Register tmp, |
twisti@4318 | 4288 | int offset) { |
twisti@4318 | 4289 | intptr_t value = *delayed_value_addr; |
twisti@4318 | 4290 | if (value != 0) |
twisti@4318 | 4291 | return RegisterOrConstant(value + offset); |
twisti@4318 | 4292 | |
twisti@4318 | 4293 | // load indirectly to solve generation ordering problem |
twisti@4318 | 4294 | movptr(tmp, ExternalAddress((address) delayed_value_addr)); |
twisti@4318 | 4295 | |
twisti@4318 | 4296 | #ifdef ASSERT |
twisti@4318 | 4297 | { Label L; |
twisti@4318 | 4298 | testptr(tmp, tmp); |
twisti@4318 | 4299 | if (WizardMode) { |
twisti@4318 | 4300 | jcc(Assembler::notZero, L); |
twisti@4318 | 4301 | char* buf = new char[40]; |
twisti@4318 | 4302 | sprintf(buf, "DelayedValue="INTPTR_FORMAT, delayed_value_addr[1]); |
twisti@4318 | 4303 | STOP(buf); |
twisti@4318 | 4304 | } else { |
twisti@4318 | 4305 | jccb(Assembler::notZero, L); |
twisti@4318 | 4306 | hlt(); |
twisti@4318 | 4307 | } |
twisti@4318 | 4308 | bind(L); |
twisti@4318 | 4309 | } |
twisti@4318 | 4310 | #endif |
twisti@4318 | 4311 | |
twisti@4318 | 4312 | if (offset != 0) |
twisti@4318 | 4313 | addptr(tmp, offset); |
twisti@4318 | 4314 | |
twisti@4318 | 4315 | return RegisterOrConstant(tmp); |
twisti@4318 | 4316 | } |
twisti@4318 | 4317 | |
twisti@4318 | 4318 | |
twisti@4318 | 4319 | Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, |
twisti@4318 | 4320 | int extra_slot_offset) { |
twisti@4318 | 4321 | // cf. TemplateTable::prepare_invoke(), if (load_receiver). |
twisti@4318 | 4322 | int stackElementSize = Interpreter::stackElementSize; |
twisti@4318 | 4323 | int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); |
twisti@4318 | 4324 | #ifdef ASSERT |
twisti@4318 | 4325 | int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); |
twisti@4318 | 4326 | assert(offset1 - offset == stackElementSize, "correct arithmetic"); |
twisti@4318 | 4327 | #endif |
twisti@4318 | 4328 | Register scale_reg = noreg; |
twisti@4318 | 4329 | Address::ScaleFactor scale_factor = Address::no_scale; |
twisti@4318 | 4330 | if (arg_slot.is_constant()) { |
twisti@4318 | 4331 | offset += arg_slot.as_constant() * stackElementSize; |
twisti@4318 | 4332 | } else { |
twisti@4318 | 4333 | scale_reg = arg_slot.as_register(); |
twisti@4318 | 4334 | scale_factor = Address::times(stackElementSize); |
twisti@4318 | 4335 | } |
twisti@4318 | 4336 | offset += wordSize; // return PC is on stack |
twisti@4318 | 4337 | return Address(rsp, scale_reg, scale_factor, offset); |
twisti@4318 | 4338 | } |
twisti@4318 | 4339 | |
twisti@4318 | 4340 | |
twisti@4318 | 4341 | void MacroAssembler::verify_oop_addr(Address addr, const char* s) { |
twisti@4318 | 4342 | if (!VerifyOops) return; |
twisti@4318 | 4343 | |
twisti@4318 | 4344 | // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord); |
twisti@4318 | 4345 | // Pass register number to verify_oop_subroutine |
twisti@4318 | 4346 | char* b = new char[strlen(s) + 50]; |
twisti@4318 | 4347 | sprintf(b, "verify_oop_addr: %s", s); |
twisti@4318 | 4348 | |
twisti@4318 | 4349 | #ifdef _LP64 |
twisti@4318 | 4350 | push(rscratch1); // save r10, trashed by movptr() |
twisti@4318 | 4351 | #endif |
twisti@4318 | 4352 | push(rax); // save rax, |
twisti@4318 | 4353 | // addr may contain rsp so we will have to adjust it based on the push |
twisti@4318 | 4354 | // we just did (and on 64 bit we do two pushes) |
twisti@4318 | 4355 | // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which |
twisti@4318 | 4356 | // stores rax into addr which is backwards of what was intended. |
twisti@4318 | 4357 | if (addr.uses(rsp)) { |
twisti@4318 | 4358 | lea(rax, addr); |
twisti@4318 | 4359 | pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord)); |
twisti@4318 | 4360 | } else { |
twisti@4318 | 4361 | pushptr(addr); |
twisti@4318 | 4362 | } |
twisti@4318 | 4363 | |
twisti@4318 | 4364 | ExternalAddress buffer((address) b); |
twisti@4318 | 4365 | // pass msg argument |
twisti@4318 | 4366 | // avoid using pushptr, as it modifies scratch registers |
twisti@4318 | 4367 | // and our contract is not to modify anything |
twisti@4318 | 4368 | movptr(rax, buffer.addr()); |
twisti@4318 | 4369 | push(rax); |
twisti@4318 | 4370 | |
twisti@4318 | 4371 | // call indirectly to solve generation ordering problem |
twisti@4318 | 4372 | movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address())); |
twisti@4318 | 4373 | call(rax); |
twisti@4318 | 4374 | // Caller pops the arguments (addr, message) and restores rax, r10. |
twisti@4318 | 4375 | } |
twisti@4318 | 4376 | |
twisti@4318 | 4377 | void MacroAssembler::verify_tlab() { |
twisti@4318 | 4378 | #ifdef ASSERT |
twisti@4318 | 4379 | if (UseTLAB && VerifyOops) { |
twisti@4318 | 4380 | Label next, ok; |
twisti@4318 | 4381 | Register t1 = rsi; |
twisti@4318 | 4382 | Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread); |
twisti@4318 | 4383 | |
twisti@4318 | 4384 | push(t1); |
twisti@4318 | 4385 | NOT_LP64(push(thread_reg)); |
twisti@4318 | 4386 | NOT_LP64(get_thread(thread_reg)); |
twisti@4318 | 4387 | |
twisti@4318 | 4388 | movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
twisti@4318 | 4389 | cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset()))); |
twisti@4318 | 4390 | jcc(Assembler::aboveEqual, next); |
twisti@4318 | 4391 | STOP("assert(top >= start)"); |
twisti@4318 | 4392 | should_not_reach_here(); |
twisti@4318 | 4393 | |
twisti@4318 | 4394 | bind(next); |
twisti@4318 | 4395 | movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset()))); |
twisti@4318 | 4396 | cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset()))); |
twisti@4318 | 4397 | jcc(Assembler::aboveEqual, ok); |
twisti@4318 | 4398 | STOP("assert(top <= end)"); |
twisti@4318 | 4399 | should_not_reach_here(); |
twisti@4318 | 4400 | |
twisti@4318 | 4401 | bind(ok); |
twisti@4318 | 4402 | NOT_LP64(pop(thread_reg)); |
twisti@4318 | 4403 | pop(t1); |
twisti@4318 | 4404 | } |
twisti@4318 | 4405 | #endif |
twisti@4318 | 4406 | } |
twisti@4318 | 4407 | |
twisti@4318 | 4408 | class ControlWord { |
twisti@4318 | 4409 | public: |
twisti@4318 | 4410 | int32_t _value; |
twisti@4318 | 4411 | |
twisti@4318 | 4412 | int rounding_control() const { return (_value >> 10) & 3 ; } |
twisti@4318 | 4413 | int precision_control() const { return (_value >> 8) & 3 ; } |
twisti@4318 | 4414 | bool precision() const { return ((_value >> 5) & 1) != 0; } |
twisti@4318 | 4415 | bool underflow() const { return ((_value >> 4) & 1) != 0; } |
twisti@4318 | 4416 | bool overflow() const { return ((_value >> 3) & 1) != 0; } |
twisti@4318 | 4417 | bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
twisti@4318 | 4418 | bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
twisti@4318 | 4419 | bool invalid() const { return ((_value >> 0) & 1) != 0; } |
twisti@4318 | 4420 | |
twisti@4318 | 4421 | void print() const { |
twisti@4318 | 4422 | // rounding control |
twisti@4318 | 4423 | const char* rc; |
twisti@4318 | 4424 | switch (rounding_control()) { |
twisti@4318 | 4425 | case 0: rc = "round near"; break; |
twisti@4318 | 4426 | case 1: rc = "round down"; break; |
twisti@4318 | 4427 | case 2: rc = "round up "; break; |
twisti@4318 | 4428 | case 3: rc = "chop "; break; |
twisti@4318 | 4429 | }; |
twisti@4318 | 4430 | // precision control |
twisti@4318 | 4431 | const char* pc; |
twisti@4318 | 4432 | switch (precision_control()) { |
twisti@4318 | 4433 | case 0: pc = "24 bits "; break; |
twisti@4318 | 4434 | case 1: pc = "reserved"; break; |
twisti@4318 | 4435 | case 2: pc = "53 bits "; break; |
twisti@4318 | 4436 | case 3: pc = "64 bits "; break; |
twisti@4318 | 4437 | }; |
twisti@4318 | 4438 | // flags |
twisti@4318 | 4439 | char f[9]; |
twisti@4318 | 4440 | f[0] = ' '; |
twisti@4318 | 4441 | f[1] = ' '; |
twisti@4318 | 4442 | f[2] = (precision ()) ? 'P' : 'p'; |
twisti@4318 | 4443 | f[3] = (underflow ()) ? 'U' : 'u'; |
twisti@4318 | 4444 | f[4] = (overflow ()) ? 'O' : 'o'; |
twisti@4318 | 4445 | f[5] = (zero_divide ()) ? 'Z' : 'z'; |
twisti@4318 | 4446 | f[6] = (denormalized()) ? 'D' : 'd'; |
twisti@4318 | 4447 | f[7] = (invalid ()) ? 'I' : 'i'; |
twisti@4318 | 4448 | f[8] = '\x0'; |
twisti@4318 | 4449 | // output |
twisti@4318 | 4450 | printf("%04x masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc); |
twisti@4318 | 4451 | } |
twisti@4318 | 4452 | |
twisti@4318 | 4453 | }; |
twisti@4318 | 4454 | |
twisti@4318 | 4455 | class StatusWord { |
twisti@4318 | 4456 | public: |
twisti@4318 | 4457 | int32_t _value; |
twisti@4318 | 4458 | |
twisti@4318 | 4459 | bool busy() const { return ((_value >> 15) & 1) != 0; } |
twisti@4318 | 4460 | bool C3() const { return ((_value >> 14) & 1) != 0; } |
twisti@4318 | 4461 | bool C2() const { return ((_value >> 10) & 1) != 0; } |
twisti@4318 | 4462 | bool C1() const { return ((_value >> 9) & 1) != 0; } |
twisti@4318 | 4463 | bool C0() const { return ((_value >> 8) & 1) != 0; } |
twisti@4318 | 4464 | int top() const { return (_value >> 11) & 7 ; } |
twisti@4318 | 4465 | bool error_status() const { return ((_value >> 7) & 1) != 0; } |
twisti@4318 | 4466 | bool stack_fault() const { return ((_value >> 6) & 1) != 0; } |
twisti@4318 | 4467 | bool precision() const { return ((_value >> 5) & 1) != 0; } |
twisti@4318 | 4468 | bool underflow() const { return ((_value >> 4) & 1) != 0; } |
twisti@4318 | 4469 | bool overflow() const { return ((_value >> 3) & 1) != 0; } |
twisti@4318 | 4470 | bool zero_divide() const { return ((_value >> 2) & 1) != 0; } |
twisti@4318 | 4471 | bool denormalized() const { return ((_value >> 1) & 1) != 0; } |
twisti@4318 | 4472 | bool invalid() const { return ((_value >> 0) & 1) != 0; } |
twisti@4318 | 4473 | |
twisti@4318 | 4474 | void print() const { |
twisti@4318 | 4475 | // condition codes |
twisti@4318 | 4476 | char c[5]; |
twisti@4318 | 4477 | c[0] = (C3()) ? '3' : '-'; |
twisti@4318 | 4478 | c[1] = (C2()) ? '2' : '-'; |
twisti@4318 | 4479 | c[2] = (C1()) ? '1' : '-'; |
twisti@4318 | 4480 | c[3] = (C0()) ? '0' : '-'; |
twisti@4318 | 4481 | c[4] = '\x0'; |
twisti@4318 | 4482 | // flags |
twisti@4318 | 4483 | char f[9]; |
twisti@4318 | 4484 | f[0] = (error_status()) ? 'E' : '-'; |
twisti@4318 | 4485 | f[1] = (stack_fault ()) ? 'S' : '-'; |
twisti@4318 | 4486 | f[2] = (precision ()) ? 'P' : '-'; |
twisti@4318 | 4487 | f[3] = (underflow ()) ? 'U' : '-'; |
twisti@4318 | 4488 | f[4] = (overflow ()) ? 'O' : '-'; |
twisti@4318 | 4489 | f[5] = (zero_divide ()) ? 'Z' : '-'; |
twisti@4318 | 4490 | f[6] = (denormalized()) ? 'D' : '-'; |
twisti@4318 | 4491 | f[7] = (invalid ()) ? 'I' : '-'; |
twisti@4318 | 4492 | f[8] = '\x0'; |
twisti@4318 | 4493 | // output |
twisti@4318 | 4494 | printf("%04x flags = %s, cc = %s, top = %d", _value & 0xFFFF, f, c, top()); |
twisti@4318 | 4495 | } |
twisti@4318 | 4496 | |
twisti@4318 | 4497 | }; |
twisti@4318 | 4498 | |
twisti@4318 | 4499 | class TagWord { |
twisti@4318 | 4500 | public: |
twisti@4318 | 4501 | int32_t _value; |
twisti@4318 | 4502 | |
twisti@4318 | 4503 | int tag_at(int i) const { return (_value >> (i*2)) & 3; } |
twisti@4318 | 4504 | |
twisti@4318 | 4505 | void print() const { |
twisti@4318 | 4506 | printf("%04x", _value & 0xFFFF); |
twisti@4318 | 4507 | } |
twisti@4318 | 4508 | |
twisti@4318 | 4509 | }; |
twisti@4318 | 4510 | |
twisti@4318 | 4511 | class FPU_Register { |
twisti@4318 | 4512 | public: |
twisti@4318 | 4513 | int32_t _m0; |
twisti@4318 | 4514 | int32_t _m1; |
twisti@4318 | 4515 | int16_t _ex; |
twisti@4318 | 4516 | |
twisti@4318 | 4517 | bool is_indefinite() const { |
twisti@4318 | 4518 | return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0; |
twisti@4318 | 4519 | } |
twisti@4318 | 4520 | |
twisti@4318 | 4521 | void print() const { |
twisti@4318 | 4522 | char sign = (_ex < 0) ? '-' : '+'; |
twisti@4318 | 4523 | const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : " "; |
twisti@4318 | 4524 | printf("%c%04hx.%08x%08x %s", sign, _ex, _m1, _m0, kind); |
twisti@4318 | 4525 | }; |
twisti@4318 | 4526 | |
twisti@4318 | 4527 | }; |
twisti@4318 | 4528 | |
twisti@4318 | 4529 | class FPU_State { |
twisti@4318 | 4530 | public: |
twisti@4318 | 4531 | enum { |
twisti@4318 | 4532 | register_size = 10, |
twisti@4318 | 4533 | number_of_registers = 8, |
twisti@4318 | 4534 | register_mask = 7 |
twisti@4318 | 4535 | }; |
twisti@4318 | 4536 | |
twisti@4318 | 4537 | ControlWord _control_word; |
twisti@4318 | 4538 | StatusWord _status_word; |
twisti@4318 | 4539 | TagWord _tag_word; |
twisti@4318 | 4540 | int32_t _error_offset; |
twisti@4318 | 4541 | int32_t _error_selector; |
twisti@4318 | 4542 | int32_t _data_offset; |
twisti@4318 | 4543 | int32_t _data_selector; |
twisti@4318 | 4544 | int8_t _register[register_size * number_of_registers]; |
twisti@4318 | 4545 | |
twisti@4318 | 4546 | int tag_for_st(int i) const { return _tag_word.tag_at((_status_word.top() + i) & register_mask); } |
twisti@4318 | 4547 | FPU_Register* st(int i) const { return (FPU_Register*)&_register[register_size * i]; } |
twisti@4318 | 4548 | |
twisti@4318 | 4549 | const char* tag_as_string(int tag) const { |
twisti@4318 | 4550 | switch (tag) { |
twisti@4318 | 4551 | case 0: return "valid"; |
twisti@4318 | 4552 | case 1: return "zero"; |
twisti@4318 | 4553 | case 2: return "special"; |
twisti@4318 | 4554 | case 3: return "empty"; |
twisti@4318 | 4555 | } |
twisti@4318 | 4556 | ShouldNotReachHere(); |
twisti@4318 | 4557 | return NULL; |
twisti@4318 | 4558 | } |
twisti@4318 | 4559 | |
twisti@4318 | 4560 | void print() const { |
twisti@4318 | 4561 | // print computation registers |
twisti@4318 | 4562 | { int t = _status_word.top(); |
twisti@4318 | 4563 | for (int i = 0; i < number_of_registers; i++) { |
twisti@4318 | 4564 | int j = (i - t) & register_mask; |
twisti@4318 | 4565 | printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j); |
twisti@4318 | 4566 | st(j)->print(); |
twisti@4318 | 4567 | printf(" %s\n", tag_as_string(_tag_word.tag_at(i))); |
twisti@4318 | 4568 | } |
twisti@4318 | 4569 | } |
twisti@4318 | 4570 | printf("\n"); |
twisti@4318 | 4571 | // print control registers |
twisti@4318 | 4572 | printf("ctrl = "); _control_word.print(); printf("\n"); |
twisti@4318 | 4573 | printf("stat = "); _status_word .print(); printf("\n"); |
twisti@4318 | 4574 | printf("tags = "); _tag_word .print(); printf("\n"); |
twisti@4318 | 4575 | } |
twisti@4318 | 4576 | |
twisti@4318 | 4577 | }; |
twisti@4318 | 4578 | |
twisti@4318 | 4579 | class Flag_Register { |
twisti@4318 | 4580 | public: |
twisti@4318 | 4581 | int32_t _value; |
twisti@4318 | 4582 | |
twisti@4318 | 4583 | bool overflow() const { return ((_value >> 11) & 1) != 0; } |
twisti@4318 | 4584 | bool direction() const { return ((_value >> 10) & 1) != 0; } |
twisti@4318 | 4585 | bool sign() const { return ((_value >> 7) & 1) != 0; } |
twisti@4318 | 4586 | bool zero() const { return ((_value >> 6) & 1) != 0; } |
twisti@4318 | 4587 | bool auxiliary_carry() const { return ((_value >> 4) & 1) != 0; } |
twisti@4318 | 4588 | bool parity() const { return ((_value >> 2) & 1) != 0; } |
twisti@4318 | 4589 | bool carry() const { return ((_value >> 0) & 1) != 0; } |
twisti@4318 | 4590 | |
twisti@4318 | 4591 | void print() const { |
twisti@4318 | 4592 | // flags |
twisti@4318 | 4593 | char f[8]; |
twisti@4318 | 4594 | f[0] = (overflow ()) ? 'O' : '-'; |
twisti@4318 | 4595 | f[1] = (direction ()) ? 'D' : '-'; |
twisti@4318 | 4596 | f[2] = (sign ()) ? 'S' : '-'; |
twisti@4318 | 4597 | f[3] = (zero ()) ? 'Z' : '-'; |
twisti@4318 | 4598 | f[4] = (auxiliary_carry()) ? 'A' : '-'; |
twisti@4318 | 4599 | f[5] = (parity ()) ? 'P' : '-'; |
twisti@4318 | 4600 | f[6] = (carry ()) ? 'C' : '-'; |
twisti@4318 | 4601 | f[7] = '\x0'; |
twisti@4318 | 4602 | // output |
twisti@4318 | 4603 | printf("%08x flags = %s", _value, f); |
twisti@4318 | 4604 | } |
twisti@4318 | 4605 | |
twisti@4318 | 4606 | }; |
twisti@4318 | 4607 | |
twisti@4318 | 4608 | class IU_Register { |
twisti@4318 | 4609 | public: |
twisti@4318 | 4610 | int32_t _value; |
twisti@4318 | 4611 | |
twisti@4318 | 4612 | void print() const { |
twisti@4318 | 4613 | printf("%08x %11d", _value, _value); |
twisti@4318 | 4614 | } |
twisti@4318 | 4615 | |
twisti@4318 | 4616 | }; |
twisti@4318 | 4617 | |
twisti@4318 | 4618 | class IU_State { |
twisti@4318 | 4619 | public: |
twisti@4318 | 4620 | Flag_Register _eflags; |
twisti@4318 | 4621 | IU_Register _rdi; |
twisti@4318 | 4622 | IU_Register _rsi; |
twisti@4318 | 4623 | IU_Register _rbp; |
twisti@4318 | 4624 | IU_Register _rsp; |
twisti@4318 | 4625 | IU_Register _rbx; |
twisti@4318 | 4626 | IU_Register _rdx; |
twisti@4318 | 4627 | IU_Register _rcx; |
twisti@4318 | 4628 | IU_Register _rax; |
twisti@4318 | 4629 | |
twisti@4318 | 4630 | void print() const { |
twisti@4318 | 4631 | // computation registers |
twisti@4318 | 4632 | printf("rax, = "); _rax.print(); printf("\n"); |
twisti@4318 | 4633 | printf("rbx, = "); _rbx.print(); printf("\n"); |
twisti@4318 | 4634 | printf("rcx = "); _rcx.print(); printf("\n"); |
twisti@4318 | 4635 | printf("rdx = "); _rdx.print(); printf("\n"); |
twisti@4318 | 4636 | printf("rdi = "); _rdi.print(); printf("\n"); |
twisti@4318 | 4637 | printf("rsi = "); _rsi.print(); printf("\n"); |
twisti@4318 | 4638 | printf("rbp, = "); _rbp.print(); printf("\n"); |
twisti@4318 | 4639 | printf("rsp = "); _rsp.print(); printf("\n"); |
twisti@4318 | 4640 | printf("\n"); |
twisti@4318 | 4641 | // control registers |
twisti@4318 | 4642 | printf("flgs = "); _eflags.print(); printf("\n"); |
twisti@4318 | 4643 | } |
twisti@4318 | 4644 | }; |
twisti@4318 | 4645 | |
twisti@4318 | 4646 | |
twisti@4318 | 4647 | class CPU_State { |
twisti@4318 | 4648 | public: |
twisti@4318 | 4649 | FPU_State _fpu_state; |
twisti@4318 | 4650 | IU_State _iu_state; |
twisti@4318 | 4651 | |
twisti@4318 | 4652 | void print() const { |
twisti@4318 | 4653 | printf("--------------------------------------------------\n"); |
twisti@4318 | 4654 | _iu_state .print(); |
twisti@4318 | 4655 | printf("\n"); |
twisti@4318 | 4656 | _fpu_state.print(); |
twisti@4318 | 4657 | printf("--------------------------------------------------\n"); |
twisti@4318 | 4658 | } |
twisti@4318 | 4659 | |
twisti@4318 | 4660 | }; |
twisti@4318 | 4661 | |
twisti@4318 | 4662 | |
twisti@4318 | 4663 | static void _print_CPU_state(CPU_State* state) { |
twisti@4318 | 4664 | state->print(); |
twisti@4318 | 4665 | }; |
twisti@4318 | 4666 | |
twisti@4318 | 4667 | |
twisti@4318 | 4668 | void MacroAssembler::print_CPU_state() { |
twisti@4318 | 4669 | push_CPU_state(); |
twisti@4318 | 4670 | push(rsp); // pass CPU state |
twisti@4318 | 4671 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state))); |
twisti@4318 | 4672 | addptr(rsp, wordSize); // discard argument |
twisti@4318 | 4673 | pop_CPU_state(); |
twisti@4318 | 4674 | } |
twisti@4318 | 4675 | |
twisti@4318 | 4676 | |
twisti@4318 | 4677 | static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) { |
twisti@4318 | 4678 | static int counter = 0; |
twisti@4318 | 4679 | FPU_State* fs = &state->_fpu_state; |
twisti@4318 | 4680 | counter++; |
twisti@4318 | 4681 | // For leaf calls, only verify that the top few elements remain empty. |
twisti@4318 | 4682 | // We only need 1 empty at the top for C2 code. |
twisti@4318 | 4683 | if( stack_depth < 0 ) { |
twisti@4318 | 4684 | if( fs->tag_for_st(7) != 3 ) { |
twisti@4318 | 4685 | printf("FPR7 not empty\n"); |
twisti@4318 | 4686 | state->print(); |
twisti@4318 | 4687 | assert(false, "error"); |
twisti@4318 | 4688 | return false; |
twisti@4318 | 4689 | } |
twisti@4318 | 4690 | return true; // All other stack states do not matter |
twisti@4318 | 4691 | } |
twisti@4318 | 4692 | |
twisti@4318 | 4693 | assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std, |
twisti@4318 | 4694 | "bad FPU control word"); |
twisti@4318 | 4695 | |
twisti@4318 | 4696 | // compute stack depth |
twisti@4318 | 4697 | int i = 0; |
twisti@4318 | 4698 | while (i < FPU_State::number_of_registers && fs->tag_for_st(i) < 3) i++; |
twisti@4318 | 4699 | int d = i; |
twisti@4318 | 4700 | while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++; |
twisti@4318 | 4701 | // verify findings |
twisti@4318 | 4702 | if (i != FPU_State::number_of_registers) { |
twisti@4318 | 4703 | // stack not contiguous |
twisti@4318 | 4704 | printf("%s: stack not contiguous at ST%d\n", s, i); |
twisti@4318 | 4705 | state->print(); |
twisti@4318 | 4706 | assert(false, "error"); |
twisti@4318 | 4707 | return false; |
twisti@4318 | 4708 | } |
twisti@4318 | 4709 | // check if computed stack depth corresponds to expected stack depth |
twisti@4318 | 4710 | if (stack_depth < 0) { |
twisti@4318 | 4711 | // expected stack depth is -stack_depth or less |
twisti@4318 | 4712 | if (d > -stack_depth) { |
twisti@4318 | 4713 | // too many elements on the stack |
twisti@4318 | 4714 | printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d); |
twisti@4318 | 4715 | state->print(); |
twisti@4318 | 4716 | assert(false, "error"); |
twisti@4318 | 4717 | return false; |
twisti@4318 | 4718 | } |
twisti@4318 | 4719 | } else { |
twisti@4318 | 4720 | // expected stack depth is stack_depth |
twisti@4318 | 4721 | if (d != stack_depth) { |
twisti@4318 | 4722 | // wrong stack depth |
twisti@4318 | 4723 | printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d); |
twisti@4318 | 4724 | state->print(); |
twisti@4318 | 4725 | assert(false, "error"); |
twisti@4318 | 4726 | return false; |
twisti@4318 | 4727 | } |
twisti@4318 | 4728 | } |
twisti@4318 | 4729 | // everything is cool |
twisti@4318 | 4730 | return true; |
twisti@4318 | 4731 | } |
twisti@4318 | 4732 | |
twisti@4318 | 4733 | |
twisti@4318 | 4734 | void MacroAssembler::verify_FPU(int stack_depth, const char* s) { |
twisti@4318 | 4735 | if (!VerifyFPU) return; |
twisti@4318 | 4736 | push_CPU_state(); |
twisti@4318 | 4737 | push(rsp); // pass CPU state |
twisti@4318 | 4738 | ExternalAddress msg((address) s); |
twisti@4318 | 4739 | // pass message string s |
twisti@4318 | 4740 | pushptr(msg.addr()); |
twisti@4318 | 4741 | push(stack_depth); // pass stack depth |
twisti@4318 | 4742 | call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU))); |
twisti@4318 | 4743 | addptr(rsp, 3 * wordSize); // discard arguments |
twisti@4318 | 4744 | // check for error |
twisti@4318 | 4745 | { Label L; |
twisti@4318 | 4746 | testl(rax, rax); |
twisti@4318 | 4747 | jcc(Assembler::notZero, L); |
twisti@4318 | 4748 | int3(); // break if error condition |
twisti@4318 | 4749 | bind(L); |
twisti@4318 | 4750 | } |
twisti@4318 | 4751 | pop_CPU_state(); |
twisti@4318 | 4752 | } |
twisti@4318 | 4753 | |
twisti@4318 | 4754 | void MacroAssembler::load_klass(Register dst, Register src) { |
twisti@4318 | 4755 | #ifdef _LP64 |
twisti@4318 | 4756 | if (UseCompressedKlassPointers) { |
twisti@4318 | 4757 | movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
twisti@4318 | 4758 | decode_klass_not_null(dst); |
twisti@4318 | 4759 | } else |
twisti@4318 | 4760 | #endif |
twisti@4318 | 4761 | movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
twisti@4318 | 4762 | } |
twisti@4318 | 4763 | |
twisti@4318 | 4764 | void MacroAssembler::load_prototype_header(Register dst, Register src) { |
twisti@4318 | 4765 | #ifdef _LP64 |
twisti@4318 | 4766 | if (UseCompressedKlassPointers) { |
twisti@4318 | 4767 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 4768 | movl(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
twisti@4318 | 4769 | if (Universe::narrow_klass_shift() != 0) { |
twisti@4318 | 4770 | assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
twisti@4318 | 4771 | assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); |
twisti@4318 | 4772 | movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset())); |
twisti@4318 | 4773 | } else { |
twisti@4318 | 4774 | movq(dst, Address(dst, Klass::prototype_header_offset())); |
twisti@4318 | 4775 | } |
twisti@4318 | 4776 | } else |
twisti@4318 | 4777 | #endif |
twisti@4318 | 4778 | { |
twisti@4318 | 4779 | movptr(dst, Address(src, oopDesc::klass_offset_in_bytes())); |
twisti@4318 | 4780 | movptr(dst, Address(dst, Klass::prototype_header_offset())); |
twisti@4318 | 4781 | } |
twisti@4318 | 4782 | } |
twisti@4318 | 4783 | |
twisti@4318 | 4784 | void MacroAssembler::store_klass(Register dst, Register src) { |
twisti@4318 | 4785 | #ifdef _LP64 |
twisti@4318 | 4786 | if (UseCompressedKlassPointers) { |
twisti@4318 | 4787 | encode_klass_not_null(src); |
twisti@4318 | 4788 | movl(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
twisti@4318 | 4789 | } else |
twisti@4318 | 4790 | #endif |
twisti@4318 | 4791 | movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src); |
twisti@4318 | 4792 | } |
twisti@4318 | 4793 | |
twisti@4318 | 4794 | void MacroAssembler::load_heap_oop(Register dst, Address src) { |
twisti@4318 | 4795 | #ifdef _LP64 |
twisti@4318 | 4796 | // FIXME: Must change all places where we try to load the klass. |
twisti@4318 | 4797 | if (UseCompressedOops) { |
twisti@4318 | 4798 | movl(dst, src); |
twisti@4318 | 4799 | decode_heap_oop(dst); |
twisti@4318 | 4800 | } else |
twisti@4318 | 4801 | #endif |
twisti@4318 | 4802 | movptr(dst, src); |
twisti@4318 | 4803 | } |
twisti@4318 | 4804 | |
twisti@4318 | 4805 | // Doesn't do verfication, generates fixed size code |
twisti@4318 | 4806 | void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) { |
twisti@4318 | 4807 | #ifdef _LP64 |
twisti@4318 | 4808 | if (UseCompressedOops) { |
twisti@4318 | 4809 | movl(dst, src); |
twisti@4318 | 4810 | decode_heap_oop_not_null(dst); |
twisti@4318 | 4811 | } else |
twisti@4318 | 4812 | #endif |
twisti@4318 | 4813 | movptr(dst, src); |
twisti@4318 | 4814 | } |
twisti@4318 | 4815 | |
twisti@4318 | 4816 | void MacroAssembler::store_heap_oop(Address dst, Register src) { |
twisti@4318 | 4817 | #ifdef _LP64 |
twisti@4318 | 4818 | if (UseCompressedOops) { |
twisti@4318 | 4819 | assert(!dst.uses(src), "not enough registers"); |
twisti@4318 | 4820 | encode_heap_oop(src); |
twisti@4318 | 4821 | movl(dst, src); |
twisti@4318 | 4822 | } else |
twisti@4318 | 4823 | #endif |
twisti@4318 | 4824 | movptr(dst, src); |
twisti@4318 | 4825 | } |
twisti@4318 | 4826 | |
twisti@4318 | 4827 | void MacroAssembler::cmp_heap_oop(Register src1, Address src2, Register tmp) { |
twisti@4318 | 4828 | assert_different_registers(src1, tmp); |
twisti@4318 | 4829 | #ifdef _LP64 |
twisti@4318 | 4830 | if (UseCompressedOops) { |
twisti@4318 | 4831 | bool did_push = false; |
twisti@4318 | 4832 | if (tmp == noreg) { |
twisti@4318 | 4833 | tmp = rax; |
twisti@4318 | 4834 | push(tmp); |
twisti@4318 | 4835 | did_push = true; |
twisti@4318 | 4836 | assert(!src2.uses(rsp), "can't push"); |
twisti@4318 | 4837 | } |
twisti@4318 | 4838 | load_heap_oop(tmp, src2); |
twisti@4318 | 4839 | cmpptr(src1, tmp); |
twisti@4318 | 4840 | if (did_push) pop(tmp); |
twisti@4318 | 4841 | } else |
twisti@4318 | 4842 | #endif |
twisti@4318 | 4843 | cmpptr(src1, src2); |
twisti@4318 | 4844 | } |
twisti@4318 | 4845 | |
twisti@4318 | 4846 | // Used for storing NULLs. |
twisti@4318 | 4847 | void MacroAssembler::store_heap_oop_null(Address dst) { |
twisti@4318 | 4848 | #ifdef _LP64 |
twisti@4318 | 4849 | if (UseCompressedOops) { |
twisti@4318 | 4850 | movl(dst, (int32_t)NULL_WORD); |
twisti@4318 | 4851 | } else { |
twisti@4318 | 4852 | movslq(dst, (int32_t)NULL_WORD); |
twisti@4318 | 4853 | } |
twisti@4318 | 4854 | #else |
twisti@4318 | 4855 | movl(dst, (int32_t)NULL_WORD); |
twisti@4318 | 4856 | #endif |
twisti@4318 | 4857 | } |
twisti@4318 | 4858 | |
twisti@4318 | 4859 | #ifdef _LP64 |
twisti@4318 | 4860 | void MacroAssembler::store_klass_gap(Register dst, Register src) { |
twisti@4318 | 4861 | if (UseCompressedKlassPointers) { |
twisti@4318 | 4862 | // Store to klass gap in destination |
twisti@4318 | 4863 | movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src); |
twisti@4318 | 4864 | } |
twisti@4318 | 4865 | } |
twisti@4318 | 4866 | |
twisti@4318 | 4867 | #ifdef ASSERT |
twisti@4318 | 4868 | void MacroAssembler::verify_heapbase(const char* msg) { |
twisti@4318 | 4869 | assert (UseCompressedOops || UseCompressedKlassPointers, "should be compressed"); |
twisti@4318 | 4870 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 4871 | if (CheckCompressedOops) { |
twisti@4318 | 4872 | Label ok; |
twisti@4318 | 4873 | push(rscratch1); // cmpptr trashes rscratch1 |
twisti@4318 | 4874 | cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr())); |
twisti@4318 | 4875 | jcc(Assembler::equal, ok); |
twisti@4318 | 4876 | STOP(msg); |
twisti@4318 | 4877 | bind(ok); |
twisti@4318 | 4878 | pop(rscratch1); |
twisti@4318 | 4879 | } |
twisti@4318 | 4880 | } |
twisti@4318 | 4881 | #endif |
twisti@4318 | 4882 | |
twisti@4318 | 4883 | // Algorithm must match oop.inline.hpp encode_heap_oop. |
twisti@4318 | 4884 | void MacroAssembler::encode_heap_oop(Register r) { |
twisti@4318 | 4885 | #ifdef ASSERT |
twisti@4318 | 4886 | verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?"); |
twisti@4318 | 4887 | #endif |
twisti@4318 | 4888 | verify_oop(r, "broken oop in encode_heap_oop"); |
twisti@4318 | 4889 | if (Universe::narrow_oop_base() == NULL) { |
twisti@4318 | 4890 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4891 | assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4892 | shrq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4893 | } |
twisti@4318 | 4894 | return; |
twisti@4318 | 4895 | } |
twisti@4318 | 4896 | testq(r, r); |
twisti@4318 | 4897 | cmovq(Assembler::equal, r, r12_heapbase); |
twisti@4318 | 4898 | subq(r, r12_heapbase); |
twisti@4318 | 4899 | shrq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4900 | } |
twisti@4318 | 4901 | |
twisti@4318 | 4902 | void MacroAssembler::encode_heap_oop_not_null(Register r) { |
twisti@4318 | 4903 | #ifdef ASSERT |
twisti@4318 | 4904 | verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?"); |
twisti@4318 | 4905 | if (CheckCompressedOops) { |
twisti@4318 | 4906 | Label ok; |
twisti@4318 | 4907 | testq(r, r); |
twisti@4318 | 4908 | jcc(Assembler::notEqual, ok); |
twisti@4318 | 4909 | STOP("null oop passed to encode_heap_oop_not_null"); |
twisti@4318 | 4910 | bind(ok); |
twisti@4318 | 4911 | } |
twisti@4318 | 4912 | #endif |
twisti@4318 | 4913 | verify_oop(r, "broken oop in encode_heap_oop_not_null"); |
twisti@4318 | 4914 | if (Universe::narrow_oop_base() != NULL) { |
twisti@4318 | 4915 | subq(r, r12_heapbase); |
twisti@4318 | 4916 | } |
twisti@4318 | 4917 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4918 | assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4919 | shrq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4920 | } |
twisti@4318 | 4921 | } |
twisti@4318 | 4922 | |
twisti@4318 | 4923 | void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { |
twisti@4318 | 4924 | #ifdef ASSERT |
twisti@4318 | 4925 | verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?"); |
twisti@4318 | 4926 | if (CheckCompressedOops) { |
twisti@4318 | 4927 | Label ok; |
twisti@4318 | 4928 | testq(src, src); |
twisti@4318 | 4929 | jcc(Assembler::notEqual, ok); |
twisti@4318 | 4930 | STOP("null oop passed to encode_heap_oop_not_null2"); |
twisti@4318 | 4931 | bind(ok); |
twisti@4318 | 4932 | } |
twisti@4318 | 4933 | #endif |
twisti@4318 | 4934 | verify_oop(src, "broken oop in encode_heap_oop_not_null2"); |
twisti@4318 | 4935 | if (dst != src) { |
twisti@4318 | 4936 | movq(dst, src); |
twisti@4318 | 4937 | } |
twisti@4318 | 4938 | if (Universe::narrow_oop_base() != NULL) { |
twisti@4318 | 4939 | subq(dst, r12_heapbase); |
twisti@4318 | 4940 | } |
twisti@4318 | 4941 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4942 | assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4943 | shrq(dst, LogMinObjAlignmentInBytes); |
twisti@4318 | 4944 | } |
twisti@4318 | 4945 | } |
twisti@4318 | 4946 | |
twisti@4318 | 4947 | void MacroAssembler::decode_heap_oop(Register r) { |
twisti@4318 | 4948 | #ifdef ASSERT |
twisti@4318 | 4949 | verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?"); |
twisti@4318 | 4950 | #endif |
twisti@4318 | 4951 | if (Universe::narrow_oop_base() == NULL) { |
twisti@4318 | 4952 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4953 | assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4954 | shlq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4955 | } |
twisti@4318 | 4956 | } else { |
twisti@4318 | 4957 | Label done; |
twisti@4318 | 4958 | shlq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4959 | jccb(Assembler::equal, done); |
twisti@4318 | 4960 | addq(r, r12_heapbase); |
twisti@4318 | 4961 | bind(done); |
twisti@4318 | 4962 | } |
twisti@4318 | 4963 | verify_oop(r, "broken oop in decode_heap_oop"); |
twisti@4318 | 4964 | } |
twisti@4318 | 4965 | |
twisti@4318 | 4966 | void MacroAssembler::decode_heap_oop_not_null(Register r) { |
twisti@4318 | 4967 | // Note: it will change flags |
twisti@4318 | 4968 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 4969 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 4970 | // Cannot assert, unverified entry point counts instructions (see .ad file) |
twisti@4318 | 4971 | // vtableStubs also counts instructions in pd_code_size_limit. |
twisti@4318 | 4972 | // Also do not verify_oop as this is called by verify_oop. |
twisti@4318 | 4973 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4974 | assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4975 | shlq(r, LogMinObjAlignmentInBytes); |
twisti@4318 | 4976 | if (Universe::narrow_oop_base() != NULL) { |
twisti@4318 | 4977 | addq(r, r12_heapbase); |
twisti@4318 | 4978 | } |
twisti@4318 | 4979 | } else { |
twisti@4318 | 4980 | assert (Universe::narrow_oop_base() == NULL, "sanity"); |
twisti@4318 | 4981 | } |
twisti@4318 | 4982 | } |
twisti@4318 | 4983 | |
twisti@4318 | 4984 | void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { |
twisti@4318 | 4985 | // Note: it will change flags |
twisti@4318 | 4986 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 4987 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 4988 | // Cannot assert, unverified entry point counts instructions (see .ad file) |
twisti@4318 | 4989 | // vtableStubs also counts instructions in pd_code_size_limit. |
twisti@4318 | 4990 | // Also do not verify_oop as this is called by verify_oop. |
twisti@4318 | 4991 | if (Universe::narrow_oop_shift() != 0) { |
twisti@4318 | 4992 | assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong"); |
twisti@4318 | 4993 | if (LogMinObjAlignmentInBytes == Address::times_8) { |
twisti@4318 | 4994 | leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); |
twisti@4318 | 4995 | } else { |
twisti@4318 | 4996 | if (dst != src) { |
twisti@4318 | 4997 | movq(dst, src); |
twisti@4318 | 4998 | } |
twisti@4318 | 4999 | shlq(dst, LogMinObjAlignmentInBytes); |
twisti@4318 | 5000 | if (Universe::narrow_oop_base() != NULL) { |
twisti@4318 | 5001 | addq(dst, r12_heapbase); |
twisti@4318 | 5002 | } |
twisti@4318 | 5003 | } |
twisti@4318 | 5004 | } else { |
twisti@4318 | 5005 | assert (Universe::narrow_oop_base() == NULL, "sanity"); |
twisti@4318 | 5006 | if (dst != src) { |
twisti@4318 | 5007 | movq(dst, src); |
twisti@4318 | 5008 | } |
twisti@4318 | 5009 | } |
twisti@4318 | 5010 | } |
twisti@4318 | 5011 | |
twisti@4318 | 5012 | void MacroAssembler::encode_klass_not_null(Register r) { |
twisti@4318 | 5013 | assert(Metaspace::is_initialized(), "metaspace should be initialized"); |
twisti@4318 | 5014 | #ifdef ASSERT |
twisti@4318 | 5015 | verify_heapbase("MacroAssembler::encode_klass_not_null: heap base corrupted?"); |
twisti@4318 | 5016 | #endif |
twisti@4318 | 5017 | if (Universe::narrow_klass_base() != NULL) { |
twisti@4318 | 5018 | subq(r, r12_heapbase); |
twisti@4318 | 5019 | } |
twisti@4318 | 5020 | if (Universe::narrow_klass_shift() != 0) { |
twisti@4318 | 5021 | assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
twisti@4318 | 5022 | shrq(r, LogKlassAlignmentInBytes); |
twisti@4318 | 5023 | } |
twisti@4318 | 5024 | } |
twisti@4318 | 5025 | |
twisti@4318 | 5026 | void MacroAssembler::encode_klass_not_null(Register dst, Register src) { |
twisti@4318 | 5027 | assert(Metaspace::is_initialized(), "metaspace should be initialized"); |
twisti@4318 | 5028 | #ifdef ASSERT |
twisti@4318 | 5029 | verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?"); |
twisti@4318 | 5030 | #endif |
twisti@4318 | 5031 | if (dst != src) { |
twisti@4318 | 5032 | movq(dst, src); |
twisti@4318 | 5033 | } |
twisti@4318 | 5034 | if (Universe::narrow_klass_base() != NULL) { |
twisti@4318 | 5035 | subq(dst, r12_heapbase); |
twisti@4318 | 5036 | } |
twisti@4318 | 5037 | if (Universe::narrow_klass_shift() != 0) { |
twisti@4318 | 5038 | assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
twisti@4318 | 5039 | shrq(dst, LogKlassAlignmentInBytes); |
twisti@4318 | 5040 | } |
twisti@4318 | 5041 | } |
twisti@4318 | 5042 | |
twisti@4318 | 5043 | void MacroAssembler::decode_klass_not_null(Register r) { |
twisti@4318 | 5044 | assert(Metaspace::is_initialized(), "metaspace should be initialized"); |
twisti@4318 | 5045 | // Note: it will change flags |
twisti@4318 | 5046 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5047 | // Cannot assert, unverified entry point counts instructions (see .ad file) |
twisti@4318 | 5048 | // vtableStubs also counts instructions in pd_code_size_limit. |
twisti@4318 | 5049 | // Also do not verify_oop as this is called by verify_oop. |
twisti@4318 | 5050 | if (Universe::narrow_klass_shift() != 0) { |
twisti@4318 | 5051 | assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
twisti@4318 | 5052 | shlq(r, LogKlassAlignmentInBytes); |
twisti@4318 | 5053 | if (Universe::narrow_klass_base() != NULL) { |
twisti@4318 | 5054 | addq(r, r12_heapbase); |
twisti@4318 | 5055 | } |
twisti@4318 | 5056 | } else { |
twisti@4318 | 5057 | assert (Universe::narrow_klass_base() == NULL, "sanity"); |
twisti@4318 | 5058 | } |
twisti@4318 | 5059 | } |
twisti@4318 | 5060 | |
twisti@4318 | 5061 | void MacroAssembler::decode_klass_not_null(Register dst, Register src) { |
twisti@4318 | 5062 | assert(Metaspace::is_initialized(), "metaspace should be initialized"); |
twisti@4318 | 5063 | // Note: it will change flags |
twisti@4318 | 5064 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5065 | // Cannot assert, unverified entry point counts instructions (see .ad file) |
twisti@4318 | 5066 | // vtableStubs also counts instructions in pd_code_size_limit. |
twisti@4318 | 5067 | // Also do not verify_oop as this is called by verify_oop. |
twisti@4318 | 5068 | if (Universe::narrow_klass_shift() != 0) { |
twisti@4318 | 5069 | assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong"); |
twisti@4318 | 5070 | assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?"); |
twisti@4318 | 5071 | leaq(dst, Address(r12_heapbase, src, Address::times_8, 0)); |
twisti@4318 | 5072 | } else { |
twisti@4318 | 5073 | assert (Universe::narrow_klass_base() == NULL, "sanity"); |
twisti@4318 | 5074 | if (dst != src) { |
twisti@4318 | 5075 | movq(dst, src); |
twisti@4318 | 5076 | } |
twisti@4318 | 5077 | } |
twisti@4318 | 5078 | } |
twisti@4318 | 5079 | |
twisti@4318 | 5080 | void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { |
twisti@4318 | 5081 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 5082 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 5083 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5084 | int oop_index = oop_recorder()->find_index(obj); |
twisti@4318 | 5085 | RelocationHolder rspec = oop_Relocation::spec(oop_index); |
twisti@4318 | 5086 | mov_narrow_oop(dst, oop_index, rspec); |
twisti@4318 | 5087 | } |
twisti@4318 | 5088 | |
twisti@4318 | 5089 | void MacroAssembler::set_narrow_oop(Address dst, jobject obj) { |
twisti@4318 | 5090 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 5091 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 5092 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5093 | int oop_index = oop_recorder()->find_index(obj); |
twisti@4318 | 5094 | RelocationHolder rspec = oop_Relocation::spec(oop_index); |
twisti@4318 | 5095 | mov_narrow_oop(dst, oop_index, rspec); |
twisti@4318 | 5096 | } |
twisti@4318 | 5097 | |
twisti@4318 | 5098 | void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { |
twisti@4318 | 5099 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5100 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5101 | int klass_index = oop_recorder()->find_index(k); |
twisti@4318 | 5102 | RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
twisti@4318 | 5103 | mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec); |
twisti@4318 | 5104 | } |
twisti@4318 | 5105 | |
twisti@4318 | 5106 | void MacroAssembler::set_narrow_klass(Address dst, Klass* k) { |
twisti@4318 | 5107 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5108 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5109 | int klass_index = oop_recorder()->find_index(k); |
twisti@4318 | 5110 | RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
twisti@4318 | 5111 | mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec); |
twisti@4318 | 5112 | } |
twisti@4318 | 5113 | |
twisti@4318 | 5114 | void MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) { |
twisti@4318 | 5115 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 5116 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 5117 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5118 | int oop_index = oop_recorder()->find_index(obj); |
twisti@4318 | 5119 | RelocationHolder rspec = oop_Relocation::spec(oop_index); |
twisti@4318 | 5120 | Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
twisti@4318 | 5121 | } |
twisti@4318 | 5122 | |
twisti@4318 | 5123 | void MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) { |
twisti@4318 | 5124 | assert (UseCompressedOops, "should only be used for compressed headers"); |
twisti@4318 | 5125 | assert (Universe::heap() != NULL, "java heap should be initialized"); |
twisti@4318 | 5126 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5127 | int oop_index = oop_recorder()->find_index(obj); |
twisti@4318 | 5128 | RelocationHolder rspec = oop_Relocation::spec(oop_index); |
twisti@4318 | 5129 | Assembler::cmp_narrow_oop(dst, oop_index, rspec); |
twisti@4318 | 5130 | } |
twisti@4318 | 5131 | |
twisti@4318 | 5132 | void MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) { |
twisti@4318 | 5133 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5134 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5135 | int klass_index = oop_recorder()->find_index(k); |
twisti@4318 | 5136 | RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
twisti@4318 | 5137 | Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec); |
twisti@4318 | 5138 | } |
twisti@4318 | 5139 | |
twisti@4318 | 5140 | void MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) { |
twisti@4318 | 5141 | assert (UseCompressedKlassPointers, "should only be used for compressed headers"); |
twisti@4318 | 5142 | assert (oop_recorder() != NULL, "this assembler needs an OopRecorder"); |
twisti@4318 | 5143 | int klass_index = oop_recorder()->find_index(k); |
twisti@4318 | 5144 | RelocationHolder rspec = metadata_Relocation::spec(klass_index); |
twisti@4318 | 5145 | Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec); |
twisti@4318 | 5146 | } |
twisti@4318 | 5147 | |
twisti@4318 | 5148 | void MacroAssembler::reinit_heapbase() { |
twisti@4318 | 5149 | if (UseCompressedOops || UseCompressedKlassPointers) { |
twisti@4318 | 5150 | movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr())); |
twisti@4318 | 5151 | } |
twisti@4318 | 5152 | } |
twisti@4318 | 5153 | #endif // _LP64 |
twisti@4318 | 5154 | |
twisti@4318 | 5155 | |
twisti@4318 | 5156 | // C2 compiled method's prolog code. |
twisti@4318 | 5157 | void MacroAssembler::verified_entry(int framesize, bool stack_bang, bool fp_mode_24b) { |
twisti@4318 | 5158 | |
twisti@4318 | 5159 | // WARNING: Initial instruction MUST be 5 bytes or longer so that |
twisti@4318 | 5160 | // NativeJump::patch_verified_entry will be able to patch out the entry |
twisti@4318 | 5161 | // code safely. The push to verify stack depth is ok at 5 bytes, |
twisti@4318 | 5162 | // the frame allocation can be either 3 or 6 bytes. So if we don't do |
twisti@4318 | 5163 | // stack bang then we must use the 6 byte frame allocation even if |
twisti@4318 | 5164 | // we have no frame. :-( |
twisti@4318 | 5165 | |
twisti@4318 | 5166 | assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned"); |
twisti@4318 | 5167 | // Remove word for return addr |
twisti@4318 | 5168 | framesize -= wordSize; |
twisti@4318 | 5169 | |
twisti@4318 | 5170 | // Calls to C2R adapters often do not accept exceptional returns. |
twisti@4318 | 5171 | // We require that their callers must bang for them. But be careful, because |
twisti@4318 | 5172 | // some VM calls (such as call site linkage) can use several kilobytes of |
twisti@4318 | 5173 | // stack. But the stack safety zone should account for that. |
twisti@4318 | 5174 | // See bugs 4446381, 4468289, 4497237. |
twisti@4318 | 5175 | if (stack_bang) { |
twisti@4318 | 5176 | generate_stack_overflow_check(framesize); |
twisti@4318 | 5177 | |
twisti@4318 | 5178 | // We always push rbp, so that on return to interpreter rbp, will be |
twisti@4318 | 5179 | // restored correctly and we can correct the stack. |
twisti@4318 | 5180 | push(rbp); |
twisti@4318 | 5181 | // Remove word for ebp |
twisti@4318 | 5182 | framesize -= wordSize; |
twisti@4318 | 5183 | |
twisti@4318 | 5184 | // Create frame |
twisti@4318 | 5185 | if (framesize) { |
twisti@4318 | 5186 | subptr(rsp, framesize); |
twisti@4318 | 5187 | } |
twisti@4318 | 5188 | } else { |
twisti@4318 | 5189 | // Create frame (force generation of a 4 byte immediate value) |
twisti@4318 | 5190 | subptr_imm32(rsp, framesize); |
twisti@4318 | 5191 | |
twisti@4318 | 5192 | // Save RBP register now. |
twisti@4318 | 5193 | framesize -= wordSize; |
twisti@4318 | 5194 | movptr(Address(rsp, framesize), rbp); |
twisti@4318 | 5195 | } |
twisti@4318 | 5196 | |
twisti@4318 | 5197 | if (VerifyStackAtCalls) { // Majik cookie to verify stack depth |
twisti@4318 | 5198 | framesize -= wordSize; |
twisti@4318 | 5199 | movptr(Address(rsp, framesize), (int32_t)0xbadb100d); |
twisti@4318 | 5200 | } |
twisti@4318 | 5201 | |
twisti@4318 | 5202 | #ifndef _LP64 |
twisti@4318 | 5203 | // If method sets FPU control word do it now |
twisti@4318 | 5204 | if (fp_mode_24b) { |
twisti@4318 | 5205 | fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24())); |
twisti@4318 | 5206 | } |
twisti@4318 | 5207 | if (UseSSE >= 2 && VerifyFPU) { |
twisti@4318 | 5208 | verify_FPU(0, "FPU stack must be clean on entry"); |
twisti@4318 | 5209 | } |
twisti@4318 | 5210 | #endif |
twisti@4318 | 5211 | |
twisti@4318 | 5212 | #ifdef ASSERT |
twisti@4318 | 5213 | if (VerifyStackAtCalls) { |
twisti@4318 | 5214 | Label L; |
twisti@4318 | 5215 | push(rax); |
twisti@4318 | 5216 | mov(rax, rsp); |
twisti@4318 | 5217 | andptr(rax, StackAlignmentInBytes-1); |
twisti@4318 | 5218 | cmpptr(rax, StackAlignmentInBytes-wordSize); |
twisti@4318 | 5219 | pop(rax); |
twisti@4318 | 5220 | jcc(Assembler::equal, L); |
twisti@4318 | 5221 | STOP("Stack is not properly aligned!"); |
twisti@4318 | 5222 | bind(L); |
twisti@4318 | 5223 | } |
twisti@4318 | 5224 | #endif |
twisti@4318 | 5225 | |
twisti@4318 | 5226 | } |
twisti@4318 | 5227 | |
kvn@4410 | 5228 | void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp) { |
kvn@4410 | 5229 | // cnt - number of qwords (8-byte words). |
kvn@4410 | 5230 | // base - start address, qword aligned. |
kvn@4410 | 5231 | assert(base==rdi, "base register must be edi for rep stos"); |
kvn@4410 | 5232 | assert(tmp==rax, "tmp register must be eax for rep stos"); |
kvn@4410 | 5233 | assert(cnt==rcx, "cnt register must be ecx for rep stos"); |
kvn@4410 | 5234 | |
kvn@4410 | 5235 | xorptr(tmp, tmp); |
kvn@4410 | 5236 | if (UseFastStosb) { |
kvn@4410 | 5237 | shlptr(cnt,3); // convert to number of bytes |
kvn@4410 | 5238 | rep_stosb(); |
kvn@4410 | 5239 | } else { |
kvn@4410 | 5240 | NOT_LP64(shlptr(cnt,1);) // convert to number of dwords for 32-bit VM |
kvn@4410 | 5241 | rep_stos(); |
kvn@4410 | 5242 | } |
kvn@4410 | 5243 | } |
twisti@4318 | 5244 | |
twisti@4318 | 5245 | // IndexOf for constant substrings with size >= 8 chars |
twisti@4318 | 5246 | // which don't need to be loaded through stack. |
twisti@4318 | 5247 | void MacroAssembler::string_indexofC8(Register str1, Register str2, |
twisti@4318 | 5248 | Register cnt1, Register cnt2, |
twisti@4318 | 5249 | int int_cnt2, Register result, |
twisti@4318 | 5250 | XMMRegister vec, Register tmp) { |
twisti@4318 | 5251 | ShortBranchVerifier sbv(this); |
twisti@4318 | 5252 | assert(UseSSE42Intrinsics, "SSE4.2 is required"); |
twisti@4318 | 5253 | |
twisti@4318 | 5254 | // This method uses pcmpestri inxtruction with bound registers |
twisti@4318 | 5255 | // inputs: |
twisti@4318 | 5256 | // xmm - substring |
twisti@4318 | 5257 | // rax - substring length (elements count) |
twisti@4318 | 5258 | // mem - scanned string |
twisti@4318 | 5259 | // rdx - string length (elements count) |
twisti@4318 | 5260 | // 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
twisti@4318 | 5261 | // outputs: |
twisti@4318 | 5262 | // rcx - matched index in string |
twisti@4318 | 5263 | assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
twisti@4318 | 5264 | |
twisti@4318 | 5265 | Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, |
twisti@4318 | 5266 | RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR, |
twisti@4318 | 5267 | MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE; |
twisti@4318 | 5268 | |
twisti@4318 | 5269 | // Note, inline_string_indexOf() generates checks: |
twisti@4318 | 5270 | // if (substr.count > string.count) return -1; |
twisti@4318 | 5271 | // if (substr.count == 0) return 0; |
twisti@4318 | 5272 | assert(int_cnt2 >= 8, "this code isused only for cnt2 >= 8 chars"); |
twisti@4318 | 5273 | |
twisti@4318 | 5274 | // Load substring. |
twisti@4318 | 5275 | movdqu(vec, Address(str2, 0)); |
twisti@4318 | 5276 | movl(cnt2, int_cnt2); |
twisti@4318 | 5277 | movptr(result, str1); // string addr |
twisti@4318 | 5278 | |
twisti@4318 | 5279 | if (int_cnt2 > 8) { |
twisti@4318 | 5280 | jmpb(SCAN_TO_SUBSTR); |
twisti@4318 | 5281 | |
twisti@4318 | 5282 | // Reload substr for rescan, this code |
twisti@4318 | 5283 | // is executed only for large substrings (> 8 chars) |
twisti@4318 | 5284 | bind(RELOAD_SUBSTR); |
twisti@4318 | 5285 | movdqu(vec, Address(str2, 0)); |
twisti@4318 | 5286 | negptr(cnt2); // Jumped here with negative cnt2, convert to positive |
twisti@4318 | 5287 | |
twisti@4318 | 5288 | bind(RELOAD_STR); |
twisti@4318 | 5289 | // We came here after the beginning of the substring was |
twisti@4318 | 5290 | // matched but the rest of it was not so we need to search |
twisti@4318 | 5291 | // again. Start from the next element after the previous match. |
twisti@4318 | 5292 | |
twisti@4318 | 5293 | // cnt2 is number of substring reminding elements and |
twisti@4318 | 5294 | // cnt1 is number of string reminding elements when cmp failed. |
twisti@4318 | 5295 | // Restored cnt1 = cnt1 - cnt2 + int_cnt2 |
twisti@4318 | 5296 | subl(cnt1, cnt2); |
twisti@4318 | 5297 | addl(cnt1, int_cnt2); |
twisti@4318 | 5298 | movl(cnt2, int_cnt2); // Now restore cnt2 |
twisti@4318 | 5299 | |
twisti@4318 | 5300 | decrementl(cnt1); // Shift to next element |
twisti@4318 | 5301 | cmpl(cnt1, cnt2); |
twisti@4318 | 5302 | jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
twisti@4318 | 5303 | |
twisti@4318 | 5304 | addptr(result, 2); |
twisti@4318 | 5305 | |
twisti@4318 | 5306 | } // (int_cnt2 > 8) |
twisti@4318 | 5307 | |
twisti@4318 | 5308 | // Scan string for start of substr in 16-byte vectors |
twisti@4318 | 5309 | bind(SCAN_TO_SUBSTR); |
twisti@4318 | 5310 | pcmpestri(vec, Address(result, 0), 0x0d); |
twisti@4318 | 5311 | jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
twisti@4318 | 5312 | subl(cnt1, 8); |
twisti@4318 | 5313 | jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
twisti@4318 | 5314 | cmpl(cnt1, cnt2); |
twisti@4318 | 5315 | jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
twisti@4318 | 5316 | addptr(result, 16); |
twisti@4318 | 5317 | jmpb(SCAN_TO_SUBSTR); |
twisti@4318 | 5318 | |
twisti@4318 | 5319 | // Found a potential substr |
twisti@4318 | 5320 | bind(FOUND_CANDIDATE); |
twisti@4318 | 5321 | // Matched whole vector if first element matched (tmp(rcx) == 0). |
twisti@4318 | 5322 | if (int_cnt2 == 8) { |
twisti@4318 | 5323 | jccb(Assembler::overflow, RET_FOUND); // OF == 1 |
twisti@4318 | 5324 | } else { // int_cnt2 > 8 |
twisti@4318 | 5325 | jccb(Assembler::overflow, FOUND_SUBSTR); |
twisti@4318 | 5326 | } |
twisti@4318 | 5327 | // After pcmpestri tmp(rcx) contains matched element index |
twisti@4318 | 5328 | // Compute start addr of substr |
twisti@4318 | 5329 | lea(result, Address(result, tmp, Address::times_2)); |
twisti@4318 | 5330 | |
twisti@4318 | 5331 | // Make sure string is still long enough |
twisti@4318 | 5332 | subl(cnt1, tmp); |
twisti@4318 | 5333 | cmpl(cnt1, cnt2); |
twisti@4318 | 5334 | if (int_cnt2 == 8) { |
twisti@4318 | 5335 | jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
twisti@4318 | 5336 | } else { // int_cnt2 > 8 |
twisti@4318 | 5337 | jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD); |
twisti@4318 | 5338 | } |
twisti@4318 | 5339 | // Left less then substring. |
twisti@4318 | 5340 | |
twisti@4318 | 5341 | bind(RET_NOT_FOUND); |
twisti@4318 | 5342 | movl(result, -1); |
twisti@4318 | 5343 | jmpb(EXIT); |
twisti@4318 | 5344 | |
twisti@4318 | 5345 | if (int_cnt2 > 8) { |
twisti@4318 | 5346 | // This code is optimized for the case when whole substring |
twisti@4318 | 5347 | // is matched if its head is matched. |
twisti@4318 | 5348 | bind(MATCH_SUBSTR_HEAD); |
twisti@4318 | 5349 | pcmpestri(vec, Address(result, 0), 0x0d); |
twisti@4318 | 5350 | // Reload only string if does not match |
twisti@4318 | 5351 | jccb(Assembler::noOverflow, RELOAD_STR); // OF == 0 |
twisti@4318 | 5352 | |
twisti@4318 | 5353 | Label CONT_SCAN_SUBSTR; |
twisti@4318 | 5354 | // Compare the rest of substring (> 8 chars). |
twisti@4318 | 5355 | bind(FOUND_SUBSTR); |
twisti@4318 | 5356 | // First 8 chars are already matched. |
twisti@4318 | 5357 | negptr(cnt2); |
twisti@4318 | 5358 | addptr(cnt2, 8); |
twisti@4318 | 5359 | |
twisti@4318 | 5360 | bind(SCAN_SUBSTR); |
twisti@4318 | 5361 | subl(cnt1, 8); |
twisti@4318 | 5362 | cmpl(cnt2, -8); // Do not read beyond substring |
twisti@4318 | 5363 | jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR); |
twisti@4318 | 5364 | // Back-up strings to avoid reading beyond substring: |
twisti@4318 | 5365 | // cnt1 = cnt1 - cnt2 + 8 |
twisti@4318 | 5366 | addl(cnt1, cnt2); // cnt2 is negative |
twisti@4318 | 5367 | addl(cnt1, 8); |
twisti@4318 | 5368 | movl(cnt2, 8); negptr(cnt2); |
twisti@4318 | 5369 | bind(CONT_SCAN_SUBSTR); |
twisti@4318 | 5370 | if (int_cnt2 < (int)G) { |
twisti@4318 | 5371 | movdqu(vec, Address(str2, cnt2, Address::times_2, int_cnt2*2)); |
twisti@4318 | 5372 | pcmpestri(vec, Address(result, cnt2, Address::times_2, int_cnt2*2), 0x0d); |
twisti@4318 | 5373 | } else { |
twisti@4318 | 5374 | // calculate index in register to avoid integer overflow (int_cnt2*2) |
twisti@4318 | 5375 | movl(tmp, int_cnt2); |
twisti@4318 | 5376 | addptr(tmp, cnt2); |
twisti@4318 | 5377 | movdqu(vec, Address(str2, tmp, Address::times_2, 0)); |
twisti@4318 | 5378 | pcmpestri(vec, Address(result, tmp, Address::times_2, 0), 0x0d); |
twisti@4318 | 5379 | } |
twisti@4318 | 5380 | // Need to reload strings pointers if not matched whole vector |
twisti@4318 | 5381 | jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
twisti@4318 | 5382 | addptr(cnt2, 8); |
twisti@4318 | 5383 | jcc(Assembler::negative, SCAN_SUBSTR); |
twisti@4318 | 5384 | // Fall through if found full substring |
twisti@4318 | 5385 | |
twisti@4318 | 5386 | } // (int_cnt2 > 8) |
twisti@4318 | 5387 | |
twisti@4318 | 5388 | bind(RET_FOUND); |
twisti@4318 | 5389 | // Found result if we matched full small substring. |
twisti@4318 | 5390 | // Compute substr offset |
twisti@4318 | 5391 | subptr(result, str1); |
twisti@4318 | 5392 | shrl(result, 1); // index |
twisti@4318 | 5393 | bind(EXIT); |
twisti@4318 | 5394 | |
twisti@4318 | 5395 | } // string_indexofC8 |
twisti@4318 | 5396 | |
twisti@4318 | 5397 | // Small strings are loaded through stack if they cross page boundary. |
twisti@4318 | 5398 | void MacroAssembler::string_indexof(Register str1, Register str2, |
twisti@4318 | 5399 | Register cnt1, Register cnt2, |
twisti@4318 | 5400 | int int_cnt2, Register result, |
twisti@4318 | 5401 | XMMRegister vec, Register tmp) { |
twisti@4318 | 5402 | ShortBranchVerifier sbv(this); |
twisti@4318 | 5403 | assert(UseSSE42Intrinsics, "SSE4.2 is required"); |
twisti@4318 | 5404 | // |
twisti@4318 | 5405 | // int_cnt2 is length of small (< 8 chars) constant substring |
twisti@4318 | 5406 | // or (-1) for non constant substring in which case its length |
twisti@4318 | 5407 | // is in cnt2 register. |
twisti@4318 | 5408 | // |
twisti@4318 | 5409 | // Note, inline_string_indexOf() generates checks: |
twisti@4318 | 5410 | // if (substr.count > string.count) return -1; |
twisti@4318 | 5411 | // if (substr.count == 0) return 0; |
twisti@4318 | 5412 | // |
twisti@4318 | 5413 | assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < 8), "should be != 0"); |
twisti@4318 | 5414 | |
twisti@4318 | 5415 | // This method uses pcmpestri inxtruction with bound registers |
twisti@4318 | 5416 | // inputs: |
twisti@4318 | 5417 | // xmm - substring |
twisti@4318 | 5418 | // rax - substring length (elements count) |
twisti@4318 | 5419 | // mem - scanned string |
twisti@4318 | 5420 | // rdx - string length (elements count) |
twisti@4318 | 5421 | // 0xd - mode: 1100 (substring search) + 01 (unsigned shorts) |
twisti@4318 | 5422 | // outputs: |
twisti@4318 | 5423 | // rcx - matched index in string |
twisti@4318 | 5424 | assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
twisti@4318 | 5425 | |
twisti@4318 | 5426 | Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR, |
twisti@4318 | 5427 | RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR, |
twisti@4318 | 5428 | FOUND_CANDIDATE; |
twisti@4318 | 5429 | |
twisti@4318 | 5430 | { //======================================================== |
twisti@4318 | 5431 | // We don't know where these strings are located |
twisti@4318 | 5432 | // and we can't read beyond them. Load them through stack. |
twisti@4318 | 5433 | Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR; |
twisti@4318 | 5434 | |
twisti@4318 | 5435 | movptr(tmp, rsp); // save old SP |
twisti@4318 | 5436 | |
twisti@4318 | 5437 | if (int_cnt2 > 0) { // small (< 8 chars) constant substring |
twisti@4318 | 5438 | if (int_cnt2 == 1) { // One char |
twisti@4318 | 5439 | load_unsigned_short(result, Address(str2, 0)); |
twisti@4318 | 5440 | movdl(vec, result); // move 32 bits |
twisti@4318 | 5441 | } else if (int_cnt2 == 2) { // Two chars |
twisti@4318 | 5442 | movdl(vec, Address(str2, 0)); // move 32 bits |
twisti@4318 | 5443 | } else if (int_cnt2 == 4) { // Four chars |
twisti@4318 | 5444 | movq(vec, Address(str2, 0)); // move 64 bits |
twisti@4318 | 5445 | } else { // cnt2 = { 3, 5, 6, 7 } |
twisti@4318 | 5446 | // Array header size is 12 bytes in 32-bit VM |
twisti@4318 | 5447 | // + 6 bytes for 3 chars == 18 bytes, |
twisti@4318 | 5448 | // enough space to load vec and shift. |
twisti@4318 | 5449 | assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity"); |
twisti@4318 | 5450 | movdqu(vec, Address(str2, (int_cnt2*2)-16)); |
twisti@4318 | 5451 | psrldq(vec, 16-(int_cnt2*2)); |
twisti@4318 | 5452 | } |
twisti@4318 | 5453 | } else { // not constant substring |
twisti@4318 | 5454 | cmpl(cnt2, 8); |
twisti@4318 | 5455 | jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough |
twisti@4318 | 5456 | |
twisti@4318 | 5457 | // We can read beyond string if srt+16 does not cross page boundary |
twisti@4318 | 5458 | // since heaps are aligned and mapped by pages. |
twisti@4318 | 5459 | assert(os::vm_page_size() < (int)G, "default page should be small"); |
twisti@4318 | 5460 | movl(result, str2); // We need only low 32 bits |
twisti@4318 | 5461 | andl(result, (os::vm_page_size()-1)); |
twisti@4318 | 5462 | cmpl(result, (os::vm_page_size()-16)); |
twisti@4318 | 5463 | jccb(Assembler::belowEqual, CHECK_STR); |
twisti@4318 | 5464 | |
twisti@4318 | 5465 | // Move small strings to stack to allow load 16 bytes into vec. |
twisti@4318 | 5466 | subptr(rsp, 16); |
twisti@4318 | 5467 | int stk_offset = wordSize-2; |
twisti@4318 | 5468 | push(cnt2); |
twisti@4318 | 5469 | |
twisti@4318 | 5470 | bind(COPY_SUBSTR); |
twisti@4318 | 5471 | load_unsigned_short(result, Address(str2, cnt2, Address::times_2, -2)); |
twisti@4318 | 5472 | movw(Address(rsp, cnt2, Address::times_2, stk_offset), result); |
twisti@4318 | 5473 | decrement(cnt2); |
twisti@4318 | 5474 | jccb(Assembler::notZero, COPY_SUBSTR); |
twisti@4318 | 5475 | |
twisti@4318 | 5476 | pop(cnt2); |
twisti@4318 | 5477 | movptr(str2, rsp); // New substring address |
twisti@4318 | 5478 | } // non constant |
twisti@4318 | 5479 | |
twisti@4318 | 5480 | bind(CHECK_STR); |
twisti@4318 | 5481 | cmpl(cnt1, 8); |
twisti@4318 | 5482 | jccb(Assembler::aboveEqual, BIG_STRINGS); |
twisti@4318 | 5483 | |
twisti@4318 | 5484 | // Check cross page boundary. |
twisti@4318 | 5485 | movl(result, str1); // We need only low 32 bits |
twisti@4318 | 5486 | andl(result, (os::vm_page_size()-1)); |
twisti@4318 | 5487 | cmpl(result, (os::vm_page_size()-16)); |
twisti@4318 | 5488 | jccb(Assembler::belowEqual, BIG_STRINGS); |
twisti@4318 | 5489 | |
twisti@4318 | 5490 | subptr(rsp, 16); |
twisti@4318 | 5491 | int stk_offset = -2; |
twisti@4318 | 5492 | if (int_cnt2 < 0) { // not constant |
twisti@4318 | 5493 | push(cnt2); |
twisti@4318 | 5494 | stk_offset += wordSize; |
twisti@4318 | 5495 | } |
twisti@4318 | 5496 | movl(cnt2, cnt1); |
twisti@4318 | 5497 | |
twisti@4318 | 5498 | bind(COPY_STR); |
twisti@4318 | 5499 | load_unsigned_short(result, Address(str1, cnt2, Address::times_2, -2)); |
twisti@4318 | 5500 | movw(Address(rsp, cnt2, Address::times_2, stk_offset), result); |
twisti@4318 | 5501 | decrement(cnt2); |
twisti@4318 | 5502 | jccb(Assembler::notZero, COPY_STR); |
twisti@4318 | 5503 | |
twisti@4318 | 5504 | if (int_cnt2 < 0) { // not constant |
twisti@4318 | 5505 | pop(cnt2); |
twisti@4318 | 5506 | } |
twisti@4318 | 5507 | movptr(str1, rsp); // New string address |
twisti@4318 | 5508 | |
twisti@4318 | 5509 | bind(BIG_STRINGS); |
twisti@4318 | 5510 | // Load substring. |
twisti@4318 | 5511 | if (int_cnt2 < 0) { // -1 |
twisti@4318 | 5512 | movdqu(vec, Address(str2, 0)); |
twisti@4318 | 5513 | push(cnt2); // substr count |
twisti@4318 | 5514 | push(str2); // substr addr |
twisti@4318 | 5515 | push(str1); // string addr |
twisti@4318 | 5516 | } else { |
twisti@4318 | 5517 | // Small (< 8 chars) constant substrings are loaded already. |
twisti@4318 | 5518 | movl(cnt2, int_cnt2); |
twisti@4318 | 5519 | } |
twisti@4318 | 5520 | push(tmp); // original SP |
twisti@4318 | 5521 | |
twisti@4318 | 5522 | } // Finished loading |
twisti@4318 | 5523 | |
twisti@4318 | 5524 | //======================================================== |
twisti@4318 | 5525 | // Start search |
twisti@4318 | 5526 | // |
twisti@4318 | 5527 | |
twisti@4318 | 5528 | movptr(result, str1); // string addr |
twisti@4318 | 5529 | |
twisti@4318 | 5530 | if (int_cnt2 < 0) { // Only for non constant substring |
twisti@4318 | 5531 | jmpb(SCAN_TO_SUBSTR); |
twisti@4318 | 5532 | |
twisti@4318 | 5533 | // SP saved at sp+0 |
twisti@4318 | 5534 | // String saved at sp+1*wordSize |
twisti@4318 | 5535 | // Substr saved at sp+2*wordSize |
twisti@4318 | 5536 | // Substr count saved at sp+3*wordSize |
twisti@4318 | 5537 | |
twisti@4318 | 5538 | // Reload substr for rescan, this code |
twisti@4318 | 5539 | // is executed only for large substrings (> 8 chars) |
twisti@4318 | 5540 | bind(RELOAD_SUBSTR); |
twisti@4318 | 5541 | movptr(str2, Address(rsp, 2*wordSize)); |
twisti@4318 | 5542 | movl(cnt2, Address(rsp, 3*wordSize)); |
twisti@4318 | 5543 | movdqu(vec, Address(str2, 0)); |
twisti@4318 | 5544 | // We came here after the beginning of the substring was |
twisti@4318 | 5545 | // matched but the rest of it was not so we need to search |
twisti@4318 | 5546 | // again. Start from the next element after the previous match. |
twisti@4318 | 5547 | subptr(str1, result); // Restore counter |
twisti@4318 | 5548 | shrl(str1, 1); |
twisti@4318 | 5549 | addl(cnt1, str1); |
twisti@4318 | 5550 | decrementl(cnt1); // Shift to next element |
twisti@4318 | 5551 | cmpl(cnt1, cnt2); |
twisti@4318 | 5552 | jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
twisti@4318 | 5553 | |
twisti@4318 | 5554 | addptr(result, 2); |
twisti@4318 | 5555 | } // non constant |
twisti@4318 | 5556 | |
twisti@4318 | 5557 | // Scan string for start of substr in 16-byte vectors |
twisti@4318 | 5558 | bind(SCAN_TO_SUBSTR); |
twisti@4318 | 5559 | assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri"); |
twisti@4318 | 5560 | pcmpestri(vec, Address(result, 0), 0x0d); |
twisti@4318 | 5561 | jccb(Assembler::below, FOUND_CANDIDATE); // CF == 1 |
twisti@4318 | 5562 | subl(cnt1, 8); |
twisti@4318 | 5563 | jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string |
twisti@4318 | 5564 | cmpl(cnt1, cnt2); |
twisti@4318 | 5565 | jccb(Assembler::negative, RET_NOT_FOUND); // Left less then substring |
twisti@4318 | 5566 | addptr(result, 16); |
twisti@4318 | 5567 | |
twisti@4318 | 5568 | bind(ADJUST_STR); |
twisti@4318 | 5569 | cmpl(cnt1, 8); // Do not read beyond string |
twisti@4318 | 5570 | jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR); |
twisti@4318 | 5571 | // Back-up string to avoid reading beyond string. |
twisti@4318 | 5572 | lea(result, Address(result, cnt1, Address::times_2, -16)); |
twisti@4318 | 5573 | movl(cnt1, 8); |
twisti@4318 | 5574 | jmpb(SCAN_TO_SUBSTR); |
twisti@4318 | 5575 | |
twisti@4318 | 5576 | // Found a potential substr |
twisti@4318 | 5577 | bind(FOUND_CANDIDATE); |
twisti@4318 | 5578 | // After pcmpestri tmp(rcx) contains matched element index |
twisti@4318 | 5579 | |
twisti@4318 | 5580 | // Make sure string is still long enough |
twisti@4318 | 5581 | subl(cnt1, tmp); |
twisti@4318 | 5582 | cmpl(cnt1, cnt2); |
twisti@4318 | 5583 | jccb(Assembler::greaterEqual, FOUND_SUBSTR); |
twisti@4318 | 5584 | // Left less then substring. |
twisti@4318 | 5585 | |
twisti@4318 | 5586 | bind(RET_NOT_FOUND); |
twisti@4318 | 5587 | movl(result, -1); |
twisti@4318 | 5588 | jmpb(CLEANUP); |
twisti@4318 | 5589 | |
twisti@4318 | 5590 | bind(FOUND_SUBSTR); |
twisti@4318 | 5591 | // Compute start addr of substr |
twisti@4318 | 5592 | lea(result, Address(result, tmp, Address::times_2)); |
twisti@4318 | 5593 | |
twisti@4318 | 5594 | if (int_cnt2 > 0) { // Constant substring |
twisti@4318 | 5595 | // Repeat search for small substring (< 8 chars) |
twisti@4318 | 5596 | // from new point without reloading substring. |
twisti@4318 | 5597 | // Have to check that we don't read beyond string. |
twisti@4318 | 5598 | cmpl(tmp, 8-int_cnt2); |
twisti@4318 | 5599 | jccb(Assembler::greater, ADJUST_STR); |
twisti@4318 | 5600 | // Fall through if matched whole substring. |
twisti@4318 | 5601 | } else { // non constant |
twisti@4318 | 5602 | assert(int_cnt2 == -1, "should be != 0"); |
twisti@4318 | 5603 | |
twisti@4318 | 5604 | addl(tmp, cnt2); |
twisti@4318 | 5605 | // Found result if we matched whole substring. |
twisti@4318 | 5606 | cmpl(tmp, 8); |
twisti@4318 | 5607 | jccb(Assembler::lessEqual, RET_FOUND); |
twisti@4318 | 5608 | |
twisti@4318 | 5609 | // Repeat search for small substring (<= 8 chars) |
twisti@4318 | 5610 | // from new point 'str1' without reloading substring. |
twisti@4318 | 5611 | cmpl(cnt2, 8); |
twisti@4318 | 5612 | // Have to check that we don't read beyond string. |
twisti@4318 | 5613 | jccb(Assembler::lessEqual, ADJUST_STR); |
twisti@4318 | 5614 | |
twisti@4318 | 5615 | Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG; |
twisti@4318 | 5616 | // Compare the rest of substring (> 8 chars). |
twisti@4318 | 5617 | movptr(str1, result); |
twisti@4318 | 5618 | |
twisti@4318 | 5619 | cmpl(tmp, cnt2); |
twisti@4318 | 5620 | // First 8 chars are already matched. |
twisti@4318 | 5621 | jccb(Assembler::equal, CHECK_NEXT); |
twisti@4318 | 5622 | |
twisti@4318 | 5623 | bind(SCAN_SUBSTR); |
twisti@4318 | 5624 | pcmpestri(vec, Address(str1, 0), 0x0d); |
twisti@4318 | 5625 | // Need to reload strings pointers if not matched whole vector |
twisti@4318 | 5626 | jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0 |
twisti@4318 | 5627 | |
twisti@4318 | 5628 | bind(CHECK_NEXT); |
twisti@4318 | 5629 | subl(cnt2, 8); |
twisti@4318 | 5630 | jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring |
twisti@4318 | 5631 | addptr(str1, 16); |
twisti@4318 | 5632 | addptr(str2, 16); |
twisti@4318 | 5633 | subl(cnt1, 8); |
twisti@4318 | 5634 | cmpl(cnt2, 8); // Do not read beyond substring |
twisti@4318 | 5635 | jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR); |
twisti@4318 | 5636 | // Back-up strings to avoid reading beyond substring. |
twisti@4318 | 5637 | lea(str2, Address(str2, cnt2, Address::times_2, -16)); |
twisti@4318 | 5638 | lea(str1, Address(str1, cnt2, Address::times_2, -16)); |
twisti@4318 | 5639 | subl(cnt1, cnt2); |
twisti@4318 | 5640 | movl(cnt2, 8); |
twisti@4318 | 5641 | addl(cnt1, 8); |
twisti@4318 | 5642 | bind(CONT_SCAN_SUBSTR); |
twisti@4318 | 5643 | movdqu(vec, Address(str2, 0)); |
twisti@4318 | 5644 | jmpb(SCAN_SUBSTR); |
twisti@4318 | 5645 | |
twisti@4318 | 5646 | bind(RET_FOUND_LONG); |
twisti@4318 | 5647 | movptr(str1, Address(rsp, wordSize)); |
twisti@4318 | 5648 | } // non constant |
twisti@4318 | 5649 | |
twisti@4318 | 5650 | bind(RET_FOUND); |
twisti@4318 | 5651 | // Compute substr offset |
twisti@4318 | 5652 | subptr(result, str1); |
twisti@4318 | 5653 | shrl(result, 1); // index |
twisti@4318 | 5654 | |
twisti@4318 | 5655 | bind(CLEANUP); |
twisti@4318 | 5656 | pop(rsp); // restore SP |
twisti@4318 | 5657 | |
twisti@4318 | 5658 | } // string_indexof |
twisti@4318 | 5659 | |
twisti@4318 | 5660 | // Compare strings. |
twisti@4318 | 5661 | void MacroAssembler::string_compare(Register str1, Register str2, |
twisti@4318 | 5662 | Register cnt1, Register cnt2, Register result, |
twisti@4318 | 5663 | XMMRegister vec1) { |
twisti@4318 | 5664 | ShortBranchVerifier sbv(this); |
twisti@4318 | 5665 | Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL; |
twisti@4318 | 5666 | |
twisti@4318 | 5667 | // Compute the minimum of the string lengths and the |
twisti@4318 | 5668 | // difference of the string lengths (stack). |
twisti@4318 | 5669 | // Do the conditional move stuff |
twisti@4318 | 5670 | movl(result, cnt1); |
twisti@4318 | 5671 | subl(cnt1, cnt2); |
twisti@4318 | 5672 | push(cnt1); |
twisti@4318 | 5673 | cmov32(Assembler::lessEqual, cnt2, result); |
twisti@4318 | 5674 | |
twisti@4318 | 5675 | // Is the minimum length zero? |
twisti@4318 | 5676 | testl(cnt2, cnt2); |
twisti@4318 | 5677 | jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
twisti@4318 | 5678 | |
kvn@4413 | 5679 | // Compare first characters |
twisti@4318 | 5680 | load_unsigned_short(result, Address(str1, 0)); |
twisti@4318 | 5681 | load_unsigned_short(cnt1, Address(str2, 0)); |
twisti@4318 | 5682 | subl(result, cnt1); |
twisti@4318 | 5683 | jcc(Assembler::notZero, POP_LABEL); |
kvn@4413 | 5684 | cmpl(cnt2, 1); |
kvn@4413 | 5685 | jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
kvn@4413 | 5686 | |
kvn@4413 | 5687 | // Check if the strings start at the same location. |
kvn@4413 | 5688 | cmpptr(str1, str2); |
kvn@4413 | 5689 | jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
twisti@4318 | 5690 | |
twisti@4318 | 5691 | Address::ScaleFactor scale = Address::times_2; |
twisti@4318 | 5692 | int stride = 8; |
twisti@4318 | 5693 | |
kvn@4413 | 5694 | if (UseAVX >= 2) { |
kvn@4413 | 5695 | Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR; |
kvn@4413 | 5696 | Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR; |
kvn@4413 | 5697 | Label COMPARE_TAIL_LONG; |
kvn@4413 | 5698 | int pcmpmask = 0x19; |
kvn@4413 | 5699 | |
kvn@4413 | 5700 | // Setup to compare 16-chars (32-bytes) vectors, |
kvn@4413 | 5701 | // start from first character again because it has aligned address. |
kvn@4413 | 5702 | int stride2 = 16; |
kvn@4413 | 5703 | int adr_stride = stride << scale; |
kvn@4413 | 5704 | int adr_stride2 = stride2 << scale; |
kvn@4413 | 5705 | |
kvn@4413 | 5706 | assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
kvn@4413 | 5707 | // rax and rdx are used by pcmpestri as elements counters |
kvn@4413 | 5708 | movl(result, cnt2); |
kvn@4413 | 5709 | andl(cnt2, ~(stride2-1)); // cnt2 holds the vector count |
kvn@4413 | 5710 | jcc(Assembler::zero, COMPARE_TAIL_LONG); |
kvn@4413 | 5711 | |
kvn@4413 | 5712 | // fast path : compare first 2 8-char vectors. |
kvn@4413 | 5713 | bind(COMPARE_16_CHARS); |
kvn@4413 | 5714 | movdqu(vec1, Address(str1, 0)); |
kvn@4413 | 5715 | pcmpestri(vec1, Address(str2, 0), pcmpmask); |
kvn@4413 | 5716 | jccb(Assembler::below, COMPARE_INDEX_CHAR); |
kvn@4413 | 5717 | |
kvn@4413 | 5718 | movdqu(vec1, Address(str1, adr_stride)); |
kvn@4413 | 5719 | pcmpestri(vec1, Address(str2, adr_stride), pcmpmask); |
kvn@4413 | 5720 | jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS); |
kvn@4413 | 5721 | addl(cnt1, stride); |
kvn@4413 | 5722 | |
kvn@4413 | 5723 | // Compare the characters at index in cnt1 |
kvn@4413 | 5724 | bind(COMPARE_INDEX_CHAR); //cnt1 has the offset of the mismatching character |
kvn@4413 | 5725 | load_unsigned_short(result, Address(str1, cnt1, scale)); |
kvn@4413 | 5726 | load_unsigned_short(cnt2, Address(str2, cnt1, scale)); |
kvn@4413 | 5727 | subl(result, cnt2); |
kvn@4413 | 5728 | jmp(POP_LABEL); |
kvn@4413 | 5729 | |
kvn@4413 | 5730 | // Setup the registers to start vector comparison loop |
kvn@4413 | 5731 | bind(COMPARE_WIDE_VECTORS); |
kvn@4413 | 5732 | lea(str1, Address(str1, result, scale)); |
kvn@4413 | 5733 | lea(str2, Address(str2, result, scale)); |
kvn@4413 | 5734 | subl(result, stride2); |
kvn@4413 | 5735 | subl(cnt2, stride2); |
kvn@4413 | 5736 | jccb(Assembler::zero, COMPARE_WIDE_TAIL); |
kvn@4413 | 5737 | negptr(result); |
kvn@4413 | 5738 | |
kvn@4413 | 5739 | // In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest) |
kvn@4413 | 5740 | bind(COMPARE_WIDE_VECTORS_LOOP); |
kvn@4413 | 5741 | vmovdqu(vec1, Address(str1, result, scale)); |
kvn@4413 | 5742 | vpxor(vec1, Address(str2, result, scale)); |
kvn@4413 | 5743 | vptest(vec1, vec1); |
kvn@4413 | 5744 | jccb(Assembler::notZero, VECTOR_NOT_EQUAL); |
kvn@4413 | 5745 | addptr(result, stride2); |
kvn@4413 | 5746 | subl(cnt2, stride2); |
kvn@4413 | 5747 | jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP); |
kvn@4413 | 5748 | |
kvn@4413 | 5749 | // compare wide vectors tail |
kvn@4413 | 5750 | bind(COMPARE_WIDE_TAIL); |
kvn@4413 | 5751 | testptr(result, result); |
kvn@4413 | 5752 | jccb(Assembler::zero, LENGTH_DIFF_LABEL); |
kvn@4413 | 5753 | |
kvn@4413 | 5754 | movl(result, stride2); |
kvn@4413 | 5755 | movl(cnt2, result); |
kvn@4413 | 5756 | negptr(result); |
kvn@4413 | 5757 | jmpb(COMPARE_WIDE_VECTORS_LOOP); |
kvn@4413 | 5758 | |
kvn@4413 | 5759 | // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors. |
kvn@4413 | 5760 | bind(VECTOR_NOT_EQUAL); |
kvn@4413 | 5761 | lea(str1, Address(str1, result, scale)); |
kvn@4413 | 5762 | lea(str2, Address(str2, result, scale)); |
kvn@4413 | 5763 | jmp(COMPARE_16_CHARS); |
kvn@4413 | 5764 | |
kvn@4413 | 5765 | // Compare tail chars, length between 1 to 15 chars |
kvn@4413 | 5766 | bind(COMPARE_TAIL_LONG); |
kvn@4413 | 5767 | movl(cnt2, result); |
kvn@4413 | 5768 | cmpl(cnt2, stride); |
kvn@4413 | 5769 | jccb(Assembler::less, COMPARE_SMALL_STR); |
kvn@4413 | 5770 | |
kvn@4413 | 5771 | movdqu(vec1, Address(str1, 0)); |
kvn@4413 | 5772 | pcmpestri(vec1, Address(str2, 0), pcmpmask); |
kvn@4413 | 5773 | jcc(Assembler::below, COMPARE_INDEX_CHAR); |
kvn@4413 | 5774 | subptr(cnt2, stride); |
kvn@4413 | 5775 | jccb(Assembler::zero, LENGTH_DIFF_LABEL); |
kvn@4413 | 5776 | lea(str1, Address(str1, result, scale)); |
kvn@4413 | 5777 | lea(str2, Address(str2, result, scale)); |
kvn@4413 | 5778 | negptr(cnt2); |
kvn@4413 | 5779 | jmpb(WHILE_HEAD_LABEL); |
kvn@4413 | 5780 | |
kvn@4413 | 5781 | bind(COMPARE_SMALL_STR); |
kvn@4413 | 5782 | } else if (UseSSE42Intrinsics) { |
twisti@4318 | 5783 | Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL; |
twisti@4318 | 5784 | int pcmpmask = 0x19; |
kvn@4413 | 5785 | // Setup to compare 8-char (16-byte) vectors, |
kvn@4413 | 5786 | // start from first character again because it has aligned address. |
twisti@4318 | 5787 | movl(result, cnt2); |
twisti@4318 | 5788 | andl(cnt2, ~(stride - 1)); // cnt2 holds the vector count |
twisti@4318 | 5789 | jccb(Assembler::zero, COMPARE_TAIL); |
twisti@4318 | 5790 | |
twisti@4318 | 5791 | lea(str1, Address(str1, result, scale)); |
twisti@4318 | 5792 | lea(str2, Address(str2, result, scale)); |
twisti@4318 | 5793 | negptr(result); |
twisti@4318 | 5794 | |
twisti@4318 | 5795 | // pcmpestri |
twisti@4318 | 5796 | // inputs: |
twisti@4318 | 5797 | // vec1- substring |
twisti@4318 | 5798 | // rax - negative string length (elements count) |
twisti@4318 | 5799 | // mem - scaned string |
twisti@4318 | 5800 | // rdx - string length (elements count) |
twisti@4318 | 5801 | // pcmpmask - cmp mode: 11000 (string compare with negated result) |
twisti@4318 | 5802 | // + 00 (unsigned bytes) or + 01 (unsigned shorts) |
twisti@4318 | 5803 | // outputs: |
twisti@4318 | 5804 | // rcx - first mismatched element index |
twisti@4318 | 5805 | assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri"); |
twisti@4318 | 5806 | |
twisti@4318 | 5807 | bind(COMPARE_WIDE_VECTORS); |
twisti@4318 | 5808 | movdqu(vec1, Address(str1, result, scale)); |
twisti@4318 | 5809 | pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
twisti@4318 | 5810 | // After pcmpestri cnt1(rcx) contains mismatched element index |
twisti@4318 | 5811 | |
twisti@4318 | 5812 | jccb(Assembler::below, VECTOR_NOT_EQUAL); // CF==1 |
twisti@4318 | 5813 | addptr(result, stride); |
twisti@4318 | 5814 | subptr(cnt2, stride); |
twisti@4318 | 5815 | jccb(Assembler::notZero, COMPARE_WIDE_VECTORS); |
twisti@4318 | 5816 | |
twisti@4318 | 5817 | // compare wide vectors tail |
kvn@4413 | 5818 | testptr(result, result); |
twisti@4318 | 5819 | jccb(Assembler::zero, LENGTH_DIFF_LABEL); |
twisti@4318 | 5820 | |
twisti@4318 | 5821 | movl(cnt2, stride); |
twisti@4318 | 5822 | movl(result, stride); |
twisti@4318 | 5823 | negptr(result); |
twisti@4318 | 5824 | movdqu(vec1, Address(str1, result, scale)); |
twisti@4318 | 5825 | pcmpestri(vec1, Address(str2, result, scale), pcmpmask); |
twisti@4318 | 5826 | jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL); |
twisti@4318 | 5827 | |
twisti@4318 | 5828 | // Mismatched characters in the vectors |
twisti@4318 | 5829 | bind(VECTOR_NOT_EQUAL); |
kvn@4413 | 5830 | addptr(cnt1, result); |
kvn@4413 | 5831 | load_unsigned_short(result, Address(str1, cnt1, scale)); |
kvn@4413 | 5832 | load_unsigned_short(cnt2, Address(str2, cnt1, scale)); |
kvn@4413 | 5833 | subl(result, cnt2); |
twisti@4318 | 5834 | jmpb(POP_LABEL); |
twisti@4318 | 5835 | |
twisti@4318 | 5836 | bind(COMPARE_TAIL); // limit is zero |
twisti@4318 | 5837 | movl(cnt2, result); |
twisti@4318 | 5838 | // Fallthru to tail compare |
twisti@4318 | 5839 | } |
twisti@4318 | 5840 | // Shift str2 and str1 to the end of the arrays, negate min |
kvn@4413 | 5841 | lea(str1, Address(str1, cnt2, scale)); |
kvn@4413 | 5842 | lea(str2, Address(str2, cnt2, scale)); |
kvn@4413 | 5843 | decrementl(cnt2); // first character was compared already |
twisti@4318 | 5844 | negptr(cnt2); |
twisti@4318 | 5845 | |
twisti@4318 | 5846 | // Compare the rest of the elements |
twisti@4318 | 5847 | bind(WHILE_HEAD_LABEL); |
twisti@4318 | 5848 | load_unsigned_short(result, Address(str1, cnt2, scale, 0)); |
twisti@4318 | 5849 | load_unsigned_short(cnt1, Address(str2, cnt2, scale, 0)); |
twisti@4318 | 5850 | subl(result, cnt1); |
twisti@4318 | 5851 | jccb(Assembler::notZero, POP_LABEL); |
twisti@4318 | 5852 | increment(cnt2); |
twisti@4318 | 5853 | jccb(Assembler::notZero, WHILE_HEAD_LABEL); |
twisti@4318 | 5854 | |
twisti@4318 | 5855 | // Strings are equal up to min length. Return the length difference. |
twisti@4318 | 5856 | bind(LENGTH_DIFF_LABEL); |
twisti@4318 | 5857 | pop(result); |
twisti@4318 | 5858 | jmpb(DONE_LABEL); |
twisti@4318 | 5859 | |
twisti@4318 | 5860 | // Discard the stored length difference |
twisti@4318 | 5861 | bind(POP_LABEL); |
twisti@4318 | 5862 | pop(cnt1); |
twisti@4318 | 5863 | |
twisti@4318 | 5864 | // That's it |
twisti@4318 | 5865 | bind(DONE_LABEL); |
twisti@4318 | 5866 | } |
twisti@4318 | 5867 | |
twisti@4318 | 5868 | // Compare char[] arrays aligned to 4 bytes or substrings. |
twisti@4318 | 5869 | void MacroAssembler::char_arrays_equals(bool is_array_equ, Register ary1, Register ary2, |
twisti@4318 | 5870 | Register limit, Register result, Register chr, |
twisti@4318 | 5871 | XMMRegister vec1, XMMRegister vec2) { |
twisti@4318 | 5872 | ShortBranchVerifier sbv(this); |
twisti@4318 | 5873 | Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR; |
twisti@4318 | 5874 | |
twisti@4318 | 5875 | int length_offset = arrayOopDesc::length_offset_in_bytes(); |
twisti@4318 | 5876 | int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); |
twisti@4318 | 5877 | |
twisti@4318 | 5878 | // Check the input args |
twisti@4318 | 5879 | cmpptr(ary1, ary2); |
twisti@4318 | 5880 | jcc(Assembler::equal, TRUE_LABEL); |
twisti@4318 | 5881 | |
twisti@4318 | 5882 | if (is_array_equ) { |
twisti@4318 | 5883 | // Need additional checks for arrays_equals. |
twisti@4318 | 5884 | testptr(ary1, ary1); |
twisti@4318 | 5885 | jcc(Assembler::zero, FALSE_LABEL); |
twisti@4318 | 5886 | testptr(ary2, ary2); |
twisti@4318 | 5887 | jcc(Assembler::zero, FALSE_LABEL); |
twisti@4318 | 5888 | |
twisti@4318 | 5889 | // Check the lengths |
twisti@4318 | 5890 | movl(limit, Address(ary1, length_offset)); |
twisti@4318 | 5891 | cmpl(limit, Address(ary2, length_offset)); |
twisti@4318 | 5892 | jcc(Assembler::notEqual, FALSE_LABEL); |
twisti@4318 | 5893 | } |
twisti@4318 | 5894 | |
twisti@4318 | 5895 | // count == 0 |
twisti@4318 | 5896 | testl(limit, limit); |
twisti@4318 | 5897 | jcc(Assembler::zero, TRUE_LABEL); |
twisti@4318 | 5898 | |
twisti@4318 | 5899 | if (is_array_equ) { |
twisti@4318 | 5900 | // Load array address |
twisti@4318 | 5901 | lea(ary1, Address(ary1, base_offset)); |
twisti@4318 | 5902 | lea(ary2, Address(ary2, base_offset)); |
twisti@4318 | 5903 | } |
twisti@4318 | 5904 | |
twisti@4318 | 5905 | shll(limit, 1); // byte count != 0 |
twisti@4318 | 5906 | movl(result, limit); // copy |
twisti@4318 | 5907 | |
kvn@4413 | 5908 | if (UseAVX >= 2) { |
kvn@4413 | 5909 | // With AVX2, use 32-byte vector compare |
kvn@4413 | 5910 | Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
kvn@4413 | 5911 | |
kvn@4413 | 5912 | // Compare 32-byte vectors |
kvn@4413 | 5913 | andl(result, 0x0000001e); // tail count (in bytes) |
kvn@4413 | 5914 | andl(limit, 0xffffffe0); // vector count (in bytes) |
kvn@4413 | 5915 | jccb(Assembler::zero, COMPARE_TAIL); |
kvn@4413 | 5916 | |
kvn@4413 | 5917 | lea(ary1, Address(ary1, limit, Address::times_1)); |
kvn@4413 | 5918 | lea(ary2, Address(ary2, limit, Address::times_1)); |
kvn@4413 | 5919 | negptr(limit); |
kvn@4413 | 5920 | |
kvn@4413 | 5921 | bind(COMPARE_WIDE_VECTORS); |
kvn@4413 | 5922 | vmovdqu(vec1, Address(ary1, limit, Address::times_1)); |
kvn@4413 | 5923 | vmovdqu(vec2, Address(ary2, limit, Address::times_1)); |
kvn@4413 | 5924 | vpxor(vec1, vec2); |
kvn@4413 | 5925 | |
kvn@4413 | 5926 | vptest(vec1, vec1); |
kvn@4413 | 5927 | jccb(Assembler::notZero, FALSE_LABEL); |
kvn@4413 | 5928 | addptr(limit, 32); |
kvn@4413 | 5929 | jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
kvn@4413 | 5930 | |
kvn@4413 | 5931 | testl(result, result); |
kvn@4413 | 5932 | jccb(Assembler::zero, TRUE_LABEL); |
kvn@4413 | 5933 | |
kvn@4413 | 5934 | vmovdqu(vec1, Address(ary1, result, Address::times_1, -32)); |
kvn@4413 | 5935 | vmovdqu(vec2, Address(ary2, result, Address::times_1, -32)); |
kvn@4413 | 5936 | vpxor(vec1, vec2); |
kvn@4413 | 5937 | |
kvn@4413 | 5938 | vptest(vec1, vec1); |
kvn@4413 | 5939 | jccb(Assembler::notZero, FALSE_LABEL); |
kvn@4413 | 5940 | jmpb(TRUE_LABEL); |
kvn@4413 | 5941 | |
kvn@4413 | 5942 | bind(COMPARE_TAIL); // limit is zero |
kvn@4413 | 5943 | movl(limit, result); |
kvn@4413 | 5944 | // Fallthru to tail compare |
kvn@4413 | 5945 | } else if (UseSSE42Intrinsics) { |
twisti@4318 | 5946 | // With SSE4.2, use double quad vector compare |
twisti@4318 | 5947 | Label COMPARE_WIDE_VECTORS, COMPARE_TAIL; |
twisti@4318 | 5948 | |
twisti@4318 | 5949 | // Compare 16-byte vectors |
twisti@4318 | 5950 | andl(result, 0x0000000e); // tail count (in bytes) |
twisti@4318 | 5951 | andl(limit, 0xfffffff0); // vector count (in bytes) |
twisti@4318 | 5952 | jccb(Assembler::zero, COMPARE_TAIL); |
twisti@4318 | 5953 | |
twisti@4318 | 5954 | lea(ary1, Address(ary1, limit, Address::times_1)); |
twisti@4318 | 5955 | lea(ary2, Address(ary2, limit, Address::times_1)); |
twisti@4318 | 5956 | negptr(limit); |
twisti@4318 | 5957 | |
twisti@4318 | 5958 | bind(COMPARE_WIDE_VECTORS); |
twisti@4318 | 5959 | movdqu(vec1, Address(ary1, limit, Address::times_1)); |
twisti@4318 | 5960 | movdqu(vec2, Address(ary2, limit, Address::times_1)); |
twisti@4318 | 5961 | pxor(vec1, vec2); |
twisti@4318 | 5962 | |
twisti@4318 | 5963 | ptest(vec1, vec1); |
twisti@4318 | 5964 | jccb(Assembler::notZero, FALSE_LABEL); |
twisti@4318 | 5965 | addptr(limit, 16); |
twisti@4318 | 5966 | jcc(Assembler::notZero, COMPARE_WIDE_VECTORS); |
twisti@4318 | 5967 | |
twisti@4318 | 5968 | testl(result, result); |
twisti@4318 | 5969 | jccb(Assembler::zero, TRUE_LABEL); |
twisti@4318 | 5970 | |
twisti@4318 | 5971 | movdqu(vec1, Address(ary1, result, Address::times_1, -16)); |
twisti@4318 | 5972 | movdqu(vec2, Address(ary2, result, Address::times_1, -16)); |
twisti@4318 | 5973 | pxor(vec1, vec2); |
twisti@4318 | 5974 | |
twisti@4318 | 5975 | ptest(vec1, vec1); |
twisti@4318 | 5976 | jccb(Assembler::notZero, FALSE_LABEL); |
twisti@4318 | 5977 | jmpb(TRUE_LABEL); |
twisti@4318 | 5978 | |
twisti@4318 | 5979 | bind(COMPARE_TAIL); // limit is zero |
twisti@4318 | 5980 | movl(limit, result); |
twisti@4318 | 5981 | // Fallthru to tail compare |
twisti@4318 | 5982 | } |
twisti@4318 | 5983 | |
twisti@4318 | 5984 | // Compare 4-byte vectors |
twisti@4318 | 5985 | andl(limit, 0xfffffffc); // vector count (in bytes) |
twisti@4318 | 5986 | jccb(Assembler::zero, COMPARE_CHAR); |
twisti@4318 | 5987 | |
twisti@4318 | 5988 | lea(ary1, Address(ary1, limit, Address::times_1)); |
twisti@4318 | 5989 | lea(ary2, Address(ary2, limit, Address::times_1)); |
twisti@4318 | 5990 | negptr(limit); |
twisti@4318 | 5991 | |
twisti@4318 | 5992 | bind(COMPARE_VECTORS); |
twisti@4318 | 5993 | movl(chr, Address(ary1, limit, Address::times_1)); |
twisti@4318 | 5994 | cmpl(chr, Address(ary2, limit, Address::times_1)); |
twisti@4318 | 5995 | jccb(Assembler::notEqual, FALSE_LABEL); |
twisti@4318 | 5996 | addptr(limit, 4); |
twisti@4318 | 5997 | jcc(Assembler::notZero, COMPARE_VECTORS); |
twisti@4318 | 5998 | |
twisti@4318 | 5999 | // Compare trailing char (final 2 bytes), if any |
twisti@4318 | 6000 | bind(COMPARE_CHAR); |
twisti@4318 | 6001 | testl(result, 0x2); // tail char |
twisti@4318 | 6002 | jccb(Assembler::zero, TRUE_LABEL); |
twisti@4318 | 6003 | load_unsigned_short(chr, Address(ary1, 0)); |
twisti@4318 | 6004 | load_unsigned_short(limit, Address(ary2, 0)); |
twisti@4318 | 6005 | cmpl(chr, limit); |
twisti@4318 | 6006 | jccb(Assembler::notEqual, FALSE_LABEL); |
twisti@4318 | 6007 | |
twisti@4318 | 6008 | bind(TRUE_LABEL); |
twisti@4318 | 6009 | movl(result, 1); // return true |
twisti@4318 | 6010 | jmpb(DONE); |
twisti@4318 | 6011 | |
twisti@4318 | 6012 | bind(FALSE_LABEL); |
twisti@4318 | 6013 | xorl(result, result); // return false |
twisti@4318 | 6014 | |
twisti@4318 | 6015 | // That's it |
twisti@4318 | 6016 | bind(DONE); |
twisti@4318 | 6017 | } |
twisti@4318 | 6018 | |
twisti@4318 | 6019 | void MacroAssembler::generate_fill(BasicType t, bool aligned, |
twisti@4318 | 6020 | Register to, Register value, Register count, |
twisti@4318 | 6021 | Register rtmp, XMMRegister xtmp) { |
twisti@4318 | 6022 | ShortBranchVerifier sbv(this); |
twisti@4318 | 6023 | assert_different_registers(to, value, count, rtmp); |
twisti@4318 | 6024 | Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte; |
twisti@4318 | 6025 | Label L_fill_2_bytes, L_fill_4_bytes; |
twisti@4318 | 6026 | |
twisti@4318 | 6027 | int shift = -1; |
twisti@4318 | 6028 | switch (t) { |
twisti@4318 | 6029 | case T_BYTE: |
twisti@4318 | 6030 | shift = 2; |
twisti@4318 | 6031 | break; |
twisti@4318 | 6032 | case T_SHORT: |
twisti@4318 | 6033 | shift = 1; |
twisti@4318 | 6034 | break; |
twisti@4318 | 6035 | case T_INT: |
twisti@4318 | 6036 | shift = 0; |
twisti@4318 | 6037 | break; |
twisti@4318 | 6038 | default: ShouldNotReachHere(); |
twisti@4318 | 6039 | } |
twisti@4318 | 6040 | |
twisti@4318 | 6041 | if (t == T_BYTE) { |
twisti@4318 | 6042 | andl(value, 0xff); |
twisti@4318 | 6043 | movl(rtmp, value); |
twisti@4318 | 6044 | shll(rtmp, 8); |
twisti@4318 | 6045 | orl(value, rtmp); |
twisti@4318 | 6046 | } |
twisti@4318 | 6047 | if (t == T_SHORT) { |
twisti@4318 | 6048 | andl(value, 0xffff); |
twisti@4318 | 6049 | } |
twisti@4318 | 6050 | if (t == T_BYTE || t == T_SHORT) { |
twisti@4318 | 6051 | movl(rtmp, value); |
twisti@4318 | 6052 | shll(rtmp, 16); |
twisti@4318 | 6053 | orl(value, rtmp); |
twisti@4318 | 6054 | } |
twisti@4318 | 6055 | |
twisti@4318 | 6056 | cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element |
twisti@4318 | 6057 | jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp |
twisti@4318 | 6058 | if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) { |
twisti@4318 | 6059 | // align source address at 4 bytes address boundary |
twisti@4318 | 6060 | if (t == T_BYTE) { |
twisti@4318 | 6061 | // One byte misalignment happens only for byte arrays |
twisti@4318 | 6062 | testptr(to, 1); |
twisti@4318 | 6063 | jccb(Assembler::zero, L_skip_align1); |
twisti@4318 | 6064 | movb(Address(to, 0), value); |
twisti@4318 | 6065 | increment(to); |
twisti@4318 | 6066 | decrement(count); |
twisti@4318 | 6067 | BIND(L_skip_align1); |
twisti@4318 | 6068 | } |
twisti@4318 | 6069 | // Two bytes misalignment happens only for byte and short (char) arrays |
twisti@4318 | 6070 | testptr(to, 2); |
twisti@4318 | 6071 | jccb(Assembler::zero, L_skip_align2); |
twisti@4318 | 6072 | movw(Address(to, 0), value); |
twisti@4318 | 6073 | addptr(to, 2); |
twisti@4318 | 6074 | subl(count, 1<<(shift-1)); |
twisti@4318 | 6075 | BIND(L_skip_align2); |
twisti@4318 | 6076 | } |
twisti@4318 | 6077 | if (UseSSE < 2) { |
twisti@4318 | 6078 | Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
twisti@4318 | 6079 | // Fill 32-byte chunks |
twisti@4318 | 6080 | subl(count, 8 << shift); |
twisti@4318 | 6081 | jcc(Assembler::less, L_check_fill_8_bytes); |
twisti@4318 | 6082 | align(16); |
twisti@4318 | 6083 | |
twisti@4318 | 6084 | BIND(L_fill_32_bytes_loop); |
twisti@4318 | 6085 | |
twisti@4318 | 6086 | for (int i = 0; i < 32; i += 4) { |
twisti@4318 | 6087 | movl(Address(to, i), value); |
twisti@4318 | 6088 | } |
twisti@4318 | 6089 | |
twisti@4318 | 6090 | addptr(to, 32); |
twisti@4318 | 6091 | subl(count, 8 << shift); |
twisti@4318 | 6092 | jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
twisti@4318 | 6093 | BIND(L_check_fill_8_bytes); |
twisti@4318 | 6094 | addl(count, 8 << shift); |
twisti@4318 | 6095 | jccb(Assembler::zero, L_exit); |
twisti@4318 | 6096 | jmpb(L_fill_8_bytes); |
twisti@4318 | 6097 | |
twisti@4318 | 6098 | // |
twisti@4318 | 6099 | // length is too short, just fill qwords |
twisti@4318 | 6100 | // |
twisti@4318 | 6101 | BIND(L_fill_8_bytes_loop); |
twisti@4318 | 6102 | movl(Address(to, 0), value); |
twisti@4318 | 6103 | movl(Address(to, 4), value); |
twisti@4318 | 6104 | addptr(to, 8); |
twisti@4318 | 6105 | BIND(L_fill_8_bytes); |
twisti@4318 | 6106 | subl(count, 1 << (shift + 1)); |
twisti@4318 | 6107 | jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
twisti@4318 | 6108 | // fall through to fill 4 bytes |
twisti@4318 | 6109 | } else { |
twisti@4318 | 6110 | Label L_fill_32_bytes; |
twisti@4318 | 6111 | if (!UseUnalignedLoadStores) { |
twisti@4318 | 6112 | // align to 8 bytes, we know we are 4 byte aligned to start |
twisti@4318 | 6113 | testptr(to, 4); |
twisti@4318 | 6114 | jccb(Assembler::zero, L_fill_32_bytes); |
twisti@4318 | 6115 | movl(Address(to, 0), value); |
twisti@4318 | 6116 | addptr(to, 4); |
twisti@4318 | 6117 | subl(count, 1<<shift); |
twisti@4318 | 6118 | } |
twisti@4318 | 6119 | BIND(L_fill_32_bytes); |
twisti@4318 | 6120 | { |
twisti@4318 | 6121 | assert( UseSSE >= 2, "supported cpu only" ); |
twisti@4318 | 6122 | Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes; |
twisti@4318 | 6123 | movdl(xtmp, value); |
kvn@4411 | 6124 | if (UseAVX >= 2 && UseUnalignedLoadStores) { |
kvn@4411 | 6125 | // Fill 64-byte chunks |
kvn@4411 | 6126 | Label L_fill_64_bytes_loop, L_check_fill_32_bytes; |
kvn@4411 | 6127 | vpbroadcastd(xtmp, xtmp); |
kvn@4411 | 6128 | |
kvn@4411 | 6129 | subl(count, 16 << shift); |
kvn@4411 | 6130 | jcc(Assembler::less, L_check_fill_32_bytes); |
kvn@4411 | 6131 | align(16); |
kvn@4411 | 6132 | |
kvn@4411 | 6133 | BIND(L_fill_64_bytes_loop); |
kvn@4411 | 6134 | vmovdqu(Address(to, 0), xtmp); |
kvn@4411 | 6135 | vmovdqu(Address(to, 32), xtmp); |
kvn@4411 | 6136 | addptr(to, 64); |
kvn@4411 | 6137 | subl(count, 16 << shift); |
kvn@4411 | 6138 | jcc(Assembler::greaterEqual, L_fill_64_bytes_loop); |
kvn@4411 | 6139 | |
kvn@4411 | 6140 | BIND(L_check_fill_32_bytes); |
kvn@4411 | 6141 | addl(count, 8 << shift); |
kvn@4411 | 6142 | jccb(Assembler::less, L_check_fill_8_bytes); |
kvn@4411 | 6143 | vmovdqu(Address(to, 0), xtmp); |
kvn@4411 | 6144 | addptr(to, 32); |
kvn@4411 | 6145 | subl(count, 8 << shift); |
twisti@4318 | 6146 | } else { |
kvn@4411 | 6147 | // Fill 32-byte chunks |
kvn@4411 | 6148 | pshufd(xtmp, xtmp, 0); |
kvn@4411 | 6149 | |
kvn@4411 | 6150 | subl(count, 8 << shift); |
kvn@4411 | 6151 | jcc(Assembler::less, L_check_fill_8_bytes); |
kvn@4411 | 6152 | align(16); |
kvn@4411 | 6153 | |
kvn@4411 | 6154 | BIND(L_fill_32_bytes_loop); |
kvn@4411 | 6155 | |
kvn@4411 | 6156 | if (UseUnalignedLoadStores) { |
kvn@4411 | 6157 | movdqu(Address(to, 0), xtmp); |
kvn@4411 | 6158 | movdqu(Address(to, 16), xtmp); |
kvn@4411 | 6159 | } else { |
kvn@4411 | 6160 | movq(Address(to, 0), xtmp); |
kvn@4411 | 6161 | movq(Address(to, 8), xtmp); |
kvn@4411 | 6162 | movq(Address(to, 16), xtmp); |
kvn@4411 | 6163 | movq(Address(to, 24), xtmp); |
kvn@4411 | 6164 | } |
kvn@4411 | 6165 | |
kvn@4411 | 6166 | addptr(to, 32); |
kvn@4411 | 6167 | subl(count, 8 << shift); |
kvn@4411 | 6168 | jcc(Assembler::greaterEqual, L_fill_32_bytes_loop); |
twisti@4318 | 6169 | } |
twisti@4318 | 6170 | BIND(L_check_fill_8_bytes); |
twisti@4318 | 6171 | addl(count, 8 << shift); |
twisti@4318 | 6172 | jccb(Assembler::zero, L_exit); |
twisti@4318 | 6173 | jmpb(L_fill_8_bytes); |
twisti@4318 | 6174 | |
twisti@4318 | 6175 | // |
twisti@4318 | 6176 | // length is too short, just fill qwords |
twisti@4318 | 6177 | // |
twisti@4318 | 6178 | BIND(L_fill_8_bytes_loop); |
twisti@4318 | 6179 | movq(Address(to, 0), xtmp); |
twisti@4318 | 6180 | addptr(to, 8); |
twisti@4318 | 6181 | BIND(L_fill_8_bytes); |
twisti@4318 | 6182 | subl(count, 1 << (shift + 1)); |
twisti@4318 | 6183 | jcc(Assembler::greaterEqual, L_fill_8_bytes_loop); |
twisti@4318 | 6184 | } |
twisti@4318 | 6185 | } |
twisti@4318 | 6186 | // fill trailing 4 bytes |
twisti@4318 | 6187 | BIND(L_fill_4_bytes); |
twisti@4318 | 6188 | testl(count, 1<<shift); |
twisti@4318 | 6189 | jccb(Assembler::zero, L_fill_2_bytes); |
twisti@4318 | 6190 | movl(Address(to, 0), value); |
twisti@4318 | 6191 | if (t == T_BYTE || t == T_SHORT) { |
twisti@4318 | 6192 | addptr(to, 4); |
twisti@4318 | 6193 | BIND(L_fill_2_bytes); |
twisti@4318 | 6194 | // fill trailing 2 bytes |
twisti@4318 | 6195 | testl(count, 1<<(shift-1)); |
twisti@4318 | 6196 | jccb(Assembler::zero, L_fill_byte); |
twisti@4318 | 6197 | movw(Address(to, 0), value); |
twisti@4318 | 6198 | if (t == T_BYTE) { |
twisti@4318 | 6199 | addptr(to, 2); |
twisti@4318 | 6200 | BIND(L_fill_byte); |
twisti@4318 | 6201 | // fill trailing byte |
twisti@4318 | 6202 | testl(count, 1); |
twisti@4318 | 6203 | jccb(Assembler::zero, L_exit); |
twisti@4318 | 6204 | movb(Address(to, 0), value); |
twisti@4318 | 6205 | } else { |
twisti@4318 | 6206 | BIND(L_fill_byte); |
twisti@4318 | 6207 | } |
twisti@4318 | 6208 | } else { |
twisti@4318 | 6209 | BIND(L_fill_2_bytes); |
twisti@4318 | 6210 | } |
twisti@4318 | 6211 | BIND(L_exit); |
twisti@4318 | 6212 | } |
twisti@4318 | 6213 | #undef BIND |
twisti@4318 | 6214 | #undef BLOCK_COMMENT |
twisti@4318 | 6215 | |
twisti@4318 | 6216 | |
twisti@4318 | 6217 | Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) { |
twisti@4318 | 6218 | switch (cond) { |
twisti@4318 | 6219 | // Note some conditions are synonyms for others |
twisti@4318 | 6220 | case Assembler::zero: return Assembler::notZero; |
twisti@4318 | 6221 | case Assembler::notZero: return Assembler::zero; |
twisti@4318 | 6222 | case Assembler::less: return Assembler::greaterEqual; |
twisti@4318 | 6223 | case Assembler::lessEqual: return Assembler::greater; |
twisti@4318 | 6224 | case Assembler::greater: return Assembler::lessEqual; |
twisti@4318 | 6225 | case Assembler::greaterEqual: return Assembler::less; |
twisti@4318 | 6226 | case Assembler::below: return Assembler::aboveEqual; |
twisti@4318 | 6227 | case Assembler::belowEqual: return Assembler::above; |
twisti@4318 | 6228 | case Assembler::above: return Assembler::belowEqual; |
twisti@4318 | 6229 | case Assembler::aboveEqual: return Assembler::below; |
twisti@4318 | 6230 | case Assembler::overflow: return Assembler::noOverflow; |
twisti@4318 | 6231 | case Assembler::noOverflow: return Assembler::overflow; |
twisti@4318 | 6232 | case Assembler::negative: return Assembler::positive; |
twisti@4318 | 6233 | case Assembler::positive: return Assembler::negative; |
twisti@4318 | 6234 | case Assembler::parity: return Assembler::noParity; |
twisti@4318 | 6235 | case Assembler::noParity: return Assembler::parity; |
twisti@4318 | 6236 | } |
twisti@4318 | 6237 | ShouldNotReachHere(); return Assembler::overflow; |
twisti@4318 | 6238 | } |
twisti@4318 | 6239 | |
twisti@4318 | 6240 | SkipIfEqual::SkipIfEqual( |
twisti@4318 | 6241 | MacroAssembler* masm, const bool* flag_addr, bool value) { |
twisti@4318 | 6242 | _masm = masm; |
twisti@4318 | 6243 | _masm->cmp8(ExternalAddress((address)flag_addr), value); |
twisti@4318 | 6244 | _masm->jcc(Assembler::equal, _label); |
twisti@4318 | 6245 | } |
twisti@4318 | 6246 | |
twisti@4318 | 6247 | SkipIfEqual::~SkipIfEqual() { |
twisti@4318 | 6248 | _masm->bind(_label); |
twisti@4318 | 6249 | } |