Fri, 12 Feb 2010 15:27:36 -0800
Merge
duke@435 | 1 | /* |
twisti@1587 | 2 | * Copyright 1999-2010 Sun Microsystems, Inc. All Rights Reserved. |
duke@435 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
duke@435 | 4 | * |
duke@435 | 5 | * This code is free software; you can redistribute it and/or modify it |
duke@435 | 6 | * under the terms of the GNU General Public License version 2 only, as |
duke@435 | 7 | * published by the Free Software Foundation. |
duke@435 | 8 | * |
duke@435 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
duke@435 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
duke@435 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
duke@435 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
duke@435 | 13 | * accompanied this code). |
duke@435 | 14 | * |
duke@435 | 15 | * You should have received a copy of the GNU General Public License version |
duke@435 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
duke@435 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
duke@435 | 18 | * |
duke@435 | 19 | * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, |
duke@435 | 20 | * CA 95054 USA or visit www.sun.com if you need additional information or |
duke@435 | 21 | * have any questions. |
duke@435 | 22 | * |
duke@435 | 23 | */ |
duke@435 | 24 | |
duke@435 | 25 | #include "incls/_precompiled.incl" |
duke@435 | 26 | #include "incls/_library_call.cpp.incl" |
duke@435 | 27 | |
duke@435 | 28 | class LibraryIntrinsic : public InlineCallGenerator { |
duke@435 | 29 | // Extend the set of intrinsics known to the runtime: |
duke@435 | 30 | public: |
duke@435 | 31 | private: |
duke@435 | 32 | bool _is_virtual; |
duke@435 | 33 | vmIntrinsics::ID _intrinsic_id; |
duke@435 | 34 | |
duke@435 | 35 | public: |
duke@435 | 36 | LibraryIntrinsic(ciMethod* m, bool is_virtual, vmIntrinsics::ID id) |
duke@435 | 37 | : InlineCallGenerator(m), |
duke@435 | 38 | _is_virtual(is_virtual), |
duke@435 | 39 | _intrinsic_id(id) |
duke@435 | 40 | { |
duke@435 | 41 | } |
duke@435 | 42 | virtual bool is_intrinsic() const { return true; } |
duke@435 | 43 | virtual bool is_virtual() const { return _is_virtual; } |
duke@435 | 44 | virtual JVMState* generate(JVMState* jvms); |
duke@435 | 45 | vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; } |
duke@435 | 46 | }; |
duke@435 | 47 | |
duke@435 | 48 | |
duke@435 | 49 | // Local helper class for LibraryIntrinsic: |
duke@435 | 50 | class LibraryCallKit : public GraphKit { |
duke@435 | 51 | private: |
duke@435 | 52 | LibraryIntrinsic* _intrinsic; // the library intrinsic being called |
duke@435 | 53 | |
duke@435 | 54 | public: |
duke@435 | 55 | LibraryCallKit(JVMState* caller, LibraryIntrinsic* intrinsic) |
duke@435 | 56 | : GraphKit(caller), |
duke@435 | 57 | _intrinsic(intrinsic) |
duke@435 | 58 | { |
duke@435 | 59 | } |
duke@435 | 60 | |
duke@435 | 61 | ciMethod* caller() const { return jvms()->method(); } |
duke@435 | 62 | int bci() const { return jvms()->bci(); } |
duke@435 | 63 | LibraryIntrinsic* intrinsic() const { return _intrinsic; } |
duke@435 | 64 | vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); } |
duke@435 | 65 | ciMethod* callee() const { return _intrinsic->method(); } |
duke@435 | 66 | ciSignature* signature() const { return callee()->signature(); } |
duke@435 | 67 | int arg_size() const { return callee()->arg_size(); } |
duke@435 | 68 | |
duke@435 | 69 | bool try_to_inline(); |
duke@435 | 70 | |
duke@435 | 71 | // Helper functions to inline natives |
duke@435 | 72 | void push_result(RegionNode* region, PhiNode* value); |
duke@435 | 73 | Node* generate_guard(Node* test, RegionNode* region, float true_prob); |
duke@435 | 74 | Node* generate_slow_guard(Node* test, RegionNode* region); |
duke@435 | 75 | Node* generate_fair_guard(Node* test, RegionNode* region); |
duke@435 | 76 | Node* generate_negative_guard(Node* index, RegionNode* region, |
duke@435 | 77 | // resulting CastII of index: |
duke@435 | 78 | Node* *pos_index = NULL); |
duke@435 | 79 | Node* generate_nonpositive_guard(Node* index, bool never_negative, |
duke@435 | 80 | // resulting CastII of index: |
duke@435 | 81 | Node* *pos_index = NULL); |
duke@435 | 82 | Node* generate_limit_guard(Node* offset, Node* subseq_length, |
duke@435 | 83 | Node* array_length, |
duke@435 | 84 | RegionNode* region); |
duke@435 | 85 | Node* generate_current_thread(Node* &tls_output); |
duke@435 | 86 | address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset, |
duke@435 | 87 | bool disjoint_bases, const char* &name); |
duke@435 | 88 | Node* load_mirror_from_klass(Node* klass); |
duke@435 | 89 | Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, |
duke@435 | 90 | int nargs, |
duke@435 | 91 | RegionNode* region, int null_path, |
duke@435 | 92 | int offset); |
duke@435 | 93 | Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs, |
duke@435 | 94 | RegionNode* region, int null_path) { |
duke@435 | 95 | int offset = java_lang_Class::klass_offset_in_bytes(); |
duke@435 | 96 | return load_klass_from_mirror_common(mirror, never_see_null, nargs, |
duke@435 | 97 | region, null_path, |
duke@435 | 98 | offset); |
duke@435 | 99 | } |
duke@435 | 100 | Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null, |
duke@435 | 101 | int nargs, |
duke@435 | 102 | RegionNode* region, int null_path) { |
duke@435 | 103 | int offset = java_lang_Class::array_klass_offset_in_bytes(); |
duke@435 | 104 | return load_klass_from_mirror_common(mirror, never_see_null, nargs, |
duke@435 | 105 | region, null_path, |
duke@435 | 106 | offset); |
duke@435 | 107 | } |
duke@435 | 108 | Node* generate_access_flags_guard(Node* kls, |
duke@435 | 109 | int modifier_mask, int modifier_bits, |
duke@435 | 110 | RegionNode* region); |
duke@435 | 111 | Node* generate_interface_guard(Node* kls, RegionNode* region); |
duke@435 | 112 | Node* generate_array_guard(Node* kls, RegionNode* region) { |
duke@435 | 113 | return generate_array_guard_common(kls, region, false, false); |
duke@435 | 114 | } |
duke@435 | 115 | Node* generate_non_array_guard(Node* kls, RegionNode* region) { |
duke@435 | 116 | return generate_array_guard_common(kls, region, false, true); |
duke@435 | 117 | } |
duke@435 | 118 | Node* generate_objArray_guard(Node* kls, RegionNode* region) { |
duke@435 | 119 | return generate_array_guard_common(kls, region, true, false); |
duke@435 | 120 | } |
duke@435 | 121 | Node* generate_non_objArray_guard(Node* kls, RegionNode* region) { |
duke@435 | 122 | return generate_array_guard_common(kls, region, true, true); |
duke@435 | 123 | } |
duke@435 | 124 | Node* generate_array_guard_common(Node* kls, RegionNode* region, |
duke@435 | 125 | bool obj_array, bool not_array); |
duke@435 | 126 | Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); |
duke@435 | 127 | CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, |
duke@435 | 128 | bool is_virtual = false, bool is_static = false); |
duke@435 | 129 | CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) { |
duke@435 | 130 | return generate_method_call(method_id, false, true); |
duke@435 | 131 | } |
duke@435 | 132 | CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { |
duke@435 | 133 | return generate_method_call(method_id, true, false); |
duke@435 | 134 | } |
duke@435 | 135 | |
kvn@1421 | 136 | Node* make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2); |
duke@435 | 137 | bool inline_string_compareTo(); |
duke@435 | 138 | bool inline_string_indexOf(); |
duke@435 | 139 | Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i); |
cfang@1116 | 140 | bool inline_string_equals(); |
duke@435 | 141 | Node* pop_math_arg(); |
duke@435 | 142 | bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName); |
duke@435 | 143 | bool inline_math_native(vmIntrinsics::ID id); |
duke@435 | 144 | bool inline_trig(vmIntrinsics::ID id); |
duke@435 | 145 | bool inline_trans(vmIntrinsics::ID id); |
duke@435 | 146 | bool inline_abs(vmIntrinsics::ID id); |
duke@435 | 147 | bool inline_sqrt(vmIntrinsics::ID id); |
duke@435 | 148 | bool inline_pow(vmIntrinsics::ID id); |
duke@435 | 149 | bool inline_exp(vmIntrinsics::ID id); |
duke@435 | 150 | bool inline_min_max(vmIntrinsics::ID id); |
duke@435 | 151 | Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); |
duke@435 | 152 | // This returns Type::AnyPtr, RawPtr, or OopPtr. |
duke@435 | 153 | int classify_unsafe_addr(Node* &base, Node* &offset); |
duke@435 | 154 | Node* make_unsafe_address(Node* base, Node* offset); |
duke@435 | 155 | bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile); |
duke@435 | 156 | bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static); |
duke@435 | 157 | bool inline_unsafe_allocate(); |
duke@435 | 158 | bool inline_unsafe_copyMemory(); |
duke@435 | 159 | bool inline_native_currentThread(); |
duke@435 | 160 | bool inline_native_time_funcs(bool isNano); |
duke@435 | 161 | bool inline_native_isInterrupted(); |
duke@435 | 162 | bool inline_native_Class_query(vmIntrinsics::ID id); |
duke@435 | 163 | bool inline_native_subtype_check(); |
duke@435 | 164 | |
duke@435 | 165 | bool inline_native_newArray(); |
duke@435 | 166 | bool inline_native_getLength(); |
duke@435 | 167 | bool inline_array_copyOf(bool is_copyOfRange); |
rasbold@604 | 168 | bool inline_array_equals(); |
kvn@1268 | 169 | void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); |
duke@435 | 170 | bool inline_native_clone(bool is_virtual); |
duke@435 | 171 | bool inline_native_Reflection_getCallerClass(); |
duke@435 | 172 | bool inline_native_AtomicLong_get(); |
duke@435 | 173 | bool inline_native_AtomicLong_attemptUpdate(); |
duke@435 | 174 | bool is_method_invoke_or_aux_frame(JVMState* jvms); |
duke@435 | 175 | // Helper function for inlining native object hash method |
duke@435 | 176 | bool inline_native_hashcode(bool is_virtual, bool is_static); |
duke@435 | 177 | bool inline_native_getClass(); |
duke@435 | 178 | |
duke@435 | 179 | // Helper functions for inlining arraycopy |
duke@435 | 180 | bool inline_arraycopy(); |
duke@435 | 181 | void generate_arraycopy(const TypePtr* adr_type, |
duke@435 | 182 | BasicType basic_elem_type, |
duke@435 | 183 | Node* src, Node* src_offset, |
duke@435 | 184 | Node* dest, Node* dest_offset, |
duke@435 | 185 | Node* copy_length, |
duke@435 | 186 | bool disjoint_bases = false, |
duke@435 | 187 | bool length_never_negative = false, |
duke@435 | 188 | RegionNode* slow_region = NULL); |
duke@435 | 189 | AllocateArrayNode* tightly_coupled_allocation(Node* ptr, |
duke@435 | 190 | RegionNode* slow_region); |
duke@435 | 191 | void generate_clear_array(const TypePtr* adr_type, |
duke@435 | 192 | Node* dest, |
duke@435 | 193 | BasicType basic_elem_type, |
duke@435 | 194 | Node* slice_off, |
duke@435 | 195 | Node* slice_len, |
duke@435 | 196 | Node* slice_end); |
duke@435 | 197 | bool generate_block_arraycopy(const TypePtr* adr_type, |
duke@435 | 198 | BasicType basic_elem_type, |
duke@435 | 199 | AllocateNode* alloc, |
duke@435 | 200 | Node* src, Node* src_offset, |
duke@435 | 201 | Node* dest, Node* dest_offset, |
duke@435 | 202 | Node* dest_size); |
duke@435 | 203 | void generate_slow_arraycopy(const TypePtr* adr_type, |
duke@435 | 204 | Node* src, Node* src_offset, |
duke@435 | 205 | Node* dest, Node* dest_offset, |
kvn@1268 | 206 | Node* copy_length); |
duke@435 | 207 | Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
duke@435 | 208 | Node* dest_elem_klass, |
duke@435 | 209 | Node* src, Node* src_offset, |
duke@435 | 210 | Node* dest, Node* dest_offset, |
kvn@1268 | 211 | Node* copy_length); |
duke@435 | 212 | Node* generate_generic_arraycopy(const TypePtr* adr_type, |
duke@435 | 213 | Node* src, Node* src_offset, |
duke@435 | 214 | Node* dest, Node* dest_offset, |
kvn@1268 | 215 | Node* copy_length); |
duke@435 | 216 | void generate_unchecked_arraycopy(const TypePtr* adr_type, |
duke@435 | 217 | BasicType basic_elem_type, |
duke@435 | 218 | bool disjoint_bases, |
duke@435 | 219 | Node* src, Node* src_offset, |
duke@435 | 220 | Node* dest, Node* dest_offset, |
duke@435 | 221 | Node* copy_length); |
duke@435 | 222 | bool inline_unsafe_CAS(BasicType type); |
duke@435 | 223 | bool inline_unsafe_ordered_store(BasicType type); |
duke@435 | 224 | bool inline_fp_conversions(vmIntrinsics::ID id); |
twisti@1210 | 225 | bool inline_numberOfLeadingZeros(vmIntrinsics::ID id); |
twisti@1210 | 226 | bool inline_numberOfTrailingZeros(vmIntrinsics::ID id); |
twisti@1078 | 227 | bool inline_bitCount(vmIntrinsics::ID id); |
duke@435 | 228 | bool inline_reverseBytes(vmIntrinsics::ID id); |
duke@435 | 229 | }; |
duke@435 | 230 | |
duke@435 | 231 | |
duke@435 | 232 | //---------------------------make_vm_intrinsic---------------------------- |
duke@435 | 233 | CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { |
duke@435 | 234 | vmIntrinsics::ID id = m->intrinsic_id(); |
duke@435 | 235 | assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); |
duke@435 | 236 | |
duke@435 | 237 | if (DisableIntrinsic[0] != '\0' |
duke@435 | 238 | && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) { |
duke@435 | 239 | // disabled by a user request on the command line: |
duke@435 | 240 | // example: -XX:DisableIntrinsic=_hashCode,_getClass |
duke@435 | 241 | return NULL; |
duke@435 | 242 | } |
duke@435 | 243 | |
duke@435 | 244 | if (!m->is_loaded()) { |
duke@435 | 245 | // do not attempt to inline unloaded methods |
duke@435 | 246 | return NULL; |
duke@435 | 247 | } |
duke@435 | 248 | |
duke@435 | 249 | // Only a few intrinsics implement a virtual dispatch. |
duke@435 | 250 | // They are expensive calls which are also frequently overridden. |
duke@435 | 251 | if (is_virtual) { |
duke@435 | 252 | switch (id) { |
duke@435 | 253 | case vmIntrinsics::_hashCode: |
duke@435 | 254 | case vmIntrinsics::_clone: |
duke@435 | 255 | // OK, Object.hashCode and Object.clone intrinsics come in both flavors |
duke@435 | 256 | break; |
duke@435 | 257 | default: |
duke@435 | 258 | return NULL; |
duke@435 | 259 | } |
duke@435 | 260 | } |
duke@435 | 261 | |
duke@435 | 262 | // -XX:-InlineNatives disables nearly all intrinsics: |
duke@435 | 263 | if (!InlineNatives) { |
duke@435 | 264 | switch (id) { |
duke@435 | 265 | case vmIntrinsics::_indexOf: |
duke@435 | 266 | case vmIntrinsics::_compareTo: |
cfang@1116 | 267 | case vmIntrinsics::_equals: |
rasbold@604 | 268 | case vmIntrinsics::_equalsC: |
duke@435 | 269 | break; // InlineNatives does not control String.compareTo |
duke@435 | 270 | default: |
duke@435 | 271 | return NULL; |
duke@435 | 272 | } |
duke@435 | 273 | } |
duke@435 | 274 | |
duke@435 | 275 | switch (id) { |
duke@435 | 276 | case vmIntrinsics::_compareTo: |
duke@435 | 277 | if (!SpecialStringCompareTo) return NULL; |
duke@435 | 278 | break; |
duke@435 | 279 | case vmIntrinsics::_indexOf: |
duke@435 | 280 | if (!SpecialStringIndexOf) return NULL; |
duke@435 | 281 | break; |
cfang@1116 | 282 | case vmIntrinsics::_equals: |
cfang@1116 | 283 | if (!SpecialStringEquals) return NULL; |
cfang@1116 | 284 | break; |
rasbold@604 | 285 | case vmIntrinsics::_equalsC: |
rasbold@604 | 286 | if (!SpecialArraysEquals) return NULL; |
rasbold@604 | 287 | break; |
duke@435 | 288 | case vmIntrinsics::_arraycopy: |
duke@435 | 289 | if (!InlineArrayCopy) return NULL; |
duke@435 | 290 | break; |
duke@435 | 291 | case vmIntrinsics::_copyMemory: |
duke@435 | 292 | if (StubRoutines::unsafe_arraycopy() == NULL) return NULL; |
duke@435 | 293 | if (!InlineArrayCopy) return NULL; |
duke@435 | 294 | break; |
duke@435 | 295 | case vmIntrinsics::_hashCode: |
duke@435 | 296 | if (!InlineObjectHash) return NULL; |
duke@435 | 297 | break; |
duke@435 | 298 | case vmIntrinsics::_clone: |
duke@435 | 299 | case vmIntrinsics::_copyOf: |
duke@435 | 300 | case vmIntrinsics::_copyOfRange: |
duke@435 | 301 | if (!InlineObjectCopy) return NULL; |
duke@435 | 302 | // These also use the arraycopy intrinsic mechanism: |
duke@435 | 303 | if (!InlineArrayCopy) return NULL; |
duke@435 | 304 | break; |
duke@435 | 305 | case vmIntrinsics::_checkIndex: |
duke@435 | 306 | // We do not intrinsify this. The optimizer does fine with it. |
duke@435 | 307 | return NULL; |
duke@435 | 308 | |
duke@435 | 309 | case vmIntrinsics::_get_AtomicLong: |
duke@435 | 310 | case vmIntrinsics::_attemptUpdate: |
duke@435 | 311 | if (!InlineAtomicLong) return NULL; |
duke@435 | 312 | break; |
duke@435 | 313 | |
duke@435 | 314 | case vmIntrinsics::_getCallerClass: |
duke@435 | 315 | if (!UseNewReflection) return NULL; |
duke@435 | 316 | if (!InlineReflectionGetCallerClass) return NULL; |
duke@435 | 317 | if (!JDK_Version::is_gte_jdk14x_version()) return NULL; |
duke@435 | 318 | break; |
duke@435 | 319 | |
twisti@1078 | 320 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 321 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 322 | if (!UsePopCountInstruction) return NULL; |
twisti@1078 | 323 | break; |
twisti@1078 | 324 | |
duke@435 | 325 | default: |
jrose@1291 | 326 | assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); |
jrose@1291 | 327 | assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); |
duke@435 | 328 | break; |
duke@435 | 329 | } |
duke@435 | 330 | |
duke@435 | 331 | // -XX:-InlineClassNatives disables natives from the Class class. |
duke@435 | 332 | // The flag applies to all reflective calls, notably Array.newArray |
duke@435 | 333 | // (visible to Java programmers as Array.newInstance). |
duke@435 | 334 | if (m->holder()->name() == ciSymbol::java_lang_Class() || |
duke@435 | 335 | m->holder()->name() == ciSymbol::java_lang_reflect_Array()) { |
duke@435 | 336 | if (!InlineClassNatives) return NULL; |
duke@435 | 337 | } |
duke@435 | 338 | |
duke@435 | 339 | // -XX:-InlineThreadNatives disables natives from the Thread class. |
duke@435 | 340 | if (m->holder()->name() == ciSymbol::java_lang_Thread()) { |
duke@435 | 341 | if (!InlineThreadNatives) return NULL; |
duke@435 | 342 | } |
duke@435 | 343 | |
duke@435 | 344 | // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes. |
duke@435 | 345 | if (m->holder()->name() == ciSymbol::java_lang_Math() || |
duke@435 | 346 | m->holder()->name() == ciSymbol::java_lang_Float() || |
duke@435 | 347 | m->holder()->name() == ciSymbol::java_lang_Double()) { |
duke@435 | 348 | if (!InlineMathNatives) return NULL; |
duke@435 | 349 | } |
duke@435 | 350 | |
duke@435 | 351 | // -XX:-InlineUnsafeOps disables natives from the Unsafe class. |
duke@435 | 352 | if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) { |
duke@435 | 353 | if (!InlineUnsafeOps) return NULL; |
duke@435 | 354 | } |
duke@435 | 355 | |
duke@435 | 356 | return new LibraryIntrinsic(m, is_virtual, (vmIntrinsics::ID) id); |
duke@435 | 357 | } |
duke@435 | 358 | |
duke@435 | 359 | //----------------------register_library_intrinsics----------------------- |
duke@435 | 360 | // Initialize this file's data structures, for each Compile instance. |
duke@435 | 361 | void Compile::register_library_intrinsics() { |
duke@435 | 362 | // Nothing to do here. |
duke@435 | 363 | } |
duke@435 | 364 | |
duke@435 | 365 | JVMState* LibraryIntrinsic::generate(JVMState* jvms) { |
duke@435 | 366 | LibraryCallKit kit(jvms, this); |
duke@435 | 367 | Compile* C = kit.C; |
duke@435 | 368 | int nodes = C->unique(); |
duke@435 | 369 | #ifndef PRODUCT |
duke@435 | 370 | if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) { |
duke@435 | 371 | char buf[1000]; |
duke@435 | 372 | const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); |
duke@435 | 373 | tty->print_cr("Intrinsic %s", str); |
duke@435 | 374 | } |
duke@435 | 375 | #endif |
duke@435 | 376 | if (kit.try_to_inline()) { |
duke@435 | 377 | if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
duke@435 | 378 | tty->print("Inlining intrinsic %s%s at bci:%d in", |
duke@435 | 379 | vmIntrinsics::name_at(intrinsic_id()), |
duke@435 | 380 | (is_virtual() ? " (virtual)" : ""), kit.bci()); |
duke@435 | 381 | kit.caller()->print_short_name(tty); |
duke@435 | 382 | tty->print_cr(" (%d bytes)", kit.caller()->code_size()); |
duke@435 | 383 | } |
duke@435 | 384 | C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
duke@435 | 385 | if (C->log()) { |
duke@435 | 386 | C->log()->elem("intrinsic id='%s'%s nodes='%d'", |
duke@435 | 387 | vmIntrinsics::name_at(intrinsic_id()), |
duke@435 | 388 | (is_virtual() ? " virtual='1'" : ""), |
duke@435 | 389 | C->unique() - nodes); |
duke@435 | 390 | } |
duke@435 | 391 | return kit.transfer_exceptions_into_jvms(); |
duke@435 | 392 | } |
duke@435 | 393 | |
duke@435 | 394 | if (PrintIntrinsics) { |
jrose@1291 | 395 | tty->print("Did not inline intrinsic %s%s at bci:%d in", |
jrose@1291 | 396 | vmIntrinsics::name_at(intrinsic_id()), |
jrose@1291 | 397 | (is_virtual() ? " (virtual)" : ""), kit.bci()); |
jrose@1291 | 398 | kit.caller()->print_short_name(tty); |
jrose@1291 | 399 | tty->print_cr(" (%d bytes)", kit.caller()->code_size()); |
duke@435 | 400 | } |
duke@435 | 401 | C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); |
duke@435 | 402 | return NULL; |
duke@435 | 403 | } |
duke@435 | 404 | |
duke@435 | 405 | bool LibraryCallKit::try_to_inline() { |
duke@435 | 406 | // Handle symbolic names for otherwise undistinguished boolean switches: |
duke@435 | 407 | const bool is_store = true; |
duke@435 | 408 | const bool is_native_ptr = true; |
duke@435 | 409 | const bool is_static = true; |
duke@435 | 410 | |
duke@435 | 411 | switch (intrinsic_id()) { |
duke@435 | 412 | case vmIntrinsics::_hashCode: |
duke@435 | 413 | return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); |
duke@435 | 414 | case vmIntrinsics::_identityHashCode: |
duke@435 | 415 | return inline_native_hashcode(/*!virtual*/ false, is_static); |
duke@435 | 416 | case vmIntrinsics::_getClass: |
duke@435 | 417 | return inline_native_getClass(); |
duke@435 | 418 | |
duke@435 | 419 | case vmIntrinsics::_dsin: |
duke@435 | 420 | case vmIntrinsics::_dcos: |
duke@435 | 421 | case vmIntrinsics::_dtan: |
duke@435 | 422 | case vmIntrinsics::_dabs: |
duke@435 | 423 | case vmIntrinsics::_datan2: |
duke@435 | 424 | case vmIntrinsics::_dsqrt: |
duke@435 | 425 | case vmIntrinsics::_dexp: |
duke@435 | 426 | case vmIntrinsics::_dlog: |
duke@435 | 427 | case vmIntrinsics::_dlog10: |
duke@435 | 428 | case vmIntrinsics::_dpow: |
duke@435 | 429 | return inline_math_native(intrinsic_id()); |
duke@435 | 430 | |
duke@435 | 431 | case vmIntrinsics::_min: |
duke@435 | 432 | case vmIntrinsics::_max: |
duke@435 | 433 | return inline_min_max(intrinsic_id()); |
duke@435 | 434 | |
duke@435 | 435 | case vmIntrinsics::_arraycopy: |
duke@435 | 436 | return inline_arraycopy(); |
duke@435 | 437 | |
duke@435 | 438 | case vmIntrinsics::_compareTo: |
duke@435 | 439 | return inline_string_compareTo(); |
duke@435 | 440 | case vmIntrinsics::_indexOf: |
duke@435 | 441 | return inline_string_indexOf(); |
cfang@1116 | 442 | case vmIntrinsics::_equals: |
cfang@1116 | 443 | return inline_string_equals(); |
duke@435 | 444 | |
duke@435 | 445 | case vmIntrinsics::_getObject: |
duke@435 | 446 | return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, false); |
duke@435 | 447 | case vmIntrinsics::_getBoolean: |
duke@435 | 448 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, false); |
duke@435 | 449 | case vmIntrinsics::_getByte: |
duke@435 | 450 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, false); |
duke@435 | 451 | case vmIntrinsics::_getShort: |
duke@435 | 452 | return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, false); |
duke@435 | 453 | case vmIntrinsics::_getChar: |
duke@435 | 454 | return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, false); |
duke@435 | 455 | case vmIntrinsics::_getInt: |
duke@435 | 456 | return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, false); |
duke@435 | 457 | case vmIntrinsics::_getLong: |
duke@435 | 458 | return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, false); |
duke@435 | 459 | case vmIntrinsics::_getFloat: |
duke@435 | 460 | return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, false); |
duke@435 | 461 | case vmIntrinsics::_getDouble: |
duke@435 | 462 | return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, false); |
duke@435 | 463 | |
duke@435 | 464 | case vmIntrinsics::_putObject: |
duke@435 | 465 | return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, false); |
duke@435 | 466 | case vmIntrinsics::_putBoolean: |
duke@435 | 467 | return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, false); |
duke@435 | 468 | case vmIntrinsics::_putByte: |
duke@435 | 469 | return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, false); |
duke@435 | 470 | case vmIntrinsics::_putShort: |
duke@435 | 471 | return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, false); |
duke@435 | 472 | case vmIntrinsics::_putChar: |
duke@435 | 473 | return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, false); |
duke@435 | 474 | case vmIntrinsics::_putInt: |
duke@435 | 475 | return inline_unsafe_access(!is_native_ptr, is_store, T_INT, false); |
duke@435 | 476 | case vmIntrinsics::_putLong: |
duke@435 | 477 | return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, false); |
duke@435 | 478 | case vmIntrinsics::_putFloat: |
duke@435 | 479 | return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, false); |
duke@435 | 480 | case vmIntrinsics::_putDouble: |
duke@435 | 481 | return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, false); |
duke@435 | 482 | |
duke@435 | 483 | case vmIntrinsics::_getByte_raw: |
duke@435 | 484 | return inline_unsafe_access(is_native_ptr, !is_store, T_BYTE, false); |
duke@435 | 485 | case vmIntrinsics::_getShort_raw: |
duke@435 | 486 | return inline_unsafe_access(is_native_ptr, !is_store, T_SHORT, false); |
duke@435 | 487 | case vmIntrinsics::_getChar_raw: |
duke@435 | 488 | return inline_unsafe_access(is_native_ptr, !is_store, T_CHAR, false); |
duke@435 | 489 | case vmIntrinsics::_getInt_raw: |
duke@435 | 490 | return inline_unsafe_access(is_native_ptr, !is_store, T_INT, false); |
duke@435 | 491 | case vmIntrinsics::_getLong_raw: |
duke@435 | 492 | return inline_unsafe_access(is_native_ptr, !is_store, T_LONG, false); |
duke@435 | 493 | case vmIntrinsics::_getFloat_raw: |
duke@435 | 494 | return inline_unsafe_access(is_native_ptr, !is_store, T_FLOAT, false); |
duke@435 | 495 | case vmIntrinsics::_getDouble_raw: |
duke@435 | 496 | return inline_unsafe_access(is_native_ptr, !is_store, T_DOUBLE, false); |
duke@435 | 497 | case vmIntrinsics::_getAddress_raw: |
duke@435 | 498 | return inline_unsafe_access(is_native_ptr, !is_store, T_ADDRESS, false); |
duke@435 | 499 | |
duke@435 | 500 | case vmIntrinsics::_putByte_raw: |
duke@435 | 501 | return inline_unsafe_access(is_native_ptr, is_store, T_BYTE, false); |
duke@435 | 502 | case vmIntrinsics::_putShort_raw: |
duke@435 | 503 | return inline_unsafe_access(is_native_ptr, is_store, T_SHORT, false); |
duke@435 | 504 | case vmIntrinsics::_putChar_raw: |
duke@435 | 505 | return inline_unsafe_access(is_native_ptr, is_store, T_CHAR, false); |
duke@435 | 506 | case vmIntrinsics::_putInt_raw: |
duke@435 | 507 | return inline_unsafe_access(is_native_ptr, is_store, T_INT, false); |
duke@435 | 508 | case vmIntrinsics::_putLong_raw: |
duke@435 | 509 | return inline_unsafe_access(is_native_ptr, is_store, T_LONG, false); |
duke@435 | 510 | case vmIntrinsics::_putFloat_raw: |
duke@435 | 511 | return inline_unsafe_access(is_native_ptr, is_store, T_FLOAT, false); |
duke@435 | 512 | case vmIntrinsics::_putDouble_raw: |
duke@435 | 513 | return inline_unsafe_access(is_native_ptr, is_store, T_DOUBLE, false); |
duke@435 | 514 | case vmIntrinsics::_putAddress_raw: |
duke@435 | 515 | return inline_unsafe_access(is_native_ptr, is_store, T_ADDRESS, false); |
duke@435 | 516 | |
duke@435 | 517 | case vmIntrinsics::_getObjectVolatile: |
duke@435 | 518 | return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, true); |
duke@435 | 519 | case vmIntrinsics::_getBooleanVolatile: |
duke@435 | 520 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, true); |
duke@435 | 521 | case vmIntrinsics::_getByteVolatile: |
duke@435 | 522 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, true); |
duke@435 | 523 | case vmIntrinsics::_getShortVolatile: |
duke@435 | 524 | return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, true); |
duke@435 | 525 | case vmIntrinsics::_getCharVolatile: |
duke@435 | 526 | return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, true); |
duke@435 | 527 | case vmIntrinsics::_getIntVolatile: |
duke@435 | 528 | return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, true); |
duke@435 | 529 | case vmIntrinsics::_getLongVolatile: |
duke@435 | 530 | return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, true); |
duke@435 | 531 | case vmIntrinsics::_getFloatVolatile: |
duke@435 | 532 | return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, true); |
duke@435 | 533 | case vmIntrinsics::_getDoubleVolatile: |
duke@435 | 534 | return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, true); |
duke@435 | 535 | |
duke@435 | 536 | case vmIntrinsics::_putObjectVolatile: |
duke@435 | 537 | return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, true); |
duke@435 | 538 | case vmIntrinsics::_putBooleanVolatile: |
duke@435 | 539 | return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, true); |
duke@435 | 540 | case vmIntrinsics::_putByteVolatile: |
duke@435 | 541 | return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, true); |
duke@435 | 542 | case vmIntrinsics::_putShortVolatile: |
duke@435 | 543 | return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, true); |
duke@435 | 544 | case vmIntrinsics::_putCharVolatile: |
duke@435 | 545 | return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, true); |
duke@435 | 546 | case vmIntrinsics::_putIntVolatile: |
duke@435 | 547 | return inline_unsafe_access(!is_native_ptr, is_store, T_INT, true); |
duke@435 | 548 | case vmIntrinsics::_putLongVolatile: |
duke@435 | 549 | return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, true); |
duke@435 | 550 | case vmIntrinsics::_putFloatVolatile: |
duke@435 | 551 | return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, true); |
duke@435 | 552 | case vmIntrinsics::_putDoubleVolatile: |
duke@435 | 553 | return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, true); |
duke@435 | 554 | |
duke@435 | 555 | case vmIntrinsics::_prefetchRead: |
duke@435 | 556 | return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static); |
duke@435 | 557 | case vmIntrinsics::_prefetchWrite: |
duke@435 | 558 | return inline_unsafe_prefetch(!is_native_ptr, is_store, !is_static); |
duke@435 | 559 | case vmIntrinsics::_prefetchReadStatic: |
duke@435 | 560 | return inline_unsafe_prefetch(!is_native_ptr, !is_store, is_static); |
duke@435 | 561 | case vmIntrinsics::_prefetchWriteStatic: |
duke@435 | 562 | return inline_unsafe_prefetch(!is_native_ptr, is_store, is_static); |
duke@435 | 563 | |
duke@435 | 564 | case vmIntrinsics::_compareAndSwapObject: |
duke@435 | 565 | return inline_unsafe_CAS(T_OBJECT); |
duke@435 | 566 | case vmIntrinsics::_compareAndSwapInt: |
duke@435 | 567 | return inline_unsafe_CAS(T_INT); |
duke@435 | 568 | case vmIntrinsics::_compareAndSwapLong: |
duke@435 | 569 | return inline_unsafe_CAS(T_LONG); |
duke@435 | 570 | |
duke@435 | 571 | case vmIntrinsics::_putOrderedObject: |
duke@435 | 572 | return inline_unsafe_ordered_store(T_OBJECT); |
duke@435 | 573 | case vmIntrinsics::_putOrderedInt: |
duke@435 | 574 | return inline_unsafe_ordered_store(T_INT); |
duke@435 | 575 | case vmIntrinsics::_putOrderedLong: |
duke@435 | 576 | return inline_unsafe_ordered_store(T_LONG); |
duke@435 | 577 | |
duke@435 | 578 | case vmIntrinsics::_currentThread: |
duke@435 | 579 | return inline_native_currentThread(); |
duke@435 | 580 | case vmIntrinsics::_isInterrupted: |
duke@435 | 581 | return inline_native_isInterrupted(); |
duke@435 | 582 | |
duke@435 | 583 | case vmIntrinsics::_currentTimeMillis: |
duke@435 | 584 | return inline_native_time_funcs(false); |
duke@435 | 585 | case vmIntrinsics::_nanoTime: |
duke@435 | 586 | return inline_native_time_funcs(true); |
duke@435 | 587 | case vmIntrinsics::_allocateInstance: |
duke@435 | 588 | return inline_unsafe_allocate(); |
duke@435 | 589 | case vmIntrinsics::_copyMemory: |
duke@435 | 590 | return inline_unsafe_copyMemory(); |
duke@435 | 591 | case vmIntrinsics::_newArray: |
duke@435 | 592 | return inline_native_newArray(); |
duke@435 | 593 | case vmIntrinsics::_getLength: |
duke@435 | 594 | return inline_native_getLength(); |
duke@435 | 595 | case vmIntrinsics::_copyOf: |
duke@435 | 596 | return inline_array_copyOf(false); |
duke@435 | 597 | case vmIntrinsics::_copyOfRange: |
duke@435 | 598 | return inline_array_copyOf(true); |
rasbold@604 | 599 | case vmIntrinsics::_equalsC: |
rasbold@604 | 600 | return inline_array_equals(); |
duke@435 | 601 | case vmIntrinsics::_clone: |
duke@435 | 602 | return inline_native_clone(intrinsic()->is_virtual()); |
duke@435 | 603 | |
duke@435 | 604 | case vmIntrinsics::_isAssignableFrom: |
duke@435 | 605 | return inline_native_subtype_check(); |
duke@435 | 606 | |
duke@435 | 607 | case vmIntrinsics::_isInstance: |
duke@435 | 608 | case vmIntrinsics::_getModifiers: |
duke@435 | 609 | case vmIntrinsics::_isInterface: |
duke@435 | 610 | case vmIntrinsics::_isArray: |
duke@435 | 611 | case vmIntrinsics::_isPrimitive: |
duke@435 | 612 | case vmIntrinsics::_getSuperclass: |
duke@435 | 613 | case vmIntrinsics::_getComponentType: |
duke@435 | 614 | case vmIntrinsics::_getClassAccessFlags: |
duke@435 | 615 | return inline_native_Class_query(intrinsic_id()); |
duke@435 | 616 | |
duke@435 | 617 | case vmIntrinsics::_floatToRawIntBits: |
duke@435 | 618 | case vmIntrinsics::_floatToIntBits: |
duke@435 | 619 | case vmIntrinsics::_intBitsToFloat: |
duke@435 | 620 | case vmIntrinsics::_doubleToRawLongBits: |
duke@435 | 621 | case vmIntrinsics::_doubleToLongBits: |
duke@435 | 622 | case vmIntrinsics::_longBitsToDouble: |
duke@435 | 623 | return inline_fp_conversions(intrinsic_id()); |
duke@435 | 624 | |
twisti@1210 | 625 | case vmIntrinsics::_numberOfLeadingZeros_i: |
twisti@1210 | 626 | case vmIntrinsics::_numberOfLeadingZeros_l: |
twisti@1210 | 627 | return inline_numberOfLeadingZeros(intrinsic_id()); |
twisti@1210 | 628 | |
twisti@1210 | 629 | case vmIntrinsics::_numberOfTrailingZeros_i: |
twisti@1210 | 630 | case vmIntrinsics::_numberOfTrailingZeros_l: |
twisti@1210 | 631 | return inline_numberOfTrailingZeros(intrinsic_id()); |
twisti@1210 | 632 | |
twisti@1078 | 633 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 634 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 635 | return inline_bitCount(intrinsic_id()); |
twisti@1078 | 636 | |
duke@435 | 637 | case vmIntrinsics::_reverseBytes_i: |
duke@435 | 638 | case vmIntrinsics::_reverseBytes_l: |
duke@435 | 639 | return inline_reverseBytes((vmIntrinsics::ID) intrinsic_id()); |
duke@435 | 640 | |
duke@435 | 641 | case vmIntrinsics::_get_AtomicLong: |
duke@435 | 642 | return inline_native_AtomicLong_get(); |
duke@435 | 643 | case vmIntrinsics::_attemptUpdate: |
duke@435 | 644 | return inline_native_AtomicLong_attemptUpdate(); |
duke@435 | 645 | |
duke@435 | 646 | case vmIntrinsics::_getCallerClass: |
duke@435 | 647 | return inline_native_Reflection_getCallerClass(); |
duke@435 | 648 | |
duke@435 | 649 | default: |
duke@435 | 650 | // If you get here, it may be that someone has added a new intrinsic |
duke@435 | 651 | // to the list in vmSymbols.hpp without implementing it here. |
duke@435 | 652 | #ifndef PRODUCT |
duke@435 | 653 | if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { |
duke@435 | 654 | tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)", |
duke@435 | 655 | vmIntrinsics::name_at(intrinsic_id()), intrinsic_id()); |
duke@435 | 656 | } |
duke@435 | 657 | #endif |
duke@435 | 658 | return false; |
duke@435 | 659 | } |
duke@435 | 660 | } |
duke@435 | 661 | |
duke@435 | 662 | //------------------------------push_result------------------------------ |
duke@435 | 663 | // Helper function for finishing intrinsics. |
duke@435 | 664 | void LibraryCallKit::push_result(RegionNode* region, PhiNode* value) { |
duke@435 | 665 | record_for_igvn(region); |
duke@435 | 666 | set_control(_gvn.transform(region)); |
duke@435 | 667 | BasicType value_type = value->type()->basic_type(); |
duke@435 | 668 | push_node(value_type, _gvn.transform(value)); |
duke@435 | 669 | } |
duke@435 | 670 | |
duke@435 | 671 | //------------------------------generate_guard--------------------------- |
duke@435 | 672 | // Helper function for generating guarded fast-slow graph structures. |
duke@435 | 673 | // The given 'test', if true, guards a slow path. If the test fails |
duke@435 | 674 | // then a fast path can be taken. (We generally hope it fails.) |
duke@435 | 675 | // In all cases, GraphKit::control() is updated to the fast path. |
duke@435 | 676 | // The returned value represents the control for the slow path. |
duke@435 | 677 | // The return value is never 'top'; it is either a valid control |
duke@435 | 678 | // or NULL if it is obvious that the slow path can never be taken. |
duke@435 | 679 | // Also, if region and the slow control are not NULL, the slow edge |
duke@435 | 680 | // is appended to the region. |
duke@435 | 681 | Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { |
duke@435 | 682 | if (stopped()) { |
duke@435 | 683 | // Already short circuited. |
duke@435 | 684 | return NULL; |
duke@435 | 685 | } |
duke@435 | 686 | |
duke@435 | 687 | // Build an if node and its projections. |
duke@435 | 688 | // If test is true we take the slow path, which we assume is uncommon. |
duke@435 | 689 | if (_gvn.type(test) == TypeInt::ZERO) { |
duke@435 | 690 | // The slow branch is never taken. No need to build this guard. |
duke@435 | 691 | return NULL; |
duke@435 | 692 | } |
duke@435 | 693 | |
duke@435 | 694 | IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); |
duke@435 | 695 | |
duke@435 | 696 | Node* if_slow = _gvn.transform( new (C, 1) IfTrueNode(iff) ); |
duke@435 | 697 | if (if_slow == top()) { |
duke@435 | 698 | // The slow branch is never taken. No need to build this guard. |
duke@435 | 699 | return NULL; |
duke@435 | 700 | } |
duke@435 | 701 | |
duke@435 | 702 | if (region != NULL) |
duke@435 | 703 | region->add_req(if_slow); |
duke@435 | 704 | |
duke@435 | 705 | Node* if_fast = _gvn.transform( new (C, 1) IfFalseNode(iff) ); |
duke@435 | 706 | set_control(if_fast); |
duke@435 | 707 | |
duke@435 | 708 | return if_slow; |
duke@435 | 709 | } |
duke@435 | 710 | |
duke@435 | 711 | inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) { |
duke@435 | 712 | return generate_guard(test, region, PROB_UNLIKELY_MAG(3)); |
duke@435 | 713 | } |
duke@435 | 714 | inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) { |
duke@435 | 715 | return generate_guard(test, region, PROB_FAIR); |
duke@435 | 716 | } |
duke@435 | 717 | |
duke@435 | 718 | inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, |
duke@435 | 719 | Node* *pos_index) { |
duke@435 | 720 | if (stopped()) |
duke@435 | 721 | return NULL; // already stopped |
duke@435 | 722 | if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] |
duke@435 | 723 | return NULL; // index is already adequately typed |
duke@435 | 724 | Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) ); |
duke@435 | 725 | Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 726 | Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); |
duke@435 | 727 | if (is_neg != NULL && pos_index != NULL) { |
duke@435 | 728 | // Emulate effect of Parse::adjust_map_after_if. |
duke@435 | 729 | Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS); |
duke@435 | 730 | ccast->set_req(0, control()); |
duke@435 | 731 | (*pos_index) = _gvn.transform(ccast); |
duke@435 | 732 | } |
duke@435 | 733 | return is_neg; |
duke@435 | 734 | } |
duke@435 | 735 | |
duke@435 | 736 | inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative, |
duke@435 | 737 | Node* *pos_index) { |
duke@435 | 738 | if (stopped()) |
duke@435 | 739 | return NULL; // already stopped |
duke@435 | 740 | if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint] |
duke@435 | 741 | return NULL; // index is already adequately typed |
duke@435 | 742 | Node* cmp_le = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) ); |
duke@435 | 743 | BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le); |
duke@435 | 744 | Node* bol_le = _gvn.transform( new (C, 2) BoolNode(cmp_le, le_or_eq) ); |
duke@435 | 745 | Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN); |
duke@435 | 746 | if (is_notp != NULL && pos_index != NULL) { |
duke@435 | 747 | // Emulate effect of Parse::adjust_map_after_if. |
duke@435 | 748 | Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS1); |
duke@435 | 749 | ccast->set_req(0, control()); |
duke@435 | 750 | (*pos_index) = _gvn.transform(ccast); |
duke@435 | 751 | } |
duke@435 | 752 | return is_notp; |
duke@435 | 753 | } |
duke@435 | 754 | |
duke@435 | 755 | // Make sure that 'position' is a valid limit index, in [0..length]. |
duke@435 | 756 | // There are two equivalent plans for checking this: |
duke@435 | 757 | // A. (offset + copyLength) unsigned<= arrayLength |
duke@435 | 758 | // B. offset <= (arrayLength - copyLength) |
duke@435 | 759 | // We require that all of the values above, except for the sum and |
duke@435 | 760 | // difference, are already known to be non-negative. |
duke@435 | 761 | // Plan A is robust in the face of overflow, if offset and copyLength |
duke@435 | 762 | // are both hugely positive. |
duke@435 | 763 | // |
duke@435 | 764 | // Plan B is less direct and intuitive, but it does not overflow at |
duke@435 | 765 | // all, since the difference of two non-negatives is always |
duke@435 | 766 | // representable. Whenever Java methods must perform the equivalent |
duke@435 | 767 | // check they generally use Plan B instead of Plan A. |
duke@435 | 768 | // For the moment we use Plan A. |
duke@435 | 769 | inline Node* LibraryCallKit::generate_limit_guard(Node* offset, |
duke@435 | 770 | Node* subseq_length, |
duke@435 | 771 | Node* array_length, |
duke@435 | 772 | RegionNode* region) { |
duke@435 | 773 | if (stopped()) |
duke@435 | 774 | return NULL; // already stopped |
duke@435 | 775 | bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; |
duke@435 | 776 | if (zero_offset && _gvn.eqv_uncast(subseq_length, array_length)) |
duke@435 | 777 | return NULL; // common case of whole-array copy |
duke@435 | 778 | Node* last = subseq_length; |
duke@435 | 779 | if (!zero_offset) // last += offset |
duke@435 | 780 | last = _gvn.transform( new (C, 3) AddINode(last, offset)); |
duke@435 | 781 | Node* cmp_lt = _gvn.transform( new (C, 3) CmpUNode(array_length, last) ); |
duke@435 | 782 | Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 783 | Node* is_over = generate_guard(bol_lt, region, PROB_MIN); |
duke@435 | 784 | return is_over; |
duke@435 | 785 | } |
duke@435 | 786 | |
duke@435 | 787 | |
duke@435 | 788 | //--------------------------generate_current_thread-------------------- |
duke@435 | 789 | Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { |
duke@435 | 790 | ciKlass* thread_klass = env()->Thread_klass(); |
duke@435 | 791 | const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); |
duke@435 | 792 | Node* thread = _gvn.transform(new (C, 1) ThreadLocalNode()); |
duke@435 | 793 | Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset())); |
duke@435 | 794 | Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT); |
duke@435 | 795 | tls_output = thread; |
duke@435 | 796 | return threadObj; |
duke@435 | 797 | } |
duke@435 | 798 | |
duke@435 | 799 | |
kvn@1421 | 800 | //------------------------------make_string_method_node------------------------ |
kvn@1421 | 801 | // Helper method for String intrinsic finctions. |
kvn@1421 | 802 | Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2) { |
kvn@1421 | 803 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
kvn@1421 | 804 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
kvn@1421 | 805 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
kvn@1421 | 806 | |
kvn@1421 | 807 | Node* no_ctrl = NULL; |
kvn@1421 | 808 | |
kvn@1421 | 809 | ciInstanceKlass* klass = env()->String_klass(); |
kvn@1421 | 810 | const TypeInstPtr* string_type = |
kvn@1421 | 811 | TypeInstPtr::make(TypePtr::BotPTR, klass, false, NULL, 0); |
kvn@1421 | 812 | |
kvn@1421 | 813 | const TypeAryPtr* value_type = |
kvn@1421 | 814 | TypeAryPtr::make(TypePtr::NotNull, |
kvn@1421 | 815 | TypeAry::make(TypeInt::CHAR,TypeInt::POS), |
kvn@1421 | 816 | ciTypeArrayKlass::make(T_CHAR), true, 0); |
kvn@1421 | 817 | |
kvn@1421 | 818 | // Get start addr of string and substring |
kvn@1421 | 819 | Node* str1_valuea = basic_plus_adr(str1, str1, value_offset); |
kvn@1421 | 820 | Node* str1_value = make_load(no_ctrl, str1_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset)); |
kvn@1421 | 821 | Node* str1_offseta = basic_plus_adr(str1, str1, offset_offset); |
kvn@1421 | 822 | Node* str1_offset = make_load(no_ctrl, str1_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
kvn@1421 | 823 | Node* str1_start = array_element_address(str1_value, str1_offset, T_CHAR); |
kvn@1421 | 824 | |
kvn@1421 | 825 | // Pin loads from String::equals() argument since it could be NULL. |
kvn@1421 | 826 | Node* str2_ctrl = (opcode == Op_StrEquals) ? control() : no_ctrl; |
kvn@1421 | 827 | Node* str2_valuea = basic_plus_adr(str2, str2, value_offset); |
kvn@1421 | 828 | Node* str2_value = make_load(str2_ctrl, str2_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset)); |
kvn@1421 | 829 | Node* str2_offseta = basic_plus_adr(str2, str2, offset_offset); |
kvn@1421 | 830 | Node* str2_offset = make_load(str2_ctrl, str2_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
kvn@1421 | 831 | Node* str2_start = array_element_address(str2_value, str2_offset, T_CHAR); |
kvn@1421 | 832 | |
kvn@1421 | 833 | Node* result = NULL; |
kvn@1421 | 834 | switch (opcode) { |
kvn@1421 | 835 | case Op_StrIndexOf: |
kvn@1421 | 836 | result = new (C, 6) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 837 | str1_start, cnt1, str2_start, cnt2); |
kvn@1421 | 838 | break; |
kvn@1421 | 839 | case Op_StrComp: |
kvn@1421 | 840 | result = new (C, 6) StrCompNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 841 | str1_start, cnt1, str2_start, cnt2); |
kvn@1421 | 842 | break; |
kvn@1421 | 843 | case Op_StrEquals: |
kvn@1421 | 844 | result = new (C, 5) StrEqualsNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 845 | str1_start, str2_start, cnt1); |
kvn@1421 | 846 | break; |
kvn@1421 | 847 | default: |
kvn@1421 | 848 | ShouldNotReachHere(); |
kvn@1421 | 849 | return NULL; |
kvn@1421 | 850 | } |
kvn@1421 | 851 | |
kvn@1421 | 852 | // All these intrinsics have checks. |
kvn@1421 | 853 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
kvn@1421 | 854 | |
kvn@1421 | 855 | return _gvn.transform(result); |
kvn@1421 | 856 | } |
kvn@1421 | 857 | |
duke@435 | 858 | //------------------------------inline_string_compareTo------------------------ |
duke@435 | 859 | bool LibraryCallKit::inline_string_compareTo() { |
duke@435 | 860 | |
cfang@1116 | 861 | if (!Matcher::has_match_rule(Op_StrComp)) return false; |
cfang@1116 | 862 | |
duke@435 | 863 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
duke@435 | 864 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
duke@435 | 865 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
duke@435 | 866 | |
duke@435 | 867 | _sp += 2; |
duke@435 | 868 | Node *argument = pop(); // pop non-receiver first: it was pushed second |
duke@435 | 869 | Node *receiver = pop(); |
duke@435 | 870 | |
duke@435 | 871 | // Null check on self without removing any arguments. The argument |
duke@435 | 872 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 873 | // invalid stack traces when string compare is inlined into a method |
duke@435 | 874 | // which handles NullPointerExceptions. |
duke@435 | 875 | _sp += 2; |
duke@435 | 876 | receiver = do_null_check(receiver, T_OBJECT); |
duke@435 | 877 | argument = do_null_check(argument, T_OBJECT); |
duke@435 | 878 | _sp -= 2; |
duke@435 | 879 | if (stopped()) { |
duke@435 | 880 | return true; |
duke@435 | 881 | } |
duke@435 | 882 | |
duke@435 | 883 | ciInstanceKlass* klass = env()->String_klass(); |
duke@435 | 884 | const TypeInstPtr* string_type = |
duke@435 | 885 | TypeInstPtr::make(TypePtr::BotPTR, klass, false, NULL, 0); |
kvn@1421 | 886 | Node* no_ctrl = NULL; |
kvn@1421 | 887 | |
kvn@1421 | 888 | // Get counts for string and argument |
kvn@1421 | 889 | Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 890 | Node* receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 891 | |
kvn@1421 | 892 | Node* argument_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@1421 | 893 | Node* argument_cnt = make_load(no_ctrl, argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 894 | |
kvn@1421 | 895 | Node* compare = make_string_method_node(Op_StrComp, receiver, receiver_cnt, argument, argument_cnt); |
duke@435 | 896 | push(compare); |
duke@435 | 897 | return true; |
duke@435 | 898 | } |
duke@435 | 899 | |
cfang@1116 | 900 | //------------------------------inline_string_equals------------------------ |
cfang@1116 | 901 | bool LibraryCallKit::inline_string_equals() { |
cfang@1116 | 902 | |
cfang@1116 | 903 | if (!Matcher::has_match_rule(Op_StrEquals)) return false; |
cfang@1116 | 904 | |
cfang@1116 | 905 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
cfang@1116 | 906 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
cfang@1116 | 907 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
cfang@1116 | 908 | |
cfang@1116 | 909 | _sp += 2; |
cfang@1116 | 910 | Node* argument = pop(); // pop non-receiver first: it was pushed second |
cfang@1116 | 911 | Node* receiver = pop(); |
cfang@1116 | 912 | |
cfang@1116 | 913 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 914 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 915 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 916 | // which handles NullPointerExceptions. |
cfang@1116 | 917 | _sp += 2; |
cfang@1116 | 918 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 919 | //should not do null check for argument for String.equals(), because spec |
cfang@1116 | 920 | //allows to specify NULL as argument. |
cfang@1116 | 921 | _sp -= 2; |
cfang@1116 | 922 | |
cfang@1116 | 923 | if (stopped()) { |
cfang@1116 | 924 | return true; |
cfang@1116 | 925 | } |
cfang@1116 | 926 | |
kvn@1421 | 927 | // paths (plus control) merge |
kvn@1421 | 928 | RegionNode* region = new (C, 5) RegionNode(5); |
kvn@1421 | 929 | Node* phi = new (C, 5) PhiNode(region, TypeInt::BOOL); |
kvn@1421 | 930 | |
kvn@1421 | 931 | // does source == target string? |
kvn@1421 | 932 | Node* cmp = _gvn.transform(new (C, 3) CmpPNode(receiver, argument)); |
kvn@1421 | 933 | Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::eq)); |
kvn@1421 | 934 | |
kvn@1421 | 935 | Node* if_eq = generate_slow_guard(bol, NULL); |
kvn@1421 | 936 | if (if_eq != NULL) { |
kvn@1421 | 937 | // receiver == argument |
kvn@1421 | 938 | phi->init_req(2, intcon(1)); |
kvn@1421 | 939 | region->init_req(2, if_eq); |
kvn@1421 | 940 | } |
kvn@1421 | 941 | |
cfang@1116 | 942 | // get String klass for instanceOf |
cfang@1116 | 943 | ciInstanceKlass* klass = env()->String_klass(); |
cfang@1116 | 944 | |
kvn@1421 | 945 | if (!stopped()) { |
kvn@1421 | 946 | Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass))); |
kvn@1421 | 947 | Node* cmp = _gvn.transform(new (C, 3) CmpINode(inst, intcon(1))); |
kvn@1421 | 948 | Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::ne)); |
kvn@1421 | 949 | |
kvn@1421 | 950 | Node* inst_false = generate_guard(bol, NULL, PROB_MIN); |
kvn@1421 | 951 | //instanceOf == true, fallthrough |
kvn@1421 | 952 | |
kvn@1421 | 953 | if (inst_false != NULL) { |
kvn@1421 | 954 | phi->init_req(3, intcon(0)); |
kvn@1421 | 955 | region->init_req(3, inst_false); |
kvn@1421 | 956 | } |
kvn@1421 | 957 | } |
cfang@1116 | 958 | |
cfang@1116 | 959 | const TypeInstPtr* string_type = |
cfang@1116 | 960 | TypeInstPtr::make(TypePtr::BotPTR, klass, false, NULL, 0); |
cfang@1116 | 961 | |
kvn@1421 | 962 | Node* no_ctrl = NULL; |
kvn@1421 | 963 | Node* receiver_cnt; |
kvn@1421 | 964 | Node* argument_cnt; |
kvn@1421 | 965 | |
kvn@1421 | 966 | if (!stopped()) { |
kvn@1421 | 967 | // Get counts for string and argument |
kvn@1421 | 968 | Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 969 | receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 970 | |
kvn@1421 | 971 | // Pin load from argument string since it could be NULL. |
kvn@1421 | 972 | Node* argument_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@1421 | 973 | argument_cnt = make_load(control(), argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 974 | |
kvn@1421 | 975 | // Check for receiver count != argument count |
kvn@1421 | 976 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(receiver_cnt, argument_cnt) ); |
kvn@1421 | 977 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::ne) ); |
kvn@1421 | 978 | Node* if_ne = generate_slow_guard(bol, NULL); |
kvn@1421 | 979 | if (if_ne != NULL) { |
kvn@1421 | 980 | phi->init_req(4, intcon(0)); |
kvn@1421 | 981 | region->init_req(4, if_ne); |
kvn@1421 | 982 | } |
kvn@1421 | 983 | } |
kvn@1421 | 984 | |
kvn@1421 | 985 | // Check for count == 0 is done by mach node StrEquals. |
kvn@1421 | 986 | |
kvn@1421 | 987 | if (!stopped()) { |
kvn@1421 | 988 | Node* equals = make_string_method_node(Op_StrEquals, receiver, receiver_cnt, argument, argument_cnt); |
kvn@1421 | 989 | phi->init_req(1, equals); |
kvn@1421 | 990 | region->init_req(1, control()); |
kvn@1421 | 991 | } |
cfang@1116 | 992 | |
cfang@1116 | 993 | // post merge |
cfang@1116 | 994 | set_control(_gvn.transform(region)); |
cfang@1116 | 995 | record_for_igvn(region); |
cfang@1116 | 996 | |
cfang@1116 | 997 | push(_gvn.transform(phi)); |
cfang@1116 | 998 | |
cfang@1116 | 999 | return true; |
cfang@1116 | 1000 | } |
cfang@1116 | 1001 | |
rasbold@604 | 1002 | //------------------------------inline_array_equals---------------------------- |
rasbold@604 | 1003 | bool LibraryCallKit::inline_array_equals() { |
rasbold@604 | 1004 | |
rasbold@609 | 1005 | if (!Matcher::has_match_rule(Op_AryEq)) return false; |
rasbold@609 | 1006 | |
rasbold@604 | 1007 | _sp += 2; |
rasbold@604 | 1008 | Node *argument2 = pop(); |
rasbold@604 | 1009 | Node *argument1 = pop(); |
rasbold@604 | 1010 | |
rasbold@604 | 1011 | Node* equals = |
kvn@1421 | 1012 | _gvn.transform(new (C, 4) AryEqNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 1013 | argument1, argument2) ); |
rasbold@604 | 1014 | push(equals); |
rasbold@604 | 1015 | return true; |
rasbold@604 | 1016 | } |
rasbold@604 | 1017 | |
duke@435 | 1018 | // Java version of String.indexOf(constant string) |
duke@435 | 1019 | // class StringDecl { |
duke@435 | 1020 | // StringDecl(char[] ca) { |
duke@435 | 1021 | // offset = 0; |
duke@435 | 1022 | // count = ca.length; |
duke@435 | 1023 | // value = ca; |
duke@435 | 1024 | // } |
duke@435 | 1025 | // int offset; |
duke@435 | 1026 | // int count; |
duke@435 | 1027 | // char[] value; |
duke@435 | 1028 | // } |
duke@435 | 1029 | // |
duke@435 | 1030 | // static int string_indexOf_J(StringDecl string_object, char[] target_object, |
duke@435 | 1031 | // int targetOffset, int cache_i, int md2) { |
duke@435 | 1032 | // int cache = cache_i; |
duke@435 | 1033 | // int sourceOffset = string_object.offset; |
duke@435 | 1034 | // int sourceCount = string_object.count; |
duke@435 | 1035 | // int targetCount = target_object.length; |
duke@435 | 1036 | // |
duke@435 | 1037 | // int targetCountLess1 = targetCount - 1; |
duke@435 | 1038 | // int sourceEnd = sourceOffset + sourceCount - targetCountLess1; |
duke@435 | 1039 | // |
duke@435 | 1040 | // char[] source = string_object.value; |
duke@435 | 1041 | // char[] target = target_object; |
duke@435 | 1042 | // int lastChar = target[targetCountLess1]; |
duke@435 | 1043 | // |
duke@435 | 1044 | // outer_loop: |
duke@435 | 1045 | // for (int i = sourceOffset; i < sourceEnd; ) { |
duke@435 | 1046 | // int src = source[i + targetCountLess1]; |
duke@435 | 1047 | // if (src == lastChar) { |
duke@435 | 1048 | // // With random strings and a 4-character alphabet, |
duke@435 | 1049 | // // reverse matching at this point sets up 0.8% fewer |
duke@435 | 1050 | // // frames, but (paradoxically) makes 0.3% more probes. |
duke@435 | 1051 | // // Since those probes are nearer the lastChar probe, |
duke@435 | 1052 | // // there is may be a net D$ win with reverse matching. |
duke@435 | 1053 | // // But, reversing loop inhibits unroll of inner loop |
duke@435 | 1054 | // // for unknown reason. So, does running outer loop from |
duke@435 | 1055 | // // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount) |
duke@435 | 1056 | // for (int j = 0; j < targetCountLess1; j++) { |
duke@435 | 1057 | // if (target[targetOffset + j] != source[i+j]) { |
duke@435 | 1058 | // if ((cache & (1 << source[i+j])) == 0) { |
duke@435 | 1059 | // if (md2 < j+1) { |
duke@435 | 1060 | // i += j+1; |
duke@435 | 1061 | // continue outer_loop; |
duke@435 | 1062 | // } |
duke@435 | 1063 | // } |
duke@435 | 1064 | // i += md2; |
duke@435 | 1065 | // continue outer_loop; |
duke@435 | 1066 | // } |
duke@435 | 1067 | // } |
duke@435 | 1068 | // return i - sourceOffset; |
duke@435 | 1069 | // } |
duke@435 | 1070 | // if ((cache & (1 << src)) == 0) { |
duke@435 | 1071 | // i += targetCountLess1; |
duke@435 | 1072 | // } // using "i += targetCount;" and an "else i++;" causes a jump to jump. |
duke@435 | 1073 | // i++; |
duke@435 | 1074 | // } |
duke@435 | 1075 | // return -1; |
duke@435 | 1076 | // } |
duke@435 | 1077 | |
duke@435 | 1078 | //------------------------------string_indexOf------------------------ |
duke@435 | 1079 | Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i, |
duke@435 | 1080 | jint cache_i, jint md2_i) { |
duke@435 | 1081 | |
duke@435 | 1082 | Node* no_ctrl = NULL; |
duke@435 | 1083 | float likely = PROB_LIKELY(0.9); |
duke@435 | 1084 | float unlikely = PROB_UNLIKELY(0.9); |
duke@435 | 1085 | |
duke@435 | 1086 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
duke@435 | 1087 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
duke@435 | 1088 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
duke@435 | 1089 | |
duke@435 | 1090 | ciInstanceKlass* klass = env()->String_klass(); |
duke@435 | 1091 | const TypeInstPtr* string_type = TypeInstPtr::make(TypePtr::BotPTR, klass, false, NULL, 0); |
duke@435 | 1092 | const TypeAryPtr* source_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::CHAR,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, 0); |
duke@435 | 1093 | |
duke@435 | 1094 | Node* sourceOffseta = basic_plus_adr(string_object, string_object, offset_offset); |
duke@435 | 1095 | Node* sourceOffset = make_load(no_ctrl, sourceOffseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
duke@435 | 1096 | Node* sourceCounta = basic_plus_adr(string_object, string_object, count_offset); |
duke@435 | 1097 | Node* sourceCount = make_load(no_ctrl, sourceCounta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
duke@435 | 1098 | Node* sourcea = basic_plus_adr(string_object, string_object, value_offset); |
duke@435 | 1099 | Node* source = make_load(no_ctrl, sourcea, source_type, T_OBJECT, string_type->add_offset(value_offset)); |
duke@435 | 1100 | |
kvn@599 | 1101 | Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array)) ); |
duke@435 | 1102 | jint target_length = target_array->length(); |
duke@435 | 1103 | const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin)); |
duke@435 | 1104 | const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot); |
duke@435 | 1105 | |
kvn@1286 | 1106 | IdealKit kit(gvn(), control(), merged_memory(), false, true); |
duke@435 | 1107 | #define __ kit. |
duke@435 | 1108 | Node* zero = __ ConI(0); |
duke@435 | 1109 | Node* one = __ ConI(1); |
duke@435 | 1110 | Node* cache = __ ConI(cache_i); |
duke@435 | 1111 | Node* md2 = __ ConI(md2_i); |
duke@435 | 1112 | Node* lastChar = __ ConI(target_array->char_at(target_length - 1)); |
duke@435 | 1113 | Node* targetCount = __ ConI(target_length); |
duke@435 | 1114 | Node* targetCountLess1 = __ ConI(target_length - 1); |
duke@435 | 1115 | Node* targetOffset = __ ConI(targetOffset_i); |
duke@435 | 1116 | Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1); |
duke@435 | 1117 | |
kvn@1286 | 1118 | IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done(); |
duke@435 | 1119 | Node* outer_loop = __ make_label(2 /* goto */); |
duke@435 | 1120 | Node* return_ = __ make_label(1); |
duke@435 | 1121 | |
duke@435 | 1122 | __ set(rtn,__ ConI(-1)); |
duke@435 | 1123 | __ loop(i, sourceOffset, BoolTest::lt, sourceEnd); { |
duke@435 | 1124 | Node* i2 = __ AddI(__ value(i), targetCountLess1); |
duke@435 | 1125 | // pin to prohibit loading of "next iteration" value which may SEGV (rare) |
duke@435 | 1126 | Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS); |
duke@435 | 1127 | __ if_then(src, BoolTest::eq, lastChar, unlikely); { |
duke@435 | 1128 | __ loop(j, zero, BoolTest::lt, targetCountLess1); { |
duke@435 | 1129 | Node* tpj = __ AddI(targetOffset, __ value(j)); |
duke@435 | 1130 | Node* targ = load_array_element(no_ctrl, target, tpj, target_type); |
duke@435 | 1131 | Node* ipj = __ AddI(__ value(i), __ value(j)); |
duke@435 | 1132 | Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS); |
duke@435 | 1133 | __ if_then(targ, BoolTest::ne, src2); { |
duke@435 | 1134 | __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); { |
duke@435 | 1135 | __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); { |
duke@435 | 1136 | __ increment(i, __ AddI(__ value(j), one)); |
duke@435 | 1137 | __ goto_(outer_loop); |
duke@435 | 1138 | } __ end_if(); __ dead(j); |
duke@435 | 1139 | }__ end_if(); __ dead(j); |
duke@435 | 1140 | __ increment(i, md2); |
duke@435 | 1141 | __ goto_(outer_loop); |
duke@435 | 1142 | }__ end_if(); |
duke@435 | 1143 | __ increment(j, one); |
duke@435 | 1144 | }__ end_loop(); __ dead(j); |
duke@435 | 1145 | __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i); |
duke@435 | 1146 | __ goto_(return_); |
duke@435 | 1147 | }__ end_if(); |
duke@435 | 1148 | __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); { |
duke@435 | 1149 | __ increment(i, targetCountLess1); |
duke@435 | 1150 | }__ end_if(); |
duke@435 | 1151 | __ increment(i, one); |
duke@435 | 1152 | __ bind(outer_loop); |
duke@435 | 1153 | }__ end_loop(); __ dead(i); |
duke@435 | 1154 | __ bind(return_); |
kvn@1286 | 1155 | |
kvn@1286 | 1156 | // Final sync IdealKit and GraphKit. |
kvn@1286 | 1157 | sync_kit(kit); |
duke@435 | 1158 | Node* result = __ value(rtn); |
duke@435 | 1159 | #undef __ |
duke@435 | 1160 | C->set_has_loops(true); |
duke@435 | 1161 | return result; |
duke@435 | 1162 | } |
duke@435 | 1163 | |
duke@435 | 1164 | //------------------------------inline_string_indexOf------------------------ |
duke@435 | 1165 | bool LibraryCallKit::inline_string_indexOf() { |
duke@435 | 1166 | |
cfang@1116 | 1167 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
cfang@1116 | 1168 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
cfang@1116 | 1169 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
cfang@1116 | 1170 | |
duke@435 | 1171 | _sp += 2; |
duke@435 | 1172 | Node *argument = pop(); // pop non-receiver first: it was pushed second |
duke@435 | 1173 | Node *receiver = pop(); |
duke@435 | 1174 | |
cfang@1116 | 1175 | Node* result; |
cfang@1116 | 1176 | if (Matcher::has_match_rule(Op_StrIndexOf) && |
cfang@1116 | 1177 | UseSSE42Intrinsics) { |
cfang@1116 | 1178 | // Generate SSE4.2 version of indexOf |
cfang@1116 | 1179 | // We currently only have match rules that use SSE4.2 |
cfang@1116 | 1180 | |
cfang@1116 | 1181 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 1182 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 1183 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 1184 | // which handles NullPointerExceptions. |
cfang@1116 | 1185 | _sp += 2; |
cfang@1116 | 1186 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 1187 | argument = do_null_check(argument, T_OBJECT); |
cfang@1116 | 1188 | _sp -= 2; |
cfang@1116 | 1189 | |
cfang@1116 | 1190 | if (stopped()) { |
cfang@1116 | 1191 | return true; |
cfang@1116 | 1192 | } |
cfang@1116 | 1193 | |
kvn@1421 | 1194 | // Make the merge point |
kvn@1421 | 1195 | RegionNode* result_rgn = new (C, 3) RegionNode(3); |
kvn@1421 | 1196 | Node* result_phi = new (C, 3) PhiNode(result_rgn, TypeInt::INT); |
kvn@1421 | 1197 | Node* no_ctrl = NULL; |
kvn@1421 | 1198 | |
cfang@1116 | 1199 | ciInstanceKlass* klass = env()->String_klass(); |
cfang@1116 | 1200 | const TypeInstPtr* string_type = |
cfang@1116 | 1201 | TypeInstPtr::make(TypePtr::BotPTR, klass, false, NULL, 0); |
cfang@1116 | 1202 | |
kvn@1421 | 1203 | // Get counts for string and substr |
kvn@1421 | 1204 | Node* source_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 1205 | Node* source_cnt = make_load(no_ctrl, source_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1206 | |
kvn@1421 | 1207 | Node* substr_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@1421 | 1208 | Node* substr_cnt = make_load(no_ctrl, substr_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1209 | |
kvn@1421 | 1210 | // Check for substr count > string count |
kvn@1421 | 1211 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, source_cnt) ); |
kvn@1421 | 1212 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::gt) ); |
kvn@1421 | 1213 | Node* if_gt = generate_slow_guard(bol, NULL); |
kvn@1421 | 1214 | if (if_gt != NULL) { |
kvn@1421 | 1215 | result_phi->init_req(2, intcon(-1)); |
kvn@1421 | 1216 | result_rgn->init_req(2, if_gt); |
kvn@1421 | 1217 | } |
kvn@1421 | 1218 | |
kvn@1421 | 1219 | if (!stopped()) { |
kvn@1421 | 1220 | result = make_string_method_node(Op_StrIndexOf, receiver, source_cnt, argument, substr_cnt); |
kvn@1421 | 1221 | result_phi->init_req(1, result); |
kvn@1421 | 1222 | result_rgn->init_req(1, control()); |
kvn@1421 | 1223 | } |
kvn@1421 | 1224 | set_control(_gvn.transform(result_rgn)); |
kvn@1421 | 1225 | record_for_igvn(result_rgn); |
kvn@1421 | 1226 | result = _gvn.transform(result_phi); |
kvn@1421 | 1227 | |
cfang@1116 | 1228 | } else { //Use LibraryCallKit::string_indexOf |
cfang@1116 | 1229 | // don't intrinsify is argument isn't a constant string. |
cfang@1116 | 1230 | if (!argument->is_Con()) { |
cfang@1116 | 1231 | return false; |
cfang@1116 | 1232 | } |
cfang@1116 | 1233 | const TypeOopPtr* str_type = _gvn.type(argument)->isa_oopptr(); |
cfang@1116 | 1234 | if (str_type == NULL) { |
cfang@1116 | 1235 | return false; |
cfang@1116 | 1236 | } |
cfang@1116 | 1237 | ciInstanceKlass* klass = env()->String_klass(); |
cfang@1116 | 1238 | ciObject* str_const = str_type->const_oop(); |
cfang@1116 | 1239 | if (str_const == NULL || str_const->klass() != klass) { |
cfang@1116 | 1240 | return false; |
cfang@1116 | 1241 | } |
cfang@1116 | 1242 | ciInstance* str = str_const->as_instance(); |
cfang@1116 | 1243 | assert(str != NULL, "must be instance"); |
cfang@1116 | 1244 | |
cfang@1116 | 1245 | ciObject* v = str->field_value_by_offset(value_offset).as_object(); |
cfang@1116 | 1246 | int o = str->field_value_by_offset(offset_offset).as_int(); |
cfang@1116 | 1247 | int c = str->field_value_by_offset(count_offset).as_int(); |
cfang@1116 | 1248 | ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array |
cfang@1116 | 1249 | |
cfang@1116 | 1250 | // constant strings have no offset and count == length which |
cfang@1116 | 1251 | // simplifies the resulting code somewhat so lets optimize for that. |
cfang@1116 | 1252 | if (o != 0 || c != pat->length()) { |
cfang@1116 | 1253 | return false; |
cfang@1116 | 1254 | } |
cfang@1116 | 1255 | |
cfang@1116 | 1256 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 1257 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 1258 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 1259 | // which handles NullPointerExceptions. |
cfang@1116 | 1260 | _sp += 2; |
cfang@1116 | 1261 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 1262 | // No null check on the argument is needed since it's a constant String oop. |
cfang@1116 | 1263 | _sp -= 2; |
cfang@1116 | 1264 | if (stopped()) { |
cfang@1116 | 1265 | return true; |
cfang@1116 | 1266 | } |
cfang@1116 | 1267 | |
cfang@1116 | 1268 | // The null string as a pattern always returns 0 (match at beginning of string) |
cfang@1116 | 1269 | if (c == 0) { |
cfang@1116 | 1270 | push(intcon(0)); |
cfang@1116 | 1271 | return true; |
cfang@1116 | 1272 | } |
cfang@1116 | 1273 | |
cfang@1116 | 1274 | // Generate default indexOf |
cfang@1116 | 1275 | jchar lastChar = pat->char_at(o + (c - 1)); |
cfang@1116 | 1276 | int cache = 0; |
cfang@1116 | 1277 | int i; |
cfang@1116 | 1278 | for (i = 0; i < c - 1; i++) { |
cfang@1116 | 1279 | assert(i < pat->length(), "out of range"); |
cfang@1116 | 1280 | cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1))); |
cfang@1116 | 1281 | } |
cfang@1116 | 1282 | |
cfang@1116 | 1283 | int md2 = c; |
cfang@1116 | 1284 | for (i = 0; i < c - 1; i++) { |
cfang@1116 | 1285 | assert(i < pat->length(), "out of range"); |
cfang@1116 | 1286 | if (pat->char_at(o + i) == lastChar) { |
cfang@1116 | 1287 | md2 = (c - 1) - i; |
cfang@1116 | 1288 | } |
cfang@1116 | 1289 | } |
cfang@1116 | 1290 | |
cfang@1116 | 1291 | result = string_indexOf(receiver, pat, o, cache, md2); |
duke@435 | 1292 | } |
cfang@1116 | 1293 | |
duke@435 | 1294 | push(result); |
duke@435 | 1295 | return true; |
duke@435 | 1296 | } |
duke@435 | 1297 | |
duke@435 | 1298 | //--------------------------pop_math_arg-------------------------------- |
duke@435 | 1299 | // Pop a double argument to a math function from the stack |
duke@435 | 1300 | // rounding it if necessary. |
duke@435 | 1301 | Node * LibraryCallKit::pop_math_arg() { |
duke@435 | 1302 | Node *arg = pop_pair(); |
duke@435 | 1303 | if( Matcher::strict_fp_requires_explicit_rounding && UseSSE<=1 ) |
duke@435 | 1304 | arg = _gvn.transform( new (C, 2) RoundDoubleNode(0, arg) ); |
duke@435 | 1305 | return arg; |
duke@435 | 1306 | } |
duke@435 | 1307 | |
duke@435 | 1308 | //------------------------------inline_trig---------------------------------- |
duke@435 | 1309 | // Inline sin/cos/tan instructions, if possible. If rounding is required, do |
duke@435 | 1310 | // argument reduction which will turn into a fast/slow diamond. |
duke@435 | 1311 | bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) { |
duke@435 | 1312 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1313 | Node* arg = pop_math_arg(); |
duke@435 | 1314 | Node* trig = NULL; |
duke@435 | 1315 | |
duke@435 | 1316 | switch (id) { |
duke@435 | 1317 | case vmIntrinsics::_dsin: |
duke@435 | 1318 | trig = _gvn.transform((Node*)new (C, 2) SinDNode(arg)); |
duke@435 | 1319 | break; |
duke@435 | 1320 | case vmIntrinsics::_dcos: |
duke@435 | 1321 | trig = _gvn.transform((Node*)new (C, 2) CosDNode(arg)); |
duke@435 | 1322 | break; |
duke@435 | 1323 | case vmIntrinsics::_dtan: |
duke@435 | 1324 | trig = _gvn.transform((Node*)new (C, 2) TanDNode(arg)); |
duke@435 | 1325 | break; |
duke@435 | 1326 | default: |
duke@435 | 1327 | assert(false, "bad intrinsic was passed in"); |
duke@435 | 1328 | return false; |
duke@435 | 1329 | } |
duke@435 | 1330 | |
duke@435 | 1331 | // Rounding required? Check for argument reduction! |
duke@435 | 1332 | if( Matcher::strict_fp_requires_explicit_rounding ) { |
duke@435 | 1333 | |
duke@435 | 1334 | static const double pi_4 = 0.7853981633974483; |
duke@435 | 1335 | static const double neg_pi_4 = -0.7853981633974483; |
duke@435 | 1336 | // pi/2 in 80-bit extended precision |
duke@435 | 1337 | // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00}; |
duke@435 | 1338 | // -pi/2 in 80-bit extended precision |
duke@435 | 1339 | // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00}; |
duke@435 | 1340 | // Cutoff value for using this argument reduction technique |
duke@435 | 1341 | //static const double pi_2_minus_epsilon = 1.564660403643354; |
duke@435 | 1342 | //static const double neg_pi_2_plus_epsilon = -1.564660403643354; |
duke@435 | 1343 | |
duke@435 | 1344 | // Pseudocode for sin: |
duke@435 | 1345 | // if (x <= Math.PI / 4.0) { |
duke@435 | 1346 | // if (x >= -Math.PI / 4.0) return fsin(x); |
duke@435 | 1347 | // if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0); |
duke@435 | 1348 | // } else { |
duke@435 | 1349 | // if (x <= Math.PI / 2.0) return fcos(x - Math.PI / 2.0); |
duke@435 | 1350 | // } |
duke@435 | 1351 | // return StrictMath.sin(x); |
duke@435 | 1352 | |
duke@435 | 1353 | // Pseudocode for cos: |
duke@435 | 1354 | // if (x <= Math.PI / 4.0) { |
duke@435 | 1355 | // if (x >= -Math.PI / 4.0) return fcos(x); |
duke@435 | 1356 | // if (x >= -Math.PI / 2.0) return fsin(x + Math.PI / 2.0); |
duke@435 | 1357 | // } else { |
duke@435 | 1358 | // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0); |
duke@435 | 1359 | // } |
duke@435 | 1360 | // return StrictMath.cos(x); |
duke@435 | 1361 | |
duke@435 | 1362 | // Actually, sticking in an 80-bit Intel value into C2 will be tough; it |
duke@435 | 1363 | // requires a special machine instruction to load it. Instead we'll try |
duke@435 | 1364 | // the 'easy' case. If we really need the extra range +/- PI/2 we'll |
duke@435 | 1365 | // probably do the math inside the SIN encoding. |
duke@435 | 1366 | |
duke@435 | 1367 | // Make the merge point |
duke@435 | 1368 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 1369 | Node *phi = new (C, 3) PhiNode(r,Type::DOUBLE); |
duke@435 | 1370 | |
duke@435 | 1371 | // Flatten arg so we need only 1 test |
duke@435 | 1372 | Node *abs = _gvn.transform(new (C, 2) AbsDNode(arg)); |
duke@435 | 1373 | // Node for PI/4 constant |
duke@435 | 1374 | Node *pi4 = makecon(TypeD::make(pi_4)); |
duke@435 | 1375 | // Check PI/4 : abs(arg) |
duke@435 | 1376 | Node *cmp = _gvn.transform(new (C, 3) CmpDNode(pi4,abs)); |
duke@435 | 1377 | // Check: If PI/4 < abs(arg) then go slow |
duke@435 | 1378 | Node *bol = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::lt ) ); |
duke@435 | 1379 | // Branch either way |
duke@435 | 1380 | IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 1381 | set_control(opt_iff(r,iff)); |
duke@435 | 1382 | |
duke@435 | 1383 | // Set fast path result |
duke@435 | 1384 | phi->init_req(2,trig); |
duke@435 | 1385 | |
duke@435 | 1386 | // Slow path - non-blocking leaf call |
duke@435 | 1387 | Node* call = NULL; |
duke@435 | 1388 | switch (id) { |
duke@435 | 1389 | case vmIntrinsics::_dsin: |
duke@435 | 1390 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1391 | CAST_FROM_FN_PTR(address, SharedRuntime::dsin), |
duke@435 | 1392 | "Sin", NULL, arg, top()); |
duke@435 | 1393 | break; |
duke@435 | 1394 | case vmIntrinsics::_dcos: |
duke@435 | 1395 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1396 | CAST_FROM_FN_PTR(address, SharedRuntime::dcos), |
duke@435 | 1397 | "Cos", NULL, arg, top()); |
duke@435 | 1398 | break; |
duke@435 | 1399 | case vmIntrinsics::_dtan: |
duke@435 | 1400 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1401 | CAST_FROM_FN_PTR(address, SharedRuntime::dtan), |
duke@435 | 1402 | "Tan", NULL, arg, top()); |
duke@435 | 1403 | break; |
duke@435 | 1404 | } |
duke@435 | 1405 | assert(control()->in(0) == call, ""); |
duke@435 | 1406 | Node* slow_result = _gvn.transform(new (C, 1) ProjNode(call,TypeFunc::Parms)); |
duke@435 | 1407 | r->init_req(1,control()); |
duke@435 | 1408 | phi->init_req(1,slow_result); |
duke@435 | 1409 | |
duke@435 | 1410 | // Post-merge |
duke@435 | 1411 | set_control(_gvn.transform(r)); |
duke@435 | 1412 | record_for_igvn(r); |
duke@435 | 1413 | trig = _gvn.transform(phi); |
duke@435 | 1414 | |
duke@435 | 1415 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1416 | } |
duke@435 | 1417 | // Push result back on JVM stack |
duke@435 | 1418 | push_pair(trig); |
duke@435 | 1419 | return true; |
duke@435 | 1420 | } |
duke@435 | 1421 | |
duke@435 | 1422 | //------------------------------inline_sqrt------------------------------------- |
duke@435 | 1423 | // Inline square root instruction, if possible. |
duke@435 | 1424 | bool LibraryCallKit::inline_sqrt(vmIntrinsics::ID id) { |
duke@435 | 1425 | assert(id == vmIntrinsics::_dsqrt, "Not square root"); |
duke@435 | 1426 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1427 | push_pair(_gvn.transform(new (C, 2) SqrtDNode(0, pop_math_arg()))); |
duke@435 | 1428 | return true; |
duke@435 | 1429 | } |
duke@435 | 1430 | |
duke@435 | 1431 | //------------------------------inline_abs------------------------------------- |
duke@435 | 1432 | // Inline absolute value instruction, if possible. |
duke@435 | 1433 | bool LibraryCallKit::inline_abs(vmIntrinsics::ID id) { |
duke@435 | 1434 | assert(id == vmIntrinsics::_dabs, "Not absolute value"); |
duke@435 | 1435 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1436 | push_pair(_gvn.transform(new (C, 2) AbsDNode(pop_math_arg()))); |
duke@435 | 1437 | return true; |
duke@435 | 1438 | } |
duke@435 | 1439 | |
duke@435 | 1440 | //------------------------------inline_exp------------------------------------- |
duke@435 | 1441 | // Inline exp instructions, if possible. The Intel hardware only misses |
duke@435 | 1442 | // really odd corner cases (+/- Infinity). Just uncommon-trap them. |
duke@435 | 1443 | bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) { |
duke@435 | 1444 | assert(id == vmIntrinsics::_dexp, "Not exp"); |
duke@435 | 1445 | |
duke@435 | 1446 | // If this inlining ever returned NaN in the past, we do not intrinsify it |
duke@435 | 1447 | // every again. NaN results requires StrictMath.exp handling. |
duke@435 | 1448 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 1449 | |
duke@435 | 1450 | // Do not intrinsify on older platforms which lack cmove. |
duke@435 | 1451 | if (ConditionalMoveLimit == 0) return false; |
duke@435 | 1452 | |
duke@435 | 1453 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1454 | Node *x = pop_math_arg(); |
duke@435 | 1455 | Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x)); |
duke@435 | 1456 | |
duke@435 | 1457 | //------------------- |
duke@435 | 1458 | //result=(result.isNaN())? StrictMath::exp():result; |
duke@435 | 1459 | // Check: If isNaN() by checking result!=result? then go to Strict Math |
duke@435 | 1460 | Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); |
duke@435 | 1461 | // Build the boolean node |
duke@435 | 1462 | Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); |
duke@435 | 1463 | |
duke@435 | 1464 | { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); |
duke@435 | 1465 | // End the current control-flow path |
duke@435 | 1466 | push_pair(x); |
duke@435 | 1467 | // Math.exp intrinsic returned a NaN, which requires StrictMath.exp |
duke@435 | 1468 | // to handle. Recompile without intrinsifying Math.exp |
duke@435 | 1469 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 1470 | Deoptimization::Action_make_not_entrant); |
duke@435 | 1471 | } |
duke@435 | 1472 | |
duke@435 | 1473 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1474 | |
duke@435 | 1475 | push_pair(result); |
duke@435 | 1476 | |
duke@435 | 1477 | return true; |
duke@435 | 1478 | } |
duke@435 | 1479 | |
duke@435 | 1480 | //------------------------------inline_pow------------------------------------- |
duke@435 | 1481 | // Inline power instructions, if possible. |
duke@435 | 1482 | bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) { |
duke@435 | 1483 | assert(id == vmIntrinsics::_dpow, "Not pow"); |
duke@435 | 1484 | |
duke@435 | 1485 | // If this inlining ever returned NaN in the past, we do not intrinsify it |
duke@435 | 1486 | // every again. NaN results requires StrictMath.pow handling. |
duke@435 | 1487 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 1488 | |
duke@435 | 1489 | // Do not intrinsify on older platforms which lack cmove. |
duke@435 | 1490 | if (ConditionalMoveLimit == 0) return false; |
duke@435 | 1491 | |
duke@435 | 1492 | // Pseudocode for pow |
duke@435 | 1493 | // if (x <= 0.0) { |
duke@435 | 1494 | // if ((double)((int)y)==y) { // if y is int |
duke@435 | 1495 | // result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y) |
duke@435 | 1496 | // } else { |
duke@435 | 1497 | // result = NaN; |
duke@435 | 1498 | // } |
duke@435 | 1499 | // } else { |
duke@435 | 1500 | // result = DPow(x,y); |
duke@435 | 1501 | // } |
duke@435 | 1502 | // if (result != result)? { |
twisti@1040 | 1503 | // uncommon_trap(); |
duke@435 | 1504 | // } |
duke@435 | 1505 | // return result; |
duke@435 | 1506 | |
duke@435 | 1507 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1508 | Node* y = pop_math_arg(); |
duke@435 | 1509 | Node* x = pop_math_arg(); |
duke@435 | 1510 | |
duke@435 | 1511 | Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) ); |
duke@435 | 1512 | |
duke@435 | 1513 | // Short form: if not top-level (i.e., Math.pow but inlining Math.pow |
duke@435 | 1514 | // inside of something) then skip the fancy tests and just check for |
duke@435 | 1515 | // NaN result. |
duke@435 | 1516 | Node *result = NULL; |
duke@435 | 1517 | if( jvms()->depth() >= 1 ) { |
duke@435 | 1518 | result = fast_result; |
duke@435 | 1519 | } else { |
duke@435 | 1520 | |
duke@435 | 1521 | // Set the merge point for If node with condition of (x <= 0.0) |
duke@435 | 1522 | // There are four possible paths to region node and phi node |
duke@435 | 1523 | RegionNode *r = new (C, 4) RegionNode(4); |
duke@435 | 1524 | Node *phi = new (C, 4) PhiNode(r, Type::DOUBLE); |
duke@435 | 1525 | |
duke@435 | 1526 | // Build the first if node: if (x <= 0.0) |
duke@435 | 1527 | // Node for 0 constant |
duke@435 | 1528 | Node *zeronode = makecon(TypeD::ZERO); |
duke@435 | 1529 | // Check x:0 |
duke@435 | 1530 | Node *cmp = _gvn.transform(new (C, 3) CmpDNode(x, zeronode)); |
duke@435 | 1531 | // Check: If (x<=0) then go complex path |
duke@435 | 1532 | Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) ); |
duke@435 | 1533 | // Branch either way |
duke@435 | 1534 | IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); |
duke@435 | 1535 | Node *opt_test = _gvn.transform(if1); |
duke@435 | 1536 | //assert( opt_test->is_If(), "Expect an IfNode"); |
duke@435 | 1537 | IfNode *opt_if1 = (IfNode*)opt_test; |
duke@435 | 1538 | // Fast path taken; set region slot 3 |
duke@435 | 1539 | Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) ); |
duke@435 | 1540 | r->init_req(3,fast_taken); // Capture fast-control |
duke@435 | 1541 | |
duke@435 | 1542 | // Fast path not-taken, i.e. slow path |
duke@435 | 1543 | Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) ); |
duke@435 | 1544 | |
duke@435 | 1545 | // Set fast path result |
duke@435 | 1546 | Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) ); |
duke@435 | 1547 | phi->init_req(3, fast_result); |
duke@435 | 1548 | |
duke@435 | 1549 | // Complex path |
duke@435 | 1550 | // Build the second if node (if y is int) |
duke@435 | 1551 | // Node for (int)y |
duke@435 | 1552 | Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y)); |
duke@435 | 1553 | // Node for (double)((int) y) |
duke@435 | 1554 | Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty)); |
duke@435 | 1555 | // Check (double)((int) y) : y |
duke@435 | 1556 | Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y)); |
duke@435 | 1557 | // Check if (y isn't int) then go to slow path |
duke@435 | 1558 | |
duke@435 | 1559 | Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) ); |
twisti@1040 | 1560 | // Branch either way |
duke@435 | 1561 | IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); |
duke@435 | 1562 | Node *slow_path = opt_iff(r,if2); // Set region path 2 |
duke@435 | 1563 | |
duke@435 | 1564 | // Calculate DPow(abs(x), y)*(1 & (int)y) |
duke@435 | 1565 | // Node for constant 1 |
duke@435 | 1566 | Node *conone = intcon(1); |
duke@435 | 1567 | // 1& (int)y |
duke@435 | 1568 | Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) ); |
duke@435 | 1569 | // zero node |
duke@435 | 1570 | Node *conzero = intcon(0); |
duke@435 | 1571 | // Check (1&(int)y)==0? |
duke@435 | 1572 | Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero)); |
duke@435 | 1573 | // Check if (1&(int)y)!=0?, if so the result is negative |
duke@435 | 1574 | Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) ); |
duke@435 | 1575 | // abs(x) |
duke@435 | 1576 | Node *absx=_gvn.transform( new (C, 2) AbsDNode(x)); |
duke@435 | 1577 | // abs(x)^y |
duke@435 | 1578 | Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) ); |
duke@435 | 1579 | // -abs(x)^y |
duke@435 | 1580 | Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy)); |
duke@435 | 1581 | // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y) |
duke@435 | 1582 | Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE)); |
duke@435 | 1583 | // Set complex path fast result |
duke@435 | 1584 | phi->init_req(2, signresult); |
duke@435 | 1585 | |
duke@435 | 1586 | static const jlong nan_bits = CONST64(0x7ff8000000000000); |
duke@435 | 1587 | Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN |
duke@435 | 1588 | r->init_req(1,slow_path); |
duke@435 | 1589 | phi->init_req(1,slow_result); |
duke@435 | 1590 | |
duke@435 | 1591 | // Post merge |
duke@435 | 1592 | set_control(_gvn.transform(r)); |
duke@435 | 1593 | record_for_igvn(r); |
duke@435 | 1594 | result=_gvn.transform(phi); |
duke@435 | 1595 | } |
duke@435 | 1596 | |
duke@435 | 1597 | //------------------- |
duke@435 | 1598 | //result=(result.isNaN())? uncommon_trap():result; |
duke@435 | 1599 | // Check: If isNaN() by checking result!=result? then go to Strict Math |
duke@435 | 1600 | Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); |
duke@435 | 1601 | // Build the boolean node |
duke@435 | 1602 | Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); |
duke@435 | 1603 | |
duke@435 | 1604 | { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); |
duke@435 | 1605 | // End the current control-flow path |
duke@435 | 1606 | push_pair(x); |
duke@435 | 1607 | push_pair(y); |
duke@435 | 1608 | // Math.pow intrinsic returned a NaN, which requires StrictMath.pow |
duke@435 | 1609 | // to handle. Recompile without intrinsifying Math.pow. |
duke@435 | 1610 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 1611 | Deoptimization::Action_make_not_entrant); |
duke@435 | 1612 | } |
duke@435 | 1613 | |
duke@435 | 1614 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1615 | |
duke@435 | 1616 | push_pair(result); |
duke@435 | 1617 | |
duke@435 | 1618 | return true; |
duke@435 | 1619 | } |
duke@435 | 1620 | |
duke@435 | 1621 | //------------------------------inline_trans------------------------------------- |
duke@435 | 1622 | // Inline transcendental instructions, if possible. The Intel hardware gets |
duke@435 | 1623 | // these right, no funny corner cases missed. |
duke@435 | 1624 | bool LibraryCallKit::inline_trans(vmIntrinsics::ID id) { |
duke@435 | 1625 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1626 | Node* arg = pop_math_arg(); |
duke@435 | 1627 | Node* trans = NULL; |
duke@435 | 1628 | |
duke@435 | 1629 | switch (id) { |
duke@435 | 1630 | case vmIntrinsics::_dlog: |
duke@435 | 1631 | trans = _gvn.transform((Node*)new (C, 2) LogDNode(arg)); |
duke@435 | 1632 | break; |
duke@435 | 1633 | case vmIntrinsics::_dlog10: |
duke@435 | 1634 | trans = _gvn.transform((Node*)new (C, 2) Log10DNode(arg)); |
duke@435 | 1635 | break; |
duke@435 | 1636 | default: |
duke@435 | 1637 | assert(false, "bad intrinsic was passed in"); |
duke@435 | 1638 | return false; |
duke@435 | 1639 | } |
duke@435 | 1640 | |
duke@435 | 1641 | // Push result back on JVM stack |
duke@435 | 1642 | push_pair(trans); |
duke@435 | 1643 | return true; |
duke@435 | 1644 | } |
duke@435 | 1645 | |
duke@435 | 1646 | //------------------------------runtime_math----------------------------- |
duke@435 | 1647 | bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) { |
duke@435 | 1648 | Node* a = NULL; |
duke@435 | 1649 | Node* b = NULL; |
duke@435 | 1650 | |
duke@435 | 1651 | assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(), |
duke@435 | 1652 | "must be (DD)D or (D)D type"); |
duke@435 | 1653 | |
duke@435 | 1654 | // Inputs |
duke@435 | 1655 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1656 | if (call_type == OptoRuntime::Math_DD_D_Type()) { |
duke@435 | 1657 | b = pop_math_arg(); |
duke@435 | 1658 | } |
duke@435 | 1659 | a = pop_math_arg(); |
duke@435 | 1660 | |
duke@435 | 1661 | const TypePtr* no_memory_effects = NULL; |
duke@435 | 1662 | Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, |
duke@435 | 1663 | no_memory_effects, |
duke@435 | 1664 | a, top(), b, b ? top() : NULL); |
duke@435 | 1665 | Node* value = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+0)); |
duke@435 | 1666 | #ifdef ASSERT |
duke@435 | 1667 | Node* value_top = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+1)); |
duke@435 | 1668 | assert(value_top == top(), "second value must be top"); |
duke@435 | 1669 | #endif |
duke@435 | 1670 | |
duke@435 | 1671 | push_pair(value); |
duke@435 | 1672 | return true; |
duke@435 | 1673 | } |
duke@435 | 1674 | |
duke@435 | 1675 | //------------------------------inline_math_native----------------------------- |
duke@435 | 1676 | bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { |
duke@435 | 1677 | switch (id) { |
duke@435 | 1678 | // These intrinsics are not properly supported on all hardware |
duke@435 | 1679 | case vmIntrinsics::_dcos: return Matcher::has_match_rule(Op_CosD) ? inline_trig(id) : |
duke@435 | 1680 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dcos), "COS"); |
duke@435 | 1681 | case vmIntrinsics::_dsin: return Matcher::has_match_rule(Op_SinD) ? inline_trig(id) : |
duke@435 | 1682 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dsin), "SIN"); |
duke@435 | 1683 | case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) : |
duke@435 | 1684 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dtan), "TAN"); |
duke@435 | 1685 | |
duke@435 | 1686 | case vmIntrinsics::_dlog: return Matcher::has_match_rule(Op_LogD) ? inline_trans(id) : |
duke@435 | 1687 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog), "LOG"); |
duke@435 | 1688 | case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_trans(id) : |
duke@435 | 1689 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), "LOG10"); |
duke@435 | 1690 | |
duke@435 | 1691 | // These intrinsics are supported on all hardware |
duke@435 | 1692 | case vmIntrinsics::_dsqrt: return Matcher::has_match_rule(Op_SqrtD) ? inline_sqrt(id) : false; |
duke@435 | 1693 | case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_abs(id) : false; |
duke@435 | 1694 | |
duke@435 | 1695 | // These intrinsics don't work on X86. The ad implementation doesn't |
duke@435 | 1696 | // handle NaN's properly. Instead of returning infinity, the ad |
duke@435 | 1697 | // implementation returns a NaN on overflow. See bug: 6304089 |
duke@435 | 1698 | // Once the ad implementations are fixed, change the code below |
duke@435 | 1699 | // to match the intrinsics above |
duke@435 | 1700 | |
duke@435 | 1701 | case vmIntrinsics::_dexp: return |
duke@435 | 1702 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP"); |
duke@435 | 1703 | case vmIntrinsics::_dpow: return |
duke@435 | 1704 | runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); |
duke@435 | 1705 | |
duke@435 | 1706 | // These intrinsics are not yet correctly implemented |
duke@435 | 1707 | case vmIntrinsics::_datan2: |
duke@435 | 1708 | return false; |
duke@435 | 1709 | |
duke@435 | 1710 | default: |
duke@435 | 1711 | ShouldNotReachHere(); |
duke@435 | 1712 | return false; |
duke@435 | 1713 | } |
duke@435 | 1714 | } |
duke@435 | 1715 | |
duke@435 | 1716 | static bool is_simple_name(Node* n) { |
duke@435 | 1717 | return (n->req() == 1 // constant |
duke@435 | 1718 | || (n->is_Type() && n->as_Type()->type()->singleton()) |
duke@435 | 1719 | || n->is_Proj() // parameter or return value |
duke@435 | 1720 | || n->is_Phi() // local of some sort |
duke@435 | 1721 | ); |
duke@435 | 1722 | } |
duke@435 | 1723 | |
duke@435 | 1724 | //----------------------------inline_min_max----------------------------------- |
duke@435 | 1725 | bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) { |
duke@435 | 1726 | push(generate_min_max(id, argument(0), argument(1))); |
duke@435 | 1727 | |
duke@435 | 1728 | return true; |
duke@435 | 1729 | } |
duke@435 | 1730 | |
duke@435 | 1731 | Node* |
duke@435 | 1732 | LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { |
duke@435 | 1733 | // These are the candidate return value: |
duke@435 | 1734 | Node* xvalue = x0; |
duke@435 | 1735 | Node* yvalue = y0; |
duke@435 | 1736 | |
duke@435 | 1737 | if (xvalue == yvalue) { |
duke@435 | 1738 | return xvalue; |
duke@435 | 1739 | } |
duke@435 | 1740 | |
duke@435 | 1741 | bool want_max = (id == vmIntrinsics::_max); |
duke@435 | 1742 | |
duke@435 | 1743 | const TypeInt* txvalue = _gvn.type(xvalue)->isa_int(); |
duke@435 | 1744 | const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int(); |
duke@435 | 1745 | if (txvalue == NULL || tyvalue == NULL) return top(); |
duke@435 | 1746 | // This is not really necessary, but it is consistent with a |
duke@435 | 1747 | // hypothetical MaxINode::Value method: |
duke@435 | 1748 | int widen = MAX2(txvalue->_widen, tyvalue->_widen); |
duke@435 | 1749 | |
duke@435 | 1750 | // %%% This folding logic should (ideally) be in a different place. |
duke@435 | 1751 | // Some should be inside IfNode, and there to be a more reliable |
duke@435 | 1752 | // transformation of ?: style patterns into cmoves. We also want |
duke@435 | 1753 | // more powerful optimizations around cmove and min/max. |
duke@435 | 1754 | |
duke@435 | 1755 | // Try to find a dominating comparison of these guys. |
duke@435 | 1756 | // It can simplify the index computation for Arrays.copyOf |
duke@435 | 1757 | // and similar uses of System.arraycopy. |
duke@435 | 1758 | // First, compute the normalized version of CmpI(x, y). |
duke@435 | 1759 | int cmp_op = Op_CmpI; |
duke@435 | 1760 | Node* xkey = xvalue; |
duke@435 | 1761 | Node* ykey = yvalue; |
duke@435 | 1762 | Node* ideal_cmpxy = _gvn.transform( new(C, 3) CmpINode(xkey, ykey) ); |
duke@435 | 1763 | if (ideal_cmpxy->is_Cmp()) { |
duke@435 | 1764 | // E.g., if we have CmpI(length - offset, count), |
duke@435 | 1765 | // it might idealize to CmpI(length, count + offset) |
duke@435 | 1766 | cmp_op = ideal_cmpxy->Opcode(); |
duke@435 | 1767 | xkey = ideal_cmpxy->in(1); |
duke@435 | 1768 | ykey = ideal_cmpxy->in(2); |
duke@435 | 1769 | } |
duke@435 | 1770 | |
duke@435 | 1771 | // Start by locating any relevant comparisons. |
duke@435 | 1772 | Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey; |
duke@435 | 1773 | Node* cmpxy = NULL; |
duke@435 | 1774 | Node* cmpyx = NULL; |
duke@435 | 1775 | for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) { |
duke@435 | 1776 | Node* cmp = start_from->fast_out(k); |
duke@435 | 1777 | if (cmp->outcnt() > 0 && // must have prior uses |
duke@435 | 1778 | cmp->in(0) == NULL && // must be context-independent |
duke@435 | 1779 | cmp->Opcode() == cmp_op) { // right kind of compare |
duke@435 | 1780 | if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp; |
duke@435 | 1781 | if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp; |
duke@435 | 1782 | } |
duke@435 | 1783 | } |
duke@435 | 1784 | |
duke@435 | 1785 | const int NCMPS = 2; |
duke@435 | 1786 | Node* cmps[NCMPS] = { cmpxy, cmpyx }; |
duke@435 | 1787 | int cmpn; |
duke@435 | 1788 | for (cmpn = 0; cmpn < NCMPS; cmpn++) { |
duke@435 | 1789 | if (cmps[cmpn] != NULL) break; // find a result |
duke@435 | 1790 | } |
duke@435 | 1791 | if (cmpn < NCMPS) { |
duke@435 | 1792 | // Look for a dominating test that tells us the min and max. |
duke@435 | 1793 | int depth = 0; // Limit search depth for speed |
duke@435 | 1794 | Node* dom = control(); |
duke@435 | 1795 | for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) { |
duke@435 | 1796 | if (++depth >= 100) break; |
duke@435 | 1797 | Node* ifproj = dom; |
duke@435 | 1798 | if (!ifproj->is_Proj()) continue; |
duke@435 | 1799 | Node* iff = ifproj->in(0); |
duke@435 | 1800 | if (!iff->is_If()) continue; |
duke@435 | 1801 | Node* bol = iff->in(1); |
duke@435 | 1802 | if (!bol->is_Bool()) continue; |
duke@435 | 1803 | Node* cmp = bol->in(1); |
duke@435 | 1804 | if (cmp == NULL) continue; |
duke@435 | 1805 | for (cmpn = 0; cmpn < NCMPS; cmpn++) |
duke@435 | 1806 | if (cmps[cmpn] == cmp) break; |
duke@435 | 1807 | if (cmpn == NCMPS) continue; |
duke@435 | 1808 | BoolTest::mask btest = bol->as_Bool()->_test._test; |
duke@435 | 1809 | if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate(); |
duke@435 | 1810 | if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); |
duke@435 | 1811 | // At this point, we know that 'x btest y' is true. |
duke@435 | 1812 | switch (btest) { |
duke@435 | 1813 | case BoolTest::eq: |
duke@435 | 1814 | // They are proven equal, so we can collapse the min/max. |
duke@435 | 1815 | // Either value is the answer. Choose the simpler. |
duke@435 | 1816 | if (is_simple_name(yvalue) && !is_simple_name(xvalue)) |
duke@435 | 1817 | return yvalue; |
duke@435 | 1818 | return xvalue; |
duke@435 | 1819 | case BoolTest::lt: // x < y |
duke@435 | 1820 | case BoolTest::le: // x <= y |
duke@435 | 1821 | return (want_max ? yvalue : xvalue); |
duke@435 | 1822 | case BoolTest::gt: // x > y |
duke@435 | 1823 | case BoolTest::ge: // x >= y |
duke@435 | 1824 | return (want_max ? xvalue : yvalue); |
duke@435 | 1825 | } |
duke@435 | 1826 | } |
duke@435 | 1827 | } |
duke@435 | 1828 | |
duke@435 | 1829 | // We failed to find a dominating test. |
duke@435 | 1830 | // Let's pick a test that might GVN with prior tests. |
duke@435 | 1831 | Node* best_bol = NULL; |
duke@435 | 1832 | BoolTest::mask best_btest = BoolTest::illegal; |
duke@435 | 1833 | for (cmpn = 0; cmpn < NCMPS; cmpn++) { |
duke@435 | 1834 | Node* cmp = cmps[cmpn]; |
duke@435 | 1835 | if (cmp == NULL) continue; |
duke@435 | 1836 | for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) { |
duke@435 | 1837 | Node* bol = cmp->fast_out(j); |
duke@435 | 1838 | if (!bol->is_Bool()) continue; |
duke@435 | 1839 | BoolTest::mask btest = bol->as_Bool()->_test._test; |
duke@435 | 1840 | if (btest == BoolTest::eq || btest == BoolTest::ne) continue; |
duke@435 | 1841 | if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); |
duke@435 | 1842 | if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) { |
duke@435 | 1843 | best_bol = bol->as_Bool(); |
duke@435 | 1844 | best_btest = btest; |
duke@435 | 1845 | } |
duke@435 | 1846 | } |
duke@435 | 1847 | } |
duke@435 | 1848 | |
duke@435 | 1849 | Node* answer_if_true = NULL; |
duke@435 | 1850 | Node* answer_if_false = NULL; |
duke@435 | 1851 | switch (best_btest) { |
duke@435 | 1852 | default: |
duke@435 | 1853 | if (cmpxy == NULL) |
duke@435 | 1854 | cmpxy = ideal_cmpxy; |
duke@435 | 1855 | best_bol = _gvn.transform( new(C, 2) BoolNode(cmpxy, BoolTest::lt) ); |
duke@435 | 1856 | // and fall through: |
duke@435 | 1857 | case BoolTest::lt: // x < y |
duke@435 | 1858 | case BoolTest::le: // x <= y |
duke@435 | 1859 | answer_if_true = (want_max ? yvalue : xvalue); |
duke@435 | 1860 | answer_if_false = (want_max ? xvalue : yvalue); |
duke@435 | 1861 | break; |
duke@435 | 1862 | case BoolTest::gt: // x > y |
duke@435 | 1863 | case BoolTest::ge: // x >= y |
duke@435 | 1864 | answer_if_true = (want_max ? xvalue : yvalue); |
duke@435 | 1865 | answer_if_false = (want_max ? yvalue : xvalue); |
duke@435 | 1866 | break; |
duke@435 | 1867 | } |
duke@435 | 1868 | |
duke@435 | 1869 | jint hi, lo; |
duke@435 | 1870 | if (want_max) { |
duke@435 | 1871 | // We can sharpen the minimum. |
duke@435 | 1872 | hi = MAX2(txvalue->_hi, tyvalue->_hi); |
duke@435 | 1873 | lo = MAX2(txvalue->_lo, tyvalue->_lo); |
duke@435 | 1874 | } else { |
duke@435 | 1875 | // We can sharpen the maximum. |
duke@435 | 1876 | hi = MIN2(txvalue->_hi, tyvalue->_hi); |
duke@435 | 1877 | lo = MIN2(txvalue->_lo, tyvalue->_lo); |
duke@435 | 1878 | } |
duke@435 | 1879 | |
duke@435 | 1880 | // Use a flow-free graph structure, to avoid creating excess control edges |
duke@435 | 1881 | // which could hinder other optimizations. |
duke@435 | 1882 | // Since Math.min/max is often used with arraycopy, we want |
duke@435 | 1883 | // tightly_coupled_allocation to be able to see beyond min/max expressions. |
duke@435 | 1884 | Node* cmov = CMoveNode::make(C, NULL, best_bol, |
duke@435 | 1885 | answer_if_false, answer_if_true, |
duke@435 | 1886 | TypeInt::make(lo, hi, widen)); |
duke@435 | 1887 | |
duke@435 | 1888 | return _gvn.transform(cmov); |
duke@435 | 1889 | |
duke@435 | 1890 | /* |
duke@435 | 1891 | // This is not as desirable as it may seem, since Min and Max |
duke@435 | 1892 | // nodes do not have a full set of optimizations. |
duke@435 | 1893 | // And they would interfere, anyway, with 'if' optimizations |
duke@435 | 1894 | // and with CMoveI canonical forms. |
duke@435 | 1895 | switch (id) { |
duke@435 | 1896 | case vmIntrinsics::_min: |
duke@435 | 1897 | result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break; |
duke@435 | 1898 | case vmIntrinsics::_max: |
duke@435 | 1899 | result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break; |
duke@435 | 1900 | default: |
duke@435 | 1901 | ShouldNotReachHere(); |
duke@435 | 1902 | } |
duke@435 | 1903 | */ |
duke@435 | 1904 | } |
duke@435 | 1905 | |
duke@435 | 1906 | inline int |
duke@435 | 1907 | LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) { |
duke@435 | 1908 | const TypePtr* base_type = TypePtr::NULL_PTR; |
duke@435 | 1909 | if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); |
duke@435 | 1910 | if (base_type == NULL) { |
duke@435 | 1911 | // Unknown type. |
duke@435 | 1912 | return Type::AnyPtr; |
duke@435 | 1913 | } else if (base_type == TypePtr::NULL_PTR) { |
duke@435 | 1914 | // Since this is a NULL+long form, we have to switch to a rawptr. |
duke@435 | 1915 | base = _gvn.transform( new (C, 2) CastX2PNode(offset) ); |
duke@435 | 1916 | offset = MakeConX(0); |
duke@435 | 1917 | return Type::RawPtr; |
duke@435 | 1918 | } else if (base_type->base() == Type::RawPtr) { |
duke@435 | 1919 | return Type::RawPtr; |
duke@435 | 1920 | } else if (base_type->isa_oopptr()) { |
duke@435 | 1921 | // Base is never null => always a heap address. |
duke@435 | 1922 | if (base_type->ptr() == TypePtr::NotNull) { |
duke@435 | 1923 | return Type::OopPtr; |
duke@435 | 1924 | } |
duke@435 | 1925 | // Offset is small => always a heap address. |
duke@435 | 1926 | const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); |
duke@435 | 1927 | if (offset_type != NULL && |
duke@435 | 1928 | base_type->offset() == 0 && // (should always be?) |
duke@435 | 1929 | offset_type->_lo >= 0 && |
duke@435 | 1930 | !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { |
duke@435 | 1931 | return Type::OopPtr; |
duke@435 | 1932 | } |
duke@435 | 1933 | // Otherwise, it might either be oop+off or NULL+addr. |
duke@435 | 1934 | return Type::AnyPtr; |
duke@435 | 1935 | } else { |
duke@435 | 1936 | // No information: |
duke@435 | 1937 | return Type::AnyPtr; |
duke@435 | 1938 | } |
duke@435 | 1939 | } |
duke@435 | 1940 | |
duke@435 | 1941 | inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) { |
duke@435 | 1942 | int kind = classify_unsafe_addr(base, offset); |
duke@435 | 1943 | if (kind == Type::RawPtr) { |
duke@435 | 1944 | return basic_plus_adr(top(), base, offset); |
duke@435 | 1945 | } else { |
duke@435 | 1946 | return basic_plus_adr(base, offset); |
duke@435 | 1947 | } |
duke@435 | 1948 | } |
duke@435 | 1949 | |
twisti@1210 | 1950 | //-------------------inline_numberOfLeadingZeros_int/long----------------------- |
twisti@1210 | 1951 | // inline int Integer.numberOfLeadingZeros(int) |
twisti@1210 | 1952 | // inline int Long.numberOfLeadingZeros(long) |
twisti@1210 | 1953 | bool LibraryCallKit::inline_numberOfLeadingZeros(vmIntrinsics::ID id) { |
twisti@1210 | 1954 | assert(id == vmIntrinsics::_numberOfLeadingZeros_i || id == vmIntrinsics::_numberOfLeadingZeros_l, "not numberOfLeadingZeros"); |
twisti@1210 | 1955 | if (id == vmIntrinsics::_numberOfLeadingZeros_i && !Matcher::match_rule_supported(Op_CountLeadingZerosI)) return false; |
twisti@1210 | 1956 | if (id == vmIntrinsics::_numberOfLeadingZeros_l && !Matcher::match_rule_supported(Op_CountLeadingZerosL)) return false; |
twisti@1210 | 1957 | _sp += arg_size(); // restore stack pointer |
twisti@1210 | 1958 | switch (id) { |
twisti@1210 | 1959 | case vmIntrinsics::_numberOfLeadingZeros_i: |
twisti@1210 | 1960 | push(_gvn.transform(new (C, 2) CountLeadingZerosINode(pop()))); |
twisti@1210 | 1961 | break; |
twisti@1210 | 1962 | case vmIntrinsics::_numberOfLeadingZeros_l: |
twisti@1210 | 1963 | push(_gvn.transform(new (C, 2) CountLeadingZerosLNode(pop_pair()))); |
twisti@1210 | 1964 | break; |
twisti@1210 | 1965 | default: |
twisti@1210 | 1966 | ShouldNotReachHere(); |
twisti@1210 | 1967 | } |
twisti@1210 | 1968 | return true; |
twisti@1210 | 1969 | } |
twisti@1210 | 1970 | |
twisti@1210 | 1971 | //-------------------inline_numberOfTrailingZeros_int/long---------------------- |
twisti@1210 | 1972 | // inline int Integer.numberOfTrailingZeros(int) |
twisti@1210 | 1973 | // inline int Long.numberOfTrailingZeros(long) |
twisti@1210 | 1974 | bool LibraryCallKit::inline_numberOfTrailingZeros(vmIntrinsics::ID id) { |
twisti@1210 | 1975 | assert(id == vmIntrinsics::_numberOfTrailingZeros_i || id == vmIntrinsics::_numberOfTrailingZeros_l, "not numberOfTrailingZeros"); |
twisti@1210 | 1976 | if (id == vmIntrinsics::_numberOfTrailingZeros_i && !Matcher::match_rule_supported(Op_CountTrailingZerosI)) return false; |
twisti@1210 | 1977 | if (id == vmIntrinsics::_numberOfTrailingZeros_l && !Matcher::match_rule_supported(Op_CountTrailingZerosL)) return false; |
twisti@1210 | 1978 | _sp += arg_size(); // restore stack pointer |
twisti@1210 | 1979 | switch (id) { |
twisti@1210 | 1980 | case vmIntrinsics::_numberOfTrailingZeros_i: |
twisti@1210 | 1981 | push(_gvn.transform(new (C, 2) CountTrailingZerosINode(pop()))); |
twisti@1210 | 1982 | break; |
twisti@1210 | 1983 | case vmIntrinsics::_numberOfTrailingZeros_l: |
twisti@1210 | 1984 | push(_gvn.transform(new (C, 2) CountTrailingZerosLNode(pop_pair()))); |
twisti@1210 | 1985 | break; |
twisti@1210 | 1986 | default: |
twisti@1210 | 1987 | ShouldNotReachHere(); |
twisti@1210 | 1988 | } |
twisti@1210 | 1989 | return true; |
twisti@1210 | 1990 | } |
twisti@1210 | 1991 | |
twisti@1078 | 1992 | //----------------------------inline_bitCount_int/long----------------------- |
twisti@1078 | 1993 | // inline int Integer.bitCount(int) |
twisti@1078 | 1994 | // inline int Long.bitCount(long) |
twisti@1078 | 1995 | bool LibraryCallKit::inline_bitCount(vmIntrinsics::ID id) { |
twisti@1078 | 1996 | assert(id == vmIntrinsics::_bitCount_i || id == vmIntrinsics::_bitCount_l, "not bitCount"); |
twisti@1078 | 1997 | if (id == vmIntrinsics::_bitCount_i && !Matcher::has_match_rule(Op_PopCountI)) return false; |
twisti@1078 | 1998 | if (id == vmIntrinsics::_bitCount_l && !Matcher::has_match_rule(Op_PopCountL)) return false; |
twisti@1078 | 1999 | _sp += arg_size(); // restore stack pointer |
twisti@1078 | 2000 | switch (id) { |
twisti@1078 | 2001 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 2002 | push(_gvn.transform(new (C, 2) PopCountINode(pop()))); |
twisti@1078 | 2003 | break; |
twisti@1078 | 2004 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 2005 | push(_gvn.transform(new (C, 2) PopCountLNode(pop_pair()))); |
twisti@1078 | 2006 | break; |
twisti@1078 | 2007 | default: |
twisti@1078 | 2008 | ShouldNotReachHere(); |
twisti@1078 | 2009 | } |
twisti@1078 | 2010 | return true; |
twisti@1078 | 2011 | } |
twisti@1078 | 2012 | |
duke@435 | 2013 | //----------------------------inline_reverseBytes_int/long------------------- |
twisti@1040 | 2014 | // inline Integer.reverseBytes(int) |
twisti@1040 | 2015 | // inline Long.reverseBytes(long) |
duke@435 | 2016 | bool LibraryCallKit::inline_reverseBytes(vmIntrinsics::ID id) { |
duke@435 | 2017 | assert(id == vmIntrinsics::_reverseBytes_i || id == vmIntrinsics::_reverseBytes_l, "not reverse Bytes"); |
duke@435 | 2018 | if (id == vmIntrinsics::_reverseBytes_i && !Matcher::has_match_rule(Op_ReverseBytesI)) return false; |
duke@435 | 2019 | if (id == vmIntrinsics::_reverseBytes_l && !Matcher::has_match_rule(Op_ReverseBytesL)) return false; |
duke@435 | 2020 | _sp += arg_size(); // restore stack pointer |
duke@435 | 2021 | switch (id) { |
duke@435 | 2022 | case vmIntrinsics::_reverseBytes_i: |
duke@435 | 2023 | push(_gvn.transform(new (C, 2) ReverseBytesINode(0, pop()))); |
duke@435 | 2024 | break; |
duke@435 | 2025 | case vmIntrinsics::_reverseBytes_l: |
duke@435 | 2026 | push_pair(_gvn.transform(new (C, 2) ReverseBytesLNode(0, pop_pair()))); |
duke@435 | 2027 | break; |
duke@435 | 2028 | default: |
duke@435 | 2029 | ; |
duke@435 | 2030 | } |
duke@435 | 2031 | return true; |
duke@435 | 2032 | } |
duke@435 | 2033 | |
duke@435 | 2034 | //----------------------------inline_unsafe_access---------------------------- |
duke@435 | 2035 | |
duke@435 | 2036 | const static BasicType T_ADDRESS_HOLDER = T_LONG; |
duke@435 | 2037 | |
duke@435 | 2038 | // Interpret Unsafe.fieldOffset cookies correctly: |
duke@435 | 2039 | extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset); |
duke@435 | 2040 | |
duke@435 | 2041 | bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) { |
duke@435 | 2042 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2043 | |
duke@435 | 2044 | #ifndef PRODUCT |
duke@435 | 2045 | { |
duke@435 | 2046 | ResourceMark rm; |
duke@435 | 2047 | // Check the signatures. |
duke@435 | 2048 | ciSignature* sig = signature(); |
duke@435 | 2049 | #ifdef ASSERT |
duke@435 | 2050 | if (!is_store) { |
duke@435 | 2051 | // Object getObject(Object base, int/long offset), etc. |
duke@435 | 2052 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2053 | if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name()) |
duke@435 | 2054 | rtype = T_ADDRESS; // it is really a C void* |
duke@435 | 2055 | assert(rtype == type, "getter must return the expected value"); |
duke@435 | 2056 | if (!is_native_ptr) { |
duke@435 | 2057 | assert(sig->count() == 2, "oop getter has 2 arguments"); |
duke@435 | 2058 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); |
duke@435 | 2059 | assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); |
duke@435 | 2060 | } else { |
duke@435 | 2061 | assert(sig->count() == 1, "native getter has 1 argument"); |
duke@435 | 2062 | assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long"); |
duke@435 | 2063 | } |
duke@435 | 2064 | } else { |
duke@435 | 2065 | // void putObject(Object base, int/long offset, Object x), etc. |
duke@435 | 2066 | assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); |
duke@435 | 2067 | if (!is_native_ptr) { |
duke@435 | 2068 | assert(sig->count() == 3, "oop putter has 3 arguments"); |
duke@435 | 2069 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); |
duke@435 | 2070 | assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); |
duke@435 | 2071 | } else { |
duke@435 | 2072 | assert(sig->count() == 2, "native putter has 2 arguments"); |
duke@435 | 2073 | assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long"); |
duke@435 | 2074 | } |
duke@435 | 2075 | BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); |
duke@435 | 2076 | if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name()) |
duke@435 | 2077 | vtype = T_ADDRESS; // it is really a C void* |
duke@435 | 2078 | assert(vtype == type, "putter must accept the expected value"); |
duke@435 | 2079 | } |
duke@435 | 2080 | #endif // ASSERT |
duke@435 | 2081 | } |
duke@435 | 2082 | #endif //PRODUCT |
duke@435 | 2083 | |
duke@435 | 2084 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2085 | |
duke@435 | 2086 | int type_words = type2size[ (type == T_ADDRESS) ? T_LONG : type ]; |
duke@435 | 2087 | |
duke@435 | 2088 | // Argument words: "this" plus (oop/offset) or (lo/hi) args plus maybe 1 or 2 value words |
duke@435 | 2089 | int nargs = 1 + (is_native_ptr ? 2 : 3) + (is_store ? type_words : 0); |
duke@435 | 2090 | |
duke@435 | 2091 | debug_only(int saved_sp = _sp); |
duke@435 | 2092 | _sp += nargs; |
duke@435 | 2093 | |
duke@435 | 2094 | Node* val; |
duke@435 | 2095 | debug_only(val = (Node*)(uintptr_t)-1); |
duke@435 | 2096 | |
duke@435 | 2097 | |
duke@435 | 2098 | if (is_store) { |
duke@435 | 2099 | // Get the value being stored. (Pop it first; it was pushed last.) |
duke@435 | 2100 | switch (type) { |
duke@435 | 2101 | case T_DOUBLE: |
duke@435 | 2102 | case T_LONG: |
duke@435 | 2103 | case T_ADDRESS: |
duke@435 | 2104 | val = pop_pair(); |
duke@435 | 2105 | break; |
duke@435 | 2106 | default: |
duke@435 | 2107 | val = pop(); |
duke@435 | 2108 | } |
duke@435 | 2109 | } |
duke@435 | 2110 | |
duke@435 | 2111 | // Build address expression. See the code in inline_unsafe_prefetch. |
duke@435 | 2112 | Node *adr; |
duke@435 | 2113 | Node *heap_base_oop = top(); |
duke@435 | 2114 | if (!is_native_ptr) { |
duke@435 | 2115 | // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset |
duke@435 | 2116 | Node* offset = pop_pair(); |
duke@435 | 2117 | // The base is either a Java object or a value produced by Unsafe.staticFieldBase |
duke@435 | 2118 | Node* base = pop(); |
duke@435 | 2119 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2120 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2121 | // by oopDesc::field_base. |
duke@435 | 2122 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 2123 | "fieldOffset must be byte-scaled"); |
duke@435 | 2124 | // 32-bit machines ignore the high half! |
duke@435 | 2125 | offset = ConvL2X(offset); |
duke@435 | 2126 | adr = make_unsafe_address(base, offset); |
duke@435 | 2127 | heap_base_oop = base; |
duke@435 | 2128 | } else { |
duke@435 | 2129 | Node* ptr = pop_pair(); |
duke@435 | 2130 | // Adjust Java long to machine word: |
duke@435 | 2131 | ptr = ConvL2X(ptr); |
duke@435 | 2132 | adr = make_unsafe_address(NULL, ptr); |
duke@435 | 2133 | } |
duke@435 | 2134 | |
duke@435 | 2135 | // Pop receiver last: it was pushed first. |
duke@435 | 2136 | Node *receiver = pop(); |
duke@435 | 2137 | |
duke@435 | 2138 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2139 | |
duke@435 | 2140 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2141 | |
duke@435 | 2142 | // First guess at the value type. |
duke@435 | 2143 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2144 | |
duke@435 | 2145 | // Try to categorize the address. If it comes up as TypeJavaPtr::BOTTOM, |
duke@435 | 2146 | // there was not enough information to nail it down. |
duke@435 | 2147 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2148 | assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); |
duke@435 | 2149 | |
duke@435 | 2150 | // We will need memory barriers unless we can determine a unique |
duke@435 | 2151 | // alias category for this reference. (Note: If for some reason |
duke@435 | 2152 | // the barriers get omitted and the unsafe reference begins to "pollute" |
duke@435 | 2153 | // the alias analysis of the rest of the graph, either Compile::can_alias |
duke@435 | 2154 | // or Compile::must_alias will throw a diagnostic assert.) |
duke@435 | 2155 | bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM); |
duke@435 | 2156 | |
duke@435 | 2157 | if (!is_store && type == T_OBJECT) { |
duke@435 | 2158 | // Attempt to infer a sharper value type from the offset and base type. |
duke@435 | 2159 | ciKlass* sharpened_klass = NULL; |
duke@435 | 2160 | |
duke@435 | 2161 | // See if it is an instance field, with an object type. |
duke@435 | 2162 | if (alias_type->field() != NULL) { |
duke@435 | 2163 | assert(!is_native_ptr, "native pointer op cannot use a java address"); |
duke@435 | 2164 | if (alias_type->field()->type()->is_klass()) { |
duke@435 | 2165 | sharpened_klass = alias_type->field()->type()->as_klass(); |
duke@435 | 2166 | } |
duke@435 | 2167 | } |
duke@435 | 2168 | |
duke@435 | 2169 | // See if it is a narrow oop array. |
duke@435 | 2170 | if (adr_type->isa_aryptr()) { |
twisti@1330 | 2171 | if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { |
duke@435 | 2172 | const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr(); |
duke@435 | 2173 | if (elem_type != NULL) { |
duke@435 | 2174 | sharpened_klass = elem_type->klass(); |
duke@435 | 2175 | } |
duke@435 | 2176 | } |
duke@435 | 2177 | } |
duke@435 | 2178 | |
duke@435 | 2179 | if (sharpened_klass != NULL) { |
duke@435 | 2180 | const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass); |
duke@435 | 2181 | |
duke@435 | 2182 | // Sharpen the value type. |
duke@435 | 2183 | value_type = tjp; |
duke@435 | 2184 | |
duke@435 | 2185 | #ifndef PRODUCT |
duke@435 | 2186 | if (PrintIntrinsics || PrintInlining || PrintOptoInlining) { |
duke@435 | 2187 | tty->print(" from base type: "); adr_type->dump(); |
duke@435 | 2188 | tty->print(" sharpened value: "); value_type->dump(); |
duke@435 | 2189 | } |
duke@435 | 2190 | #endif |
duke@435 | 2191 | } |
duke@435 | 2192 | } |
duke@435 | 2193 | |
duke@435 | 2194 | // Null check on self without removing any arguments. The argument |
duke@435 | 2195 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 2196 | // invalid stack traces when the primitive is inlined into a method |
duke@435 | 2197 | // which handles NullPointerExceptions. |
duke@435 | 2198 | _sp += nargs; |
duke@435 | 2199 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2200 | _sp -= nargs; |
duke@435 | 2201 | if (stopped()) { |
duke@435 | 2202 | return true; |
duke@435 | 2203 | } |
duke@435 | 2204 | // Heap pointers get a null-check from the interpreter, |
duke@435 | 2205 | // as a courtesy. However, this is not guaranteed by Unsafe, |
duke@435 | 2206 | // and it is not possible to fully distinguish unintended nulls |
duke@435 | 2207 | // from intended ones in this API. |
duke@435 | 2208 | |
duke@435 | 2209 | if (is_volatile) { |
duke@435 | 2210 | // We need to emit leading and trailing CPU membars (see below) in |
duke@435 | 2211 | // addition to memory membars when is_volatile. This is a little |
duke@435 | 2212 | // too strong, but avoids the need to insert per-alias-type |
duke@435 | 2213 | // volatile membars (for stores; compare Parse::do_put_xxx), which |
twisti@1040 | 2214 | // we cannot do effectively here because we probably only have a |
duke@435 | 2215 | // rough approximation of type. |
duke@435 | 2216 | need_mem_bar = true; |
duke@435 | 2217 | // For Stores, place a memory ordering barrier now. |
duke@435 | 2218 | if (is_store) |
duke@435 | 2219 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2220 | } |
duke@435 | 2221 | |
duke@435 | 2222 | // Memory barrier to prevent normal and 'unsafe' accesses from |
duke@435 | 2223 | // bypassing each other. Happens after null checks, so the |
duke@435 | 2224 | // exception paths do not take memory state from the memory barrier, |
duke@435 | 2225 | // so there's no problems making a strong assert about mixing users |
duke@435 | 2226 | // of safe & unsafe memory. Otherwise fails in a CTW of rt.jar |
duke@435 | 2227 | // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl. |
duke@435 | 2228 | if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2229 | |
duke@435 | 2230 | if (!is_store) { |
duke@435 | 2231 | Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile); |
duke@435 | 2232 | // load value and push onto stack |
duke@435 | 2233 | switch (type) { |
duke@435 | 2234 | case T_BOOLEAN: |
duke@435 | 2235 | case T_CHAR: |
duke@435 | 2236 | case T_BYTE: |
duke@435 | 2237 | case T_SHORT: |
duke@435 | 2238 | case T_INT: |
duke@435 | 2239 | case T_FLOAT: |
duke@435 | 2240 | case T_OBJECT: |
duke@435 | 2241 | push( p ); |
duke@435 | 2242 | break; |
duke@435 | 2243 | case T_ADDRESS: |
duke@435 | 2244 | // Cast to an int type. |
duke@435 | 2245 | p = _gvn.transform( new (C, 2) CastP2XNode(NULL,p) ); |
duke@435 | 2246 | p = ConvX2L(p); |
duke@435 | 2247 | push_pair(p); |
duke@435 | 2248 | break; |
duke@435 | 2249 | case T_DOUBLE: |
duke@435 | 2250 | case T_LONG: |
duke@435 | 2251 | push_pair( p ); |
duke@435 | 2252 | break; |
duke@435 | 2253 | default: ShouldNotReachHere(); |
duke@435 | 2254 | } |
duke@435 | 2255 | } else { |
duke@435 | 2256 | // place effect of store into memory |
duke@435 | 2257 | switch (type) { |
duke@435 | 2258 | case T_DOUBLE: |
duke@435 | 2259 | val = dstore_rounding(val); |
duke@435 | 2260 | break; |
duke@435 | 2261 | case T_ADDRESS: |
duke@435 | 2262 | // Repackage the long as a pointer. |
duke@435 | 2263 | val = ConvL2X(val); |
duke@435 | 2264 | val = _gvn.transform( new (C, 2) CastX2PNode(val) ); |
duke@435 | 2265 | break; |
duke@435 | 2266 | } |
duke@435 | 2267 | |
duke@435 | 2268 | if (type != T_OBJECT ) { |
duke@435 | 2269 | (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile); |
duke@435 | 2270 | } else { |
duke@435 | 2271 | // Possibly an oop being stored to Java heap or native memory |
duke@435 | 2272 | if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) { |
duke@435 | 2273 | // oop to Java heap. |
never@1260 | 2274 | (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type); |
duke@435 | 2275 | } else { |
duke@435 | 2276 | // We can't tell at compile time if we are storing in the Java heap or outside |
duke@435 | 2277 | // of it. So we need to emit code to conditionally do the proper type of |
duke@435 | 2278 | // store. |
duke@435 | 2279 | |
kvn@1286 | 2280 | IdealKit ideal(gvn(), control(), merged_memory()); |
kvn@1286 | 2281 | #define __ ideal. |
duke@435 | 2282 | // QQQ who knows what probability is here?? |
kvn@1286 | 2283 | __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); { |
kvn@1286 | 2284 | // Sync IdealKit and graphKit. |
kvn@1286 | 2285 | set_all_memory( __ merged_memory()); |
kvn@1286 | 2286 | set_control(__ ctrl()); |
kvn@1286 | 2287 | Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type); |
kvn@1286 | 2288 | // Update IdealKit memory. |
kvn@1286 | 2289 | __ set_all_memory(merged_memory()); |
kvn@1286 | 2290 | __ set_ctrl(control()); |
kvn@1286 | 2291 | } __ else_(); { |
kvn@1286 | 2292 | __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile); |
kvn@1286 | 2293 | } __ end_if(); |
kvn@1286 | 2294 | // Final sync IdealKit and GraphKit. |
kvn@1286 | 2295 | sync_kit(ideal); |
kvn@1286 | 2296 | #undef __ |
duke@435 | 2297 | } |
duke@435 | 2298 | } |
duke@435 | 2299 | } |
duke@435 | 2300 | |
duke@435 | 2301 | if (is_volatile) { |
duke@435 | 2302 | if (!is_store) |
duke@435 | 2303 | insert_mem_bar(Op_MemBarAcquire); |
duke@435 | 2304 | else |
duke@435 | 2305 | insert_mem_bar(Op_MemBarVolatile); |
duke@435 | 2306 | } |
duke@435 | 2307 | |
duke@435 | 2308 | if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2309 | |
duke@435 | 2310 | return true; |
duke@435 | 2311 | } |
duke@435 | 2312 | |
duke@435 | 2313 | //----------------------------inline_unsafe_prefetch---------------------------- |
duke@435 | 2314 | |
duke@435 | 2315 | bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) { |
duke@435 | 2316 | #ifndef PRODUCT |
duke@435 | 2317 | { |
duke@435 | 2318 | ResourceMark rm; |
duke@435 | 2319 | // Check the signatures. |
duke@435 | 2320 | ciSignature* sig = signature(); |
duke@435 | 2321 | #ifdef ASSERT |
duke@435 | 2322 | // Object getObject(Object base, int/long offset), etc. |
duke@435 | 2323 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2324 | if (!is_native_ptr) { |
duke@435 | 2325 | assert(sig->count() == 2, "oop prefetch has 2 arguments"); |
duke@435 | 2326 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object"); |
duke@435 | 2327 | assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct"); |
duke@435 | 2328 | } else { |
duke@435 | 2329 | assert(sig->count() == 1, "native prefetch has 1 argument"); |
duke@435 | 2330 | assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long"); |
duke@435 | 2331 | } |
duke@435 | 2332 | #endif // ASSERT |
duke@435 | 2333 | } |
duke@435 | 2334 | #endif // !PRODUCT |
duke@435 | 2335 | |
duke@435 | 2336 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2337 | |
duke@435 | 2338 | // Argument words: "this" if not static, plus (oop/offset) or (lo/hi) args |
duke@435 | 2339 | int nargs = (is_static ? 0 : 1) + (is_native_ptr ? 2 : 3); |
duke@435 | 2340 | |
duke@435 | 2341 | debug_only(int saved_sp = _sp); |
duke@435 | 2342 | _sp += nargs; |
duke@435 | 2343 | |
duke@435 | 2344 | // Build address expression. See the code in inline_unsafe_access. |
duke@435 | 2345 | Node *adr; |
duke@435 | 2346 | if (!is_native_ptr) { |
duke@435 | 2347 | // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset |
duke@435 | 2348 | Node* offset = pop_pair(); |
duke@435 | 2349 | // The base is either a Java object or a value produced by Unsafe.staticFieldBase |
duke@435 | 2350 | Node* base = pop(); |
duke@435 | 2351 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2352 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2353 | // by oopDesc::field_base. |
duke@435 | 2354 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 2355 | "fieldOffset must be byte-scaled"); |
duke@435 | 2356 | // 32-bit machines ignore the high half! |
duke@435 | 2357 | offset = ConvL2X(offset); |
duke@435 | 2358 | adr = make_unsafe_address(base, offset); |
duke@435 | 2359 | } else { |
duke@435 | 2360 | Node* ptr = pop_pair(); |
duke@435 | 2361 | // Adjust Java long to machine word: |
duke@435 | 2362 | ptr = ConvL2X(ptr); |
duke@435 | 2363 | adr = make_unsafe_address(NULL, ptr); |
duke@435 | 2364 | } |
duke@435 | 2365 | |
duke@435 | 2366 | if (is_static) { |
duke@435 | 2367 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2368 | } else { |
duke@435 | 2369 | // Pop receiver last: it was pushed first. |
duke@435 | 2370 | Node *receiver = pop(); |
duke@435 | 2371 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2372 | |
duke@435 | 2373 | // Null check on self without removing any arguments. The argument |
duke@435 | 2374 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 2375 | // invalid stack traces when the primitive is inlined into a method |
duke@435 | 2376 | // which handles NullPointerExceptions. |
duke@435 | 2377 | _sp += nargs; |
duke@435 | 2378 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2379 | _sp -= nargs; |
duke@435 | 2380 | if (stopped()) { |
duke@435 | 2381 | return true; |
duke@435 | 2382 | } |
duke@435 | 2383 | } |
duke@435 | 2384 | |
duke@435 | 2385 | // Generate the read or write prefetch |
duke@435 | 2386 | Node *prefetch; |
duke@435 | 2387 | if (is_store) { |
duke@435 | 2388 | prefetch = new (C, 3) PrefetchWriteNode(i_o(), adr); |
duke@435 | 2389 | } else { |
duke@435 | 2390 | prefetch = new (C, 3) PrefetchReadNode(i_o(), adr); |
duke@435 | 2391 | } |
duke@435 | 2392 | prefetch->init_req(0, control()); |
duke@435 | 2393 | set_i_o(_gvn.transform(prefetch)); |
duke@435 | 2394 | |
duke@435 | 2395 | return true; |
duke@435 | 2396 | } |
duke@435 | 2397 | |
duke@435 | 2398 | //----------------------------inline_unsafe_CAS---------------------------- |
duke@435 | 2399 | |
duke@435 | 2400 | bool LibraryCallKit::inline_unsafe_CAS(BasicType type) { |
duke@435 | 2401 | // This basic scheme here is the same as inline_unsafe_access, but |
duke@435 | 2402 | // differs in enough details that combining them would make the code |
duke@435 | 2403 | // overly confusing. (This is a true fact! I originally combined |
duke@435 | 2404 | // them, but even I was confused by it!) As much code/comments as |
duke@435 | 2405 | // possible are retained from inline_unsafe_access though to make |
twisti@1040 | 2406 | // the correspondences clearer. - dl |
duke@435 | 2407 | |
duke@435 | 2408 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2409 | |
duke@435 | 2410 | #ifndef PRODUCT |
duke@435 | 2411 | { |
duke@435 | 2412 | ResourceMark rm; |
duke@435 | 2413 | // Check the signatures. |
duke@435 | 2414 | ciSignature* sig = signature(); |
duke@435 | 2415 | #ifdef ASSERT |
duke@435 | 2416 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2417 | assert(rtype == T_BOOLEAN, "CAS must return boolean"); |
duke@435 | 2418 | assert(sig->count() == 4, "CAS has 4 arguments"); |
duke@435 | 2419 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); |
duke@435 | 2420 | assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); |
duke@435 | 2421 | #endif // ASSERT |
duke@435 | 2422 | } |
duke@435 | 2423 | #endif //PRODUCT |
duke@435 | 2424 | |
duke@435 | 2425 | // number of stack slots per value argument (1 or 2) |
duke@435 | 2426 | int type_words = type2size[type]; |
duke@435 | 2427 | |
duke@435 | 2428 | // Cannot inline wide CAS on machines that don't support it natively |
kvn@464 | 2429 | if (type2aelembytes(type) > BytesPerInt && !VM_Version::supports_cx8()) |
duke@435 | 2430 | return false; |
duke@435 | 2431 | |
duke@435 | 2432 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2433 | |
duke@435 | 2434 | // Argument words: "this" plus oop plus offset plus oldvalue plus newvalue; |
duke@435 | 2435 | int nargs = 1 + 1 + 2 + type_words + type_words; |
duke@435 | 2436 | |
duke@435 | 2437 | // pop arguments: newval, oldval, offset, base, and receiver |
duke@435 | 2438 | debug_only(int saved_sp = _sp); |
duke@435 | 2439 | _sp += nargs; |
duke@435 | 2440 | Node* newval = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2441 | Node* oldval = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2442 | Node *offset = pop_pair(); |
duke@435 | 2443 | Node *base = pop(); |
duke@435 | 2444 | Node *receiver = pop(); |
duke@435 | 2445 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2446 | |
duke@435 | 2447 | // Null check receiver. |
duke@435 | 2448 | _sp += nargs; |
duke@435 | 2449 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2450 | _sp -= nargs; |
duke@435 | 2451 | if (stopped()) { |
duke@435 | 2452 | return true; |
duke@435 | 2453 | } |
duke@435 | 2454 | |
duke@435 | 2455 | // Build field offset expression. |
duke@435 | 2456 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2457 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2458 | // by oopDesc::field_base. |
duke@435 | 2459 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); |
duke@435 | 2460 | // 32-bit machines ignore the high half of long offsets |
duke@435 | 2461 | offset = ConvL2X(offset); |
duke@435 | 2462 | Node* adr = make_unsafe_address(base, offset); |
duke@435 | 2463 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2464 | |
duke@435 | 2465 | // (Unlike inline_unsafe_access, there seems no point in trying |
duke@435 | 2466 | // to refine types. Just use the coarse types here. |
duke@435 | 2467 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2468 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2469 | assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); |
duke@435 | 2470 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 2471 | |
duke@435 | 2472 | // Memory-model-wise, a CAS acts like a little synchronized block, |
twisti@1040 | 2473 | // so needs barriers on each side. These don't translate into |
duke@435 | 2474 | // actual barriers on most machines, but we still need rest of |
duke@435 | 2475 | // compiler to respect ordering. |
duke@435 | 2476 | |
duke@435 | 2477 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2478 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2479 | |
duke@435 | 2480 | // 4984716: MemBars must be inserted before this |
duke@435 | 2481 | // memory node in order to avoid a false |
duke@435 | 2482 | // dependency which will confuse the scheduler. |
duke@435 | 2483 | Node *mem = memory(alias_idx); |
duke@435 | 2484 | |
duke@435 | 2485 | // For now, we handle only those cases that actually exist: ints, |
duke@435 | 2486 | // longs, and Object. Adding others should be straightforward. |
duke@435 | 2487 | Node* cas; |
duke@435 | 2488 | switch(type) { |
duke@435 | 2489 | case T_INT: |
duke@435 | 2490 | cas = _gvn.transform(new (C, 5) CompareAndSwapINode(control(), mem, adr, newval, oldval)); |
duke@435 | 2491 | break; |
duke@435 | 2492 | case T_LONG: |
duke@435 | 2493 | cas = _gvn.transform(new (C, 5) CompareAndSwapLNode(control(), mem, adr, newval, oldval)); |
duke@435 | 2494 | break; |
duke@435 | 2495 | case T_OBJECT: |
coleenp@548 | 2496 | // reference stores need a store barrier. |
duke@435 | 2497 | // (They don't if CAS fails, but it isn't worth checking.) |
never@1262 | 2498 | pre_barrier(control(), base, adr, alias_idx, newval, value_type->make_oopptr(), T_OBJECT); |
coleenp@548 | 2499 | #ifdef _LP64 |
kvn@598 | 2500 | if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
kvn@656 | 2501 | Node *newval_enc = _gvn.transform(new (C, 2) EncodePNode(newval, newval->bottom_type()->make_narrowoop())); |
kvn@656 | 2502 | Node *oldval_enc = _gvn.transform(new (C, 2) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop())); |
coleenp@548 | 2503 | cas = _gvn.transform(new (C, 5) CompareAndSwapNNode(control(), mem, adr, |
kvn@656 | 2504 | newval_enc, oldval_enc)); |
coleenp@548 | 2505 | } else |
coleenp@548 | 2506 | #endif |
kvn@656 | 2507 | { |
kvn@656 | 2508 | cas = _gvn.transform(new (C, 5) CompareAndSwapPNode(control(), mem, adr, newval, oldval)); |
kvn@656 | 2509 | } |
duke@435 | 2510 | post_barrier(control(), cas, base, adr, alias_idx, newval, T_OBJECT, true); |
duke@435 | 2511 | break; |
duke@435 | 2512 | default: |
duke@435 | 2513 | ShouldNotReachHere(); |
duke@435 | 2514 | break; |
duke@435 | 2515 | } |
duke@435 | 2516 | |
duke@435 | 2517 | // SCMemProjNodes represent the memory state of CAS. Their main |
duke@435 | 2518 | // role is to prevent CAS nodes from being optimized away when their |
duke@435 | 2519 | // results aren't used. |
duke@435 | 2520 | Node* proj = _gvn.transform( new (C, 1) SCMemProjNode(cas)); |
duke@435 | 2521 | set_memory(proj, alias_idx); |
duke@435 | 2522 | |
duke@435 | 2523 | // Add the trailing membar surrounding the access |
duke@435 | 2524 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2525 | insert_mem_bar(Op_MemBarAcquire); |
duke@435 | 2526 | |
duke@435 | 2527 | push(cas); |
duke@435 | 2528 | return true; |
duke@435 | 2529 | } |
duke@435 | 2530 | |
duke@435 | 2531 | bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) { |
duke@435 | 2532 | // This is another variant of inline_unsafe_access, differing in |
duke@435 | 2533 | // that it always issues store-store ("release") barrier and ensures |
duke@435 | 2534 | // store-atomicity (which only matters for "long"). |
duke@435 | 2535 | |
duke@435 | 2536 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2537 | |
duke@435 | 2538 | #ifndef PRODUCT |
duke@435 | 2539 | { |
duke@435 | 2540 | ResourceMark rm; |
duke@435 | 2541 | // Check the signatures. |
duke@435 | 2542 | ciSignature* sig = signature(); |
duke@435 | 2543 | #ifdef ASSERT |
duke@435 | 2544 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2545 | assert(rtype == T_VOID, "must return void"); |
duke@435 | 2546 | assert(sig->count() == 3, "has 3 arguments"); |
duke@435 | 2547 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object"); |
duke@435 | 2548 | assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long"); |
duke@435 | 2549 | #endif // ASSERT |
duke@435 | 2550 | } |
duke@435 | 2551 | #endif //PRODUCT |
duke@435 | 2552 | |
duke@435 | 2553 | // number of stack slots per value argument (1 or 2) |
duke@435 | 2554 | int type_words = type2size[type]; |
duke@435 | 2555 | |
duke@435 | 2556 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2557 | |
duke@435 | 2558 | // Argument words: "this" plus oop plus offset plus value; |
duke@435 | 2559 | int nargs = 1 + 1 + 2 + type_words; |
duke@435 | 2560 | |
duke@435 | 2561 | // pop arguments: val, offset, base, and receiver |
duke@435 | 2562 | debug_only(int saved_sp = _sp); |
duke@435 | 2563 | _sp += nargs; |
duke@435 | 2564 | Node* val = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2565 | Node *offset = pop_pair(); |
duke@435 | 2566 | Node *base = pop(); |
duke@435 | 2567 | Node *receiver = pop(); |
duke@435 | 2568 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2569 | |
duke@435 | 2570 | // Null check receiver. |
duke@435 | 2571 | _sp += nargs; |
duke@435 | 2572 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2573 | _sp -= nargs; |
duke@435 | 2574 | if (stopped()) { |
duke@435 | 2575 | return true; |
duke@435 | 2576 | } |
duke@435 | 2577 | |
duke@435 | 2578 | // Build field offset expression. |
duke@435 | 2579 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); |
duke@435 | 2580 | // 32-bit machines ignore the high half of long offsets |
duke@435 | 2581 | offset = ConvL2X(offset); |
duke@435 | 2582 | Node* adr = make_unsafe_address(base, offset); |
duke@435 | 2583 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2584 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2585 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2586 | |
duke@435 | 2587 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2588 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2589 | // Ensure that the store is atomic for longs: |
duke@435 | 2590 | bool require_atomic_access = true; |
duke@435 | 2591 | Node* store; |
duke@435 | 2592 | if (type == T_OBJECT) // reference stores need a store barrier. |
never@1260 | 2593 | store = store_oop_to_unknown(control(), base, adr, adr_type, val, type); |
duke@435 | 2594 | else { |
duke@435 | 2595 | store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access); |
duke@435 | 2596 | } |
duke@435 | 2597 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2598 | return true; |
duke@435 | 2599 | } |
duke@435 | 2600 | |
duke@435 | 2601 | bool LibraryCallKit::inline_unsafe_allocate() { |
duke@435 | 2602 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2603 | int nargs = 1 + 1; |
duke@435 | 2604 | assert(signature()->size() == nargs-1, "alloc has 1 argument"); |
duke@435 | 2605 | null_check_receiver(callee()); // check then ignore argument(0) |
duke@435 | 2606 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 2607 | Node* cls = do_null_check(argument(1), T_OBJECT); |
duke@435 | 2608 | _sp -= nargs; |
duke@435 | 2609 | if (stopped()) return true; |
duke@435 | 2610 | |
duke@435 | 2611 | Node* kls = load_klass_from_mirror(cls, false, nargs, NULL, 0); |
duke@435 | 2612 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 2613 | kls = do_null_check(kls, T_OBJECT); |
duke@435 | 2614 | _sp -= nargs; |
duke@435 | 2615 | if (stopped()) return true; // argument was like int.class |
duke@435 | 2616 | |
duke@435 | 2617 | // Note: The argument might still be an illegal value like |
duke@435 | 2618 | // Serializable.class or Object[].class. The runtime will handle it. |
duke@435 | 2619 | // But we must make an explicit check for initialization. |
duke@435 | 2620 | Node* insp = basic_plus_adr(kls, instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc)); |
duke@435 | 2621 | Node* inst = make_load(NULL, insp, TypeInt::INT, T_INT); |
duke@435 | 2622 | Node* bits = intcon(instanceKlass::fully_initialized); |
duke@435 | 2623 | Node* test = _gvn.transform( new (C, 3) SubINode(inst, bits) ); |
duke@435 | 2624 | // The 'test' is non-zero if we need to take a slow path. |
duke@435 | 2625 | |
duke@435 | 2626 | Node* obj = new_instance(kls, test); |
duke@435 | 2627 | push(obj); |
duke@435 | 2628 | |
duke@435 | 2629 | return true; |
duke@435 | 2630 | } |
duke@435 | 2631 | |
duke@435 | 2632 | //------------------------inline_native_time_funcs-------------- |
duke@435 | 2633 | // inline code for System.currentTimeMillis() and System.nanoTime() |
duke@435 | 2634 | // these have the same type and signature |
duke@435 | 2635 | bool LibraryCallKit::inline_native_time_funcs(bool isNano) { |
duke@435 | 2636 | address funcAddr = isNano ? CAST_FROM_FN_PTR(address, os::javaTimeNanos) : |
duke@435 | 2637 | CAST_FROM_FN_PTR(address, os::javaTimeMillis); |
duke@435 | 2638 | const char * funcName = isNano ? "nanoTime" : "currentTimeMillis"; |
duke@435 | 2639 | const TypeFunc *tf = OptoRuntime::current_time_millis_Type(); |
duke@435 | 2640 | const TypePtr* no_memory_effects = NULL; |
duke@435 | 2641 | Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); |
duke@435 | 2642 | Node* value = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms+0)); |
duke@435 | 2643 | #ifdef ASSERT |
duke@435 | 2644 | Node* value_top = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms + 1)); |
duke@435 | 2645 | assert(value_top == top(), "second value must be top"); |
duke@435 | 2646 | #endif |
duke@435 | 2647 | push_pair(value); |
duke@435 | 2648 | return true; |
duke@435 | 2649 | } |
duke@435 | 2650 | |
duke@435 | 2651 | //------------------------inline_native_currentThread------------------ |
duke@435 | 2652 | bool LibraryCallKit::inline_native_currentThread() { |
duke@435 | 2653 | Node* junk = NULL; |
duke@435 | 2654 | push(generate_current_thread(junk)); |
duke@435 | 2655 | return true; |
duke@435 | 2656 | } |
duke@435 | 2657 | |
duke@435 | 2658 | //------------------------inline_native_isInterrupted------------------ |
duke@435 | 2659 | bool LibraryCallKit::inline_native_isInterrupted() { |
duke@435 | 2660 | const int nargs = 1+1; // receiver + boolean |
duke@435 | 2661 | assert(nargs == arg_size(), "sanity"); |
duke@435 | 2662 | // Add a fast path to t.isInterrupted(clear_int): |
duke@435 | 2663 | // (t == Thread.current() && (!TLS._osthread._interrupted || !clear_int)) |
duke@435 | 2664 | // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int) |
duke@435 | 2665 | // So, in the common case that the interrupt bit is false, |
duke@435 | 2666 | // we avoid making a call into the VM. Even if the interrupt bit |
duke@435 | 2667 | // is true, if the clear_int argument is false, we avoid the VM call. |
duke@435 | 2668 | // However, if the receiver is not currentThread, we must call the VM, |
duke@435 | 2669 | // because there must be some locking done around the operation. |
duke@435 | 2670 | |
duke@435 | 2671 | // We only go to the fast case code if we pass two guards. |
duke@435 | 2672 | // Paths which do not pass are accumulated in the slow_region. |
duke@435 | 2673 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 2674 | record_for_igvn(slow_region); |
duke@435 | 2675 | RegionNode* result_rgn = new (C, 4) RegionNode(1+3); // fast1, fast2, slow |
duke@435 | 2676 | PhiNode* result_val = new (C, 4) PhiNode(result_rgn, TypeInt::BOOL); |
duke@435 | 2677 | enum { no_int_result_path = 1, |
duke@435 | 2678 | no_clear_result_path = 2, |
duke@435 | 2679 | slow_result_path = 3 |
duke@435 | 2680 | }; |
duke@435 | 2681 | |
duke@435 | 2682 | // (a) Receiving thread must be the current thread. |
duke@435 | 2683 | Node* rec_thr = argument(0); |
duke@435 | 2684 | Node* tls_ptr = NULL; |
duke@435 | 2685 | Node* cur_thr = generate_current_thread(tls_ptr); |
duke@435 | 2686 | Node* cmp_thr = _gvn.transform( new (C, 3) CmpPNode(cur_thr, rec_thr) ); |
duke@435 | 2687 | Node* bol_thr = _gvn.transform( new (C, 2) BoolNode(cmp_thr, BoolTest::ne) ); |
duke@435 | 2688 | |
duke@435 | 2689 | bool known_current_thread = (_gvn.type(bol_thr) == TypeInt::ZERO); |
duke@435 | 2690 | if (!known_current_thread) |
duke@435 | 2691 | generate_slow_guard(bol_thr, slow_region); |
duke@435 | 2692 | |
duke@435 | 2693 | // (b) Interrupt bit on TLS must be false. |
duke@435 | 2694 | Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset())); |
duke@435 | 2695 | Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS); |
duke@435 | 2696 | p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset())); |
kvn@1222 | 2697 | // Set the control input on the field _interrupted read to prevent it floating up. |
kvn@1222 | 2698 | Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT); |
duke@435 | 2699 | Node* cmp_bit = _gvn.transform( new (C, 3) CmpINode(int_bit, intcon(0)) ); |
duke@435 | 2700 | Node* bol_bit = _gvn.transform( new (C, 2) BoolNode(cmp_bit, BoolTest::ne) ); |
duke@435 | 2701 | |
duke@435 | 2702 | IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); |
duke@435 | 2703 | |
duke@435 | 2704 | // First fast path: if (!TLS._interrupted) return false; |
duke@435 | 2705 | Node* false_bit = _gvn.transform( new (C, 1) IfFalseNode(iff_bit) ); |
duke@435 | 2706 | result_rgn->init_req(no_int_result_path, false_bit); |
duke@435 | 2707 | result_val->init_req(no_int_result_path, intcon(0)); |
duke@435 | 2708 | |
duke@435 | 2709 | // drop through to next case |
duke@435 | 2710 | set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_bit)) ); |
duke@435 | 2711 | |
duke@435 | 2712 | // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path. |
duke@435 | 2713 | Node* clr_arg = argument(1); |
duke@435 | 2714 | Node* cmp_arg = _gvn.transform( new (C, 3) CmpINode(clr_arg, intcon(0)) ); |
duke@435 | 2715 | Node* bol_arg = _gvn.transform( new (C, 2) BoolNode(cmp_arg, BoolTest::ne) ); |
duke@435 | 2716 | IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN); |
duke@435 | 2717 | |
duke@435 | 2718 | // Second fast path: ... else if (!clear_int) return true; |
duke@435 | 2719 | Node* false_arg = _gvn.transform( new (C, 1) IfFalseNode(iff_arg) ); |
duke@435 | 2720 | result_rgn->init_req(no_clear_result_path, false_arg); |
duke@435 | 2721 | result_val->init_req(no_clear_result_path, intcon(1)); |
duke@435 | 2722 | |
duke@435 | 2723 | // drop through to next case |
duke@435 | 2724 | set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_arg)) ); |
duke@435 | 2725 | |
duke@435 | 2726 | // (d) Otherwise, go to the slow path. |
duke@435 | 2727 | slow_region->add_req(control()); |
duke@435 | 2728 | set_control( _gvn.transform(slow_region) ); |
duke@435 | 2729 | |
duke@435 | 2730 | if (stopped()) { |
duke@435 | 2731 | // There is no slow path. |
duke@435 | 2732 | result_rgn->init_req(slow_result_path, top()); |
duke@435 | 2733 | result_val->init_req(slow_result_path, top()); |
duke@435 | 2734 | } else { |
duke@435 | 2735 | // non-virtual because it is a private non-static |
duke@435 | 2736 | CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted); |
duke@435 | 2737 | |
duke@435 | 2738 | Node* slow_val = set_results_for_java_call(slow_call); |
duke@435 | 2739 | // this->control() comes from set_results_for_java_call |
duke@435 | 2740 | |
duke@435 | 2741 | // If we know that the result of the slow call will be true, tell the optimizer! |
duke@435 | 2742 | if (known_current_thread) slow_val = intcon(1); |
duke@435 | 2743 | |
duke@435 | 2744 | Node* fast_io = slow_call->in(TypeFunc::I_O); |
duke@435 | 2745 | Node* fast_mem = slow_call->in(TypeFunc::Memory); |
duke@435 | 2746 | // These two phis are pre-filled with copies of of the fast IO and Memory |
duke@435 | 2747 | Node* io_phi = PhiNode::make(result_rgn, fast_io, Type::ABIO); |
duke@435 | 2748 | Node* mem_phi = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM); |
duke@435 | 2749 | |
duke@435 | 2750 | result_rgn->init_req(slow_result_path, control()); |
duke@435 | 2751 | io_phi ->init_req(slow_result_path, i_o()); |
duke@435 | 2752 | mem_phi ->init_req(slow_result_path, reset_memory()); |
duke@435 | 2753 | result_val->init_req(slow_result_path, slow_val); |
duke@435 | 2754 | |
duke@435 | 2755 | set_all_memory( _gvn.transform(mem_phi) ); |
duke@435 | 2756 | set_i_o( _gvn.transform(io_phi) ); |
duke@435 | 2757 | } |
duke@435 | 2758 | |
duke@435 | 2759 | push_result(result_rgn, result_val); |
duke@435 | 2760 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 2761 | |
duke@435 | 2762 | return true; |
duke@435 | 2763 | } |
duke@435 | 2764 | |
duke@435 | 2765 | //---------------------------load_mirror_from_klass---------------------------- |
duke@435 | 2766 | // Given a klass oop, load its java mirror (a java.lang.Class oop). |
duke@435 | 2767 | Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { |
duke@435 | 2768 | Node* p = basic_plus_adr(klass, Klass::java_mirror_offset_in_bytes() + sizeof(oopDesc)); |
duke@435 | 2769 | return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT); |
duke@435 | 2770 | } |
duke@435 | 2771 | |
duke@435 | 2772 | //-----------------------load_klass_from_mirror_common------------------------- |
duke@435 | 2773 | // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop. |
duke@435 | 2774 | // Test the klass oop for null (signifying a primitive Class like Integer.TYPE), |
duke@435 | 2775 | // and branch to the given path on the region. |
duke@435 | 2776 | // If never_see_null, take an uncommon trap on null, so we can optimistically |
duke@435 | 2777 | // compile for the non-null case. |
duke@435 | 2778 | // If the region is NULL, force never_see_null = true. |
duke@435 | 2779 | Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror, |
duke@435 | 2780 | bool never_see_null, |
duke@435 | 2781 | int nargs, |
duke@435 | 2782 | RegionNode* region, |
duke@435 | 2783 | int null_path, |
duke@435 | 2784 | int offset) { |
duke@435 | 2785 | if (region == NULL) never_see_null = true; |
duke@435 | 2786 | Node* p = basic_plus_adr(mirror, offset); |
duke@435 | 2787 | const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; |
kvn@599 | 2788 | Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) ); |
duke@435 | 2789 | _sp += nargs; // any deopt will start just before call to enclosing method |
duke@435 | 2790 | Node* null_ctl = top(); |
duke@435 | 2791 | kls = null_check_oop(kls, &null_ctl, never_see_null); |
duke@435 | 2792 | if (region != NULL) { |
duke@435 | 2793 | // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). |
duke@435 | 2794 | region->init_req(null_path, null_ctl); |
duke@435 | 2795 | } else { |
duke@435 | 2796 | assert(null_ctl == top(), "no loose ends"); |
duke@435 | 2797 | } |
duke@435 | 2798 | _sp -= nargs; |
duke@435 | 2799 | return kls; |
duke@435 | 2800 | } |
duke@435 | 2801 | |
duke@435 | 2802 | //--------------------(inline_native_Class_query helpers)--------------------- |
duke@435 | 2803 | // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER. |
duke@435 | 2804 | // Fall through if (mods & mask) == bits, take the guard otherwise. |
duke@435 | 2805 | Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) { |
duke@435 | 2806 | // Branch around if the given klass has the given modifier bit set. |
duke@435 | 2807 | // Like generate_guard, adds a new path onto the region. |
duke@435 | 2808 | Node* modp = basic_plus_adr(kls, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc)); |
duke@435 | 2809 | Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT); |
duke@435 | 2810 | Node* mask = intcon(modifier_mask); |
duke@435 | 2811 | Node* bits = intcon(modifier_bits); |
duke@435 | 2812 | Node* mbit = _gvn.transform( new (C, 3) AndINode(mods, mask) ); |
duke@435 | 2813 | Node* cmp = _gvn.transform( new (C, 3) CmpINode(mbit, bits) ); |
duke@435 | 2814 | Node* bol = _gvn.transform( new (C, 2) BoolNode(cmp, BoolTest::ne) ); |
duke@435 | 2815 | return generate_fair_guard(bol, region); |
duke@435 | 2816 | } |
duke@435 | 2817 | Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) { |
duke@435 | 2818 | return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region); |
duke@435 | 2819 | } |
duke@435 | 2820 | |
duke@435 | 2821 | //-------------------------inline_native_Class_query------------------- |
duke@435 | 2822 | bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { |
duke@435 | 2823 | int nargs = 1+0; // just the Class mirror, in most cases |
duke@435 | 2824 | const Type* return_type = TypeInt::BOOL; |
duke@435 | 2825 | Node* prim_return_value = top(); // what happens if it's a primitive class? |
duke@435 | 2826 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 2827 | bool expect_prim = false; // most of these guys expect to work on refs |
duke@435 | 2828 | |
duke@435 | 2829 | enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; |
duke@435 | 2830 | |
duke@435 | 2831 | switch (id) { |
duke@435 | 2832 | case vmIntrinsics::_isInstance: |
duke@435 | 2833 | nargs = 1+1; // the Class mirror, plus the object getting queried about |
duke@435 | 2834 | // nothing is an instance of a primitive type |
duke@435 | 2835 | prim_return_value = intcon(0); |
duke@435 | 2836 | break; |
duke@435 | 2837 | case vmIntrinsics::_getModifiers: |
duke@435 | 2838 | prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); |
duke@435 | 2839 | assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line"); |
duke@435 | 2840 | return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin); |
duke@435 | 2841 | break; |
duke@435 | 2842 | case vmIntrinsics::_isInterface: |
duke@435 | 2843 | prim_return_value = intcon(0); |
duke@435 | 2844 | break; |
duke@435 | 2845 | case vmIntrinsics::_isArray: |
duke@435 | 2846 | prim_return_value = intcon(0); |
duke@435 | 2847 | expect_prim = true; // cf. ObjectStreamClass.getClassSignature |
duke@435 | 2848 | break; |
duke@435 | 2849 | case vmIntrinsics::_isPrimitive: |
duke@435 | 2850 | prim_return_value = intcon(1); |
duke@435 | 2851 | expect_prim = true; // obviously |
duke@435 | 2852 | break; |
duke@435 | 2853 | case vmIntrinsics::_getSuperclass: |
duke@435 | 2854 | prim_return_value = null(); |
duke@435 | 2855 | return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); |
duke@435 | 2856 | break; |
duke@435 | 2857 | case vmIntrinsics::_getComponentType: |
duke@435 | 2858 | prim_return_value = null(); |
duke@435 | 2859 | return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); |
duke@435 | 2860 | break; |
duke@435 | 2861 | case vmIntrinsics::_getClassAccessFlags: |
duke@435 | 2862 | prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); |
duke@435 | 2863 | return_type = TypeInt::INT; // not bool! 6297094 |
duke@435 | 2864 | break; |
duke@435 | 2865 | default: |
duke@435 | 2866 | ShouldNotReachHere(); |
duke@435 | 2867 | } |
duke@435 | 2868 | |
duke@435 | 2869 | Node* mirror = argument(0); |
duke@435 | 2870 | Node* obj = (nargs <= 1)? top(): argument(1); |
duke@435 | 2871 | |
duke@435 | 2872 | const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); |
duke@435 | 2873 | if (mirror_con == NULL) return false; // cannot happen? |
duke@435 | 2874 | |
duke@435 | 2875 | #ifndef PRODUCT |
duke@435 | 2876 | if (PrintIntrinsics || PrintInlining || PrintOptoInlining) { |
duke@435 | 2877 | ciType* k = mirror_con->java_mirror_type(); |
duke@435 | 2878 | if (k) { |
duke@435 | 2879 | tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id())); |
duke@435 | 2880 | k->print_name(); |
duke@435 | 2881 | tty->cr(); |
duke@435 | 2882 | } |
duke@435 | 2883 | } |
duke@435 | 2884 | #endif |
duke@435 | 2885 | |
duke@435 | 2886 | // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive). |
duke@435 | 2887 | RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 2888 | record_for_igvn(region); |
duke@435 | 2889 | PhiNode* phi = new (C, PATH_LIMIT) PhiNode(region, return_type); |
duke@435 | 2890 | |
duke@435 | 2891 | // The mirror will never be null of Reflection.getClassAccessFlags, however |
duke@435 | 2892 | // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE |
duke@435 | 2893 | // if it is. See bug 4774291. |
duke@435 | 2894 | |
duke@435 | 2895 | // For Reflection.getClassAccessFlags(), the null check occurs in |
duke@435 | 2896 | // the wrong place; see inline_unsafe_access(), above, for a similar |
duke@435 | 2897 | // situation. |
duke@435 | 2898 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 2899 | mirror = do_null_check(mirror, T_OBJECT); |
duke@435 | 2900 | _sp -= nargs; |
duke@435 | 2901 | // If mirror or obj is dead, only null-path is taken. |
duke@435 | 2902 | if (stopped()) return true; |
duke@435 | 2903 | |
duke@435 | 2904 | if (expect_prim) never_see_null = false; // expect nulls (meaning prims) |
duke@435 | 2905 | |
duke@435 | 2906 | // Now load the mirror's klass metaobject, and null-check it. |
duke@435 | 2907 | // Side-effects region with the control path if the klass is null. |
duke@435 | 2908 | Node* kls = load_klass_from_mirror(mirror, never_see_null, nargs, |
duke@435 | 2909 | region, _prim_path); |
duke@435 | 2910 | // If kls is null, we have a primitive mirror. |
duke@435 | 2911 | phi->init_req(_prim_path, prim_return_value); |
duke@435 | 2912 | if (stopped()) { push_result(region, phi); return true; } |
duke@435 | 2913 | |
duke@435 | 2914 | Node* p; // handy temp |
duke@435 | 2915 | Node* null_ctl; |
duke@435 | 2916 | |
duke@435 | 2917 | // Now that we have the non-null klass, we can perform the real query. |
duke@435 | 2918 | // For constant classes, the query will constant-fold in LoadNode::Value. |
duke@435 | 2919 | Node* query_value = top(); |
duke@435 | 2920 | switch (id) { |
duke@435 | 2921 | case vmIntrinsics::_isInstance: |
duke@435 | 2922 | // nothing is an instance of a primitive type |
duke@435 | 2923 | query_value = gen_instanceof(obj, kls); |
duke@435 | 2924 | break; |
duke@435 | 2925 | |
duke@435 | 2926 | case vmIntrinsics::_getModifiers: |
duke@435 | 2927 | p = basic_plus_adr(kls, Klass::modifier_flags_offset_in_bytes() + sizeof(oopDesc)); |
duke@435 | 2928 | query_value = make_load(NULL, p, TypeInt::INT, T_INT); |
duke@435 | 2929 | break; |
duke@435 | 2930 | |
duke@435 | 2931 | case vmIntrinsics::_isInterface: |
duke@435 | 2932 | // (To verify this code sequence, check the asserts in JVM_IsInterface.) |
duke@435 | 2933 | if (generate_interface_guard(kls, region) != NULL) |
duke@435 | 2934 | // A guard was added. If the guard is taken, it was an interface. |
duke@435 | 2935 | phi->add_req(intcon(1)); |
duke@435 | 2936 | // If we fall through, it's a plain class. |
duke@435 | 2937 | query_value = intcon(0); |
duke@435 | 2938 | break; |
duke@435 | 2939 | |
duke@435 | 2940 | case vmIntrinsics::_isArray: |
duke@435 | 2941 | // (To verify this code sequence, check the asserts in JVM_IsArrayClass.) |
duke@435 | 2942 | if (generate_array_guard(kls, region) != NULL) |
duke@435 | 2943 | // A guard was added. If the guard is taken, it was an array. |
duke@435 | 2944 | phi->add_req(intcon(1)); |
duke@435 | 2945 | // If we fall through, it's a plain class. |
duke@435 | 2946 | query_value = intcon(0); |
duke@435 | 2947 | break; |
duke@435 | 2948 | |
duke@435 | 2949 | case vmIntrinsics::_isPrimitive: |
duke@435 | 2950 | query_value = intcon(0); // "normal" path produces false |
duke@435 | 2951 | break; |
duke@435 | 2952 | |
duke@435 | 2953 | case vmIntrinsics::_getSuperclass: |
duke@435 | 2954 | // The rules here are somewhat unfortunate, but we can still do better |
duke@435 | 2955 | // with random logic than with a JNI call. |
duke@435 | 2956 | // Interfaces store null or Object as _super, but must report null. |
duke@435 | 2957 | // Arrays store an intermediate super as _super, but must report Object. |
duke@435 | 2958 | // Other types can report the actual _super. |
duke@435 | 2959 | // (To verify this code sequence, check the asserts in JVM_IsInterface.) |
duke@435 | 2960 | if (generate_interface_guard(kls, region) != NULL) |
duke@435 | 2961 | // A guard was added. If the guard is taken, it was an interface. |
duke@435 | 2962 | phi->add_req(null()); |
duke@435 | 2963 | if (generate_array_guard(kls, region) != NULL) |
duke@435 | 2964 | // A guard was added. If the guard is taken, it was an array. |
duke@435 | 2965 | phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); |
duke@435 | 2966 | // If we fall through, it's a plain class. Get its _super. |
duke@435 | 2967 | p = basic_plus_adr(kls, Klass::super_offset_in_bytes() + sizeof(oopDesc)); |
kvn@599 | 2968 | kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) ); |
duke@435 | 2969 | null_ctl = top(); |
duke@435 | 2970 | kls = null_check_oop(kls, &null_ctl); |
duke@435 | 2971 | if (null_ctl != top()) { |
duke@435 | 2972 | // If the guard is taken, Object.superClass is null (both klass and mirror). |
duke@435 | 2973 | region->add_req(null_ctl); |
duke@435 | 2974 | phi ->add_req(null()); |
duke@435 | 2975 | } |
duke@435 | 2976 | if (!stopped()) { |
duke@435 | 2977 | query_value = load_mirror_from_klass(kls); |
duke@435 | 2978 | } |
duke@435 | 2979 | break; |
duke@435 | 2980 | |
duke@435 | 2981 | case vmIntrinsics::_getComponentType: |
duke@435 | 2982 | if (generate_array_guard(kls, region) != NULL) { |
duke@435 | 2983 | // Be sure to pin the oop load to the guard edge just created: |
duke@435 | 2984 | Node* is_array_ctrl = region->in(region->req()-1); |
duke@435 | 2985 | Node* cma = basic_plus_adr(kls, in_bytes(arrayKlass::component_mirror_offset()) + sizeof(oopDesc)); |
duke@435 | 2986 | Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT); |
duke@435 | 2987 | phi->add_req(cmo); |
duke@435 | 2988 | } |
duke@435 | 2989 | query_value = null(); // non-array case is null |
duke@435 | 2990 | break; |
duke@435 | 2991 | |
duke@435 | 2992 | case vmIntrinsics::_getClassAccessFlags: |
duke@435 | 2993 | p = basic_plus_adr(kls, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc)); |
duke@435 | 2994 | query_value = make_load(NULL, p, TypeInt::INT, T_INT); |
duke@435 | 2995 | break; |
duke@435 | 2996 | |
duke@435 | 2997 | default: |
duke@435 | 2998 | ShouldNotReachHere(); |
duke@435 | 2999 | } |
duke@435 | 3000 | |
duke@435 | 3001 | // Fall-through is the normal case of a query to a real class. |
duke@435 | 3002 | phi->init_req(1, query_value); |
duke@435 | 3003 | region->init_req(1, control()); |
duke@435 | 3004 | |
duke@435 | 3005 | push_result(region, phi); |
duke@435 | 3006 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3007 | |
duke@435 | 3008 | return true; |
duke@435 | 3009 | } |
duke@435 | 3010 | |
duke@435 | 3011 | //--------------------------inline_native_subtype_check------------------------ |
duke@435 | 3012 | // This intrinsic takes the JNI calls out of the heart of |
duke@435 | 3013 | // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc. |
duke@435 | 3014 | bool LibraryCallKit::inline_native_subtype_check() { |
duke@435 | 3015 | int nargs = 1+1; // the Class mirror, plus the other class getting examined |
duke@435 | 3016 | |
duke@435 | 3017 | // Pull both arguments off the stack. |
duke@435 | 3018 | Node* args[2]; // two java.lang.Class mirrors: superc, subc |
duke@435 | 3019 | args[0] = argument(0); |
duke@435 | 3020 | args[1] = argument(1); |
duke@435 | 3021 | Node* klasses[2]; // corresponding Klasses: superk, subk |
duke@435 | 3022 | klasses[0] = klasses[1] = top(); |
duke@435 | 3023 | |
duke@435 | 3024 | enum { |
duke@435 | 3025 | // A full decision tree on {superc is prim, subc is prim}: |
duke@435 | 3026 | _prim_0_path = 1, // {P,N} => false |
duke@435 | 3027 | // {P,P} & superc!=subc => false |
duke@435 | 3028 | _prim_same_path, // {P,P} & superc==subc => true |
duke@435 | 3029 | _prim_1_path, // {N,P} => false |
duke@435 | 3030 | _ref_subtype_path, // {N,N} & subtype check wins => true |
duke@435 | 3031 | _both_ref_path, // {N,N} & subtype check loses => false |
duke@435 | 3032 | PATH_LIMIT |
duke@435 | 3033 | }; |
duke@435 | 3034 | |
duke@435 | 3035 | RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3036 | Node* phi = new (C, PATH_LIMIT) PhiNode(region, TypeInt::BOOL); |
duke@435 | 3037 | record_for_igvn(region); |
duke@435 | 3038 | |
duke@435 | 3039 | const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads |
duke@435 | 3040 | const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; |
duke@435 | 3041 | int class_klass_offset = java_lang_Class::klass_offset_in_bytes(); |
duke@435 | 3042 | |
duke@435 | 3043 | // First null-check both mirrors and load each mirror's klass metaobject. |
duke@435 | 3044 | int which_arg; |
duke@435 | 3045 | for (which_arg = 0; which_arg <= 1; which_arg++) { |
duke@435 | 3046 | Node* arg = args[which_arg]; |
duke@435 | 3047 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3048 | arg = do_null_check(arg, T_OBJECT); |
duke@435 | 3049 | _sp -= nargs; |
duke@435 | 3050 | if (stopped()) break; |
duke@435 | 3051 | args[which_arg] = _gvn.transform(arg); |
duke@435 | 3052 | |
duke@435 | 3053 | Node* p = basic_plus_adr(arg, class_klass_offset); |
kvn@599 | 3054 | Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type); |
duke@435 | 3055 | klasses[which_arg] = _gvn.transform(kls); |
duke@435 | 3056 | } |
duke@435 | 3057 | |
duke@435 | 3058 | // Having loaded both klasses, test each for null. |
duke@435 | 3059 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 3060 | for (which_arg = 0; which_arg <= 1; which_arg++) { |
duke@435 | 3061 | Node* kls = klasses[which_arg]; |
duke@435 | 3062 | Node* null_ctl = top(); |
duke@435 | 3063 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3064 | kls = null_check_oop(kls, &null_ctl, never_see_null); |
duke@435 | 3065 | _sp -= nargs; |
duke@435 | 3066 | int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path); |
duke@435 | 3067 | region->init_req(prim_path, null_ctl); |
duke@435 | 3068 | if (stopped()) break; |
duke@435 | 3069 | klasses[which_arg] = kls; |
duke@435 | 3070 | } |
duke@435 | 3071 | |
duke@435 | 3072 | if (!stopped()) { |
duke@435 | 3073 | // now we have two reference types, in klasses[0..1] |
duke@435 | 3074 | Node* subk = klasses[1]; // the argument to isAssignableFrom |
duke@435 | 3075 | Node* superk = klasses[0]; // the receiver |
duke@435 | 3076 | region->set_req(_both_ref_path, gen_subtype_check(subk, superk)); |
duke@435 | 3077 | // now we have a successful reference subtype check |
duke@435 | 3078 | region->set_req(_ref_subtype_path, control()); |
duke@435 | 3079 | } |
duke@435 | 3080 | |
duke@435 | 3081 | // If both operands are primitive (both klasses null), then |
duke@435 | 3082 | // we must return true when they are identical primitives. |
duke@435 | 3083 | // It is convenient to test this after the first null klass check. |
duke@435 | 3084 | set_control(region->in(_prim_0_path)); // go back to first null check |
duke@435 | 3085 | if (!stopped()) { |
duke@435 | 3086 | // Since superc is primitive, make a guard for the superc==subc case. |
duke@435 | 3087 | Node* cmp_eq = _gvn.transform( new (C, 3) CmpPNode(args[0], args[1]) ); |
duke@435 | 3088 | Node* bol_eq = _gvn.transform( new (C, 2) BoolNode(cmp_eq, BoolTest::eq) ); |
duke@435 | 3089 | generate_guard(bol_eq, region, PROB_FAIR); |
duke@435 | 3090 | if (region->req() == PATH_LIMIT+1) { |
duke@435 | 3091 | // A guard was added. If the added guard is taken, superc==subc. |
duke@435 | 3092 | region->swap_edges(PATH_LIMIT, _prim_same_path); |
duke@435 | 3093 | region->del_req(PATH_LIMIT); |
duke@435 | 3094 | } |
duke@435 | 3095 | region->set_req(_prim_0_path, control()); // Not equal after all. |
duke@435 | 3096 | } |
duke@435 | 3097 | |
duke@435 | 3098 | // these are the only paths that produce 'true': |
duke@435 | 3099 | phi->set_req(_prim_same_path, intcon(1)); |
duke@435 | 3100 | phi->set_req(_ref_subtype_path, intcon(1)); |
duke@435 | 3101 | |
duke@435 | 3102 | // pull together the cases: |
duke@435 | 3103 | assert(region->req() == PATH_LIMIT, "sane region"); |
duke@435 | 3104 | for (uint i = 1; i < region->req(); i++) { |
duke@435 | 3105 | Node* ctl = region->in(i); |
duke@435 | 3106 | if (ctl == NULL || ctl == top()) { |
duke@435 | 3107 | region->set_req(i, top()); |
duke@435 | 3108 | phi ->set_req(i, top()); |
duke@435 | 3109 | } else if (phi->in(i) == NULL) { |
duke@435 | 3110 | phi->set_req(i, intcon(0)); // all other paths produce 'false' |
duke@435 | 3111 | } |
duke@435 | 3112 | } |
duke@435 | 3113 | |
duke@435 | 3114 | set_control(_gvn.transform(region)); |
duke@435 | 3115 | push(_gvn.transform(phi)); |
duke@435 | 3116 | |
duke@435 | 3117 | return true; |
duke@435 | 3118 | } |
duke@435 | 3119 | |
duke@435 | 3120 | //---------------------generate_array_guard_common------------------------ |
duke@435 | 3121 | Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, |
duke@435 | 3122 | bool obj_array, bool not_array) { |
duke@435 | 3123 | // If obj_array/non_array==false/false: |
duke@435 | 3124 | // Branch around if the given klass is in fact an array (either obj or prim). |
duke@435 | 3125 | // If obj_array/non_array==false/true: |
duke@435 | 3126 | // Branch around if the given klass is not an array klass of any kind. |
duke@435 | 3127 | // If obj_array/non_array==true/true: |
duke@435 | 3128 | // Branch around if the kls is not an oop array (kls is int[], String, etc.) |
duke@435 | 3129 | // If obj_array/non_array==true/false: |
duke@435 | 3130 | // Branch around if the kls is an oop array (Object[] or subtype) |
duke@435 | 3131 | // |
duke@435 | 3132 | // Like generate_guard, adds a new path onto the region. |
duke@435 | 3133 | jint layout_con = 0; |
duke@435 | 3134 | Node* layout_val = get_layout_helper(kls, layout_con); |
duke@435 | 3135 | if (layout_val == NULL) { |
duke@435 | 3136 | bool query = (obj_array |
duke@435 | 3137 | ? Klass::layout_helper_is_objArray(layout_con) |
duke@435 | 3138 | : Klass::layout_helper_is_javaArray(layout_con)); |
duke@435 | 3139 | if (query == not_array) { |
duke@435 | 3140 | return NULL; // never a branch |
duke@435 | 3141 | } else { // always a branch |
duke@435 | 3142 | Node* always_branch = control(); |
duke@435 | 3143 | if (region != NULL) |
duke@435 | 3144 | region->add_req(always_branch); |
duke@435 | 3145 | set_control(top()); |
duke@435 | 3146 | return always_branch; |
duke@435 | 3147 | } |
duke@435 | 3148 | } |
duke@435 | 3149 | // Now test the correct condition. |
duke@435 | 3150 | jint nval = (obj_array |
duke@435 | 3151 | ? ((jint)Klass::_lh_array_tag_type_value |
duke@435 | 3152 | << Klass::_lh_array_tag_shift) |
duke@435 | 3153 | : Klass::_lh_neutral_value); |
duke@435 | 3154 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(nval)) ); |
duke@435 | 3155 | BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array |
duke@435 | 3156 | // invert the test if we are looking for a non-array |
duke@435 | 3157 | if (not_array) btest = BoolTest(btest).negate(); |
duke@435 | 3158 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, btest) ); |
duke@435 | 3159 | return generate_fair_guard(bol, region); |
duke@435 | 3160 | } |
duke@435 | 3161 | |
duke@435 | 3162 | |
duke@435 | 3163 | //-----------------------inline_native_newArray-------------------------- |
duke@435 | 3164 | bool LibraryCallKit::inline_native_newArray() { |
duke@435 | 3165 | int nargs = 2; |
duke@435 | 3166 | Node* mirror = argument(0); |
duke@435 | 3167 | Node* count_val = argument(1); |
duke@435 | 3168 | |
duke@435 | 3169 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3170 | mirror = do_null_check(mirror, T_OBJECT); |
duke@435 | 3171 | _sp -= nargs; |
kvn@598 | 3172 | // If mirror or obj is dead, only null-path is taken. |
kvn@598 | 3173 | if (stopped()) return true; |
duke@435 | 3174 | |
duke@435 | 3175 | enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT }; |
duke@435 | 3176 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3177 | PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
duke@435 | 3178 | TypeInstPtr::NOTNULL); |
duke@435 | 3179 | PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
duke@435 | 3180 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
duke@435 | 3181 | TypePtr::BOTTOM); |
duke@435 | 3182 | |
duke@435 | 3183 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 3184 | Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null, |
duke@435 | 3185 | nargs, |
duke@435 | 3186 | result_reg, _slow_path); |
duke@435 | 3187 | Node* normal_ctl = control(); |
duke@435 | 3188 | Node* no_array_ctl = result_reg->in(_slow_path); |
duke@435 | 3189 | |
duke@435 | 3190 | // Generate code for the slow case. We make a call to newArray(). |
duke@435 | 3191 | set_control(no_array_ctl); |
duke@435 | 3192 | if (!stopped()) { |
duke@435 | 3193 | // Either the input type is void.class, or else the |
duke@435 | 3194 | // array klass has not yet been cached. Either the |
duke@435 | 3195 | // ensuing call will throw an exception, or else it |
duke@435 | 3196 | // will cache the array klass for next time. |
duke@435 | 3197 | PreserveJVMState pjvms(this); |
duke@435 | 3198 | CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray); |
duke@435 | 3199 | Node* slow_result = set_results_for_java_call(slow_call); |
duke@435 | 3200 | // this->control() comes from set_results_for_java_call |
duke@435 | 3201 | result_reg->set_req(_slow_path, control()); |
duke@435 | 3202 | result_val->set_req(_slow_path, slow_result); |
duke@435 | 3203 | result_io ->set_req(_slow_path, i_o()); |
duke@435 | 3204 | result_mem->set_req(_slow_path, reset_memory()); |
duke@435 | 3205 | } |
duke@435 | 3206 | |
duke@435 | 3207 | set_control(normal_ctl); |
duke@435 | 3208 | if (!stopped()) { |
duke@435 | 3209 | // Normal case: The array type has been cached in the java.lang.Class. |
duke@435 | 3210 | // The following call works fine even if the array type is polymorphic. |
duke@435 | 3211 | // It could be a dynamic mix of int[], boolean[], Object[], etc. |
cfang@1165 | 3212 | Node* obj = new_array(klass_node, count_val, nargs); |
duke@435 | 3213 | result_reg->init_req(_normal_path, control()); |
duke@435 | 3214 | result_val->init_req(_normal_path, obj); |
duke@435 | 3215 | result_io ->init_req(_normal_path, i_o()); |
duke@435 | 3216 | result_mem->init_req(_normal_path, reset_memory()); |
duke@435 | 3217 | } |
duke@435 | 3218 | |
duke@435 | 3219 | // Return the combined state. |
duke@435 | 3220 | set_i_o( _gvn.transform(result_io) ); |
duke@435 | 3221 | set_all_memory( _gvn.transform(result_mem) ); |
duke@435 | 3222 | push_result(result_reg, result_val); |
duke@435 | 3223 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3224 | |
duke@435 | 3225 | return true; |
duke@435 | 3226 | } |
duke@435 | 3227 | |
duke@435 | 3228 | //----------------------inline_native_getLength-------------------------- |
duke@435 | 3229 | bool LibraryCallKit::inline_native_getLength() { |
duke@435 | 3230 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 3231 | |
duke@435 | 3232 | int nargs = 1; |
duke@435 | 3233 | Node* array = argument(0); |
duke@435 | 3234 | |
duke@435 | 3235 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3236 | array = do_null_check(array, T_OBJECT); |
duke@435 | 3237 | _sp -= nargs; |
duke@435 | 3238 | |
duke@435 | 3239 | // If array is dead, only null-path is taken. |
duke@435 | 3240 | if (stopped()) return true; |
duke@435 | 3241 | |
duke@435 | 3242 | // Deoptimize if it is a non-array. |
duke@435 | 3243 | Node* non_array = generate_non_array_guard(load_object_klass(array), NULL); |
duke@435 | 3244 | |
duke@435 | 3245 | if (non_array != NULL) { |
duke@435 | 3246 | PreserveJVMState pjvms(this); |
duke@435 | 3247 | set_control(non_array); |
duke@435 | 3248 | _sp += nargs; // push the arguments back on the stack |
duke@435 | 3249 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 3250 | Deoptimization::Action_maybe_recompile); |
duke@435 | 3251 | } |
duke@435 | 3252 | |
duke@435 | 3253 | // If control is dead, only non-array-path is taken. |
duke@435 | 3254 | if (stopped()) return true; |
duke@435 | 3255 | |
duke@435 | 3256 | // The works fine even if the array type is polymorphic. |
duke@435 | 3257 | // It could be a dynamic mix of int[], boolean[], Object[], etc. |
duke@435 | 3258 | push( load_array_length(array) ); |
duke@435 | 3259 | |
duke@435 | 3260 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3261 | |
duke@435 | 3262 | return true; |
duke@435 | 3263 | } |
duke@435 | 3264 | |
duke@435 | 3265 | //------------------------inline_array_copyOf---------------------------- |
duke@435 | 3266 | bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { |
duke@435 | 3267 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 3268 | |
duke@435 | 3269 | // Restore the stack and pop off the arguments. |
duke@435 | 3270 | int nargs = 3 + (is_copyOfRange? 1: 0); |
duke@435 | 3271 | Node* original = argument(0); |
duke@435 | 3272 | Node* start = is_copyOfRange? argument(1): intcon(0); |
duke@435 | 3273 | Node* end = is_copyOfRange? argument(2): argument(1); |
duke@435 | 3274 | Node* array_type_mirror = is_copyOfRange? argument(3): argument(2); |
duke@435 | 3275 | |
cfang@1337 | 3276 | Node* newcopy; |
cfang@1337 | 3277 | |
cfang@1337 | 3278 | //set the original stack and the reexecute bit for the interpreter to reexecute |
cfang@1337 | 3279 | //the bytecode that invokes Arrays.copyOf if deoptimization happens |
cfang@1337 | 3280 | { PreserveReexecuteState preexecs(this); |
cfang@1337 | 3281 | _sp += nargs; |
cfang@1337 | 3282 | jvms()->set_should_reexecute(true); |
cfang@1337 | 3283 | |
cfang@1337 | 3284 | array_type_mirror = do_null_check(array_type_mirror, T_OBJECT); |
cfang@1337 | 3285 | original = do_null_check(original, T_OBJECT); |
cfang@1337 | 3286 | |
cfang@1337 | 3287 | // Check if a null path was taken unconditionally. |
cfang@1337 | 3288 | if (stopped()) return true; |
cfang@1337 | 3289 | |
cfang@1337 | 3290 | Node* orig_length = load_array_length(original); |
cfang@1337 | 3291 | |
cfang@1337 | 3292 | Node* klass_node = load_klass_from_mirror(array_type_mirror, false, 0, |
cfang@1337 | 3293 | NULL, 0); |
cfang@1337 | 3294 | klass_node = do_null_check(klass_node, T_OBJECT); |
cfang@1337 | 3295 | |
cfang@1337 | 3296 | RegionNode* bailout = new (C, 1) RegionNode(1); |
cfang@1337 | 3297 | record_for_igvn(bailout); |
cfang@1337 | 3298 | |
cfang@1337 | 3299 | // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc. |
cfang@1337 | 3300 | // Bail out if that is so. |
cfang@1337 | 3301 | Node* not_objArray = generate_non_objArray_guard(klass_node, bailout); |
cfang@1337 | 3302 | if (not_objArray != NULL) { |
cfang@1337 | 3303 | // Improve the klass node's type from the new optimistic assumption: |
cfang@1337 | 3304 | ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); |
cfang@1337 | 3305 | const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/); |
cfang@1337 | 3306 | Node* cast = new (C, 2) CastPPNode(klass_node, akls); |
cfang@1337 | 3307 | cast->init_req(0, control()); |
cfang@1337 | 3308 | klass_node = _gvn.transform(cast); |
cfang@1337 | 3309 | } |
cfang@1337 | 3310 | |
cfang@1337 | 3311 | // Bail out if either start or end is negative. |
cfang@1337 | 3312 | generate_negative_guard(start, bailout, &start); |
cfang@1337 | 3313 | generate_negative_guard(end, bailout, &end); |
cfang@1337 | 3314 | |
cfang@1337 | 3315 | Node* length = end; |
cfang@1337 | 3316 | if (_gvn.type(start) != TypeInt::ZERO) { |
cfang@1337 | 3317 | length = _gvn.transform( new (C, 3) SubINode(end, start) ); |
cfang@1337 | 3318 | } |
cfang@1337 | 3319 | |
cfang@1337 | 3320 | // Bail out if length is negative. |
cfang@1337 | 3321 | // ...Not needed, since the new_array will throw the right exception. |
cfang@1337 | 3322 | //generate_negative_guard(length, bailout, &length); |
cfang@1337 | 3323 | |
cfang@1337 | 3324 | if (bailout->req() > 1) { |
cfang@1337 | 3325 | PreserveJVMState pjvms(this); |
cfang@1337 | 3326 | set_control( _gvn.transform(bailout) ); |
cfang@1337 | 3327 | uncommon_trap(Deoptimization::Reason_intrinsic, |
cfang@1337 | 3328 | Deoptimization::Action_maybe_recompile); |
cfang@1337 | 3329 | } |
cfang@1337 | 3330 | |
cfang@1337 | 3331 | if (!stopped()) { |
cfang@1335 | 3332 | |
cfang@1335 | 3333 | // How many elements will we copy from the original? |
cfang@1335 | 3334 | // The answer is MinI(orig_length - start, length). |
cfang@1335 | 3335 | Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); |
cfang@1335 | 3336 | Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); |
cfang@1335 | 3337 | |
cfang@1335 | 3338 | const bool raw_mem_only = true; |
cfang@1335 | 3339 | newcopy = new_array(klass_node, length, 0, raw_mem_only); |
cfang@1335 | 3340 | |
cfang@1335 | 3341 | // Generate a direct call to the right arraycopy function(s). |
cfang@1335 | 3342 | // We know the copy is disjoint but we might not know if the |
cfang@1335 | 3343 | // oop stores need checking. |
cfang@1335 | 3344 | // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). |
cfang@1335 | 3345 | // This will fail a store-check if x contains any non-nulls. |
cfang@1335 | 3346 | bool disjoint_bases = true; |
cfang@1335 | 3347 | bool length_never_negative = true; |
cfang@1335 | 3348 | generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
cfang@1335 | 3349 | original, start, newcopy, intcon(0), moved, |
cfang@1335 | 3350 | disjoint_bases, length_never_negative); |
cfang@1337 | 3351 | } |
cfang@1337 | 3352 | } //original reexecute and sp are set back here |
cfang@1337 | 3353 | |
cfang@1337 | 3354 | if(!stopped()) { |
duke@435 | 3355 | push(newcopy); |
duke@435 | 3356 | } |
duke@435 | 3357 | |
duke@435 | 3358 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3359 | |
duke@435 | 3360 | return true; |
duke@435 | 3361 | } |
duke@435 | 3362 | |
duke@435 | 3363 | |
duke@435 | 3364 | //----------------------generate_virtual_guard--------------------------- |
duke@435 | 3365 | // Helper for hashCode and clone. Peeks inside the vtable to avoid a call. |
duke@435 | 3366 | Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, |
duke@435 | 3367 | RegionNode* slow_region) { |
duke@435 | 3368 | ciMethod* method = callee(); |
duke@435 | 3369 | int vtable_index = method->vtable_index(); |
duke@435 | 3370 | // Get the methodOop out of the appropriate vtable entry. |
duke@435 | 3371 | int entry_offset = (instanceKlass::vtable_start_offset() + |
duke@435 | 3372 | vtable_index*vtableEntry::size()) * wordSize + |
duke@435 | 3373 | vtableEntry::method_offset_in_bytes(); |
duke@435 | 3374 | Node* entry_addr = basic_plus_adr(obj_klass, entry_offset); |
duke@435 | 3375 | Node* target_call = make_load(NULL, entry_addr, TypeInstPtr::NOTNULL, T_OBJECT); |
duke@435 | 3376 | |
duke@435 | 3377 | // Compare the target method with the expected method (e.g., Object.hashCode). |
duke@435 | 3378 | const TypeInstPtr* native_call_addr = TypeInstPtr::make(method); |
duke@435 | 3379 | |
duke@435 | 3380 | Node* native_call = makecon(native_call_addr); |
duke@435 | 3381 | Node* chk_native = _gvn.transform( new(C, 3) CmpPNode(target_call, native_call) ); |
duke@435 | 3382 | Node* test_native = _gvn.transform( new(C, 2) BoolNode(chk_native, BoolTest::ne) ); |
duke@435 | 3383 | |
duke@435 | 3384 | return generate_slow_guard(test_native, slow_region); |
duke@435 | 3385 | } |
duke@435 | 3386 | |
duke@435 | 3387 | //-----------------------generate_method_call---------------------------- |
duke@435 | 3388 | // Use generate_method_call to make a slow-call to the real |
duke@435 | 3389 | // method if the fast path fails. An alternative would be to |
duke@435 | 3390 | // use a stub like OptoRuntime::slow_arraycopy_Java. |
duke@435 | 3391 | // This only works for expanding the current library call, |
duke@435 | 3392 | // not another intrinsic. (E.g., don't use this for making an |
duke@435 | 3393 | // arraycopy call inside of the copyOf intrinsic.) |
duke@435 | 3394 | CallJavaNode* |
duke@435 | 3395 | LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) { |
duke@435 | 3396 | // When compiling the intrinsic method itself, do not use this technique. |
duke@435 | 3397 | guarantee(callee() != C->method(), "cannot make slow-call to self"); |
duke@435 | 3398 | |
duke@435 | 3399 | ciMethod* method = callee(); |
duke@435 | 3400 | // ensure the JVMS we have will be correct for this call |
duke@435 | 3401 | guarantee(method_id == method->intrinsic_id(), "must match"); |
duke@435 | 3402 | |
duke@435 | 3403 | const TypeFunc* tf = TypeFunc::make(method); |
duke@435 | 3404 | int tfdc = tf->domain()->cnt(); |
duke@435 | 3405 | CallJavaNode* slow_call; |
duke@435 | 3406 | if (is_static) { |
duke@435 | 3407 | assert(!is_virtual, ""); |
duke@435 | 3408 | slow_call = new(C, tfdc) CallStaticJavaNode(tf, |
duke@435 | 3409 | SharedRuntime::get_resolve_static_call_stub(), |
duke@435 | 3410 | method, bci()); |
duke@435 | 3411 | } else if (is_virtual) { |
duke@435 | 3412 | null_check_receiver(method); |
duke@435 | 3413 | int vtable_index = methodOopDesc::invalid_vtable_index; |
duke@435 | 3414 | if (UseInlineCaches) { |
duke@435 | 3415 | // Suppress the vtable call |
duke@435 | 3416 | } else { |
duke@435 | 3417 | // hashCode and clone are not a miranda methods, |
duke@435 | 3418 | // so the vtable index is fixed. |
duke@435 | 3419 | // No need to use the linkResolver to get it. |
duke@435 | 3420 | vtable_index = method->vtable_index(); |
duke@435 | 3421 | } |
duke@435 | 3422 | slow_call = new(C, tfdc) CallDynamicJavaNode(tf, |
duke@435 | 3423 | SharedRuntime::get_resolve_virtual_call_stub(), |
duke@435 | 3424 | method, vtable_index, bci()); |
duke@435 | 3425 | } else { // neither virtual nor static: opt_virtual |
duke@435 | 3426 | null_check_receiver(method); |
duke@435 | 3427 | slow_call = new(C, tfdc) CallStaticJavaNode(tf, |
duke@435 | 3428 | SharedRuntime::get_resolve_opt_virtual_call_stub(), |
duke@435 | 3429 | method, bci()); |
duke@435 | 3430 | slow_call->set_optimized_virtual(true); |
duke@435 | 3431 | } |
duke@435 | 3432 | set_arguments_for_java_call(slow_call); |
duke@435 | 3433 | set_edges_for_java_call(slow_call); |
duke@435 | 3434 | return slow_call; |
duke@435 | 3435 | } |
duke@435 | 3436 | |
duke@435 | 3437 | |
duke@435 | 3438 | //------------------------------inline_native_hashcode-------------------- |
duke@435 | 3439 | // Build special case code for calls to hashCode on an object. |
duke@435 | 3440 | bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { |
duke@435 | 3441 | assert(is_static == callee()->is_static(), "correct intrinsic selection"); |
duke@435 | 3442 | assert(!(is_virtual && is_static), "either virtual, special, or static"); |
duke@435 | 3443 | |
duke@435 | 3444 | enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT }; |
duke@435 | 3445 | |
duke@435 | 3446 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3447 | PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
duke@435 | 3448 | TypeInt::INT); |
duke@435 | 3449 | PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
duke@435 | 3450 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
duke@435 | 3451 | TypePtr::BOTTOM); |
duke@435 | 3452 | Node* obj = NULL; |
duke@435 | 3453 | if (!is_static) { |
duke@435 | 3454 | // Check for hashing null object |
duke@435 | 3455 | obj = null_check_receiver(callee()); |
duke@435 | 3456 | if (stopped()) return true; // unconditionally null |
duke@435 | 3457 | result_reg->init_req(_null_path, top()); |
duke@435 | 3458 | result_val->init_req(_null_path, top()); |
duke@435 | 3459 | } else { |
duke@435 | 3460 | // Do a null check, and return zero if null. |
duke@435 | 3461 | // System.identityHashCode(null) == 0 |
duke@435 | 3462 | obj = argument(0); |
duke@435 | 3463 | Node* null_ctl = top(); |
duke@435 | 3464 | obj = null_check_oop(obj, &null_ctl); |
duke@435 | 3465 | result_reg->init_req(_null_path, null_ctl); |
duke@435 | 3466 | result_val->init_req(_null_path, _gvn.intcon(0)); |
duke@435 | 3467 | } |
duke@435 | 3468 | |
duke@435 | 3469 | // Unconditionally null? Then return right away. |
duke@435 | 3470 | if (stopped()) { |
duke@435 | 3471 | set_control( result_reg->in(_null_path) ); |
duke@435 | 3472 | if (!stopped()) |
duke@435 | 3473 | push( result_val ->in(_null_path) ); |
duke@435 | 3474 | return true; |
duke@435 | 3475 | } |
duke@435 | 3476 | |
duke@435 | 3477 | // After null check, get the object's klass. |
duke@435 | 3478 | Node* obj_klass = load_object_klass(obj); |
duke@435 | 3479 | |
duke@435 | 3480 | // This call may be virtual (invokevirtual) or bound (invokespecial). |
duke@435 | 3481 | // For each case we generate slightly different code. |
duke@435 | 3482 | |
duke@435 | 3483 | // We only go to the fast case code if we pass a number of guards. The |
duke@435 | 3484 | // paths which do not pass are accumulated in the slow_region. |
duke@435 | 3485 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 3486 | record_for_igvn(slow_region); |
duke@435 | 3487 | |
duke@435 | 3488 | // If this is a virtual call, we generate a funny guard. We pull out |
duke@435 | 3489 | // the vtable entry corresponding to hashCode() from the target object. |
duke@435 | 3490 | // If the target method which we are calling happens to be the native |
duke@435 | 3491 | // Object hashCode() method, we pass the guard. We do not need this |
duke@435 | 3492 | // guard for non-virtual calls -- the caller is known to be the native |
duke@435 | 3493 | // Object hashCode(). |
duke@435 | 3494 | if (is_virtual) { |
duke@435 | 3495 | generate_virtual_guard(obj_klass, slow_region); |
duke@435 | 3496 | } |
duke@435 | 3497 | |
duke@435 | 3498 | // Get the header out of the object, use LoadMarkNode when available |
duke@435 | 3499 | Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); |
duke@435 | 3500 | Node* header = make_load(NULL, header_addr, TypeRawPtr::BOTTOM, T_ADDRESS); |
duke@435 | 3501 | header = _gvn.transform( new (C, 2) CastP2XNode(NULL, header) ); |
duke@435 | 3502 | |
duke@435 | 3503 | // Test the header to see if it is unlocked. |
duke@435 | 3504 | Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place); |
duke@435 | 3505 | Node *lmasked_header = _gvn.transform( new (C, 3) AndXNode(header, lock_mask) ); |
duke@435 | 3506 | Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value); |
duke@435 | 3507 | Node *chk_unlocked = _gvn.transform( new (C, 3) CmpXNode( lmasked_header, unlocked_val)); |
duke@435 | 3508 | Node *test_unlocked = _gvn.transform( new (C, 2) BoolNode( chk_unlocked, BoolTest::ne) ); |
duke@435 | 3509 | |
duke@435 | 3510 | generate_slow_guard(test_unlocked, slow_region); |
duke@435 | 3511 | |
duke@435 | 3512 | // Get the hash value and check to see that it has been properly assigned. |
duke@435 | 3513 | // We depend on hash_mask being at most 32 bits and avoid the use of |
duke@435 | 3514 | // hash_mask_in_place because it could be larger than 32 bits in a 64-bit |
duke@435 | 3515 | // vm: see markOop.hpp. |
duke@435 | 3516 | Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask); |
duke@435 | 3517 | Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift); |
duke@435 | 3518 | Node *hshifted_header= _gvn.transform( new (C, 3) URShiftXNode(header, hash_shift) ); |
duke@435 | 3519 | // This hack lets the hash bits live anywhere in the mark object now, as long |
twisti@1040 | 3520 | // as the shift drops the relevant bits into the low 32 bits. Note that |
duke@435 | 3521 | // Java spec says that HashCode is an int so there's no point in capturing |
duke@435 | 3522 | // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build). |
duke@435 | 3523 | hshifted_header = ConvX2I(hshifted_header); |
duke@435 | 3524 | Node *hash_val = _gvn.transform( new (C, 3) AndINode(hshifted_header, hash_mask) ); |
duke@435 | 3525 | |
duke@435 | 3526 | Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash); |
duke@435 | 3527 | Node *chk_assigned = _gvn.transform( new (C, 3) CmpINode( hash_val, no_hash_val)); |
duke@435 | 3528 | Node *test_assigned = _gvn.transform( new (C, 2) BoolNode( chk_assigned, BoolTest::eq) ); |
duke@435 | 3529 | |
duke@435 | 3530 | generate_slow_guard(test_assigned, slow_region); |
duke@435 | 3531 | |
duke@435 | 3532 | Node* init_mem = reset_memory(); |
duke@435 | 3533 | // fill in the rest of the null path: |
duke@435 | 3534 | result_io ->init_req(_null_path, i_o()); |
duke@435 | 3535 | result_mem->init_req(_null_path, init_mem); |
duke@435 | 3536 | |
duke@435 | 3537 | result_val->init_req(_fast_path, hash_val); |
duke@435 | 3538 | result_reg->init_req(_fast_path, control()); |
duke@435 | 3539 | result_io ->init_req(_fast_path, i_o()); |
duke@435 | 3540 | result_mem->init_req(_fast_path, init_mem); |
duke@435 | 3541 | |
duke@435 | 3542 | // Generate code for the slow case. We make a call to hashCode(). |
duke@435 | 3543 | set_control(_gvn.transform(slow_region)); |
duke@435 | 3544 | if (!stopped()) { |
duke@435 | 3545 | // No need for PreserveJVMState, because we're using up the present state. |
duke@435 | 3546 | set_all_memory(init_mem); |
duke@435 | 3547 | vmIntrinsics::ID hashCode_id = vmIntrinsics::_hashCode; |
duke@435 | 3548 | if (is_static) hashCode_id = vmIntrinsics::_identityHashCode; |
duke@435 | 3549 | CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static); |
duke@435 | 3550 | Node* slow_result = set_results_for_java_call(slow_call); |
duke@435 | 3551 | // this->control() comes from set_results_for_java_call |
duke@435 | 3552 | result_reg->init_req(_slow_path, control()); |
duke@435 | 3553 | result_val->init_req(_slow_path, slow_result); |
duke@435 | 3554 | result_io ->set_req(_slow_path, i_o()); |
duke@435 | 3555 | result_mem ->set_req(_slow_path, reset_memory()); |
duke@435 | 3556 | } |
duke@435 | 3557 | |
duke@435 | 3558 | // Return the combined state. |
duke@435 | 3559 | set_i_o( _gvn.transform(result_io) ); |
duke@435 | 3560 | set_all_memory( _gvn.transform(result_mem) ); |
duke@435 | 3561 | push_result(result_reg, result_val); |
duke@435 | 3562 | |
duke@435 | 3563 | return true; |
duke@435 | 3564 | } |
duke@435 | 3565 | |
duke@435 | 3566 | //---------------------------inline_native_getClass---------------------------- |
twisti@1040 | 3567 | // Build special case code for calls to getClass on an object. |
duke@435 | 3568 | bool LibraryCallKit::inline_native_getClass() { |
duke@435 | 3569 | Node* obj = null_check_receiver(callee()); |
duke@435 | 3570 | if (stopped()) return true; |
duke@435 | 3571 | push( load_mirror_from_klass(load_object_klass(obj)) ); |
duke@435 | 3572 | return true; |
duke@435 | 3573 | } |
duke@435 | 3574 | |
duke@435 | 3575 | //-----------------inline_native_Reflection_getCallerClass--------------------- |
duke@435 | 3576 | // In the presence of deep enough inlining, getCallerClass() becomes a no-op. |
duke@435 | 3577 | // |
duke@435 | 3578 | // NOTE that this code must perform the same logic as |
duke@435 | 3579 | // vframeStream::security_get_caller_frame in that it must skip |
duke@435 | 3580 | // Method.invoke() and auxiliary frames. |
duke@435 | 3581 | |
duke@435 | 3582 | |
duke@435 | 3583 | |
duke@435 | 3584 | |
duke@435 | 3585 | bool LibraryCallKit::inline_native_Reflection_getCallerClass() { |
duke@435 | 3586 | ciMethod* method = callee(); |
duke@435 | 3587 | |
duke@435 | 3588 | #ifndef PRODUCT |
duke@435 | 3589 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3590 | tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass"); |
duke@435 | 3591 | } |
duke@435 | 3592 | #endif |
duke@435 | 3593 | |
duke@435 | 3594 | debug_only(int saved_sp = _sp); |
duke@435 | 3595 | |
duke@435 | 3596 | // Argument words: (int depth) |
duke@435 | 3597 | int nargs = 1; |
duke@435 | 3598 | |
duke@435 | 3599 | _sp += nargs; |
duke@435 | 3600 | Node* caller_depth_node = pop(); |
duke@435 | 3601 | |
duke@435 | 3602 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 3603 | |
duke@435 | 3604 | // The depth value must be a constant in order for the runtime call |
duke@435 | 3605 | // to be eliminated. |
duke@435 | 3606 | const TypeInt* caller_depth_type = _gvn.type(caller_depth_node)->isa_int(); |
duke@435 | 3607 | if (caller_depth_type == NULL || !caller_depth_type->is_con()) { |
duke@435 | 3608 | #ifndef PRODUCT |
duke@435 | 3609 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3610 | tty->print_cr(" Bailing out because caller depth was not a constant"); |
duke@435 | 3611 | } |
duke@435 | 3612 | #endif |
duke@435 | 3613 | return false; |
duke@435 | 3614 | } |
duke@435 | 3615 | // Note that the JVM state at this point does not include the |
duke@435 | 3616 | // getCallerClass() frame which we are trying to inline. The |
duke@435 | 3617 | // semantics of getCallerClass(), however, are that the "first" |
duke@435 | 3618 | // frame is the getCallerClass() frame, so we subtract one from the |
duke@435 | 3619 | // requested depth before continuing. We don't inline requests of |
duke@435 | 3620 | // getCallerClass(0). |
duke@435 | 3621 | int caller_depth = caller_depth_type->get_con() - 1; |
duke@435 | 3622 | if (caller_depth < 0) { |
duke@435 | 3623 | #ifndef PRODUCT |
duke@435 | 3624 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3625 | tty->print_cr(" Bailing out because caller depth was %d", caller_depth); |
duke@435 | 3626 | } |
duke@435 | 3627 | #endif |
duke@435 | 3628 | return false; |
duke@435 | 3629 | } |
duke@435 | 3630 | |
duke@435 | 3631 | if (!jvms()->has_method()) { |
duke@435 | 3632 | #ifndef PRODUCT |
duke@435 | 3633 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3634 | tty->print_cr(" Bailing out because intrinsic was inlined at top level"); |
duke@435 | 3635 | } |
duke@435 | 3636 | #endif |
duke@435 | 3637 | return false; |
duke@435 | 3638 | } |
duke@435 | 3639 | int _depth = jvms()->depth(); // cache call chain depth |
duke@435 | 3640 | |
duke@435 | 3641 | // Walk back up the JVM state to find the caller at the required |
duke@435 | 3642 | // depth. NOTE that this code must perform the same logic as |
duke@435 | 3643 | // vframeStream::security_get_caller_frame in that it must skip |
duke@435 | 3644 | // Method.invoke() and auxiliary frames. Note also that depth is |
duke@435 | 3645 | // 1-based (1 is the bottom of the inlining). |
duke@435 | 3646 | int inlining_depth = _depth; |
duke@435 | 3647 | JVMState* caller_jvms = NULL; |
duke@435 | 3648 | |
duke@435 | 3649 | if (inlining_depth > 0) { |
duke@435 | 3650 | caller_jvms = jvms(); |
duke@435 | 3651 | assert(caller_jvms = jvms()->of_depth(inlining_depth), "inlining_depth == our depth"); |
duke@435 | 3652 | do { |
duke@435 | 3653 | // The following if-tests should be performed in this order |
duke@435 | 3654 | if (is_method_invoke_or_aux_frame(caller_jvms)) { |
duke@435 | 3655 | // Skip a Method.invoke() or auxiliary frame |
duke@435 | 3656 | } else if (caller_depth > 0) { |
duke@435 | 3657 | // Skip real frame |
duke@435 | 3658 | --caller_depth; |
duke@435 | 3659 | } else { |
duke@435 | 3660 | // We're done: reached desired caller after skipping. |
duke@435 | 3661 | break; |
duke@435 | 3662 | } |
duke@435 | 3663 | caller_jvms = caller_jvms->caller(); |
duke@435 | 3664 | --inlining_depth; |
duke@435 | 3665 | } while (inlining_depth > 0); |
duke@435 | 3666 | } |
duke@435 | 3667 | |
duke@435 | 3668 | if (inlining_depth == 0) { |
duke@435 | 3669 | #ifndef PRODUCT |
duke@435 | 3670 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3671 | tty->print_cr(" Bailing out because caller depth (%d) exceeded inlining depth (%d)", caller_depth_type->get_con(), _depth); |
duke@435 | 3672 | tty->print_cr(" JVM state at this point:"); |
duke@435 | 3673 | for (int i = _depth; i >= 1; i--) { |
duke@435 | 3674 | tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8()); |
duke@435 | 3675 | } |
duke@435 | 3676 | } |
duke@435 | 3677 | #endif |
duke@435 | 3678 | return false; // Reached end of inlining |
duke@435 | 3679 | } |
duke@435 | 3680 | |
duke@435 | 3681 | // Acquire method holder as java.lang.Class |
duke@435 | 3682 | ciInstanceKlass* caller_klass = caller_jvms->method()->holder(); |
duke@435 | 3683 | ciInstance* caller_mirror = caller_klass->java_mirror(); |
duke@435 | 3684 | // Push this as a constant |
duke@435 | 3685 | push(makecon(TypeInstPtr::make(caller_mirror))); |
duke@435 | 3686 | #ifndef PRODUCT |
duke@435 | 3687 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3688 | tty->print_cr(" Succeeded: caller = %s.%s, caller depth = %d, depth = %d", caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), caller_depth_type->get_con(), _depth); |
duke@435 | 3689 | tty->print_cr(" JVM state at this point:"); |
duke@435 | 3690 | for (int i = _depth; i >= 1; i--) { |
duke@435 | 3691 | tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8()); |
duke@435 | 3692 | } |
duke@435 | 3693 | } |
duke@435 | 3694 | #endif |
duke@435 | 3695 | return true; |
duke@435 | 3696 | } |
duke@435 | 3697 | |
duke@435 | 3698 | // Helper routine for above |
duke@435 | 3699 | bool LibraryCallKit::is_method_invoke_or_aux_frame(JVMState* jvms) { |
twisti@1587 | 3700 | ciMethod* method = jvms->method(); |
twisti@1587 | 3701 | |
duke@435 | 3702 | // Is this the Method.invoke method itself? |
twisti@1587 | 3703 | if (method->intrinsic_id() == vmIntrinsics::_invoke) |
duke@435 | 3704 | return true; |
duke@435 | 3705 | |
duke@435 | 3706 | // Is this a helper, defined somewhere underneath MethodAccessorImpl. |
twisti@1587 | 3707 | ciKlass* k = method->holder(); |
duke@435 | 3708 | if (k->is_instance_klass()) { |
duke@435 | 3709 | ciInstanceKlass* ik = k->as_instance_klass(); |
duke@435 | 3710 | for (; ik != NULL; ik = ik->super()) { |
duke@435 | 3711 | if (ik->name() == ciSymbol::sun_reflect_MethodAccessorImpl() && |
duke@435 | 3712 | ik == env()->find_system_klass(ik->name())) { |
duke@435 | 3713 | return true; |
duke@435 | 3714 | } |
duke@435 | 3715 | } |
duke@435 | 3716 | } |
twisti@1587 | 3717 | else if (method->is_method_handle_adapter()) { |
twisti@1587 | 3718 | // This is an internal adapter frame from the MethodHandleCompiler -- skip it |
twisti@1587 | 3719 | return true; |
twisti@1587 | 3720 | } |
duke@435 | 3721 | |
duke@435 | 3722 | return false; |
duke@435 | 3723 | } |
duke@435 | 3724 | |
duke@435 | 3725 | static int value_field_offset = -1; // offset of the "value" field of AtomicLongCSImpl. This is needed by |
duke@435 | 3726 | // inline_native_AtomicLong_attemptUpdate() but it has no way of |
duke@435 | 3727 | // computing it since there is no lookup field by name function in the |
duke@435 | 3728 | // CI interface. This is computed and set by inline_native_AtomicLong_get(). |
duke@435 | 3729 | // Using a static variable here is safe even if we have multiple compilation |
duke@435 | 3730 | // threads because the offset is constant. At worst the same offset will be |
duke@435 | 3731 | // computed and stored multiple |
duke@435 | 3732 | |
duke@435 | 3733 | bool LibraryCallKit::inline_native_AtomicLong_get() { |
duke@435 | 3734 | // Restore the stack and pop off the argument |
duke@435 | 3735 | _sp+=1; |
duke@435 | 3736 | Node *obj = pop(); |
duke@435 | 3737 | |
duke@435 | 3738 | // get the offset of the "value" field. Since the CI interfaces |
duke@435 | 3739 | // does not provide a way to look up a field by name, we scan the bytecodes |
duke@435 | 3740 | // to get the field index. We expect the first 2 instructions of the method |
duke@435 | 3741 | // to be: |
duke@435 | 3742 | // 0 aload_0 |
duke@435 | 3743 | // 1 getfield "value" |
duke@435 | 3744 | ciMethod* method = callee(); |
duke@435 | 3745 | if (value_field_offset == -1) |
duke@435 | 3746 | { |
duke@435 | 3747 | ciField* value_field; |
duke@435 | 3748 | ciBytecodeStream iter(method); |
duke@435 | 3749 | Bytecodes::Code bc = iter.next(); |
duke@435 | 3750 | |
duke@435 | 3751 | if ((bc != Bytecodes::_aload_0) && |
duke@435 | 3752 | ((bc != Bytecodes::_aload) || (iter.get_index() != 0))) |
duke@435 | 3753 | return false; |
duke@435 | 3754 | bc = iter.next(); |
duke@435 | 3755 | if (bc != Bytecodes::_getfield) |
duke@435 | 3756 | return false; |
duke@435 | 3757 | bool ignore; |
duke@435 | 3758 | value_field = iter.get_field(ignore); |
duke@435 | 3759 | value_field_offset = value_field->offset_in_bytes(); |
duke@435 | 3760 | } |
duke@435 | 3761 | |
duke@435 | 3762 | // Null check without removing any arguments. |
duke@435 | 3763 | _sp++; |
duke@435 | 3764 | obj = do_null_check(obj, T_OBJECT); |
duke@435 | 3765 | _sp--; |
duke@435 | 3766 | // Check for locking null object |
duke@435 | 3767 | if (stopped()) return true; |
duke@435 | 3768 | |
duke@435 | 3769 | Node *adr = basic_plus_adr(obj, obj, value_field_offset); |
duke@435 | 3770 | const TypePtr *adr_type = _gvn.type(adr)->is_ptr(); |
duke@435 | 3771 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 3772 | |
duke@435 | 3773 | Node *result = _gvn.transform(new (C, 3) LoadLLockedNode(control(), memory(alias_idx), adr)); |
duke@435 | 3774 | |
duke@435 | 3775 | push_pair(result); |
duke@435 | 3776 | |
duke@435 | 3777 | return true; |
duke@435 | 3778 | } |
duke@435 | 3779 | |
duke@435 | 3780 | bool LibraryCallKit::inline_native_AtomicLong_attemptUpdate() { |
duke@435 | 3781 | // Restore the stack and pop off the arguments |
duke@435 | 3782 | _sp+=5; |
duke@435 | 3783 | Node *newVal = pop_pair(); |
duke@435 | 3784 | Node *oldVal = pop_pair(); |
duke@435 | 3785 | Node *obj = pop(); |
duke@435 | 3786 | |
duke@435 | 3787 | // we need the offset of the "value" field which was computed when |
duke@435 | 3788 | // inlining the get() method. Give up if we don't have it. |
duke@435 | 3789 | if (value_field_offset == -1) |
duke@435 | 3790 | return false; |
duke@435 | 3791 | |
duke@435 | 3792 | // Null check without removing any arguments. |
duke@435 | 3793 | _sp+=5; |
duke@435 | 3794 | obj = do_null_check(obj, T_OBJECT); |
duke@435 | 3795 | _sp-=5; |
duke@435 | 3796 | // Check for locking null object |
duke@435 | 3797 | if (stopped()) return true; |
duke@435 | 3798 | |
duke@435 | 3799 | Node *adr = basic_plus_adr(obj, obj, value_field_offset); |
duke@435 | 3800 | const TypePtr *adr_type = _gvn.type(adr)->is_ptr(); |
duke@435 | 3801 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 3802 | |
kvn@855 | 3803 | Node *cas = _gvn.transform(new (C, 5) StoreLConditionalNode(control(), memory(alias_idx), adr, newVal, oldVal)); |
kvn@855 | 3804 | Node *store_proj = _gvn.transform( new (C, 1) SCMemProjNode(cas)); |
duke@435 | 3805 | set_memory(store_proj, alias_idx); |
kvn@855 | 3806 | Node *bol = _gvn.transform( new (C, 2) BoolNode( cas, BoolTest::eq ) ); |
kvn@855 | 3807 | |
kvn@855 | 3808 | Node *result; |
kvn@855 | 3809 | // CMove node is not used to be able fold a possible check code |
kvn@855 | 3810 | // after attemptUpdate() call. This code could be transformed |
kvn@855 | 3811 | // into CMove node by loop optimizations. |
kvn@855 | 3812 | { |
kvn@855 | 3813 | RegionNode *r = new (C, 3) RegionNode(3); |
kvn@855 | 3814 | result = new (C, 3) PhiNode(r, TypeInt::BOOL); |
kvn@855 | 3815 | |
kvn@855 | 3816 | Node *iff = create_and_xform_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN); |
kvn@855 | 3817 | Node *iftrue = opt_iff(r, iff); |
kvn@855 | 3818 | r->init_req(1, iftrue); |
kvn@855 | 3819 | result->init_req(1, intcon(1)); |
kvn@855 | 3820 | result->init_req(2, intcon(0)); |
kvn@855 | 3821 | |
kvn@855 | 3822 | set_control(_gvn.transform(r)); |
kvn@855 | 3823 | record_for_igvn(r); |
kvn@855 | 3824 | |
kvn@855 | 3825 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
kvn@855 | 3826 | } |
kvn@855 | 3827 | |
kvn@855 | 3828 | push(_gvn.transform(result)); |
duke@435 | 3829 | return true; |
duke@435 | 3830 | } |
duke@435 | 3831 | |
duke@435 | 3832 | bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) { |
duke@435 | 3833 | // restore the arguments |
duke@435 | 3834 | _sp += arg_size(); |
duke@435 | 3835 | |
duke@435 | 3836 | switch (id) { |
duke@435 | 3837 | case vmIntrinsics::_floatToRawIntBits: |
duke@435 | 3838 | push(_gvn.transform( new (C, 2) MoveF2INode(pop()))); |
duke@435 | 3839 | break; |
duke@435 | 3840 | |
duke@435 | 3841 | case vmIntrinsics::_intBitsToFloat: |
duke@435 | 3842 | push(_gvn.transform( new (C, 2) MoveI2FNode(pop()))); |
duke@435 | 3843 | break; |
duke@435 | 3844 | |
duke@435 | 3845 | case vmIntrinsics::_doubleToRawLongBits: |
duke@435 | 3846 | push_pair(_gvn.transform( new (C, 2) MoveD2LNode(pop_pair()))); |
duke@435 | 3847 | break; |
duke@435 | 3848 | |
duke@435 | 3849 | case vmIntrinsics::_longBitsToDouble: |
duke@435 | 3850 | push_pair(_gvn.transform( new (C, 2) MoveL2DNode(pop_pair()))); |
duke@435 | 3851 | break; |
duke@435 | 3852 | |
duke@435 | 3853 | case vmIntrinsics::_doubleToLongBits: { |
duke@435 | 3854 | Node* value = pop_pair(); |
duke@435 | 3855 | |
duke@435 | 3856 | // two paths (plus control) merge in a wood |
duke@435 | 3857 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 3858 | Node *phi = new (C, 3) PhiNode(r, TypeLong::LONG); |
duke@435 | 3859 | |
duke@435 | 3860 | Node *cmpisnan = _gvn.transform( new (C, 3) CmpDNode(value, value)); |
duke@435 | 3861 | // Build the boolean node |
duke@435 | 3862 | Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) ); |
duke@435 | 3863 | |
duke@435 | 3864 | // Branch either way. |
duke@435 | 3865 | // NaN case is less traveled, which makes all the difference. |
duke@435 | 3866 | IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 3867 | Node *opt_isnan = _gvn.transform(ifisnan); |
duke@435 | 3868 | assert( opt_isnan->is_If(), "Expect an IfNode"); |
duke@435 | 3869 | IfNode *opt_ifisnan = (IfNode*)opt_isnan; |
duke@435 | 3870 | Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) ); |
duke@435 | 3871 | |
duke@435 | 3872 | set_control(iftrue); |
duke@435 | 3873 | |
duke@435 | 3874 | static const jlong nan_bits = CONST64(0x7ff8000000000000); |
duke@435 | 3875 | Node *slow_result = longcon(nan_bits); // return NaN |
duke@435 | 3876 | phi->init_req(1, _gvn.transform( slow_result )); |
duke@435 | 3877 | r->init_req(1, iftrue); |
duke@435 | 3878 | |
duke@435 | 3879 | // Else fall through |
duke@435 | 3880 | Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) ); |
duke@435 | 3881 | set_control(iffalse); |
duke@435 | 3882 | |
duke@435 | 3883 | phi->init_req(2, _gvn.transform( new (C, 2) MoveD2LNode(value))); |
duke@435 | 3884 | r->init_req(2, iffalse); |
duke@435 | 3885 | |
duke@435 | 3886 | // Post merge |
duke@435 | 3887 | set_control(_gvn.transform(r)); |
duke@435 | 3888 | record_for_igvn(r); |
duke@435 | 3889 | |
duke@435 | 3890 | Node* result = _gvn.transform(phi); |
duke@435 | 3891 | assert(result->bottom_type()->isa_long(), "must be"); |
duke@435 | 3892 | push_pair(result); |
duke@435 | 3893 | |
duke@435 | 3894 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3895 | |
duke@435 | 3896 | break; |
duke@435 | 3897 | } |
duke@435 | 3898 | |
duke@435 | 3899 | case vmIntrinsics::_floatToIntBits: { |
duke@435 | 3900 | Node* value = pop(); |
duke@435 | 3901 | |
duke@435 | 3902 | // two paths (plus control) merge in a wood |
duke@435 | 3903 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 3904 | Node *phi = new (C, 3) PhiNode(r, TypeInt::INT); |
duke@435 | 3905 | |
duke@435 | 3906 | Node *cmpisnan = _gvn.transform( new (C, 3) CmpFNode(value, value)); |
duke@435 | 3907 | // Build the boolean node |
duke@435 | 3908 | Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) ); |
duke@435 | 3909 | |
duke@435 | 3910 | // Branch either way. |
duke@435 | 3911 | // NaN case is less traveled, which makes all the difference. |
duke@435 | 3912 | IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 3913 | Node *opt_isnan = _gvn.transform(ifisnan); |
duke@435 | 3914 | assert( opt_isnan->is_If(), "Expect an IfNode"); |
duke@435 | 3915 | IfNode *opt_ifisnan = (IfNode*)opt_isnan; |
duke@435 | 3916 | Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) ); |
duke@435 | 3917 | |
duke@435 | 3918 | set_control(iftrue); |
duke@435 | 3919 | |
duke@435 | 3920 | static const jint nan_bits = 0x7fc00000; |
duke@435 | 3921 | Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN |
duke@435 | 3922 | phi->init_req(1, _gvn.transform( slow_result )); |
duke@435 | 3923 | r->init_req(1, iftrue); |
duke@435 | 3924 | |
duke@435 | 3925 | // Else fall through |
duke@435 | 3926 | Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) ); |
duke@435 | 3927 | set_control(iffalse); |
duke@435 | 3928 | |
duke@435 | 3929 | phi->init_req(2, _gvn.transform( new (C, 2) MoveF2INode(value))); |
duke@435 | 3930 | r->init_req(2, iffalse); |
duke@435 | 3931 | |
duke@435 | 3932 | // Post merge |
duke@435 | 3933 | set_control(_gvn.transform(r)); |
duke@435 | 3934 | record_for_igvn(r); |
duke@435 | 3935 | |
duke@435 | 3936 | Node* result = _gvn.transform(phi); |
duke@435 | 3937 | assert(result->bottom_type()->isa_int(), "must be"); |
duke@435 | 3938 | push(result); |
duke@435 | 3939 | |
duke@435 | 3940 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3941 | |
duke@435 | 3942 | break; |
duke@435 | 3943 | } |
duke@435 | 3944 | |
duke@435 | 3945 | default: |
duke@435 | 3946 | ShouldNotReachHere(); |
duke@435 | 3947 | } |
duke@435 | 3948 | |
duke@435 | 3949 | return true; |
duke@435 | 3950 | } |
duke@435 | 3951 | |
duke@435 | 3952 | #ifdef _LP64 |
duke@435 | 3953 | #define XTOP ,top() /*additional argument*/ |
duke@435 | 3954 | #else //_LP64 |
duke@435 | 3955 | #define XTOP /*no additional argument*/ |
duke@435 | 3956 | #endif //_LP64 |
duke@435 | 3957 | |
duke@435 | 3958 | //----------------------inline_unsafe_copyMemory------------------------- |
duke@435 | 3959 | bool LibraryCallKit::inline_unsafe_copyMemory() { |
duke@435 | 3960 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 3961 | int nargs = 1 + 5 + 3; // 5 args: (src: ptr,off, dst: ptr,off, size) |
duke@435 | 3962 | assert(signature()->size() == nargs-1, "copy has 5 arguments"); |
duke@435 | 3963 | null_check_receiver(callee()); // check then ignore argument(0) |
duke@435 | 3964 | if (stopped()) return true; |
duke@435 | 3965 | |
duke@435 | 3966 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 3967 | |
duke@435 | 3968 | Node* src_ptr = argument(1); |
duke@435 | 3969 | Node* src_off = ConvL2X(argument(2)); |
duke@435 | 3970 | assert(argument(3)->is_top(), "2nd half of long"); |
duke@435 | 3971 | Node* dst_ptr = argument(4); |
duke@435 | 3972 | Node* dst_off = ConvL2X(argument(5)); |
duke@435 | 3973 | assert(argument(6)->is_top(), "2nd half of long"); |
duke@435 | 3974 | Node* size = ConvL2X(argument(7)); |
duke@435 | 3975 | assert(argument(8)->is_top(), "2nd half of long"); |
duke@435 | 3976 | |
duke@435 | 3977 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 3978 | "fieldOffset must be byte-scaled"); |
duke@435 | 3979 | |
duke@435 | 3980 | Node* src = make_unsafe_address(src_ptr, src_off); |
duke@435 | 3981 | Node* dst = make_unsafe_address(dst_ptr, dst_off); |
duke@435 | 3982 | |
duke@435 | 3983 | // Conservatively insert a memory barrier on all memory slices. |
duke@435 | 3984 | // Do not let writes of the copy source or destination float below the copy. |
duke@435 | 3985 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 3986 | |
duke@435 | 3987 | // Call it. Note that the length argument is not scaled. |
duke@435 | 3988 | make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 3989 | OptoRuntime::fast_arraycopy_Type(), |
duke@435 | 3990 | StubRoutines::unsafe_arraycopy(), |
duke@435 | 3991 | "unsafe_arraycopy", |
duke@435 | 3992 | TypeRawPtr::BOTTOM, |
duke@435 | 3993 | src, dst, size XTOP); |
duke@435 | 3994 | |
duke@435 | 3995 | // Do not let reads of the copy destination float above the copy. |
duke@435 | 3996 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 3997 | |
duke@435 | 3998 | return true; |
duke@435 | 3999 | } |
duke@435 | 4000 | |
kvn@1268 | 4001 | //------------------------clone_coping----------------------------------- |
kvn@1268 | 4002 | // Helper function for inline_native_clone. |
kvn@1268 | 4003 | void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { |
kvn@1268 | 4004 | assert(obj_size != NULL, ""); |
kvn@1268 | 4005 | Node* raw_obj = alloc_obj->in(1); |
kvn@1268 | 4006 | assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); |
kvn@1268 | 4007 | |
kvn@1268 | 4008 | if (ReduceBulkZeroing) { |
kvn@1268 | 4009 | // We will be completely responsible for initializing this object - |
kvn@1268 | 4010 | // mark Initialize node as complete. |
kvn@1268 | 4011 | AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); |
kvn@1268 | 4012 | // The object was just allocated - there should be no any stores! |
kvn@1268 | 4013 | guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); |
kvn@1268 | 4014 | } |
kvn@1268 | 4015 | |
kvn@1268 | 4016 | // Copy the fastest available way. |
kvn@1268 | 4017 | // TODO: generate fields copies for small objects instead. |
kvn@1268 | 4018 | Node* src = obj; |
kvn@1393 | 4019 | Node* dest = alloc_obj; |
kvn@1268 | 4020 | Node* size = _gvn.transform(obj_size); |
kvn@1268 | 4021 | |
kvn@1268 | 4022 | // Exclude the header but include array length to copy by 8 bytes words. |
kvn@1268 | 4023 | // Can't use base_offset_in_bytes(bt) since basic type is unknown. |
kvn@1268 | 4024 | int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : |
kvn@1268 | 4025 | instanceOopDesc::base_offset_in_bytes(); |
kvn@1268 | 4026 | // base_off: |
kvn@1268 | 4027 | // 8 - 32-bit VM |
kvn@1268 | 4028 | // 12 - 64-bit VM, compressed oops |
kvn@1268 | 4029 | // 16 - 64-bit VM, normal oops |
kvn@1268 | 4030 | if (base_off % BytesPerLong != 0) { |
kvn@1268 | 4031 | assert(UseCompressedOops, ""); |
kvn@1268 | 4032 | if (is_array) { |
kvn@1268 | 4033 | // Exclude length to copy by 8 bytes words. |
kvn@1268 | 4034 | base_off += sizeof(int); |
kvn@1268 | 4035 | } else { |
kvn@1268 | 4036 | // Include klass to copy by 8 bytes words. |
kvn@1268 | 4037 | base_off = instanceOopDesc::klass_offset_in_bytes(); |
kvn@1268 | 4038 | } |
kvn@1268 | 4039 | assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); |
kvn@1268 | 4040 | } |
kvn@1268 | 4041 | src = basic_plus_adr(src, base_off); |
kvn@1268 | 4042 | dest = basic_plus_adr(dest, base_off); |
kvn@1268 | 4043 | |
kvn@1268 | 4044 | // Compute the length also, if needed: |
kvn@1268 | 4045 | Node* countx = size; |
kvn@1268 | 4046 | countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); |
kvn@1268 | 4047 | countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); |
kvn@1268 | 4048 | |
kvn@1268 | 4049 | const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
kvn@1268 | 4050 | bool disjoint_bases = true; |
kvn@1268 | 4051 | generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, |
kvn@1268 | 4052 | src, NULL, dest, NULL, countx); |
kvn@1268 | 4053 | |
kvn@1268 | 4054 | // If necessary, emit some card marks afterwards. (Non-arrays only.) |
kvn@1268 | 4055 | if (card_mark) { |
kvn@1268 | 4056 | assert(!is_array, ""); |
kvn@1268 | 4057 | // Put in store barrier for any and all oops we are sticking |
kvn@1268 | 4058 | // into this object. (We could avoid this if we could prove |
kvn@1268 | 4059 | // that the object type contains no oop fields at all.) |
kvn@1268 | 4060 | Node* no_particular_value = NULL; |
kvn@1268 | 4061 | Node* no_particular_field = NULL; |
kvn@1268 | 4062 | int raw_adr_idx = Compile::AliasIdxRaw; |
kvn@1268 | 4063 | post_barrier(control(), |
kvn@1268 | 4064 | memory(raw_adr_type), |
kvn@1393 | 4065 | alloc_obj, |
kvn@1268 | 4066 | no_particular_field, |
kvn@1268 | 4067 | raw_adr_idx, |
kvn@1268 | 4068 | no_particular_value, |
kvn@1268 | 4069 | T_OBJECT, |
kvn@1268 | 4070 | false); |
kvn@1268 | 4071 | } |
kvn@1268 | 4072 | |
kvn@1393 | 4073 | // Do not let reads from the cloned object float above the arraycopy. |
kvn@1393 | 4074 | insert_mem_bar(Op_MemBarCPUOrder); |
kvn@1268 | 4075 | } |
duke@435 | 4076 | |
duke@435 | 4077 | //------------------------inline_native_clone---------------------------- |
duke@435 | 4078 | // Here are the simple edge cases: |
duke@435 | 4079 | // null receiver => normal trap |
duke@435 | 4080 | // virtual and clone was overridden => slow path to out-of-line clone |
duke@435 | 4081 | // not cloneable or finalizer => slow path to out-of-line Object.clone |
duke@435 | 4082 | // |
duke@435 | 4083 | // The general case has two steps, allocation and copying. |
duke@435 | 4084 | // Allocation has two cases, and uses GraphKit::new_instance or new_array. |
duke@435 | 4085 | // |
duke@435 | 4086 | // Copying also has two cases, oop arrays and everything else. |
duke@435 | 4087 | // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy). |
duke@435 | 4088 | // Everything else uses the tight inline loop supplied by CopyArrayNode. |
duke@435 | 4089 | // |
duke@435 | 4090 | // These steps fold up nicely if and when the cloned object's klass |
duke@435 | 4091 | // can be sharply typed as an object array, a type array, or an instance. |
duke@435 | 4092 | // |
duke@435 | 4093 | bool LibraryCallKit::inline_native_clone(bool is_virtual) { |
duke@435 | 4094 | int nargs = 1; |
cfang@1337 | 4095 | PhiNode* result_val; |
duke@435 | 4096 | |
cfang@1335 | 4097 | //set the original stack and the reexecute bit for the interpreter to reexecute |
cfang@1335 | 4098 | //the bytecode that invokes Object.clone if deoptimization happens |
cfang@1335 | 4099 | { PreserveReexecuteState preexecs(this); |
cfang@1337 | 4100 | jvms()->set_should_reexecute(true); |
cfang@1337 | 4101 | |
cfang@1337 | 4102 | //null_check_receiver will adjust _sp (push and pop) |
cfang@1337 | 4103 | Node* obj = null_check_receiver(callee()); |
cfang@1337 | 4104 | if (stopped()) return true; |
cfang@1337 | 4105 | |
cfang@1335 | 4106 | _sp += nargs; |
cfang@1337 | 4107 | |
cfang@1337 | 4108 | Node* obj_klass = load_object_klass(obj); |
cfang@1337 | 4109 | const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr(); |
cfang@1337 | 4110 | const TypeOopPtr* toop = ((tklass != NULL) |
cfang@1337 | 4111 | ? tklass->as_instance_type() |
cfang@1337 | 4112 | : TypeInstPtr::NOTNULL); |
cfang@1337 | 4113 | |
cfang@1337 | 4114 | // Conservatively insert a memory barrier on all memory slices. |
cfang@1337 | 4115 | // Do not let writes into the original float below the clone. |
cfang@1337 | 4116 | insert_mem_bar(Op_MemBarCPUOrder); |
cfang@1337 | 4117 | |
cfang@1337 | 4118 | // paths into result_reg: |
cfang@1337 | 4119 | enum { |
cfang@1337 | 4120 | _slow_path = 1, // out-of-line call to clone method (virtual or not) |
cfang@1337 | 4121 | _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy |
cfang@1337 | 4122 | _array_path, // plain array allocation, plus arrayof_long_arraycopy |
cfang@1337 | 4123 | _instance_path, // plain instance allocation, plus arrayof_long_arraycopy |
cfang@1337 | 4124 | PATH_LIMIT |
cfang@1337 | 4125 | }; |
cfang@1337 | 4126 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
cfang@1337 | 4127 | result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
cfang@1337 | 4128 | TypeInstPtr::NOTNULL); |
cfang@1337 | 4129 | PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
cfang@1337 | 4130 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
cfang@1337 | 4131 | TypePtr::BOTTOM); |
cfang@1337 | 4132 | record_for_igvn(result_reg); |
cfang@1337 | 4133 | |
cfang@1337 | 4134 | const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
cfang@1337 | 4135 | int raw_adr_idx = Compile::AliasIdxRaw; |
cfang@1337 | 4136 | const bool raw_mem_only = true; |
cfang@1337 | 4137 | |
cfang@1335 | 4138 | |
cfang@1335 | 4139 | Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); |
cfang@1335 | 4140 | if (array_ctl != NULL) { |
cfang@1335 | 4141 | // It's an array. |
cfang@1335 | 4142 | PreserveJVMState pjvms(this); |
cfang@1335 | 4143 | set_control(array_ctl); |
cfang@1335 | 4144 | Node* obj_length = load_array_length(obj); |
cfang@1335 | 4145 | Node* obj_size = NULL; |
cfang@1335 | 4146 | Node* alloc_obj = new_array(obj_klass, obj_length, 0, |
cfang@1335 | 4147 | raw_mem_only, &obj_size); |
cfang@1335 | 4148 | |
cfang@1335 | 4149 | if (!use_ReduceInitialCardMarks()) { |
cfang@1335 | 4150 | // If it is an oop array, it requires very special treatment, |
cfang@1335 | 4151 | // because card marking is required on each card of the array. |
cfang@1335 | 4152 | Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); |
cfang@1335 | 4153 | if (is_obja != NULL) { |
cfang@1335 | 4154 | PreserveJVMState pjvms2(this); |
cfang@1335 | 4155 | set_control(is_obja); |
cfang@1335 | 4156 | // Generate a direct call to the right arraycopy function(s). |
cfang@1335 | 4157 | bool disjoint_bases = true; |
cfang@1335 | 4158 | bool length_never_negative = true; |
cfang@1335 | 4159 | generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
cfang@1335 | 4160 | obj, intcon(0), alloc_obj, intcon(0), |
cfang@1335 | 4161 | obj_length, |
cfang@1335 | 4162 | disjoint_bases, length_never_negative); |
cfang@1335 | 4163 | result_reg->init_req(_objArray_path, control()); |
cfang@1335 | 4164 | result_val->init_req(_objArray_path, alloc_obj); |
cfang@1335 | 4165 | result_i_o ->set_req(_objArray_path, i_o()); |
cfang@1335 | 4166 | result_mem ->set_req(_objArray_path, reset_memory()); |
cfang@1335 | 4167 | } |
cfang@1335 | 4168 | } |
cfang@1335 | 4169 | // Otherwise, there are no card marks to worry about. |
ysr@1462 | 4170 | // (We can dispense with card marks if we know the allocation |
ysr@1462 | 4171 | // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks |
ysr@1462 | 4172 | // causes the non-eden paths to take compensating steps to |
ysr@1462 | 4173 | // simulate a fresh allocation, so that no further |
ysr@1462 | 4174 | // card marks are required in compiled code to initialize |
ysr@1462 | 4175 | // the object.) |
cfang@1335 | 4176 | |
cfang@1335 | 4177 | if (!stopped()) { |
cfang@1335 | 4178 | copy_to_clone(obj, alloc_obj, obj_size, true, false); |
cfang@1335 | 4179 | |
cfang@1335 | 4180 | // Present the results of the copy. |
cfang@1335 | 4181 | result_reg->init_req(_array_path, control()); |
cfang@1335 | 4182 | result_val->init_req(_array_path, alloc_obj); |
cfang@1335 | 4183 | result_i_o ->set_req(_array_path, i_o()); |
cfang@1335 | 4184 | result_mem ->set_req(_array_path, reset_memory()); |
duke@435 | 4185 | } |
duke@435 | 4186 | } |
cfang@1335 | 4187 | |
cfang@1335 | 4188 | // We only go to the instance fast case code if we pass a number of guards. |
cfang@1335 | 4189 | // The paths which do not pass are accumulated in the slow_region. |
cfang@1335 | 4190 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
cfang@1335 | 4191 | record_for_igvn(slow_region); |
kvn@1268 | 4192 | if (!stopped()) { |
cfang@1335 | 4193 | // It's an instance (we did array above). Make the slow-path tests. |
cfang@1335 | 4194 | // If this is a virtual call, we generate a funny guard. We grab |
cfang@1335 | 4195 | // the vtable entry corresponding to clone() from the target object. |
cfang@1335 | 4196 | // If the target method which we are calling happens to be the |
cfang@1335 | 4197 | // Object clone() method, we pass the guard. We do not need this |
cfang@1335 | 4198 | // guard for non-virtual calls; the caller is known to be the native |
cfang@1335 | 4199 | // Object clone(). |
cfang@1335 | 4200 | if (is_virtual) { |
cfang@1335 | 4201 | generate_virtual_guard(obj_klass, slow_region); |
cfang@1335 | 4202 | } |
cfang@1335 | 4203 | |
cfang@1335 | 4204 | // The object must be cloneable and must not have a finalizer. |
cfang@1335 | 4205 | // Both of these conditions may be checked in a single test. |
cfang@1335 | 4206 | // We could optimize the cloneable test further, but we don't care. |
cfang@1335 | 4207 | generate_access_flags_guard(obj_klass, |
cfang@1335 | 4208 | // Test both conditions: |
cfang@1335 | 4209 | JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER, |
cfang@1335 | 4210 | // Must be cloneable but not finalizer: |
cfang@1335 | 4211 | JVM_ACC_IS_CLONEABLE, |
cfang@1335 | 4212 | slow_region); |
kvn@1268 | 4213 | } |
cfang@1335 | 4214 | |
cfang@1335 | 4215 | if (!stopped()) { |
cfang@1335 | 4216 | // It's an instance, and it passed the slow-path tests. |
cfang@1335 | 4217 | PreserveJVMState pjvms(this); |
cfang@1335 | 4218 | Node* obj_size = NULL; |
cfang@1335 | 4219 | Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); |
cfang@1335 | 4220 | |
cfang@1335 | 4221 | copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks()); |
cfang@1335 | 4222 | |
cfang@1335 | 4223 | // Present the results of the slow call. |
cfang@1335 | 4224 | result_reg->init_req(_instance_path, control()); |
cfang@1335 | 4225 | result_val->init_req(_instance_path, alloc_obj); |
cfang@1335 | 4226 | result_i_o ->set_req(_instance_path, i_o()); |
cfang@1335 | 4227 | result_mem ->set_req(_instance_path, reset_memory()); |
duke@435 | 4228 | } |
duke@435 | 4229 | |
cfang@1335 | 4230 | // Generate code for the slow case. We make a call to clone(). |
cfang@1335 | 4231 | set_control(_gvn.transform(slow_region)); |
cfang@1335 | 4232 | if (!stopped()) { |
cfang@1335 | 4233 | PreserveJVMState pjvms(this); |
cfang@1335 | 4234 | CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual); |
cfang@1335 | 4235 | Node* slow_result = set_results_for_java_call(slow_call); |
cfang@1335 | 4236 | // this->control() comes from set_results_for_java_call |
cfang@1335 | 4237 | result_reg->init_req(_slow_path, control()); |
cfang@1335 | 4238 | result_val->init_req(_slow_path, slow_result); |
cfang@1335 | 4239 | result_i_o ->set_req(_slow_path, i_o()); |
cfang@1335 | 4240 | result_mem ->set_req(_slow_path, reset_memory()); |
cfang@1335 | 4241 | } |
cfang@1337 | 4242 | |
cfang@1337 | 4243 | // Return the combined state. |
cfang@1337 | 4244 | set_control( _gvn.transform(result_reg) ); |
cfang@1337 | 4245 | set_i_o( _gvn.transform(result_i_o) ); |
cfang@1337 | 4246 | set_all_memory( _gvn.transform(result_mem) ); |
cfang@1335 | 4247 | } //original reexecute and sp are set back here |
duke@435 | 4248 | |
kvn@1268 | 4249 | push(_gvn.transform(result_val)); |
duke@435 | 4250 | |
duke@435 | 4251 | return true; |
duke@435 | 4252 | } |
duke@435 | 4253 | |
duke@435 | 4254 | |
duke@435 | 4255 | // constants for computing the copy function |
duke@435 | 4256 | enum { |
duke@435 | 4257 | COPYFUNC_UNALIGNED = 0, |
duke@435 | 4258 | COPYFUNC_ALIGNED = 1, // src, dest aligned to HeapWordSize |
duke@435 | 4259 | COPYFUNC_CONJOINT = 0, |
duke@435 | 4260 | COPYFUNC_DISJOINT = 2 // src != dest, or transfer can descend |
duke@435 | 4261 | }; |
duke@435 | 4262 | |
duke@435 | 4263 | // Note: The condition "disjoint" applies also for overlapping copies |
duke@435 | 4264 | // where an descending copy is permitted (i.e., dest_offset <= src_offset). |
duke@435 | 4265 | static address |
duke@435 | 4266 | select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) { |
duke@435 | 4267 | int selector = |
duke@435 | 4268 | (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) + |
duke@435 | 4269 | (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT); |
duke@435 | 4270 | |
duke@435 | 4271 | #define RETURN_STUB(xxx_arraycopy) { \ |
duke@435 | 4272 | name = #xxx_arraycopy; \ |
duke@435 | 4273 | return StubRoutines::xxx_arraycopy(); } |
duke@435 | 4274 | |
duke@435 | 4275 | switch (t) { |
duke@435 | 4276 | case T_BYTE: |
duke@435 | 4277 | case T_BOOLEAN: |
duke@435 | 4278 | switch (selector) { |
duke@435 | 4279 | case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy); |
duke@435 | 4280 | case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_arraycopy); |
duke@435 | 4281 | case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_disjoint_arraycopy); |
duke@435 | 4282 | case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_disjoint_arraycopy); |
duke@435 | 4283 | } |
duke@435 | 4284 | case T_CHAR: |
duke@435 | 4285 | case T_SHORT: |
duke@435 | 4286 | switch (selector) { |
duke@435 | 4287 | case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_arraycopy); |
duke@435 | 4288 | case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_arraycopy); |
duke@435 | 4289 | case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_disjoint_arraycopy); |
duke@435 | 4290 | case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_disjoint_arraycopy); |
duke@435 | 4291 | } |
duke@435 | 4292 | case T_INT: |
duke@435 | 4293 | case T_FLOAT: |
duke@435 | 4294 | switch (selector) { |
duke@435 | 4295 | case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_arraycopy); |
duke@435 | 4296 | case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_arraycopy); |
duke@435 | 4297 | case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_disjoint_arraycopy); |
duke@435 | 4298 | case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_disjoint_arraycopy); |
duke@435 | 4299 | } |
duke@435 | 4300 | case T_DOUBLE: |
duke@435 | 4301 | case T_LONG: |
duke@435 | 4302 | switch (selector) { |
duke@435 | 4303 | case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_arraycopy); |
duke@435 | 4304 | case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_arraycopy); |
duke@435 | 4305 | case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_disjoint_arraycopy); |
duke@435 | 4306 | case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy); |
duke@435 | 4307 | } |
duke@435 | 4308 | case T_ARRAY: |
duke@435 | 4309 | case T_OBJECT: |
duke@435 | 4310 | switch (selector) { |
duke@435 | 4311 | case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy); |
duke@435 | 4312 | case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy); |
duke@435 | 4313 | case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy); |
duke@435 | 4314 | case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy); |
duke@435 | 4315 | } |
duke@435 | 4316 | default: |
duke@435 | 4317 | ShouldNotReachHere(); |
duke@435 | 4318 | return NULL; |
duke@435 | 4319 | } |
duke@435 | 4320 | |
duke@435 | 4321 | #undef RETURN_STUB |
duke@435 | 4322 | } |
duke@435 | 4323 | |
duke@435 | 4324 | //------------------------------basictype2arraycopy---------------------------- |
duke@435 | 4325 | address LibraryCallKit::basictype2arraycopy(BasicType t, |
duke@435 | 4326 | Node* src_offset, |
duke@435 | 4327 | Node* dest_offset, |
duke@435 | 4328 | bool disjoint_bases, |
duke@435 | 4329 | const char* &name) { |
duke@435 | 4330 | const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);; |
duke@435 | 4331 | const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);; |
duke@435 | 4332 | |
duke@435 | 4333 | bool aligned = false; |
duke@435 | 4334 | bool disjoint = disjoint_bases; |
duke@435 | 4335 | |
duke@435 | 4336 | // if the offsets are the same, we can treat the memory regions as |
duke@435 | 4337 | // disjoint, because either the memory regions are in different arrays, |
duke@435 | 4338 | // or they are identical (which we can treat as disjoint.) We can also |
duke@435 | 4339 | // treat a copy with a destination index less that the source index |
duke@435 | 4340 | // as disjoint since a low->high copy will work correctly in this case. |
duke@435 | 4341 | if (src_offset_inttype != NULL && src_offset_inttype->is_con() && |
duke@435 | 4342 | dest_offset_inttype != NULL && dest_offset_inttype->is_con()) { |
duke@435 | 4343 | // both indices are constants |
duke@435 | 4344 | int s_offs = src_offset_inttype->get_con(); |
duke@435 | 4345 | int d_offs = dest_offset_inttype->get_con(); |
kvn@464 | 4346 | int element_size = type2aelembytes(t); |
duke@435 | 4347 | aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) && |
duke@435 | 4348 | ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0); |
duke@435 | 4349 | if (s_offs >= d_offs) disjoint = true; |
duke@435 | 4350 | } else if (src_offset == dest_offset && src_offset != NULL) { |
duke@435 | 4351 | // This can occur if the offsets are identical non-constants. |
duke@435 | 4352 | disjoint = true; |
duke@435 | 4353 | } |
duke@435 | 4354 | |
duke@435 | 4355 | return select_arraycopy_function(t, aligned, disjoint, name); |
duke@435 | 4356 | } |
duke@435 | 4357 | |
duke@435 | 4358 | |
duke@435 | 4359 | //------------------------------inline_arraycopy----------------------- |
duke@435 | 4360 | bool LibraryCallKit::inline_arraycopy() { |
duke@435 | 4361 | // Restore the stack and pop off the arguments. |
duke@435 | 4362 | int nargs = 5; // 2 oops, 3 ints, no size_t or long |
duke@435 | 4363 | assert(callee()->signature()->size() == nargs, "copy has 5 arguments"); |
duke@435 | 4364 | |
duke@435 | 4365 | Node *src = argument(0); |
duke@435 | 4366 | Node *src_offset = argument(1); |
duke@435 | 4367 | Node *dest = argument(2); |
duke@435 | 4368 | Node *dest_offset = argument(3); |
duke@435 | 4369 | Node *length = argument(4); |
duke@435 | 4370 | |
duke@435 | 4371 | // Compile time checks. If any of these checks cannot be verified at compile time, |
duke@435 | 4372 | // we do not make a fast path for this call. Instead, we let the call remain as it |
duke@435 | 4373 | // is. The checks we choose to mandate at compile time are: |
duke@435 | 4374 | // |
duke@435 | 4375 | // (1) src and dest are arrays. |
duke@435 | 4376 | const Type* src_type = src->Value(&_gvn); |
duke@435 | 4377 | const Type* dest_type = dest->Value(&_gvn); |
duke@435 | 4378 | const TypeAryPtr* top_src = src_type->isa_aryptr(); |
duke@435 | 4379 | const TypeAryPtr* top_dest = dest_type->isa_aryptr(); |
duke@435 | 4380 | if (top_src == NULL || top_src->klass() == NULL || |
duke@435 | 4381 | top_dest == NULL || top_dest->klass() == NULL) { |
duke@435 | 4382 | // Conservatively insert a memory barrier on all memory slices. |
duke@435 | 4383 | // Do not let writes into the source float below the arraycopy. |
duke@435 | 4384 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4385 | |
duke@435 | 4386 | // Call StubRoutines::generic_arraycopy stub. |
duke@435 | 4387 | generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, |
kvn@1268 | 4388 | src, src_offset, dest, dest_offset, length); |
duke@435 | 4389 | |
duke@435 | 4390 | // Do not let reads from the destination float above the arraycopy. |
duke@435 | 4391 | // Since we cannot type the arrays, we don't know which slices |
duke@435 | 4392 | // might be affected. We could restrict this barrier only to those |
duke@435 | 4393 | // memory slices which pertain to array elements--but don't bother. |
duke@435 | 4394 | if (!InsertMemBarAfterArraycopy) |
duke@435 | 4395 | // (If InsertMemBarAfterArraycopy, there is already one in place.) |
duke@435 | 4396 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4397 | return true; |
duke@435 | 4398 | } |
duke@435 | 4399 | |
duke@435 | 4400 | // (2) src and dest arrays must have elements of the same BasicType |
duke@435 | 4401 | // Figure out the size and type of the elements we will be copying. |
duke@435 | 4402 | BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); |
duke@435 | 4403 | BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type(); |
duke@435 | 4404 | if (src_elem == T_ARRAY) src_elem = T_OBJECT; |
duke@435 | 4405 | if (dest_elem == T_ARRAY) dest_elem = T_OBJECT; |
duke@435 | 4406 | |
duke@435 | 4407 | if (src_elem != dest_elem || dest_elem == T_VOID) { |
duke@435 | 4408 | // The component types are not the same or are not recognized. Punt. |
duke@435 | 4409 | // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
duke@435 | 4410 | generate_slow_arraycopy(TypePtr::BOTTOM, |
kvn@1268 | 4411 | src, src_offset, dest, dest_offset, length); |
duke@435 | 4412 | return true; |
duke@435 | 4413 | } |
duke@435 | 4414 | |
duke@435 | 4415 | //--------------------------------------------------------------------------- |
duke@435 | 4416 | // We will make a fast path for this call to arraycopy. |
duke@435 | 4417 | |
duke@435 | 4418 | // We have the following tests left to perform: |
duke@435 | 4419 | // |
duke@435 | 4420 | // (3) src and dest must not be null. |
duke@435 | 4421 | // (4) src_offset must not be negative. |
duke@435 | 4422 | // (5) dest_offset must not be negative. |
duke@435 | 4423 | // (6) length must not be negative. |
duke@435 | 4424 | // (7) src_offset + length must not exceed length of src. |
duke@435 | 4425 | // (8) dest_offset + length must not exceed length of dest. |
duke@435 | 4426 | // (9) each element of an oop array must be assignable |
duke@435 | 4427 | |
duke@435 | 4428 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 4429 | record_for_igvn(slow_region); |
duke@435 | 4430 | |
duke@435 | 4431 | // (3) operands must not be null |
duke@435 | 4432 | // We currently perform our null checks with the do_null_check routine. |
duke@435 | 4433 | // This means that the null exceptions will be reported in the caller |
duke@435 | 4434 | // rather than (correctly) reported inside of the native arraycopy call. |
duke@435 | 4435 | // This should be corrected, given time. We do our null check with the |
duke@435 | 4436 | // stack pointer restored. |
duke@435 | 4437 | _sp += nargs; |
duke@435 | 4438 | src = do_null_check(src, T_ARRAY); |
duke@435 | 4439 | dest = do_null_check(dest, T_ARRAY); |
duke@435 | 4440 | _sp -= nargs; |
duke@435 | 4441 | |
duke@435 | 4442 | // (4) src_offset must not be negative. |
duke@435 | 4443 | generate_negative_guard(src_offset, slow_region); |
duke@435 | 4444 | |
duke@435 | 4445 | // (5) dest_offset must not be negative. |
duke@435 | 4446 | generate_negative_guard(dest_offset, slow_region); |
duke@435 | 4447 | |
duke@435 | 4448 | // (6) length must not be negative (moved to generate_arraycopy()). |
duke@435 | 4449 | // generate_negative_guard(length, slow_region); |
duke@435 | 4450 | |
duke@435 | 4451 | // (7) src_offset + length must not exceed length of src. |
duke@435 | 4452 | generate_limit_guard(src_offset, length, |
duke@435 | 4453 | load_array_length(src), |
duke@435 | 4454 | slow_region); |
duke@435 | 4455 | |
duke@435 | 4456 | // (8) dest_offset + length must not exceed length of dest. |
duke@435 | 4457 | generate_limit_guard(dest_offset, length, |
duke@435 | 4458 | load_array_length(dest), |
duke@435 | 4459 | slow_region); |
duke@435 | 4460 | |
duke@435 | 4461 | // (9) each element of an oop array must be assignable |
duke@435 | 4462 | // The generate_arraycopy subroutine checks this. |
duke@435 | 4463 | |
duke@435 | 4464 | // This is where the memory effects are placed: |
duke@435 | 4465 | const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); |
duke@435 | 4466 | generate_arraycopy(adr_type, dest_elem, |
duke@435 | 4467 | src, src_offset, dest, dest_offset, length, |
kvn@1268 | 4468 | false, false, slow_region); |
duke@435 | 4469 | |
duke@435 | 4470 | return true; |
duke@435 | 4471 | } |
duke@435 | 4472 | |
duke@435 | 4473 | //-----------------------------generate_arraycopy---------------------- |
duke@435 | 4474 | // Generate an optimized call to arraycopy. |
duke@435 | 4475 | // Caller must guard against non-arrays. |
duke@435 | 4476 | // Caller must determine a common array basic-type for both arrays. |
duke@435 | 4477 | // Caller must validate offsets against array bounds. |
duke@435 | 4478 | // The slow_region has already collected guard failure paths |
duke@435 | 4479 | // (such as out of bounds length or non-conformable array types). |
duke@435 | 4480 | // The generated code has this shape, in general: |
duke@435 | 4481 | // |
duke@435 | 4482 | // if (length == 0) return // via zero_path |
duke@435 | 4483 | // slowval = -1 |
duke@435 | 4484 | // if (types unknown) { |
duke@435 | 4485 | // slowval = call generic copy loop |
duke@435 | 4486 | // if (slowval == 0) return // via checked_path |
duke@435 | 4487 | // } else if (indexes in bounds) { |
duke@435 | 4488 | // if ((is object array) && !(array type check)) { |
duke@435 | 4489 | // slowval = call checked copy loop |
duke@435 | 4490 | // if (slowval == 0) return // via checked_path |
duke@435 | 4491 | // } else { |
duke@435 | 4492 | // call bulk copy loop |
duke@435 | 4493 | // return // via fast_path |
duke@435 | 4494 | // } |
duke@435 | 4495 | // } |
duke@435 | 4496 | // // adjust params for remaining work: |
duke@435 | 4497 | // if (slowval != -1) { |
duke@435 | 4498 | // n = -1^slowval; src_offset += n; dest_offset += n; length -= n |
duke@435 | 4499 | // } |
duke@435 | 4500 | // slow_region: |
duke@435 | 4501 | // call slow arraycopy(src, src_offset, dest, dest_offset, length) |
duke@435 | 4502 | // return // via slow_call_path |
duke@435 | 4503 | // |
duke@435 | 4504 | // This routine is used from several intrinsics: System.arraycopy, |
duke@435 | 4505 | // Object.clone (the array subcase), and Arrays.copyOf[Range]. |
duke@435 | 4506 | // |
duke@435 | 4507 | void |
duke@435 | 4508 | LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, |
duke@435 | 4509 | BasicType basic_elem_type, |
duke@435 | 4510 | Node* src, Node* src_offset, |
duke@435 | 4511 | Node* dest, Node* dest_offset, |
duke@435 | 4512 | Node* copy_length, |
duke@435 | 4513 | bool disjoint_bases, |
duke@435 | 4514 | bool length_never_negative, |
duke@435 | 4515 | RegionNode* slow_region) { |
duke@435 | 4516 | |
duke@435 | 4517 | if (slow_region == NULL) { |
duke@435 | 4518 | slow_region = new(C,1) RegionNode(1); |
duke@435 | 4519 | record_for_igvn(slow_region); |
duke@435 | 4520 | } |
duke@435 | 4521 | |
duke@435 | 4522 | Node* original_dest = dest; |
duke@435 | 4523 | AllocateArrayNode* alloc = NULL; // used for zeroing, if needed |
duke@435 | 4524 | bool must_clear_dest = false; |
duke@435 | 4525 | |
duke@435 | 4526 | // See if this is the initialization of a newly-allocated array. |
duke@435 | 4527 | // If so, we will take responsibility here for initializing it to zero. |
duke@435 | 4528 | // (Note: Because tightly_coupled_allocation performs checks on the |
duke@435 | 4529 | // out-edges of the dest, we need to avoid making derived pointers |
duke@435 | 4530 | // from it until we have checked its uses.) |
duke@435 | 4531 | if (ReduceBulkZeroing |
duke@435 | 4532 | && !ZeroTLAB // pointless if already zeroed |
duke@435 | 4533 | && basic_elem_type != T_CONFLICT // avoid corner case |
duke@435 | 4534 | && !_gvn.eqv_uncast(src, dest) |
duke@435 | 4535 | && ((alloc = tightly_coupled_allocation(dest, slow_region)) |
duke@435 | 4536 | != NULL) |
kvn@469 | 4537 | && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 |
duke@435 | 4538 | && alloc->maybe_set_complete(&_gvn)) { |
duke@435 | 4539 | // "You break it, you buy it." |
duke@435 | 4540 | InitializeNode* init = alloc->initialization(); |
duke@435 | 4541 | assert(init->is_complete(), "we just did this"); |
kvn@1268 | 4542 | assert(dest->is_CheckCastPP(), "sanity"); |
duke@435 | 4543 | assert(dest->in(0)->in(0) == init, "dest pinned"); |
duke@435 | 4544 | adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
duke@435 | 4545 | // From this point on, every exit path is responsible for |
duke@435 | 4546 | // initializing any non-copied parts of the object to zero. |
duke@435 | 4547 | must_clear_dest = true; |
duke@435 | 4548 | } else { |
duke@435 | 4549 | // No zeroing elimination here. |
duke@435 | 4550 | alloc = NULL; |
duke@435 | 4551 | //original_dest = dest; |
duke@435 | 4552 | //must_clear_dest = false; |
duke@435 | 4553 | } |
duke@435 | 4554 | |
duke@435 | 4555 | // Results are placed here: |
duke@435 | 4556 | enum { fast_path = 1, // normal void-returning assembly stub |
duke@435 | 4557 | checked_path = 2, // special assembly stub with cleanup |
duke@435 | 4558 | slow_call_path = 3, // something went wrong; call the VM |
duke@435 | 4559 | zero_path = 4, // bypass when length of copy is zero |
duke@435 | 4560 | bcopy_path = 5, // copy primitive array by 64-bit blocks |
duke@435 | 4561 | PATH_LIMIT = 6 |
duke@435 | 4562 | }; |
duke@435 | 4563 | RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 4564 | PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO); |
duke@435 | 4565 | PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type); |
duke@435 | 4566 | record_for_igvn(result_region); |
duke@435 | 4567 | _gvn.set_type_bottom(result_i_o); |
duke@435 | 4568 | _gvn.set_type_bottom(result_memory); |
duke@435 | 4569 | assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice"); |
duke@435 | 4570 | |
duke@435 | 4571 | // The slow_control path: |
duke@435 | 4572 | Node* slow_control; |
duke@435 | 4573 | Node* slow_i_o = i_o(); |
duke@435 | 4574 | Node* slow_mem = memory(adr_type); |
duke@435 | 4575 | debug_only(slow_control = (Node*) badAddress); |
duke@435 | 4576 | |
duke@435 | 4577 | // Checked control path: |
duke@435 | 4578 | Node* checked_control = top(); |
duke@435 | 4579 | Node* checked_mem = NULL; |
duke@435 | 4580 | Node* checked_i_o = NULL; |
duke@435 | 4581 | Node* checked_value = NULL; |
duke@435 | 4582 | |
duke@435 | 4583 | if (basic_elem_type == T_CONFLICT) { |
duke@435 | 4584 | assert(!must_clear_dest, ""); |
duke@435 | 4585 | Node* cv = generate_generic_arraycopy(adr_type, |
duke@435 | 4586 | src, src_offset, dest, dest_offset, |
kvn@1268 | 4587 | copy_length); |
duke@435 | 4588 | if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
duke@435 | 4589 | checked_control = control(); |
duke@435 | 4590 | checked_i_o = i_o(); |
duke@435 | 4591 | checked_mem = memory(adr_type); |
duke@435 | 4592 | checked_value = cv; |
duke@435 | 4593 | set_control(top()); // no fast path |
duke@435 | 4594 | } |
duke@435 | 4595 | |
duke@435 | 4596 | Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative); |
duke@435 | 4597 | if (not_pos != NULL) { |
duke@435 | 4598 | PreserveJVMState pjvms(this); |
duke@435 | 4599 | set_control(not_pos); |
duke@435 | 4600 | |
duke@435 | 4601 | // (6) length must not be negative. |
duke@435 | 4602 | if (!length_never_negative) { |
duke@435 | 4603 | generate_negative_guard(copy_length, slow_region); |
duke@435 | 4604 | } |
duke@435 | 4605 | |
kvn@1271 | 4606 | // copy_length is 0. |
duke@435 | 4607 | if (!stopped() && must_clear_dest) { |
duke@435 | 4608 | Node* dest_length = alloc->in(AllocateNode::ALength); |
duke@435 | 4609 | if (_gvn.eqv_uncast(copy_length, dest_length) |
duke@435 | 4610 | || _gvn.find_int_con(dest_length, 1) <= 0) { |
kvn@1271 | 4611 | // There is no zeroing to do. No need for a secondary raw memory barrier. |
duke@435 | 4612 | } else { |
duke@435 | 4613 | // Clear the whole thing since there are no source elements to copy. |
duke@435 | 4614 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4615 | intcon(0), NULL, |
duke@435 | 4616 | alloc->in(AllocateNode::AllocSize)); |
kvn@1271 | 4617 | // Use a secondary InitializeNode as raw memory barrier. |
kvn@1271 | 4618 | // Currently it is needed only on this path since other |
kvn@1271 | 4619 | // paths have stub or runtime calls as raw memory barriers. |
kvn@1271 | 4620 | InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, |
kvn@1271 | 4621 | Compile::AliasIdxRaw, |
kvn@1271 | 4622 | top())->as_Initialize(); |
kvn@1271 | 4623 | init->set_complete(&_gvn); // (there is no corresponding AllocateNode) |
duke@435 | 4624 | } |
duke@435 | 4625 | } |
duke@435 | 4626 | |
duke@435 | 4627 | // Present the results of the fast call. |
duke@435 | 4628 | result_region->init_req(zero_path, control()); |
duke@435 | 4629 | result_i_o ->init_req(zero_path, i_o()); |
duke@435 | 4630 | result_memory->init_req(zero_path, memory(adr_type)); |
duke@435 | 4631 | } |
duke@435 | 4632 | |
duke@435 | 4633 | if (!stopped() && must_clear_dest) { |
duke@435 | 4634 | // We have to initialize the *uncopied* part of the array to zero. |
duke@435 | 4635 | // The copy destination is the slice dest[off..off+len]. The other slices |
duke@435 | 4636 | // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. |
duke@435 | 4637 | Node* dest_size = alloc->in(AllocateNode::AllocSize); |
duke@435 | 4638 | Node* dest_length = alloc->in(AllocateNode::ALength); |
duke@435 | 4639 | Node* dest_tail = _gvn.transform( new(C,3) AddINode(dest_offset, |
duke@435 | 4640 | copy_length) ); |
duke@435 | 4641 | |
duke@435 | 4642 | // If there is a head section that needs zeroing, do it now. |
duke@435 | 4643 | if (find_int_con(dest_offset, -1) != 0) { |
duke@435 | 4644 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4645 | intcon(0), dest_offset, |
duke@435 | 4646 | NULL); |
duke@435 | 4647 | } |
duke@435 | 4648 | |
duke@435 | 4649 | // Next, perform a dynamic check on the tail length. |
duke@435 | 4650 | // It is often zero, and we can win big if we prove this. |
duke@435 | 4651 | // There are two wins: Avoid generating the ClearArray |
duke@435 | 4652 | // with its attendant messy index arithmetic, and upgrade |
duke@435 | 4653 | // the copy to a more hardware-friendly word size of 64 bits. |
duke@435 | 4654 | Node* tail_ctl = NULL; |
duke@435 | 4655 | if (!stopped() && !_gvn.eqv_uncast(dest_tail, dest_length)) { |
duke@435 | 4656 | Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) ); |
duke@435 | 4657 | Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 4658 | tail_ctl = generate_slow_guard(bol_lt, NULL); |
duke@435 | 4659 | assert(tail_ctl != NULL || !stopped(), "must be an outcome"); |
duke@435 | 4660 | } |
duke@435 | 4661 | |
duke@435 | 4662 | // At this point, let's assume there is no tail. |
duke@435 | 4663 | if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) { |
duke@435 | 4664 | // There is no tail. Try an upgrade to a 64-bit copy. |
duke@435 | 4665 | bool didit = false; |
duke@435 | 4666 | { PreserveJVMState pjvms(this); |
duke@435 | 4667 | didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc, |
duke@435 | 4668 | src, src_offset, dest, dest_offset, |
duke@435 | 4669 | dest_size); |
duke@435 | 4670 | if (didit) { |
duke@435 | 4671 | // Present the results of the block-copying fast call. |
duke@435 | 4672 | result_region->init_req(bcopy_path, control()); |
duke@435 | 4673 | result_i_o ->init_req(bcopy_path, i_o()); |
duke@435 | 4674 | result_memory->init_req(bcopy_path, memory(adr_type)); |
duke@435 | 4675 | } |
duke@435 | 4676 | } |
duke@435 | 4677 | if (didit) |
duke@435 | 4678 | set_control(top()); // no regular fast path |
duke@435 | 4679 | } |
duke@435 | 4680 | |
duke@435 | 4681 | // Clear the tail, if any. |
duke@435 | 4682 | if (tail_ctl != NULL) { |
duke@435 | 4683 | Node* notail_ctl = stopped() ? NULL : control(); |
duke@435 | 4684 | set_control(tail_ctl); |
duke@435 | 4685 | if (notail_ctl == NULL) { |
duke@435 | 4686 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4687 | dest_tail, NULL, |
duke@435 | 4688 | dest_size); |
duke@435 | 4689 | } else { |
duke@435 | 4690 | // Make a local merge. |
duke@435 | 4691 | Node* done_ctl = new(C,3) RegionNode(3); |
duke@435 | 4692 | Node* done_mem = new(C,3) PhiNode(done_ctl, Type::MEMORY, adr_type); |
duke@435 | 4693 | done_ctl->init_req(1, notail_ctl); |
duke@435 | 4694 | done_mem->init_req(1, memory(adr_type)); |
duke@435 | 4695 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4696 | dest_tail, NULL, |
duke@435 | 4697 | dest_size); |
duke@435 | 4698 | done_ctl->init_req(2, control()); |
duke@435 | 4699 | done_mem->init_req(2, memory(adr_type)); |
duke@435 | 4700 | set_control( _gvn.transform(done_ctl) ); |
duke@435 | 4701 | set_memory( _gvn.transform(done_mem), adr_type ); |
duke@435 | 4702 | } |
duke@435 | 4703 | } |
duke@435 | 4704 | } |
duke@435 | 4705 | |
duke@435 | 4706 | BasicType copy_type = basic_elem_type; |
duke@435 | 4707 | assert(basic_elem_type != T_ARRAY, "caller must fix this"); |
duke@435 | 4708 | if (!stopped() && copy_type == T_OBJECT) { |
duke@435 | 4709 | // If src and dest have compatible element types, we can copy bits. |
duke@435 | 4710 | // Types S[] and D[] are compatible if D is a supertype of S. |
duke@435 | 4711 | // |
duke@435 | 4712 | // If they are not, we will use checked_oop_disjoint_arraycopy, |
duke@435 | 4713 | // which performs a fast optimistic per-oop check, and backs off |
duke@435 | 4714 | // further to JVM_ArrayCopy on the first per-oop check that fails. |
duke@435 | 4715 | // (Actually, we don't move raw bits only; the GC requires card marks.) |
duke@435 | 4716 | |
duke@435 | 4717 | // Get the klassOop for both src and dest |
duke@435 | 4718 | Node* src_klass = load_object_klass(src); |
duke@435 | 4719 | Node* dest_klass = load_object_klass(dest); |
duke@435 | 4720 | |
duke@435 | 4721 | // Generate the subtype check. |
duke@435 | 4722 | // This might fold up statically, or then again it might not. |
duke@435 | 4723 | // |
duke@435 | 4724 | // Non-static example: Copying List<String>.elements to a new String[]. |
duke@435 | 4725 | // The backing store for a List<String> is always an Object[], |
duke@435 | 4726 | // but its elements are always type String, if the generic types |
duke@435 | 4727 | // are correct at the source level. |
duke@435 | 4728 | // |
duke@435 | 4729 | // Test S[] against D[], not S against D, because (probably) |
duke@435 | 4730 | // the secondary supertype cache is less busy for S[] than S. |
duke@435 | 4731 | // This usually only matters when D is an interface. |
duke@435 | 4732 | Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass); |
duke@435 | 4733 | // Plug failing path into checked_oop_disjoint_arraycopy |
duke@435 | 4734 | if (not_subtype_ctrl != top()) { |
duke@435 | 4735 | PreserveJVMState pjvms(this); |
duke@435 | 4736 | set_control(not_subtype_ctrl); |
duke@435 | 4737 | // (At this point we can assume disjoint_bases, since types differ.) |
duke@435 | 4738 | int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc); |
duke@435 | 4739 | Node* p1 = basic_plus_adr(dest_klass, ek_offset); |
kvn@599 | 4740 | Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
duke@435 | 4741 | Node* dest_elem_klass = _gvn.transform(n1); |
duke@435 | 4742 | Node* cv = generate_checkcast_arraycopy(adr_type, |
duke@435 | 4743 | dest_elem_klass, |
duke@435 | 4744 | src, src_offset, dest, dest_offset, |
kvn@1268 | 4745 | copy_length); |
duke@435 | 4746 | if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
duke@435 | 4747 | checked_control = control(); |
duke@435 | 4748 | checked_i_o = i_o(); |
duke@435 | 4749 | checked_mem = memory(adr_type); |
duke@435 | 4750 | checked_value = cv; |
duke@435 | 4751 | } |
duke@435 | 4752 | // At this point we know we do not need type checks on oop stores. |
duke@435 | 4753 | |
duke@435 | 4754 | // Let's see if we need card marks: |
duke@435 | 4755 | if (alloc != NULL && use_ReduceInitialCardMarks()) { |
duke@435 | 4756 | // If we do not need card marks, copy using the jint or jlong stub. |
coleenp@548 | 4757 | copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); |
kvn@464 | 4758 | assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), |
duke@435 | 4759 | "sizes agree"); |
duke@435 | 4760 | } |
duke@435 | 4761 | } |
duke@435 | 4762 | |
duke@435 | 4763 | if (!stopped()) { |
duke@435 | 4764 | // Generate the fast path, if possible. |
duke@435 | 4765 | PreserveJVMState pjvms(this); |
duke@435 | 4766 | generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases, |
duke@435 | 4767 | src, src_offset, dest, dest_offset, |
duke@435 | 4768 | ConvI2X(copy_length)); |
duke@435 | 4769 | |
duke@435 | 4770 | // Present the results of the fast call. |
duke@435 | 4771 | result_region->init_req(fast_path, control()); |
duke@435 | 4772 | result_i_o ->init_req(fast_path, i_o()); |
duke@435 | 4773 | result_memory->init_req(fast_path, memory(adr_type)); |
duke@435 | 4774 | } |
duke@435 | 4775 | |
duke@435 | 4776 | // Here are all the slow paths up to this point, in one bundle: |
duke@435 | 4777 | slow_control = top(); |
duke@435 | 4778 | if (slow_region != NULL) |
duke@435 | 4779 | slow_control = _gvn.transform(slow_region); |
duke@435 | 4780 | debug_only(slow_region = (RegionNode*)badAddress); |
duke@435 | 4781 | |
duke@435 | 4782 | set_control(checked_control); |
duke@435 | 4783 | if (!stopped()) { |
duke@435 | 4784 | // Clean up after the checked call. |
duke@435 | 4785 | // The returned value is either 0 or -1^K, |
duke@435 | 4786 | // where K = number of partially transferred array elements. |
duke@435 | 4787 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) ); |
duke@435 | 4788 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) ); |
duke@435 | 4789 | IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN); |
duke@435 | 4790 | |
duke@435 | 4791 | // If it is 0, we are done, so transfer to the end. |
duke@435 | 4792 | Node* checks_done = _gvn.transform( new(C, 1) IfTrueNode(iff) ); |
duke@435 | 4793 | result_region->init_req(checked_path, checks_done); |
duke@435 | 4794 | result_i_o ->init_req(checked_path, checked_i_o); |
duke@435 | 4795 | result_memory->init_req(checked_path, checked_mem); |
duke@435 | 4796 | |
duke@435 | 4797 | // If it is not zero, merge into the slow call. |
duke@435 | 4798 | set_control( _gvn.transform( new(C, 1) IfFalseNode(iff) )); |
duke@435 | 4799 | RegionNode* slow_reg2 = new(C, 3) RegionNode(3); |
duke@435 | 4800 | PhiNode* slow_i_o2 = new(C, 3) PhiNode(slow_reg2, Type::ABIO); |
duke@435 | 4801 | PhiNode* slow_mem2 = new(C, 3) PhiNode(slow_reg2, Type::MEMORY, adr_type); |
duke@435 | 4802 | record_for_igvn(slow_reg2); |
duke@435 | 4803 | slow_reg2 ->init_req(1, slow_control); |
duke@435 | 4804 | slow_i_o2 ->init_req(1, slow_i_o); |
duke@435 | 4805 | slow_mem2 ->init_req(1, slow_mem); |
duke@435 | 4806 | slow_reg2 ->init_req(2, control()); |
kvn@1268 | 4807 | slow_i_o2 ->init_req(2, checked_i_o); |
kvn@1268 | 4808 | slow_mem2 ->init_req(2, checked_mem); |
duke@435 | 4809 | |
duke@435 | 4810 | slow_control = _gvn.transform(slow_reg2); |
duke@435 | 4811 | slow_i_o = _gvn.transform(slow_i_o2); |
duke@435 | 4812 | slow_mem = _gvn.transform(slow_mem2); |
duke@435 | 4813 | |
duke@435 | 4814 | if (alloc != NULL) { |
duke@435 | 4815 | // We'll restart from the very beginning, after zeroing the whole thing. |
duke@435 | 4816 | // This can cause double writes, but that's OK since dest is brand new. |
duke@435 | 4817 | // So we ignore the low 31 bits of the value returned from the stub. |
duke@435 | 4818 | } else { |
duke@435 | 4819 | // We must continue the copy exactly where it failed, or else |
duke@435 | 4820 | // another thread might see the wrong number of writes to dest. |
duke@435 | 4821 | Node* checked_offset = _gvn.transform( new(C, 3) XorINode(checked_value, intcon(-1)) ); |
duke@435 | 4822 | Node* slow_offset = new(C, 3) PhiNode(slow_reg2, TypeInt::INT); |
duke@435 | 4823 | slow_offset->init_req(1, intcon(0)); |
duke@435 | 4824 | slow_offset->init_req(2, checked_offset); |
duke@435 | 4825 | slow_offset = _gvn.transform(slow_offset); |
duke@435 | 4826 | |
duke@435 | 4827 | // Adjust the arguments by the conditionally incoming offset. |
duke@435 | 4828 | Node* src_off_plus = _gvn.transform( new(C, 3) AddINode(src_offset, slow_offset) ); |
duke@435 | 4829 | Node* dest_off_plus = _gvn.transform( new(C, 3) AddINode(dest_offset, slow_offset) ); |
duke@435 | 4830 | Node* length_minus = _gvn.transform( new(C, 3) SubINode(copy_length, slow_offset) ); |
duke@435 | 4831 | |
duke@435 | 4832 | // Tweak the node variables to adjust the code produced below: |
duke@435 | 4833 | src_offset = src_off_plus; |
duke@435 | 4834 | dest_offset = dest_off_plus; |
duke@435 | 4835 | copy_length = length_minus; |
duke@435 | 4836 | } |
duke@435 | 4837 | } |
duke@435 | 4838 | |
duke@435 | 4839 | set_control(slow_control); |
duke@435 | 4840 | if (!stopped()) { |
duke@435 | 4841 | // Generate the slow path, if needed. |
duke@435 | 4842 | PreserveJVMState pjvms(this); // replace_in_map may trash the map |
duke@435 | 4843 | |
duke@435 | 4844 | set_memory(slow_mem, adr_type); |
duke@435 | 4845 | set_i_o(slow_i_o); |
duke@435 | 4846 | |
duke@435 | 4847 | if (must_clear_dest) { |
duke@435 | 4848 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4849 | intcon(0), NULL, |
duke@435 | 4850 | alloc->in(AllocateNode::AllocSize)); |
duke@435 | 4851 | } |
duke@435 | 4852 | |
duke@435 | 4853 | generate_slow_arraycopy(adr_type, |
duke@435 | 4854 | src, src_offset, dest, dest_offset, |
kvn@1268 | 4855 | copy_length); |
duke@435 | 4856 | |
duke@435 | 4857 | result_region->init_req(slow_call_path, control()); |
duke@435 | 4858 | result_i_o ->init_req(slow_call_path, i_o()); |
duke@435 | 4859 | result_memory->init_req(slow_call_path, memory(adr_type)); |
duke@435 | 4860 | } |
duke@435 | 4861 | |
duke@435 | 4862 | // Remove unused edges. |
duke@435 | 4863 | for (uint i = 1; i < result_region->req(); i++) { |
duke@435 | 4864 | if (result_region->in(i) == NULL) |
duke@435 | 4865 | result_region->init_req(i, top()); |
duke@435 | 4866 | } |
duke@435 | 4867 | |
duke@435 | 4868 | // Finished; return the combined state. |
duke@435 | 4869 | set_control( _gvn.transform(result_region) ); |
duke@435 | 4870 | set_i_o( _gvn.transform(result_i_o) ); |
duke@435 | 4871 | set_memory( _gvn.transform(result_memory), adr_type ); |
duke@435 | 4872 | |
duke@435 | 4873 | // The memory edges above are precise in order to model effects around |
twisti@1040 | 4874 | // array copies accurately to allow value numbering of field loads around |
duke@435 | 4875 | // arraycopy. Such field loads, both before and after, are common in Java |
duke@435 | 4876 | // collections and similar classes involving header/array data structures. |
duke@435 | 4877 | // |
duke@435 | 4878 | // But with low number of register or when some registers are used or killed |
duke@435 | 4879 | // by arraycopy calls it causes registers spilling on stack. See 6544710. |
duke@435 | 4880 | // The next memory barrier is added to avoid it. If the arraycopy can be |
duke@435 | 4881 | // optimized away (which it can, sometimes) then we can manually remove |
duke@435 | 4882 | // the membar also. |
kvn@1393 | 4883 | // |
kvn@1393 | 4884 | // Do not let reads from the cloned object float above the arraycopy. |
kvn@1393 | 4885 | if (InsertMemBarAfterArraycopy || alloc != NULL) |
duke@435 | 4886 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4887 | } |
duke@435 | 4888 | |
duke@435 | 4889 | |
duke@435 | 4890 | // Helper function which determines if an arraycopy immediately follows |
duke@435 | 4891 | // an allocation, with no intervening tests or other escapes for the object. |
duke@435 | 4892 | AllocateArrayNode* |
duke@435 | 4893 | LibraryCallKit::tightly_coupled_allocation(Node* ptr, |
duke@435 | 4894 | RegionNode* slow_region) { |
duke@435 | 4895 | if (stopped()) return NULL; // no fast path |
duke@435 | 4896 | if (C->AliasLevel() == 0) return NULL; // no MergeMems around |
duke@435 | 4897 | |
duke@435 | 4898 | AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); |
duke@435 | 4899 | if (alloc == NULL) return NULL; |
duke@435 | 4900 | |
duke@435 | 4901 | Node* rawmem = memory(Compile::AliasIdxRaw); |
duke@435 | 4902 | // Is the allocation's memory state untouched? |
duke@435 | 4903 | if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) { |
duke@435 | 4904 | // Bail out if there have been raw-memory effects since the allocation. |
duke@435 | 4905 | // (Example: There might have been a call or safepoint.) |
duke@435 | 4906 | return NULL; |
duke@435 | 4907 | } |
duke@435 | 4908 | rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw); |
duke@435 | 4909 | if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) { |
duke@435 | 4910 | return NULL; |
duke@435 | 4911 | } |
duke@435 | 4912 | |
duke@435 | 4913 | // There must be no unexpected observers of this allocation. |
duke@435 | 4914 | for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) { |
duke@435 | 4915 | Node* obs = ptr->fast_out(i); |
duke@435 | 4916 | if (obs != this->map()) { |
duke@435 | 4917 | return NULL; |
duke@435 | 4918 | } |
duke@435 | 4919 | } |
duke@435 | 4920 | |
duke@435 | 4921 | // This arraycopy must unconditionally follow the allocation of the ptr. |
duke@435 | 4922 | Node* alloc_ctl = ptr->in(0); |
duke@435 | 4923 | assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo"); |
duke@435 | 4924 | |
duke@435 | 4925 | Node* ctl = control(); |
duke@435 | 4926 | while (ctl != alloc_ctl) { |
duke@435 | 4927 | // There may be guards which feed into the slow_region. |
duke@435 | 4928 | // Any other control flow means that we might not get a chance |
duke@435 | 4929 | // to finish initializing the allocated object. |
duke@435 | 4930 | if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) { |
duke@435 | 4931 | IfNode* iff = ctl->in(0)->as_If(); |
duke@435 | 4932 | Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con); |
duke@435 | 4933 | assert(not_ctl != NULL && not_ctl != ctl, "found alternate"); |
duke@435 | 4934 | if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) { |
duke@435 | 4935 | ctl = iff->in(0); // This test feeds the known slow_region. |
duke@435 | 4936 | continue; |
duke@435 | 4937 | } |
duke@435 | 4938 | // One more try: Various low-level checks bottom out in |
duke@435 | 4939 | // uncommon traps. If the debug-info of the trap omits |
duke@435 | 4940 | // any reference to the allocation, as we've already |
duke@435 | 4941 | // observed, then there can be no objection to the trap. |
duke@435 | 4942 | bool found_trap = false; |
duke@435 | 4943 | for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) { |
duke@435 | 4944 | Node* obs = not_ctl->fast_out(j); |
duke@435 | 4945 | if (obs->in(0) == not_ctl && obs->is_Call() && |
duke@435 | 4946 | (obs->as_Call()->entry_point() == |
duke@435 | 4947 | SharedRuntime::uncommon_trap_blob()->instructions_begin())) { |
duke@435 | 4948 | found_trap = true; break; |
duke@435 | 4949 | } |
duke@435 | 4950 | } |
duke@435 | 4951 | if (found_trap) { |
duke@435 | 4952 | ctl = iff->in(0); // This test feeds a harmless uncommon trap. |
duke@435 | 4953 | continue; |
duke@435 | 4954 | } |
duke@435 | 4955 | } |
duke@435 | 4956 | return NULL; |
duke@435 | 4957 | } |
duke@435 | 4958 | |
duke@435 | 4959 | // If we get this far, we have an allocation which immediately |
duke@435 | 4960 | // precedes the arraycopy, and we can take over zeroing the new object. |
duke@435 | 4961 | // The arraycopy will finish the initialization, and provide |
duke@435 | 4962 | // a new control state to which we will anchor the destination pointer. |
duke@435 | 4963 | |
duke@435 | 4964 | return alloc; |
duke@435 | 4965 | } |
duke@435 | 4966 | |
duke@435 | 4967 | // Helper for initialization of arrays, creating a ClearArray. |
duke@435 | 4968 | // It writes zero bits in [start..end), within the body of an array object. |
duke@435 | 4969 | // The memory effects are all chained onto the 'adr_type' alias category. |
duke@435 | 4970 | // |
duke@435 | 4971 | // Since the object is otherwise uninitialized, we are free |
duke@435 | 4972 | // to put a little "slop" around the edges of the cleared area, |
duke@435 | 4973 | // as long as it does not go back into the array's header, |
duke@435 | 4974 | // or beyond the array end within the heap. |
duke@435 | 4975 | // |
duke@435 | 4976 | // The lower edge can be rounded down to the nearest jint and the |
duke@435 | 4977 | // upper edge can be rounded up to the nearest MinObjAlignmentInBytes. |
duke@435 | 4978 | // |
duke@435 | 4979 | // Arguments: |
duke@435 | 4980 | // adr_type memory slice where writes are generated |
duke@435 | 4981 | // dest oop of the destination array |
duke@435 | 4982 | // basic_elem_type element type of the destination |
duke@435 | 4983 | // slice_idx array index of first element to store |
duke@435 | 4984 | // slice_len number of elements to store (or NULL) |
duke@435 | 4985 | // dest_size total size in bytes of the array object |
duke@435 | 4986 | // |
duke@435 | 4987 | // Exactly one of slice_len or dest_size must be non-NULL. |
duke@435 | 4988 | // If dest_size is non-NULL, zeroing extends to the end of the object. |
duke@435 | 4989 | // If slice_len is non-NULL, the slice_idx value must be a constant. |
duke@435 | 4990 | void |
duke@435 | 4991 | LibraryCallKit::generate_clear_array(const TypePtr* adr_type, |
duke@435 | 4992 | Node* dest, |
duke@435 | 4993 | BasicType basic_elem_type, |
duke@435 | 4994 | Node* slice_idx, |
duke@435 | 4995 | Node* slice_len, |
duke@435 | 4996 | Node* dest_size) { |
duke@435 | 4997 | // one or the other but not both of slice_len and dest_size: |
duke@435 | 4998 | assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, ""); |
duke@435 | 4999 | if (slice_len == NULL) slice_len = top(); |
duke@435 | 5000 | if (dest_size == NULL) dest_size = top(); |
duke@435 | 5001 | |
duke@435 | 5002 | // operate on this memory slice: |
duke@435 | 5003 | Node* mem = memory(adr_type); // memory slice to operate on |
duke@435 | 5004 | |
duke@435 | 5005 | // scaling and rounding of indexes: |
kvn@464 | 5006 | int scale = exact_log2(type2aelembytes(basic_elem_type)); |
duke@435 | 5007 | int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); |
duke@435 | 5008 | int clear_low = (-1 << scale) & (BytesPerInt - 1); |
duke@435 | 5009 | int bump_bit = (-1 << scale) & BytesPerInt; |
duke@435 | 5010 | |
duke@435 | 5011 | // determine constant starts and ends |
duke@435 | 5012 | const intptr_t BIG_NEG = -128; |
duke@435 | 5013 | assert(BIG_NEG + 2*abase < 0, "neg enough"); |
duke@435 | 5014 | intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG); |
duke@435 | 5015 | intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG); |
duke@435 | 5016 | if (slice_len_con == 0) { |
duke@435 | 5017 | return; // nothing to do here |
duke@435 | 5018 | } |
duke@435 | 5019 | intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low; |
duke@435 | 5020 | intptr_t end_con = find_intptr_t_con(dest_size, -1); |
duke@435 | 5021 | if (slice_idx_con >= 0 && slice_len_con >= 0) { |
duke@435 | 5022 | assert(end_con < 0, "not two cons"); |
duke@435 | 5023 | end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale), |
duke@435 | 5024 | BytesPerLong); |
duke@435 | 5025 | } |
duke@435 | 5026 | |
duke@435 | 5027 | if (start_con >= 0 && end_con >= 0) { |
duke@435 | 5028 | // Constant start and end. Simple. |
duke@435 | 5029 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5030 | start_con, end_con, &_gvn); |
duke@435 | 5031 | } else if (start_con >= 0 && dest_size != top()) { |
duke@435 | 5032 | // Constant start, pre-rounded end after the tail of the array. |
duke@435 | 5033 | Node* end = dest_size; |
duke@435 | 5034 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5035 | start_con, end, &_gvn); |
duke@435 | 5036 | } else if (start_con >= 0 && slice_len != top()) { |
duke@435 | 5037 | // Constant start, non-constant end. End needs rounding up. |
duke@435 | 5038 | // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8) |
duke@435 | 5039 | intptr_t end_base = abase + (slice_idx_con << scale); |
duke@435 | 5040 | int end_round = (-1 << scale) & (BytesPerLong - 1); |
duke@435 | 5041 | Node* end = ConvI2X(slice_len); |
duke@435 | 5042 | if (scale != 0) |
duke@435 | 5043 | end = _gvn.transform( new(C,3) LShiftXNode(end, intcon(scale) )); |
duke@435 | 5044 | end_base += end_round; |
duke@435 | 5045 | end = _gvn.transform( new(C,3) AddXNode(end, MakeConX(end_base)) ); |
duke@435 | 5046 | end = _gvn.transform( new(C,3) AndXNode(end, MakeConX(~end_round)) ); |
duke@435 | 5047 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5048 | start_con, end, &_gvn); |
duke@435 | 5049 | } else if (start_con < 0 && dest_size != top()) { |
duke@435 | 5050 | // Non-constant start, pre-rounded end after the tail of the array. |
duke@435 | 5051 | // This is almost certainly a "round-to-end" operation. |
duke@435 | 5052 | Node* start = slice_idx; |
duke@435 | 5053 | start = ConvI2X(start); |
duke@435 | 5054 | if (scale != 0) |
duke@435 | 5055 | start = _gvn.transform( new(C,3) LShiftXNode( start, intcon(scale) )); |
duke@435 | 5056 | start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(abase)) ); |
duke@435 | 5057 | if ((bump_bit | clear_low) != 0) { |
duke@435 | 5058 | int to_clear = (bump_bit | clear_low); |
duke@435 | 5059 | // Align up mod 8, then store a jint zero unconditionally |
duke@435 | 5060 | // just before the mod-8 boundary. |
coleenp@548 | 5061 | if (((abase + bump_bit) & ~to_clear) - bump_bit |
coleenp@548 | 5062 | < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) { |
coleenp@548 | 5063 | bump_bit = 0; |
coleenp@548 | 5064 | assert((abase & to_clear) == 0, "array base must be long-aligned"); |
coleenp@548 | 5065 | } else { |
coleenp@548 | 5066 | // Bump 'start' up to (or past) the next jint boundary: |
coleenp@548 | 5067 | start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(bump_bit)) ); |
coleenp@548 | 5068 | assert((abase & clear_low) == 0, "array base must be int-aligned"); |
coleenp@548 | 5069 | } |
duke@435 | 5070 | // Round bumped 'start' down to jlong boundary in body of array. |
duke@435 | 5071 | start = _gvn.transform( new(C,3) AndXNode(start, MakeConX(~to_clear)) ); |
coleenp@548 | 5072 | if (bump_bit != 0) { |
coleenp@548 | 5073 | // Store a zero to the immediately preceding jint: |
coleenp@548 | 5074 | Node* x1 = _gvn.transform( new(C,3) AddXNode(start, MakeConX(-bump_bit)) ); |
coleenp@548 | 5075 | Node* p1 = basic_plus_adr(dest, x1); |
coleenp@548 | 5076 | mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT); |
coleenp@548 | 5077 | mem = _gvn.transform(mem); |
coleenp@548 | 5078 | } |
duke@435 | 5079 | } |
duke@435 | 5080 | Node* end = dest_size; // pre-rounded |
duke@435 | 5081 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5082 | start, end, &_gvn); |
duke@435 | 5083 | } else { |
duke@435 | 5084 | // Non-constant start, unrounded non-constant end. |
duke@435 | 5085 | // (Nobody zeroes a random midsection of an array using this routine.) |
duke@435 | 5086 | ShouldNotReachHere(); // fix caller |
duke@435 | 5087 | } |
duke@435 | 5088 | |
duke@435 | 5089 | // Done. |
duke@435 | 5090 | set_memory(mem, adr_type); |
duke@435 | 5091 | } |
duke@435 | 5092 | |
duke@435 | 5093 | |
duke@435 | 5094 | bool |
duke@435 | 5095 | LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type, |
duke@435 | 5096 | BasicType basic_elem_type, |
duke@435 | 5097 | AllocateNode* alloc, |
duke@435 | 5098 | Node* src, Node* src_offset, |
duke@435 | 5099 | Node* dest, Node* dest_offset, |
duke@435 | 5100 | Node* dest_size) { |
duke@435 | 5101 | // See if there is an advantage from block transfer. |
kvn@464 | 5102 | int scale = exact_log2(type2aelembytes(basic_elem_type)); |
duke@435 | 5103 | if (scale >= LogBytesPerLong) |
duke@435 | 5104 | return false; // it is already a block transfer |
duke@435 | 5105 | |
duke@435 | 5106 | // Look at the alignment of the starting offsets. |
duke@435 | 5107 | int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); |
duke@435 | 5108 | const intptr_t BIG_NEG = -128; |
duke@435 | 5109 | assert(BIG_NEG + 2*abase < 0, "neg enough"); |
duke@435 | 5110 | |
duke@435 | 5111 | intptr_t src_off = abase + ((intptr_t) find_int_con(src_offset, -1) << scale); |
duke@435 | 5112 | intptr_t dest_off = abase + ((intptr_t) find_int_con(dest_offset, -1) << scale); |
duke@435 | 5113 | if (src_off < 0 || dest_off < 0) |
duke@435 | 5114 | // At present, we can only understand constants. |
duke@435 | 5115 | return false; |
duke@435 | 5116 | |
duke@435 | 5117 | if (((src_off | dest_off) & (BytesPerLong-1)) != 0) { |
duke@435 | 5118 | // Non-aligned; too bad. |
duke@435 | 5119 | // One more chance: Pick off an initial 32-bit word. |
duke@435 | 5120 | // This is a common case, since abase can be odd mod 8. |
duke@435 | 5121 | if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt && |
duke@435 | 5122 | ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) { |
duke@435 | 5123 | Node* sptr = basic_plus_adr(src, src_off); |
duke@435 | 5124 | Node* dptr = basic_plus_adr(dest, dest_off); |
duke@435 | 5125 | Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type); |
duke@435 | 5126 | store_to_memory(control(), dptr, sval, T_INT, adr_type); |
duke@435 | 5127 | src_off += BytesPerInt; |
duke@435 | 5128 | dest_off += BytesPerInt; |
duke@435 | 5129 | } else { |
duke@435 | 5130 | return false; |
duke@435 | 5131 | } |
duke@435 | 5132 | } |
duke@435 | 5133 | assert(src_off % BytesPerLong == 0, ""); |
duke@435 | 5134 | assert(dest_off % BytesPerLong == 0, ""); |
duke@435 | 5135 | |
duke@435 | 5136 | // Do this copy by giant steps. |
duke@435 | 5137 | Node* sptr = basic_plus_adr(src, src_off); |
duke@435 | 5138 | Node* dptr = basic_plus_adr(dest, dest_off); |
duke@435 | 5139 | Node* countx = dest_size; |
duke@435 | 5140 | countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) ); |
duke@435 | 5141 | countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) ); |
duke@435 | 5142 | |
duke@435 | 5143 | bool disjoint_bases = true; // since alloc != NULL |
duke@435 | 5144 | generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, |
duke@435 | 5145 | sptr, NULL, dptr, NULL, countx); |
duke@435 | 5146 | |
duke@435 | 5147 | return true; |
duke@435 | 5148 | } |
duke@435 | 5149 | |
duke@435 | 5150 | |
duke@435 | 5151 | // Helper function; generates code for the slow case. |
duke@435 | 5152 | // We make a call to a runtime method which emulates the native method, |
duke@435 | 5153 | // but without the native wrapper overhead. |
duke@435 | 5154 | void |
duke@435 | 5155 | LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
duke@435 | 5156 | Node* src, Node* src_offset, |
duke@435 | 5157 | Node* dest, Node* dest_offset, |
kvn@1268 | 5158 | Node* copy_length) { |
duke@435 | 5159 | Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
duke@435 | 5160 | OptoRuntime::slow_arraycopy_Type(), |
duke@435 | 5161 | OptoRuntime::slow_arraycopy_Java(), |
duke@435 | 5162 | "slow_arraycopy", adr_type, |
duke@435 | 5163 | src, src_offset, dest, dest_offset, |
duke@435 | 5164 | copy_length); |
duke@435 | 5165 | |
duke@435 | 5166 | // Handle exceptions thrown by this fellow: |
duke@435 | 5167 | make_slow_call_ex(call, env()->Throwable_klass(), false); |
duke@435 | 5168 | } |
duke@435 | 5169 | |
duke@435 | 5170 | // Helper function; generates code for cases requiring runtime checks. |
duke@435 | 5171 | Node* |
duke@435 | 5172 | LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
duke@435 | 5173 | Node* dest_elem_klass, |
duke@435 | 5174 | Node* src, Node* src_offset, |
duke@435 | 5175 | Node* dest, Node* dest_offset, |
kvn@1268 | 5176 | Node* copy_length) { |
duke@435 | 5177 | if (stopped()) return NULL; |
duke@435 | 5178 | |
duke@435 | 5179 | address copyfunc_addr = StubRoutines::checkcast_arraycopy(); |
duke@435 | 5180 | if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
duke@435 | 5181 | return NULL; |
duke@435 | 5182 | } |
duke@435 | 5183 | |
duke@435 | 5184 | // Pick out the parameters required to perform a store-check |
duke@435 | 5185 | // for the target array. This is an optimistic check. It will |
duke@435 | 5186 | // look in each non-null element's class, at the desired klass's |
duke@435 | 5187 | // super_check_offset, for the desired klass. |
duke@435 | 5188 | int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc); |
duke@435 | 5189 | Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset); |
duke@435 | 5190 | Node* n3 = new(C, 3) LoadINode(NULL, immutable_memory(), p3, TypeRawPtr::BOTTOM); |
duke@435 | 5191 | Node* check_offset = _gvn.transform(n3); |
duke@435 | 5192 | Node* check_value = dest_elem_klass; |
duke@435 | 5193 | |
duke@435 | 5194 | Node* src_start = array_element_address(src, src_offset, T_OBJECT); |
duke@435 | 5195 | Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT); |
duke@435 | 5196 | |
duke@435 | 5197 | // (We know the arrays are never conjoint, because their types differ.) |
duke@435 | 5198 | Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5199 | OptoRuntime::checkcast_arraycopy_Type(), |
duke@435 | 5200 | copyfunc_addr, "checkcast_arraycopy", adr_type, |
duke@435 | 5201 | // five arguments, of which two are |
duke@435 | 5202 | // intptr_t (jlong in LP64) |
duke@435 | 5203 | src_start, dest_start, |
duke@435 | 5204 | copy_length XTOP, |
duke@435 | 5205 | check_offset XTOP, |
duke@435 | 5206 | check_value); |
duke@435 | 5207 | |
duke@435 | 5208 | return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms)); |
duke@435 | 5209 | } |
duke@435 | 5210 | |
duke@435 | 5211 | |
duke@435 | 5212 | // Helper function; generates code for cases requiring runtime checks. |
duke@435 | 5213 | Node* |
duke@435 | 5214 | LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
duke@435 | 5215 | Node* src, Node* src_offset, |
duke@435 | 5216 | Node* dest, Node* dest_offset, |
kvn@1268 | 5217 | Node* copy_length) { |
duke@435 | 5218 | if (stopped()) return NULL; |
duke@435 | 5219 | |
duke@435 | 5220 | address copyfunc_addr = StubRoutines::generic_arraycopy(); |
duke@435 | 5221 | if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
duke@435 | 5222 | return NULL; |
duke@435 | 5223 | } |
duke@435 | 5224 | |
duke@435 | 5225 | Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5226 | OptoRuntime::generic_arraycopy_Type(), |
duke@435 | 5227 | copyfunc_addr, "generic_arraycopy", adr_type, |
duke@435 | 5228 | src, src_offset, dest, dest_offset, copy_length); |
duke@435 | 5229 | |
duke@435 | 5230 | return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms)); |
duke@435 | 5231 | } |
duke@435 | 5232 | |
duke@435 | 5233 | // Helper function; generates the fast out-of-line call to an arraycopy stub. |
duke@435 | 5234 | void |
duke@435 | 5235 | LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type, |
duke@435 | 5236 | BasicType basic_elem_type, |
duke@435 | 5237 | bool disjoint_bases, |
duke@435 | 5238 | Node* src, Node* src_offset, |
duke@435 | 5239 | Node* dest, Node* dest_offset, |
duke@435 | 5240 | Node* copy_length) { |
duke@435 | 5241 | if (stopped()) return; // nothing to do |
duke@435 | 5242 | |
duke@435 | 5243 | Node* src_start = src; |
duke@435 | 5244 | Node* dest_start = dest; |
duke@435 | 5245 | if (src_offset != NULL || dest_offset != NULL) { |
duke@435 | 5246 | assert(src_offset != NULL && dest_offset != NULL, ""); |
duke@435 | 5247 | src_start = array_element_address(src, src_offset, basic_elem_type); |
duke@435 | 5248 | dest_start = array_element_address(dest, dest_offset, basic_elem_type); |
duke@435 | 5249 | } |
duke@435 | 5250 | |
duke@435 | 5251 | // Figure out which arraycopy runtime method to call. |
duke@435 | 5252 | const char* copyfunc_name = "arraycopy"; |
duke@435 | 5253 | address copyfunc_addr = |
duke@435 | 5254 | basictype2arraycopy(basic_elem_type, src_offset, dest_offset, |
duke@435 | 5255 | disjoint_bases, copyfunc_name); |
duke@435 | 5256 | |
duke@435 | 5257 | // Call it. Note that the count_ix value is not scaled to a byte-size. |
duke@435 | 5258 | make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5259 | OptoRuntime::fast_arraycopy_Type(), |
duke@435 | 5260 | copyfunc_addr, copyfunc_name, adr_type, |
duke@435 | 5261 | src_start, dest_start, copy_length XTOP); |
duke@435 | 5262 | } |