Thu, 16 Feb 2012 17:12:49 -0800
7145346: VerifyStackAtCalls is broken
Summary: Replace call_epilog() encoding with macroassembler use. Moved duplicated code to x86.ad. Fixed return_addr() definition.
Reviewed-by: never
duke@435 | 1 | /* |
kvn@2602 | 2 | * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved. |
duke@435 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
duke@435 | 4 | * |
duke@435 | 5 | * This code is free software; you can redistribute it and/or modify it |
duke@435 | 6 | * under the terms of the GNU General Public License version 2 only, as |
duke@435 | 7 | * published by the Free Software Foundation. |
duke@435 | 8 | * |
duke@435 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
duke@435 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
duke@435 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
duke@435 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
duke@435 | 13 | * accompanied this code). |
duke@435 | 14 | * |
duke@435 | 15 | * You should have received a copy of the GNU General Public License version |
duke@435 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
duke@435 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
duke@435 | 18 | * |
trims@1907 | 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
trims@1907 | 20 | * or visit www.oracle.com if you need additional information or have any |
trims@1907 | 21 | * questions. |
duke@435 | 22 | * |
duke@435 | 23 | */ |
duke@435 | 24 | |
stefank@2314 | 25 | #include "precompiled.hpp" |
stefank@2314 | 26 | #include "classfile/systemDictionary.hpp" |
stefank@2314 | 27 | #include "classfile/vmSymbols.hpp" |
twisti@2687 | 28 | #include "compiler/compileBroker.hpp" |
stefank@2314 | 29 | #include "compiler/compileLog.hpp" |
stefank@2314 | 30 | #include "oops/objArrayKlass.hpp" |
stefank@2314 | 31 | #include "opto/addnode.hpp" |
stefank@2314 | 32 | #include "opto/callGenerator.hpp" |
stefank@2314 | 33 | #include "opto/cfgnode.hpp" |
stefank@2314 | 34 | #include "opto/idealKit.hpp" |
stefank@2314 | 35 | #include "opto/mulnode.hpp" |
stefank@2314 | 36 | #include "opto/parse.hpp" |
stefank@2314 | 37 | #include "opto/runtime.hpp" |
stefank@2314 | 38 | #include "opto/subnode.hpp" |
stefank@2314 | 39 | #include "prims/nativeLookup.hpp" |
stefank@2314 | 40 | #include "runtime/sharedRuntime.hpp" |
duke@435 | 41 | |
duke@435 | 42 | class LibraryIntrinsic : public InlineCallGenerator { |
duke@435 | 43 | // Extend the set of intrinsics known to the runtime: |
duke@435 | 44 | public: |
duke@435 | 45 | private: |
duke@435 | 46 | bool _is_virtual; |
duke@435 | 47 | vmIntrinsics::ID _intrinsic_id; |
duke@435 | 48 | |
duke@435 | 49 | public: |
duke@435 | 50 | LibraryIntrinsic(ciMethod* m, bool is_virtual, vmIntrinsics::ID id) |
duke@435 | 51 | : InlineCallGenerator(m), |
duke@435 | 52 | _is_virtual(is_virtual), |
duke@435 | 53 | _intrinsic_id(id) |
duke@435 | 54 | { |
duke@435 | 55 | } |
duke@435 | 56 | virtual bool is_intrinsic() const { return true; } |
duke@435 | 57 | virtual bool is_virtual() const { return _is_virtual; } |
duke@435 | 58 | virtual JVMState* generate(JVMState* jvms); |
duke@435 | 59 | vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; } |
duke@435 | 60 | }; |
duke@435 | 61 | |
duke@435 | 62 | |
duke@435 | 63 | // Local helper class for LibraryIntrinsic: |
duke@435 | 64 | class LibraryCallKit : public GraphKit { |
duke@435 | 65 | private: |
duke@435 | 66 | LibraryIntrinsic* _intrinsic; // the library intrinsic being called |
duke@435 | 67 | |
duke@435 | 68 | public: |
duke@435 | 69 | LibraryCallKit(JVMState* caller, LibraryIntrinsic* intrinsic) |
duke@435 | 70 | : GraphKit(caller), |
duke@435 | 71 | _intrinsic(intrinsic) |
duke@435 | 72 | { |
duke@435 | 73 | } |
duke@435 | 74 | |
duke@435 | 75 | ciMethod* caller() const { return jvms()->method(); } |
duke@435 | 76 | int bci() const { return jvms()->bci(); } |
duke@435 | 77 | LibraryIntrinsic* intrinsic() const { return _intrinsic; } |
duke@435 | 78 | vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); } |
duke@435 | 79 | ciMethod* callee() const { return _intrinsic->method(); } |
duke@435 | 80 | ciSignature* signature() const { return callee()->signature(); } |
duke@435 | 81 | int arg_size() const { return callee()->arg_size(); } |
duke@435 | 82 | |
duke@435 | 83 | bool try_to_inline(); |
duke@435 | 84 | |
duke@435 | 85 | // Helper functions to inline natives |
duke@435 | 86 | void push_result(RegionNode* region, PhiNode* value); |
duke@435 | 87 | Node* generate_guard(Node* test, RegionNode* region, float true_prob); |
duke@435 | 88 | Node* generate_slow_guard(Node* test, RegionNode* region); |
duke@435 | 89 | Node* generate_fair_guard(Node* test, RegionNode* region); |
duke@435 | 90 | Node* generate_negative_guard(Node* index, RegionNode* region, |
duke@435 | 91 | // resulting CastII of index: |
duke@435 | 92 | Node* *pos_index = NULL); |
duke@435 | 93 | Node* generate_nonpositive_guard(Node* index, bool never_negative, |
duke@435 | 94 | // resulting CastII of index: |
duke@435 | 95 | Node* *pos_index = NULL); |
duke@435 | 96 | Node* generate_limit_guard(Node* offset, Node* subseq_length, |
duke@435 | 97 | Node* array_length, |
duke@435 | 98 | RegionNode* region); |
duke@435 | 99 | Node* generate_current_thread(Node* &tls_output); |
duke@435 | 100 | address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset, |
iveresov@2606 | 101 | bool disjoint_bases, const char* &name, bool dest_uninitialized); |
duke@435 | 102 | Node* load_mirror_from_klass(Node* klass); |
duke@435 | 103 | Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, |
duke@435 | 104 | int nargs, |
duke@435 | 105 | RegionNode* region, int null_path, |
duke@435 | 106 | int offset); |
duke@435 | 107 | Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs, |
duke@435 | 108 | RegionNode* region, int null_path) { |
duke@435 | 109 | int offset = java_lang_Class::klass_offset_in_bytes(); |
duke@435 | 110 | return load_klass_from_mirror_common(mirror, never_see_null, nargs, |
duke@435 | 111 | region, null_path, |
duke@435 | 112 | offset); |
duke@435 | 113 | } |
duke@435 | 114 | Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null, |
duke@435 | 115 | int nargs, |
duke@435 | 116 | RegionNode* region, int null_path) { |
duke@435 | 117 | int offset = java_lang_Class::array_klass_offset_in_bytes(); |
duke@435 | 118 | return load_klass_from_mirror_common(mirror, never_see_null, nargs, |
duke@435 | 119 | region, null_path, |
duke@435 | 120 | offset); |
duke@435 | 121 | } |
duke@435 | 122 | Node* generate_access_flags_guard(Node* kls, |
duke@435 | 123 | int modifier_mask, int modifier_bits, |
duke@435 | 124 | RegionNode* region); |
duke@435 | 125 | Node* generate_interface_guard(Node* kls, RegionNode* region); |
duke@435 | 126 | Node* generate_array_guard(Node* kls, RegionNode* region) { |
duke@435 | 127 | return generate_array_guard_common(kls, region, false, false); |
duke@435 | 128 | } |
duke@435 | 129 | Node* generate_non_array_guard(Node* kls, RegionNode* region) { |
duke@435 | 130 | return generate_array_guard_common(kls, region, false, true); |
duke@435 | 131 | } |
duke@435 | 132 | Node* generate_objArray_guard(Node* kls, RegionNode* region) { |
duke@435 | 133 | return generate_array_guard_common(kls, region, true, false); |
duke@435 | 134 | } |
duke@435 | 135 | Node* generate_non_objArray_guard(Node* kls, RegionNode* region) { |
duke@435 | 136 | return generate_array_guard_common(kls, region, true, true); |
duke@435 | 137 | } |
duke@435 | 138 | Node* generate_array_guard_common(Node* kls, RegionNode* region, |
duke@435 | 139 | bool obj_array, bool not_array); |
duke@435 | 140 | Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region); |
duke@435 | 141 | CallJavaNode* generate_method_call(vmIntrinsics::ID method_id, |
duke@435 | 142 | bool is_virtual = false, bool is_static = false); |
duke@435 | 143 | CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) { |
duke@435 | 144 | return generate_method_call(method_id, false, true); |
duke@435 | 145 | } |
duke@435 | 146 | CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) { |
duke@435 | 147 | return generate_method_call(method_id, true, false); |
duke@435 | 148 | } |
duke@435 | 149 | |
kvn@1421 | 150 | Node* make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2); |
duke@435 | 151 | bool inline_string_compareTo(); |
duke@435 | 152 | bool inline_string_indexOf(); |
duke@435 | 153 | Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i); |
cfang@1116 | 154 | bool inline_string_equals(); |
duke@435 | 155 | Node* pop_math_arg(); |
duke@435 | 156 | bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName); |
duke@435 | 157 | bool inline_math_native(vmIntrinsics::ID id); |
duke@435 | 158 | bool inline_trig(vmIntrinsics::ID id); |
duke@435 | 159 | bool inline_trans(vmIntrinsics::ID id); |
duke@435 | 160 | bool inline_abs(vmIntrinsics::ID id); |
duke@435 | 161 | bool inline_sqrt(vmIntrinsics::ID id); |
duke@435 | 162 | bool inline_pow(vmIntrinsics::ID id); |
duke@435 | 163 | bool inline_exp(vmIntrinsics::ID id); |
duke@435 | 164 | bool inline_min_max(vmIntrinsics::ID id); |
duke@435 | 165 | Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); |
duke@435 | 166 | // This returns Type::AnyPtr, RawPtr, or OopPtr. |
duke@435 | 167 | int classify_unsafe_addr(Node* &base, Node* &offset); |
duke@435 | 168 | Node* make_unsafe_address(Node* base, Node* offset); |
johnc@2781 | 169 | // Helper for inline_unsafe_access. |
johnc@2781 | 170 | // Generates the guards that check whether the result of |
johnc@2781 | 171 | // Unsafe.getObject should be recorded in an SATB log buffer. |
johnc@2781 | 172 | void insert_g1_pre_barrier(Node* base_oop, Node* offset, Node* pre_val); |
duke@435 | 173 | bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile); |
duke@435 | 174 | bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static); |
duke@435 | 175 | bool inline_unsafe_allocate(); |
duke@435 | 176 | bool inline_unsafe_copyMemory(); |
duke@435 | 177 | bool inline_native_currentThread(); |
duke@435 | 178 | bool inline_native_time_funcs(bool isNano); |
duke@435 | 179 | bool inline_native_isInterrupted(); |
duke@435 | 180 | bool inline_native_Class_query(vmIntrinsics::ID id); |
duke@435 | 181 | bool inline_native_subtype_check(); |
duke@435 | 182 | |
duke@435 | 183 | bool inline_native_newArray(); |
duke@435 | 184 | bool inline_native_getLength(); |
duke@435 | 185 | bool inline_array_copyOf(bool is_copyOfRange); |
rasbold@604 | 186 | bool inline_array_equals(); |
kvn@1268 | 187 | void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); |
duke@435 | 188 | bool inline_native_clone(bool is_virtual); |
duke@435 | 189 | bool inline_native_Reflection_getCallerClass(); |
duke@435 | 190 | bool inline_native_AtomicLong_get(); |
duke@435 | 191 | bool inline_native_AtomicLong_attemptUpdate(); |
duke@435 | 192 | bool is_method_invoke_or_aux_frame(JVMState* jvms); |
duke@435 | 193 | // Helper function for inlining native object hash method |
duke@435 | 194 | bool inline_native_hashcode(bool is_virtual, bool is_static); |
duke@435 | 195 | bool inline_native_getClass(); |
duke@435 | 196 | |
duke@435 | 197 | // Helper functions for inlining arraycopy |
duke@435 | 198 | bool inline_arraycopy(); |
duke@435 | 199 | void generate_arraycopy(const TypePtr* adr_type, |
duke@435 | 200 | BasicType basic_elem_type, |
duke@435 | 201 | Node* src, Node* src_offset, |
duke@435 | 202 | Node* dest, Node* dest_offset, |
duke@435 | 203 | Node* copy_length, |
duke@435 | 204 | bool disjoint_bases = false, |
duke@435 | 205 | bool length_never_negative = false, |
duke@435 | 206 | RegionNode* slow_region = NULL); |
duke@435 | 207 | AllocateArrayNode* tightly_coupled_allocation(Node* ptr, |
duke@435 | 208 | RegionNode* slow_region); |
duke@435 | 209 | void generate_clear_array(const TypePtr* adr_type, |
duke@435 | 210 | Node* dest, |
duke@435 | 211 | BasicType basic_elem_type, |
duke@435 | 212 | Node* slice_off, |
duke@435 | 213 | Node* slice_len, |
duke@435 | 214 | Node* slice_end); |
duke@435 | 215 | bool generate_block_arraycopy(const TypePtr* adr_type, |
duke@435 | 216 | BasicType basic_elem_type, |
duke@435 | 217 | AllocateNode* alloc, |
duke@435 | 218 | Node* src, Node* src_offset, |
duke@435 | 219 | Node* dest, Node* dest_offset, |
iveresov@2606 | 220 | Node* dest_size, bool dest_uninitialized); |
duke@435 | 221 | void generate_slow_arraycopy(const TypePtr* adr_type, |
duke@435 | 222 | Node* src, Node* src_offset, |
duke@435 | 223 | Node* dest, Node* dest_offset, |
iveresov@2606 | 224 | Node* copy_length, bool dest_uninitialized); |
duke@435 | 225 | Node* generate_checkcast_arraycopy(const TypePtr* adr_type, |
duke@435 | 226 | Node* dest_elem_klass, |
duke@435 | 227 | Node* src, Node* src_offset, |
duke@435 | 228 | Node* dest, Node* dest_offset, |
iveresov@2606 | 229 | Node* copy_length, bool dest_uninitialized); |
duke@435 | 230 | Node* generate_generic_arraycopy(const TypePtr* adr_type, |
duke@435 | 231 | Node* src, Node* src_offset, |
duke@435 | 232 | Node* dest, Node* dest_offset, |
iveresov@2606 | 233 | Node* copy_length, bool dest_uninitialized); |
duke@435 | 234 | void generate_unchecked_arraycopy(const TypePtr* adr_type, |
duke@435 | 235 | BasicType basic_elem_type, |
duke@435 | 236 | bool disjoint_bases, |
duke@435 | 237 | Node* src, Node* src_offset, |
duke@435 | 238 | Node* dest, Node* dest_offset, |
iveresov@2606 | 239 | Node* copy_length, bool dest_uninitialized); |
duke@435 | 240 | bool inline_unsafe_CAS(BasicType type); |
duke@435 | 241 | bool inline_unsafe_ordered_store(BasicType type); |
duke@435 | 242 | bool inline_fp_conversions(vmIntrinsics::ID id); |
twisti@1210 | 243 | bool inline_numberOfLeadingZeros(vmIntrinsics::ID id); |
twisti@1210 | 244 | bool inline_numberOfTrailingZeros(vmIntrinsics::ID id); |
twisti@1078 | 245 | bool inline_bitCount(vmIntrinsics::ID id); |
duke@435 | 246 | bool inline_reverseBytes(vmIntrinsics::ID id); |
johnc@2781 | 247 | |
johnc@2781 | 248 | bool inline_reference_get(); |
duke@435 | 249 | }; |
duke@435 | 250 | |
duke@435 | 251 | |
duke@435 | 252 | //---------------------------make_vm_intrinsic---------------------------- |
duke@435 | 253 | CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { |
duke@435 | 254 | vmIntrinsics::ID id = m->intrinsic_id(); |
duke@435 | 255 | assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); |
duke@435 | 256 | |
duke@435 | 257 | if (DisableIntrinsic[0] != '\0' |
duke@435 | 258 | && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) { |
duke@435 | 259 | // disabled by a user request on the command line: |
duke@435 | 260 | // example: -XX:DisableIntrinsic=_hashCode,_getClass |
duke@435 | 261 | return NULL; |
duke@435 | 262 | } |
duke@435 | 263 | |
duke@435 | 264 | if (!m->is_loaded()) { |
duke@435 | 265 | // do not attempt to inline unloaded methods |
duke@435 | 266 | return NULL; |
duke@435 | 267 | } |
duke@435 | 268 | |
duke@435 | 269 | // Only a few intrinsics implement a virtual dispatch. |
duke@435 | 270 | // They are expensive calls which are also frequently overridden. |
duke@435 | 271 | if (is_virtual) { |
duke@435 | 272 | switch (id) { |
duke@435 | 273 | case vmIntrinsics::_hashCode: |
duke@435 | 274 | case vmIntrinsics::_clone: |
duke@435 | 275 | // OK, Object.hashCode and Object.clone intrinsics come in both flavors |
duke@435 | 276 | break; |
duke@435 | 277 | default: |
duke@435 | 278 | return NULL; |
duke@435 | 279 | } |
duke@435 | 280 | } |
duke@435 | 281 | |
duke@435 | 282 | // -XX:-InlineNatives disables nearly all intrinsics: |
duke@435 | 283 | if (!InlineNatives) { |
duke@435 | 284 | switch (id) { |
duke@435 | 285 | case vmIntrinsics::_indexOf: |
duke@435 | 286 | case vmIntrinsics::_compareTo: |
cfang@1116 | 287 | case vmIntrinsics::_equals: |
rasbold@604 | 288 | case vmIntrinsics::_equalsC: |
duke@435 | 289 | break; // InlineNatives does not control String.compareTo |
duke@435 | 290 | default: |
duke@435 | 291 | return NULL; |
duke@435 | 292 | } |
duke@435 | 293 | } |
duke@435 | 294 | |
duke@435 | 295 | switch (id) { |
duke@435 | 296 | case vmIntrinsics::_compareTo: |
duke@435 | 297 | if (!SpecialStringCompareTo) return NULL; |
duke@435 | 298 | break; |
duke@435 | 299 | case vmIntrinsics::_indexOf: |
duke@435 | 300 | if (!SpecialStringIndexOf) return NULL; |
duke@435 | 301 | break; |
cfang@1116 | 302 | case vmIntrinsics::_equals: |
cfang@1116 | 303 | if (!SpecialStringEquals) return NULL; |
cfang@1116 | 304 | break; |
rasbold@604 | 305 | case vmIntrinsics::_equalsC: |
rasbold@604 | 306 | if (!SpecialArraysEquals) return NULL; |
rasbold@604 | 307 | break; |
duke@435 | 308 | case vmIntrinsics::_arraycopy: |
duke@435 | 309 | if (!InlineArrayCopy) return NULL; |
duke@435 | 310 | break; |
duke@435 | 311 | case vmIntrinsics::_copyMemory: |
duke@435 | 312 | if (StubRoutines::unsafe_arraycopy() == NULL) return NULL; |
duke@435 | 313 | if (!InlineArrayCopy) return NULL; |
duke@435 | 314 | break; |
duke@435 | 315 | case vmIntrinsics::_hashCode: |
duke@435 | 316 | if (!InlineObjectHash) return NULL; |
duke@435 | 317 | break; |
duke@435 | 318 | case vmIntrinsics::_clone: |
duke@435 | 319 | case vmIntrinsics::_copyOf: |
duke@435 | 320 | case vmIntrinsics::_copyOfRange: |
duke@435 | 321 | if (!InlineObjectCopy) return NULL; |
duke@435 | 322 | // These also use the arraycopy intrinsic mechanism: |
duke@435 | 323 | if (!InlineArrayCopy) return NULL; |
duke@435 | 324 | break; |
duke@435 | 325 | case vmIntrinsics::_checkIndex: |
duke@435 | 326 | // We do not intrinsify this. The optimizer does fine with it. |
duke@435 | 327 | return NULL; |
duke@435 | 328 | |
duke@435 | 329 | case vmIntrinsics::_get_AtomicLong: |
duke@435 | 330 | case vmIntrinsics::_attemptUpdate: |
duke@435 | 331 | if (!InlineAtomicLong) return NULL; |
duke@435 | 332 | break; |
duke@435 | 333 | |
duke@435 | 334 | case vmIntrinsics::_getCallerClass: |
duke@435 | 335 | if (!UseNewReflection) return NULL; |
duke@435 | 336 | if (!InlineReflectionGetCallerClass) return NULL; |
duke@435 | 337 | if (!JDK_Version::is_gte_jdk14x_version()) return NULL; |
duke@435 | 338 | break; |
duke@435 | 339 | |
twisti@1078 | 340 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 341 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 342 | if (!UsePopCountInstruction) return NULL; |
twisti@1078 | 343 | break; |
twisti@1078 | 344 | |
johnc@2781 | 345 | case vmIntrinsics::_Reference_get: |
johnc@2781 | 346 | // It is only when G1 is enabled that we absolutely |
johnc@2781 | 347 | // need to use the intrinsic version of Reference.get() |
johnc@2781 | 348 | // so that the value in the referent field, if necessary, |
johnc@2781 | 349 | // can be registered by the pre-barrier code. |
johnc@2781 | 350 | if (!UseG1GC) return NULL; |
johnc@2781 | 351 | break; |
johnc@2781 | 352 | |
duke@435 | 353 | default: |
jrose@1291 | 354 | assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); |
jrose@1291 | 355 | assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); |
duke@435 | 356 | break; |
duke@435 | 357 | } |
duke@435 | 358 | |
duke@435 | 359 | // -XX:-InlineClassNatives disables natives from the Class class. |
duke@435 | 360 | // The flag applies to all reflective calls, notably Array.newArray |
duke@435 | 361 | // (visible to Java programmers as Array.newInstance). |
duke@435 | 362 | if (m->holder()->name() == ciSymbol::java_lang_Class() || |
duke@435 | 363 | m->holder()->name() == ciSymbol::java_lang_reflect_Array()) { |
duke@435 | 364 | if (!InlineClassNatives) return NULL; |
duke@435 | 365 | } |
duke@435 | 366 | |
duke@435 | 367 | // -XX:-InlineThreadNatives disables natives from the Thread class. |
duke@435 | 368 | if (m->holder()->name() == ciSymbol::java_lang_Thread()) { |
duke@435 | 369 | if (!InlineThreadNatives) return NULL; |
duke@435 | 370 | } |
duke@435 | 371 | |
duke@435 | 372 | // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes. |
duke@435 | 373 | if (m->holder()->name() == ciSymbol::java_lang_Math() || |
duke@435 | 374 | m->holder()->name() == ciSymbol::java_lang_Float() || |
duke@435 | 375 | m->holder()->name() == ciSymbol::java_lang_Double()) { |
duke@435 | 376 | if (!InlineMathNatives) return NULL; |
duke@435 | 377 | } |
duke@435 | 378 | |
duke@435 | 379 | // -XX:-InlineUnsafeOps disables natives from the Unsafe class. |
duke@435 | 380 | if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) { |
duke@435 | 381 | if (!InlineUnsafeOps) return NULL; |
duke@435 | 382 | } |
duke@435 | 383 | |
duke@435 | 384 | return new LibraryIntrinsic(m, is_virtual, (vmIntrinsics::ID) id); |
duke@435 | 385 | } |
duke@435 | 386 | |
duke@435 | 387 | //----------------------register_library_intrinsics----------------------- |
duke@435 | 388 | // Initialize this file's data structures, for each Compile instance. |
duke@435 | 389 | void Compile::register_library_intrinsics() { |
duke@435 | 390 | // Nothing to do here. |
duke@435 | 391 | } |
duke@435 | 392 | |
duke@435 | 393 | JVMState* LibraryIntrinsic::generate(JVMState* jvms) { |
duke@435 | 394 | LibraryCallKit kit(jvms, this); |
duke@435 | 395 | Compile* C = kit.C; |
duke@435 | 396 | int nodes = C->unique(); |
duke@435 | 397 | #ifndef PRODUCT |
duke@435 | 398 | if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) { |
duke@435 | 399 | char buf[1000]; |
duke@435 | 400 | const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); |
duke@435 | 401 | tty->print_cr("Intrinsic %s", str); |
duke@435 | 402 | } |
duke@435 | 403 | #endif |
johnc@2781 | 404 | |
duke@435 | 405 | if (kit.try_to_inline()) { |
duke@435 | 406 | if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
twisti@2687 | 407 | CompileTask::print_inlining(kit.callee(), jvms->depth() - 1, kit.bci(), is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"); |
duke@435 | 408 | } |
duke@435 | 409 | C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
duke@435 | 410 | if (C->log()) { |
duke@435 | 411 | C->log()->elem("intrinsic id='%s'%s nodes='%d'", |
duke@435 | 412 | vmIntrinsics::name_at(intrinsic_id()), |
duke@435 | 413 | (is_virtual() ? " virtual='1'" : ""), |
duke@435 | 414 | C->unique() - nodes); |
duke@435 | 415 | } |
duke@435 | 416 | return kit.transfer_exceptions_into_jvms(); |
duke@435 | 417 | } |
duke@435 | 418 | |
duke@435 | 419 | if (PrintIntrinsics) { |
johnc@2781 | 420 | if (jvms->has_method()) { |
johnc@2781 | 421 | // Not a root compile. |
johnc@2781 | 422 | tty->print("Did not inline intrinsic %s%s at bci:%d in", |
johnc@2781 | 423 | vmIntrinsics::name_at(intrinsic_id()), |
johnc@2781 | 424 | (is_virtual() ? " (virtual)" : ""), kit.bci()); |
johnc@2781 | 425 | kit.caller()->print_short_name(tty); |
johnc@2781 | 426 | tty->print_cr(" (%d bytes)", kit.caller()->code_size()); |
johnc@2781 | 427 | } else { |
johnc@2781 | 428 | // Root compile |
johnc@2781 | 429 | tty->print("Did not generate intrinsic %s%s at bci:%d in", |
jrose@1291 | 430 | vmIntrinsics::name_at(intrinsic_id()), |
jrose@1291 | 431 | (is_virtual() ? " (virtual)" : ""), kit.bci()); |
johnc@2781 | 432 | } |
duke@435 | 433 | } |
duke@435 | 434 | C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); |
duke@435 | 435 | return NULL; |
duke@435 | 436 | } |
duke@435 | 437 | |
duke@435 | 438 | bool LibraryCallKit::try_to_inline() { |
duke@435 | 439 | // Handle symbolic names for otherwise undistinguished boolean switches: |
duke@435 | 440 | const bool is_store = true; |
duke@435 | 441 | const bool is_native_ptr = true; |
duke@435 | 442 | const bool is_static = true; |
duke@435 | 443 | |
johnc@2781 | 444 | if (!jvms()->has_method()) { |
johnc@2781 | 445 | // Root JVMState has a null method. |
johnc@2781 | 446 | assert(map()->memory()->Opcode() == Op_Parm, ""); |
johnc@2781 | 447 | // Insert the memory aliasing node |
johnc@2781 | 448 | set_all_memory(reset_memory()); |
johnc@2781 | 449 | } |
johnc@2781 | 450 | assert(merged_memory(), ""); |
johnc@2781 | 451 | |
duke@435 | 452 | switch (intrinsic_id()) { |
duke@435 | 453 | case vmIntrinsics::_hashCode: |
duke@435 | 454 | return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); |
duke@435 | 455 | case vmIntrinsics::_identityHashCode: |
duke@435 | 456 | return inline_native_hashcode(/*!virtual*/ false, is_static); |
duke@435 | 457 | case vmIntrinsics::_getClass: |
duke@435 | 458 | return inline_native_getClass(); |
duke@435 | 459 | |
duke@435 | 460 | case vmIntrinsics::_dsin: |
duke@435 | 461 | case vmIntrinsics::_dcos: |
duke@435 | 462 | case vmIntrinsics::_dtan: |
duke@435 | 463 | case vmIntrinsics::_dabs: |
duke@435 | 464 | case vmIntrinsics::_datan2: |
duke@435 | 465 | case vmIntrinsics::_dsqrt: |
duke@435 | 466 | case vmIntrinsics::_dexp: |
duke@435 | 467 | case vmIntrinsics::_dlog: |
duke@435 | 468 | case vmIntrinsics::_dlog10: |
duke@435 | 469 | case vmIntrinsics::_dpow: |
duke@435 | 470 | return inline_math_native(intrinsic_id()); |
duke@435 | 471 | |
duke@435 | 472 | case vmIntrinsics::_min: |
duke@435 | 473 | case vmIntrinsics::_max: |
duke@435 | 474 | return inline_min_max(intrinsic_id()); |
duke@435 | 475 | |
duke@435 | 476 | case vmIntrinsics::_arraycopy: |
duke@435 | 477 | return inline_arraycopy(); |
duke@435 | 478 | |
duke@435 | 479 | case vmIntrinsics::_compareTo: |
duke@435 | 480 | return inline_string_compareTo(); |
duke@435 | 481 | case vmIntrinsics::_indexOf: |
duke@435 | 482 | return inline_string_indexOf(); |
cfang@1116 | 483 | case vmIntrinsics::_equals: |
cfang@1116 | 484 | return inline_string_equals(); |
duke@435 | 485 | |
duke@435 | 486 | case vmIntrinsics::_getObject: |
duke@435 | 487 | return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, false); |
duke@435 | 488 | case vmIntrinsics::_getBoolean: |
duke@435 | 489 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, false); |
duke@435 | 490 | case vmIntrinsics::_getByte: |
duke@435 | 491 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, false); |
duke@435 | 492 | case vmIntrinsics::_getShort: |
duke@435 | 493 | return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, false); |
duke@435 | 494 | case vmIntrinsics::_getChar: |
duke@435 | 495 | return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, false); |
duke@435 | 496 | case vmIntrinsics::_getInt: |
duke@435 | 497 | return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, false); |
duke@435 | 498 | case vmIntrinsics::_getLong: |
duke@435 | 499 | return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, false); |
duke@435 | 500 | case vmIntrinsics::_getFloat: |
duke@435 | 501 | return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, false); |
duke@435 | 502 | case vmIntrinsics::_getDouble: |
duke@435 | 503 | return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, false); |
duke@435 | 504 | |
duke@435 | 505 | case vmIntrinsics::_putObject: |
duke@435 | 506 | return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, false); |
duke@435 | 507 | case vmIntrinsics::_putBoolean: |
duke@435 | 508 | return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, false); |
duke@435 | 509 | case vmIntrinsics::_putByte: |
duke@435 | 510 | return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, false); |
duke@435 | 511 | case vmIntrinsics::_putShort: |
duke@435 | 512 | return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, false); |
duke@435 | 513 | case vmIntrinsics::_putChar: |
duke@435 | 514 | return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, false); |
duke@435 | 515 | case vmIntrinsics::_putInt: |
duke@435 | 516 | return inline_unsafe_access(!is_native_ptr, is_store, T_INT, false); |
duke@435 | 517 | case vmIntrinsics::_putLong: |
duke@435 | 518 | return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, false); |
duke@435 | 519 | case vmIntrinsics::_putFloat: |
duke@435 | 520 | return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, false); |
duke@435 | 521 | case vmIntrinsics::_putDouble: |
duke@435 | 522 | return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, false); |
duke@435 | 523 | |
duke@435 | 524 | case vmIntrinsics::_getByte_raw: |
duke@435 | 525 | return inline_unsafe_access(is_native_ptr, !is_store, T_BYTE, false); |
duke@435 | 526 | case vmIntrinsics::_getShort_raw: |
duke@435 | 527 | return inline_unsafe_access(is_native_ptr, !is_store, T_SHORT, false); |
duke@435 | 528 | case vmIntrinsics::_getChar_raw: |
duke@435 | 529 | return inline_unsafe_access(is_native_ptr, !is_store, T_CHAR, false); |
duke@435 | 530 | case vmIntrinsics::_getInt_raw: |
duke@435 | 531 | return inline_unsafe_access(is_native_ptr, !is_store, T_INT, false); |
duke@435 | 532 | case vmIntrinsics::_getLong_raw: |
duke@435 | 533 | return inline_unsafe_access(is_native_ptr, !is_store, T_LONG, false); |
duke@435 | 534 | case vmIntrinsics::_getFloat_raw: |
duke@435 | 535 | return inline_unsafe_access(is_native_ptr, !is_store, T_FLOAT, false); |
duke@435 | 536 | case vmIntrinsics::_getDouble_raw: |
duke@435 | 537 | return inline_unsafe_access(is_native_ptr, !is_store, T_DOUBLE, false); |
duke@435 | 538 | case vmIntrinsics::_getAddress_raw: |
duke@435 | 539 | return inline_unsafe_access(is_native_ptr, !is_store, T_ADDRESS, false); |
duke@435 | 540 | |
duke@435 | 541 | case vmIntrinsics::_putByte_raw: |
duke@435 | 542 | return inline_unsafe_access(is_native_ptr, is_store, T_BYTE, false); |
duke@435 | 543 | case vmIntrinsics::_putShort_raw: |
duke@435 | 544 | return inline_unsafe_access(is_native_ptr, is_store, T_SHORT, false); |
duke@435 | 545 | case vmIntrinsics::_putChar_raw: |
duke@435 | 546 | return inline_unsafe_access(is_native_ptr, is_store, T_CHAR, false); |
duke@435 | 547 | case vmIntrinsics::_putInt_raw: |
duke@435 | 548 | return inline_unsafe_access(is_native_ptr, is_store, T_INT, false); |
duke@435 | 549 | case vmIntrinsics::_putLong_raw: |
duke@435 | 550 | return inline_unsafe_access(is_native_ptr, is_store, T_LONG, false); |
duke@435 | 551 | case vmIntrinsics::_putFloat_raw: |
duke@435 | 552 | return inline_unsafe_access(is_native_ptr, is_store, T_FLOAT, false); |
duke@435 | 553 | case vmIntrinsics::_putDouble_raw: |
duke@435 | 554 | return inline_unsafe_access(is_native_ptr, is_store, T_DOUBLE, false); |
duke@435 | 555 | case vmIntrinsics::_putAddress_raw: |
duke@435 | 556 | return inline_unsafe_access(is_native_ptr, is_store, T_ADDRESS, false); |
duke@435 | 557 | |
duke@435 | 558 | case vmIntrinsics::_getObjectVolatile: |
duke@435 | 559 | return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, true); |
duke@435 | 560 | case vmIntrinsics::_getBooleanVolatile: |
duke@435 | 561 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, true); |
duke@435 | 562 | case vmIntrinsics::_getByteVolatile: |
duke@435 | 563 | return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, true); |
duke@435 | 564 | case vmIntrinsics::_getShortVolatile: |
duke@435 | 565 | return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, true); |
duke@435 | 566 | case vmIntrinsics::_getCharVolatile: |
duke@435 | 567 | return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, true); |
duke@435 | 568 | case vmIntrinsics::_getIntVolatile: |
duke@435 | 569 | return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, true); |
duke@435 | 570 | case vmIntrinsics::_getLongVolatile: |
duke@435 | 571 | return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, true); |
duke@435 | 572 | case vmIntrinsics::_getFloatVolatile: |
duke@435 | 573 | return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, true); |
duke@435 | 574 | case vmIntrinsics::_getDoubleVolatile: |
duke@435 | 575 | return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, true); |
duke@435 | 576 | |
duke@435 | 577 | case vmIntrinsics::_putObjectVolatile: |
duke@435 | 578 | return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, true); |
duke@435 | 579 | case vmIntrinsics::_putBooleanVolatile: |
duke@435 | 580 | return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, true); |
duke@435 | 581 | case vmIntrinsics::_putByteVolatile: |
duke@435 | 582 | return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, true); |
duke@435 | 583 | case vmIntrinsics::_putShortVolatile: |
duke@435 | 584 | return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, true); |
duke@435 | 585 | case vmIntrinsics::_putCharVolatile: |
duke@435 | 586 | return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, true); |
duke@435 | 587 | case vmIntrinsics::_putIntVolatile: |
duke@435 | 588 | return inline_unsafe_access(!is_native_ptr, is_store, T_INT, true); |
duke@435 | 589 | case vmIntrinsics::_putLongVolatile: |
duke@435 | 590 | return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, true); |
duke@435 | 591 | case vmIntrinsics::_putFloatVolatile: |
duke@435 | 592 | return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, true); |
duke@435 | 593 | case vmIntrinsics::_putDoubleVolatile: |
duke@435 | 594 | return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, true); |
duke@435 | 595 | |
duke@435 | 596 | case vmIntrinsics::_prefetchRead: |
duke@435 | 597 | return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static); |
duke@435 | 598 | case vmIntrinsics::_prefetchWrite: |
duke@435 | 599 | return inline_unsafe_prefetch(!is_native_ptr, is_store, !is_static); |
duke@435 | 600 | case vmIntrinsics::_prefetchReadStatic: |
duke@435 | 601 | return inline_unsafe_prefetch(!is_native_ptr, !is_store, is_static); |
duke@435 | 602 | case vmIntrinsics::_prefetchWriteStatic: |
duke@435 | 603 | return inline_unsafe_prefetch(!is_native_ptr, is_store, is_static); |
duke@435 | 604 | |
duke@435 | 605 | case vmIntrinsics::_compareAndSwapObject: |
duke@435 | 606 | return inline_unsafe_CAS(T_OBJECT); |
duke@435 | 607 | case vmIntrinsics::_compareAndSwapInt: |
duke@435 | 608 | return inline_unsafe_CAS(T_INT); |
duke@435 | 609 | case vmIntrinsics::_compareAndSwapLong: |
duke@435 | 610 | return inline_unsafe_CAS(T_LONG); |
duke@435 | 611 | |
duke@435 | 612 | case vmIntrinsics::_putOrderedObject: |
duke@435 | 613 | return inline_unsafe_ordered_store(T_OBJECT); |
duke@435 | 614 | case vmIntrinsics::_putOrderedInt: |
duke@435 | 615 | return inline_unsafe_ordered_store(T_INT); |
duke@435 | 616 | case vmIntrinsics::_putOrderedLong: |
duke@435 | 617 | return inline_unsafe_ordered_store(T_LONG); |
duke@435 | 618 | |
duke@435 | 619 | case vmIntrinsics::_currentThread: |
duke@435 | 620 | return inline_native_currentThread(); |
duke@435 | 621 | case vmIntrinsics::_isInterrupted: |
duke@435 | 622 | return inline_native_isInterrupted(); |
duke@435 | 623 | |
duke@435 | 624 | case vmIntrinsics::_currentTimeMillis: |
duke@435 | 625 | return inline_native_time_funcs(false); |
duke@435 | 626 | case vmIntrinsics::_nanoTime: |
duke@435 | 627 | return inline_native_time_funcs(true); |
duke@435 | 628 | case vmIntrinsics::_allocateInstance: |
duke@435 | 629 | return inline_unsafe_allocate(); |
duke@435 | 630 | case vmIntrinsics::_copyMemory: |
duke@435 | 631 | return inline_unsafe_copyMemory(); |
duke@435 | 632 | case vmIntrinsics::_newArray: |
duke@435 | 633 | return inline_native_newArray(); |
duke@435 | 634 | case vmIntrinsics::_getLength: |
duke@435 | 635 | return inline_native_getLength(); |
duke@435 | 636 | case vmIntrinsics::_copyOf: |
duke@435 | 637 | return inline_array_copyOf(false); |
duke@435 | 638 | case vmIntrinsics::_copyOfRange: |
duke@435 | 639 | return inline_array_copyOf(true); |
rasbold@604 | 640 | case vmIntrinsics::_equalsC: |
rasbold@604 | 641 | return inline_array_equals(); |
duke@435 | 642 | case vmIntrinsics::_clone: |
duke@435 | 643 | return inline_native_clone(intrinsic()->is_virtual()); |
duke@435 | 644 | |
duke@435 | 645 | case vmIntrinsics::_isAssignableFrom: |
duke@435 | 646 | return inline_native_subtype_check(); |
duke@435 | 647 | |
duke@435 | 648 | case vmIntrinsics::_isInstance: |
duke@435 | 649 | case vmIntrinsics::_getModifiers: |
duke@435 | 650 | case vmIntrinsics::_isInterface: |
duke@435 | 651 | case vmIntrinsics::_isArray: |
duke@435 | 652 | case vmIntrinsics::_isPrimitive: |
duke@435 | 653 | case vmIntrinsics::_getSuperclass: |
duke@435 | 654 | case vmIntrinsics::_getComponentType: |
duke@435 | 655 | case vmIntrinsics::_getClassAccessFlags: |
duke@435 | 656 | return inline_native_Class_query(intrinsic_id()); |
duke@435 | 657 | |
duke@435 | 658 | case vmIntrinsics::_floatToRawIntBits: |
duke@435 | 659 | case vmIntrinsics::_floatToIntBits: |
duke@435 | 660 | case vmIntrinsics::_intBitsToFloat: |
duke@435 | 661 | case vmIntrinsics::_doubleToRawLongBits: |
duke@435 | 662 | case vmIntrinsics::_doubleToLongBits: |
duke@435 | 663 | case vmIntrinsics::_longBitsToDouble: |
duke@435 | 664 | return inline_fp_conversions(intrinsic_id()); |
duke@435 | 665 | |
twisti@1210 | 666 | case vmIntrinsics::_numberOfLeadingZeros_i: |
twisti@1210 | 667 | case vmIntrinsics::_numberOfLeadingZeros_l: |
twisti@1210 | 668 | return inline_numberOfLeadingZeros(intrinsic_id()); |
twisti@1210 | 669 | |
twisti@1210 | 670 | case vmIntrinsics::_numberOfTrailingZeros_i: |
twisti@1210 | 671 | case vmIntrinsics::_numberOfTrailingZeros_l: |
twisti@1210 | 672 | return inline_numberOfTrailingZeros(intrinsic_id()); |
twisti@1210 | 673 | |
twisti@1078 | 674 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 675 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 676 | return inline_bitCount(intrinsic_id()); |
twisti@1078 | 677 | |
duke@435 | 678 | case vmIntrinsics::_reverseBytes_i: |
duke@435 | 679 | case vmIntrinsics::_reverseBytes_l: |
never@1831 | 680 | case vmIntrinsics::_reverseBytes_s: |
never@1831 | 681 | case vmIntrinsics::_reverseBytes_c: |
duke@435 | 682 | return inline_reverseBytes((vmIntrinsics::ID) intrinsic_id()); |
duke@435 | 683 | |
duke@435 | 684 | case vmIntrinsics::_get_AtomicLong: |
duke@435 | 685 | return inline_native_AtomicLong_get(); |
duke@435 | 686 | case vmIntrinsics::_attemptUpdate: |
duke@435 | 687 | return inline_native_AtomicLong_attemptUpdate(); |
duke@435 | 688 | |
duke@435 | 689 | case vmIntrinsics::_getCallerClass: |
duke@435 | 690 | return inline_native_Reflection_getCallerClass(); |
duke@435 | 691 | |
johnc@2781 | 692 | case vmIntrinsics::_Reference_get: |
johnc@2781 | 693 | return inline_reference_get(); |
johnc@2781 | 694 | |
duke@435 | 695 | default: |
duke@435 | 696 | // If you get here, it may be that someone has added a new intrinsic |
duke@435 | 697 | // to the list in vmSymbols.hpp without implementing it here. |
duke@435 | 698 | #ifndef PRODUCT |
duke@435 | 699 | if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { |
duke@435 | 700 | tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)", |
duke@435 | 701 | vmIntrinsics::name_at(intrinsic_id()), intrinsic_id()); |
duke@435 | 702 | } |
duke@435 | 703 | #endif |
duke@435 | 704 | return false; |
duke@435 | 705 | } |
duke@435 | 706 | } |
duke@435 | 707 | |
duke@435 | 708 | //------------------------------push_result------------------------------ |
duke@435 | 709 | // Helper function for finishing intrinsics. |
duke@435 | 710 | void LibraryCallKit::push_result(RegionNode* region, PhiNode* value) { |
duke@435 | 711 | record_for_igvn(region); |
duke@435 | 712 | set_control(_gvn.transform(region)); |
duke@435 | 713 | BasicType value_type = value->type()->basic_type(); |
duke@435 | 714 | push_node(value_type, _gvn.transform(value)); |
duke@435 | 715 | } |
duke@435 | 716 | |
duke@435 | 717 | //------------------------------generate_guard--------------------------- |
duke@435 | 718 | // Helper function for generating guarded fast-slow graph structures. |
duke@435 | 719 | // The given 'test', if true, guards a slow path. If the test fails |
duke@435 | 720 | // then a fast path can be taken. (We generally hope it fails.) |
duke@435 | 721 | // In all cases, GraphKit::control() is updated to the fast path. |
duke@435 | 722 | // The returned value represents the control for the slow path. |
duke@435 | 723 | // The return value is never 'top'; it is either a valid control |
duke@435 | 724 | // or NULL if it is obvious that the slow path can never be taken. |
duke@435 | 725 | // Also, if region and the slow control are not NULL, the slow edge |
duke@435 | 726 | // is appended to the region. |
duke@435 | 727 | Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { |
duke@435 | 728 | if (stopped()) { |
duke@435 | 729 | // Already short circuited. |
duke@435 | 730 | return NULL; |
duke@435 | 731 | } |
duke@435 | 732 | |
duke@435 | 733 | // Build an if node and its projections. |
duke@435 | 734 | // If test is true we take the slow path, which we assume is uncommon. |
duke@435 | 735 | if (_gvn.type(test) == TypeInt::ZERO) { |
duke@435 | 736 | // The slow branch is never taken. No need to build this guard. |
duke@435 | 737 | return NULL; |
duke@435 | 738 | } |
duke@435 | 739 | |
duke@435 | 740 | IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); |
duke@435 | 741 | |
duke@435 | 742 | Node* if_slow = _gvn.transform( new (C, 1) IfTrueNode(iff) ); |
duke@435 | 743 | if (if_slow == top()) { |
duke@435 | 744 | // The slow branch is never taken. No need to build this guard. |
duke@435 | 745 | return NULL; |
duke@435 | 746 | } |
duke@435 | 747 | |
duke@435 | 748 | if (region != NULL) |
duke@435 | 749 | region->add_req(if_slow); |
duke@435 | 750 | |
duke@435 | 751 | Node* if_fast = _gvn.transform( new (C, 1) IfFalseNode(iff) ); |
duke@435 | 752 | set_control(if_fast); |
duke@435 | 753 | |
duke@435 | 754 | return if_slow; |
duke@435 | 755 | } |
duke@435 | 756 | |
duke@435 | 757 | inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) { |
duke@435 | 758 | return generate_guard(test, region, PROB_UNLIKELY_MAG(3)); |
duke@435 | 759 | } |
duke@435 | 760 | inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) { |
duke@435 | 761 | return generate_guard(test, region, PROB_FAIR); |
duke@435 | 762 | } |
duke@435 | 763 | |
duke@435 | 764 | inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, |
duke@435 | 765 | Node* *pos_index) { |
duke@435 | 766 | if (stopped()) |
duke@435 | 767 | return NULL; // already stopped |
duke@435 | 768 | if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] |
duke@435 | 769 | return NULL; // index is already adequately typed |
duke@435 | 770 | Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) ); |
duke@435 | 771 | Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 772 | Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); |
duke@435 | 773 | if (is_neg != NULL && pos_index != NULL) { |
duke@435 | 774 | // Emulate effect of Parse::adjust_map_after_if. |
duke@435 | 775 | Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS); |
duke@435 | 776 | ccast->set_req(0, control()); |
duke@435 | 777 | (*pos_index) = _gvn.transform(ccast); |
duke@435 | 778 | } |
duke@435 | 779 | return is_neg; |
duke@435 | 780 | } |
duke@435 | 781 | |
duke@435 | 782 | inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative, |
duke@435 | 783 | Node* *pos_index) { |
duke@435 | 784 | if (stopped()) |
duke@435 | 785 | return NULL; // already stopped |
duke@435 | 786 | if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint] |
duke@435 | 787 | return NULL; // index is already adequately typed |
duke@435 | 788 | Node* cmp_le = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) ); |
duke@435 | 789 | BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le); |
duke@435 | 790 | Node* bol_le = _gvn.transform( new (C, 2) BoolNode(cmp_le, le_or_eq) ); |
duke@435 | 791 | Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN); |
duke@435 | 792 | if (is_notp != NULL && pos_index != NULL) { |
duke@435 | 793 | // Emulate effect of Parse::adjust_map_after_if. |
duke@435 | 794 | Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS1); |
duke@435 | 795 | ccast->set_req(0, control()); |
duke@435 | 796 | (*pos_index) = _gvn.transform(ccast); |
duke@435 | 797 | } |
duke@435 | 798 | return is_notp; |
duke@435 | 799 | } |
duke@435 | 800 | |
duke@435 | 801 | // Make sure that 'position' is a valid limit index, in [0..length]. |
duke@435 | 802 | // There are two equivalent plans for checking this: |
duke@435 | 803 | // A. (offset + copyLength) unsigned<= arrayLength |
duke@435 | 804 | // B. offset <= (arrayLength - copyLength) |
duke@435 | 805 | // We require that all of the values above, except for the sum and |
duke@435 | 806 | // difference, are already known to be non-negative. |
duke@435 | 807 | // Plan A is robust in the face of overflow, if offset and copyLength |
duke@435 | 808 | // are both hugely positive. |
duke@435 | 809 | // |
duke@435 | 810 | // Plan B is less direct and intuitive, but it does not overflow at |
duke@435 | 811 | // all, since the difference of two non-negatives is always |
duke@435 | 812 | // representable. Whenever Java methods must perform the equivalent |
duke@435 | 813 | // check they generally use Plan B instead of Plan A. |
duke@435 | 814 | // For the moment we use Plan A. |
duke@435 | 815 | inline Node* LibraryCallKit::generate_limit_guard(Node* offset, |
duke@435 | 816 | Node* subseq_length, |
duke@435 | 817 | Node* array_length, |
duke@435 | 818 | RegionNode* region) { |
duke@435 | 819 | if (stopped()) |
duke@435 | 820 | return NULL; // already stopped |
duke@435 | 821 | bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; |
kvn@3407 | 822 | if (zero_offset && subseq_length->eqv_uncast(array_length)) |
duke@435 | 823 | return NULL; // common case of whole-array copy |
duke@435 | 824 | Node* last = subseq_length; |
duke@435 | 825 | if (!zero_offset) // last += offset |
duke@435 | 826 | last = _gvn.transform( new (C, 3) AddINode(last, offset)); |
duke@435 | 827 | Node* cmp_lt = _gvn.transform( new (C, 3) CmpUNode(array_length, last) ); |
duke@435 | 828 | Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 829 | Node* is_over = generate_guard(bol_lt, region, PROB_MIN); |
duke@435 | 830 | return is_over; |
duke@435 | 831 | } |
duke@435 | 832 | |
duke@435 | 833 | |
duke@435 | 834 | //--------------------------generate_current_thread-------------------- |
duke@435 | 835 | Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { |
duke@435 | 836 | ciKlass* thread_klass = env()->Thread_klass(); |
duke@435 | 837 | const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); |
duke@435 | 838 | Node* thread = _gvn.transform(new (C, 1) ThreadLocalNode()); |
duke@435 | 839 | Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset())); |
duke@435 | 840 | Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT); |
duke@435 | 841 | tls_output = thread; |
duke@435 | 842 | return threadObj; |
duke@435 | 843 | } |
duke@435 | 844 | |
duke@435 | 845 | |
kvn@1421 | 846 | //------------------------------make_string_method_node------------------------ |
kvn@1421 | 847 | // Helper method for String intrinsic finctions. |
kvn@1421 | 848 | Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2) { |
kvn@1421 | 849 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
kvn@1421 | 850 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
kvn@1421 | 851 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
kvn@1421 | 852 | |
kvn@1421 | 853 | Node* no_ctrl = NULL; |
kvn@1421 | 854 | |
kvn@1421 | 855 | ciInstanceKlass* klass = env()->String_klass(); |
never@1851 | 856 | const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass); |
kvn@1421 | 857 | |
kvn@1421 | 858 | const TypeAryPtr* value_type = |
kvn@1421 | 859 | TypeAryPtr::make(TypePtr::NotNull, |
kvn@1421 | 860 | TypeAry::make(TypeInt::CHAR,TypeInt::POS), |
kvn@1421 | 861 | ciTypeArrayKlass::make(T_CHAR), true, 0); |
kvn@1421 | 862 | |
kvn@1421 | 863 | // Get start addr of string and substring |
kvn@1421 | 864 | Node* str1_valuea = basic_plus_adr(str1, str1, value_offset); |
kvn@1421 | 865 | Node* str1_value = make_load(no_ctrl, str1_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset)); |
kvn@1421 | 866 | Node* str1_offseta = basic_plus_adr(str1, str1, offset_offset); |
kvn@1421 | 867 | Node* str1_offset = make_load(no_ctrl, str1_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
kvn@1421 | 868 | Node* str1_start = array_element_address(str1_value, str1_offset, T_CHAR); |
kvn@1421 | 869 | |
kvn@1421 | 870 | Node* str2_valuea = basic_plus_adr(str2, str2, value_offset); |
kvn@2869 | 871 | Node* str2_value = make_load(no_ctrl, str2_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset)); |
kvn@1421 | 872 | Node* str2_offseta = basic_plus_adr(str2, str2, offset_offset); |
kvn@2869 | 873 | Node* str2_offset = make_load(no_ctrl, str2_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
kvn@1421 | 874 | Node* str2_start = array_element_address(str2_value, str2_offset, T_CHAR); |
kvn@1421 | 875 | |
kvn@1421 | 876 | Node* result = NULL; |
kvn@1421 | 877 | switch (opcode) { |
kvn@1421 | 878 | case Op_StrIndexOf: |
kvn@1421 | 879 | result = new (C, 6) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 880 | str1_start, cnt1, str2_start, cnt2); |
kvn@1421 | 881 | break; |
kvn@1421 | 882 | case Op_StrComp: |
kvn@1421 | 883 | result = new (C, 6) StrCompNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 884 | str1_start, cnt1, str2_start, cnt2); |
kvn@1421 | 885 | break; |
kvn@1421 | 886 | case Op_StrEquals: |
kvn@1421 | 887 | result = new (C, 5) StrEqualsNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 888 | str1_start, str2_start, cnt1); |
kvn@1421 | 889 | break; |
kvn@1421 | 890 | default: |
kvn@1421 | 891 | ShouldNotReachHere(); |
kvn@1421 | 892 | return NULL; |
kvn@1421 | 893 | } |
kvn@1421 | 894 | |
kvn@1421 | 895 | // All these intrinsics have checks. |
kvn@1421 | 896 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
kvn@1421 | 897 | |
kvn@1421 | 898 | return _gvn.transform(result); |
kvn@1421 | 899 | } |
kvn@1421 | 900 | |
duke@435 | 901 | //------------------------------inline_string_compareTo------------------------ |
duke@435 | 902 | bool LibraryCallKit::inline_string_compareTo() { |
duke@435 | 903 | |
cfang@1116 | 904 | if (!Matcher::has_match_rule(Op_StrComp)) return false; |
cfang@1116 | 905 | |
duke@435 | 906 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
duke@435 | 907 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
duke@435 | 908 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
duke@435 | 909 | |
duke@435 | 910 | _sp += 2; |
duke@435 | 911 | Node *argument = pop(); // pop non-receiver first: it was pushed second |
duke@435 | 912 | Node *receiver = pop(); |
duke@435 | 913 | |
duke@435 | 914 | // Null check on self without removing any arguments. The argument |
duke@435 | 915 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 916 | // invalid stack traces when string compare is inlined into a method |
duke@435 | 917 | // which handles NullPointerExceptions. |
duke@435 | 918 | _sp += 2; |
duke@435 | 919 | receiver = do_null_check(receiver, T_OBJECT); |
duke@435 | 920 | argument = do_null_check(argument, T_OBJECT); |
duke@435 | 921 | _sp -= 2; |
duke@435 | 922 | if (stopped()) { |
duke@435 | 923 | return true; |
duke@435 | 924 | } |
duke@435 | 925 | |
duke@435 | 926 | ciInstanceKlass* klass = env()->String_klass(); |
never@1851 | 927 | const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass); |
kvn@1421 | 928 | Node* no_ctrl = NULL; |
kvn@1421 | 929 | |
kvn@1421 | 930 | // Get counts for string and argument |
kvn@1421 | 931 | Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 932 | Node* receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 933 | |
kvn@1421 | 934 | Node* argument_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@1421 | 935 | Node* argument_cnt = make_load(no_ctrl, argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 936 | |
kvn@1421 | 937 | Node* compare = make_string_method_node(Op_StrComp, receiver, receiver_cnt, argument, argument_cnt); |
duke@435 | 938 | push(compare); |
duke@435 | 939 | return true; |
duke@435 | 940 | } |
duke@435 | 941 | |
cfang@1116 | 942 | //------------------------------inline_string_equals------------------------ |
cfang@1116 | 943 | bool LibraryCallKit::inline_string_equals() { |
cfang@1116 | 944 | |
cfang@1116 | 945 | if (!Matcher::has_match_rule(Op_StrEquals)) return false; |
cfang@1116 | 946 | |
cfang@1116 | 947 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
cfang@1116 | 948 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
cfang@1116 | 949 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
cfang@1116 | 950 | |
jrose@2101 | 951 | int nargs = 2; |
jrose@2101 | 952 | _sp += nargs; |
cfang@1116 | 953 | Node* argument = pop(); // pop non-receiver first: it was pushed second |
cfang@1116 | 954 | Node* receiver = pop(); |
cfang@1116 | 955 | |
cfang@1116 | 956 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 957 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 958 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 959 | // which handles NullPointerExceptions. |
jrose@2101 | 960 | _sp += nargs; |
cfang@1116 | 961 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 962 | //should not do null check for argument for String.equals(), because spec |
cfang@1116 | 963 | //allows to specify NULL as argument. |
jrose@2101 | 964 | _sp -= nargs; |
cfang@1116 | 965 | |
cfang@1116 | 966 | if (stopped()) { |
cfang@1116 | 967 | return true; |
cfang@1116 | 968 | } |
cfang@1116 | 969 | |
kvn@1421 | 970 | // paths (plus control) merge |
kvn@1421 | 971 | RegionNode* region = new (C, 5) RegionNode(5); |
kvn@1421 | 972 | Node* phi = new (C, 5) PhiNode(region, TypeInt::BOOL); |
kvn@1421 | 973 | |
kvn@1421 | 974 | // does source == target string? |
kvn@1421 | 975 | Node* cmp = _gvn.transform(new (C, 3) CmpPNode(receiver, argument)); |
kvn@1421 | 976 | Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::eq)); |
kvn@1421 | 977 | |
kvn@1421 | 978 | Node* if_eq = generate_slow_guard(bol, NULL); |
kvn@1421 | 979 | if (if_eq != NULL) { |
kvn@1421 | 980 | // receiver == argument |
kvn@1421 | 981 | phi->init_req(2, intcon(1)); |
kvn@1421 | 982 | region->init_req(2, if_eq); |
kvn@1421 | 983 | } |
kvn@1421 | 984 | |
cfang@1116 | 985 | // get String klass for instanceOf |
cfang@1116 | 986 | ciInstanceKlass* klass = env()->String_klass(); |
cfang@1116 | 987 | |
kvn@1421 | 988 | if (!stopped()) { |
jrose@2101 | 989 | _sp += nargs; // gen_instanceof might do an uncommon trap |
kvn@1421 | 990 | Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass))); |
jrose@2101 | 991 | _sp -= nargs; |
kvn@1421 | 992 | Node* cmp = _gvn.transform(new (C, 3) CmpINode(inst, intcon(1))); |
kvn@1421 | 993 | Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::ne)); |
kvn@1421 | 994 | |
kvn@1421 | 995 | Node* inst_false = generate_guard(bol, NULL, PROB_MIN); |
kvn@1421 | 996 | //instanceOf == true, fallthrough |
kvn@1421 | 997 | |
kvn@1421 | 998 | if (inst_false != NULL) { |
kvn@1421 | 999 | phi->init_req(3, intcon(0)); |
kvn@1421 | 1000 | region->init_req(3, inst_false); |
kvn@1421 | 1001 | } |
kvn@1421 | 1002 | } |
cfang@1116 | 1003 | |
never@1851 | 1004 | const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass); |
cfang@1116 | 1005 | |
kvn@1421 | 1006 | Node* no_ctrl = NULL; |
kvn@1421 | 1007 | Node* receiver_cnt; |
kvn@1421 | 1008 | Node* argument_cnt; |
kvn@1421 | 1009 | |
kvn@1421 | 1010 | if (!stopped()) { |
never@1851 | 1011 | // Properly cast the argument to String |
never@1851 | 1012 | argument = _gvn.transform(new (C, 2) CheckCastPPNode(control(), argument, string_type)); |
kvn@2869 | 1013 | // This path is taken only when argument's type is String:NotNull. |
kvn@2869 | 1014 | argument = cast_not_null(argument, false); |
never@1851 | 1015 | |
kvn@1421 | 1016 | // Get counts for string and argument |
kvn@1421 | 1017 | Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 1018 | receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1019 | |
kvn@1421 | 1020 | Node* argument_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@2869 | 1021 | argument_cnt = make_load(no_ctrl, argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1022 | |
kvn@1421 | 1023 | // Check for receiver count != argument count |
kvn@1421 | 1024 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(receiver_cnt, argument_cnt) ); |
kvn@1421 | 1025 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::ne) ); |
kvn@1421 | 1026 | Node* if_ne = generate_slow_guard(bol, NULL); |
kvn@1421 | 1027 | if (if_ne != NULL) { |
kvn@1421 | 1028 | phi->init_req(4, intcon(0)); |
kvn@1421 | 1029 | region->init_req(4, if_ne); |
kvn@1421 | 1030 | } |
kvn@1421 | 1031 | } |
kvn@1421 | 1032 | |
kvn@1421 | 1033 | // Check for count == 0 is done by mach node StrEquals. |
kvn@1421 | 1034 | |
kvn@1421 | 1035 | if (!stopped()) { |
kvn@1421 | 1036 | Node* equals = make_string_method_node(Op_StrEquals, receiver, receiver_cnt, argument, argument_cnt); |
kvn@1421 | 1037 | phi->init_req(1, equals); |
kvn@1421 | 1038 | region->init_req(1, control()); |
kvn@1421 | 1039 | } |
cfang@1116 | 1040 | |
cfang@1116 | 1041 | // post merge |
cfang@1116 | 1042 | set_control(_gvn.transform(region)); |
cfang@1116 | 1043 | record_for_igvn(region); |
cfang@1116 | 1044 | |
cfang@1116 | 1045 | push(_gvn.transform(phi)); |
cfang@1116 | 1046 | |
cfang@1116 | 1047 | return true; |
cfang@1116 | 1048 | } |
cfang@1116 | 1049 | |
rasbold@604 | 1050 | //------------------------------inline_array_equals---------------------------- |
rasbold@604 | 1051 | bool LibraryCallKit::inline_array_equals() { |
rasbold@604 | 1052 | |
rasbold@609 | 1053 | if (!Matcher::has_match_rule(Op_AryEq)) return false; |
rasbold@609 | 1054 | |
rasbold@604 | 1055 | _sp += 2; |
rasbold@604 | 1056 | Node *argument2 = pop(); |
rasbold@604 | 1057 | Node *argument1 = pop(); |
rasbold@604 | 1058 | |
rasbold@604 | 1059 | Node* equals = |
kvn@1421 | 1060 | _gvn.transform(new (C, 4) AryEqNode(control(), memory(TypeAryPtr::CHARS), |
kvn@1421 | 1061 | argument1, argument2) ); |
rasbold@604 | 1062 | push(equals); |
rasbold@604 | 1063 | return true; |
rasbold@604 | 1064 | } |
rasbold@604 | 1065 | |
duke@435 | 1066 | // Java version of String.indexOf(constant string) |
duke@435 | 1067 | // class StringDecl { |
duke@435 | 1068 | // StringDecl(char[] ca) { |
duke@435 | 1069 | // offset = 0; |
duke@435 | 1070 | // count = ca.length; |
duke@435 | 1071 | // value = ca; |
duke@435 | 1072 | // } |
duke@435 | 1073 | // int offset; |
duke@435 | 1074 | // int count; |
duke@435 | 1075 | // char[] value; |
duke@435 | 1076 | // } |
duke@435 | 1077 | // |
duke@435 | 1078 | // static int string_indexOf_J(StringDecl string_object, char[] target_object, |
duke@435 | 1079 | // int targetOffset, int cache_i, int md2) { |
duke@435 | 1080 | // int cache = cache_i; |
duke@435 | 1081 | // int sourceOffset = string_object.offset; |
duke@435 | 1082 | // int sourceCount = string_object.count; |
duke@435 | 1083 | // int targetCount = target_object.length; |
duke@435 | 1084 | // |
duke@435 | 1085 | // int targetCountLess1 = targetCount - 1; |
duke@435 | 1086 | // int sourceEnd = sourceOffset + sourceCount - targetCountLess1; |
duke@435 | 1087 | // |
duke@435 | 1088 | // char[] source = string_object.value; |
duke@435 | 1089 | // char[] target = target_object; |
duke@435 | 1090 | // int lastChar = target[targetCountLess1]; |
duke@435 | 1091 | // |
duke@435 | 1092 | // outer_loop: |
duke@435 | 1093 | // for (int i = sourceOffset; i < sourceEnd; ) { |
duke@435 | 1094 | // int src = source[i + targetCountLess1]; |
duke@435 | 1095 | // if (src == lastChar) { |
duke@435 | 1096 | // // With random strings and a 4-character alphabet, |
duke@435 | 1097 | // // reverse matching at this point sets up 0.8% fewer |
duke@435 | 1098 | // // frames, but (paradoxically) makes 0.3% more probes. |
duke@435 | 1099 | // // Since those probes are nearer the lastChar probe, |
duke@435 | 1100 | // // there is may be a net D$ win with reverse matching. |
duke@435 | 1101 | // // But, reversing loop inhibits unroll of inner loop |
duke@435 | 1102 | // // for unknown reason. So, does running outer loop from |
duke@435 | 1103 | // // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount) |
duke@435 | 1104 | // for (int j = 0; j < targetCountLess1; j++) { |
duke@435 | 1105 | // if (target[targetOffset + j] != source[i+j]) { |
duke@435 | 1106 | // if ((cache & (1 << source[i+j])) == 0) { |
duke@435 | 1107 | // if (md2 < j+1) { |
duke@435 | 1108 | // i += j+1; |
duke@435 | 1109 | // continue outer_loop; |
duke@435 | 1110 | // } |
duke@435 | 1111 | // } |
duke@435 | 1112 | // i += md2; |
duke@435 | 1113 | // continue outer_loop; |
duke@435 | 1114 | // } |
duke@435 | 1115 | // } |
duke@435 | 1116 | // return i - sourceOffset; |
duke@435 | 1117 | // } |
duke@435 | 1118 | // if ((cache & (1 << src)) == 0) { |
duke@435 | 1119 | // i += targetCountLess1; |
duke@435 | 1120 | // } // using "i += targetCount;" and an "else i++;" causes a jump to jump. |
duke@435 | 1121 | // i++; |
duke@435 | 1122 | // } |
duke@435 | 1123 | // return -1; |
duke@435 | 1124 | // } |
duke@435 | 1125 | |
duke@435 | 1126 | //------------------------------string_indexOf------------------------ |
duke@435 | 1127 | Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i, |
duke@435 | 1128 | jint cache_i, jint md2_i) { |
duke@435 | 1129 | |
duke@435 | 1130 | Node* no_ctrl = NULL; |
duke@435 | 1131 | float likely = PROB_LIKELY(0.9); |
duke@435 | 1132 | float unlikely = PROB_UNLIKELY(0.9); |
duke@435 | 1133 | |
kvn@2665 | 1134 | const int nargs = 2; // number of arguments to push back for uncommon trap in predicate |
kvn@2665 | 1135 | |
duke@435 | 1136 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
duke@435 | 1137 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
duke@435 | 1138 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
duke@435 | 1139 | |
duke@435 | 1140 | ciInstanceKlass* klass = env()->String_klass(); |
never@1851 | 1141 | const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass); |
duke@435 | 1142 | const TypeAryPtr* source_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::CHAR,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, 0); |
duke@435 | 1143 | |
duke@435 | 1144 | Node* sourceOffseta = basic_plus_adr(string_object, string_object, offset_offset); |
duke@435 | 1145 | Node* sourceOffset = make_load(no_ctrl, sourceOffseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset)); |
duke@435 | 1146 | Node* sourceCounta = basic_plus_adr(string_object, string_object, count_offset); |
duke@435 | 1147 | Node* sourceCount = make_load(no_ctrl, sourceCounta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
duke@435 | 1148 | Node* sourcea = basic_plus_adr(string_object, string_object, value_offset); |
duke@435 | 1149 | Node* source = make_load(no_ctrl, sourcea, source_type, T_OBJECT, string_type->add_offset(value_offset)); |
duke@435 | 1150 | |
jcoomes@2661 | 1151 | Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) ); |
duke@435 | 1152 | jint target_length = target_array->length(); |
duke@435 | 1153 | const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin)); |
duke@435 | 1154 | const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot); |
duke@435 | 1155 | |
kvn@2726 | 1156 | IdealKit kit(this, false, true); |
duke@435 | 1157 | #define __ kit. |
duke@435 | 1158 | Node* zero = __ ConI(0); |
duke@435 | 1159 | Node* one = __ ConI(1); |
duke@435 | 1160 | Node* cache = __ ConI(cache_i); |
duke@435 | 1161 | Node* md2 = __ ConI(md2_i); |
duke@435 | 1162 | Node* lastChar = __ ConI(target_array->char_at(target_length - 1)); |
duke@435 | 1163 | Node* targetCount = __ ConI(target_length); |
duke@435 | 1164 | Node* targetCountLess1 = __ ConI(target_length - 1); |
duke@435 | 1165 | Node* targetOffset = __ ConI(targetOffset_i); |
duke@435 | 1166 | Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1); |
duke@435 | 1167 | |
kvn@1286 | 1168 | IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done(); |
duke@435 | 1169 | Node* outer_loop = __ make_label(2 /* goto */); |
duke@435 | 1170 | Node* return_ = __ make_label(1); |
duke@435 | 1171 | |
duke@435 | 1172 | __ set(rtn,__ ConI(-1)); |
kvn@2665 | 1173 | __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); { |
duke@435 | 1174 | Node* i2 = __ AddI(__ value(i), targetCountLess1); |
duke@435 | 1175 | // pin to prohibit loading of "next iteration" value which may SEGV (rare) |
duke@435 | 1176 | Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS); |
duke@435 | 1177 | __ if_then(src, BoolTest::eq, lastChar, unlikely); { |
kvn@2665 | 1178 | __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); { |
duke@435 | 1179 | Node* tpj = __ AddI(targetOffset, __ value(j)); |
duke@435 | 1180 | Node* targ = load_array_element(no_ctrl, target, tpj, target_type); |
duke@435 | 1181 | Node* ipj = __ AddI(__ value(i), __ value(j)); |
duke@435 | 1182 | Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS); |
duke@435 | 1183 | __ if_then(targ, BoolTest::ne, src2); { |
duke@435 | 1184 | __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); { |
duke@435 | 1185 | __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); { |
duke@435 | 1186 | __ increment(i, __ AddI(__ value(j), one)); |
duke@435 | 1187 | __ goto_(outer_loop); |
duke@435 | 1188 | } __ end_if(); __ dead(j); |
duke@435 | 1189 | }__ end_if(); __ dead(j); |
duke@435 | 1190 | __ increment(i, md2); |
duke@435 | 1191 | __ goto_(outer_loop); |
duke@435 | 1192 | }__ end_if(); |
duke@435 | 1193 | __ increment(j, one); |
duke@435 | 1194 | }__ end_loop(); __ dead(j); |
duke@435 | 1195 | __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i); |
duke@435 | 1196 | __ goto_(return_); |
duke@435 | 1197 | }__ end_if(); |
duke@435 | 1198 | __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); { |
duke@435 | 1199 | __ increment(i, targetCountLess1); |
duke@435 | 1200 | }__ end_if(); |
duke@435 | 1201 | __ increment(i, one); |
duke@435 | 1202 | __ bind(outer_loop); |
duke@435 | 1203 | }__ end_loop(); __ dead(i); |
duke@435 | 1204 | __ bind(return_); |
kvn@1286 | 1205 | |
kvn@1286 | 1206 | // Final sync IdealKit and GraphKit. |
kvn@2726 | 1207 | final_sync(kit); |
duke@435 | 1208 | Node* result = __ value(rtn); |
duke@435 | 1209 | #undef __ |
duke@435 | 1210 | C->set_has_loops(true); |
duke@435 | 1211 | return result; |
duke@435 | 1212 | } |
duke@435 | 1213 | |
duke@435 | 1214 | //------------------------------inline_string_indexOf------------------------ |
duke@435 | 1215 | bool LibraryCallKit::inline_string_indexOf() { |
duke@435 | 1216 | |
cfang@1116 | 1217 | const int value_offset = java_lang_String::value_offset_in_bytes(); |
cfang@1116 | 1218 | const int count_offset = java_lang_String::count_offset_in_bytes(); |
cfang@1116 | 1219 | const int offset_offset = java_lang_String::offset_offset_in_bytes(); |
cfang@1116 | 1220 | |
duke@435 | 1221 | _sp += 2; |
duke@435 | 1222 | Node *argument = pop(); // pop non-receiver first: it was pushed second |
duke@435 | 1223 | Node *receiver = pop(); |
duke@435 | 1224 | |
cfang@1116 | 1225 | Node* result; |
iveresov@1859 | 1226 | // Disable the use of pcmpestri until it can be guaranteed that |
iveresov@1859 | 1227 | // the load doesn't cross into the uncommited space. |
kvn@2602 | 1228 | if (Matcher::has_match_rule(Op_StrIndexOf) && |
cfang@1116 | 1229 | UseSSE42Intrinsics) { |
cfang@1116 | 1230 | // Generate SSE4.2 version of indexOf |
cfang@1116 | 1231 | // We currently only have match rules that use SSE4.2 |
cfang@1116 | 1232 | |
cfang@1116 | 1233 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 1234 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 1235 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 1236 | // which handles NullPointerExceptions. |
cfang@1116 | 1237 | _sp += 2; |
cfang@1116 | 1238 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 1239 | argument = do_null_check(argument, T_OBJECT); |
cfang@1116 | 1240 | _sp -= 2; |
cfang@1116 | 1241 | |
cfang@1116 | 1242 | if (stopped()) { |
cfang@1116 | 1243 | return true; |
cfang@1116 | 1244 | } |
cfang@1116 | 1245 | |
kvn@2602 | 1246 | ciInstanceKlass* str_klass = env()->String_klass(); |
kvn@2602 | 1247 | const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass); |
kvn@2602 | 1248 | |
kvn@1421 | 1249 | // Make the merge point |
kvn@2602 | 1250 | RegionNode* result_rgn = new (C, 4) RegionNode(4); |
kvn@2602 | 1251 | Node* result_phi = new (C, 4) PhiNode(result_rgn, TypeInt::INT); |
kvn@1421 | 1252 | Node* no_ctrl = NULL; |
kvn@1421 | 1253 | |
kvn@1421 | 1254 | // Get counts for string and substr |
kvn@1421 | 1255 | Node* source_cnta = basic_plus_adr(receiver, receiver, count_offset); |
kvn@1421 | 1256 | Node* source_cnt = make_load(no_ctrl, source_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1257 | |
kvn@1421 | 1258 | Node* substr_cnta = basic_plus_adr(argument, argument, count_offset); |
kvn@1421 | 1259 | Node* substr_cnt = make_load(no_ctrl, substr_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset)); |
kvn@1421 | 1260 | |
kvn@1421 | 1261 | // Check for substr count > string count |
kvn@1421 | 1262 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, source_cnt) ); |
kvn@1421 | 1263 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::gt) ); |
kvn@1421 | 1264 | Node* if_gt = generate_slow_guard(bol, NULL); |
kvn@1421 | 1265 | if (if_gt != NULL) { |
kvn@1421 | 1266 | result_phi->init_req(2, intcon(-1)); |
kvn@1421 | 1267 | result_rgn->init_req(2, if_gt); |
kvn@1421 | 1268 | } |
kvn@1421 | 1269 | |
kvn@1421 | 1270 | if (!stopped()) { |
kvn@2602 | 1271 | // Check for substr count == 0 |
kvn@2602 | 1272 | cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, intcon(0)) ); |
kvn@2602 | 1273 | bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) ); |
kvn@2602 | 1274 | Node* if_zero = generate_slow_guard(bol, NULL); |
kvn@2602 | 1275 | if (if_zero != NULL) { |
kvn@2602 | 1276 | result_phi->init_req(3, intcon(0)); |
kvn@2602 | 1277 | result_rgn->init_req(3, if_zero); |
kvn@2602 | 1278 | } |
kvn@2602 | 1279 | } |
kvn@2602 | 1280 | |
kvn@2602 | 1281 | if (!stopped()) { |
kvn@1421 | 1282 | result = make_string_method_node(Op_StrIndexOf, receiver, source_cnt, argument, substr_cnt); |
kvn@1421 | 1283 | result_phi->init_req(1, result); |
kvn@1421 | 1284 | result_rgn->init_req(1, control()); |
kvn@1421 | 1285 | } |
kvn@1421 | 1286 | set_control(_gvn.transform(result_rgn)); |
kvn@1421 | 1287 | record_for_igvn(result_rgn); |
kvn@1421 | 1288 | result = _gvn.transform(result_phi); |
kvn@1421 | 1289 | |
kvn@2602 | 1290 | } else { // Use LibraryCallKit::string_indexOf |
kvn@2602 | 1291 | // don't intrinsify if argument isn't a constant string. |
cfang@1116 | 1292 | if (!argument->is_Con()) { |
cfang@1116 | 1293 | return false; |
cfang@1116 | 1294 | } |
cfang@1116 | 1295 | const TypeOopPtr* str_type = _gvn.type(argument)->isa_oopptr(); |
cfang@1116 | 1296 | if (str_type == NULL) { |
cfang@1116 | 1297 | return false; |
cfang@1116 | 1298 | } |
cfang@1116 | 1299 | ciInstanceKlass* klass = env()->String_klass(); |
cfang@1116 | 1300 | ciObject* str_const = str_type->const_oop(); |
cfang@1116 | 1301 | if (str_const == NULL || str_const->klass() != klass) { |
cfang@1116 | 1302 | return false; |
cfang@1116 | 1303 | } |
cfang@1116 | 1304 | ciInstance* str = str_const->as_instance(); |
cfang@1116 | 1305 | assert(str != NULL, "must be instance"); |
cfang@1116 | 1306 | |
cfang@1116 | 1307 | ciObject* v = str->field_value_by_offset(value_offset).as_object(); |
cfang@1116 | 1308 | int o = str->field_value_by_offset(offset_offset).as_int(); |
cfang@1116 | 1309 | int c = str->field_value_by_offset(count_offset).as_int(); |
cfang@1116 | 1310 | ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array |
cfang@1116 | 1311 | |
cfang@1116 | 1312 | // constant strings have no offset and count == length which |
cfang@1116 | 1313 | // simplifies the resulting code somewhat so lets optimize for that. |
cfang@1116 | 1314 | if (o != 0 || c != pat->length()) { |
cfang@1116 | 1315 | return false; |
cfang@1116 | 1316 | } |
cfang@1116 | 1317 | |
cfang@1116 | 1318 | // Null check on self without removing any arguments. The argument |
cfang@1116 | 1319 | // null check technically happens in the wrong place, which can lead to |
cfang@1116 | 1320 | // invalid stack traces when string compare is inlined into a method |
cfang@1116 | 1321 | // which handles NullPointerExceptions. |
cfang@1116 | 1322 | _sp += 2; |
cfang@1116 | 1323 | receiver = do_null_check(receiver, T_OBJECT); |
cfang@1116 | 1324 | // No null check on the argument is needed since it's a constant String oop. |
cfang@1116 | 1325 | _sp -= 2; |
cfang@1116 | 1326 | if (stopped()) { |
kvn@2602 | 1327 | return true; |
cfang@1116 | 1328 | } |
cfang@1116 | 1329 | |
cfang@1116 | 1330 | // The null string as a pattern always returns 0 (match at beginning of string) |
cfang@1116 | 1331 | if (c == 0) { |
cfang@1116 | 1332 | push(intcon(0)); |
cfang@1116 | 1333 | return true; |
cfang@1116 | 1334 | } |
cfang@1116 | 1335 | |
cfang@1116 | 1336 | // Generate default indexOf |
cfang@1116 | 1337 | jchar lastChar = pat->char_at(o + (c - 1)); |
cfang@1116 | 1338 | int cache = 0; |
cfang@1116 | 1339 | int i; |
cfang@1116 | 1340 | for (i = 0; i < c - 1; i++) { |
cfang@1116 | 1341 | assert(i < pat->length(), "out of range"); |
cfang@1116 | 1342 | cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1))); |
cfang@1116 | 1343 | } |
cfang@1116 | 1344 | |
cfang@1116 | 1345 | int md2 = c; |
cfang@1116 | 1346 | for (i = 0; i < c - 1; i++) { |
cfang@1116 | 1347 | assert(i < pat->length(), "out of range"); |
cfang@1116 | 1348 | if (pat->char_at(o + i) == lastChar) { |
cfang@1116 | 1349 | md2 = (c - 1) - i; |
cfang@1116 | 1350 | } |
cfang@1116 | 1351 | } |
cfang@1116 | 1352 | |
cfang@1116 | 1353 | result = string_indexOf(receiver, pat, o, cache, md2); |
duke@435 | 1354 | } |
cfang@1116 | 1355 | |
duke@435 | 1356 | push(result); |
duke@435 | 1357 | return true; |
duke@435 | 1358 | } |
duke@435 | 1359 | |
duke@435 | 1360 | //--------------------------pop_math_arg-------------------------------- |
duke@435 | 1361 | // Pop a double argument to a math function from the stack |
duke@435 | 1362 | // rounding it if necessary. |
duke@435 | 1363 | Node * LibraryCallKit::pop_math_arg() { |
duke@435 | 1364 | Node *arg = pop_pair(); |
duke@435 | 1365 | if( Matcher::strict_fp_requires_explicit_rounding && UseSSE<=1 ) |
duke@435 | 1366 | arg = _gvn.transform( new (C, 2) RoundDoubleNode(0, arg) ); |
duke@435 | 1367 | return arg; |
duke@435 | 1368 | } |
duke@435 | 1369 | |
duke@435 | 1370 | //------------------------------inline_trig---------------------------------- |
duke@435 | 1371 | // Inline sin/cos/tan instructions, if possible. If rounding is required, do |
duke@435 | 1372 | // argument reduction which will turn into a fast/slow diamond. |
duke@435 | 1373 | bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) { |
duke@435 | 1374 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1375 | Node* arg = pop_math_arg(); |
duke@435 | 1376 | Node* trig = NULL; |
duke@435 | 1377 | |
duke@435 | 1378 | switch (id) { |
duke@435 | 1379 | case vmIntrinsics::_dsin: |
duke@435 | 1380 | trig = _gvn.transform((Node*)new (C, 2) SinDNode(arg)); |
duke@435 | 1381 | break; |
duke@435 | 1382 | case vmIntrinsics::_dcos: |
duke@435 | 1383 | trig = _gvn.transform((Node*)new (C, 2) CosDNode(arg)); |
duke@435 | 1384 | break; |
duke@435 | 1385 | case vmIntrinsics::_dtan: |
duke@435 | 1386 | trig = _gvn.transform((Node*)new (C, 2) TanDNode(arg)); |
duke@435 | 1387 | break; |
duke@435 | 1388 | default: |
duke@435 | 1389 | assert(false, "bad intrinsic was passed in"); |
duke@435 | 1390 | return false; |
duke@435 | 1391 | } |
duke@435 | 1392 | |
duke@435 | 1393 | // Rounding required? Check for argument reduction! |
duke@435 | 1394 | if( Matcher::strict_fp_requires_explicit_rounding ) { |
duke@435 | 1395 | |
duke@435 | 1396 | static const double pi_4 = 0.7853981633974483; |
duke@435 | 1397 | static const double neg_pi_4 = -0.7853981633974483; |
duke@435 | 1398 | // pi/2 in 80-bit extended precision |
duke@435 | 1399 | // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00}; |
duke@435 | 1400 | // -pi/2 in 80-bit extended precision |
duke@435 | 1401 | // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00}; |
duke@435 | 1402 | // Cutoff value for using this argument reduction technique |
duke@435 | 1403 | //static const double pi_2_minus_epsilon = 1.564660403643354; |
duke@435 | 1404 | //static const double neg_pi_2_plus_epsilon = -1.564660403643354; |
duke@435 | 1405 | |
duke@435 | 1406 | // Pseudocode for sin: |
duke@435 | 1407 | // if (x <= Math.PI / 4.0) { |
duke@435 | 1408 | // if (x >= -Math.PI / 4.0) return fsin(x); |
duke@435 | 1409 | // if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0); |
duke@435 | 1410 | // } else { |
duke@435 | 1411 | // if (x <= Math.PI / 2.0) return fcos(x - Math.PI / 2.0); |
duke@435 | 1412 | // } |
duke@435 | 1413 | // return StrictMath.sin(x); |
duke@435 | 1414 | |
duke@435 | 1415 | // Pseudocode for cos: |
duke@435 | 1416 | // if (x <= Math.PI / 4.0) { |
duke@435 | 1417 | // if (x >= -Math.PI / 4.0) return fcos(x); |
duke@435 | 1418 | // if (x >= -Math.PI / 2.0) return fsin(x + Math.PI / 2.0); |
duke@435 | 1419 | // } else { |
duke@435 | 1420 | // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0); |
duke@435 | 1421 | // } |
duke@435 | 1422 | // return StrictMath.cos(x); |
duke@435 | 1423 | |
duke@435 | 1424 | // Actually, sticking in an 80-bit Intel value into C2 will be tough; it |
duke@435 | 1425 | // requires a special machine instruction to load it. Instead we'll try |
duke@435 | 1426 | // the 'easy' case. If we really need the extra range +/- PI/2 we'll |
duke@435 | 1427 | // probably do the math inside the SIN encoding. |
duke@435 | 1428 | |
duke@435 | 1429 | // Make the merge point |
duke@435 | 1430 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 1431 | Node *phi = new (C, 3) PhiNode(r,Type::DOUBLE); |
duke@435 | 1432 | |
duke@435 | 1433 | // Flatten arg so we need only 1 test |
duke@435 | 1434 | Node *abs = _gvn.transform(new (C, 2) AbsDNode(arg)); |
duke@435 | 1435 | // Node for PI/4 constant |
duke@435 | 1436 | Node *pi4 = makecon(TypeD::make(pi_4)); |
duke@435 | 1437 | // Check PI/4 : abs(arg) |
duke@435 | 1438 | Node *cmp = _gvn.transform(new (C, 3) CmpDNode(pi4,abs)); |
duke@435 | 1439 | // Check: If PI/4 < abs(arg) then go slow |
duke@435 | 1440 | Node *bol = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::lt ) ); |
duke@435 | 1441 | // Branch either way |
duke@435 | 1442 | IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 1443 | set_control(opt_iff(r,iff)); |
duke@435 | 1444 | |
duke@435 | 1445 | // Set fast path result |
duke@435 | 1446 | phi->init_req(2,trig); |
duke@435 | 1447 | |
duke@435 | 1448 | // Slow path - non-blocking leaf call |
duke@435 | 1449 | Node* call = NULL; |
duke@435 | 1450 | switch (id) { |
duke@435 | 1451 | case vmIntrinsics::_dsin: |
duke@435 | 1452 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1453 | CAST_FROM_FN_PTR(address, SharedRuntime::dsin), |
duke@435 | 1454 | "Sin", NULL, arg, top()); |
duke@435 | 1455 | break; |
duke@435 | 1456 | case vmIntrinsics::_dcos: |
duke@435 | 1457 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1458 | CAST_FROM_FN_PTR(address, SharedRuntime::dcos), |
duke@435 | 1459 | "Cos", NULL, arg, top()); |
duke@435 | 1460 | break; |
duke@435 | 1461 | case vmIntrinsics::_dtan: |
duke@435 | 1462 | call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(), |
duke@435 | 1463 | CAST_FROM_FN_PTR(address, SharedRuntime::dtan), |
duke@435 | 1464 | "Tan", NULL, arg, top()); |
duke@435 | 1465 | break; |
duke@435 | 1466 | } |
duke@435 | 1467 | assert(control()->in(0) == call, ""); |
duke@435 | 1468 | Node* slow_result = _gvn.transform(new (C, 1) ProjNode(call,TypeFunc::Parms)); |
duke@435 | 1469 | r->init_req(1,control()); |
duke@435 | 1470 | phi->init_req(1,slow_result); |
duke@435 | 1471 | |
duke@435 | 1472 | // Post-merge |
duke@435 | 1473 | set_control(_gvn.transform(r)); |
duke@435 | 1474 | record_for_igvn(r); |
duke@435 | 1475 | trig = _gvn.transform(phi); |
duke@435 | 1476 | |
duke@435 | 1477 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1478 | } |
duke@435 | 1479 | // Push result back on JVM stack |
duke@435 | 1480 | push_pair(trig); |
duke@435 | 1481 | return true; |
duke@435 | 1482 | } |
duke@435 | 1483 | |
duke@435 | 1484 | //------------------------------inline_sqrt------------------------------------- |
duke@435 | 1485 | // Inline square root instruction, if possible. |
duke@435 | 1486 | bool LibraryCallKit::inline_sqrt(vmIntrinsics::ID id) { |
duke@435 | 1487 | assert(id == vmIntrinsics::_dsqrt, "Not square root"); |
duke@435 | 1488 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1489 | push_pair(_gvn.transform(new (C, 2) SqrtDNode(0, pop_math_arg()))); |
duke@435 | 1490 | return true; |
duke@435 | 1491 | } |
duke@435 | 1492 | |
duke@435 | 1493 | //------------------------------inline_abs------------------------------------- |
duke@435 | 1494 | // Inline absolute value instruction, if possible. |
duke@435 | 1495 | bool LibraryCallKit::inline_abs(vmIntrinsics::ID id) { |
duke@435 | 1496 | assert(id == vmIntrinsics::_dabs, "Not absolute value"); |
duke@435 | 1497 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1498 | push_pair(_gvn.transform(new (C, 2) AbsDNode(pop_math_arg()))); |
duke@435 | 1499 | return true; |
duke@435 | 1500 | } |
duke@435 | 1501 | |
duke@435 | 1502 | //------------------------------inline_exp------------------------------------- |
duke@435 | 1503 | // Inline exp instructions, if possible. The Intel hardware only misses |
duke@435 | 1504 | // really odd corner cases (+/- Infinity). Just uncommon-trap them. |
duke@435 | 1505 | bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) { |
duke@435 | 1506 | assert(id == vmIntrinsics::_dexp, "Not exp"); |
duke@435 | 1507 | |
duke@435 | 1508 | // If this inlining ever returned NaN in the past, we do not intrinsify it |
duke@435 | 1509 | // every again. NaN results requires StrictMath.exp handling. |
duke@435 | 1510 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 1511 | |
duke@435 | 1512 | // Do not intrinsify on older platforms which lack cmove. |
duke@435 | 1513 | if (ConditionalMoveLimit == 0) return false; |
duke@435 | 1514 | |
duke@435 | 1515 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1516 | Node *x = pop_math_arg(); |
duke@435 | 1517 | Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x)); |
duke@435 | 1518 | |
duke@435 | 1519 | //------------------- |
duke@435 | 1520 | //result=(result.isNaN())? StrictMath::exp():result; |
duke@435 | 1521 | // Check: If isNaN() by checking result!=result? then go to Strict Math |
duke@435 | 1522 | Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); |
duke@435 | 1523 | // Build the boolean node |
duke@435 | 1524 | Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); |
duke@435 | 1525 | |
duke@435 | 1526 | { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); |
duke@435 | 1527 | // End the current control-flow path |
duke@435 | 1528 | push_pair(x); |
duke@435 | 1529 | // Math.exp intrinsic returned a NaN, which requires StrictMath.exp |
duke@435 | 1530 | // to handle. Recompile without intrinsifying Math.exp |
duke@435 | 1531 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 1532 | Deoptimization::Action_make_not_entrant); |
duke@435 | 1533 | } |
duke@435 | 1534 | |
duke@435 | 1535 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1536 | |
duke@435 | 1537 | push_pair(result); |
duke@435 | 1538 | |
duke@435 | 1539 | return true; |
duke@435 | 1540 | } |
duke@435 | 1541 | |
duke@435 | 1542 | //------------------------------inline_pow------------------------------------- |
duke@435 | 1543 | // Inline power instructions, if possible. |
duke@435 | 1544 | bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) { |
duke@435 | 1545 | assert(id == vmIntrinsics::_dpow, "Not pow"); |
duke@435 | 1546 | |
duke@435 | 1547 | // If this inlining ever returned NaN in the past, we do not intrinsify it |
duke@435 | 1548 | // every again. NaN results requires StrictMath.pow handling. |
duke@435 | 1549 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 1550 | |
duke@435 | 1551 | // Do not intrinsify on older platforms which lack cmove. |
duke@435 | 1552 | if (ConditionalMoveLimit == 0) return false; |
duke@435 | 1553 | |
duke@435 | 1554 | // Pseudocode for pow |
duke@435 | 1555 | // if (x <= 0.0) { |
duke@435 | 1556 | // if ((double)((int)y)==y) { // if y is int |
duke@435 | 1557 | // result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y) |
duke@435 | 1558 | // } else { |
duke@435 | 1559 | // result = NaN; |
duke@435 | 1560 | // } |
duke@435 | 1561 | // } else { |
duke@435 | 1562 | // result = DPow(x,y); |
duke@435 | 1563 | // } |
duke@435 | 1564 | // if (result != result)? { |
twisti@1040 | 1565 | // uncommon_trap(); |
duke@435 | 1566 | // } |
duke@435 | 1567 | // return result; |
duke@435 | 1568 | |
duke@435 | 1569 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1570 | Node* y = pop_math_arg(); |
duke@435 | 1571 | Node* x = pop_math_arg(); |
duke@435 | 1572 | |
duke@435 | 1573 | Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) ); |
duke@435 | 1574 | |
duke@435 | 1575 | // Short form: if not top-level (i.e., Math.pow but inlining Math.pow |
duke@435 | 1576 | // inside of something) then skip the fancy tests and just check for |
duke@435 | 1577 | // NaN result. |
duke@435 | 1578 | Node *result = NULL; |
duke@435 | 1579 | if( jvms()->depth() >= 1 ) { |
duke@435 | 1580 | result = fast_result; |
duke@435 | 1581 | } else { |
duke@435 | 1582 | |
duke@435 | 1583 | // Set the merge point for If node with condition of (x <= 0.0) |
duke@435 | 1584 | // There are four possible paths to region node and phi node |
duke@435 | 1585 | RegionNode *r = new (C, 4) RegionNode(4); |
duke@435 | 1586 | Node *phi = new (C, 4) PhiNode(r, Type::DOUBLE); |
duke@435 | 1587 | |
duke@435 | 1588 | // Build the first if node: if (x <= 0.0) |
duke@435 | 1589 | // Node for 0 constant |
duke@435 | 1590 | Node *zeronode = makecon(TypeD::ZERO); |
duke@435 | 1591 | // Check x:0 |
duke@435 | 1592 | Node *cmp = _gvn.transform(new (C, 3) CmpDNode(x, zeronode)); |
duke@435 | 1593 | // Check: If (x<=0) then go complex path |
duke@435 | 1594 | Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) ); |
duke@435 | 1595 | // Branch either way |
duke@435 | 1596 | IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); |
duke@435 | 1597 | Node *opt_test = _gvn.transform(if1); |
duke@435 | 1598 | //assert( opt_test->is_If(), "Expect an IfNode"); |
duke@435 | 1599 | IfNode *opt_if1 = (IfNode*)opt_test; |
duke@435 | 1600 | // Fast path taken; set region slot 3 |
duke@435 | 1601 | Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) ); |
duke@435 | 1602 | r->init_req(3,fast_taken); // Capture fast-control |
duke@435 | 1603 | |
duke@435 | 1604 | // Fast path not-taken, i.e. slow path |
duke@435 | 1605 | Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) ); |
duke@435 | 1606 | |
duke@435 | 1607 | // Set fast path result |
duke@435 | 1608 | Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) ); |
duke@435 | 1609 | phi->init_req(3, fast_result); |
duke@435 | 1610 | |
duke@435 | 1611 | // Complex path |
duke@435 | 1612 | // Build the second if node (if y is int) |
duke@435 | 1613 | // Node for (int)y |
duke@435 | 1614 | Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y)); |
duke@435 | 1615 | // Node for (double)((int) y) |
duke@435 | 1616 | Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty)); |
duke@435 | 1617 | // Check (double)((int) y) : y |
duke@435 | 1618 | Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y)); |
duke@435 | 1619 | // Check if (y isn't int) then go to slow path |
duke@435 | 1620 | |
duke@435 | 1621 | Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) ); |
twisti@1040 | 1622 | // Branch either way |
duke@435 | 1623 | IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); |
duke@435 | 1624 | Node *slow_path = opt_iff(r,if2); // Set region path 2 |
duke@435 | 1625 | |
duke@435 | 1626 | // Calculate DPow(abs(x), y)*(1 & (int)y) |
duke@435 | 1627 | // Node for constant 1 |
duke@435 | 1628 | Node *conone = intcon(1); |
duke@435 | 1629 | // 1& (int)y |
duke@435 | 1630 | Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) ); |
duke@435 | 1631 | // zero node |
duke@435 | 1632 | Node *conzero = intcon(0); |
duke@435 | 1633 | // Check (1&(int)y)==0? |
duke@435 | 1634 | Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero)); |
duke@435 | 1635 | // Check if (1&(int)y)!=0?, if so the result is negative |
duke@435 | 1636 | Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) ); |
duke@435 | 1637 | // abs(x) |
duke@435 | 1638 | Node *absx=_gvn.transform( new (C, 2) AbsDNode(x)); |
duke@435 | 1639 | // abs(x)^y |
duke@435 | 1640 | Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) ); |
duke@435 | 1641 | // -abs(x)^y |
duke@435 | 1642 | Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy)); |
duke@435 | 1643 | // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y) |
duke@435 | 1644 | Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE)); |
duke@435 | 1645 | // Set complex path fast result |
duke@435 | 1646 | phi->init_req(2, signresult); |
duke@435 | 1647 | |
duke@435 | 1648 | static const jlong nan_bits = CONST64(0x7ff8000000000000); |
duke@435 | 1649 | Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN |
duke@435 | 1650 | r->init_req(1,slow_path); |
duke@435 | 1651 | phi->init_req(1,slow_result); |
duke@435 | 1652 | |
duke@435 | 1653 | // Post merge |
duke@435 | 1654 | set_control(_gvn.transform(r)); |
duke@435 | 1655 | record_for_igvn(r); |
duke@435 | 1656 | result=_gvn.transform(phi); |
duke@435 | 1657 | } |
duke@435 | 1658 | |
duke@435 | 1659 | //------------------- |
duke@435 | 1660 | //result=(result.isNaN())? uncommon_trap():result; |
duke@435 | 1661 | // Check: If isNaN() by checking result!=result? then go to Strict Math |
duke@435 | 1662 | Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); |
duke@435 | 1663 | // Build the boolean node |
duke@435 | 1664 | Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); |
duke@435 | 1665 | |
duke@435 | 1666 | { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); |
duke@435 | 1667 | // End the current control-flow path |
duke@435 | 1668 | push_pair(x); |
duke@435 | 1669 | push_pair(y); |
duke@435 | 1670 | // Math.pow intrinsic returned a NaN, which requires StrictMath.pow |
duke@435 | 1671 | // to handle. Recompile without intrinsifying Math.pow. |
duke@435 | 1672 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 1673 | Deoptimization::Action_make_not_entrant); |
duke@435 | 1674 | } |
duke@435 | 1675 | |
duke@435 | 1676 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 1677 | |
duke@435 | 1678 | push_pair(result); |
duke@435 | 1679 | |
duke@435 | 1680 | return true; |
duke@435 | 1681 | } |
duke@435 | 1682 | |
duke@435 | 1683 | //------------------------------inline_trans------------------------------------- |
duke@435 | 1684 | // Inline transcendental instructions, if possible. The Intel hardware gets |
duke@435 | 1685 | // these right, no funny corner cases missed. |
duke@435 | 1686 | bool LibraryCallKit::inline_trans(vmIntrinsics::ID id) { |
duke@435 | 1687 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1688 | Node* arg = pop_math_arg(); |
duke@435 | 1689 | Node* trans = NULL; |
duke@435 | 1690 | |
duke@435 | 1691 | switch (id) { |
duke@435 | 1692 | case vmIntrinsics::_dlog: |
duke@435 | 1693 | trans = _gvn.transform((Node*)new (C, 2) LogDNode(arg)); |
duke@435 | 1694 | break; |
duke@435 | 1695 | case vmIntrinsics::_dlog10: |
duke@435 | 1696 | trans = _gvn.transform((Node*)new (C, 2) Log10DNode(arg)); |
duke@435 | 1697 | break; |
duke@435 | 1698 | default: |
duke@435 | 1699 | assert(false, "bad intrinsic was passed in"); |
duke@435 | 1700 | return false; |
duke@435 | 1701 | } |
duke@435 | 1702 | |
duke@435 | 1703 | // Push result back on JVM stack |
duke@435 | 1704 | push_pair(trans); |
duke@435 | 1705 | return true; |
duke@435 | 1706 | } |
duke@435 | 1707 | |
duke@435 | 1708 | //------------------------------runtime_math----------------------------- |
duke@435 | 1709 | bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) { |
duke@435 | 1710 | Node* a = NULL; |
duke@435 | 1711 | Node* b = NULL; |
duke@435 | 1712 | |
duke@435 | 1713 | assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(), |
duke@435 | 1714 | "must be (DD)D or (D)D type"); |
duke@435 | 1715 | |
duke@435 | 1716 | // Inputs |
duke@435 | 1717 | _sp += arg_size(); // restore stack pointer |
duke@435 | 1718 | if (call_type == OptoRuntime::Math_DD_D_Type()) { |
duke@435 | 1719 | b = pop_math_arg(); |
duke@435 | 1720 | } |
duke@435 | 1721 | a = pop_math_arg(); |
duke@435 | 1722 | |
duke@435 | 1723 | const TypePtr* no_memory_effects = NULL; |
duke@435 | 1724 | Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, |
duke@435 | 1725 | no_memory_effects, |
duke@435 | 1726 | a, top(), b, b ? top() : NULL); |
duke@435 | 1727 | Node* value = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+0)); |
duke@435 | 1728 | #ifdef ASSERT |
duke@435 | 1729 | Node* value_top = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+1)); |
duke@435 | 1730 | assert(value_top == top(), "second value must be top"); |
duke@435 | 1731 | #endif |
duke@435 | 1732 | |
duke@435 | 1733 | push_pair(value); |
duke@435 | 1734 | return true; |
duke@435 | 1735 | } |
duke@435 | 1736 | |
duke@435 | 1737 | //------------------------------inline_math_native----------------------------- |
duke@435 | 1738 | bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { |
duke@435 | 1739 | switch (id) { |
duke@435 | 1740 | // These intrinsics are not properly supported on all hardware |
duke@435 | 1741 | case vmIntrinsics::_dcos: return Matcher::has_match_rule(Op_CosD) ? inline_trig(id) : |
duke@435 | 1742 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dcos), "COS"); |
duke@435 | 1743 | case vmIntrinsics::_dsin: return Matcher::has_match_rule(Op_SinD) ? inline_trig(id) : |
duke@435 | 1744 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dsin), "SIN"); |
duke@435 | 1745 | case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) : |
duke@435 | 1746 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dtan), "TAN"); |
duke@435 | 1747 | |
duke@435 | 1748 | case vmIntrinsics::_dlog: return Matcher::has_match_rule(Op_LogD) ? inline_trans(id) : |
duke@435 | 1749 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog), "LOG"); |
duke@435 | 1750 | case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_trans(id) : |
duke@435 | 1751 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), "LOG10"); |
duke@435 | 1752 | |
duke@435 | 1753 | // These intrinsics are supported on all hardware |
duke@435 | 1754 | case vmIntrinsics::_dsqrt: return Matcher::has_match_rule(Op_SqrtD) ? inline_sqrt(id) : false; |
duke@435 | 1755 | case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_abs(id) : false; |
duke@435 | 1756 | |
duke@435 | 1757 | // These intrinsics don't work on X86. The ad implementation doesn't |
duke@435 | 1758 | // handle NaN's properly. Instead of returning infinity, the ad |
duke@435 | 1759 | // implementation returns a NaN on overflow. See bug: 6304089 |
duke@435 | 1760 | // Once the ad implementations are fixed, change the code below |
duke@435 | 1761 | // to match the intrinsics above |
duke@435 | 1762 | |
duke@435 | 1763 | case vmIntrinsics::_dexp: return |
duke@435 | 1764 | runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP"); |
duke@435 | 1765 | case vmIntrinsics::_dpow: return |
duke@435 | 1766 | runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); |
duke@435 | 1767 | |
duke@435 | 1768 | // These intrinsics are not yet correctly implemented |
duke@435 | 1769 | case vmIntrinsics::_datan2: |
duke@435 | 1770 | return false; |
duke@435 | 1771 | |
duke@435 | 1772 | default: |
duke@435 | 1773 | ShouldNotReachHere(); |
duke@435 | 1774 | return false; |
duke@435 | 1775 | } |
duke@435 | 1776 | } |
duke@435 | 1777 | |
duke@435 | 1778 | static bool is_simple_name(Node* n) { |
duke@435 | 1779 | return (n->req() == 1 // constant |
duke@435 | 1780 | || (n->is_Type() && n->as_Type()->type()->singleton()) |
duke@435 | 1781 | || n->is_Proj() // parameter or return value |
duke@435 | 1782 | || n->is_Phi() // local of some sort |
duke@435 | 1783 | ); |
duke@435 | 1784 | } |
duke@435 | 1785 | |
duke@435 | 1786 | //----------------------------inline_min_max----------------------------------- |
duke@435 | 1787 | bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) { |
duke@435 | 1788 | push(generate_min_max(id, argument(0), argument(1))); |
duke@435 | 1789 | |
duke@435 | 1790 | return true; |
duke@435 | 1791 | } |
duke@435 | 1792 | |
duke@435 | 1793 | Node* |
duke@435 | 1794 | LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { |
duke@435 | 1795 | // These are the candidate return value: |
duke@435 | 1796 | Node* xvalue = x0; |
duke@435 | 1797 | Node* yvalue = y0; |
duke@435 | 1798 | |
duke@435 | 1799 | if (xvalue == yvalue) { |
duke@435 | 1800 | return xvalue; |
duke@435 | 1801 | } |
duke@435 | 1802 | |
duke@435 | 1803 | bool want_max = (id == vmIntrinsics::_max); |
duke@435 | 1804 | |
duke@435 | 1805 | const TypeInt* txvalue = _gvn.type(xvalue)->isa_int(); |
duke@435 | 1806 | const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int(); |
duke@435 | 1807 | if (txvalue == NULL || tyvalue == NULL) return top(); |
duke@435 | 1808 | // This is not really necessary, but it is consistent with a |
duke@435 | 1809 | // hypothetical MaxINode::Value method: |
duke@435 | 1810 | int widen = MAX2(txvalue->_widen, tyvalue->_widen); |
duke@435 | 1811 | |
duke@435 | 1812 | // %%% This folding logic should (ideally) be in a different place. |
duke@435 | 1813 | // Some should be inside IfNode, and there to be a more reliable |
duke@435 | 1814 | // transformation of ?: style patterns into cmoves. We also want |
duke@435 | 1815 | // more powerful optimizations around cmove and min/max. |
duke@435 | 1816 | |
duke@435 | 1817 | // Try to find a dominating comparison of these guys. |
duke@435 | 1818 | // It can simplify the index computation for Arrays.copyOf |
duke@435 | 1819 | // and similar uses of System.arraycopy. |
duke@435 | 1820 | // First, compute the normalized version of CmpI(x, y). |
duke@435 | 1821 | int cmp_op = Op_CmpI; |
duke@435 | 1822 | Node* xkey = xvalue; |
duke@435 | 1823 | Node* ykey = yvalue; |
duke@435 | 1824 | Node* ideal_cmpxy = _gvn.transform( new(C, 3) CmpINode(xkey, ykey) ); |
duke@435 | 1825 | if (ideal_cmpxy->is_Cmp()) { |
duke@435 | 1826 | // E.g., if we have CmpI(length - offset, count), |
duke@435 | 1827 | // it might idealize to CmpI(length, count + offset) |
duke@435 | 1828 | cmp_op = ideal_cmpxy->Opcode(); |
duke@435 | 1829 | xkey = ideal_cmpxy->in(1); |
duke@435 | 1830 | ykey = ideal_cmpxy->in(2); |
duke@435 | 1831 | } |
duke@435 | 1832 | |
duke@435 | 1833 | // Start by locating any relevant comparisons. |
duke@435 | 1834 | Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey; |
duke@435 | 1835 | Node* cmpxy = NULL; |
duke@435 | 1836 | Node* cmpyx = NULL; |
duke@435 | 1837 | for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) { |
duke@435 | 1838 | Node* cmp = start_from->fast_out(k); |
duke@435 | 1839 | if (cmp->outcnt() > 0 && // must have prior uses |
duke@435 | 1840 | cmp->in(0) == NULL && // must be context-independent |
duke@435 | 1841 | cmp->Opcode() == cmp_op) { // right kind of compare |
duke@435 | 1842 | if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp; |
duke@435 | 1843 | if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp; |
duke@435 | 1844 | } |
duke@435 | 1845 | } |
duke@435 | 1846 | |
duke@435 | 1847 | const int NCMPS = 2; |
duke@435 | 1848 | Node* cmps[NCMPS] = { cmpxy, cmpyx }; |
duke@435 | 1849 | int cmpn; |
duke@435 | 1850 | for (cmpn = 0; cmpn < NCMPS; cmpn++) { |
duke@435 | 1851 | if (cmps[cmpn] != NULL) break; // find a result |
duke@435 | 1852 | } |
duke@435 | 1853 | if (cmpn < NCMPS) { |
duke@435 | 1854 | // Look for a dominating test that tells us the min and max. |
duke@435 | 1855 | int depth = 0; // Limit search depth for speed |
duke@435 | 1856 | Node* dom = control(); |
duke@435 | 1857 | for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) { |
duke@435 | 1858 | if (++depth >= 100) break; |
duke@435 | 1859 | Node* ifproj = dom; |
duke@435 | 1860 | if (!ifproj->is_Proj()) continue; |
duke@435 | 1861 | Node* iff = ifproj->in(0); |
duke@435 | 1862 | if (!iff->is_If()) continue; |
duke@435 | 1863 | Node* bol = iff->in(1); |
duke@435 | 1864 | if (!bol->is_Bool()) continue; |
duke@435 | 1865 | Node* cmp = bol->in(1); |
duke@435 | 1866 | if (cmp == NULL) continue; |
duke@435 | 1867 | for (cmpn = 0; cmpn < NCMPS; cmpn++) |
duke@435 | 1868 | if (cmps[cmpn] == cmp) break; |
duke@435 | 1869 | if (cmpn == NCMPS) continue; |
duke@435 | 1870 | BoolTest::mask btest = bol->as_Bool()->_test._test; |
duke@435 | 1871 | if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate(); |
duke@435 | 1872 | if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); |
duke@435 | 1873 | // At this point, we know that 'x btest y' is true. |
duke@435 | 1874 | switch (btest) { |
duke@435 | 1875 | case BoolTest::eq: |
duke@435 | 1876 | // They are proven equal, so we can collapse the min/max. |
duke@435 | 1877 | // Either value is the answer. Choose the simpler. |
duke@435 | 1878 | if (is_simple_name(yvalue) && !is_simple_name(xvalue)) |
duke@435 | 1879 | return yvalue; |
duke@435 | 1880 | return xvalue; |
duke@435 | 1881 | case BoolTest::lt: // x < y |
duke@435 | 1882 | case BoolTest::le: // x <= y |
duke@435 | 1883 | return (want_max ? yvalue : xvalue); |
duke@435 | 1884 | case BoolTest::gt: // x > y |
duke@435 | 1885 | case BoolTest::ge: // x >= y |
duke@435 | 1886 | return (want_max ? xvalue : yvalue); |
duke@435 | 1887 | } |
duke@435 | 1888 | } |
duke@435 | 1889 | } |
duke@435 | 1890 | |
duke@435 | 1891 | // We failed to find a dominating test. |
duke@435 | 1892 | // Let's pick a test that might GVN with prior tests. |
duke@435 | 1893 | Node* best_bol = NULL; |
duke@435 | 1894 | BoolTest::mask best_btest = BoolTest::illegal; |
duke@435 | 1895 | for (cmpn = 0; cmpn < NCMPS; cmpn++) { |
duke@435 | 1896 | Node* cmp = cmps[cmpn]; |
duke@435 | 1897 | if (cmp == NULL) continue; |
duke@435 | 1898 | for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) { |
duke@435 | 1899 | Node* bol = cmp->fast_out(j); |
duke@435 | 1900 | if (!bol->is_Bool()) continue; |
duke@435 | 1901 | BoolTest::mask btest = bol->as_Bool()->_test._test; |
duke@435 | 1902 | if (btest == BoolTest::eq || btest == BoolTest::ne) continue; |
duke@435 | 1903 | if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); |
duke@435 | 1904 | if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) { |
duke@435 | 1905 | best_bol = bol->as_Bool(); |
duke@435 | 1906 | best_btest = btest; |
duke@435 | 1907 | } |
duke@435 | 1908 | } |
duke@435 | 1909 | } |
duke@435 | 1910 | |
duke@435 | 1911 | Node* answer_if_true = NULL; |
duke@435 | 1912 | Node* answer_if_false = NULL; |
duke@435 | 1913 | switch (best_btest) { |
duke@435 | 1914 | default: |
duke@435 | 1915 | if (cmpxy == NULL) |
duke@435 | 1916 | cmpxy = ideal_cmpxy; |
duke@435 | 1917 | best_bol = _gvn.transform( new(C, 2) BoolNode(cmpxy, BoolTest::lt) ); |
duke@435 | 1918 | // and fall through: |
duke@435 | 1919 | case BoolTest::lt: // x < y |
duke@435 | 1920 | case BoolTest::le: // x <= y |
duke@435 | 1921 | answer_if_true = (want_max ? yvalue : xvalue); |
duke@435 | 1922 | answer_if_false = (want_max ? xvalue : yvalue); |
duke@435 | 1923 | break; |
duke@435 | 1924 | case BoolTest::gt: // x > y |
duke@435 | 1925 | case BoolTest::ge: // x >= y |
duke@435 | 1926 | answer_if_true = (want_max ? xvalue : yvalue); |
duke@435 | 1927 | answer_if_false = (want_max ? yvalue : xvalue); |
duke@435 | 1928 | break; |
duke@435 | 1929 | } |
duke@435 | 1930 | |
duke@435 | 1931 | jint hi, lo; |
duke@435 | 1932 | if (want_max) { |
duke@435 | 1933 | // We can sharpen the minimum. |
duke@435 | 1934 | hi = MAX2(txvalue->_hi, tyvalue->_hi); |
duke@435 | 1935 | lo = MAX2(txvalue->_lo, tyvalue->_lo); |
duke@435 | 1936 | } else { |
duke@435 | 1937 | // We can sharpen the maximum. |
duke@435 | 1938 | hi = MIN2(txvalue->_hi, tyvalue->_hi); |
duke@435 | 1939 | lo = MIN2(txvalue->_lo, tyvalue->_lo); |
duke@435 | 1940 | } |
duke@435 | 1941 | |
duke@435 | 1942 | // Use a flow-free graph structure, to avoid creating excess control edges |
duke@435 | 1943 | // which could hinder other optimizations. |
duke@435 | 1944 | // Since Math.min/max is often used with arraycopy, we want |
duke@435 | 1945 | // tightly_coupled_allocation to be able to see beyond min/max expressions. |
duke@435 | 1946 | Node* cmov = CMoveNode::make(C, NULL, best_bol, |
duke@435 | 1947 | answer_if_false, answer_if_true, |
duke@435 | 1948 | TypeInt::make(lo, hi, widen)); |
duke@435 | 1949 | |
duke@435 | 1950 | return _gvn.transform(cmov); |
duke@435 | 1951 | |
duke@435 | 1952 | /* |
duke@435 | 1953 | // This is not as desirable as it may seem, since Min and Max |
duke@435 | 1954 | // nodes do not have a full set of optimizations. |
duke@435 | 1955 | // And they would interfere, anyway, with 'if' optimizations |
duke@435 | 1956 | // and with CMoveI canonical forms. |
duke@435 | 1957 | switch (id) { |
duke@435 | 1958 | case vmIntrinsics::_min: |
duke@435 | 1959 | result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break; |
duke@435 | 1960 | case vmIntrinsics::_max: |
duke@435 | 1961 | result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break; |
duke@435 | 1962 | default: |
duke@435 | 1963 | ShouldNotReachHere(); |
duke@435 | 1964 | } |
duke@435 | 1965 | */ |
duke@435 | 1966 | } |
duke@435 | 1967 | |
duke@435 | 1968 | inline int |
duke@435 | 1969 | LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) { |
duke@435 | 1970 | const TypePtr* base_type = TypePtr::NULL_PTR; |
duke@435 | 1971 | if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); |
duke@435 | 1972 | if (base_type == NULL) { |
duke@435 | 1973 | // Unknown type. |
duke@435 | 1974 | return Type::AnyPtr; |
duke@435 | 1975 | } else if (base_type == TypePtr::NULL_PTR) { |
duke@435 | 1976 | // Since this is a NULL+long form, we have to switch to a rawptr. |
duke@435 | 1977 | base = _gvn.transform( new (C, 2) CastX2PNode(offset) ); |
duke@435 | 1978 | offset = MakeConX(0); |
duke@435 | 1979 | return Type::RawPtr; |
duke@435 | 1980 | } else if (base_type->base() == Type::RawPtr) { |
duke@435 | 1981 | return Type::RawPtr; |
duke@435 | 1982 | } else if (base_type->isa_oopptr()) { |
duke@435 | 1983 | // Base is never null => always a heap address. |
duke@435 | 1984 | if (base_type->ptr() == TypePtr::NotNull) { |
duke@435 | 1985 | return Type::OopPtr; |
duke@435 | 1986 | } |
duke@435 | 1987 | // Offset is small => always a heap address. |
duke@435 | 1988 | const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); |
duke@435 | 1989 | if (offset_type != NULL && |
duke@435 | 1990 | base_type->offset() == 0 && // (should always be?) |
duke@435 | 1991 | offset_type->_lo >= 0 && |
duke@435 | 1992 | !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { |
duke@435 | 1993 | return Type::OopPtr; |
duke@435 | 1994 | } |
duke@435 | 1995 | // Otherwise, it might either be oop+off or NULL+addr. |
duke@435 | 1996 | return Type::AnyPtr; |
duke@435 | 1997 | } else { |
duke@435 | 1998 | // No information: |
duke@435 | 1999 | return Type::AnyPtr; |
duke@435 | 2000 | } |
duke@435 | 2001 | } |
duke@435 | 2002 | |
duke@435 | 2003 | inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) { |
duke@435 | 2004 | int kind = classify_unsafe_addr(base, offset); |
duke@435 | 2005 | if (kind == Type::RawPtr) { |
duke@435 | 2006 | return basic_plus_adr(top(), base, offset); |
duke@435 | 2007 | } else { |
duke@435 | 2008 | return basic_plus_adr(base, offset); |
duke@435 | 2009 | } |
duke@435 | 2010 | } |
duke@435 | 2011 | |
twisti@1210 | 2012 | //-------------------inline_numberOfLeadingZeros_int/long----------------------- |
twisti@1210 | 2013 | // inline int Integer.numberOfLeadingZeros(int) |
twisti@1210 | 2014 | // inline int Long.numberOfLeadingZeros(long) |
twisti@1210 | 2015 | bool LibraryCallKit::inline_numberOfLeadingZeros(vmIntrinsics::ID id) { |
twisti@1210 | 2016 | assert(id == vmIntrinsics::_numberOfLeadingZeros_i || id == vmIntrinsics::_numberOfLeadingZeros_l, "not numberOfLeadingZeros"); |
twisti@1210 | 2017 | if (id == vmIntrinsics::_numberOfLeadingZeros_i && !Matcher::match_rule_supported(Op_CountLeadingZerosI)) return false; |
twisti@1210 | 2018 | if (id == vmIntrinsics::_numberOfLeadingZeros_l && !Matcher::match_rule_supported(Op_CountLeadingZerosL)) return false; |
twisti@1210 | 2019 | _sp += arg_size(); // restore stack pointer |
twisti@1210 | 2020 | switch (id) { |
twisti@1210 | 2021 | case vmIntrinsics::_numberOfLeadingZeros_i: |
twisti@1210 | 2022 | push(_gvn.transform(new (C, 2) CountLeadingZerosINode(pop()))); |
twisti@1210 | 2023 | break; |
twisti@1210 | 2024 | case vmIntrinsics::_numberOfLeadingZeros_l: |
twisti@1210 | 2025 | push(_gvn.transform(new (C, 2) CountLeadingZerosLNode(pop_pair()))); |
twisti@1210 | 2026 | break; |
twisti@1210 | 2027 | default: |
twisti@1210 | 2028 | ShouldNotReachHere(); |
twisti@1210 | 2029 | } |
twisti@1210 | 2030 | return true; |
twisti@1210 | 2031 | } |
twisti@1210 | 2032 | |
twisti@1210 | 2033 | //-------------------inline_numberOfTrailingZeros_int/long---------------------- |
twisti@1210 | 2034 | // inline int Integer.numberOfTrailingZeros(int) |
twisti@1210 | 2035 | // inline int Long.numberOfTrailingZeros(long) |
twisti@1210 | 2036 | bool LibraryCallKit::inline_numberOfTrailingZeros(vmIntrinsics::ID id) { |
twisti@1210 | 2037 | assert(id == vmIntrinsics::_numberOfTrailingZeros_i || id == vmIntrinsics::_numberOfTrailingZeros_l, "not numberOfTrailingZeros"); |
twisti@1210 | 2038 | if (id == vmIntrinsics::_numberOfTrailingZeros_i && !Matcher::match_rule_supported(Op_CountTrailingZerosI)) return false; |
twisti@1210 | 2039 | if (id == vmIntrinsics::_numberOfTrailingZeros_l && !Matcher::match_rule_supported(Op_CountTrailingZerosL)) return false; |
twisti@1210 | 2040 | _sp += arg_size(); // restore stack pointer |
twisti@1210 | 2041 | switch (id) { |
twisti@1210 | 2042 | case vmIntrinsics::_numberOfTrailingZeros_i: |
twisti@1210 | 2043 | push(_gvn.transform(new (C, 2) CountTrailingZerosINode(pop()))); |
twisti@1210 | 2044 | break; |
twisti@1210 | 2045 | case vmIntrinsics::_numberOfTrailingZeros_l: |
twisti@1210 | 2046 | push(_gvn.transform(new (C, 2) CountTrailingZerosLNode(pop_pair()))); |
twisti@1210 | 2047 | break; |
twisti@1210 | 2048 | default: |
twisti@1210 | 2049 | ShouldNotReachHere(); |
twisti@1210 | 2050 | } |
twisti@1210 | 2051 | return true; |
twisti@1210 | 2052 | } |
twisti@1210 | 2053 | |
twisti@1078 | 2054 | //----------------------------inline_bitCount_int/long----------------------- |
twisti@1078 | 2055 | // inline int Integer.bitCount(int) |
twisti@1078 | 2056 | // inline int Long.bitCount(long) |
twisti@1078 | 2057 | bool LibraryCallKit::inline_bitCount(vmIntrinsics::ID id) { |
twisti@1078 | 2058 | assert(id == vmIntrinsics::_bitCount_i || id == vmIntrinsics::_bitCount_l, "not bitCount"); |
twisti@1078 | 2059 | if (id == vmIntrinsics::_bitCount_i && !Matcher::has_match_rule(Op_PopCountI)) return false; |
twisti@1078 | 2060 | if (id == vmIntrinsics::_bitCount_l && !Matcher::has_match_rule(Op_PopCountL)) return false; |
twisti@1078 | 2061 | _sp += arg_size(); // restore stack pointer |
twisti@1078 | 2062 | switch (id) { |
twisti@1078 | 2063 | case vmIntrinsics::_bitCount_i: |
twisti@1078 | 2064 | push(_gvn.transform(new (C, 2) PopCountINode(pop()))); |
twisti@1078 | 2065 | break; |
twisti@1078 | 2066 | case vmIntrinsics::_bitCount_l: |
twisti@1078 | 2067 | push(_gvn.transform(new (C, 2) PopCountLNode(pop_pair()))); |
twisti@1078 | 2068 | break; |
twisti@1078 | 2069 | default: |
twisti@1078 | 2070 | ShouldNotReachHere(); |
twisti@1078 | 2071 | } |
twisti@1078 | 2072 | return true; |
twisti@1078 | 2073 | } |
twisti@1078 | 2074 | |
never@1831 | 2075 | //----------------------------inline_reverseBytes_int/long/char/short------------------- |
twisti@1040 | 2076 | // inline Integer.reverseBytes(int) |
twisti@1040 | 2077 | // inline Long.reverseBytes(long) |
never@1831 | 2078 | // inline Character.reverseBytes(char) |
never@1831 | 2079 | // inline Short.reverseBytes(short) |
duke@435 | 2080 | bool LibraryCallKit::inline_reverseBytes(vmIntrinsics::ID id) { |
never@1831 | 2081 | assert(id == vmIntrinsics::_reverseBytes_i || id == vmIntrinsics::_reverseBytes_l || |
never@1831 | 2082 | id == vmIntrinsics::_reverseBytes_c || id == vmIntrinsics::_reverseBytes_s, |
never@1831 | 2083 | "not reverse Bytes"); |
never@1831 | 2084 | if (id == vmIntrinsics::_reverseBytes_i && !Matcher::has_match_rule(Op_ReverseBytesI)) return false; |
never@1831 | 2085 | if (id == vmIntrinsics::_reverseBytes_l && !Matcher::has_match_rule(Op_ReverseBytesL)) return false; |
never@1831 | 2086 | if (id == vmIntrinsics::_reverseBytes_c && !Matcher::has_match_rule(Op_ReverseBytesUS)) return false; |
never@1831 | 2087 | if (id == vmIntrinsics::_reverseBytes_s && !Matcher::has_match_rule(Op_ReverseBytesS)) return false; |
duke@435 | 2088 | _sp += arg_size(); // restore stack pointer |
duke@435 | 2089 | switch (id) { |
duke@435 | 2090 | case vmIntrinsics::_reverseBytes_i: |
duke@435 | 2091 | push(_gvn.transform(new (C, 2) ReverseBytesINode(0, pop()))); |
duke@435 | 2092 | break; |
duke@435 | 2093 | case vmIntrinsics::_reverseBytes_l: |
duke@435 | 2094 | push_pair(_gvn.transform(new (C, 2) ReverseBytesLNode(0, pop_pair()))); |
duke@435 | 2095 | break; |
never@1831 | 2096 | case vmIntrinsics::_reverseBytes_c: |
never@1831 | 2097 | push(_gvn.transform(new (C, 2) ReverseBytesUSNode(0, pop()))); |
never@1831 | 2098 | break; |
never@1831 | 2099 | case vmIntrinsics::_reverseBytes_s: |
never@1831 | 2100 | push(_gvn.transform(new (C, 2) ReverseBytesSNode(0, pop()))); |
never@1831 | 2101 | break; |
duke@435 | 2102 | default: |
duke@435 | 2103 | ; |
duke@435 | 2104 | } |
duke@435 | 2105 | return true; |
duke@435 | 2106 | } |
duke@435 | 2107 | |
duke@435 | 2108 | //----------------------------inline_unsafe_access---------------------------- |
duke@435 | 2109 | |
duke@435 | 2110 | const static BasicType T_ADDRESS_HOLDER = T_LONG; |
duke@435 | 2111 | |
johnc@2781 | 2112 | // Helper that guards and inserts a G1 pre-barrier. |
johnc@2781 | 2113 | void LibraryCallKit::insert_g1_pre_barrier(Node* base_oop, Node* offset, Node* pre_val) { |
johnc@2781 | 2114 | assert(UseG1GC, "should not call this otherwise"); |
johnc@2781 | 2115 | |
johnc@2781 | 2116 | // We could be accessing the referent field of a reference object. If so, when G1 |
johnc@2781 | 2117 | // is enabled, we need to log the value in the referent field in an SATB buffer. |
johnc@2781 | 2118 | // This routine performs some compile time filters and generates suitable |
johnc@2781 | 2119 | // runtime filters that guard the pre-barrier code. |
johnc@2781 | 2120 | |
johnc@2781 | 2121 | // Some compile time checks. |
johnc@2781 | 2122 | |
johnc@2781 | 2123 | // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? |
johnc@2781 | 2124 | const TypeX* otype = offset->find_intptr_t_type(); |
johnc@2781 | 2125 | if (otype != NULL && otype->is_con() && |
johnc@2781 | 2126 | otype->get_con() != java_lang_ref_Reference::referent_offset) { |
johnc@2781 | 2127 | // Constant offset but not the reference_offset so just return |
johnc@2781 | 2128 | return; |
johnc@2781 | 2129 | } |
johnc@2781 | 2130 | |
johnc@2781 | 2131 | // We only need to generate the runtime guards for instances. |
johnc@2781 | 2132 | const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr(); |
johnc@2781 | 2133 | if (btype != NULL) { |
johnc@2781 | 2134 | if (btype->isa_aryptr()) { |
johnc@2781 | 2135 | // Array type so nothing to do |
johnc@2781 | 2136 | return; |
johnc@2781 | 2137 | } |
johnc@2781 | 2138 | |
johnc@2781 | 2139 | const TypeInstPtr* itype = btype->isa_instptr(); |
johnc@2781 | 2140 | if (itype != NULL) { |
johnc@2781 | 2141 | // Can the klass of base_oop be statically determined |
johnc@2781 | 2142 | // to be _not_ a sub-class of Reference? |
johnc@2781 | 2143 | ciKlass* klass = itype->klass(); |
johnc@2781 | 2144 | if (klass->is_subtype_of(env()->Reference_klass()) && |
johnc@2781 | 2145 | !env()->Reference_klass()->is_subtype_of(klass)) { |
johnc@2781 | 2146 | return; |
johnc@2781 | 2147 | } |
johnc@2781 | 2148 | } |
johnc@2781 | 2149 | } |
johnc@2781 | 2150 | |
johnc@2781 | 2151 | // The compile time filters did not reject base_oop/offset so |
johnc@2781 | 2152 | // we need to generate the following runtime filters |
johnc@2781 | 2153 | // |
johnc@2781 | 2154 | // if (offset == java_lang_ref_Reference::_reference_offset) { |
johnc@2781 | 2155 | // if (base != null) { |
johnc@2781 | 2156 | // if (klass(base)->reference_type() != REF_NONE)) { |
johnc@2781 | 2157 | // pre_barrier(_, pre_val, ...); |
johnc@2781 | 2158 | // } |
johnc@2781 | 2159 | // } |
johnc@2781 | 2160 | // } |
johnc@2781 | 2161 | |
johnc@2781 | 2162 | float likely = PROB_LIKELY(0.999); |
johnc@2781 | 2163 | float unlikely = PROB_UNLIKELY(0.999); |
johnc@2781 | 2164 | |
johnc@2787 | 2165 | IdealKit ideal(this); |
johnc@2781 | 2166 | #define __ ideal. |
johnc@2781 | 2167 | |
stefank@3391 | 2168 | const int reference_type_offset = in_bytes(instanceKlass::reference_type_offset()); |
johnc@2781 | 2169 | |
johnc@2786 | 2170 | Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset); |
johnc@2781 | 2171 | |
johnc@2781 | 2172 | __ if_then(offset, BoolTest::eq, referent_off, unlikely); { |
johnc@2781 | 2173 | __ if_then(base_oop, BoolTest::ne, null(), likely); { |
johnc@2781 | 2174 | |
johnc@2781 | 2175 | // Update graphKit memory and control from IdealKit. |
johnc@2787 | 2176 | sync_kit(ideal); |
johnc@2781 | 2177 | |
johnc@2781 | 2178 | Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass())); |
johnc@2781 | 2179 | Node* is_instof = gen_instanceof(base_oop, ref_klass_con); |
johnc@2781 | 2180 | |
johnc@2781 | 2181 | // Update IdealKit memory and control from graphKit. |
johnc@2787 | 2182 | __ sync_kit(this); |
johnc@2781 | 2183 | |
johnc@2781 | 2184 | Node* one = __ ConI(1); |
johnc@2781 | 2185 | |
johnc@2781 | 2186 | __ if_then(is_instof, BoolTest::eq, one, unlikely); { |
johnc@2781 | 2187 | |
johnc@2781 | 2188 | // Update graphKit from IdeakKit. |
johnc@2787 | 2189 | sync_kit(ideal); |
johnc@2781 | 2190 | |
johnc@2781 | 2191 | // Use the pre-barrier to record the value in the referent field |
johnc@2781 | 2192 | pre_barrier(false /* do_load */, |
johnc@2781 | 2193 | __ ctrl(), |
johnc@2790 | 2194 | NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */, |
johnc@2781 | 2195 | pre_val /* pre_val */, |
johnc@2781 | 2196 | T_OBJECT); |
johnc@2781 | 2197 | |
johnc@2781 | 2198 | // Update IdealKit from graphKit. |
johnc@2787 | 2199 | __ sync_kit(this); |
johnc@2781 | 2200 | |
johnc@2781 | 2201 | } __ end_if(); // _ref_type != ref_none |
johnc@2781 | 2202 | } __ end_if(); // base != NULL |
johnc@2781 | 2203 | } __ end_if(); // offset == referent_offset |
johnc@2781 | 2204 | |
johnc@2781 | 2205 | // Final sync IdealKit and GraphKit. |
johnc@2787 | 2206 | final_sync(ideal); |
johnc@2781 | 2207 | #undef __ |
johnc@2781 | 2208 | } |
johnc@2781 | 2209 | |
johnc@2781 | 2210 | |
duke@435 | 2211 | // Interpret Unsafe.fieldOffset cookies correctly: |
duke@435 | 2212 | extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset); |
duke@435 | 2213 | |
duke@435 | 2214 | bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) { |
duke@435 | 2215 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2216 | |
duke@435 | 2217 | #ifndef PRODUCT |
duke@435 | 2218 | { |
duke@435 | 2219 | ResourceMark rm; |
duke@435 | 2220 | // Check the signatures. |
duke@435 | 2221 | ciSignature* sig = signature(); |
duke@435 | 2222 | #ifdef ASSERT |
duke@435 | 2223 | if (!is_store) { |
duke@435 | 2224 | // Object getObject(Object base, int/long offset), etc. |
duke@435 | 2225 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2226 | if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name()) |
duke@435 | 2227 | rtype = T_ADDRESS; // it is really a C void* |
duke@435 | 2228 | assert(rtype == type, "getter must return the expected value"); |
duke@435 | 2229 | if (!is_native_ptr) { |
duke@435 | 2230 | assert(sig->count() == 2, "oop getter has 2 arguments"); |
duke@435 | 2231 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); |
duke@435 | 2232 | assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); |
duke@435 | 2233 | } else { |
duke@435 | 2234 | assert(sig->count() == 1, "native getter has 1 argument"); |
duke@435 | 2235 | assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long"); |
duke@435 | 2236 | } |
duke@435 | 2237 | } else { |
duke@435 | 2238 | // void putObject(Object base, int/long offset, Object x), etc. |
duke@435 | 2239 | assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); |
duke@435 | 2240 | if (!is_native_ptr) { |
duke@435 | 2241 | assert(sig->count() == 3, "oop putter has 3 arguments"); |
duke@435 | 2242 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); |
duke@435 | 2243 | assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); |
duke@435 | 2244 | } else { |
duke@435 | 2245 | assert(sig->count() == 2, "native putter has 2 arguments"); |
duke@435 | 2246 | assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long"); |
duke@435 | 2247 | } |
duke@435 | 2248 | BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); |
duke@435 | 2249 | if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name()) |
duke@435 | 2250 | vtype = T_ADDRESS; // it is really a C void* |
duke@435 | 2251 | assert(vtype == type, "putter must accept the expected value"); |
duke@435 | 2252 | } |
duke@435 | 2253 | #endif // ASSERT |
duke@435 | 2254 | } |
duke@435 | 2255 | #endif //PRODUCT |
duke@435 | 2256 | |
duke@435 | 2257 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2258 | |
duke@435 | 2259 | int type_words = type2size[ (type == T_ADDRESS) ? T_LONG : type ]; |
duke@435 | 2260 | |
duke@435 | 2261 | // Argument words: "this" plus (oop/offset) or (lo/hi) args plus maybe 1 or 2 value words |
duke@435 | 2262 | int nargs = 1 + (is_native_ptr ? 2 : 3) + (is_store ? type_words : 0); |
duke@435 | 2263 | |
duke@435 | 2264 | debug_only(int saved_sp = _sp); |
duke@435 | 2265 | _sp += nargs; |
duke@435 | 2266 | |
duke@435 | 2267 | Node* val; |
duke@435 | 2268 | debug_only(val = (Node*)(uintptr_t)-1); |
duke@435 | 2269 | |
duke@435 | 2270 | |
duke@435 | 2271 | if (is_store) { |
duke@435 | 2272 | // Get the value being stored. (Pop it first; it was pushed last.) |
duke@435 | 2273 | switch (type) { |
duke@435 | 2274 | case T_DOUBLE: |
duke@435 | 2275 | case T_LONG: |
duke@435 | 2276 | case T_ADDRESS: |
duke@435 | 2277 | val = pop_pair(); |
duke@435 | 2278 | break; |
duke@435 | 2279 | default: |
duke@435 | 2280 | val = pop(); |
duke@435 | 2281 | } |
duke@435 | 2282 | } |
duke@435 | 2283 | |
duke@435 | 2284 | // Build address expression. See the code in inline_unsafe_prefetch. |
duke@435 | 2285 | Node *adr; |
duke@435 | 2286 | Node *heap_base_oop = top(); |
johnc@2781 | 2287 | Node* offset = top(); |
johnc@2781 | 2288 | |
duke@435 | 2289 | if (!is_native_ptr) { |
duke@435 | 2290 | // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset |
johnc@2781 | 2291 | offset = pop_pair(); |
duke@435 | 2292 | // The base is either a Java object or a value produced by Unsafe.staticFieldBase |
duke@435 | 2293 | Node* base = pop(); |
duke@435 | 2294 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2295 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2296 | // by oopDesc::field_base. |
duke@435 | 2297 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 2298 | "fieldOffset must be byte-scaled"); |
duke@435 | 2299 | // 32-bit machines ignore the high half! |
duke@435 | 2300 | offset = ConvL2X(offset); |
duke@435 | 2301 | adr = make_unsafe_address(base, offset); |
duke@435 | 2302 | heap_base_oop = base; |
duke@435 | 2303 | } else { |
duke@435 | 2304 | Node* ptr = pop_pair(); |
duke@435 | 2305 | // Adjust Java long to machine word: |
duke@435 | 2306 | ptr = ConvL2X(ptr); |
duke@435 | 2307 | adr = make_unsafe_address(NULL, ptr); |
duke@435 | 2308 | } |
duke@435 | 2309 | |
duke@435 | 2310 | // Pop receiver last: it was pushed first. |
duke@435 | 2311 | Node *receiver = pop(); |
duke@435 | 2312 | |
duke@435 | 2313 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2314 | |
duke@435 | 2315 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2316 | |
duke@435 | 2317 | // First guess at the value type. |
duke@435 | 2318 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2319 | |
duke@435 | 2320 | // Try to categorize the address. If it comes up as TypeJavaPtr::BOTTOM, |
duke@435 | 2321 | // there was not enough information to nail it down. |
duke@435 | 2322 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2323 | assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); |
duke@435 | 2324 | |
duke@435 | 2325 | // We will need memory barriers unless we can determine a unique |
duke@435 | 2326 | // alias category for this reference. (Note: If for some reason |
duke@435 | 2327 | // the barriers get omitted and the unsafe reference begins to "pollute" |
duke@435 | 2328 | // the alias analysis of the rest of the graph, either Compile::can_alias |
duke@435 | 2329 | // or Compile::must_alias will throw a diagnostic assert.) |
duke@435 | 2330 | bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM); |
duke@435 | 2331 | |
johnc@2781 | 2332 | // If we are reading the value of the referent field of a Reference |
johnc@2781 | 2333 | // object (either by using Unsafe directly or through reflection) |
johnc@2781 | 2334 | // then, if G1 is enabled, we need to record the referent in an |
johnc@2781 | 2335 | // SATB log buffer using the pre-barrier mechanism. |
johnc@2781 | 2336 | bool need_read_barrier = UseG1GC && !is_native_ptr && !is_store && |
johnc@2781 | 2337 | offset != top() && heap_base_oop != top(); |
johnc@2781 | 2338 | |
duke@435 | 2339 | if (!is_store && type == T_OBJECT) { |
duke@435 | 2340 | // Attempt to infer a sharper value type from the offset and base type. |
duke@435 | 2341 | ciKlass* sharpened_klass = NULL; |
duke@435 | 2342 | |
duke@435 | 2343 | // See if it is an instance field, with an object type. |
duke@435 | 2344 | if (alias_type->field() != NULL) { |
duke@435 | 2345 | assert(!is_native_ptr, "native pointer op cannot use a java address"); |
duke@435 | 2346 | if (alias_type->field()->type()->is_klass()) { |
duke@435 | 2347 | sharpened_klass = alias_type->field()->type()->as_klass(); |
duke@435 | 2348 | } |
duke@435 | 2349 | } |
duke@435 | 2350 | |
duke@435 | 2351 | // See if it is a narrow oop array. |
duke@435 | 2352 | if (adr_type->isa_aryptr()) { |
twisti@1330 | 2353 | if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { |
duke@435 | 2354 | const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr(); |
duke@435 | 2355 | if (elem_type != NULL) { |
duke@435 | 2356 | sharpened_klass = elem_type->klass(); |
duke@435 | 2357 | } |
duke@435 | 2358 | } |
duke@435 | 2359 | } |
duke@435 | 2360 | |
duke@435 | 2361 | if (sharpened_klass != NULL) { |
duke@435 | 2362 | const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass); |
duke@435 | 2363 | |
duke@435 | 2364 | // Sharpen the value type. |
duke@435 | 2365 | value_type = tjp; |
duke@435 | 2366 | |
duke@435 | 2367 | #ifndef PRODUCT |
duke@435 | 2368 | if (PrintIntrinsics || PrintInlining || PrintOptoInlining) { |
duke@435 | 2369 | tty->print(" from base type: "); adr_type->dump(); |
duke@435 | 2370 | tty->print(" sharpened value: "); value_type->dump(); |
duke@435 | 2371 | } |
duke@435 | 2372 | #endif |
duke@435 | 2373 | } |
duke@435 | 2374 | } |
duke@435 | 2375 | |
duke@435 | 2376 | // Null check on self without removing any arguments. The argument |
duke@435 | 2377 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 2378 | // invalid stack traces when the primitive is inlined into a method |
duke@435 | 2379 | // which handles NullPointerExceptions. |
duke@435 | 2380 | _sp += nargs; |
duke@435 | 2381 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2382 | _sp -= nargs; |
duke@435 | 2383 | if (stopped()) { |
duke@435 | 2384 | return true; |
duke@435 | 2385 | } |
duke@435 | 2386 | // Heap pointers get a null-check from the interpreter, |
duke@435 | 2387 | // as a courtesy. However, this is not guaranteed by Unsafe, |
duke@435 | 2388 | // and it is not possible to fully distinguish unintended nulls |
duke@435 | 2389 | // from intended ones in this API. |
duke@435 | 2390 | |
duke@435 | 2391 | if (is_volatile) { |
duke@435 | 2392 | // We need to emit leading and trailing CPU membars (see below) in |
duke@435 | 2393 | // addition to memory membars when is_volatile. This is a little |
duke@435 | 2394 | // too strong, but avoids the need to insert per-alias-type |
duke@435 | 2395 | // volatile membars (for stores; compare Parse::do_put_xxx), which |
twisti@1040 | 2396 | // we cannot do effectively here because we probably only have a |
duke@435 | 2397 | // rough approximation of type. |
duke@435 | 2398 | need_mem_bar = true; |
duke@435 | 2399 | // For Stores, place a memory ordering barrier now. |
duke@435 | 2400 | if (is_store) |
duke@435 | 2401 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2402 | } |
duke@435 | 2403 | |
duke@435 | 2404 | // Memory barrier to prevent normal and 'unsafe' accesses from |
duke@435 | 2405 | // bypassing each other. Happens after null checks, so the |
duke@435 | 2406 | // exception paths do not take memory state from the memory barrier, |
duke@435 | 2407 | // so there's no problems making a strong assert about mixing users |
duke@435 | 2408 | // of safe & unsafe memory. Otherwise fails in a CTW of rt.jar |
duke@435 | 2409 | // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl. |
duke@435 | 2410 | if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2411 | |
duke@435 | 2412 | if (!is_store) { |
duke@435 | 2413 | Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile); |
duke@435 | 2414 | // load value and push onto stack |
duke@435 | 2415 | switch (type) { |
duke@435 | 2416 | case T_BOOLEAN: |
duke@435 | 2417 | case T_CHAR: |
duke@435 | 2418 | case T_BYTE: |
duke@435 | 2419 | case T_SHORT: |
duke@435 | 2420 | case T_INT: |
duke@435 | 2421 | case T_FLOAT: |
johnc@2781 | 2422 | push(p); |
johnc@2781 | 2423 | break; |
duke@435 | 2424 | case T_OBJECT: |
johnc@2781 | 2425 | if (need_read_barrier) { |
johnc@2781 | 2426 | insert_g1_pre_barrier(heap_base_oop, offset, p); |
johnc@2781 | 2427 | } |
johnc@2781 | 2428 | push(p); |
duke@435 | 2429 | break; |
duke@435 | 2430 | case T_ADDRESS: |
duke@435 | 2431 | // Cast to an int type. |
duke@435 | 2432 | p = _gvn.transform( new (C, 2) CastP2XNode(NULL,p) ); |
duke@435 | 2433 | p = ConvX2L(p); |
duke@435 | 2434 | push_pair(p); |
duke@435 | 2435 | break; |
duke@435 | 2436 | case T_DOUBLE: |
duke@435 | 2437 | case T_LONG: |
duke@435 | 2438 | push_pair( p ); |
duke@435 | 2439 | break; |
duke@435 | 2440 | default: ShouldNotReachHere(); |
duke@435 | 2441 | } |
duke@435 | 2442 | } else { |
duke@435 | 2443 | // place effect of store into memory |
duke@435 | 2444 | switch (type) { |
duke@435 | 2445 | case T_DOUBLE: |
duke@435 | 2446 | val = dstore_rounding(val); |
duke@435 | 2447 | break; |
duke@435 | 2448 | case T_ADDRESS: |
duke@435 | 2449 | // Repackage the long as a pointer. |
duke@435 | 2450 | val = ConvL2X(val); |
duke@435 | 2451 | val = _gvn.transform( new (C, 2) CastX2PNode(val) ); |
duke@435 | 2452 | break; |
duke@435 | 2453 | } |
duke@435 | 2454 | |
duke@435 | 2455 | if (type != T_OBJECT ) { |
duke@435 | 2456 | (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile); |
duke@435 | 2457 | } else { |
duke@435 | 2458 | // Possibly an oop being stored to Java heap or native memory |
duke@435 | 2459 | if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) { |
duke@435 | 2460 | // oop to Java heap. |
never@1260 | 2461 | (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type); |
duke@435 | 2462 | } else { |
duke@435 | 2463 | // We can't tell at compile time if we are storing in the Java heap or outside |
duke@435 | 2464 | // of it. So we need to emit code to conditionally do the proper type of |
duke@435 | 2465 | // store. |
duke@435 | 2466 | |
kvn@2726 | 2467 | IdealKit ideal(this); |
kvn@1286 | 2468 | #define __ ideal. |
duke@435 | 2469 | // QQQ who knows what probability is here?? |
kvn@1286 | 2470 | __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); { |
kvn@1286 | 2471 | // Sync IdealKit and graphKit. |
kvn@2726 | 2472 | sync_kit(ideal); |
kvn@1286 | 2473 | Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type); |
kvn@1286 | 2474 | // Update IdealKit memory. |
kvn@2726 | 2475 | __ sync_kit(this); |
kvn@1286 | 2476 | } __ else_(); { |
kvn@1286 | 2477 | __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile); |
kvn@1286 | 2478 | } __ end_if(); |
kvn@1286 | 2479 | // Final sync IdealKit and GraphKit. |
kvn@2726 | 2480 | final_sync(ideal); |
kvn@1286 | 2481 | #undef __ |
duke@435 | 2482 | } |
duke@435 | 2483 | } |
duke@435 | 2484 | } |
duke@435 | 2485 | |
duke@435 | 2486 | if (is_volatile) { |
duke@435 | 2487 | if (!is_store) |
duke@435 | 2488 | insert_mem_bar(Op_MemBarAcquire); |
duke@435 | 2489 | else |
duke@435 | 2490 | insert_mem_bar(Op_MemBarVolatile); |
duke@435 | 2491 | } |
duke@435 | 2492 | |
duke@435 | 2493 | if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2494 | |
duke@435 | 2495 | return true; |
duke@435 | 2496 | } |
duke@435 | 2497 | |
duke@435 | 2498 | //----------------------------inline_unsafe_prefetch---------------------------- |
duke@435 | 2499 | |
duke@435 | 2500 | bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) { |
duke@435 | 2501 | #ifndef PRODUCT |
duke@435 | 2502 | { |
duke@435 | 2503 | ResourceMark rm; |
duke@435 | 2504 | // Check the signatures. |
duke@435 | 2505 | ciSignature* sig = signature(); |
duke@435 | 2506 | #ifdef ASSERT |
duke@435 | 2507 | // Object getObject(Object base, int/long offset), etc. |
duke@435 | 2508 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2509 | if (!is_native_ptr) { |
duke@435 | 2510 | assert(sig->count() == 2, "oop prefetch has 2 arguments"); |
duke@435 | 2511 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object"); |
duke@435 | 2512 | assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct"); |
duke@435 | 2513 | } else { |
duke@435 | 2514 | assert(sig->count() == 1, "native prefetch has 1 argument"); |
duke@435 | 2515 | assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long"); |
duke@435 | 2516 | } |
duke@435 | 2517 | #endif // ASSERT |
duke@435 | 2518 | } |
duke@435 | 2519 | #endif // !PRODUCT |
duke@435 | 2520 | |
duke@435 | 2521 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2522 | |
duke@435 | 2523 | // Argument words: "this" if not static, plus (oop/offset) or (lo/hi) args |
duke@435 | 2524 | int nargs = (is_static ? 0 : 1) + (is_native_ptr ? 2 : 3); |
duke@435 | 2525 | |
duke@435 | 2526 | debug_only(int saved_sp = _sp); |
duke@435 | 2527 | _sp += nargs; |
duke@435 | 2528 | |
duke@435 | 2529 | // Build address expression. See the code in inline_unsafe_access. |
duke@435 | 2530 | Node *adr; |
duke@435 | 2531 | if (!is_native_ptr) { |
duke@435 | 2532 | // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset |
duke@435 | 2533 | Node* offset = pop_pair(); |
duke@435 | 2534 | // The base is either a Java object or a value produced by Unsafe.staticFieldBase |
duke@435 | 2535 | Node* base = pop(); |
duke@435 | 2536 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2537 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2538 | // by oopDesc::field_base. |
duke@435 | 2539 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 2540 | "fieldOffset must be byte-scaled"); |
duke@435 | 2541 | // 32-bit machines ignore the high half! |
duke@435 | 2542 | offset = ConvL2X(offset); |
duke@435 | 2543 | adr = make_unsafe_address(base, offset); |
duke@435 | 2544 | } else { |
duke@435 | 2545 | Node* ptr = pop_pair(); |
duke@435 | 2546 | // Adjust Java long to machine word: |
duke@435 | 2547 | ptr = ConvL2X(ptr); |
duke@435 | 2548 | adr = make_unsafe_address(NULL, ptr); |
duke@435 | 2549 | } |
duke@435 | 2550 | |
duke@435 | 2551 | if (is_static) { |
duke@435 | 2552 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2553 | } else { |
duke@435 | 2554 | // Pop receiver last: it was pushed first. |
duke@435 | 2555 | Node *receiver = pop(); |
duke@435 | 2556 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2557 | |
duke@435 | 2558 | // Null check on self without removing any arguments. The argument |
duke@435 | 2559 | // null check technically happens in the wrong place, which can lead to |
duke@435 | 2560 | // invalid stack traces when the primitive is inlined into a method |
duke@435 | 2561 | // which handles NullPointerExceptions. |
duke@435 | 2562 | _sp += nargs; |
duke@435 | 2563 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2564 | _sp -= nargs; |
duke@435 | 2565 | if (stopped()) { |
duke@435 | 2566 | return true; |
duke@435 | 2567 | } |
duke@435 | 2568 | } |
duke@435 | 2569 | |
duke@435 | 2570 | // Generate the read or write prefetch |
duke@435 | 2571 | Node *prefetch; |
duke@435 | 2572 | if (is_store) { |
duke@435 | 2573 | prefetch = new (C, 3) PrefetchWriteNode(i_o(), adr); |
duke@435 | 2574 | } else { |
duke@435 | 2575 | prefetch = new (C, 3) PrefetchReadNode(i_o(), adr); |
duke@435 | 2576 | } |
duke@435 | 2577 | prefetch->init_req(0, control()); |
duke@435 | 2578 | set_i_o(_gvn.transform(prefetch)); |
duke@435 | 2579 | |
duke@435 | 2580 | return true; |
duke@435 | 2581 | } |
duke@435 | 2582 | |
duke@435 | 2583 | //----------------------------inline_unsafe_CAS---------------------------- |
duke@435 | 2584 | |
duke@435 | 2585 | bool LibraryCallKit::inline_unsafe_CAS(BasicType type) { |
duke@435 | 2586 | // This basic scheme here is the same as inline_unsafe_access, but |
duke@435 | 2587 | // differs in enough details that combining them would make the code |
duke@435 | 2588 | // overly confusing. (This is a true fact! I originally combined |
duke@435 | 2589 | // them, but even I was confused by it!) As much code/comments as |
duke@435 | 2590 | // possible are retained from inline_unsafe_access though to make |
twisti@1040 | 2591 | // the correspondences clearer. - dl |
duke@435 | 2592 | |
duke@435 | 2593 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2594 | |
duke@435 | 2595 | #ifndef PRODUCT |
duke@435 | 2596 | { |
duke@435 | 2597 | ResourceMark rm; |
duke@435 | 2598 | // Check the signatures. |
duke@435 | 2599 | ciSignature* sig = signature(); |
duke@435 | 2600 | #ifdef ASSERT |
duke@435 | 2601 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2602 | assert(rtype == T_BOOLEAN, "CAS must return boolean"); |
duke@435 | 2603 | assert(sig->count() == 4, "CAS has 4 arguments"); |
duke@435 | 2604 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); |
duke@435 | 2605 | assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); |
duke@435 | 2606 | #endif // ASSERT |
duke@435 | 2607 | } |
duke@435 | 2608 | #endif //PRODUCT |
duke@435 | 2609 | |
duke@435 | 2610 | // number of stack slots per value argument (1 or 2) |
duke@435 | 2611 | int type_words = type2size[type]; |
duke@435 | 2612 | |
duke@435 | 2613 | // Cannot inline wide CAS on machines that don't support it natively |
kvn@464 | 2614 | if (type2aelembytes(type) > BytesPerInt && !VM_Version::supports_cx8()) |
duke@435 | 2615 | return false; |
duke@435 | 2616 | |
duke@435 | 2617 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2618 | |
duke@435 | 2619 | // Argument words: "this" plus oop plus offset plus oldvalue plus newvalue; |
duke@435 | 2620 | int nargs = 1 + 1 + 2 + type_words + type_words; |
duke@435 | 2621 | |
duke@435 | 2622 | // pop arguments: newval, oldval, offset, base, and receiver |
duke@435 | 2623 | debug_only(int saved_sp = _sp); |
duke@435 | 2624 | _sp += nargs; |
duke@435 | 2625 | Node* newval = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2626 | Node* oldval = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2627 | Node *offset = pop_pair(); |
duke@435 | 2628 | Node *base = pop(); |
duke@435 | 2629 | Node *receiver = pop(); |
duke@435 | 2630 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2631 | |
duke@435 | 2632 | // Null check receiver. |
duke@435 | 2633 | _sp += nargs; |
duke@435 | 2634 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2635 | _sp -= nargs; |
duke@435 | 2636 | if (stopped()) { |
duke@435 | 2637 | return true; |
duke@435 | 2638 | } |
duke@435 | 2639 | |
duke@435 | 2640 | // Build field offset expression. |
duke@435 | 2641 | // We currently rely on the cookies produced by Unsafe.xxxFieldOffset |
duke@435 | 2642 | // to be plain byte offsets, which are also the same as those accepted |
duke@435 | 2643 | // by oopDesc::field_base. |
duke@435 | 2644 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); |
duke@435 | 2645 | // 32-bit machines ignore the high half of long offsets |
duke@435 | 2646 | offset = ConvL2X(offset); |
duke@435 | 2647 | Node* adr = make_unsafe_address(base, offset); |
duke@435 | 2648 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2649 | |
duke@435 | 2650 | // (Unlike inline_unsafe_access, there seems no point in trying |
duke@435 | 2651 | // to refine types. Just use the coarse types here. |
duke@435 | 2652 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2653 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2654 | assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); |
duke@435 | 2655 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 2656 | |
duke@435 | 2657 | // Memory-model-wise, a CAS acts like a little synchronized block, |
twisti@1040 | 2658 | // so needs barriers on each side. These don't translate into |
duke@435 | 2659 | // actual barriers on most machines, but we still need rest of |
duke@435 | 2660 | // compiler to respect ordering. |
duke@435 | 2661 | |
duke@435 | 2662 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2663 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2664 | |
duke@435 | 2665 | // 4984716: MemBars must be inserted before this |
duke@435 | 2666 | // memory node in order to avoid a false |
duke@435 | 2667 | // dependency which will confuse the scheduler. |
duke@435 | 2668 | Node *mem = memory(alias_idx); |
duke@435 | 2669 | |
duke@435 | 2670 | // For now, we handle only those cases that actually exist: ints, |
duke@435 | 2671 | // longs, and Object. Adding others should be straightforward. |
duke@435 | 2672 | Node* cas; |
duke@435 | 2673 | switch(type) { |
duke@435 | 2674 | case T_INT: |
duke@435 | 2675 | cas = _gvn.transform(new (C, 5) CompareAndSwapINode(control(), mem, adr, newval, oldval)); |
duke@435 | 2676 | break; |
duke@435 | 2677 | case T_LONG: |
duke@435 | 2678 | cas = _gvn.transform(new (C, 5) CompareAndSwapLNode(control(), mem, adr, newval, oldval)); |
duke@435 | 2679 | break; |
duke@435 | 2680 | case T_OBJECT: |
kvn@3521 | 2681 | // Transformation of a value which could be NULL pointer (CastPP #NULL) |
kvn@3521 | 2682 | // could be delayed during Parse (for example, in adjust_map_after_if()). |
kvn@3521 | 2683 | // Execute transformation here to avoid barrier generation in such case. |
kvn@3521 | 2684 | if (_gvn.type(newval) == TypePtr::NULL_PTR) |
kvn@3521 | 2685 | newval = _gvn.makecon(TypePtr::NULL_PTR); |
kvn@3521 | 2686 | |
kvn@3521 | 2687 | // Reference stores need a store barrier. |
duke@435 | 2688 | // (They don't if CAS fails, but it isn't worth checking.) |
johnc@2781 | 2689 | pre_barrier(true /* do_load*/, |
johnc@2781 | 2690 | control(), base, adr, alias_idx, newval, value_type->make_oopptr(), |
johnc@2781 | 2691 | NULL /* pre_val*/, |
johnc@2781 | 2692 | T_OBJECT); |
coleenp@548 | 2693 | #ifdef _LP64 |
kvn@598 | 2694 | if (adr->bottom_type()->is_ptr_to_narrowoop()) { |
kvn@656 | 2695 | Node *newval_enc = _gvn.transform(new (C, 2) EncodePNode(newval, newval->bottom_type()->make_narrowoop())); |
kvn@656 | 2696 | Node *oldval_enc = _gvn.transform(new (C, 2) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop())); |
coleenp@548 | 2697 | cas = _gvn.transform(new (C, 5) CompareAndSwapNNode(control(), mem, adr, |
kvn@656 | 2698 | newval_enc, oldval_enc)); |
coleenp@548 | 2699 | } else |
coleenp@548 | 2700 | #endif |
kvn@656 | 2701 | { |
kvn@656 | 2702 | cas = _gvn.transform(new (C, 5) CompareAndSwapPNode(control(), mem, adr, newval, oldval)); |
kvn@656 | 2703 | } |
duke@435 | 2704 | post_barrier(control(), cas, base, adr, alias_idx, newval, T_OBJECT, true); |
duke@435 | 2705 | break; |
duke@435 | 2706 | default: |
duke@435 | 2707 | ShouldNotReachHere(); |
duke@435 | 2708 | break; |
duke@435 | 2709 | } |
duke@435 | 2710 | |
duke@435 | 2711 | // SCMemProjNodes represent the memory state of CAS. Their main |
duke@435 | 2712 | // role is to prevent CAS nodes from being optimized away when their |
duke@435 | 2713 | // results aren't used. |
duke@435 | 2714 | Node* proj = _gvn.transform( new (C, 1) SCMemProjNode(cas)); |
duke@435 | 2715 | set_memory(proj, alias_idx); |
duke@435 | 2716 | |
duke@435 | 2717 | // Add the trailing membar surrounding the access |
duke@435 | 2718 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2719 | insert_mem_bar(Op_MemBarAcquire); |
duke@435 | 2720 | |
duke@435 | 2721 | push(cas); |
duke@435 | 2722 | return true; |
duke@435 | 2723 | } |
duke@435 | 2724 | |
duke@435 | 2725 | bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) { |
duke@435 | 2726 | // This is another variant of inline_unsafe_access, differing in |
duke@435 | 2727 | // that it always issues store-store ("release") barrier and ensures |
duke@435 | 2728 | // store-atomicity (which only matters for "long"). |
duke@435 | 2729 | |
duke@435 | 2730 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2731 | |
duke@435 | 2732 | #ifndef PRODUCT |
duke@435 | 2733 | { |
duke@435 | 2734 | ResourceMark rm; |
duke@435 | 2735 | // Check the signatures. |
duke@435 | 2736 | ciSignature* sig = signature(); |
duke@435 | 2737 | #ifdef ASSERT |
duke@435 | 2738 | BasicType rtype = sig->return_type()->basic_type(); |
duke@435 | 2739 | assert(rtype == T_VOID, "must return void"); |
duke@435 | 2740 | assert(sig->count() == 3, "has 3 arguments"); |
duke@435 | 2741 | assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object"); |
duke@435 | 2742 | assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long"); |
duke@435 | 2743 | #endif // ASSERT |
duke@435 | 2744 | } |
duke@435 | 2745 | #endif //PRODUCT |
duke@435 | 2746 | |
duke@435 | 2747 | // number of stack slots per value argument (1 or 2) |
duke@435 | 2748 | int type_words = type2size[type]; |
duke@435 | 2749 | |
duke@435 | 2750 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 2751 | |
duke@435 | 2752 | // Argument words: "this" plus oop plus offset plus value; |
duke@435 | 2753 | int nargs = 1 + 1 + 2 + type_words; |
duke@435 | 2754 | |
duke@435 | 2755 | // pop arguments: val, offset, base, and receiver |
duke@435 | 2756 | debug_only(int saved_sp = _sp); |
duke@435 | 2757 | _sp += nargs; |
duke@435 | 2758 | Node* val = (type_words == 1) ? pop() : pop_pair(); |
duke@435 | 2759 | Node *offset = pop_pair(); |
duke@435 | 2760 | Node *base = pop(); |
duke@435 | 2761 | Node *receiver = pop(); |
duke@435 | 2762 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 2763 | |
duke@435 | 2764 | // Null check receiver. |
duke@435 | 2765 | _sp += nargs; |
duke@435 | 2766 | do_null_check(receiver, T_OBJECT); |
duke@435 | 2767 | _sp -= nargs; |
duke@435 | 2768 | if (stopped()) { |
duke@435 | 2769 | return true; |
duke@435 | 2770 | } |
duke@435 | 2771 | |
duke@435 | 2772 | // Build field offset expression. |
duke@435 | 2773 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); |
duke@435 | 2774 | // 32-bit machines ignore the high half of long offsets |
duke@435 | 2775 | offset = ConvL2X(offset); |
duke@435 | 2776 | Node* adr = make_unsafe_address(base, offset); |
duke@435 | 2777 | const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); |
duke@435 | 2778 | const Type *value_type = Type::get_const_basic_type(type); |
duke@435 | 2779 | Compile::AliasType* alias_type = C->alias_type(adr_type); |
duke@435 | 2780 | |
duke@435 | 2781 | insert_mem_bar(Op_MemBarRelease); |
duke@435 | 2782 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2783 | // Ensure that the store is atomic for longs: |
duke@435 | 2784 | bool require_atomic_access = true; |
duke@435 | 2785 | Node* store; |
duke@435 | 2786 | if (type == T_OBJECT) // reference stores need a store barrier. |
never@1260 | 2787 | store = store_oop_to_unknown(control(), base, adr, adr_type, val, type); |
duke@435 | 2788 | else { |
duke@435 | 2789 | store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access); |
duke@435 | 2790 | } |
duke@435 | 2791 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 2792 | return true; |
duke@435 | 2793 | } |
duke@435 | 2794 | |
duke@435 | 2795 | bool LibraryCallKit::inline_unsafe_allocate() { |
duke@435 | 2796 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 2797 | int nargs = 1 + 1; |
duke@435 | 2798 | assert(signature()->size() == nargs-1, "alloc has 1 argument"); |
duke@435 | 2799 | null_check_receiver(callee()); // check then ignore argument(0) |
duke@435 | 2800 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 2801 | Node* cls = do_null_check(argument(1), T_OBJECT); |
duke@435 | 2802 | _sp -= nargs; |
duke@435 | 2803 | if (stopped()) return true; |
duke@435 | 2804 | |
duke@435 | 2805 | Node* kls = load_klass_from_mirror(cls, false, nargs, NULL, 0); |
duke@435 | 2806 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 2807 | kls = do_null_check(kls, T_OBJECT); |
duke@435 | 2808 | _sp -= nargs; |
duke@435 | 2809 | if (stopped()) return true; // argument was like int.class |
duke@435 | 2810 | |
duke@435 | 2811 | // Note: The argument might still be an illegal value like |
duke@435 | 2812 | // Serializable.class or Object[].class. The runtime will handle it. |
duke@435 | 2813 | // But we must make an explicit check for initialization. |
stefank@3391 | 2814 | Node* insp = basic_plus_adr(kls, in_bytes(instanceKlass::init_state_offset())); |
coleenp@3368 | 2815 | // Use T_BOOLEAN for instanceKlass::_init_state so the compiler |
coleenp@3368 | 2816 | // can generate code to load it as unsigned byte. |
coleenp@3368 | 2817 | Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN); |
duke@435 | 2818 | Node* bits = intcon(instanceKlass::fully_initialized); |
duke@435 | 2819 | Node* test = _gvn.transform( new (C, 3) SubINode(inst, bits) ); |
duke@435 | 2820 | // The 'test' is non-zero if we need to take a slow path. |
duke@435 | 2821 | |
duke@435 | 2822 | Node* obj = new_instance(kls, test); |
duke@435 | 2823 | push(obj); |
duke@435 | 2824 | |
duke@435 | 2825 | return true; |
duke@435 | 2826 | } |
duke@435 | 2827 | |
duke@435 | 2828 | //------------------------inline_native_time_funcs-------------- |
duke@435 | 2829 | // inline code for System.currentTimeMillis() and System.nanoTime() |
duke@435 | 2830 | // these have the same type and signature |
duke@435 | 2831 | bool LibraryCallKit::inline_native_time_funcs(bool isNano) { |
duke@435 | 2832 | address funcAddr = isNano ? CAST_FROM_FN_PTR(address, os::javaTimeNanos) : |
duke@435 | 2833 | CAST_FROM_FN_PTR(address, os::javaTimeMillis); |
duke@435 | 2834 | const char * funcName = isNano ? "nanoTime" : "currentTimeMillis"; |
duke@435 | 2835 | const TypeFunc *tf = OptoRuntime::current_time_millis_Type(); |
duke@435 | 2836 | const TypePtr* no_memory_effects = NULL; |
duke@435 | 2837 | Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); |
duke@435 | 2838 | Node* value = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms+0)); |
duke@435 | 2839 | #ifdef ASSERT |
duke@435 | 2840 | Node* value_top = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms + 1)); |
duke@435 | 2841 | assert(value_top == top(), "second value must be top"); |
duke@435 | 2842 | #endif |
duke@435 | 2843 | push_pair(value); |
duke@435 | 2844 | return true; |
duke@435 | 2845 | } |
duke@435 | 2846 | |
duke@435 | 2847 | //------------------------inline_native_currentThread------------------ |
duke@435 | 2848 | bool LibraryCallKit::inline_native_currentThread() { |
duke@435 | 2849 | Node* junk = NULL; |
duke@435 | 2850 | push(generate_current_thread(junk)); |
duke@435 | 2851 | return true; |
duke@435 | 2852 | } |
duke@435 | 2853 | |
duke@435 | 2854 | //------------------------inline_native_isInterrupted------------------ |
duke@435 | 2855 | bool LibraryCallKit::inline_native_isInterrupted() { |
duke@435 | 2856 | const int nargs = 1+1; // receiver + boolean |
duke@435 | 2857 | assert(nargs == arg_size(), "sanity"); |
duke@435 | 2858 | // Add a fast path to t.isInterrupted(clear_int): |
duke@435 | 2859 | // (t == Thread.current() && (!TLS._osthread._interrupted || !clear_int)) |
duke@435 | 2860 | // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int) |
duke@435 | 2861 | // So, in the common case that the interrupt bit is false, |
duke@435 | 2862 | // we avoid making a call into the VM. Even if the interrupt bit |
duke@435 | 2863 | // is true, if the clear_int argument is false, we avoid the VM call. |
duke@435 | 2864 | // However, if the receiver is not currentThread, we must call the VM, |
duke@435 | 2865 | // because there must be some locking done around the operation. |
duke@435 | 2866 | |
duke@435 | 2867 | // We only go to the fast case code if we pass two guards. |
duke@435 | 2868 | // Paths which do not pass are accumulated in the slow_region. |
duke@435 | 2869 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 2870 | record_for_igvn(slow_region); |
duke@435 | 2871 | RegionNode* result_rgn = new (C, 4) RegionNode(1+3); // fast1, fast2, slow |
duke@435 | 2872 | PhiNode* result_val = new (C, 4) PhiNode(result_rgn, TypeInt::BOOL); |
duke@435 | 2873 | enum { no_int_result_path = 1, |
duke@435 | 2874 | no_clear_result_path = 2, |
duke@435 | 2875 | slow_result_path = 3 |
duke@435 | 2876 | }; |
duke@435 | 2877 | |
duke@435 | 2878 | // (a) Receiving thread must be the current thread. |
duke@435 | 2879 | Node* rec_thr = argument(0); |
duke@435 | 2880 | Node* tls_ptr = NULL; |
duke@435 | 2881 | Node* cur_thr = generate_current_thread(tls_ptr); |
duke@435 | 2882 | Node* cmp_thr = _gvn.transform( new (C, 3) CmpPNode(cur_thr, rec_thr) ); |
duke@435 | 2883 | Node* bol_thr = _gvn.transform( new (C, 2) BoolNode(cmp_thr, BoolTest::ne) ); |
duke@435 | 2884 | |
duke@435 | 2885 | bool known_current_thread = (_gvn.type(bol_thr) == TypeInt::ZERO); |
duke@435 | 2886 | if (!known_current_thread) |
duke@435 | 2887 | generate_slow_guard(bol_thr, slow_region); |
duke@435 | 2888 | |
duke@435 | 2889 | // (b) Interrupt bit on TLS must be false. |
duke@435 | 2890 | Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset())); |
duke@435 | 2891 | Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS); |
duke@435 | 2892 | p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset())); |
kvn@1222 | 2893 | // Set the control input on the field _interrupted read to prevent it floating up. |
kvn@1222 | 2894 | Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT); |
duke@435 | 2895 | Node* cmp_bit = _gvn.transform( new (C, 3) CmpINode(int_bit, intcon(0)) ); |
duke@435 | 2896 | Node* bol_bit = _gvn.transform( new (C, 2) BoolNode(cmp_bit, BoolTest::ne) ); |
duke@435 | 2897 | |
duke@435 | 2898 | IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); |
duke@435 | 2899 | |
duke@435 | 2900 | // First fast path: if (!TLS._interrupted) return false; |
duke@435 | 2901 | Node* false_bit = _gvn.transform( new (C, 1) IfFalseNode(iff_bit) ); |
duke@435 | 2902 | result_rgn->init_req(no_int_result_path, false_bit); |
duke@435 | 2903 | result_val->init_req(no_int_result_path, intcon(0)); |
duke@435 | 2904 | |
duke@435 | 2905 | // drop through to next case |
duke@435 | 2906 | set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_bit)) ); |
duke@435 | 2907 | |
duke@435 | 2908 | // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path. |
duke@435 | 2909 | Node* clr_arg = argument(1); |
duke@435 | 2910 | Node* cmp_arg = _gvn.transform( new (C, 3) CmpINode(clr_arg, intcon(0)) ); |
duke@435 | 2911 | Node* bol_arg = _gvn.transform( new (C, 2) BoolNode(cmp_arg, BoolTest::ne) ); |
duke@435 | 2912 | IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN); |
duke@435 | 2913 | |
duke@435 | 2914 | // Second fast path: ... else if (!clear_int) return true; |
duke@435 | 2915 | Node* false_arg = _gvn.transform( new (C, 1) IfFalseNode(iff_arg) ); |
duke@435 | 2916 | result_rgn->init_req(no_clear_result_path, false_arg); |
duke@435 | 2917 | result_val->init_req(no_clear_result_path, intcon(1)); |
duke@435 | 2918 | |
duke@435 | 2919 | // drop through to next case |
duke@435 | 2920 | set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_arg)) ); |
duke@435 | 2921 | |
duke@435 | 2922 | // (d) Otherwise, go to the slow path. |
duke@435 | 2923 | slow_region->add_req(control()); |
duke@435 | 2924 | set_control( _gvn.transform(slow_region) ); |
duke@435 | 2925 | |
duke@435 | 2926 | if (stopped()) { |
duke@435 | 2927 | // There is no slow path. |
duke@435 | 2928 | result_rgn->init_req(slow_result_path, top()); |
duke@435 | 2929 | result_val->init_req(slow_result_path, top()); |
duke@435 | 2930 | } else { |
duke@435 | 2931 | // non-virtual because it is a private non-static |
duke@435 | 2932 | CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted); |
duke@435 | 2933 | |
duke@435 | 2934 | Node* slow_val = set_results_for_java_call(slow_call); |
duke@435 | 2935 | // this->control() comes from set_results_for_java_call |
duke@435 | 2936 | |
duke@435 | 2937 | // If we know that the result of the slow call will be true, tell the optimizer! |
duke@435 | 2938 | if (known_current_thread) slow_val = intcon(1); |
duke@435 | 2939 | |
duke@435 | 2940 | Node* fast_io = slow_call->in(TypeFunc::I_O); |
duke@435 | 2941 | Node* fast_mem = slow_call->in(TypeFunc::Memory); |
duke@435 | 2942 | // These two phis are pre-filled with copies of of the fast IO and Memory |
duke@435 | 2943 | Node* io_phi = PhiNode::make(result_rgn, fast_io, Type::ABIO); |
duke@435 | 2944 | Node* mem_phi = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM); |
duke@435 | 2945 | |
duke@435 | 2946 | result_rgn->init_req(slow_result_path, control()); |
duke@435 | 2947 | io_phi ->init_req(slow_result_path, i_o()); |
duke@435 | 2948 | mem_phi ->init_req(slow_result_path, reset_memory()); |
duke@435 | 2949 | result_val->init_req(slow_result_path, slow_val); |
duke@435 | 2950 | |
duke@435 | 2951 | set_all_memory( _gvn.transform(mem_phi) ); |
duke@435 | 2952 | set_i_o( _gvn.transform(io_phi) ); |
duke@435 | 2953 | } |
duke@435 | 2954 | |
duke@435 | 2955 | push_result(result_rgn, result_val); |
duke@435 | 2956 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 2957 | |
duke@435 | 2958 | return true; |
duke@435 | 2959 | } |
duke@435 | 2960 | |
duke@435 | 2961 | //---------------------------load_mirror_from_klass---------------------------- |
duke@435 | 2962 | // Given a klass oop, load its java mirror (a java.lang.Class oop). |
duke@435 | 2963 | Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { |
stefank@3391 | 2964 | Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset())); |
duke@435 | 2965 | return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT); |
duke@435 | 2966 | } |
duke@435 | 2967 | |
duke@435 | 2968 | //-----------------------load_klass_from_mirror_common------------------------- |
duke@435 | 2969 | // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop. |
duke@435 | 2970 | // Test the klass oop for null (signifying a primitive Class like Integer.TYPE), |
duke@435 | 2971 | // and branch to the given path on the region. |
duke@435 | 2972 | // If never_see_null, take an uncommon trap on null, so we can optimistically |
duke@435 | 2973 | // compile for the non-null case. |
duke@435 | 2974 | // If the region is NULL, force never_see_null = true. |
duke@435 | 2975 | Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror, |
duke@435 | 2976 | bool never_see_null, |
duke@435 | 2977 | int nargs, |
duke@435 | 2978 | RegionNode* region, |
duke@435 | 2979 | int null_path, |
duke@435 | 2980 | int offset) { |
duke@435 | 2981 | if (region == NULL) never_see_null = true; |
duke@435 | 2982 | Node* p = basic_plus_adr(mirror, offset); |
duke@435 | 2983 | const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; |
kvn@599 | 2984 | Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) ); |
duke@435 | 2985 | _sp += nargs; // any deopt will start just before call to enclosing method |
duke@435 | 2986 | Node* null_ctl = top(); |
duke@435 | 2987 | kls = null_check_oop(kls, &null_ctl, never_see_null); |
duke@435 | 2988 | if (region != NULL) { |
duke@435 | 2989 | // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). |
duke@435 | 2990 | region->init_req(null_path, null_ctl); |
duke@435 | 2991 | } else { |
duke@435 | 2992 | assert(null_ctl == top(), "no loose ends"); |
duke@435 | 2993 | } |
duke@435 | 2994 | _sp -= nargs; |
duke@435 | 2995 | return kls; |
duke@435 | 2996 | } |
duke@435 | 2997 | |
duke@435 | 2998 | //--------------------(inline_native_Class_query helpers)--------------------- |
duke@435 | 2999 | // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER. |
duke@435 | 3000 | // Fall through if (mods & mask) == bits, take the guard otherwise. |
duke@435 | 3001 | Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) { |
duke@435 | 3002 | // Branch around if the given klass has the given modifier bit set. |
duke@435 | 3003 | // Like generate_guard, adds a new path onto the region. |
stefank@3391 | 3004 | Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); |
duke@435 | 3005 | Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT); |
duke@435 | 3006 | Node* mask = intcon(modifier_mask); |
duke@435 | 3007 | Node* bits = intcon(modifier_bits); |
duke@435 | 3008 | Node* mbit = _gvn.transform( new (C, 3) AndINode(mods, mask) ); |
duke@435 | 3009 | Node* cmp = _gvn.transform( new (C, 3) CmpINode(mbit, bits) ); |
duke@435 | 3010 | Node* bol = _gvn.transform( new (C, 2) BoolNode(cmp, BoolTest::ne) ); |
duke@435 | 3011 | return generate_fair_guard(bol, region); |
duke@435 | 3012 | } |
duke@435 | 3013 | Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) { |
duke@435 | 3014 | return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region); |
duke@435 | 3015 | } |
duke@435 | 3016 | |
duke@435 | 3017 | //-------------------------inline_native_Class_query------------------- |
duke@435 | 3018 | bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { |
duke@435 | 3019 | int nargs = 1+0; // just the Class mirror, in most cases |
duke@435 | 3020 | const Type* return_type = TypeInt::BOOL; |
duke@435 | 3021 | Node* prim_return_value = top(); // what happens if it's a primitive class? |
duke@435 | 3022 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 3023 | bool expect_prim = false; // most of these guys expect to work on refs |
duke@435 | 3024 | |
duke@435 | 3025 | enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; |
duke@435 | 3026 | |
duke@435 | 3027 | switch (id) { |
duke@435 | 3028 | case vmIntrinsics::_isInstance: |
duke@435 | 3029 | nargs = 1+1; // the Class mirror, plus the object getting queried about |
duke@435 | 3030 | // nothing is an instance of a primitive type |
duke@435 | 3031 | prim_return_value = intcon(0); |
duke@435 | 3032 | break; |
duke@435 | 3033 | case vmIntrinsics::_getModifiers: |
duke@435 | 3034 | prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); |
duke@435 | 3035 | assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line"); |
duke@435 | 3036 | return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin); |
duke@435 | 3037 | break; |
duke@435 | 3038 | case vmIntrinsics::_isInterface: |
duke@435 | 3039 | prim_return_value = intcon(0); |
duke@435 | 3040 | break; |
duke@435 | 3041 | case vmIntrinsics::_isArray: |
duke@435 | 3042 | prim_return_value = intcon(0); |
duke@435 | 3043 | expect_prim = true; // cf. ObjectStreamClass.getClassSignature |
duke@435 | 3044 | break; |
duke@435 | 3045 | case vmIntrinsics::_isPrimitive: |
duke@435 | 3046 | prim_return_value = intcon(1); |
duke@435 | 3047 | expect_prim = true; // obviously |
duke@435 | 3048 | break; |
duke@435 | 3049 | case vmIntrinsics::_getSuperclass: |
duke@435 | 3050 | prim_return_value = null(); |
duke@435 | 3051 | return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); |
duke@435 | 3052 | break; |
duke@435 | 3053 | case vmIntrinsics::_getComponentType: |
duke@435 | 3054 | prim_return_value = null(); |
duke@435 | 3055 | return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); |
duke@435 | 3056 | break; |
duke@435 | 3057 | case vmIntrinsics::_getClassAccessFlags: |
duke@435 | 3058 | prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); |
duke@435 | 3059 | return_type = TypeInt::INT; // not bool! 6297094 |
duke@435 | 3060 | break; |
duke@435 | 3061 | default: |
duke@435 | 3062 | ShouldNotReachHere(); |
duke@435 | 3063 | } |
duke@435 | 3064 | |
duke@435 | 3065 | Node* mirror = argument(0); |
duke@435 | 3066 | Node* obj = (nargs <= 1)? top(): argument(1); |
duke@435 | 3067 | |
duke@435 | 3068 | const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); |
duke@435 | 3069 | if (mirror_con == NULL) return false; // cannot happen? |
duke@435 | 3070 | |
duke@435 | 3071 | #ifndef PRODUCT |
duke@435 | 3072 | if (PrintIntrinsics || PrintInlining || PrintOptoInlining) { |
duke@435 | 3073 | ciType* k = mirror_con->java_mirror_type(); |
duke@435 | 3074 | if (k) { |
duke@435 | 3075 | tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id())); |
duke@435 | 3076 | k->print_name(); |
duke@435 | 3077 | tty->cr(); |
duke@435 | 3078 | } |
duke@435 | 3079 | } |
duke@435 | 3080 | #endif |
duke@435 | 3081 | |
duke@435 | 3082 | // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive). |
duke@435 | 3083 | RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3084 | record_for_igvn(region); |
duke@435 | 3085 | PhiNode* phi = new (C, PATH_LIMIT) PhiNode(region, return_type); |
duke@435 | 3086 | |
duke@435 | 3087 | // The mirror will never be null of Reflection.getClassAccessFlags, however |
duke@435 | 3088 | // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE |
duke@435 | 3089 | // if it is. See bug 4774291. |
duke@435 | 3090 | |
duke@435 | 3091 | // For Reflection.getClassAccessFlags(), the null check occurs in |
duke@435 | 3092 | // the wrong place; see inline_unsafe_access(), above, for a similar |
duke@435 | 3093 | // situation. |
duke@435 | 3094 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3095 | mirror = do_null_check(mirror, T_OBJECT); |
duke@435 | 3096 | _sp -= nargs; |
duke@435 | 3097 | // If mirror or obj is dead, only null-path is taken. |
duke@435 | 3098 | if (stopped()) return true; |
duke@435 | 3099 | |
duke@435 | 3100 | if (expect_prim) never_see_null = false; // expect nulls (meaning prims) |
duke@435 | 3101 | |
duke@435 | 3102 | // Now load the mirror's klass metaobject, and null-check it. |
duke@435 | 3103 | // Side-effects region with the control path if the klass is null. |
duke@435 | 3104 | Node* kls = load_klass_from_mirror(mirror, never_see_null, nargs, |
duke@435 | 3105 | region, _prim_path); |
duke@435 | 3106 | // If kls is null, we have a primitive mirror. |
duke@435 | 3107 | phi->init_req(_prim_path, prim_return_value); |
duke@435 | 3108 | if (stopped()) { push_result(region, phi); return true; } |
duke@435 | 3109 | |
duke@435 | 3110 | Node* p; // handy temp |
duke@435 | 3111 | Node* null_ctl; |
duke@435 | 3112 | |
duke@435 | 3113 | // Now that we have the non-null klass, we can perform the real query. |
duke@435 | 3114 | // For constant classes, the query will constant-fold in LoadNode::Value. |
duke@435 | 3115 | Node* query_value = top(); |
duke@435 | 3116 | switch (id) { |
duke@435 | 3117 | case vmIntrinsics::_isInstance: |
duke@435 | 3118 | // nothing is an instance of a primitive type |
jrose@2101 | 3119 | _sp += nargs; // gen_instanceof might do an uncommon trap |
duke@435 | 3120 | query_value = gen_instanceof(obj, kls); |
jrose@2101 | 3121 | _sp -= nargs; |
duke@435 | 3122 | break; |
duke@435 | 3123 | |
duke@435 | 3124 | case vmIntrinsics::_getModifiers: |
stefank@3391 | 3125 | p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset())); |
duke@435 | 3126 | query_value = make_load(NULL, p, TypeInt::INT, T_INT); |
duke@435 | 3127 | break; |
duke@435 | 3128 | |
duke@435 | 3129 | case vmIntrinsics::_isInterface: |
duke@435 | 3130 | // (To verify this code sequence, check the asserts in JVM_IsInterface.) |
duke@435 | 3131 | if (generate_interface_guard(kls, region) != NULL) |
duke@435 | 3132 | // A guard was added. If the guard is taken, it was an interface. |
duke@435 | 3133 | phi->add_req(intcon(1)); |
duke@435 | 3134 | // If we fall through, it's a plain class. |
duke@435 | 3135 | query_value = intcon(0); |
duke@435 | 3136 | break; |
duke@435 | 3137 | |
duke@435 | 3138 | case vmIntrinsics::_isArray: |
duke@435 | 3139 | // (To verify this code sequence, check the asserts in JVM_IsArrayClass.) |
duke@435 | 3140 | if (generate_array_guard(kls, region) != NULL) |
duke@435 | 3141 | // A guard was added. If the guard is taken, it was an array. |
duke@435 | 3142 | phi->add_req(intcon(1)); |
duke@435 | 3143 | // If we fall through, it's a plain class. |
duke@435 | 3144 | query_value = intcon(0); |
duke@435 | 3145 | break; |
duke@435 | 3146 | |
duke@435 | 3147 | case vmIntrinsics::_isPrimitive: |
duke@435 | 3148 | query_value = intcon(0); // "normal" path produces false |
duke@435 | 3149 | break; |
duke@435 | 3150 | |
duke@435 | 3151 | case vmIntrinsics::_getSuperclass: |
duke@435 | 3152 | // The rules here are somewhat unfortunate, but we can still do better |
duke@435 | 3153 | // with random logic than with a JNI call. |
duke@435 | 3154 | // Interfaces store null or Object as _super, but must report null. |
duke@435 | 3155 | // Arrays store an intermediate super as _super, but must report Object. |
duke@435 | 3156 | // Other types can report the actual _super. |
duke@435 | 3157 | // (To verify this code sequence, check the asserts in JVM_IsInterface.) |
duke@435 | 3158 | if (generate_interface_guard(kls, region) != NULL) |
duke@435 | 3159 | // A guard was added. If the guard is taken, it was an interface. |
duke@435 | 3160 | phi->add_req(null()); |
duke@435 | 3161 | if (generate_array_guard(kls, region) != NULL) |
duke@435 | 3162 | // A guard was added. If the guard is taken, it was an array. |
duke@435 | 3163 | phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); |
duke@435 | 3164 | // If we fall through, it's a plain class. Get its _super. |
stefank@3391 | 3165 | p = basic_plus_adr(kls, in_bytes(Klass::super_offset())); |
kvn@599 | 3166 | kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) ); |
duke@435 | 3167 | null_ctl = top(); |
duke@435 | 3168 | kls = null_check_oop(kls, &null_ctl); |
duke@435 | 3169 | if (null_ctl != top()) { |
duke@435 | 3170 | // If the guard is taken, Object.superClass is null (both klass and mirror). |
duke@435 | 3171 | region->add_req(null_ctl); |
duke@435 | 3172 | phi ->add_req(null()); |
duke@435 | 3173 | } |
duke@435 | 3174 | if (!stopped()) { |
duke@435 | 3175 | query_value = load_mirror_from_klass(kls); |
duke@435 | 3176 | } |
duke@435 | 3177 | break; |
duke@435 | 3178 | |
duke@435 | 3179 | case vmIntrinsics::_getComponentType: |
duke@435 | 3180 | if (generate_array_guard(kls, region) != NULL) { |
duke@435 | 3181 | // Be sure to pin the oop load to the guard edge just created: |
duke@435 | 3182 | Node* is_array_ctrl = region->in(region->req()-1); |
stefank@3391 | 3183 | Node* cma = basic_plus_adr(kls, in_bytes(arrayKlass::component_mirror_offset())); |
duke@435 | 3184 | Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT); |
duke@435 | 3185 | phi->add_req(cmo); |
duke@435 | 3186 | } |
duke@435 | 3187 | query_value = null(); // non-array case is null |
duke@435 | 3188 | break; |
duke@435 | 3189 | |
duke@435 | 3190 | case vmIntrinsics::_getClassAccessFlags: |
stefank@3391 | 3191 | p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); |
duke@435 | 3192 | query_value = make_load(NULL, p, TypeInt::INT, T_INT); |
duke@435 | 3193 | break; |
duke@435 | 3194 | |
duke@435 | 3195 | default: |
duke@435 | 3196 | ShouldNotReachHere(); |
duke@435 | 3197 | } |
duke@435 | 3198 | |
duke@435 | 3199 | // Fall-through is the normal case of a query to a real class. |
duke@435 | 3200 | phi->init_req(1, query_value); |
duke@435 | 3201 | region->init_req(1, control()); |
duke@435 | 3202 | |
duke@435 | 3203 | push_result(region, phi); |
duke@435 | 3204 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3205 | |
duke@435 | 3206 | return true; |
duke@435 | 3207 | } |
duke@435 | 3208 | |
duke@435 | 3209 | //--------------------------inline_native_subtype_check------------------------ |
duke@435 | 3210 | // This intrinsic takes the JNI calls out of the heart of |
duke@435 | 3211 | // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc. |
duke@435 | 3212 | bool LibraryCallKit::inline_native_subtype_check() { |
duke@435 | 3213 | int nargs = 1+1; // the Class mirror, plus the other class getting examined |
duke@435 | 3214 | |
duke@435 | 3215 | // Pull both arguments off the stack. |
duke@435 | 3216 | Node* args[2]; // two java.lang.Class mirrors: superc, subc |
duke@435 | 3217 | args[0] = argument(0); |
duke@435 | 3218 | args[1] = argument(1); |
duke@435 | 3219 | Node* klasses[2]; // corresponding Klasses: superk, subk |
duke@435 | 3220 | klasses[0] = klasses[1] = top(); |
duke@435 | 3221 | |
duke@435 | 3222 | enum { |
duke@435 | 3223 | // A full decision tree on {superc is prim, subc is prim}: |
duke@435 | 3224 | _prim_0_path = 1, // {P,N} => false |
duke@435 | 3225 | // {P,P} & superc!=subc => false |
duke@435 | 3226 | _prim_same_path, // {P,P} & superc==subc => true |
duke@435 | 3227 | _prim_1_path, // {N,P} => false |
duke@435 | 3228 | _ref_subtype_path, // {N,N} & subtype check wins => true |
duke@435 | 3229 | _both_ref_path, // {N,N} & subtype check loses => false |
duke@435 | 3230 | PATH_LIMIT |
duke@435 | 3231 | }; |
duke@435 | 3232 | |
duke@435 | 3233 | RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3234 | Node* phi = new (C, PATH_LIMIT) PhiNode(region, TypeInt::BOOL); |
duke@435 | 3235 | record_for_igvn(region); |
duke@435 | 3236 | |
duke@435 | 3237 | const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads |
duke@435 | 3238 | const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; |
duke@435 | 3239 | int class_klass_offset = java_lang_Class::klass_offset_in_bytes(); |
duke@435 | 3240 | |
duke@435 | 3241 | // First null-check both mirrors and load each mirror's klass metaobject. |
duke@435 | 3242 | int which_arg; |
duke@435 | 3243 | for (which_arg = 0; which_arg <= 1; which_arg++) { |
duke@435 | 3244 | Node* arg = args[which_arg]; |
duke@435 | 3245 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3246 | arg = do_null_check(arg, T_OBJECT); |
duke@435 | 3247 | _sp -= nargs; |
duke@435 | 3248 | if (stopped()) break; |
duke@435 | 3249 | args[which_arg] = _gvn.transform(arg); |
duke@435 | 3250 | |
duke@435 | 3251 | Node* p = basic_plus_adr(arg, class_klass_offset); |
kvn@599 | 3252 | Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type); |
duke@435 | 3253 | klasses[which_arg] = _gvn.transform(kls); |
duke@435 | 3254 | } |
duke@435 | 3255 | |
duke@435 | 3256 | // Having loaded both klasses, test each for null. |
duke@435 | 3257 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 3258 | for (which_arg = 0; which_arg <= 1; which_arg++) { |
duke@435 | 3259 | Node* kls = klasses[which_arg]; |
duke@435 | 3260 | Node* null_ctl = top(); |
duke@435 | 3261 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3262 | kls = null_check_oop(kls, &null_ctl, never_see_null); |
duke@435 | 3263 | _sp -= nargs; |
duke@435 | 3264 | int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path); |
duke@435 | 3265 | region->init_req(prim_path, null_ctl); |
duke@435 | 3266 | if (stopped()) break; |
duke@435 | 3267 | klasses[which_arg] = kls; |
duke@435 | 3268 | } |
duke@435 | 3269 | |
duke@435 | 3270 | if (!stopped()) { |
duke@435 | 3271 | // now we have two reference types, in klasses[0..1] |
duke@435 | 3272 | Node* subk = klasses[1]; // the argument to isAssignableFrom |
duke@435 | 3273 | Node* superk = klasses[0]; // the receiver |
duke@435 | 3274 | region->set_req(_both_ref_path, gen_subtype_check(subk, superk)); |
duke@435 | 3275 | // now we have a successful reference subtype check |
duke@435 | 3276 | region->set_req(_ref_subtype_path, control()); |
duke@435 | 3277 | } |
duke@435 | 3278 | |
duke@435 | 3279 | // If both operands are primitive (both klasses null), then |
duke@435 | 3280 | // we must return true when they are identical primitives. |
duke@435 | 3281 | // It is convenient to test this after the first null klass check. |
duke@435 | 3282 | set_control(region->in(_prim_0_path)); // go back to first null check |
duke@435 | 3283 | if (!stopped()) { |
duke@435 | 3284 | // Since superc is primitive, make a guard for the superc==subc case. |
duke@435 | 3285 | Node* cmp_eq = _gvn.transform( new (C, 3) CmpPNode(args[0], args[1]) ); |
duke@435 | 3286 | Node* bol_eq = _gvn.transform( new (C, 2) BoolNode(cmp_eq, BoolTest::eq) ); |
duke@435 | 3287 | generate_guard(bol_eq, region, PROB_FAIR); |
duke@435 | 3288 | if (region->req() == PATH_LIMIT+1) { |
duke@435 | 3289 | // A guard was added. If the added guard is taken, superc==subc. |
duke@435 | 3290 | region->swap_edges(PATH_LIMIT, _prim_same_path); |
duke@435 | 3291 | region->del_req(PATH_LIMIT); |
duke@435 | 3292 | } |
duke@435 | 3293 | region->set_req(_prim_0_path, control()); // Not equal after all. |
duke@435 | 3294 | } |
duke@435 | 3295 | |
duke@435 | 3296 | // these are the only paths that produce 'true': |
duke@435 | 3297 | phi->set_req(_prim_same_path, intcon(1)); |
duke@435 | 3298 | phi->set_req(_ref_subtype_path, intcon(1)); |
duke@435 | 3299 | |
duke@435 | 3300 | // pull together the cases: |
duke@435 | 3301 | assert(region->req() == PATH_LIMIT, "sane region"); |
duke@435 | 3302 | for (uint i = 1; i < region->req(); i++) { |
duke@435 | 3303 | Node* ctl = region->in(i); |
duke@435 | 3304 | if (ctl == NULL || ctl == top()) { |
duke@435 | 3305 | region->set_req(i, top()); |
duke@435 | 3306 | phi ->set_req(i, top()); |
duke@435 | 3307 | } else if (phi->in(i) == NULL) { |
duke@435 | 3308 | phi->set_req(i, intcon(0)); // all other paths produce 'false' |
duke@435 | 3309 | } |
duke@435 | 3310 | } |
duke@435 | 3311 | |
duke@435 | 3312 | set_control(_gvn.transform(region)); |
duke@435 | 3313 | push(_gvn.transform(phi)); |
duke@435 | 3314 | |
duke@435 | 3315 | return true; |
duke@435 | 3316 | } |
duke@435 | 3317 | |
duke@435 | 3318 | //---------------------generate_array_guard_common------------------------ |
duke@435 | 3319 | Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, |
duke@435 | 3320 | bool obj_array, bool not_array) { |
duke@435 | 3321 | // If obj_array/non_array==false/false: |
duke@435 | 3322 | // Branch around if the given klass is in fact an array (either obj or prim). |
duke@435 | 3323 | // If obj_array/non_array==false/true: |
duke@435 | 3324 | // Branch around if the given klass is not an array klass of any kind. |
duke@435 | 3325 | // If obj_array/non_array==true/true: |
duke@435 | 3326 | // Branch around if the kls is not an oop array (kls is int[], String, etc.) |
duke@435 | 3327 | // If obj_array/non_array==true/false: |
duke@435 | 3328 | // Branch around if the kls is an oop array (Object[] or subtype) |
duke@435 | 3329 | // |
duke@435 | 3330 | // Like generate_guard, adds a new path onto the region. |
duke@435 | 3331 | jint layout_con = 0; |
duke@435 | 3332 | Node* layout_val = get_layout_helper(kls, layout_con); |
duke@435 | 3333 | if (layout_val == NULL) { |
duke@435 | 3334 | bool query = (obj_array |
duke@435 | 3335 | ? Klass::layout_helper_is_objArray(layout_con) |
duke@435 | 3336 | : Klass::layout_helper_is_javaArray(layout_con)); |
duke@435 | 3337 | if (query == not_array) { |
duke@435 | 3338 | return NULL; // never a branch |
duke@435 | 3339 | } else { // always a branch |
duke@435 | 3340 | Node* always_branch = control(); |
duke@435 | 3341 | if (region != NULL) |
duke@435 | 3342 | region->add_req(always_branch); |
duke@435 | 3343 | set_control(top()); |
duke@435 | 3344 | return always_branch; |
duke@435 | 3345 | } |
duke@435 | 3346 | } |
duke@435 | 3347 | // Now test the correct condition. |
duke@435 | 3348 | jint nval = (obj_array |
duke@435 | 3349 | ? ((jint)Klass::_lh_array_tag_type_value |
duke@435 | 3350 | << Klass::_lh_array_tag_shift) |
duke@435 | 3351 | : Klass::_lh_neutral_value); |
duke@435 | 3352 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(nval)) ); |
duke@435 | 3353 | BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array |
duke@435 | 3354 | // invert the test if we are looking for a non-array |
duke@435 | 3355 | if (not_array) btest = BoolTest(btest).negate(); |
duke@435 | 3356 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, btest) ); |
duke@435 | 3357 | return generate_fair_guard(bol, region); |
duke@435 | 3358 | } |
duke@435 | 3359 | |
duke@435 | 3360 | |
duke@435 | 3361 | //-----------------------inline_native_newArray-------------------------- |
duke@435 | 3362 | bool LibraryCallKit::inline_native_newArray() { |
duke@435 | 3363 | int nargs = 2; |
duke@435 | 3364 | Node* mirror = argument(0); |
duke@435 | 3365 | Node* count_val = argument(1); |
duke@435 | 3366 | |
duke@435 | 3367 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3368 | mirror = do_null_check(mirror, T_OBJECT); |
duke@435 | 3369 | _sp -= nargs; |
kvn@598 | 3370 | // If mirror or obj is dead, only null-path is taken. |
kvn@598 | 3371 | if (stopped()) return true; |
duke@435 | 3372 | |
duke@435 | 3373 | enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT }; |
duke@435 | 3374 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3375 | PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
duke@435 | 3376 | TypeInstPtr::NOTNULL); |
duke@435 | 3377 | PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
duke@435 | 3378 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
duke@435 | 3379 | TypePtr::BOTTOM); |
duke@435 | 3380 | |
duke@435 | 3381 | bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); |
duke@435 | 3382 | Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null, |
duke@435 | 3383 | nargs, |
duke@435 | 3384 | result_reg, _slow_path); |
duke@435 | 3385 | Node* normal_ctl = control(); |
duke@435 | 3386 | Node* no_array_ctl = result_reg->in(_slow_path); |
duke@435 | 3387 | |
duke@435 | 3388 | // Generate code for the slow case. We make a call to newArray(). |
duke@435 | 3389 | set_control(no_array_ctl); |
duke@435 | 3390 | if (!stopped()) { |
duke@435 | 3391 | // Either the input type is void.class, or else the |
duke@435 | 3392 | // array klass has not yet been cached. Either the |
duke@435 | 3393 | // ensuing call will throw an exception, or else it |
duke@435 | 3394 | // will cache the array klass for next time. |
duke@435 | 3395 | PreserveJVMState pjvms(this); |
duke@435 | 3396 | CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray); |
duke@435 | 3397 | Node* slow_result = set_results_for_java_call(slow_call); |
duke@435 | 3398 | // this->control() comes from set_results_for_java_call |
duke@435 | 3399 | result_reg->set_req(_slow_path, control()); |
duke@435 | 3400 | result_val->set_req(_slow_path, slow_result); |
duke@435 | 3401 | result_io ->set_req(_slow_path, i_o()); |
duke@435 | 3402 | result_mem->set_req(_slow_path, reset_memory()); |
duke@435 | 3403 | } |
duke@435 | 3404 | |
duke@435 | 3405 | set_control(normal_ctl); |
duke@435 | 3406 | if (!stopped()) { |
duke@435 | 3407 | // Normal case: The array type has been cached in the java.lang.Class. |
duke@435 | 3408 | // The following call works fine even if the array type is polymorphic. |
duke@435 | 3409 | // It could be a dynamic mix of int[], boolean[], Object[], etc. |
cfang@1165 | 3410 | Node* obj = new_array(klass_node, count_val, nargs); |
duke@435 | 3411 | result_reg->init_req(_normal_path, control()); |
duke@435 | 3412 | result_val->init_req(_normal_path, obj); |
duke@435 | 3413 | result_io ->init_req(_normal_path, i_o()); |
duke@435 | 3414 | result_mem->init_req(_normal_path, reset_memory()); |
duke@435 | 3415 | } |
duke@435 | 3416 | |
duke@435 | 3417 | // Return the combined state. |
duke@435 | 3418 | set_i_o( _gvn.transform(result_io) ); |
duke@435 | 3419 | set_all_memory( _gvn.transform(result_mem) ); |
duke@435 | 3420 | push_result(result_reg, result_val); |
duke@435 | 3421 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3422 | |
duke@435 | 3423 | return true; |
duke@435 | 3424 | } |
duke@435 | 3425 | |
duke@435 | 3426 | //----------------------inline_native_getLength-------------------------- |
duke@435 | 3427 | bool LibraryCallKit::inline_native_getLength() { |
duke@435 | 3428 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 3429 | |
duke@435 | 3430 | int nargs = 1; |
duke@435 | 3431 | Node* array = argument(0); |
duke@435 | 3432 | |
duke@435 | 3433 | _sp += nargs; // set original stack for use by uncommon_trap |
duke@435 | 3434 | array = do_null_check(array, T_OBJECT); |
duke@435 | 3435 | _sp -= nargs; |
duke@435 | 3436 | |
duke@435 | 3437 | // If array is dead, only null-path is taken. |
duke@435 | 3438 | if (stopped()) return true; |
duke@435 | 3439 | |
duke@435 | 3440 | // Deoptimize if it is a non-array. |
duke@435 | 3441 | Node* non_array = generate_non_array_guard(load_object_klass(array), NULL); |
duke@435 | 3442 | |
duke@435 | 3443 | if (non_array != NULL) { |
duke@435 | 3444 | PreserveJVMState pjvms(this); |
duke@435 | 3445 | set_control(non_array); |
duke@435 | 3446 | _sp += nargs; // push the arguments back on the stack |
duke@435 | 3447 | uncommon_trap(Deoptimization::Reason_intrinsic, |
duke@435 | 3448 | Deoptimization::Action_maybe_recompile); |
duke@435 | 3449 | } |
duke@435 | 3450 | |
duke@435 | 3451 | // If control is dead, only non-array-path is taken. |
duke@435 | 3452 | if (stopped()) return true; |
duke@435 | 3453 | |
duke@435 | 3454 | // The works fine even if the array type is polymorphic. |
duke@435 | 3455 | // It could be a dynamic mix of int[], boolean[], Object[], etc. |
duke@435 | 3456 | push( load_array_length(array) ); |
duke@435 | 3457 | |
duke@435 | 3458 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3459 | |
duke@435 | 3460 | return true; |
duke@435 | 3461 | } |
duke@435 | 3462 | |
duke@435 | 3463 | //------------------------inline_array_copyOf---------------------------- |
duke@435 | 3464 | bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { |
duke@435 | 3465 | if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; |
duke@435 | 3466 | |
duke@435 | 3467 | // Restore the stack and pop off the arguments. |
duke@435 | 3468 | int nargs = 3 + (is_copyOfRange? 1: 0); |
duke@435 | 3469 | Node* original = argument(0); |
duke@435 | 3470 | Node* start = is_copyOfRange? argument(1): intcon(0); |
duke@435 | 3471 | Node* end = is_copyOfRange? argument(2): argument(1); |
duke@435 | 3472 | Node* array_type_mirror = is_copyOfRange? argument(3): argument(2); |
duke@435 | 3473 | |
cfang@1337 | 3474 | Node* newcopy; |
cfang@1337 | 3475 | |
cfang@1337 | 3476 | //set the original stack and the reexecute bit for the interpreter to reexecute |
cfang@1337 | 3477 | //the bytecode that invokes Arrays.copyOf if deoptimization happens |
cfang@1337 | 3478 | { PreserveReexecuteState preexecs(this); |
cfang@1337 | 3479 | _sp += nargs; |
cfang@1337 | 3480 | jvms()->set_should_reexecute(true); |
cfang@1337 | 3481 | |
cfang@1337 | 3482 | array_type_mirror = do_null_check(array_type_mirror, T_OBJECT); |
cfang@1337 | 3483 | original = do_null_check(original, T_OBJECT); |
cfang@1337 | 3484 | |
cfang@1337 | 3485 | // Check if a null path was taken unconditionally. |
cfang@1337 | 3486 | if (stopped()) return true; |
cfang@1337 | 3487 | |
cfang@1337 | 3488 | Node* orig_length = load_array_length(original); |
cfang@1337 | 3489 | |
cfang@1337 | 3490 | Node* klass_node = load_klass_from_mirror(array_type_mirror, false, 0, |
cfang@1337 | 3491 | NULL, 0); |
cfang@1337 | 3492 | klass_node = do_null_check(klass_node, T_OBJECT); |
cfang@1337 | 3493 | |
cfang@1337 | 3494 | RegionNode* bailout = new (C, 1) RegionNode(1); |
cfang@1337 | 3495 | record_for_igvn(bailout); |
cfang@1337 | 3496 | |
cfang@1337 | 3497 | // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc. |
cfang@1337 | 3498 | // Bail out if that is so. |
cfang@1337 | 3499 | Node* not_objArray = generate_non_objArray_guard(klass_node, bailout); |
cfang@1337 | 3500 | if (not_objArray != NULL) { |
cfang@1337 | 3501 | // Improve the klass node's type from the new optimistic assumption: |
cfang@1337 | 3502 | ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); |
cfang@1337 | 3503 | const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/); |
cfang@1337 | 3504 | Node* cast = new (C, 2) CastPPNode(klass_node, akls); |
cfang@1337 | 3505 | cast->init_req(0, control()); |
cfang@1337 | 3506 | klass_node = _gvn.transform(cast); |
cfang@1337 | 3507 | } |
cfang@1337 | 3508 | |
cfang@1337 | 3509 | // Bail out if either start or end is negative. |
cfang@1337 | 3510 | generate_negative_guard(start, bailout, &start); |
cfang@1337 | 3511 | generate_negative_guard(end, bailout, &end); |
cfang@1337 | 3512 | |
cfang@1337 | 3513 | Node* length = end; |
cfang@1337 | 3514 | if (_gvn.type(start) != TypeInt::ZERO) { |
cfang@1337 | 3515 | length = _gvn.transform( new (C, 3) SubINode(end, start) ); |
cfang@1337 | 3516 | } |
cfang@1337 | 3517 | |
cfang@1337 | 3518 | // Bail out if length is negative. |
cfang@1337 | 3519 | // ...Not needed, since the new_array will throw the right exception. |
cfang@1337 | 3520 | //generate_negative_guard(length, bailout, &length); |
cfang@1337 | 3521 | |
cfang@1337 | 3522 | if (bailout->req() > 1) { |
cfang@1337 | 3523 | PreserveJVMState pjvms(this); |
cfang@1337 | 3524 | set_control( _gvn.transform(bailout) ); |
cfang@1337 | 3525 | uncommon_trap(Deoptimization::Reason_intrinsic, |
cfang@1337 | 3526 | Deoptimization::Action_maybe_recompile); |
cfang@1337 | 3527 | } |
cfang@1337 | 3528 | |
cfang@1337 | 3529 | if (!stopped()) { |
cfang@1335 | 3530 | |
cfang@1335 | 3531 | // How many elements will we copy from the original? |
cfang@1335 | 3532 | // The answer is MinI(orig_length - start, length). |
cfang@1335 | 3533 | Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); |
cfang@1335 | 3534 | Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); |
cfang@1335 | 3535 | |
kvn@2810 | 3536 | newcopy = new_array(klass_node, length, 0); |
cfang@1335 | 3537 | |
cfang@1335 | 3538 | // Generate a direct call to the right arraycopy function(s). |
cfang@1335 | 3539 | // We know the copy is disjoint but we might not know if the |
cfang@1335 | 3540 | // oop stores need checking. |
cfang@1335 | 3541 | // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). |
cfang@1335 | 3542 | // This will fail a store-check if x contains any non-nulls. |
cfang@1335 | 3543 | bool disjoint_bases = true; |
cfang@1335 | 3544 | bool length_never_negative = true; |
cfang@1335 | 3545 | generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
cfang@1335 | 3546 | original, start, newcopy, intcon(0), moved, |
cfang@1335 | 3547 | disjoint_bases, length_never_negative); |
cfang@1337 | 3548 | } |
cfang@1337 | 3549 | } //original reexecute and sp are set back here |
cfang@1337 | 3550 | |
cfang@1337 | 3551 | if(!stopped()) { |
duke@435 | 3552 | push(newcopy); |
duke@435 | 3553 | } |
duke@435 | 3554 | |
duke@435 | 3555 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 3556 | |
duke@435 | 3557 | return true; |
duke@435 | 3558 | } |
duke@435 | 3559 | |
duke@435 | 3560 | |
duke@435 | 3561 | //----------------------generate_virtual_guard--------------------------- |
duke@435 | 3562 | // Helper for hashCode and clone. Peeks inside the vtable to avoid a call. |
duke@435 | 3563 | Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, |
duke@435 | 3564 | RegionNode* slow_region) { |
duke@435 | 3565 | ciMethod* method = callee(); |
duke@435 | 3566 | int vtable_index = method->vtable_index(); |
duke@435 | 3567 | // Get the methodOop out of the appropriate vtable entry. |
duke@435 | 3568 | int entry_offset = (instanceKlass::vtable_start_offset() + |
duke@435 | 3569 | vtable_index*vtableEntry::size()) * wordSize + |
duke@435 | 3570 | vtableEntry::method_offset_in_bytes(); |
duke@435 | 3571 | Node* entry_addr = basic_plus_adr(obj_klass, entry_offset); |
duke@435 | 3572 | Node* target_call = make_load(NULL, entry_addr, TypeInstPtr::NOTNULL, T_OBJECT); |
duke@435 | 3573 | |
duke@435 | 3574 | // Compare the target method with the expected method (e.g., Object.hashCode). |
duke@435 | 3575 | const TypeInstPtr* native_call_addr = TypeInstPtr::make(method); |
duke@435 | 3576 | |
duke@435 | 3577 | Node* native_call = makecon(native_call_addr); |
duke@435 | 3578 | Node* chk_native = _gvn.transform( new(C, 3) CmpPNode(target_call, native_call) ); |
duke@435 | 3579 | Node* test_native = _gvn.transform( new(C, 2) BoolNode(chk_native, BoolTest::ne) ); |
duke@435 | 3580 | |
duke@435 | 3581 | return generate_slow_guard(test_native, slow_region); |
duke@435 | 3582 | } |
duke@435 | 3583 | |
duke@435 | 3584 | //-----------------------generate_method_call---------------------------- |
duke@435 | 3585 | // Use generate_method_call to make a slow-call to the real |
duke@435 | 3586 | // method if the fast path fails. An alternative would be to |
duke@435 | 3587 | // use a stub like OptoRuntime::slow_arraycopy_Java. |
duke@435 | 3588 | // This only works for expanding the current library call, |
duke@435 | 3589 | // not another intrinsic. (E.g., don't use this for making an |
duke@435 | 3590 | // arraycopy call inside of the copyOf intrinsic.) |
duke@435 | 3591 | CallJavaNode* |
duke@435 | 3592 | LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) { |
duke@435 | 3593 | // When compiling the intrinsic method itself, do not use this technique. |
duke@435 | 3594 | guarantee(callee() != C->method(), "cannot make slow-call to self"); |
duke@435 | 3595 | |
duke@435 | 3596 | ciMethod* method = callee(); |
duke@435 | 3597 | // ensure the JVMS we have will be correct for this call |
duke@435 | 3598 | guarantee(method_id == method->intrinsic_id(), "must match"); |
duke@435 | 3599 | |
duke@435 | 3600 | const TypeFunc* tf = TypeFunc::make(method); |
duke@435 | 3601 | int tfdc = tf->domain()->cnt(); |
duke@435 | 3602 | CallJavaNode* slow_call; |
duke@435 | 3603 | if (is_static) { |
duke@435 | 3604 | assert(!is_virtual, ""); |
duke@435 | 3605 | slow_call = new(C, tfdc) CallStaticJavaNode(tf, |
duke@435 | 3606 | SharedRuntime::get_resolve_static_call_stub(), |
duke@435 | 3607 | method, bci()); |
duke@435 | 3608 | } else if (is_virtual) { |
duke@435 | 3609 | null_check_receiver(method); |
duke@435 | 3610 | int vtable_index = methodOopDesc::invalid_vtable_index; |
duke@435 | 3611 | if (UseInlineCaches) { |
duke@435 | 3612 | // Suppress the vtable call |
duke@435 | 3613 | } else { |
duke@435 | 3614 | // hashCode and clone are not a miranda methods, |
duke@435 | 3615 | // so the vtable index is fixed. |
duke@435 | 3616 | // No need to use the linkResolver to get it. |
duke@435 | 3617 | vtable_index = method->vtable_index(); |
duke@435 | 3618 | } |
duke@435 | 3619 | slow_call = new(C, tfdc) CallDynamicJavaNode(tf, |
duke@435 | 3620 | SharedRuntime::get_resolve_virtual_call_stub(), |
duke@435 | 3621 | method, vtable_index, bci()); |
duke@435 | 3622 | } else { // neither virtual nor static: opt_virtual |
duke@435 | 3623 | null_check_receiver(method); |
duke@435 | 3624 | slow_call = new(C, tfdc) CallStaticJavaNode(tf, |
duke@435 | 3625 | SharedRuntime::get_resolve_opt_virtual_call_stub(), |
duke@435 | 3626 | method, bci()); |
duke@435 | 3627 | slow_call->set_optimized_virtual(true); |
duke@435 | 3628 | } |
duke@435 | 3629 | set_arguments_for_java_call(slow_call); |
duke@435 | 3630 | set_edges_for_java_call(slow_call); |
duke@435 | 3631 | return slow_call; |
duke@435 | 3632 | } |
duke@435 | 3633 | |
duke@435 | 3634 | |
duke@435 | 3635 | //------------------------------inline_native_hashcode-------------------- |
duke@435 | 3636 | // Build special case code for calls to hashCode on an object. |
duke@435 | 3637 | bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { |
duke@435 | 3638 | assert(is_static == callee()->is_static(), "correct intrinsic selection"); |
duke@435 | 3639 | assert(!(is_virtual && is_static), "either virtual, special, or static"); |
duke@435 | 3640 | |
duke@435 | 3641 | enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT }; |
duke@435 | 3642 | |
duke@435 | 3643 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 3644 | PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
duke@435 | 3645 | TypeInt::INT); |
duke@435 | 3646 | PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
duke@435 | 3647 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
duke@435 | 3648 | TypePtr::BOTTOM); |
duke@435 | 3649 | Node* obj = NULL; |
duke@435 | 3650 | if (!is_static) { |
duke@435 | 3651 | // Check for hashing null object |
duke@435 | 3652 | obj = null_check_receiver(callee()); |
duke@435 | 3653 | if (stopped()) return true; // unconditionally null |
duke@435 | 3654 | result_reg->init_req(_null_path, top()); |
duke@435 | 3655 | result_val->init_req(_null_path, top()); |
duke@435 | 3656 | } else { |
duke@435 | 3657 | // Do a null check, and return zero if null. |
duke@435 | 3658 | // System.identityHashCode(null) == 0 |
duke@435 | 3659 | obj = argument(0); |
duke@435 | 3660 | Node* null_ctl = top(); |
duke@435 | 3661 | obj = null_check_oop(obj, &null_ctl); |
duke@435 | 3662 | result_reg->init_req(_null_path, null_ctl); |
duke@435 | 3663 | result_val->init_req(_null_path, _gvn.intcon(0)); |
duke@435 | 3664 | } |
duke@435 | 3665 | |
duke@435 | 3666 | // Unconditionally null? Then return right away. |
duke@435 | 3667 | if (stopped()) { |
duke@435 | 3668 | set_control( result_reg->in(_null_path) ); |
duke@435 | 3669 | if (!stopped()) |
duke@435 | 3670 | push( result_val ->in(_null_path) ); |
duke@435 | 3671 | return true; |
duke@435 | 3672 | } |
duke@435 | 3673 | |
duke@435 | 3674 | // After null check, get the object's klass. |
duke@435 | 3675 | Node* obj_klass = load_object_klass(obj); |
duke@435 | 3676 | |
duke@435 | 3677 | // This call may be virtual (invokevirtual) or bound (invokespecial). |
duke@435 | 3678 | // For each case we generate slightly different code. |
duke@435 | 3679 | |
duke@435 | 3680 | // We only go to the fast case code if we pass a number of guards. The |
duke@435 | 3681 | // paths which do not pass are accumulated in the slow_region. |
duke@435 | 3682 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 3683 | record_for_igvn(slow_region); |
duke@435 | 3684 | |
duke@435 | 3685 | // If this is a virtual call, we generate a funny guard. We pull out |
duke@435 | 3686 | // the vtable entry corresponding to hashCode() from the target object. |
duke@435 | 3687 | // If the target method which we are calling happens to be the native |
duke@435 | 3688 | // Object hashCode() method, we pass the guard. We do not need this |
duke@435 | 3689 | // guard for non-virtual calls -- the caller is known to be the native |
duke@435 | 3690 | // Object hashCode(). |
duke@435 | 3691 | if (is_virtual) { |
duke@435 | 3692 | generate_virtual_guard(obj_klass, slow_region); |
duke@435 | 3693 | } |
duke@435 | 3694 | |
duke@435 | 3695 | // Get the header out of the object, use LoadMarkNode when available |
duke@435 | 3696 | Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); |
kvn@1964 | 3697 | Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type()); |
duke@435 | 3698 | |
duke@435 | 3699 | // Test the header to see if it is unlocked. |
duke@435 | 3700 | Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place); |
duke@435 | 3701 | Node *lmasked_header = _gvn.transform( new (C, 3) AndXNode(header, lock_mask) ); |
duke@435 | 3702 | Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value); |
duke@435 | 3703 | Node *chk_unlocked = _gvn.transform( new (C, 3) CmpXNode( lmasked_header, unlocked_val)); |
duke@435 | 3704 | Node *test_unlocked = _gvn.transform( new (C, 2) BoolNode( chk_unlocked, BoolTest::ne) ); |
duke@435 | 3705 | |
duke@435 | 3706 | generate_slow_guard(test_unlocked, slow_region); |
duke@435 | 3707 | |
duke@435 | 3708 | // Get the hash value and check to see that it has been properly assigned. |
duke@435 | 3709 | // We depend on hash_mask being at most 32 bits and avoid the use of |
duke@435 | 3710 | // hash_mask_in_place because it could be larger than 32 bits in a 64-bit |
duke@435 | 3711 | // vm: see markOop.hpp. |
duke@435 | 3712 | Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask); |
duke@435 | 3713 | Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift); |
duke@435 | 3714 | Node *hshifted_header= _gvn.transform( new (C, 3) URShiftXNode(header, hash_shift) ); |
duke@435 | 3715 | // This hack lets the hash bits live anywhere in the mark object now, as long |
twisti@1040 | 3716 | // as the shift drops the relevant bits into the low 32 bits. Note that |
duke@435 | 3717 | // Java spec says that HashCode is an int so there's no point in capturing |
duke@435 | 3718 | // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build). |
duke@435 | 3719 | hshifted_header = ConvX2I(hshifted_header); |
duke@435 | 3720 | Node *hash_val = _gvn.transform( new (C, 3) AndINode(hshifted_header, hash_mask) ); |
duke@435 | 3721 | |
duke@435 | 3722 | Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash); |
duke@435 | 3723 | Node *chk_assigned = _gvn.transform( new (C, 3) CmpINode( hash_val, no_hash_val)); |
duke@435 | 3724 | Node *test_assigned = _gvn.transform( new (C, 2) BoolNode( chk_assigned, BoolTest::eq) ); |
duke@435 | 3725 | |
duke@435 | 3726 | generate_slow_guard(test_assigned, slow_region); |
duke@435 | 3727 | |
duke@435 | 3728 | Node* init_mem = reset_memory(); |
duke@435 | 3729 | // fill in the rest of the null path: |
duke@435 | 3730 | result_io ->init_req(_null_path, i_o()); |
duke@435 | 3731 | result_mem->init_req(_null_path, init_mem); |
duke@435 | 3732 | |
duke@435 | 3733 | result_val->init_req(_fast_path, hash_val); |
duke@435 | 3734 | result_reg->init_req(_fast_path, control()); |
duke@435 | 3735 | result_io ->init_req(_fast_path, i_o()); |
duke@435 | 3736 | result_mem->init_req(_fast_path, init_mem); |
duke@435 | 3737 | |
duke@435 | 3738 | // Generate code for the slow case. We make a call to hashCode(). |
duke@435 | 3739 | set_control(_gvn.transform(slow_region)); |
duke@435 | 3740 | if (!stopped()) { |
duke@435 | 3741 | // No need for PreserveJVMState, because we're using up the present state. |
duke@435 | 3742 | set_all_memory(init_mem); |
duke@435 | 3743 | vmIntrinsics::ID hashCode_id = vmIntrinsics::_hashCode; |
duke@435 | 3744 | if (is_static) hashCode_id = vmIntrinsics::_identityHashCode; |
duke@435 | 3745 | CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static); |
duke@435 | 3746 | Node* slow_result = set_results_for_java_call(slow_call); |
duke@435 | 3747 | // this->control() comes from set_results_for_java_call |
duke@435 | 3748 | result_reg->init_req(_slow_path, control()); |
duke@435 | 3749 | result_val->init_req(_slow_path, slow_result); |
duke@435 | 3750 | result_io ->set_req(_slow_path, i_o()); |
duke@435 | 3751 | result_mem ->set_req(_slow_path, reset_memory()); |
duke@435 | 3752 | } |
duke@435 | 3753 | |
duke@435 | 3754 | // Return the combined state. |
duke@435 | 3755 | set_i_o( _gvn.transform(result_io) ); |
duke@435 | 3756 | set_all_memory( _gvn.transform(result_mem) ); |
duke@435 | 3757 | push_result(result_reg, result_val); |
duke@435 | 3758 | |
duke@435 | 3759 | return true; |
duke@435 | 3760 | } |
duke@435 | 3761 | |
duke@435 | 3762 | //---------------------------inline_native_getClass---------------------------- |
twisti@1040 | 3763 | // Build special case code for calls to getClass on an object. |
duke@435 | 3764 | bool LibraryCallKit::inline_native_getClass() { |
duke@435 | 3765 | Node* obj = null_check_receiver(callee()); |
duke@435 | 3766 | if (stopped()) return true; |
duke@435 | 3767 | push( load_mirror_from_klass(load_object_klass(obj)) ); |
duke@435 | 3768 | return true; |
duke@435 | 3769 | } |
duke@435 | 3770 | |
duke@435 | 3771 | //-----------------inline_native_Reflection_getCallerClass--------------------- |
duke@435 | 3772 | // In the presence of deep enough inlining, getCallerClass() becomes a no-op. |
duke@435 | 3773 | // |
duke@435 | 3774 | // NOTE that this code must perform the same logic as |
duke@435 | 3775 | // vframeStream::security_get_caller_frame in that it must skip |
duke@435 | 3776 | // Method.invoke() and auxiliary frames. |
duke@435 | 3777 | |
duke@435 | 3778 | |
duke@435 | 3779 | |
duke@435 | 3780 | |
duke@435 | 3781 | bool LibraryCallKit::inline_native_Reflection_getCallerClass() { |
duke@435 | 3782 | ciMethod* method = callee(); |
duke@435 | 3783 | |
duke@435 | 3784 | #ifndef PRODUCT |
duke@435 | 3785 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3786 | tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass"); |
duke@435 | 3787 | } |
duke@435 | 3788 | #endif |
duke@435 | 3789 | |
duke@435 | 3790 | debug_only(int saved_sp = _sp); |
duke@435 | 3791 | |
duke@435 | 3792 | // Argument words: (int depth) |
duke@435 | 3793 | int nargs = 1; |
duke@435 | 3794 | |
duke@435 | 3795 | _sp += nargs; |
duke@435 | 3796 | Node* caller_depth_node = pop(); |
duke@435 | 3797 | |
duke@435 | 3798 | assert(saved_sp == _sp, "must have correct argument count"); |
duke@435 | 3799 | |
duke@435 | 3800 | // The depth value must be a constant in order for the runtime call |
duke@435 | 3801 | // to be eliminated. |
duke@435 | 3802 | const TypeInt* caller_depth_type = _gvn.type(caller_depth_node)->isa_int(); |
duke@435 | 3803 | if (caller_depth_type == NULL || !caller_depth_type->is_con()) { |
duke@435 | 3804 | #ifndef PRODUCT |
duke@435 | 3805 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3806 | tty->print_cr(" Bailing out because caller depth was not a constant"); |
duke@435 | 3807 | } |
duke@435 | 3808 | #endif |
duke@435 | 3809 | return false; |
duke@435 | 3810 | } |
duke@435 | 3811 | // Note that the JVM state at this point does not include the |
duke@435 | 3812 | // getCallerClass() frame which we are trying to inline. The |
duke@435 | 3813 | // semantics of getCallerClass(), however, are that the "first" |
duke@435 | 3814 | // frame is the getCallerClass() frame, so we subtract one from the |
duke@435 | 3815 | // requested depth before continuing. We don't inline requests of |
duke@435 | 3816 | // getCallerClass(0). |
duke@435 | 3817 | int caller_depth = caller_depth_type->get_con() - 1; |
duke@435 | 3818 | if (caller_depth < 0) { |
duke@435 | 3819 | #ifndef PRODUCT |
duke@435 | 3820 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3821 | tty->print_cr(" Bailing out because caller depth was %d", caller_depth); |
duke@435 | 3822 | } |
duke@435 | 3823 | #endif |
duke@435 | 3824 | return false; |
duke@435 | 3825 | } |
duke@435 | 3826 | |
duke@435 | 3827 | if (!jvms()->has_method()) { |
duke@435 | 3828 | #ifndef PRODUCT |
duke@435 | 3829 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3830 | tty->print_cr(" Bailing out because intrinsic was inlined at top level"); |
duke@435 | 3831 | } |
duke@435 | 3832 | #endif |
duke@435 | 3833 | return false; |
duke@435 | 3834 | } |
duke@435 | 3835 | int _depth = jvms()->depth(); // cache call chain depth |
duke@435 | 3836 | |
duke@435 | 3837 | // Walk back up the JVM state to find the caller at the required |
duke@435 | 3838 | // depth. NOTE that this code must perform the same logic as |
duke@435 | 3839 | // vframeStream::security_get_caller_frame in that it must skip |
duke@435 | 3840 | // Method.invoke() and auxiliary frames. Note also that depth is |
duke@435 | 3841 | // 1-based (1 is the bottom of the inlining). |
duke@435 | 3842 | int inlining_depth = _depth; |
duke@435 | 3843 | JVMState* caller_jvms = NULL; |
duke@435 | 3844 | |
duke@435 | 3845 | if (inlining_depth > 0) { |
duke@435 | 3846 | caller_jvms = jvms(); |
duke@435 | 3847 | assert(caller_jvms = jvms()->of_depth(inlining_depth), "inlining_depth == our depth"); |
duke@435 | 3848 | do { |
duke@435 | 3849 | // The following if-tests should be performed in this order |
duke@435 | 3850 | if (is_method_invoke_or_aux_frame(caller_jvms)) { |
duke@435 | 3851 | // Skip a Method.invoke() or auxiliary frame |
duke@435 | 3852 | } else if (caller_depth > 0) { |
duke@435 | 3853 | // Skip real frame |
duke@435 | 3854 | --caller_depth; |
duke@435 | 3855 | } else { |
duke@435 | 3856 | // We're done: reached desired caller after skipping. |
duke@435 | 3857 | break; |
duke@435 | 3858 | } |
duke@435 | 3859 | caller_jvms = caller_jvms->caller(); |
duke@435 | 3860 | --inlining_depth; |
duke@435 | 3861 | } while (inlining_depth > 0); |
duke@435 | 3862 | } |
duke@435 | 3863 | |
duke@435 | 3864 | if (inlining_depth == 0) { |
duke@435 | 3865 | #ifndef PRODUCT |
duke@435 | 3866 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3867 | tty->print_cr(" Bailing out because caller depth (%d) exceeded inlining depth (%d)", caller_depth_type->get_con(), _depth); |
duke@435 | 3868 | tty->print_cr(" JVM state at this point:"); |
duke@435 | 3869 | for (int i = _depth; i >= 1; i--) { |
duke@435 | 3870 | tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8()); |
duke@435 | 3871 | } |
duke@435 | 3872 | } |
duke@435 | 3873 | #endif |
duke@435 | 3874 | return false; // Reached end of inlining |
duke@435 | 3875 | } |
duke@435 | 3876 | |
duke@435 | 3877 | // Acquire method holder as java.lang.Class |
duke@435 | 3878 | ciInstanceKlass* caller_klass = caller_jvms->method()->holder(); |
duke@435 | 3879 | ciInstance* caller_mirror = caller_klass->java_mirror(); |
duke@435 | 3880 | // Push this as a constant |
duke@435 | 3881 | push(makecon(TypeInstPtr::make(caller_mirror))); |
duke@435 | 3882 | #ifndef PRODUCT |
duke@435 | 3883 | if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) { |
duke@435 | 3884 | tty->print_cr(" Succeeded: caller = %s.%s, caller depth = %d, depth = %d", caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), caller_depth_type->get_con(), _depth); |
duke@435 | 3885 | tty->print_cr(" JVM state at this point:"); |
duke@435 | 3886 | for (int i = _depth; i >= 1; i--) { |
duke@435 | 3887 | tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8()); |
duke@435 | 3888 | } |
duke@435 | 3889 | } |
duke@435 | 3890 | #endif |
duke@435 | 3891 | return true; |
duke@435 | 3892 | } |
duke@435 | 3893 | |
duke@435 | 3894 | // Helper routine for above |
duke@435 | 3895 | bool LibraryCallKit::is_method_invoke_or_aux_frame(JVMState* jvms) { |
twisti@1587 | 3896 | ciMethod* method = jvms->method(); |
twisti@1587 | 3897 | |
duke@435 | 3898 | // Is this the Method.invoke method itself? |
twisti@1587 | 3899 | if (method->intrinsic_id() == vmIntrinsics::_invoke) |
duke@435 | 3900 | return true; |
duke@435 | 3901 | |
duke@435 | 3902 | // Is this a helper, defined somewhere underneath MethodAccessorImpl. |
twisti@1587 | 3903 | ciKlass* k = method->holder(); |
duke@435 | 3904 | if (k->is_instance_klass()) { |
duke@435 | 3905 | ciInstanceKlass* ik = k->as_instance_klass(); |
duke@435 | 3906 | for (; ik != NULL; ik = ik->super()) { |
duke@435 | 3907 | if (ik->name() == ciSymbol::sun_reflect_MethodAccessorImpl() && |
duke@435 | 3908 | ik == env()->find_system_klass(ik->name())) { |
duke@435 | 3909 | return true; |
duke@435 | 3910 | } |
duke@435 | 3911 | } |
duke@435 | 3912 | } |
twisti@1587 | 3913 | else if (method->is_method_handle_adapter()) { |
twisti@1587 | 3914 | // This is an internal adapter frame from the MethodHandleCompiler -- skip it |
twisti@1587 | 3915 | return true; |
twisti@1587 | 3916 | } |
duke@435 | 3917 | |
duke@435 | 3918 | return false; |
duke@435 | 3919 | } |
duke@435 | 3920 | |
duke@435 | 3921 | static int value_field_offset = -1; // offset of the "value" field of AtomicLongCSImpl. This is needed by |
duke@435 | 3922 | // inline_native_AtomicLong_attemptUpdate() but it has no way of |
duke@435 | 3923 | // computing it since there is no lookup field by name function in the |
duke@435 | 3924 | // CI interface. This is computed and set by inline_native_AtomicLong_get(). |
duke@435 | 3925 | // Using a static variable here is safe even if we have multiple compilation |
duke@435 | 3926 | // threads because the offset is constant. At worst the same offset will be |
duke@435 | 3927 | // computed and stored multiple |
duke@435 | 3928 | |
duke@435 | 3929 | bool LibraryCallKit::inline_native_AtomicLong_get() { |
duke@435 | 3930 | // Restore the stack and pop off the argument |
duke@435 | 3931 | _sp+=1; |
duke@435 | 3932 | Node *obj = pop(); |
duke@435 | 3933 | |
duke@435 | 3934 | // get the offset of the "value" field. Since the CI interfaces |
duke@435 | 3935 | // does not provide a way to look up a field by name, we scan the bytecodes |
duke@435 | 3936 | // to get the field index. We expect the first 2 instructions of the method |
duke@435 | 3937 | // to be: |
duke@435 | 3938 | // 0 aload_0 |
duke@435 | 3939 | // 1 getfield "value" |
duke@435 | 3940 | ciMethod* method = callee(); |
duke@435 | 3941 | if (value_field_offset == -1) |
duke@435 | 3942 | { |
duke@435 | 3943 | ciField* value_field; |
duke@435 | 3944 | ciBytecodeStream iter(method); |
duke@435 | 3945 | Bytecodes::Code bc = iter.next(); |
duke@435 | 3946 | |
duke@435 | 3947 | if ((bc != Bytecodes::_aload_0) && |
duke@435 | 3948 | ((bc != Bytecodes::_aload) || (iter.get_index() != 0))) |
duke@435 | 3949 | return false; |
duke@435 | 3950 | bc = iter.next(); |
duke@435 | 3951 | if (bc != Bytecodes::_getfield) |
duke@435 | 3952 | return false; |
duke@435 | 3953 | bool ignore; |
duke@435 | 3954 | value_field = iter.get_field(ignore); |
duke@435 | 3955 | value_field_offset = value_field->offset_in_bytes(); |
duke@435 | 3956 | } |
duke@435 | 3957 | |
duke@435 | 3958 | // Null check without removing any arguments. |
duke@435 | 3959 | _sp++; |
duke@435 | 3960 | obj = do_null_check(obj, T_OBJECT); |
duke@435 | 3961 | _sp--; |
duke@435 | 3962 | // Check for locking null object |
duke@435 | 3963 | if (stopped()) return true; |
duke@435 | 3964 | |
duke@435 | 3965 | Node *adr = basic_plus_adr(obj, obj, value_field_offset); |
duke@435 | 3966 | const TypePtr *adr_type = _gvn.type(adr)->is_ptr(); |
duke@435 | 3967 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 3968 | |
duke@435 | 3969 | Node *result = _gvn.transform(new (C, 3) LoadLLockedNode(control(), memory(alias_idx), adr)); |
duke@435 | 3970 | |
duke@435 | 3971 | push_pair(result); |
duke@435 | 3972 | |
duke@435 | 3973 | return true; |
duke@435 | 3974 | } |
duke@435 | 3975 | |
duke@435 | 3976 | bool LibraryCallKit::inline_native_AtomicLong_attemptUpdate() { |
duke@435 | 3977 | // Restore the stack and pop off the arguments |
duke@435 | 3978 | _sp+=5; |
duke@435 | 3979 | Node *newVal = pop_pair(); |
duke@435 | 3980 | Node *oldVal = pop_pair(); |
duke@435 | 3981 | Node *obj = pop(); |
duke@435 | 3982 | |
duke@435 | 3983 | // we need the offset of the "value" field which was computed when |
duke@435 | 3984 | // inlining the get() method. Give up if we don't have it. |
duke@435 | 3985 | if (value_field_offset == -1) |
duke@435 | 3986 | return false; |
duke@435 | 3987 | |
duke@435 | 3988 | // Null check without removing any arguments. |
duke@435 | 3989 | _sp+=5; |
duke@435 | 3990 | obj = do_null_check(obj, T_OBJECT); |
duke@435 | 3991 | _sp-=5; |
duke@435 | 3992 | // Check for locking null object |
duke@435 | 3993 | if (stopped()) return true; |
duke@435 | 3994 | |
duke@435 | 3995 | Node *adr = basic_plus_adr(obj, obj, value_field_offset); |
duke@435 | 3996 | const TypePtr *adr_type = _gvn.type(adr)->is_ptr(); |
duke@435 | 3997 | int alias_idx = C->get_alias_index(adr_type); |
duke@435 | 3998 | |
kvn@855 | 3999 | Node *cas = _gvn.transform(new (C, 5) StoreLConditionalNode(control(), memory(alias_idx), adr, newVal, oldVal)); |
kvn@855 | 4000 | Node *store_proj = _gvn.transform( new (C, 1) SCMemProjNode(cas)); |
duke@435 | 4001 | set_memory(store_proj, alias_idx); |
kvn@855 | 4002 | Node *bol = _gvn.transform( new (C, 2) BoolNode( cas, BoolTest::eq ) ); |
kvn@855 | 4003 | |
kvn@855 | 4004 | Node *result; |
kvn@855 | 4005 | // CMove node is not used to be able fold a possible check code |
kvn@855 | 4006 | // after attemptUpdate() call. This code could be transformed |
kvn@855 | 4007 | // into CMove node by loop optimizations. |
kvn@855 | 4008 | { |
kvn@855 | 4009 | RegionNode *r = new (C, 3) RegionNode(3); |
kvn@855 | 4010 | result = new (C, 3) PhiNode(r, TypeInt::BOOL); |
kvn@855 | 4011 | |
kvn@855 | 4012 | Node *iff = create_and_xform_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN); |
kvn@855 | 4013 | Node *iftrue = opt_iff(r, iff); |
kvn@855 | 4014 | r->init_req(1, iftrue); |
kvn@855 | 4015 | result->init_req(1, intcon(1)); |
kvn@855 | 4016 | result->init_req(2, intcon(0)); |
kvn@855 | 4017 | |
kvn@855 | 4018 | set_control(_gvn.transform(r)); |
kvn@855 | 4019 | record_for_igvn(r); |
kvn@855 | 4020 | |
kvn@855 | 4021 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
kvn@855 | 4022 | } |
kvn@855 | 4023 | |
kvn@855 | 4024 | push(_gvn.transform(result)); |
duke@435 | 4025 | return true; |
duke@435 | 4026 | } |
duke@435 | 4027 | |
duke@435 | 4028 | bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) { |
duke@435 | 4029 | // restore the arguments |
duke@435 | 4030 | _sp += arg_size(); |
duke@435 | 4031 | |
duke@435 | 4032 | switch (id) { |
duke@435 | 4033 | case vmIntrinsics::_floatToRawIntBits: |
duke@435 | 4034 | push(_gvn.transform( new (C, 2) MoveF2INode(pop()))); |
duke@435 | 4035 | break; |
duke@435 | 4036 | |
duke@435 | 4037 | case vmIntrinsics::_intBitsToFloat: |
duke@435 | 4038 | push(_gvn.transform( new (C, 2) MoveI2FNode(pop()))); |
duke@435 | 4039 | break; |
duke@435 | 4040 | |
duke@435 | 4041 | case vmIntrinsics::_doubleToRawLongBits: |
duke@435 | 4042 | push_pair(_gvn.transform( new (C, 2) MoveD2LNode(pop_pair()))); |
duke@435 | 4043 | break; |
duke@435 | 4044 | |
duke@435 | 4045 | case vmIntrinsics::_longBitsToDouble: |
duke@435 | 4046 | push_pair(_gvn.transform( new (C, 2) MoveL2DNode(pop_pair()))); |
duke@435 | 4047 | break; |
duke@435 | 4048 | |
duke@435 | 4049 | case vmIntrinsics::_doubleToLongBits: { |
duke@435 | 4050 | Node* value = pop_pair(); |
duke@435 | 4051 | |
duke@435 | 4052 | // two paths (plus control) merge in a wood |
duke@435 | 4053 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 4054 | Node *phi = new (C, 3) PhiNode(r, TypeLong::LONG); |
duke@435 | 4055 | |
duke@435 | 4056 | Node *cmpisnan = _gvn.transform( new (C, 3) CmpDNode(value, value)); |
duke@435 | 4057 | // Build the boolean node |
duke@435 | 4058 | Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) ); |
duke@435 | 4059 | |
duke@435 | 4060 | // Branch either way. |
duke@435 | 4061 | // NaN case is less traveled, which makes all the difference. |
duke@435 | 4062 | IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 4063 | Node *opt_isnan = _gvn.transform(ifisnan); |
duke@435 | 4064 | assert( opt_isnan->is_If(), "Expect an IfNode"); |
duke@435 | 4065 | IfNode *opt_ifisnan = (IfNode*)opt_isnan; |
duke@435 | 4066 | Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) ); |
duke@435 | 4067 | |
duke@435 | 4068 | set_control(iftrue); |
duke@435 | 4069 | |
duke@435 | 4070 | static const jlong nan_bits = CONST64(0x7ff8000000000000); |
duke@435 | 4071 | Node *slow_result = longcon(nan_bits); // return NaN |
duke@435 | 4072 | phi->init_req(1, _gvn.transform( slow_result )); |
duke@435 | 4073 | r->init_req(1, iftrue); |
duke@435 | 4074 | |
duke@435 | 4075 | // Else fall through |
duke@435 | 4076 | Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) ); |
duke@435 | 4077 | set_control(iffalse); |
duke@435 | 4078 | |
duke@435 | 4079 | phi->init_req(2, _gvn.transform( new (C, 2) MoveD2LNode(value))); |
duke@435 | 4080 | r->init_req(2, iffalse); |
duke@435 | 4081 | |
duke@435 | 4082 | // Post merge |
duke@435 | 4083 | set_control(_gvn.transform(r)); |
duke@435 | 4084 | record_for_igvn(r); |
duke@435 | 4085 | |
duke@435 | 4086 | Node* result = _gvn.transform(phi); |
duke@435 | 4087 | assert(result->bottom_type()->isa_long(), "must be"); |
duke@435 | 4088 | push_pair(result); |
duke@435 | 4089 | |
duke@435 | 4090 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 4091 | |
duke@435 | 4092 | break; |
duke@435 | 4093 | } |
duke@435 | 4094 | |
duke@435 | 4095 | case vmIntrinsics::_floatToIntBits: { |
duke@435 | 4096 | Node* value = pop(); |
duke@435 | 4097 | |
duke@435 | 4098 | // two paths (plus control) merge in a wood |
duke@435 | 4099 | RegionNode *r = new (C, 3) RegionNode(3); |
duke@435 | 4100 | Node *phi = new (C, 3) PhiNode(r, TypeInt::INT); |
duke@435 | 4101 | |
duke@435 | 4102 | Node *cmpisnan = _gvn.transform( new (C, 3) CmpFNode(value, value)); |
duke@435 | 4103 | // Build the boolean node |
duke@435 | 4104 | Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) ); |
duke@435 | 4105 | |
duke@435 | 4106 | // Branch either way. |
duke@435 | 4107 | // NaN case is less traveled, which makes all the difference. |
duke@435 | 4108 | IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); |
duke@435 | 4109 | Node *opt_isnan = _gvn.transform(ifisnan); |
duke@435 | 4110 | assert( opt_isnan->is_If(), "Expect an IfNode"); |
duke@435 | 4111 | IfNode *opt_ifisnan = (IfNode*)opt_isnan; |
duke@435 | 4112 | Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) ); |
duke@435 | 4113 | |
duke@435 | 4114 | set_control(iftrue); |
duke@435 | 4115 | |
duke@435 | 4116 | static const jint nan_bits = 0x7fc00000; |
duke@435 | 4117 | Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN |
duke@435 | 4118 | phi->init_req(1, _gvn.transform( slow_result )); |
duke@435 | 4119 | r->init_req(1, iftrue); |
duke@435 | 4120 | |
duke@435 | 4121 | // Else fall through |
duke@435 | 4122 | Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) ); |
duke@435 | 4123 | set_control(iffalse); |
duke@435 | 4124 | |
duke@435 | 4125 | phi->init_req(2, _gvn.transform( new (C, 2) MoveF2INode(value))); |
duke@435 | 4126 | r->init_req(2, iffalse); |
duke@435 | 4127 | |
duke@435 | 4128 | // Post merge |
duke@435 | 4129 | set_control(_gvn.transform(r)); |
duke@435 | 4130 | record_for_igvn(r); |
duke@435 | 4131 | |
duke@435 | 4132 | Node* result = _gvn.transform(phi); |
duke@435 | 4133 | assert(result->bottom_type()->isa_int(), "must be"); |
duke@435 | 4134 | push(result); |
duke@435 | 4135 | |
duke@435 | 4136 | C->set_has_split_ifs(true); // Has chance for split-if optimization |
duke@435 | 4137 | |
duke@435 | 4138 | break; |
duke@435 | 4139 | } |
duke@435 | 4140 | |
duke@435 | 4141 | default: |
duke@435 | 4142 | ShouldNotReachHere(); |
duke@435 | 4143 | } |
duke@435 | 4144 | |
duke@435 | 4145 | return true; |
duke@435 | 4146 | } |
duke@435 | 4147 | |
duke@435 | 4148 | #ifdef _LP64 |
duke@435 | 4149 | #define XTOP ,top() /*additional argument*/ |
duke@435 | 4150 | #else //_LP64 |
duke@435 | 4151 | #define XTOP /*no additional argument*/ |
duke@435 | 4152 | #endif //_LP64 |
duke@435 | 4153 | |
duke@435 | 4154 | //----------------------inline_unsafe_copyMemory------------------------- |
duke@435 | 4155 | bool LibraryCallKit::inline_unsafe_copyMemory() { |
duke@435 | 4156 | if (callee()->is_static()) return false; // caller must have the capability! |
duke@435 | 4157 | int nargs = 1 + 5 + 3; // 5 args: (src: ptr,off, dst: ptr,off, size) |
duke@435 | 4158 | assert(signature()->size() == nargs-1, "copy has 5 arguments"); |
duke@435 | 4159 | null_check_receiver(callee()); // check then ignore argument(0) |
duke@435 | 4160 | if (stopped()) return true; |
duke@435 | 4161 | |
duke@435 | 4162 | C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". |
duke@435 | 4163 | |
duke@435 | 4164 | Node* src_ptr = argument(1); |
duke@435 | 4165 | Node* src_off = ConvL2X(argument(2)); |
duke@435 | 4166 | assert(argument(3)->is_top(), "2nd half of long"); |
duke@435 | 4167 | Node* dst_ptr = argument(4); |
duke@435 | 4168 | Node* dst_off = ConvL2X(argument(5)); |
duke@435 | 4169 | assert(argument(6)->is_top(), "2nd half of long"); |
duke@435 | 4170 | Node* size = ConvL2X(argument(7)); |
duke@435 | 4171 | assert(argument(8)->is_top(), "2nd half of long"); |
duke@435 | 4172 | |
duke@435 | 4173 | assert(Unsafe_field_offset_to_byte_offset(11) == 11, |
duke@435 | 4174 | "fieldOffset must be byte-scaled"); |
duke@435 | 4175 | |
duke@435 | 4176 | Node* src = make_unsafe_address(src_ptr, src_off); |
duke@435 | 4177 | Node* dst = make_unsafe_address(dst_ptr, dst_off); |
duke@435 | 4178 | |
duke@435 | 4179 | // Conservatively insert a memory barrier on all memory slices. |
duke@435 | 4180 | // Do not let writes of the copy source or destination float below the copy. |
duke@435 | 4181 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4182 | |
duke@435 | 4183 | // Call it. Note that the length argument is not scaled. |
duke@435 | 4184 | make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 4185 | OptoRuntime::fast_arraycopy_Type(), |
duke@435 | 4186 | StubRoutines::unsafe_arraycopy(), |
duke@435 | 4187 | "unsafe_arraycopy", |
duke@435 | 4188 | TypeRawPtr::BOTTOM, |
duke@435 | 4189 | src, dst, size XTOP); |
duke@435 | 4190 | |
duke@435 | 4191 | // Do not let reads of the copy destination float above the copy. |
duke@435 | 4192 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4193 | |
duke@435 | 4194 | return true; |
duke@435 | 4195 | } |
duke@435 | 4196 | |
kvn@1268 | 4197 | //------------------------clone_coping----------------------------------- |
kvn@1268 | 4198 | // Helper function for inline_native_clone. |
kvn@1268 | 4199 | void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { |
kvn@1268 | 4200 | assert(obj_size != NULL, ""); |
kvn@1268 | 4201 | Node* raw_obj = alloc_obj->in(1); |
kvn@1268 | 4202 | assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); |
kvn@1268 | 4203 | |
roland@3392 | 4204 | AllocateNode* alloc = NULL; |
kvn@1268 | 4205 | if (ReduceBulkZeroing) { |
kvn@1268 | 4206 | // We will be completely responsible for initializing this object - |
kvn@1268 | 4207 | // mark Initialize node as complete. |
roland@3392 | 4208 | alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); |
kvn@1268 | 4209 | // The object was just allocated - there should be no any stores! |
kvn@1268 | 4210 | guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); |
roland@3392 | 4211 | // Mark as complete_with_arraycopy so that on AllocateNode |
roland@3392 | 4212 | // expansion, we know this AllocateNode is initialized by an array |
roland@3392 | 4213 | // copy and a StoreStore barrier exists after the array copy. |
roland@3392 | 4214 | alloc->initialization()->set_complete_with_arraycopy(); |
kvn@1268 | 4215 | } |
kvn@1268 | 4216 | |
kvn@1268 | 4217 | // Copy the fastest available way. |
kvn@1268 | 4218 | // TODO: generate fields copies for small objects instead. |
kvn@1268 | 4219 | Node* src = obj; |
kvn@1393 | 4220 | Node* dest = alloc_obj; |
kvn@1268 | 4221 | Node* size = _gvn.transform(obj_size); |
kvn@1268 | 4222 | |
kvn@1268 | 4223 | // Exclude the header but include array length to copy by 8 bytes words. |
kvn@1268 | 4224 | // Can't use base_offset_in_bytes(bt) since basic type is unknown. |
kvn@1268 | 4225 | int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : |
kvn@1268 | 4226 | instanceOopDesc::base_offset_in_bytes(); |
kvn@1268 | 4227 | // base_off: |
kvn@1268 | 4228 | // 8 - 32-bit VM |
kvn@1268 | 4229 | // 12 - 64-bit VM, compressed oops |
kvn@1268 | 4230 | // 16 - 64-bit VM, normal oops |
kvn@1268 | 4231 | if (base_off % BytesPerLong != 0) { |
kvn@1268 | 4232 | assert(UseCompressedOops, ""); |
kvn@1268 | 4233 | if (is_array) { |
kvn@1268 | 4234 | // Exclude length to copy by 8 bytes words. |
kvn@1268 | 4235 | base_off += sizeof(int); |
kvn@1268 | 4236 | } else { |
kvn@1268 | 4237 | // Include klass to copy by 8 bytes words. |
kvn@1268 | 4238 | base_off = instanceOopDesc::klass_offset_in_bytes(); |
kvn@1268 | 4239 | } |
kvn@1268 | 4240 | assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); |
kvn@1268 | 4241 | } |
kvn@1268 | 4242 | src = basic_plus_adr(src, base_off); |
kvn@1268 | 4243 | dest = basic_plus_adr(dest, base_off); |
kvn@1268 | 4244 | |
kvn@1268 | 4245 | // Compute the length also, if needed: |
kvn@1268 | 4246 | Node* countx = size; |
kvn@1268 | 4247 | countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); |
kvn@1268 | 4248 | countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); |
kvn@1268 | 4249 | |
kvn@1268 | 4250 | const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
kvn@1268 | 4251 | bool disjoint_bases = true; |
kvn@1268 | 4252 | generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, |
iveresov@2606 | 4253 | src, NULL, dest, NULL, countx, |
iveresov@2606 | 4254 | /*dest_uninitialized*/true); |
kvn@1268 | 4255 | |
kvn@1268 | 4256 | // If necessary, emit some card marks afterwards. (Non-arrays only.) |
kvn@1268 | 4257 | if (card_mark) { |
kvn@1268 | 4258 | assert(!is_array, ""); |
kvn@1268 | 4259 | // Put in store barrier for any and all oops we are sticking |
kvn@1268 | 4260 | // into this object. (We could avoid this if we could prove |
kvn@1268 | 4261 | // that the object type contains no oop fields at all.) |
kvn@1268 | 4262 | Node* no_particular_value = NULL; |
kvn@1268 | 4263 | Node* no_particular_field = NULL; |
kvn@1268 | 4264 | int raw_adr_idx = Compile::AliasIdxRaw; |
kvn@1268 | 4265 | post_barrier(control(), |
kvn@1268 | 4266 | memory(raw_adr_type), |
kvn@1393 | 4267 | alloc_obj, |
kvn@1268 | 4268 | no_particular_field, |
kvn@1268 | 4269 | raw_adr_idx, |
kvn@1268 | 4270 | no_particular_value, |
kvn@1268 | 4271 | T_OBJECT, |
kvn@1268 | 4272 | false); |
kvn@1268 | 4273 | } |
kvn@1268 | 4274 | |
kvn@1393 | 4275 | // Do not let reads from the cloned object float above the arraycopy. |
roland@3392 | 4276 | if (alloc != NULL) { |
roland@3392 | 4277 | // Do not let stores that initialize this object be reordered with |
roland@3392 | 4278 | // a subsequent store that would make this object accessible by |
roland@3392 | 4279 | // other threads. |
roland@3392 | 4280 | // Record what AllocateNode this StoreStore protects so that |
roland@3392 | 4281 | // escape analysis can go from the MemBarStoreStoreNode to the |
roland@3392 | 4282 | // AllocateNode and eliminate the MemBarStoreStoreNode if possible |
roland@3392 | 4283 | // based on the escape status of the AllocateNode. |
roland@3392 | 4284 | insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress)); |
roland@3392 | 4285 | } else { |
roland@3392 | 4286 | insert_mem_bar(Op_MemBarCPUOrder); |
roland@3392 | 4287 | } |
kvn@1268 | 4288 | } |
duke@435 | 4289 | |
duke@435 | 4290 | //------------------------inline_native_clone---------------------------- |
duke@435 | 4291 | // Here are the simple edge cases: |
duke@435 | 4292 | // null receiver => normal trap |
duke@435 | 4293 | // virtual and clone was overridden => slow path to out-of-line clone |
duke@435 | 4294 | // not cloneable or finalizer => slow path to out-of-line Object.clone |
duke@435 | 4295 | // |
duke@435 | 4296 | // The general case has two steps, allocation and copying. |
duke@435 | 4297 | // Allocation has two cases, and uses GraphKit::new_instance or new_array. |
duke@435 | 4298 | // |
duke@435 | 4299 | // Copying also has two cases, oop arrays and everything else. |
duke@435 | 4300 | // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy). |
duke@435 | 4301 | // Everything else uses the tight inline loop supplied by CopyArrayNode. |
duke@435 | 4302 | // |
duke@435 | 4303 | // These steps fold up nicely if and when the cloned object's klass |
duke@435 | 4304 | // can be sharply typed as an object array, a type array, or an instance. |
duke@435 | 4305 | // |
duke@435 | 4306 | bool LibraryCallKit::inline_native_clone(bool is_virtual) { |
duke@435 | 4307 | int nargs = 1; |
cfang@1337 | 4308 | PhiNode* result_val; |
duke@435 | 4309 | |
cfang@1335 | 4310 | //set the original stack and the reexecute bit for the interpreter to reexecute |
cfang@1335 | 4311 | //the bytecode that invokes Object.clone if deoptimization happens |
cfang@1335 | 4312 | { PreserveReexecuteState preexecs(this); |
cfang@1337 | 4313 | jvms()->set_should_reexecute(true); |
cfang@1337 | 4314 | |
cfang@1337 | 4315 | //null_check_receiver will adjust _sp (push and pop) |
cfang@1337 | 4316 | Node* obj = null_check_receiver(callee()); |
cfang@1337 | 4317 | if (stopped()) return true; |
cfang@1337 | 4318 | |
cfang@1335 | 4319 | _sp += nargs; |
cfang@1337 | 4320 | |
cfang@1337 | 4321 | Node* obj_klass = load_object_klass(obj); |
cfang@1337 | 4322 | const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr(); |
cfang@1337 | 4323 | const TypeOopPtr* toop = ((tklass != NULL) |
cfang@1337 | 4324 | ? tklass->as_instance_type() |
cfang@1337 | 4325 | : TypeInstPtr::NOTNULL); |
cfang@1337 | 4326 | |
cfang@1337 | 4327 | // Conservatively insert a memory barrier on all memory slices. |
cfang@1337 | 4328 | // Do not let writes into the original float below the clone. |
cfang@1337 | 4329 | insert_mem_bar(Op_MemBarCPUOrder); |
cfang@1337 | 4330 | |
cfang@1337 | 4331 | // paths into result_reg: |
cfang@1337 | 4332 | enum { |
cfang@1337 | 4333 | _slow_path = 1, // out-of-line call to clone method (virtual or not) |
cfang@1337 | 4334 | _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy |
cfang@1337 | 4335 | _array_path, // plain array allocation, plus arrayof_long_arraycopy |
cfang@1337 | 4336 | _instance_path, // plain instance allocation, plus arrayof_long_arraycopy |
cfang@1337 | 4337 | PATH_LIMIT |
cfang@1337 | 4338 | }; |
cfang@1337 | 4339 | RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
cfang@1337 | 4340 | result_val = new(C, PATH_LIMIT) PhiNode(result_reg, |
cfang@1337 | 4341 | TypeInstPtr::NOTNULL); |
cfang@1337 | 4342 | PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO); |
cfang@1337 | 4343 | PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY, |
cfang@1337 | 4344 | TypePtr::BOTTOM); |
cfang@1337 | 4345 | record_for_igvn(result_reg); |
cfang@1337 | 4346 | |
cfang@1337 | 4347 | const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; |
cfang@1337 | 4348 | int raw_adr_idx = Compile::AliasIdxRaw; |
cfang@1335 | 4349 | |
cfang@1335 | 4350 | Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); |
cfang@1335 | 4351 | if (array_ctl != NULL) { |
cfang@1335 | 4352 | // It's an array. |
cfang@1335 | 4353 | PreserveJVMState pjvms(this); |
cfang@1335 | 4354 | set_control(array_ctl); |
cfang@1335 | 4355 | Node* obj_length = load_array_length(obj); |
cfang@1335 | 4356 | Node* obj_size = NULL; |
kvn@2810 | 4357 | Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size); |
cfang@1335 | 4358 | |
cfang@1335 | 4359 | if (!use_ReduceInitialCardMarks()) { |
cfang@1335 | 4360 | // If it is an oop array, it requires very special treatment, |
cfang@1335 | 4361 | // because card marking is required on each card of the array. |
cfang@1335 | 4362 | Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); |
cfang@1335 | 4363 | if (is_obja != NULL) { |
cfang@1335 | 4364 | PreserveJVMState pjvms2(this); |
cfang@1335 | 4365 | set_control(is_obja); |
cfang@1335 | 4366 | // Generate a direct call to the right arraycopy function(s). |
cfang@1335 | 4367 | bool disjoint_bases = true; |
cfang@1335 | 4368 | bool length_never_negative = true; |
cfang@1335 | 4369 | generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, |
cfang@1335 | 4370 | obj, intcon(0), alloc_obj, intcon(0), |
cfang@1335 | 4371 | obj_length, |
cfang@1335 | 4372 | disjoint_bases, length_never_negative); |
cfang@1335 | 4373 | result_reg->init_req(_objArray_path, control()); |
cfang@1335 | 4374 | result_val->init_req(_objArray_path, alloc_obj); |
cfang@1335 | 4375 | result_i_o ->set_req(_objArray_path, i_o()); |
cfang@1335 | 4376 | result_mem ->set_req(_objArray_path, reset_memory()); |
cfang@1335 | 4377 | } |
cfang@1335 | 4378 | } |
cfang@1335 | 4379 | // Otherwise, there are no card marks to worry about. |
ysr@1462 | 4380 | // (We can dispense with card marks if we know the allocation |
ysr@1462 | 4381 | // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks |
ysr@1462 | 4382 | // causes the non-eden paths to take compensating steps to |
ysr@1462 | 4383 | // simulate a fresh allocation, so that no further |
ysr@1462 | 4384 | // card marks are required in compiled code to initialize |
ysr@1462 | 4385 | // the object.) |
cfang@1335 | 4386 | |
cfang@1335 | 4387 | if (!stopped()) { |
cfang@1335 | 4388 | copy_to_clone(obj, alloc_obj, obj_size, true, false); |
cfang@1335 | 4389 | |
cfang@1335 | 4390 | // Present the results of the copy. |
cfang@1335 | 4391 | result_reg->init_req(_array_path, control()); |
cfang@1335 | 4392 | result_val->init_req(_array_path, alloc_obj); |
cfang@1335 | 4393 | result_i_o ->set_req(_array_path, i_o()); |
cfang@1335 | 4394 | result_mem ->set_req(_array_path, reset_memory()); |
duke@435 | 4395 | } |
duke@435 | 4396 | } |
cfang@1335 | 4397 | |
cfang@1335 | 4398 | // We only go to the instance fast case code if we pass a number of guards. |
cfang@1335 | 4399 | // The paths which do not pass are accumulated in the slow_region. |
cfang@1335 | 4400 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
cfang@1335 | 4401 | record_for_igvn(slow_region); |
kvn@1268 | 4402 | if (!stopped()) { |
cfang@1335 | 4403 | // It's an instance (we did array above). Make the slow-path tests. |
cfang@1335 | 4404 | // If this is a virtual call, we generate a funny guard. We grab |
cfang@1335 | 4405 | // the vtable entry corresponding to clone() from the target object. |
cfang@1335 | 4406 | // If the target method which we are calling happens to be the |
cfang@1335 | 4407 | // Object clone() method, we pass the guard. We do not need this |
cfang@1335 | 4408 | // guard for non-virtual calls; the caller is known to be the native |
cfang@1335 | 4409 | // Object clone(). |
cfang@1335 | 4410 | if (is_virtual) { |
cfang@1335 | 4411 | generate_virtual_guard(obj_klass, slow_region); |
cfang@1335 | 4412 | } |
cfang@1335 | 4413 | |
cfang@1335 | 4414 | // The object must be cloneable and must not have a finalizer. |
cfang@1335 | 4415 | // Both of these conditions may be checked in a single test. |
cfang@1335 | 4416 | // We could optimize the cloneable test further, but we don't care. |
cfang@1335 | 4417 | generate_access_flags_guard(obj_klass, |
cfang@1335 | 4418 | // Test both conditions: |
cfang@1335 | 4419 | JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER, |
cfang@1335 | 4420 | // Must be cloneable but not finalizer: |
cfang@1335 | 4421 | JVM_ACC_IS_CLONEABLE, |
cfang@1335 | 4422 | slow_region); |
kvn@1268 | 4423 | } |
cfang@1335 | 4424 | |
cfang@1335 | 4425 | if (!stopped()) { |
cfang@1335 | 4426 | // It's an instance, and it passed the slow-path tests. |
cfang@1335 | 4427 | PreserveJVMState pjvms(this); |
cfang@1335 | 4428 | Node* obj_size = NULL; |
kvn@2810 | 4429 | Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size); |
cfang@1335 | 4430 | |
cfang@1335 | 4431 | copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks()); |
cfang@1335 | 4432 | |
cfang@1335 | 4433 | // Present the results of the slow call. |
cfang@1335 | 4434 | result_reg->init_req(_instance_path, control()); |
cfang@1335 | 4435 | result_val->init_req(_instance_path, alloc_obj); |
cfang@1335 | 4436 | result_i_o ->set_req(_instance_path, i_o()); |
cfang@1335 | 4437 | result_mem ->set_req(_instance_path, reset_memory()); |
duke@435 | 4438 | } |
duke@435 | 4439 | |
cfang@1335 | 4440 | // Generate code for the slow case. We make a call to clone(). |
cfang@1335 | 4441 | set_control(_gvn.transform(slow_region)); |
cfang@1335 | 4442 | if (!stopped()) { |
cfang@1335 | 4443 | PreserveJVMState pjvms(this); |
cfang@1335 | 4444 | CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual); |
cfang@1335 | 4445 | Node* slow_result = set_results_for_java_call(slow_call); |
cfang@1335 | 4446 | // this->control() comes from set_results_for_java_call |
cfang@1335 | 4447 | result_reg->init_req(_slow_path, control()); |
cfang@1335 | 4448 | result_val->init_req(_slow_path, slow_result); |
cfang@1335 | 4449 | result_i_o ->set_req(_slow_path, i_o()); |
cfang@1335 | 4450 | result_mem ->set_req(_slow_path, reset_memory()); |
cfang@1335 | 4451 | } |
cfang@1337 | 4452 | |
cfang@1337 | 4453 | // Return the combined state. |
cfang@1337 | 4454 | set_control( _gvn.transform(result_reg) ); |
cfang@1337 | 4455 | set_i_o( _gvn.transform(result_i_o) ); |
cfang@1337 | 4456 | set_all_memory( _gvn.transform(result_mem) ); |
cfang@1335 | 4457 | } //original reexecute and sp are set back here |
duke@435 | 4458 | |
kvn@1268 | 4459 | push(_gvn.transform(result_val)); |
duke@435 | 4460 | |
duke@435 | 4461 | return true; |
duke@435 | 4462 | } |
duke@435 | 4463 | |
duke@435 | 4464 | //------------------------------basictype2arraycopy---------------------------- |
duke@435 | 4465 | address LibraryCallKit::basictype2arraycopy(BasicType t, |
duke@435 | 4466 | Node* src_offset, |
duke@435 | 4467 | Node* dest_offset, |
duke@435 | 4468 | bool disjoint_bases, |
iveresov@2606 | 4469 | const char* &name, |
iveresov@2606 | 4470 | bool dest_uninitialized) { |
duke@435 | 4471 | const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);; |
duke@435 | 4472 | const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);; |
duke@435 | 4473 | |
duke@435 | 4474 | bool aligned = false; |
duke@435 | 4475 | bool disjoint = disjoint_bases; |
duke@435 | 4476 | |
duke@435 | 4477 | // if the offsets are the same, we can treat the memory regions as |
duke@435 | 4478 | // disjoint, because either the memory regions are in different arrays, |
duke@435 | 4479 | // or they are identical (which we can treat as disjoint.) We can also |
duke@435 | 4480 | // treat a copy with a destination index less that the source index |
duke@435 | 4481 | // as disjoint since a low->high copy will work correctly in this case. |
duke@435 | 4482 | if (src_offset_inttype != NULL && src_offset_inttype->is_con() && |
duke@435 | 4483 | dest_offset_inttype != NULL && dest_offset_inttype->is_con()) { |
duke@435 | 4484 | // both indices are constants |
duke@435 | 4485 | int s_offs = src_offset_inttype->get_con(); |
duke@435 | 4486 | int d_offs = dest_offset_inttype->get_con(); |
kvn@464 | 4487 | int element_size = type2aelembytes(t); |
duke@435 | 4488 | aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) && |
duke@435 | 4489 | ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0); |
duke@435 | 4490 | if (s_offs >= d_offs) disjoint = true; |
duke@435 | 4491 | } else if (src_offset == dest_offset && src_offset != NULL) { |
duke@435 | 4492 | // This can occur if the offsets are identical non-constants. |
duke@435 | 4493 | disjoint = true; |
duke@435 | 4494 | } |
duke@435 | 4495 | |
roland@2728 | 4496 | return StubRoutines::select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized); |
duke@435 | 4497 | } |
duke@435 | 4498 | |
duke@435 | 4499 | |
duke@435 | 4500 | //------------------------------inline_arraycopy----------------------- |
duke@435 | 4501 | bool LibraryCallKit::inline_arraycopy() { |
duke@435 | 4502 | // Restore the stack and pop off the arguments. |
duke@435 | 4503 | int nargs = 5; // 2 oops, 3 ints, no size_t or long |
duke@435 | 4504 | assert(callee()->signature()->size() == nargs, "copy has 5 arguments"); |
duke@435 | 4505 | |
duke@435 | 4506 | Node *src = argument(0); |
duke@435 | 4507 | Node *src_offset = argument(1); |
duke@435 | 4508 | Node *dest = argument(2); |
duke@435 | 4509 | Node *dest_offset = argument(3); |
duke@435 | 4510 | Node *length = argument(4); |
duke@435 | 4511 | |
duke@435 | 4512 | // Compile time checks. If any of these checks cannot be verified at compile time, |
duke@435 | 4513 | // we do not make a fast path for this call. Instead, we let the call remain as it |
duke@435 | 4514 | // is. The checks we choose to mandate at compile time are: |
duke@435 | 4515 | // |
duke@435 | 4516 | // (1) src and dest are arrays. |
duke@435 | 4517 | const Type* src_type = src->Value(&_gvn); |
duke@435 | 4518 | const Type* dest_type = dest->Value(&_gvn); |
duke@435 | 4519 | const TypeAryPtr* top_src = src_type->isa_aryptr(); |
duke@435 | 4520 | const TypeAryPtr* top_dest = dest_type->isa_aryptr(); |
duke@435 | 4521 | if (top_src == NULL || top_src->klass() == NULL || |
duke@435 | 4522 | top_dest == NULL || top_dest->klass() == NULL) { |
duke@435 | 4523 | // Conservatively insert a memory barrier on all memory slices. |
duke@435 | 4524 | // Do not let writes into the source float below the arraycopy. |
duke@435 | 4525 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4526 | |
duke@435 | 4527 | // Call StubRoutines::generic_arraycopy stub. |
duke@435 | 4528 | generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, |
kvn@1268 | 4529 | src, src_offset, dest, dest_offset, length); |
duke@435 | 4530 | |
duke@435 | 4531 | // Do not let reads from the destination float above the arraycopy. |
duke@435 | 4532 | // Since we cannot type the arrays, we don't know which slices |
duke@435 | 4533 | // might be affected. We could restrict this barrier only to those |
duke@435 | 4534 | // memory slices which pertain to array elements--but don't bother. |
duke@435 | 4535 | if (!InsertMemBarAfterArraycopy) |
duke@435 | 4536 | // (If InsertMemBarAfterArraycopy, there is already one in place.) |
duke@435 | 4537 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 4538 | return true; |
duke@435 | 4539 | } |
duke@435 | 4540 | |
duke@435 | 4541 | // (2) src and dest arrays must have elements of the same BasicType |
duke@435 | 4542 | // Figure out the size and type of the elements we will be copying. |
duke@435 | 4543 | BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); |
duke@435 | 4544 | BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type(); |
duke@435 | 4545 | if (src_elem == T_ARRAY) src_elem = T_OBJECT; |
duke@435 | 4546 | if (dest_elem == T_ARRAY) dest_elem = T_OBJECT; |
duke@435 | 4547 | |
duke@435 | 4548 | if (src_elem != dest_elem || dest_elem == T_VOID) { |
duke@435 | 4549 | // The component types are not the same or are not recognized. Punt. |
duke@435 | 4550 | // (But, avoid the native method wrapper to JVM_ArrayCopy.) |
duke@435 | 4551 | generate_slow_arraycopy(TypePtr::BOTTOM, |
iveresov@2606 | 4552 | src, src_offset, dest, dest_offset, length, |
iveresov@2606 | 4553 | /*dest_uninitialized*/false); |
duke@435 | 4554 | return true; |
duke@435 | 4555 | } |
duke@435 | 4556 | |
duke@435 | 4557 | //--------------------------------------------------------------------------- |
duke@435 | 4558 | // We will make a fast path for this call to arraycopy. |
duke@435 | 4559 | |
duke@435 | 4560 | // We have the following tests left to perform: |
duke@435 | 4561 | // |
duke@435 | 4562 | // (3) src and dest must not be null. |
duke@435 | 4563 | // (4) src_offset must not be negative. |
duke@435 | 4564 | // (5) dest_offset must not be negative. |
duke@435 | 4565 | // (6) length must not be negative. |
duke@435 | 4566 | // (7) src_offset + length must not exceed length of src. |
duke@435 | 4567 | // (8) dest_offset + length must not exceed length of dest. |
duke@435 | 4568 | // (9) each element of an oop array must be assignable |
duke@435 | 4569 | |
duke@435 | 4570 | RegionNode* slow_region = new (C, 1) RegionNode(1); |
duke@435 | 4571 | record_for_igvn(slow_region); |
duke@435 | 4572 | |
duke@435 | 4573 | // (3) operands must not be null |
duke@435 | 4574 | // We currently perform our null checks with the do_null_check routine. |
duke@435 | 4575 | // This means that the null exceptions will be reported in the caller |
duke@435 | 4576 | // rather than (correctly) reported inside of the native arraycopy call. |
duke@435 | 4577 | // This should be corrected, given time. We do our null check with the |
duke@435 | 4578 | // stack pointer restored. |
duke@435 | 4579 | _sp += nargs; |
duke@435 | 4580 | src = do_null_check(src, T_ARRAY); |
duke@435 | 4581 | dest = do_null_check(dest, T_ARRAY); |
duke@435 | 4582 | _sp -= nargs; |
duke@435 | 4583 | |
duke@435 | 4584 | // (4) src_offset must not be negative. |
duke@435 | 4585 | generate_negative_guard(src_offset, slow_region); |
duke@435 | 4586 | |
duke@435 | 4587 | // (5) dest_offset must not be negative. |
duke@435 | 4588 | generate_negative_guard(dest_offset, slow_region); |
duke@435 | 4589 | |
duke@435 | 4590 | // (6) length must not be negative (moved to generate_arraycopy()). |
duke@435 | 4591 | // generate_negative_guard(length, slow_region); |
duke@435 | 4592 | |
duke@435 | 4593 | // (7) src_offset + length must not exceed length of src. |
duke@435 | 4594 | generate_limit_guard(src_offset, length, |
duke@435 | 4595 | load_array_length(src), |
duke@435 | 4596 | slow_region); |
duke@435 | 4597 | |
duke@435 | 4598 | // (8) dest_offset + length must not exceed length of dest. |
duke@435 | 4599 | generate_limit_guard(dest_offset, length, |
duke@435 | 4600 | load_array_length(dest), |
duke@435 | 4601 | slow_region); |
duke@435 | 4602 | |
duke@435 | 4603 | // (9) each element of an oop array must be assignable |
duke@435 | 4604 | // The generate_arraycopy subroutine checks this. |
duke@435 | 4605 | |
duke@435 | 4606 | // This is where the memory effects are placed: |
duke@435 | 4607 | const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); |
duke@435 | 4608 | generate_arraycopy(adr_type, dest_elem, |
duke@435 | 4609 | src, src_offset, dest, dest_offset, length, |
kvn@1268 | 4610 | false, false, slow_region); |
duke@435 | 4611 | |
duke@435 | 4612 | return true; |
duke@435 | 4613 | } |
duke@435 | 4614 | |
duke@435 | 4615 | //-----------------------------generate_arraycopy---------------------- |
duke@435 | 4616 | // Generate an optimized call to arraycopy. |
duke@435 | 4617 | // Caller must guard against non-arrays. |
duke@435 | 4618 | // Caller must determine a common array basic-type for both arrays. |
duke@435 | 4619 | // Caller must validate offsets against array bounds. |
duke@435 | 4620 | // The slow_region has already collected guard failure paths |
duke@435 | 4621 | // (such as out of bounds length or non-conformable array types). |
duke@435 | 4622 | // The generated code has this shape, in general: |
duke@435 | 4623 | // |
duke@435 | 4624 | // if (length == 0) return // via zero_path |
duke@435 | 4625 | // slowval = -1 |
duke@435 | 4626 | // if (types unknown) { |
duke@435 | 4627 | // slowval = call generic copy loop |
duke@435 | 4628 | // if (slowval == 0) return // via checked_path |
duke@435 | 4629 | // } else if (indexes in bounds) { |
duke@435 | 4630 | // if ((is object array) && !(array type check)) { |
duke@435 | 4631 | // slowval = call checked copy loop |
duke@435 | 4632 | // if (slowval == 0) return // via checked_path |
duke@435 | 4633 | // } else { |
duke@435 | 4634 | // call bulk copy loop |
duke@435 | 4635 | // return // via fast_path |
duke@435 | 4636 | // } |
duke@435 | 4637 | // } |
duke@435 | 4638 | // // adjust params for remaining work: |
duke@435 | 4639 | // if (slowval != -1) { |
duke@435 | 4640 | // n = -1^slowval; src_offset += n; dest_offset += n; length -= n |
duke@435 | 4641 | // } |
duke@435 | 4642 | // slow_region: |
duke@435 | 4643 | // call slow arraycopy(src, src_offset, dest, dest_offset, length) |
duke@435 | 4644 | // return // via slow_call_path |
duke@435 | 4645 | // |
duke@435 | 4646 | // This routine is used from several intrinsics: System.arraycopy, |
duke@435 | 4647 | // Object.clone (the array subcase), and Arrays.copyOf[Range]. |
duke@435 | 4648 | // |
duke@435 | 4649 | void |
duke@435 | 4650 | LibraryCallKit::generate_arraycopy(const TypePtr* adr_type, |
duke@435 | 4651 | BasicType basic_elem_type, |
duke@435 | 4652 | Node* src, Node* src_offset, |
duke@435 | 4653 | Node* dest, Node* dest_offset, |
duke@435 | 4654 | Node* copy_length, |
duke@435 | 4655 | bool disjoint_bases, |
duke@435 | 4656 | bool length_never_negative, |
duke@435 | 4657 | RegionNode* slow_region) { |
duke@435 | 4658 | |
duke@435 | 4659 | if (slow_region == NULL) { |
duke@435 | 4660 | slow_region = new(C,1) RegionNode(1); |
duke@435 | 4661 | record_for_igvn(slow_region); |
duke@435 | 4662 | } |
duke@435 | 4663 | |
duke@435 | 4664 | Node* original_dest = dest; |
duke@435 | 4665 | AllocateArrayNode* alloc = NULL; // used for zeroing, if needed |
iveresov@2606 | 4666 | bool dest_uninitialized = false; |
duke@435 | 4667 | |
duke@435 | 4668 | // See if this is the initialization of a newly-allocated array. |
duke@435 | 4669 | // If so, we will take responsibility here for initializing it to zero. |
duke@435 | 4670 | // (Note: Because tightly_coupled_allocation performs checks on the |
duke@435 | 4671 | // out-edges of the dest, we need to avoid making derived pointers |
duke@435 | 4672 | // from it until we have checked its uses.) |
duke@435 | 4673 | if (ReduceBulkZeroing |
duke@435 | 4674 | && !ZeroTLAB // pointless if already zeroed |
duke@435 | 4675 | && basic_elem_type != T_CONFLICT // avoid corner case |
kvn@3407 | 4676 | && !src->eqv_uncast(dest) |
duke@435 | 4677 | && ((alloc = tightly_coupled_allocation(dest, slow_region)) |
duke@435 | 4678 | != NULL) |
kvn@469 | 4679 | && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0 |
duke@435 | 4680 | && alloc->maybe_set_complete(&_gvn)) { |
duke@435 | 4681 | // "You break it, you buy it." |
duke@435 | 4682 | InitializeNode* init = alloc->initialization(); |
duke@435 | 4683 | assert(init->is_complete(), "we just did this"); |
kvn@3157 | 4684 | init->set_complete_with_arraycopy(); |
kvn@1268 | 4685 | assert(dest->is_CheckCastPP(), "sanity"); |
duke@435 | 4686 | assert(dest->in(0)->in(0) == init, "dest pinned"); |
duke@435 | 4687 | adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory |
duke@435 | 4688 | // From this point on, every exit path is responsible for |
duke@435 | 4689 | // initializing any non-copied parts of the object to zero. |
iveresov@2606 | 4690 | // Also, if this flag is set we make sure that arraycopy interacts properly |
iveresov@2606 | 4691 | // with G1, eliding pre-barriers. See CR 6627983. |
iveresov@2606 | 4692 | dest_uninitialized = true; |
duke@435 | 4693 | } else { |
duke@435 | 4694 | // No zeroing elimination here. |
duke@435 | 4695 | alloc = NULL; |
duke@435 | 4696 | //original_dest = dest; |
iveresov@2606 | 4697 | //dest_uninitialized = false; |
duke@435 | 4698 | } |
duke@435 | 4699 | |
duke@435 | 4700 | // Results are placed here: |
duke@435 | 4701 | enum { fast_path = 1, // normal void-returning assembly stub |
duke@435 | 4702 | checked_path = 2, // special assembly stub with cleanup |
duke@435 | 4703 | slow_call_path = 3, // something went wrong; call the VM |
duke@435 | 4704 | zero_path = 4, // bypass when length of copy is zero |
duke@435 | 4705 | bcopy_path = 5, // copy primitive array by 64-bit blocks |
duke@435 | 4706 | PATH_LIMIT = 6 |
duke@435 | 4707 | }; |
duke@435 | 4708 | RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); |
duke@435 | 4709 | PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO); |
duke@435 | 4710 | PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type); |
duke@435 | 4711 | record_for_igvn(result_region); |
duke@435 | 4712 | _gvn.set_type_bottom(result_i_o); |
duke@435 | 4713 | _gvn.set_type_bottom(result_memory); |
duke@435 | 4714 | assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice"); |
duke@435 | 4715 | |
duke@435 | 4716 | // The slow_control path: |
duke@435 | 4717 | Node* slow_control; |
duke@435 | 4718 | Node* slow_i_o = i_o(); |
duke@435 | 4719 | Node* slow_mem = memory(adr_type); |
duke@435 | 4720 | debug_only(slow_control = (Node*) badAddress); |
duke@435 | 4721 | |
duke@435 | 4722 | // Checked control path: |
duke@435 | 4723 | Node* checked_control = top(); |
duke@435 | 4724 | Node* checked_mem = NULL; |
duke@435 | 4725 | Node* checked_i_o = NULL; |
duke@435 | 4726 | Node* checked_value = NULL; |
duke@435 | 4727 | |
duke@435 | 4728 | if (basic_elem_type == T_CONFLICT) { |
iveresov@2606 | 4729 | assert(!dest_uninitialized, ""); |
duke@435 | 4730 | Node* cv = generate_generic_arraycopy(adr_type, |
duke@435 | 4731 | src, src_offset, dest, dest_offset, |
iveresov@2606 | 4732 | copy_length, dest_uninitialized); |
duke@435 | 4733 | if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
duke@435 | 4734 | checked_control = control(); |
duke@435 | 4735 | checked_i_o = i_o(); |
duke@435 | 4736 | checked_mem = memory(adr_type); |
duke@435 | 4737 | checked_value = cv; |
duke@435 | 4738 | set_control(top()); // no fast path |
duke@435 | 4739 | } |
duke@435 | 4740 | |
duke@435 | 4741 | Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative); |
duke@435 | 4742 | if (not_pos != NULL) { |
duke@435 | 4743 | PreserveJVMState pjvms(this); |
duke@435 | 4744 | set_control(not_pos); |
duke@435 | 4745 | |
duke@435 | 4746 | // (6) length must not be negative. |
duke@435 | 4747 | if (!length_never_negative) { |
duke@435 | 4748 | generate_negative_guard(copy_length, slow_region); |
duke@435 | 4749 | } |
duke@435 | 4750 | |
kvn@1271 | 4751 | // copy_length is 0. |
iveresov@2606 | 4752 | if (!stopped() && dest_uninitialized) { |
duke@435 | 4753 | Node* dest_length = alloc->in(AllocateNode::ALength); |
kvn@3407 | 4754 | if (copy_length->eqv_uncast(dest_length) |
duke@435 | 4755 | || _gvn.find_int_con(dest_length, 1) <= 0) { |
kvn@1271 | 4756 | // There is no zeroing to do. No need for a secondary raw memory barrier. |
duke@435 | 4757 | } else { |
duke@435 | 4758 | // Clear the whole thing since there are no source elements to copy. |
duke@435 | 4759 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4760 | intcon(0), NULL, |
duke@435 | 4761 | alloc->in(AllocateNode::AllocSize)); |
kvn@1271 | 4762 | // Use a secondary InitializeNode as raw memory barrier. |
kvn@1271 | 4763 | // Currently it is needed only on this path since other |
kvn@1271 | 4764 | // paths have stub or runtime calls as raw memory barriers. |
kvn@1271 | 4765 | InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, |
kvn@1271 | 4766 | Compile::AliasIdxRaw, |
kvn@1271 | 4767 | top())->as_Initialize(); |
kvn@1271 | 4768 | init->set_complete(&_gvn); // (there is no corresponding AllocateNode) |
duke@435 | 4769 | } |
duke@435 | 4770 | } |
duke@435 | 4771 | |
duke@435 | 4772 | // Present the results of the fast call. |
duke@435 | 4773 | result_region->init_req(zero_path, control()); |
duke@435 | 4774 | result_i_o ->init_req(zero_path, i_o()); |
duke@435 | 4775 | result_memory->init_req(zero_path, memory(adr_type)); |
duke@435 | 4776 | } |
duke@435 | 4777 | |
iveresov@2606 | 4778 | if (!stopped() && dest_uninitialized) { |
duke@435 | 4779 | // We have to initialize the *uncopied* part of the array to zero. |
duke@435 | 4780 | // The copy destination is the slice dest[off..off+len]. The other slices |
duke@435 | 4781 | // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. |
duke@435 | 4782 | Node* dest_size = alloc->in(AllocateNode::AllocSize); |
duke@435 | 4783 | Node* dest_length = alloc->in(AllocateNode::ALength); |
duke@435 | 4784 | Node* dest_tail = _gvn.transform( new(C,3) AddINode(dest_offset, |
duke@435 | 4785 | copy_length) ); |
duke@435 | 4786 | |
duke@435 | 4787 | // If there is a head section that needs zeroing, do it now. |
duke@435 | 4788 | if (find_int_con(dest_offset, -1) != 0) { |
duke@435 | 4789 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4790 | intcon(0), dest_offset, |
duke@435 | 4791 | NULL); |
duke@435 | 4792 | } |
duke@435 | 4793 | |
duke@435 | 4794 | // Next, perform a dynamic check on the tail length. |
duke@435 | 4795 | // It is often zero, and we can win big if we prove this. |
duke@435 | 4796 | // There are two wins: Avoid generating the ClearArray |
duke@435 | 4797 | // with its attendant messy index arithmetic, and upgrade |
duke@435 | 4798 | // the copy to a more hardware-friendly word size of 64 bits. |
duke@435 | 4799 | Node* tail_ctl = NULL; |
kvn@3407 | 4800 | if (!stopped() && !dest_tail->eqv_uncast(dest_length)) { |
duke@435 | 4801 | Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) ); |
duke@435 | 4802 | Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) ); |
duke@435 | 4803 | tail_ctl = generate_slow_guard(bol_lt, NULL); |
duke@435 | 4804 | assert(tail_ctl != NULL || !stopped(), "must be an outcome"); |
duke@435 | 4805 | } |
duke@435 | 4806 | |
duke@435 | 4807 | // At this point, let's assume there is no tail. |
duke@435 | 4808 | if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) { |
duke@435 | 4809 | // There is no tail. Try an upgrade to a 64-bit copy. |
duke@435 | 4810 | bool didit = false; |
duke@435 | 4811 | { PreserveJVMState pjvms(this); |
duke@435 | 4812 | didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc, |
duke@435 | 4813 | src, src_offset, dest, dest_offset, |
iveresov@2606 | 4814 | dest_size, dest_uninitialized); |
duke@435 | 4815 | if (didit) { |
duke@435 | 4816 | // Present the results of the block-copying fast call. |
duke@435 | 4817 | result_region->init_req(bcopy_path, control()); |
duke@435 | 4818 | result_i_o ->init_req(bcopy_path, i_o()); |
duke@435 | 4819 | result_memory->init_req(bcopy_path, memory(adr_type)); |
duke@435 | 4820 | } |
duke@435 | 4821 | } |
duke@435 | 4822 | if (didit) |
duke@435 | 4823 | set_control(top()); // no regular fast path |
duke@435 | 4824 | } |
duke@435 | 4825 | |
duke@435 | 4826 | // Clear the tail, if any. |
duke@435 | 4827 | if (tail_ctl != NULL) { |
duke@435 | 4828 | Node* notail_ctl = stopped() ? NULL : control(); |
duke@435 | 4829 | set_control(tail_ctl); |
duke@435 | 4830 | if (notail_ctl == NULL) { |
duke@435 | 4831 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4832 | dest_tail, NULL, |
duke@435 | 4833 | dest_size); |
duke@435 | 4834 | } else { |
duke@435 | 4835 | // Make a local merge. |
duke@435 | 4836 | Node* done_ctl = new(C,3) RegionNode(3); |
duke@435 | 4837 | Node* done_mem = new(C,3) PhiNode(done_ctl, Type::MEMORY, adr_type); |
duke@435 | 4838 | done_ctl->init_req(1, notail_ctl); |
duke@435 | 4839 | done_mem->init_req(1, memory(adr_type)); |
duke@435 | 4840 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4841 | dest_tail, NULL, |
duke@435 | 4842 | dest_size); |
duke@435 | 4843 | done_ctl->init_req(2, control()); |
duke@435 | 4844 | done_mem->init_req(2, memory(adr_type)); |
duke@435 | 4845 | set_control( _gvn.transform(done_ctl) ); |
duke@435 | 4846 | set_memory( _gvn.transform(done_mem), adr_type ); |
duke@435 | 4847 | } |
duke@435 | 4848 | } |
duke@435 | 4849 | } |
duke@435 | 4850 | |
duke@435 | 4851 | BasicType copy_type = basic_elem_type; |
duke@435 | 4852 | assert(basic_elem_type != T_ARRAY, "caller must fix this"); |
duke@435 | 4853 | if (!stopped() && copy_type == T_OBJECT) { |
duke@435 | 4854 | // If src and dest have compatible element types, we can copy bits. |
duke@435 | 4855 | // Types S[] and D[] are compatible if D is a supertype of S. |
duke@435 | 4856 | // |
duke@435 | 4857 | // If they are not, we will use checked_oop_disjoint_arraycopy, |
duke@435 | 4858 | // which performs a fast optimistic per-oop check, and backs off |
duke@435 | 4859 | // further to JVM_ArrayCopy on the first per-oop check that fails. |
duke@435 | 4860 | // (Actually, we don't move raw bits only; the GC requires card marks.) |
duke@435 | 4861 | |
duke@435 | 4862 | // Get the klassOop for both src and dest |
duke@435 | 4863 | Node* src_klass = load_object_klass(src); |
duke@435 | 4864 | Node* dest_klass = load_object_klass(dest); |
duke@435 | 4865 | |
duke@435 | 4866 | // Generate the subtype check. |
duke@435 | 4867 | // This might fold up statically, or then again it might not. |
duke@435 | 4868 | // |
duke@435 | 4869 | // Non-static example: Copying List<String>.elements to a new String[]. |
duke@435 | 4870 | // The backing store for a List<String> is always an Object[], |
duke@435 | 4871 | // but its elements are always type String, if the generic types |
duke@435 | 4872 | // are correct at the source level. |
duke@435 | 4873 | // |
duke@435 | 4874 | // Test S[] against D[], not S against D, because (probably) |
duke@435 | 4875 | // the secondary supertype cache is less busy for S[] than S. |
duke@435 | 4876 | // This usually only matters when D is an interface. |
duke@435 | 4877 | Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass); |
duke@435 | 4878 | // Plug failing path into checked_oop_disjoint_arraycopy |
duke@435 | 4879 | if (not_subtype_ctrl != top()) { |
duke@435 | 4880 | PreserveJVMState pjvms(this); |
duke@435 | 4881 | set_control(not_subtype_ctrl); |
duke@435 | 4882 | // (At this point we can assume disjoint_bases, since types differ.) |
stefank@3391 | 4883 | int ek_offset = in_bytes(objArrayKlass::element_klass_offset()); |
duke@435 | 4884 | Node* p1 = basic_plus_adr(dest_klass, ek_offset); |
kvn@599 | 4885 | Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM); |
duke@435 | 4886 | Node* dest_elem_klass = _gvn.transform(n1); |
duke@435 | 4887 | Node* cv = generate_checkcast_arraycopy(adr_type, |
duke@435 | 4888 | dest_elem_klass, |
duke@435 | 4889 | src, src_offset, dest, dest_offset, |
iveresov@2606 | 4890 | ConvI2X(copy_length), dest_uninitialized); |
duke@435 | 4891 | if (cv == NULL) cv = intcon(-1); // failure (no stub available) |
duke@435 | 4892 | checked_control = control(); |
duke@435 | 4893 | checked_i_o = i_o(); |
duke@435 | 4894 | checked_mem = memory(adr_type); |
duke@435 | 4895 | checked_value = cv; |
duke@435 | 4896 | } |
duke@435 | 4897 | // At this point we know we do not need type checks on oop stores. |
duke@435 | 4898 | |
duke@435 | 4899 | // Let's see if we need card marks: |
duke@435 | 4900 | if (alloc != NULL && use_ReduceInitialCardMarks()) { |
duke@435 | 4901 | // If we do not need card marks, copy using the jint or jlong stub. |
coleenp@548 | 4902 | copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); |
kvn@464 | 4903 | assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), |
duke@435 | 4904 | "sizes agree"); |
duke@435 | 4905 | } |
duke@435 | 4906 | } |
duke@435 | 4907 | |
duke@435 | 4908 | if (!stopped()) { |
duke@435 | 4909 | // Generate the fast path, if possible. |
duke@435 | 4910 | PreserveJVMState pjvms(this); |
duke@435 | 4911 | generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases, |
duke@435 | 4912 | src, src_offset, dest, dest_offset, |
iveresov@2606 | 4913 | ConvI2X(copy_length), dest_uninitialized); |
duke@435 | 4914 | |
duke@435 | 4915 | // Present the results of the fast call. |
duke@435 | 4916 | result_region->init_req(fast_path, control()); |
duke@435 | 4917 | result_i_o ->init_req(fast_path, i_o()); |
duke@435 | 4918 | result_memory->init_req(fast_path, memory(adr_type)); |
duke@435 | 4919 | } |
duke@435 | 4920 | |
duke@435 | 4921 | // Here are all the slow paths up to this point, in one bundle: |
duke@435 | 4922 | slow_control = top(); |
duke@435 | 4923 | if (slow_region != NULL) |
duke@435 | 4924 | slow_control = _gvn.transform(slow_region); |
duke@435 | 4925 | debug_only(slow_region = (RegionNode*)badAddress); |
duke@435 | 4926 | |
duke@435 | 4927 | set_control(checked_control); |
duke@435 | 4928 | if (!stopped()) { |
duke@435 | 4929 | // Clean up after the checked call. |
duke@435 | 4930 | // The returned value is either 0 or -1^K, |
duke@435 | 4931 | // where K = number of partially transferred array elements. |
duke@435 | 4932 | Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) ); |
duke@435 | 4933 | Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) ); |
duke@435 | 4934 | IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN); |
duke@435 | 4935 | |
duke@435 | 4936 | // If it is 0, we are done, so transfer to the end. |
duke@435 | 4937 | Node* checks_done = _gvn.transform( new(C, 1) IfTrueNode(iff) ); |
duke@435 | 4938 | result_region->init_req(checked_path, checks_done); |
duke@435 | 4939 | result_i_o ->init_req(checked_path, checked_i_o); |
duke@435 | 4940 | result_memory->init_req(checked_path, checked_mem); |
duke@435 | 4941 | |
duke@435 | 4942 | // If it is not zero, merge into the slow call. |
duke@435 | 4943 | set_control( _gvn.transform( new(C, 1) IfFalseNode(iff) )); |
duke@435 | 4944 | RegionNode* slow_reg2 = new(C, 3) RegionNode(3); |
duke@435 | 4945 | PhiNode* slow_i_o2 = new(C, 3) PhiNode(slow_reg2, Type::ABIO); |
duke@435 | 4946 | PhiNode* slow_mem2 = new(C, 3) PhiNode(slow_reg2, Type::MEMORY, adr_type); |
duke@435 | 4947 | record_for_igvn(slow_reg2); |
duke@435 | 4948 | slow_reg2 ->init_req(1, slow_control); |
duke@435 | 4949 | slow_i_o2 ->init_req(1, slow_i_o); |
duke@435 | 4950 | slow_mem2 ->init_req(1, slow_mem); |
duke@435 | 4951 | slow_reg2 ->init_req(2, control()); |
kvn@1268 | 4952 | slow_i_o2 ->init_req(2, checked_i_o); |
kvn@1268 | 4953 | slow_mem2 ->init_req(2, checked_mem); |
duke@435 | 4954 | |
duke@435 | 4955 | slow_control = _gvn.transform(slow_reg2); |
duke@435 | 4956 | slow_i_o = _gvn.transform(slow_i_o2); |
duke@435 | 4957 | slow_mem = _gvn.transform(slow_mem2); |
duke@435 | 4958 | |
duke@435 | 4959 | if (alloc != NULL) { |
duke@435 | 4960 | // We'll restart from the very beginning, after zeroing the whole thing. |
duke@435 | 4961 | // This can cause double writes, but that's OK since dest is brand new. |
duke@435 | 4962 | // So we ignore the low 31 bits of the value returned from the stub. |
duke@435 | 4963 | } else { |
duke@435 | 4964 | // We must continue the copy exactly where it failed, or else |
duke@435 | 4965 | // another thread might see the wrong number of writes to dest. |
duke@435 | 4966 | Node* checked_offset = _gvn.transform( new(C, 3) XorINode(checked_value, intcon(-1)) ); |
duke@435 | 4967 | Node* slow_offset = new(C, 3) PhiNode(slow_reg2, TypeInt::INT); |
duke@435 | 4968 | slow_offset->init_req(1, intcon(0)); |
duke@435 | 4969 | slow_offset->init_req(2, checked_offset); |
duke@435 | 4970 | slow_offset = _gvn.transform(slow_offset); |
duke@435 | 4971 | |
duke@435 | 4972 | // Adjust the arguments by the conditionally incoming offset. |
duke@435 | 4973 | Node* src_off_plus = _gvn.transform( new(C, 3) AddINode(src_offset, slow_offset) ); |
duke@435 | 4974 | Node* dest_off_plus = _gvn.transform( new(C, 3) AddINode(dest_offset, slow_offset) ); |
duke@435 | 4975 | Node* length_minus = _gvn.transform( new(C, 3) SubINode(copy_length, slow_offset) ); |
duke@435 | 4976 | |
duke@435 | 4977 | // Tweak the node variables to adjust the code produced below: |
duke@435 | 4978 | src_offset = src_off_plus; |
duke@435 | 4979 | dest_offset = dest_off_plus; |
duke@435 | 4980 | copy_length = length_minus; |
duke@435 | 4981 | } |
duke@435 | 4982 | } |
duke@435 | 4983 | |
duke@435 | 4984 | set_control(slow_control); |
duke@435 | 4985 | if (!stopped()) { |
duke@435 | 4986 | // Generate the slow path, if needed. |
duke@435 | 4987 | PreserveJVMState pjvms(this); // replace_in_map may trash the map |
duke@435 | 4988 | |
duke@435 | 4989 | set_memory(slow_mem, adr_type); |
duke@435 | 4990 | set_i_o(slow_i_o); |
duke@435 | 4991 | |
iveresov@2606 | 4992 | if (dest_uninitialized) { |
duke@435 | 4993 | generate_clear_array(adr_type, dest, basic_elem_type, |
duke@435 | 4994 | intcon(0), NULL, |
duke@435 | 4995 | alloc->in(AllocateNode::AllocSize)); |
duke@435 | 4996 | } |
duke@435 | 4997 | |
duke@435 | 4998 | generate_slow_arraycopy(adr_type, |
duke@435 | 4999 | src, src_offset, dest, dest_offset, |
iveresov@2606 | 5000 | copy_length, /*dest_uninitialized*/false); |
duke@435 | 5001 | |
duke@435 | 5002 | result_region->init_req(slow_call_path, control()); |
duke@435 | 5003 | result_i_o ->init_req(slow_call_path, i_o()); |
duke@435 | 5004 | result_memory->init_req(slow_call_path, memory(adr_type)); |
duke@435 | 5005 | } |
duke@435 | 5006 | |
duke@435 | 5007 | // Remove unused edges. |
duke@435 | 5008 | for (uint i = 1; i < result_region->req(); i++) { |
duke@435 | 5009 | if (result_region->in(i) == NULL) |
duke@435 | 5010 | result_region->init_req(i, top()); |
duke@435 | 5011 | } |
duke@435 | 5012 | |
duke@435 | 5013 | // Finished; return the combined state. |
duke@435 | 5014 | set_control( _gvn.transform(result_region) ); |
duke@435 | 5015 | set_i_o( _gvn.transform(result_i_o) ); |
duke@435 | 5016 | set_memory( _gvn.transform(result_memory), adr_type ); |
duke@435 | 5017 | |
duke@435 | 5018 | // The memory edges above are precise in order to model effects around |
twisti@1040 | 5019 | // array copies accurately to allow value numbering of field loads around |
duke@435 | 5020 | // arraycopy. Such field loads, both before and after, are common in Java |
duke@435 | 5021 | // collections and similar classes involving header/array data structures. |
duke@435 | 5022 | // |
duke@435 | 5023 | // But with low number of register or when some registers are used or killed |
duke@435 | 5024 | // by arraycopy calls it causes registers spilling on stack. See 6544710. |
duke@435 | 5025 | // The next memory barrier is added to avoid it. If the arraycopy can be |
duke@435 | 5026 | // optimized away (which it can, sometimes) then we can manually remove |
duke@435 | 5027 | // the membar also. |
kvn@1393 | 5028 | // |
kvn@1393 | 5029 | // Do not let reads from the cloned object float above the arraycopy. |
roland@3392 | 5030 | if (alloc != NULL) { |
roland@3392 | 5031 | // Do not let stores that initialize this object be reordered with |
roland@3392 | 5032 | // a subsequent store that would make this object accessible by |
roland@3392 | 5033 | // other threads. |
roland@3392 | 5034 | // Record what AllocateNode this StoreStore protects so that |
roland@3392 | 5035 | // escape analysis can go from the MemBarStoreStoreNode to the |
roland@3392 | 5036 | // AllocateNode and eliminate the MemBarStoreStoreNode if possible |
roland@3392 | 5037 | // based on the escape status of the AllocateNode. |
roland@3392 | 5038 | insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out(AllocateNode::RawAddress)); |
roland@3392 | 5039 | } else if (InsertMemBarAfterArraycopy) |
duke@435 | 5040 | insert_mem_bar(Op_MemBarCPUOrder); |
duke@435 | 5041 | } |
duke@435 | 5042 | |
duke@435 | 5043 | |
duke@435 | 5044 | // Helper function which determines if an arraycopy immediately follows |
duke@435 | 5045 | // an allocation, with no intervening tests or other escapes for the object. |
duke@435 | 5046 | AllocateArrayNode* |
duke@435 | 5047 | LibraryCallKit::tightly_coupled_allocation(Node* ptr, |
duke@435 | 5048 | RegionNode* slow_region) { |
duke@435 | 5049 | if (stopped()) return NULL; // no fast path |
duke@435 | 5050 | if (C->AliasLevel() == 0) return NULL; // no MergeMems around |
duke@435 | 5051 | |
duke@435 | 5052 | AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); |
duke@435 | 5053 | if (alloc == NULL) return NULL; |
duke@435 | 5054 | |
duke@435 | 5055 | Node* rawmem = memory(Compile::AliasIdxRaw); |
duke@435 | 5056 | // Is the allocation's memory state untouched? |
duke@435 | 5057 | if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) { |
duke@435 | 5058 | // Bail out if there have been raw-memory effects since the allocation. |
duke@435 | 5059 | // (Example: There might have been a call or safepoint.) |
duke@435 | 5060 | return NULL; |
duke@435 | 5061 | } |
duke@435 | 5062 | rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw); |
duke@435 | 5063 | if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) { |
duke@435 | 5064 | return NULL; |
duke@435 | 5065 | } |
duke@435 | 5066 | |
duke@435 | 5067 | // There must be no unexpected observers of this allocation. |
duke@435 | 5068 | for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) { |
duke@435 | 5069 | Node* obs = ptr->fast_out(i); |
duke@435 | 5070 | if (obs != this->map()) { |
duke@435 | 5071 | return NULL; |
duke@435 | 5072 | } |
duke@435 | 5073 | } |
duke@435 | 5074 | |
duke@435 | 5075 | // This arraycopy must unconditionally follow the allocation of the ptr. |
duke@435 | 5076 | Node* alloc_ctl = ptr->in(0); |
duke@435 | 5077 | assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo"); |
duke@435 | 5078 | |
duke@435 | 5079 | Node* ctl = control(); |
duke@435 | 5080 | while (ctl != alloc_ctl) { |
duke@435 | 5081 | // There may be guards which feed into the slow_region. |
duke@435 | 5082 | // Any other control flow means that we might not get a chance |
duke@435 | 5083 | // to finish initializing the allocated object. |
duke@435 | 5084 | if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) { |
duke@435 | 5085 | IfNode* iff = ctl->in(0)->as_If(); |
duke@435 | 5086 | Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con); |
duke@435 | 5087 | assert(not_ctl != NULL && not_ctl != ctl, "found alternate"); |
duke@435 | 5088 | if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) { |
duke@435 | 5089 | ctl = iff->in(0); // This test feeds the known slow_region. |
duke@435 | 5090 | continue; |
duke@435 | 5091 | } |
duke@435 | 5092 | // One more try: Various low-level checks bottom out in |
duke@435 | 5093 | // uncommon traps. If the debug-info of the trap omits |
duke@435 | 5094 | // any reference to the allocation, as we've already |
duke@435 | 5095 | // observed, then there can be no objection to the trap. |
duke@435 | 5096 | bool found_trap = false; |
duke@435 | 5097 | for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) { |
duke@435 | 5098 | Node* obs = not_ctl->fast_out(j); |
duke@435 | 5099 | if (obs->in(0) == not_ctl && obs->is_Call() && |
twisti@2103 | 5100 | (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) { |
duke@435 | 5101 | found_trap = true; break; |
duke@435 | 5102 | } |
duke@435 | 5103 | } |
duke@435 | 5104 | if (found_trap) { |
duke@435 | 5105 | ctl = iff->in(0); // This test feeds a harmless uncommon trap. |
duke@435 | 5106 | continue; |
duke@435 | 5107 | } |
duke@435 | 5108 | } |
duke@435 | 5109 | return NULL; |
duke@435 | 5110 | } |
duke@435 | 5111 | |
duke@435 | 5112 | // If we get this far, we have an allocation which immediately |
duke@435 | 5113 | // precedes the arraycopy, and we can take over zeroing the new object. |
duke@435 | 5114 | // The arraycopy will finish the initialization, and provide |
duke@435 | 5115 | // a new control state to which we will anchor the destination pointer. |
duke@435 | 5116 | |
duke@435 | 5117 | return alloc; |
duke@435 | 5118 | } |
duke@435 | 5119 | |
duke@435 | 5120 | // Helper for initialization of arrays, creating a ClearArray. |
duke@435 | 5121 | // It writes zero bits in [start..end), within the body of an array object. |
duke@435 | 5122 | // The memory effects are all chained onto the 'adr_type' alias category. |
duke@435 | 5123 | // |
duke@435 | 5124 | // Since the object is otherwise uninitialized, we are free |
duke@435 | 5125 | // to put a little "slop" around the edges of the cleared area, |
duke@435 | 5126 | // as long as it does not go back into the array's header, |
duke@435 | 5127 | // or beyond the array end within the heap. |
duke@435 | 5128 | // |
duke@435 | 5129 | // The lower edge can be rounded down to the nearest jint and the |
duke@435 | 5130 | // upper edge can be rounded up to the nearest MinObjAlignmentInBytes. |
duke@435 | 5131 | // |
duke@435 | 5132 | // Arguments: |
duke@435 | 5133 | // adr_type memory slice where writes are generated |
duke@435 | 5134 | // dest oop of the destination array |
duke@435 | 5135 | // basic_elem_type element type of the destination |
duke@435 | 5136 | // slice_idx array index of first element to store |
duke@435 | 5137 | // slice_len number of elements to store (or NULL) |
duke@435 | 5138 | // dest_size total size in bytes of the array object |
duke@435 | 5139 | // |
duke@435 | 5140 | // Exactly one of slice_len or dest_size must be non-NULL. |
duke@435 | 5141 | // If dest_size is non-NULL, zeroing extends to the end of the object. |
duke@435 | 5142 | // If slice_len is non-NULL, the slice_idx value must be a constant. |
duke@435 | 5143 | void |
duke@435 | 5144 | LibraryCallKit::generate_clear_array(const TypePtr* adr_type, |
duke@435 | 5145 | Node* dest, |
duke@435 | 5146 | BasicType basic_elem_type, |
duke@435 | 5147 | Node* slice_idx, |
duke@435 | 5148 | Node* slice_len, |
duke@435 | 5149 | Node* dest_size) { |
duke@435 | 5150 | // one or the other but not both of slice_len and dest_size: |
duke@435 | 5151 | assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, ""); |
duke@435 | 5152 | if (slice_len == NULL) slice_len = top(); |
duke@435 | 5153 | if (dest_size == NULL) dest_size = top(); |
duke@435 | 5154 | |
duke@435 | 5155 | // operate on this memory slice: |
duke@435 | 5156 | Node* mem = memory(adr_type); // memory slice to operate on |
duke@435 | 5157 | |
duke@435 | 5158 | // scaling and rounding of indexes: |
kvn@464 | 5159 | int scale = exact_log2(type2aelembytes(basic_elem_type)); |
duke@435 | 5160 | int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); |
duke@435 | 5161 | int clear_low = (-1 << scale) & (BytesPerInt - 1); |
duke@435 | 5162 | int bump_bit = (-1 << scale) & BytesPerInt; |
duke@435 | 5163 | |
duke@435 | 5164 | // determine constant starts and ends |
duke@435 | 5165 | const intptr_t BIG_NEG = -128; |
duke@435 | 5166 | assert(BIG_NEG + 2*abase < 0, "neg enough"); |
duke@435 | 5167 | intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG); |
duke@435 | 5168 | intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG); |
duke@435 | 5169 | if (slice_len_con == 0) { |
duke@435 | 5170 | return; // nothing to do here |
duke@435 | 5171 | } |
duke@435 | 5172 | intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low; |
duke@435 | 5173 | intptr_t end_con = find_intptr_t_con(dest_size, -1); |
duke@435 | 5174 | if (slice_idx_con >= 0 && slice_len_con >= 0) { |
duke@435 | 5175 | assert(end_con < 0, "not two cons"); |
duke@435 | 5176 | end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale), |
duke@435 | 5177 | BytesPerLong); |
duke@435 | 5178 | } |
duke@435 | 5179 | |
duke@435 | 5180 | if (start_con >= 0 && end_con >= 0) { |
duke@435 | 5181 | // Constant start and end. Simple. |
duke@435 | 5182 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5183 | start_con, end_con, &_gvn); |
duke@435 | 5184 | } else if (start_con >= 0 && dest_size != top()) { |
duke@435 | 5185 | // Constant start, pre-rounded end after the tail of the array. |
duke@435 | 5186 | Node* end = dest_size; |
duke@435 | 5187 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5188 | start_con, end, &_gvn); |
duke@435 | 5189 | } else if (start_con >= 0 && slice_len != top()) { |
duke@435 | 5190 | // Constant start, non-constant end. End needs rounding up. |
duke@435 | 5191 | // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8) |
duke@435 | 5192 | intptr_t end_base = abase + (slice_idx_con << scale); |
duke@435 | 5193 | int end_round = (-1 << scale) & (BytesPerLong - 1); |
duke@435 | 5194 | Node* end = ConvI2X(slice_len); |
duke@435 | 5195 | if (scale != 0) |
duke@435 | 5196 | end = _gvn.transform( new(C,3) LShiftXNode(end, intcon(scale) )); |
duke@435 | 5197 | end_base += end_round; |
duke@435 | 5198 | end = _gvn.transform( new(C,3) AddXNode(end, MakeConX(end_base)) ); |
duke@435 | 5199 | end = _gvn.transform( new(C,3) AndXNode(end, MakeConX(~end_round)) ); |
duke@435 | 5200 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5201 | start_con, end, &_gvn); |
duke@435 | 5202 | } else if (start_con < 0 && dest_size != top()) { |
duke@435 | 5203 | // Non-constant start, pre-rounded end after the tail of the array. |
duke@435 | 5204 | // This is almost certainly a "round-to-end" operation. |
duke@435 | 5205 | Node* start = slice_idx; |
duke@435 | 5206 | start = ConvI2X(start); |
duke@435 | 5207 | if (scale != 0) |
duke@435 | 5208 | start = _gvn.transform( new(C,3) LShiftXNode( start, intcon(scale) )); |
duke@435 | 5209 | start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(abase)) ); |
duke@435 | 5210 | if ((bump_bit | clear_low) != 0) { |
duke@435 | 5211 | int to_clear = (bump_bit | clear_low); |
duke@435 | 5212 | // Align up mod 8, then store a jint zero unconditionally |
duke@435 | 5213 | // just before the mod-8 boundary. |
coleenp@548 | 5214 | if (((abase + bump_bit) & ~to_clear) - bump_bit |
coleenp@548 | 5215 | < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) { |
coleenp@548 | 5216 | bump_bit = 0; |
coleenp@548 | 5217 | assert((abase & to_clear) == 0, "array base must be long-aligned"); |
coleenp@548 | 5218 | } else { |
coleenp@548 | 5219 | // Bump 'start' up to (or past) the next jint boundary: |
coleenp@548 | 5220 | start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(bump_bit)) ); |
coleenp@548 | 5221 | assert((abase & clear_low) == 0, "array base must be int-aligned"); |
coleenp@548 | 5222 | } |
duke@435 | 5223 | // Round bumped 'start' down to jlong boundary in body of array. |
duke@435 | 5224 | start = _gvn.transform( new(C,3) AndXNode(start, MakeConX(~to_clear)) ); |
coleenp@548 | 5225 | if (bump_bit != 0) { |
coleenp@548 | 5226 | // Store a zero to the immediately preceding jint: |
coleenp@548 | 5227 | Node* x1 = _gvn.transform( new(C,3) AddXNode(start, MakeConX(-bump_bit)) ); |
coleenp@548 | 5228 | Node* p1 = basic_plus_adr(dest, x1); |
coleenp@548 | 5229 | mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT); |
coleenp@548 | 5230 | mem = _gvn.transform(mem); |
coleenp@548 | 5231 | } |
duke@435 | 5232 | } |
duke@435 | 5233 | Node* end = dest_size; // pre-rounded |
duke@435 | 5234 | mem = ClearArrayNode::clear_memory(control(), mem, dest, |
duke@435 | 5235 | start, end, &_gvn); |
duke@435 | 5236 | } else { |
duke@435 | 5237 | // Non-constant start, unrounded non-constant end. |
duke@435 | 5238 | // (Nobody zeroes a random midsection of an array using this routine.) |
duke@435 | 5239 | ShouldNotReachHere(); // fix caller |
duke@435 | 5240 | } |
duke@435 | 5241 | |
duke@435 | 5242 | // Done. |
duke@435 | 5243 | set_memory(mem, adr_type); |
duke@435 | 5244 | } |
duke@435 | 5245 | |
duke@435 | 5246 | |
duke@435 | 5247 | bool |
duke@435 | 5248 | LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type, |
duke@435 | 5249 | BasicType basic_elem_type, |
duke@435 | 5250 | AllocateNode* alloc, |
duke@435 | 5251 | Node* src, Node* src_offset, |
duke@435 | 5252 | Node* dest, Node* dest_offset, |
iveresov@2606 | 5253 | Node* dest_size, bool dest_uninitialized) { |
duke@435 | 5254 | // See if there is an advantage from block transfer. |
kvn@464 | 5255 | int scale = exact_log2(type2aelembytes(basic_elem_type)); |
duke@435 | 5256 | if (scale >= LogBytesPerLong) |
duke@435 | 5257 | return false; // it is already a block transfer |
duke@435 | 5258 | |
duke@435 | 5259 | // Look at the alignment of the starting offsets. |
duke@435 | 5260 | int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type); |
kvn@2939 | 5261 | |
kvn@2939 | 5262 | intptr_t src_off_con = (intptr_t) find_int_con(src_offset, -1); |
kvn@2939 | 5263 | intptr_t dest_off_con = (intptr_t) find_int_con(dest_offset, -1); |
kvn@2939 | 5264 | if (src_off_con < 0 || dest_off_con < 0) |
duke@435 | 5265 | // At present, we can only understand constants. |
duke@435 | 5266 | return false; |
duke@435 | 5267 | |
kvn@2939 | 5268 | intptr_t src_off = abase + (src_off_con << scale); |
kvn@2939 | 5269 | intptr_t dest_off = abase + (dest_off_con << scale); |
kvn@2939 | 5270 | |
duke@435 | 5271 | if (((src_off | dest_off) & (BytesPerLong-1)) != 0) { |
duke@435 | 5272 | // Non-aligned; too bad. |
duke@435 | 5273 | // One more chance: Pick off an initial 32-bit word. |
duke@435 | 5274 | // This is a common case, since abase can be odd mod 8. |
duke@435 | 5275 | if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt && |
duke@435 | 5276 | ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) { |
duke@435 | 5277 | Node* sptr = basic_plus_adr(src, src_off); |
duke@435 | 5278 | Node* dptr = basic_plus_adr(dest, dest_off); |
duke@435 | 5279 | Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type); |
duke@435 | 5280 | store_to_memory(control(), dptr, sval, T_INT, adr_type); |
duke@435 | 5281 | src_off += BytesPerInt; |
duke@435 | 5282 | dest_off += BytesPerInt; |
duke@435 | 5283 | } else { |
duke@435 | 5284 | return false; |
duke@435 | 5285 | } |
duke@435 | 5286 | } |
duke@435 | 5287 | assert(src_off % BytesPerLong == 0, ""); |
duke@435 | 5288 | assert(dest_off % BytesPerLong == 0, ""); |
duke@435 | 5289 | |
duke@435 | 5290 | // Do this copy by giant steps. |
duke@435 | 5291 | Node* sptr = basic_plus_adr(src, src_off); |
duke@435 | 5292 | Node* dptr = basic_plus_adr(dest, dest_off); |
duke@435 | 5293 | Node* countx = dest_size; |
duke@435 | 5294 | countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) ); |
duke@435 | 5295 | countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) ); |
duke@435 | 5296 | |
duke@435 | 5297 | bool disjoint_bases = true; // since alloc != NULL |
duke@435 | 5298 | generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, |
iveresov@2606 | 5299 | sptr, NULL, dptr, NULL, countx, dest_uninitialized); |
duke@435 | 5300 | |
duke@435 | 5301 | return true; |
duke@435 | 5302 | } |
duke@435 | 5303 | |
duke@435 | 5304 | |
duke@435 | 5305 | // Helper function; generates code for the slow case. |
duke@435 | 5306 | // We make a call to a runtime method which emulates the native method, |
duke@435 | 5307 | // but without the native wrapper overhead. |
duke@435 | 5308 | void |
duke@435 | 5309 | LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, |
duke@435 | 5310 | Node* src, Node* src_offset, |
duke@435 | 5311 | Node* dest, Node* dest_offset, |
iveresov@2606 | 5312 | Node* copy_length, bool dest_uninitialized) { |
iveresov@2606 | 5313 | assert(!dest_uninitialized, "Invariant"); |
duke@435 | 5314 | Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, |
duke@435 | 5315 | OptoRuntime::slow_arraycopy_Type(), |
duke@435 | 5316 | OptoRuntime::slow_arraycopy_Java(), |
duke@435 | 5317 | "slow_arraycopy", adr_type, |
duke@435 | 5318 | src, src_offset, dest, dest_offset, |
duke@435 | 5319 | copy_length); |
duke@435 | 5320 | |
duke@435 | 5321 | // Handle exceptions thrown by this fellow: |
duke@435 | 5322 | make_slow_call_ex(call, env()->Throwable_klass(), false); |
duke@435 | 5323 | } |
duke@435 | 5324 | |
duke@435 | 5325 | // Helper function; generates code for cases requiring runtime checks. |
duke@435 | 5326 | Node* |
duke@435 | 5327 | LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type, |
duke@435 | 5328 | Node* dest_elem_klass, |
duke@435 | 5329 | Node* src, Node* src_offset, |
duke@435 | 5330 | Node* dest, Node* dest_offset, |
iveresov@2606 | 5331 | Node* copy_length, bool dest_uninitialized) { |
duke@435 | 5332 | if (stopped()) return NULL; |
duke@435 | 5333 | |
iveresov@2606 | 5334 | address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized); |
duke@435 | 5335 | if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
duke@435 | 5336 | return NULL; |
duke@435 | 5337 | } |
duke@435 | 5338 | |
duke@435 | 5339 | // Pick out the parameters required to perform a store-check |
duke@435 | 5340 | // for the target array. This is an optimistic check. It will |
duke@435 | 5341 | // look in each non-null element's class, at the desired klass's |
duke@435 | 5342 | // super_check_offset, for the desired klass. |
stefank@3391 | 5343 | int sco_offset = in_bytes(Klass::super_check_offset_offset()); |
duke@435 | 5344 | Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset); |
kvn@1964 | 5345 | Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr()); |
never@2199 | 5346 | Node* check_offset = ConvI2X(_gvn.transform(n3)); |
duke@435 | 5347 | Node* check_value = dest_elem_klass; |
duke@435 | 5348 | |
duke@435 | 5349 | Node* src_start = array_element_address(src, src_offset, T_OBJECT); |
duke@435 | 5350 | Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT); |
duke@435 | 5351 | |
duke@435 | 5352 | // (We know the arrays are never conjoint, because their types differ.) |
duke@435 | 5353 | Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5354 | OptoRuntime::checkcast_arraycopy_Type(), |
duke@435 | 5355 | copyfunc_addr, "checkcast_arraycopy", adr_type, |
duke@435 | 5356 | // five arguments, of which two are |
duke@435 | 5357 | // intptr_t (jlong in LP64) |
duke@435 | 5358 | src_start, dest_start, |
duke@435 | 5359 | copy_length XTOP, |
duke@435 | 5360 | check_offset XTOP, |
duke@435 | 5361 | check_value); |
duke@435 | 5362 | |
duke@435 | 5363 | return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms)); |
duke@435 | 5364 | } |
duke@435 | 5365 | |
duke@435 | 5366 | |
duke@435 | 5367 | // Helper function; generates code for cases requiring runtime checks. |
duke@435 | 5368 | Node* |
duke@435 | 5369 | LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, |
duke@435 | 5370 | Node* src, Node* src_offset, |
duke@435 | 5371 | Node* dest, Node* dest_offset, |
iveresov@2606 | 5372 | Node* copy_length, bool dest_uninitialized) { |
iveresov@2606 | 5373 | assert(!dest_uninitialized, "Invariant"); |
duke@435 | 5374 | if (stopped()) return NULL; |
duke@435 | 5375 | address copyfunc_addr = StubRoutines::generic_arraycopy(); |
duke@435 | 5376 | if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. |
duke@435 | 5377 | return NULL; |
duke@435 | 5378 | } |
duke@435 | 5379 | |
duke@435 | 5380 | Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5381 | OptoRuntime::generic_arraycopy_Type(), |
duke@435 | 5382 | copyfunc_addr, "generic_arraycopy", adr_type, |
duke@435 | 5383 | src, src_offset, dest, dest_offset, copy_length); |
duke@435 | 5384 | |
duke@435 | 5385 | return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms)); |
duke@435 | 5386 | } |
duke@435 | 5387 | |
duke@435 | 5388 | // Helper function; generates the fast out-of-line call to an arraycopy stub. |
duke@435 | 5389 | void |
duke@435 | 5390 | LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type, |
duke@435 | 5391 | BasicType basic_elem_type, |
duke@435 | 5392 | bool disjoint_bases, |
duke@435 | 5393 | Node* src, Node* src_offset, |
duke@435 | 5394 | Node* dest, Node* dest_offset, |
iveresov@2606 | 5395 | Node* copy_length, bool dest_uninitialized) { |
duke@435 | 5396 | if (stopped()) return; // nothing to do |
duke@435 | 5397 | |
duke@435 | 5398 | Node* src_start = src; |
duke@435 | 5399 | Node* dest_start = dest; |
duke@435 | 5400 | if (src_offset != NULL || dest_offset != NULL) { |
duke@435 | 5401 | assert(src_offset != NULL && dest_offset != NULL, ""); |
duke@435 | 5402 | src_start = array_element_address(src, src_offset, basic_elem_type); |
duke@435 | 5403 | dest_start = array_element_address(dest, dest_offset, basic_elem_type); |
duke@435 | 5404 | } |
duke@435 | 5405 | |
duke@435 | 5406 | // Figure out which arraycopy runtime method to call. |
duke@435 | 5407 | const char* copyfunc_name = "arraycopy"; |
duke@435 | 5408 | address copyfunc_addr = |
duke@435 | 5409 | basictype2arraycopy(basic_elem_type, src_offset, dest_offset, |
iveresov@2606 | 5410 | disjoint_bases, copyfunc_name, dest_uninitialized); |
duke@435 | 5411 | |
duke@435 | 5412 | // Call it. Note that the count_ix value is not scaled to a byte-size. |
duke@435 | 5413 | make_runtime_call(RC_LEAF|RC_NO_FP, |
duke@435 | 5414 | OptoRuntime::fast_arraycopy_Type(), |
duke@435 | 5415 | copyfunc_addr, copyfunc_name, adr_type, |
duke@435 | 5416 | src_start, dest_start, copy_length XTOP); |
duke@435 | 5417 | } |
johnc@2781 | 5418 | |
johnc@2781 | 5419 | //----------------------------inline_reference_get---------------------------- |
johnc@2781 | 5420 | |
johnc@2781 | 5421 | bool LibraryCallKit::inline_reference_get() { |
johnc@2781 | 5422 | const int nargs = 1; // self |
johnc@2781 | 5423 | |
johnc@2781 | 5424 | guarantee(java_lang_ref_Reference::referent_offset > 0, |
johnc@2781 | 5425 | "should have already been set"); |
johnc@2781 | 5426 | |
johnc@2781 | 5427 | int referent_offset = java_lang_ref_Reference::referent_offset; |
johnc@2781 | 5428 | |
johnc@2781 | 5429 | // Restore the stack and pop off the argument |
johnc@2781 | 5430 | _sp += nargs; |
johnc@2781 | 5431 | Node *reference_obj = pop(); |
johnc@2781 | 5432 | |
johnc@2781 | 5433 | // Null check on self without removing any arguments. |
johnc@2781 | 5434 | _sp += nargs; |
johnc@2781 | 5435 | reference_obj = do_null_check(reference_obj, T_OBJECT); |
johnc@2781 | 5436 | _sp -= nargs;; |
johnc@2781 | 5437 | |
johnc@2781 | 5438 | if (stopped()) return true; |
johnc@2781 | 5439 | |
johnc@2781 | 5440 | Node *adr = basic_plus_adr(reference_obj, reference_obj, referent_offset); |
johnc@2781 | 5441 | |
johnc@2781 | 5442 | ciInstanceKlass* klass = env()->Object_klass(); |
johnc@2781 | 5443 | const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass); |
johnc@2781 | 5444 | |
johnc@2781 | 5445 | Node* no_ctrl = NULL; |
johnc@2781 | 5446 | Node *result = make_load(no_ctrl, adr, object_type, T_OBJECT); |
johnc@2781 | 5447 | |
johnc@2781 | 5448 | // Use the pre-barrier to record the value in the referent field |
johnc@2781 | 5449 | pre_barrier(false /* do_load */, |
johnc@2781 | 5450 | control(), |
johnc@2790 | 5451 | NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */, |
johnc@2781 | 5452 | result /* pre_val */, |
johnc@2781 | 5453 | T_OBJECT); |
johnc@2781 | 5454 | |
johnc@2781 | 5455 | push(result); |
johnc@2781 | 5456 | return true; |
johnc@2781 | 5457 | } |
johnc@2781 | 5458 |