Thu, 07 Apr 2011 09:53:20 -0700
7009266: G1: assert(obj->is_oop_or_null(true )) failed: Error
Summary: A referent object that is only weakly reachable at the start of concurrent marking but is re-attached to the strongly reachable object graph during marking may not be marked as live. This can cause the reference object to be processed prematurely and leave dangling pointers to the referent object. Implement a read barrier for the java.lang.ref.Reference::referent field by intrinsifying the Reference.get() method, and intercepting accesses though JNI, reflection, and Unsafe, so that when a non-null referent object is read it is also logged in an SATB buffer.
Reviewed-by: kvn, iveresov, never, tonyp, dholmes
1 /*
2 * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
25 #include "precompiled.hpp"
26 #include "classfile/systemDictionary.hpp"
27 #include "classfile/vmSymbols.hpp"
28 #include "compiler/compileLog.hpp"
29 #include "oops/objArrayKlass.hpp"
30 #include "opto/addnode.hpp"
31 #include "opto/callGenerator.hpp"
32 #include "opto/cfgnode.hpp"
33 #include "opto/idealKit.hpp"
34 #include "opto/mulnode.hpp"
35 #include "opto/parse.hpp"
36 #include "opto/runtime.hpp"
37 #include "opto/subnode.hpp"
38 #include "prims/nativeLookup.hpp"
39 #include "runtime/sharedRuntime.hpp"
41 class LibraryIntrinsic : public InlineCallGenerator {
42 // Extend the set of intrinsics known to the runtime:
43 public:
44 private:
45 bool _is_virtual;
46 vmIntrinsics::ID _intrinsic_id;
48 public:
49 LibraryIntrinsic(ciMethod* m, bool is_virtual, vmIntrinsics::ID id)
50 : InlineCallGenerator(m),
51 _is_virtual(is_virtual),
52 _intrinsic_id(id)
53 {
54 }
55 virtual bool is_intrinsic() const { return true; }
56 virtual bool is_virtual() const { return _is_virtual; }
57 virtual JVMState* generate(JVMState* jvms);
58 vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }
59 };
62 // Local helper class for LibraryIntrinsic:
63 class LibraryCallKit : public GraphKit {
64 private:
65 LibraryIntrinsic* _intrinsic; // the library intrinsic being called
67 public:
68 LibraryCallKit(JVMState* caller, LibraryIntrinsic* intrinsic)
69 : GraphKit(caller),
70 _intrinsic(intrinsic)
71 {
72 }
74 ciMethod* caller() const { return jvms()->method(); }
75 int bci() const { return jvms()->bci(); }
76 LibraryIntrinsic* intrinsic() const { return _intrinsic; }
77 vmIntrinsics::ID intrinsic_id() const { return _intrinsic->intrinsic_id(); }
78 ciMethod* callee() const { return _intrinsic->method(); }
79 ciSignature* signature() const { return callee()->signature(); }
80 int arg_size() const { return callee()->arg_size(); }
82 bool try_to_inline();
84 // Helper functions to inline natives
85 void push_result(RegionNode* region, PhiNode* value);
86 Node* generate_guard(Node* test, RegionNode* region, float true_prob);
87 Node* generate_slow_guard(Node* test, RegionNode* region);
88 Node* generate_fair_guard(Node* test, RegionNode* region);
89 Node* generate_negative_guard(Node* index, RegionNode* region,
90 // resulting CastII of index:
91 Node* *pos_index = NULL);
92 Node* generate_nonpositive_guard(Node* index, bool never_negative,
93 // resulting CastII of index:
94 Node* *pos_index = NULL);
95 Node* generate_limit_guard(Node* offset, Node* subseq_length,
96 Node* array_length,
97 RegionNode* region);
98 Node* generate_current_thread(Node* &tls_output);
99 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
100 bool disjoint_bases, const char* &name, bool dest_uninitialized);
101 Node* load_mirror_from_klass(Node* klass);
102 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
103 int nargs,
104 RegionNode* region, int null_path,
105 int offset);
106 Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,
107 RegionNode* region, int null_path) {
108 int offset = java_lang_Class::klass_offset_in_bytes();
109 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
110 region, null_path,
111 offset);
112 }
113 Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,
114 int nargs,
115 RegionNode* region, int null_path) {
116 int offset = java_lang_Class::array_klass_offset_in_bytes();
117 return load_klass_from_mirror_common(mirror, never_see_null, nargs,
118 region, null_path,
119 offset);
120 }
121 Node* generate_access_flags_guard(Node* kls,
122 int modifier_mask, int modifier_bits,
123 RegionNode* region);
124 Node* generate_interface_guard(Node* kls, RegionNode* region);
125 Node* generate_array_guard(Node* kls, RegionNode* region) {
126 return generate_array_guard_common(kls, region, false, false);
127 }
128 Node* generate_non_array_guard(Node* kls, RegionNode* region) {
129 return generate_array_guard_common(kls, region, false, true);
130 }
131 Node* generate_objArray_guard(Node* kls, RegionNode* region) {
132 return generate_array_guard_common(kls, region, true, false);
133 }
134 Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {
135 return generate_array_guard_common(kls, region, true, true);
136 }
137 Node* generate_array_guard_common(Node* kls, RegionNode* region,
138 bool obj_array, bool not_array);
139 Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);
140 CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,
141 bool is_virtual = false, bool is_static = false);
142 CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {
143 return generate_method_call(method_id, false, true);
144 }
145 CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {
146 return generate_method_call(method_id, true, false);
147 }
149 Node* make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2);
150 bool inline_string_compareTo();
151 bool inline_string_indexOf();
152 Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);
153 bool inline_string_equals();
154 Node* pop_math_arg();
155 bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);
156 bool inline_math_native(vmIntrinsics::ID id);
157 bool inline_trig(vmIntrinsics::ID id);
158 bool inline_trans(vmIntrinsics::ID id);
159 bool inline_abs(vmIntrinsics::ID id);
160 bool inline_sqrt(vmIntrinsics::ID id);
161 bool inline_pow(vmIntrinsics::ID id);
162 bool inline_exp(vmIntrinsics::ID id);
163 bool inline_min_max(vmIntrinsics::ID id);
164 Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);
165 // This returns Type::AnyPtr, RawPtr, or OopPtr.
166 int classify_unsafe_addr(Node* &base, Node* &offset);
167 Node* make_unsafe_address(Node* base, Node* offset);
168 // Helper for inline_unsafe_access.
169 // Generates the guards that check whether the result of
170 // Unsafe.getObject should be recorded in an SATB log buffer.
171 void insert_g1_pre_barrier(Node* base_oop, Node* offset, Node* pre_val);
172 bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile);
173 bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);
174 bool inline_unsafe_allocate();
175 bool inline_unsafe_copyMemory();
176 bool inline_native_currentThread();
177 bool inline_native_time_funcs(bool isNano);
178 bool inline_native_isInterrupted();
179 bool inline_native_Class_query(vmIntrinsics::ID id);
180 bool inline_native_subtype_check();
182 bool inline_native_newArray();
183 bool inline_native_getLength();
184 bool inline_array_copyOf(bool is_copyOfRange);
185 bool inline_array_equals();
186 void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);
187 bool inline_native_clone(bool is_virtual);
188 bool inline_native_Reflection_getCallerClass();
189 bool inline_native_AtomicLong_get();
190 bool inline_native_AtomicLong_attemptUpdate();
191 bool is_method_invoke_or_aux_frame(JVMState* jvms);
192 // Helper function for inlining native object hash method
193 bool inline_native_hashcode(bool is_virtual, bool is_static);
194 bool inline_native_getClass();
196 // Helper functions for inlining arraycopy
197 bool inline_arraycopy();
198 void generate_arraycopy(const TypePtr* adr_type,
199 BasicType basic_elem_type,
200 Node* src, Node* src_offset,
201 Node* dest, Node* dest_offset,
202 Node* copy_length,
203 bool disjoint_bases = false,
204 bool length_never_negative = false,
205 RegionNode* slow_region = NULL);
206 AllocateArrayNode* tightly_coupled_allocation(Node* ptr,
207 RegionNode* slow_region);
208 void generate_clear_array(const TypePtr* adr_type,
209 Node* dest,
210 BasicType basic_elem_type,
211 Node* slice_off,
212 Node* slice_len,
213 Node* slice_end);
214 bool generate_block_arraycopy(const TypePtr* adr_type,
215 BasicType basic_elem_type,
216 AllocateNode* alloc,
217 Node* src, Node* src_offset,
218 Node* dest, Node* dest_offset,
219 Node* dest_size, bool dest_uninitialized);
220 void generate_slow_arraycopy(const TypePtr* adr_type,
221 Node* src, Node* src_offset,
222 Node* dest, Node* dest_offset,
223 Node* copy_length, bool dest_uninitialized);
224 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
225 Node* dest_elem_klass,
226 Node* src, Node* src_offset,
227 Node* dest, Node* dest_offset,
228 Node* copy_length, bool dest_uninitialized);
229 Node* generate_generic_arraycopy(const TypePtr* adr_type,
230 Node* src, Node* src_offset,
231 Node* dest, Node* dest_offset,
232 Node* copy_length, bool dest_uninitialized);
233 void generate_unchecked_arraycopy(const TypePtr* adr_type,
234 BasicType basic_elem_type,
235 bool disjoint_bases,
236 Node* src, Node* src_offset,
237 Node* dest, Node* dest_offset,
238 Node* copy_length, bool dest_uninitialized);
239 bool inline_unsafe_CAS(BasicType type);
240 bool inline_unsafe_ordered_store(BasicType type);
241 bool inline_fp_conversions(vmIntrinsics::ID id);
242 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
243 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
244 bool inline_bitCount(vmIntrinsics::ID id);
245 bool inline_reverseBytes(vmIntrinsics::ID id);
247 bool inline_reference_get();
248 };
251 //---------------------------make_vm_intrinsic----------------------------
252 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
253 vmIntrinsics::ID id = m->intrinsic_id();
254 assert(id != vmIntrinsics::_none, "must be a VM intrinsic");
256 if (DisableIntrinsic[0] != '\0'
257 && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {
258 // disabled by a user request on the command line:
259 // example: -XX:DisableIntrinsic=_hashCode,_getClass
260 return NULL;
261 }
263 if (!m->is_loaded()) {
264 // do not attempt to inline unloaded methods
265 return NULL;
266 }
268 // Only a few intrinsics implement a virtual dispatch.
269 // They are expensive calls which are also frequently overridden.
270 if (is_virtual) {
271 switch (id) {
272 case vmIntrinsics::_hashCode:
273 case vmIntrinsics::_clone:
274 // OK, Object.hashCode and Object.clone intrinsics come in both flavors
275 break;
276 default:
277 return NULL;
278 }
279 }
281 // -XX:-InlineNatives disables nearly all intrinsics:
282 if (!InlineNatives) {
283 switch (id) {
284 case vmIntrinsics::_indexOf:
285 case vmIntrinsics::_compareTo:
286 case vmIntrinsics::_equals:
287 case vmIntrinsics::_equalsC:
288 break; // InlineNatives does not control String.compareTo
289 default:
290 return NULL;
291 }
292 }
294 switch (id) {
295 case vmIntrinsics::_compareTo:
296 if (!SpecialStringCompareTo) return NULL;
297 break;
298 case vmIntrinsics::_indexOf:
299 if (!SpecialStringIndexOf) return NULL;
300 break;
301 case vmIntrinsics::_equals:
302 if (!SpecialStringEquals) return NULL;
303 break;
304 case vmIntrinsics::_equalsC:
305 if (!SpecialArraysEquals) return NULL;
306 break;
307 case vmIntrinsics::_arraycopy:
308 if (!InlineArrayCopy) return NULL;
309 break;
310 case vmIntrinsics::_copyMemory:
311 if (StubRoutines::unsafe_arraycopy() == NULL) return NULL;
312 if (!InlineArrayCopy) return NULL;
313 break;
314 case vmIntrinsics::_hashCode:
315 if (!InlineObjectHash) return NULL;
316 break;
317 case vmIntrinsics::_clone:
318 case vmIntrinsics::_copyOf:
319 case vmIntrinsics::_copyOfRange:
320 if (!InlineObjectCopy) return NULL;
321 // These also use the arraycopy intrinsic mechanism:
322 if (!InlineArrayCopy) return NULL;
323 break;
324 case vmIntrinsics::_checkIndex:
325 // We do not intrinsify this. The optimizer does fine with it.
326 return NULL;
328 case vmIntrinsics::_get_AtomicLong:
329 case vmIntrinsics::_attemptUpdate:
330 if (!InlineAtomicLong) return NULL;
331 break;
333 case vmIntrinsics::_getCallerClass:
334 if (!UseNewReflection) return NULL;
335 if (!InlineReflectionGetCallerClass) return NULL;
336 if (!JDK_Version::is_gte_jdk14x_version()) return NULL;
337 break;
339 case vmIntrinsics::_bitCount_i:
340 case vmIntrinsics::_bitCount_l:
341 if (!UsePopCountInstruction) return NULL;
342 break;
344 case vmIntrinsics::_Reference_get:
345 // It is only when G1 is enabled that we absolutely
346 // need to use the intrinsic version of Reference.get()
347 // so that the value in the referent field, if necessary,
348 // can be registered by the pre-barrier code.
349 if (!UseG1GC) return NULL;
350 break;
352 default:
353 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");
354 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");
355 break;
356 }
358 // -XX:-InlineClassNatives disables natives from the Class class.
359 // The flag applies to all reflective calls, notably Array.newArray
360 // (visible to Java programmers as Array.newInstance).
361 if (m->holder()->name() == ciSymbol::java_lang_Class() ||
362 m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {
363 if (!InlineClassNatives) return NULL;
364 }
366 // -XX:-InlineThreadNatives disables natives from the Thread class.
367 if (m->holder()->name() == ciSymbol::java_lang_Thread()) {
368 if (!InlineThreadNatives) return NULL;
369 }
371 // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.
372 if (m->holder()->name() == ciSymbol::java_lang_Math() ||
373 m->holder()->name() == ciSymbol::java_lang_Float() ||
374 m->holder()->name() == ciSymbol::java_lang_Double()) {
375 if (!InlineMathNatives) return NULL;
376 }
378 // -XX:-InlineUnsafeOps disables natives from the Unsafe class.
379 if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {
380 if (!InlineUnsafeOps) return NULL;
381 }
383 return new LibraryIntrinsic(m, is_virtual, (vmIntrinsics::ID) id);
384 }
386 //----------------------register_library_intrinsics-----------------------
387 // Initialize this file's data structures, for each Compile instance.
388 void Compile::register_library_intrinsics() {
389 // Nothing to do here.
390 }
392 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {
393 LibraryCallKit kit(jvms, this);
394 Compile* C = kit.C;
395 int nodes = C->unique();
396 #ifndef PRODUCT
397 if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {
398 char buf[1000];
399 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));
400 tty->print_cr("Intrinsic %s", str);
401 }
402 #endif
404 if (kit.try_to_inline()) {
405 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {
406 if (jvms->has_method()) {
407 // Not a root compile.
408 tty->print("Inlining intrinsic %s%s at bci:%d in",
409 vmIntrinsics::name_at(intrinsic_id()),
410 (is_virtual() ? " (virtual)" : ""), kit.bci());
411 kit.caller()->print_short_name(tty);
412 tty->print_cr(" (%d bytes)", kit.caller()->code_size());
413 } else {
414 // Root compile
415 tty->print_cr("Generating intrinsic %s%s at bci:%d",
416 vmIntrinsics::name_at(intrinsic_id()),
417 (is_virtual() ? " (virtual)" : ""), kit.bci());
418 }
419 }
420 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);
421 if (C->log()) {
422 C->log()->elem("intrinsic id='%s'%s nodes='%d'",
423 vmIntrinsics::name_at(intrinsic_id()),
424 (is_virtual() ? " virtual='1'" : ""),
425 C->unique() - nodes);
426 }
427 return kit.transfer_exceptions_into_jvms();
428 }
430 if (PrintIntrinsics) {
431 if (jvms->has_method()) {
432 // Not a root compile.
433 tty->print("Did not inline intrinsic %s%s at bci:%d in",
434 vmIntrinsics::name_at(intrinsic_id()),
435 (is_virtual() ? " (virtual)" : ""), kit.bci());
436 kit.caller()->print_short_name(tty);
437 tty->print_cr(" (%d bytes)", kit.caller()->code_size());
438 } else {
439 // Root compile
440 tty->print("Did not generate intrinsic %s%s at bci:%d in",
441 vmIntrinsics::name_at(intrinsic_id()),
442 (is_virtual() ? " (virtual)" : ""), kit.bci());
443 }
444 }
445 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);
446 return NULL;
447 }
449 bool LibraryCallKit::try_to_inline() {
450 // Handle symbolic names for otherwise undistinguished boolean switches:
451 const bool is_store = true;
452 const bool is_native_ptr = true;
453 const bool is_static = true;
455 if (!jvms()->has_method()) {
456 // Root JVMState has a null method.
457 assert(map()->memory()->Opcode() == Op_Parm, "");
458 // Insert the memory aliasing node
459 set_all_memory(reset_memory());
460 }
461 assert(merged_memory(), "");
463 switch (intrinsic_id()) {
464 case vmIntrinsics::_hashCode:
465 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);
466 case vmIntrinsics::_identityHashCode:
467 return inline_native_hashcode(/*!virtual*/ false, is_static);
468 case vmIntrinsics::_getClass:
469 return inline_native_getClass();
471 case vmIntrinsics::_dsin:
472 case vmIntrinsics::_dcos:
473 case vmIntrinsics::_dtan:
474 case vmIntrinsics::_dabs:
475 case vmIntrinsics::_datan2:
476 case vmIntrinsics::_dsqrt:
477 case vmIntrinsics::_dexp:
478 case vmIntrinsics::_dlog:
479 case vmIntrinsics::_dlog10:
480 case vmIntrinsics::_dpow:
481 return inline_math_native(intrinsic_id());
483 case vmIntrinsics::_min:
484 case vmIntrinsics::_max:
485 return inline_min_max(intrinsic_id());
487 case vmIntrinsics::_arraycopy:
488 return inline_arraycopy();
490 case vmIntrinsics::_compareTo:
491 return inline_string_compareTo();
492 case vmIntrinsics::_indexOf:
493 return inline_string_indexOf();
494 case vmIntrinsics::_equals:
495 return inline_string_equals();
497 case vmIntrinsics::_getObject:
498 return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, false);
499 case vmIntrinsics::_getBoolean:
500 return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, false);
501 case vmIntrinsics::_getByte:
502 return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, false);
503 case vmIntrinsics::_getShort:
504 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, false);
505 case vmIntrinsics::_getChar:
506 return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, false);
507 case vmIntrinsics::_getInt:
508 return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, false);
509 case vmIntrinsics::_getLong:
510 return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, false);
511 case vmIntrinsics::_getFloat:
512 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, false);
513 case vmIntrinsics::_getDouble:
514 return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, false);
516 case vmIntrinsics::_putObject:
517 return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, false);
518 case vmIntrinsics::_putBoolean:
519 return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, false);
520 case vmIntrinsics::_putByte:
521 return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, false);
522 case vmIntrinsics::_putShort:
523 return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, false);
524 case vmIntrinsics::_putChar:
525 return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, false);
526 case vmIntrinsics::_putInt:
527 return inline_unsafe_access(!is_native_ptr, is_store, T_INT, false);
528 case vmIntrinsics::_putLong:
529 return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, false);
530 case vmIntrinsics::_putFloat:
531 return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, false);
532 case vmIntrinsics::_putDouble:
533 return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, false);
535 case vmIntrinsics::_getByte_raw:
536 return inline_unsafe_access(is_native_ptr, !is_store, T_BYTE, false);
537 case vmIntrinsics::_getShort_raw:
538 return inline_unsafe_access(is_native_ptr, !is_store, T_SHORT, false);
539 case vmIntrinsics::_getChar_raw:
540 return inline_unsafe_access(is_native_ptr, !is_store, T_CHAR, false);
541 case vmIntrinsics::_getInt_raw:
542 return inline_unsafe_access(is_native_ptr, !is_store, T_INT, false);
543 case vmIntrinsics::_getLong_raw:
544 return inline_unsafe_access(is_native_ptr, !is_store, T_LONG, false);
545 case vmIntrinsics::_getFloat_raw:
546 return inline_unsafe_access(is_native_ptr, !is_store, T_FLOAT, false);
547 case vmIntrinsics::_getDouble_raw:
548 return inline_unsafe_access(is_native_ptr, !is_store, T_DOUBLE, false);
549 case vmIntrinsics::_getAddress_raw:
550 return inline_unsafe_access(is_native_ptr, !is_store, T_ADDRESS, false);
552 case vmIntrinsics::_putByte_raw:
553 return inline_unsafe_access(is_native_ptr, is_store, T_BYTE, false);
554 case vmIntrinsics::_putShort_raw:
555 return inline_unsafe_access(is_native_ptr, is_store, T_SHORT, false);
556 case vmIntrinsics::_putChar_raw:
557 return inline_unsafe_access(is_native_ptr, is_store, T_CHAR, false);
558 case vmIntrinsics::_putInt_raw:
559 return inline_unsafe_access(is_native_ptr, is_store, T_INT, false);
560 case vmIntrinsics::_putLong_raw:
561 return inline_unsafe_access(is_native_ptr, is_store, T_LONG, false);
562 case vmIntrinsics::_putFloat_raw:
563 return inline_unsafe_access(is_native_ptr, is_store, T_FLOAT, false);
564 case vmIntrinsics::_putDouble_raw:
565 return inline_unsafe_access(is_native_ptr, is_store, T_DOUBLE, false);
566 case vmIntrinsics::_putAddress_raw:
567 return inline_unsafe_access(is_native_ptr, is_store, T_ADDRESS, false);
569 case vmIntrinsics::_getObjectVolatile:
570 return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, true);
571 case vmIntrinsics::_getBooleanVolatile:
572 return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, true);
573 case vmIntrinsics::_getByteVolatile:
574 return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, true);
575 case vmIntrinsics::_getShortVolatile:
576 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, true);
577 case vmIntrinsics::_getCharVolatile:
578 return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, true);
579 case vmIntrinsics::_getIntVolatile:
580 return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, true);
581 case vmIntrinsics::_getLongVolatile:
582 return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, true);
583 case vmIntrinsics::_getFloatVolatile:
584 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, true);
585 case vmIntrinsics::_getDoubleVolatile:
586 return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, true);
588 case vmIntrinsics::_putObjectVolatile:
589 return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, true);
590 case vmIntrinsics::_putBooleanVolatile:
591 return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, true);
592 case vmIntrinsics::_putByteVolatile:
593 return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, true);
594 case vmIntrinsics::_putShortVolatile:
595 return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, true);
596 case vmIntrinsics::_putCharVolatile:
597 return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, true);
598 case vmIntrinsics::_putIntVolatile:
599 return inline_unsafe_access(!is_native_ptr, is_store, T_INT, true);
600 case vmIntrinsics::_putLongVolatile:
601 return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, true);
602 case vmIntrinsics::_putFloatVolatile:
603 return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, true);
604 case vmIntrinsics::_putDoubleVolatile:
605 return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, true);
607 case vmIntrinsics::_prefetchRead:
608 return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);
609 case vmIntrinsics::_prefetchWrite:
610 return inline_unsafe_prefetch(!is_native_ptr, is_store, !is_static);
611 case vmIntrinsics::_prefetchReadStatic:
612 return inline_unsafe_prefetch(!is_native_ptr, !is_store, is_static);
613 case vmIntrinsics::_prefetchWriteStatic:
614 return inline_unsafe_prefetch(!is_native_ptr, is_store, is_static);
616 case vmIntrinsics::_compareAndSwapObject:
617 return inline_unsafe_CAS(T_OBJECT);
618 case vmIntrinsics::_compareAndSwapInt:
619 return inline_unsafe_CAS(T_INT);
620 case vmIntrinsics::_compareAndSwapLong:
621 return inline_unsafe_CAS(T_LONG);
623 case vmIntrinsics::_putOrderedObject:
624 return inline_unsafe_ordered_store(T_OBJECT);
625 case vmIntrinsics::_putOrderedInt:
626 return inline_unsafe_ordered_store(T_INT);
627 case vmIntrinsics::_putOrderedLong:
628 return inline_unsafe_ordered_store(T_LONG);
630 case vmIntrinsics::_currentThread:
631 return inline_native_currentThread();
632 case vmIntrinsics::_isInterrupted:
633 return inline_native_isInterrupted();
635 case vmIntrinsics::_currentTimeMillis:
636 return inline_native_time_funcs(false);
637 case vmIntrinsics::_nanoTime:
638 return inline_native_time_funcs(true);
639 case vmIntrinsics::_allocateInstance:
640 return inline_unsafe_allocate();
641 case vmIntrinsics::_copyMemory:
642 return inline_unsafe_copyMemory();
643 case vmIntrinsics::_newArray:
644 return inline_native_newArray();
645 case vmIntrinsics::_getLength:
646 return inline_native_getLength();
647 case vmIntrinsics::_copyOf:
648 return inline_array_copyOf(false);
649 case vmIntrinsics::_copyOfRange:
650 return inline_array_copyOf(true);
651 case vmIntrinsics::_equalsC:
652 return inline_array_equals();
653 case vmIntrinsics::_clone:
654 return inline_native_clone(intrinsic()->is_virtual());
656 case vmIntrinsics::_isAssignableFrom:
657 return inline_native_subtype_check();
659 case vmIntrinsics::_isInstance:
660 case vmIntrinsics::_getModifiers:
661 case vmIntrinsics::_isInterface:
662 case vmIntrinsics::_isArray:
663 case vmIntrinsics::_isPrimitive:
664 case vmIntrinsics::_getSuperclass:
665 case vmIntrinsics::_getComponentType:
666 case vmIntrinsics::_getClassAccessFlags:
667 return inline_native_Class_query(intrinsic_id());
669 case vmIntrinsics::_floatToRawIntBits:
670 case vmIntrinsics::_floatToIntBits:
671 case vmIntrinsics::_intBitsToFloat:
672 case vmIntrinsics::_doubleToRawLongBits:
673 case vmIntrinsics::_doubleToLongBits:
674 case vmIntrinsics::_longBitsToDouble:
675 return inline_fp_conversions(intrinsic_id());
677 case vmIntrinsics::_numberOfLeadingZeros_i:
678 case vmIntrinsics::_numberOfLeadingZeros_l:
679 return inline_numberOfLeadingZeros(intrinsic_id());
681 case vmIntrinsics::_numberOfTrailingZeros_i:
682 case vmIntrinsics::_numberOfTrailingZeros_l:
683 return inline_numberOfTrailingZeros(intrinsic_id());
685 case vmIntrinsics::_bitCount_i:
686 case vmIntrinsics::_bitCount_l:
687 return inline_bitCount(intrinsic_id());
689 case vmIntrinsics::_reverseBytes_i:
690 case vmIntrinsics::_reverseBytes_l:
691 case vmIntrinsics::_reverseBytes_s:
692 case vmIntrinsics::_reverseBytes_c:
693 return inline_reverseBytes((vmIntrinsics::ID) intrinsic_id());
695 case vmIntrinsics::_get_AtomicLong:
696 return inline_native_AtomicLong_get();
697 case vmIntrinsics::_attemptUpdate:
698 return inline_native_AtomicLong_attemptUpdate();
700 case vmIntrinsics::_getCallerClass:
701 return inline_native_Reflection_getCallerClass();
703 case vmIntrinsics::_Reference_get:
704 return inline_reference_get();
706 default:
707 // If you get here, it may be that someone has added a new intrinsic
708 // to the list in vmSymbols.hpp without implementing it here.
709 #ifndef PRODUCT
710 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {
711 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",
712 vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());
713 }
714 #endif
715 return false;
716 }
717 }
719 //------------------------------push_result------------------------------
720 // Helper function for finishing intrinsics.
721 void LibraryCallKit::push_result(RegionNode* region, PhiNode* value) {
722 record_for_igvn(region);
723 set_control(_gvn.transform(region));
724 BasicType value_type = value->type()->basic_type();
725 push_node(value_type, _gvn.transform(value));
726 }
728 //------------------------------generate_guard---------------------------
729 // Helper function for generating guarded fast-slow graph structures.
730 // The given 'test', if true, guards a slow path. If the test fails
731 // then a fast path can be taken. (We generally hope it fails.)
732 // In all cases, GraphKit::control() is updated to the fast path.
733 // The returned value represents the control for the slow path.
734 // The return value is never 'top'; it is either a valid control
735 // or NULL if it is obvious that the slow path can never be taken.
736 // Also, if region and the slow control are not NULL, the slow edge
737 // is appended to the region.
738 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {
739 if (stopped()) {
740 // Already short circuited.
741 return NULL;
742 }
744 // Build an if node and its projections.
745 // If test is true we take the slow path, which we assume is uncommon.
746 if (_gvn.type(test) == TypeInt::ZERO) {
747 // The slow branch is never taken. No need to build this guard.
748 return NULL;
749 }
751 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);
753 Node* if_slow = _gvn.transform( new (C, 1) IfTrueNode(iff) );
754 if (if_slow == top()) {
755 // The slow branch is never taken. No need to build this guard.
756 return NULL;
757 }
759 if (region != NULL)
760 region->add_req(if_slow);
762 Node* if_fast = _gvn.transform( new (C, 1) IfFalseNode(iff) );
763 set_control(if_fast);
765 return if_slow;
766 }
768 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {
769 return generate_guard(test, region, PROB_UNLIKELY_MAG(3));
770 }
771 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {
772 return generate_guard(test, region, PROB_FAIR);
773 }
775 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,
776 Node* *pos_index) {
777 if (stopped())
778 return NULL; // already stopped
779 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]
780 return NULL; // index is already adequately typed
781 Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );
782 Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );
783 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);
784 if (is_neg != NULL && pos_index != NULL) {
785 // Emulate effect of Parse::adjust_map_after_if.
786 Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS);
787 ccast->set_req(0, control());
788 (*pos_index) = _gvn.transform(ccast);
789 }
790 return is_neg;
791 }
793 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,
794 Node* *pos_index) {
795 if (stopped())
796 return NULL; // already stopped
797 if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]
798 return NULL; // index is already adequately typed
799 Node* cmp_le = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );
800 BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);
801 Node* bol_le = _gvn.transform( new (C, 2) BoolNode(cmp_le, le_or_eq) );
802 Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);
803 if (is_notp != NULL && pos_index != NULL) {
804 // Emulate effect of Parse::adjust_map_after_if.
805 Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS1);
806 ccast->set_req(0, control());
807 (*pos_index) = _gvn.transform(ccast);
808 }
809 return is_notp;
810 }
812 // Make sure that 'position' is a valid limit index, in [0..length].
813 // There are two equivalent plans for checking this:
814 // A. (offset + copyLength) unsigned<= arrayLength
815 // B. offset <= (arrayLength - copyLength)
816 // We require that all of the values above, except for the sum and
817 // difference, are already known to be non-negative.
818 // Plan A is robust in the face of overflow, if offset and copyLength
819 // are both hugely positive.
820 //
821 // Plan B is less direct and intuitive, but it does not overflow at
822 // all, since the difference of two non-negatives is always
823 // representable. Whenever Java methods must perform the equivalent
824 // check they generally use Plan B instead of Plan A.
825 // For the moment we use Plan A.
826 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,
827 Node* subseq_length,
828 Node* array_length,
829 RegionNode* region) {
830 if (stopped())
831 return NULL; // already stopped
832 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;
833 if (zero_offset && _gvn.eqv_uncast(subseq_length, array_length))
834 return NULL; // common case of whole-array copy
835 Node* last = subseq_length;
836 if (!zero_offset) // last += offset
837 last = _gvn.transform( new (C, 3) AddINode(last, offset));
838 Node* cmp_lt = _gvn.transform( new (C, 3) CmpUNode(array_length, last) );
839 Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );
840 Node* is_over = generate_guard(bol_lt, region, PROB_MIN);
841 return is_over;
842 }
845 //--------------------------generate_current_thread--------------------
846 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {
847 ciKlass* thread_klass = env()->Thread_klass();
848 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);
849 Node* thread = _gvn.transform(new (C, 1) ThreadLocalNode());
850 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));
851 Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);
852 tls_output = thread;
853 return threadObj;
854 }
857 //------------------------------make_string_method_node------------------------
858 // Helper method for String intrinsic finctions.
859 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2) {
860 const int value_offset = java_lang_String::value_offset_in_bytes();
861 const int count_offset = java_lang_String::count_offset_in_bytes();
862 const int offset_offset = java_lang_String::offset_offset_in_bytes();
864 Node* no_ctrl = NULL;
866 ciInstanceKlass* klass = env()->String_klass();
867 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
869 const TypeAryPtr* value_type =
870 TypeAryPtr::make(TypePtr::NotNull,
871 TypeAry::make(TypeInt::CHAR,TypeInt::POS),
872 ciTypeArrayKlass::make(T_CHAR), true, 0);
874 // Get start addr of string and substring
875 Node* str1_valuea = basic_plus_adr(str1, str1, value_offset);
876 Node* str1_value = make_load(no_ctrl, str1_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset));
877 Node* str1_offseta = basic_plus_adr(str1, str1, offset_offset);
878 Node* str1_offset = make_load(no_ctrl, str1_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));
879 Node* str1_start = array_element_address(str1_value, str1_offset, T_CHAR);
881 // Pin loads from String::equals() argument since it could be NULL.
882 Node* str2_ctrl = (opcode == Op_StrEquals) ? control() : no_ctrl;
883 Node* str2_valuea = basic_plus_adr(str2, str2, value_offset);
884 Node* str2_value = make_load(str2_ctrl, str2_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset));
885 Node* str2_offseta = basic_plus_adr(str2, str2, offset_offset);
886 Node* str2_offset = make_load(str2_ctrl, str2_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));
887 Node* str2_start = array_element_address(str2_value, str2_offset, T_CHAR);
889 Node* result = NULL;
890 switch (opcode) {
891 case Op_StrIndexOf:
892 result = new (C, 6) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),
893 str1_start, cnt1, str2_start, cnt2);
894 break;
895 case Op_StrComp:
896 result = new (C, 6) StrCompNode(control(), memory(TypeAryPtr::CHARS),
897 str1_start, cnt1, str2_start, cnt2);
898 break;
899 case Op_StrEquals:
900 result = new (C, 5) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),
901 str1_start, str2_start, cnt1);
902 break;
903 default:
904 ShouldNotReachHere();
905 return NULL;
906 }
908 // All these intrinsics have checks.
909 C->set_has_split_ifs(true); // Has chance for split-if optimization
911 return _gvn.transform(result);
912 }
914 //------------------------------inline_string_compareTo------------------------
915 bool LibraryCallKit::inline_string_compareTo() {
917 if (!Matcher::has_match_rule(Op_StrComp)) return false;
919 const int value_offset = java_lang_String::value_offset_in_bytes();
920 const int count_offset = java_lang_String::count_offset_in_bytes();
921 const int offset_offset = java_lang_String::offset_offset_in_bytes();
923 _sp += 2;
924 Node *argument = pop(); // pop non-receiver first: it was pushed second
925 Node *receiver = pop();
927 // Null check on self without removing any arguments. The argument
928 // null check technically happens in the wrong place, which can lead to
929 // invalid stack traces when string compare is inlined into a method
930 // which handles NullPointerExceptions.
931 _sp += 2;
932 receiver = do_null_check(receiver, T_OBJECT);
933 argument = do_null_check(argument, T_OBJECT);
934 _sp -= 2;
935 if (stopped()) {
936 return true;
937 }
939 ciInstanceKlass* klass = env()->String_klass();
940 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
941 Node* no_ctrl = NULL;
943 // Get counts for string and argument
944 Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset);
945 Node* receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
947 Node* argument_cnta = basic_plus_adr(argument, argument, count_offset);
948 Node* argument_cnt = make_load(no_ctrl, argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
950 Node* compare = make_string_method_node(Op_StrComp, receiver, receiver_cnt, argument, argument_cnt);
951 push(compare);
952 return true;
953 }
955 //------------------------------inline_string_equals------------------------
956 bool LibraryCallKit::inline_string_equals() {
958 if (!Matcher::has_match_rule(Op_StrEquals)) return false;
960 const int value_offset = java_lang_String::value_offset_in_bytes();
961 const int count_offset = java_lang_String::count_offset_in_bytes();
962 const int offset_offset = java_lang_String::offset_offset_in_bytes();
964 int nargs = 2;
965 _sp += nargs;
966 Node* argument = pop(); // pop non-receiver first: it was pushed second
967 Node* receiver = pop();
969 // Null check on self without removing any arguments. The argument
970 // null check technically happens in the wrong place, which can lead to
971 // invalid stack traces when string compare is inlined into a method
972 // which handles NullPointerExceptions.
973 _sp += nargs;
974 receiver = do_null_check(receiver, T_OBJECT);
975 //should not do null check for argument for String.equals(), because spec
976 //allows to specify NULL as argument.
977 _sp -= nargs;
979 if (stopped()) {
980 return true;
981 }
983 // paths (plus control) merge
984 RegionNode* region = new (C, 5) RegionNode(5);
985 Node* phi = new (C, 5) PhiNode(region, TypeInt::BOOL);
987 // does source == target string?
988 Node* cmp = _gvn.transform(new (C, 3) CmpPNode(receiver, argument));
989 Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::eq));
991 Node* if_eq = generate_slow_guard(bol, NULL);
992 if (if_eq != NULL) {
993 // receiver == argument
994 phi->init_req(2, intcon(1));
995 region->init_req(2, if_eq);
996 }
998 // get String klass for instanceOf
999 ciInstanceKlass* klass = env()->String_klass();
1001 if (!stopped()) {
1002 _sp += nargs; // gen_instanceof might do an uncommon trap
1003 Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));
1004 _sp -= nargs;
1005 Node* cmp = _gvn.transform(new (C, 3) CmpINode(inst, intcon(1)));
1006 Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::ne));
1008 Node* inst_false = generate_guard(bol, NULL, PROB_MIN);
1009 //instanceOf == true, fallthrough
1011 if (inst_false != NULL) {
1012 phi->init_req(3, intcon(0));
1013 region->init_req(3, inst_false);
1014 }
1015 }
1017 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
1019 Node* no_ctrl = NULL;
1020 Node* receiver_cnt;
1021 Node* argument_cnt;
1023 if (!stopped()) {
1024 // Properly cast the argument to String
1025 argument = _gvn.transform(new (C, 2) CheckCastPPNode(control(), argument, string_type));
1027 // Get counts for string and argument
1028 Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset);
1029 receiver_cnt = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
1031 // Pin load from argument string since it could be NULL.
1032 Node* argument_cnta = basic_plus_adr(argument, argument, count_offset);
1033 argument_cnt = make_load(control(), argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
1035 // Check for receiver count != argument count
1036 Node* cmp = _gvn.transform( new(C, 3) CmpINode(receiver_cnt, argument_cnt) );
1037 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::ne) );
1038 Node* if_ne = generate_slow_guard(bol, NULL);
1039 if (if_ne != NULL) {
1040 phi->init_req(4, intcon(0));
1041 region->init_req(4, if_ne);
1042 }
1043 }
1045 // Check for count == 0 is done by mach node StrEquals.
1047 if (!stopped()) {
1048 Node* equals = make_string_method_node(Op_StrEquals, receiver, receiver_cnt, argument, argument_cnt);
1049 phi->init_req(1, equals);
1050 region->init_req(1, control());
1051 }
1053 // post merge
1054 set_control(_gvn.transform(region));
1055 record_for_igvn(region);
1057 push(_gvn.transform(phi));
1059 return true;
1060 }
1062 //------------------------------inline_array_equals----------------------------
1063 bool LibraryCallKit::inline_array_equals() {
1065 if (!Matcher::has_match_rule(Op_AryEq)) return false;
1067 _sp += 2;
1068 Node *argument2 = pop();
1069 Node *argument1 = pop();
1071 Node* equals =
1072 _gvn.transform(new (C, 4) AryEqNode(control(), memory(TypeAryPtr::CHARS),
1073 argument1, argument2) );
1074 push(equals);
1075 return true;
1076 }
1078 // Java version of String.indexOf(constant string)
1079 // class StringDecl {
1080 // StringDecl(char[] ca) {
1081 // offset = 0;
1082 // count = ca.length;
1083 // value = ca;
1084 // }
1085 // int offset;
1086 // int count;
1087 // char[] value;
1088 // }
1089 //
1090 // static int string_indexOf_J(StringDecl string_object, char[] target_object,
1091 // int targetOffset, int cache_i, int md2) {
1092 // int cache = cache_i;
1093 // int sourceOffset = string_object.offset;
1094 // int sourceCount = string_object.count;
1095 // int targetCount = target_object.length;
1096 //
1097 // int targetCountLess1 = targetCount - 1;
1098 // int sourceEnd = sourceOffset + sourceCount - targetCountLess1;
1099 //
1100 // char[] source = string_object.value;
1101 // char[] target = target_object;
1102 // int lastChar = target[targetCountLess1];
1103 //
1104 // outer_loop:
1105 // for (int i = sourceOffset; i < sourceEnd; ) {
1106 // int src = source[i + targetCountLess1];
1107 // if (src == lastChar) {
1108 // // With random strings and a 4-character alphabet,
1109 // // reverse matching at this point sets up 0.8% fewer
1110 // // frames, but (paradoxically) makes 0.3% more probes.
1111 // // Since those probes are nearer the lastChar probe,
1112 // // there is may be a net D$ win with reverse matching.
1113 // // But, reversing loop inhibits unroll of inner loop
1114 // // for unknown reason. So, does running outer loop from
1115 // // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)
1116 // for (int j = 0; j < targetCountLess1; j++) {
1117 // if (target[targetOffset + j] != source[i+j]) {
1118 // if ((cache & (1 << source[i+j])) == 0) {
1119 // if (md2 < j+1) {
1120 // i += j+1;
1121 // continue outer_loop;
1122 // }
1123 // }
1124 // i += md2;
1125 // continue outer_loop;
1126 // }
1127 // }
1128 // return i - sourceOffset;
1129 // }
1130 // if ((cache & (1 << src)) == 0) {
1131 // i += targetCountLess1;
1132 // } // using "i += targetCount;" and an "else i++;" causes a jump to jump.
1133 // i++;
1134 // }
1135 // return -1;
1136 // }
1138 //------------------------------string_indexOf------------------------
1139 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,
1140 jint cache_i, jint md2_i) {
1142 Node* no_ctrl = NULL;
1143 float likely = PROB_LIKELY(0.9);
1144 float unlikely = PROB_UNLIKELY(0.9);
1146 const int nargs = 2; // number of arguments to push back for uncommon trap in predicate
1148 const int value_offset = java_lang_String::value_offset_in_bytes();
1149 const int count_offset = java_lang_String::count_offset_in_bytes();
1150 const int offset_offset = java_lang_String::offset_offset_in_bytes();
1152 ciInstanceKlass* klass = env()->String_klass();
1153 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);
1154 const TypeAryPtr* source_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::CHAR,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, 0);
1156 Node* sourceOffseta = basic_plus_adr(string_object, string_object, offset_offset);
1157 Node* sourceOffset = make_load(no_ctrl, sourceOffseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));
1158 Node* sourceCounta = basic_plus_adr(string_object, string_object, count_offset);
1159 Node* sourceCount = make_load(no_ctrl, sourceCounta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
1160 Node* sourcea = basic_plus_adr(string_object, string_object, value_offset);
1161 Node* source = make_load(no_ctrl, sourcea, source_type, T_OBJECT, string_type->add_offset(value_offset));
1163 Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)) );
1164 jint target_length = target_array->length();
1165 const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));
1166 const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);
1168 IdealKit kit(gvn(), control(), merged_memory(), false, true);
1169 #define __ kit.
1170 Node* zero = __ ConI(0);
1171 Node* one = __ ConI(1);
1172 Node* cache = __ ConI(cache_i);
1173 Node* md2 = __ ConI(md2_i);
1174 Node* lastChar = __ ConI(target_array->char_at(target_length - 1));
1175 Node* targetCount = __ ConI(target_length);
1176 Node* targetCountLess1 = __ ConI(target_length - 1);
1177 Node* targetOffset = __ ConI(targetOffset_i);
1178 Node* sourceEnd = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);
1180 IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();
1181 Node* outer_loop = __ make_label(2 /* goto */);
1182 Node* return_ = __ make_label(1);
1184 __ set(rtn,__ ConI(-1));
1185 __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {
1186 Node* i2 = __ AddI(__ value(i), targetCountLess1);
1187 // pin to prohibit loading of "next iteration" value which may SEGV (rare)
1188 Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);
1189 __ if_then(src, BoolTest::eq, lastChar, unlikely); {
1190 __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {
1191 Node* tpj = __ AddI(targetOffset, __ value(j));
1192 Node* targ = load_array_element(no_ctrl, target, tpj, target_type);
1193 Node* ipj = __ AddI(__ value(i), __ value(j));
1194 Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);
1195 __ if_then(targ, BoolTest::ne, src2); {
1196 __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {
1197 __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {
1198 __ increment(i, __ AddI(__ value(j), one));
1199 __ goto_(outer_loop);
1200 } __ end_if(); __ dead(j);
1201 }__ end_if(); __ dead(j);
1202 __ increment(i, md2);
1203 __ goto_(outer_loop);
1204 }__ end_if();
1205 __ increment(j, one);
1206 }__ end_loop(); __ dead(j);
1207 __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);
1208 __ goto_(return_);
1209 }__ end_if();
1210 __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {
1211 __ increment(i, targetCountLess1);
1212 }__ end_if();
1213 __ increment(i, one);
1214 __ bind(outer_loop);
1215 }__ end_loop(); __ dead(i);
1216 __ bind(return_);
1218 // Final sync IdealKit and GraphKit.
1219 sync_kit(kit);
1220 Node* result = __ value(rtn);
1221 #undef __
1222 C->set_has_loops(true);
1223 return result;
1224 }
1226 //------------------------------inline_string_indexOf------------------------
1227 bool LibraryCallKit::inline_string_indexOf() {
1229 const int value_offset = java_lang_String::value_offset_in_bytes();
1230 const int count_offset = java_lang_String::count_offset_in_bytes();
1231 const int offset_offset = java_lang_String::offset_offset_in_bytes();
1233 _sp += 2;
1234 Node *argument = pop(); // pop non-receiver first: it was pushed second
1235 Node *receiver = pop();
1237 Node* result;
1238 // Disable the use of pcmpestri until it can be guaranteed that
1239 // the load doesn't cross into the uncommited space.
1240 if (Matcher::has_match_rule(Op_StrIndexOf) &&
1241 UseSSE42Intrinsics) {
1242 // Generate SSE4.2 version of indexOf
1243 // We currently only have match rules that use SSE4.2
1245 // Null check on self without removing any arguments. The argument
1246 // null check technically happens in the wrong place, which can lead to
1247 // invalid stack traces when string compare is inlined into a method
1248 // which handles NullPointerExceptions.
1249 _sp += 2;
1250 receiver = do_null_check(receiver, T_OBJECT);
1251 argument = do_null_check(argument, T_OBJECT);
1252 _sp -= 2;
1254 if (stopped()) {
1255 return true;
1256 }
1258 ciInstanceKlass* str_klass = env()->String_klass();
1259 const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);
1261 // Make the merge point
1262 RegionNode* result_rgn = new (C, 4) RegionNode(4);
1263 Node* result_phi = new (C, 4) PhiNode(result_rgn, TypeInt::INT);
1264 Node* no_ctrl = NULL;
1266 // Get counts for string and substr
1267 Node* source_cnta = basic_plus_adr(receiver, receiver, count_offset);
1268 Node* source_cnt = make_load(no_ctrl, source_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
1270 Node* substr_cnta = basic_plus_adr(argument, argument, count_offset);
1271 Node* substr_cnt = make_load(no_ctrl, substr_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));
1273 // Check for substr count > string count
1274 Node* cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, source_cnt) );
1275 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::gt) );
1276 Node* if_gt = generate_slow_guard(bol, NULL);
1277 if (if_gt != NULL) {
1278 result_phi->init_req(2, intcon(-1));
1279 result_rgn->init_req(2, if_gt);
1280 }
1282 if (!stopped()) {
1283 // Check for substr count == 0
1284 cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, intcon(0)) );
1285 bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
1286 Node* if_zero = generate_slow_guard(bol, NULL);
1287 if (if_zero != NULL) {
1288 result_phi->init_req(3, intcon(0));
1289 result_rgn->init_req(3, if_zero);
1290 }
1291 }
1293 if (!stopped()) {
1294 result = make_string_method_node(Op_StrIndexOf, receiver, source_cnt, argument, substr_cnt);
1295 result_phi->init_req(1, result);
1296 result_rgn->init_req(1, control());
1297 }
1298 set_control(_gvn.transform(result_rgn));
1299 record_for_igvn(result_rgn);
1300 result = _gvn.transform(result_phi);
1302 } else { // Use LibraryCallKit::string_indexOf
1303 // don't intrinsify if argument isn't a constant string.
1304 if (!argument->is_Con()) {
1305 return false;
1306 }
1307 const TypeOopPtr* str_type = _gvn.type(argument)->isa_oopptr();
1308 if (str_type == NULL) {
1309 return false;
1310 }
1311 ciInstanceKlass* klass = env()->String_klass();
1312 ciObject* str_const = str_type->const_oop();
1313 if (str_const == NULL || str_const->klass() != klass) {
1314 return false;
1315 }
1316 ciInstance* str = str_const->as_instance();
1317 assert(str != NULL, "must be instance");
1319 ciObject* v = str->field_value_by_offset(value_offset).as_object();
1320 int o = str->field_value_by_offset(offset_offset).as_int();
1321 int c = str->field_value_by_offset(count_offset).as_int();
1322 ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array
1324 // constant strings have no offset and count == length which
1325 // simplifies the resulting code somewhat so lets optimize for that.
1326 if (o != 0 || c != pat->length()) {
1327 return false;
1328 }
1330 // Null check on self without removing any arguments. The argument
1331 // null check technically happens in the wrong place, which can lead to
1332 // invalid stack traces when string compare is inlined into a method
1333 // which handles NullPointerExceptions.
1334 _sp += 2;
1335 receiver = do_null_check(receiver, T_OBJECT);
1336 // No null check on the argument is needed since it's a constant String oop.
1337 _sp -= 2;
1338 if (stopped()) {
1339 return true;
1340 }
1342 // The null string as a pattern always returns 0 (match at beginning of string)
1343 if (c == 0) {
1344 push(intcon(0));
1345 return true;
1346 }
1348 // Generate default indexOf
1349 jchar lastChar = pat->char_at(o + (c - 1));
1350 int cache = 0;
1351 int i;
1352 for (i = 0; i < c - 1; i++) {
1353 assert(i < pat->length(), "out of range");
1354 cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));
1355 }
1357 int md2 = c;
1358 for (i = 0; i < c - 1; i++) {
1359 assert(i < pat->length(), "out of range");
1360 if (pat->char_at(o + i) == lastChar) {
1361 md2 = (c - 1) - i;
1362 }
1363 }
1365 result = string_indexOf(receiver, pat, o, cache, md2);
1366 }
1368 push(result);
1369 return true;
1370 }
1372 //--------------------------pop_math_arg--------------------------------
1373 // Pop a double argument to a math function from the stack
1374 // rounding it if necessary.
1375 Node * LibraryCallKit::pop_math_arg() {
1376 Node *arg = pop_pair();
1377 if( Matcher::strict_fp_requires_explicit_rounding && UseSSE<=1 )
1378 arg = _gvn.transform( new (C, 2) RoundDoubleNode(0, arg) );
1379 return arg;
1380 }
1382 //------------------------------inline_trig----------------------------------
1383 // Inline sin/cos/tan instructions, if possible. If rounding is required, do
1384 // argument reduction which will turn into a fast/slow diamond.
1385 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {
1386 _sp += arg_size(); // restore stack pointer
1387 Node* arg = pop_math_arg();
1388 Node* trig = NULL;
1390 switch (id) {
1391 case vmIntrinsics::_dsin:
1392 trig = _gvn.transform((Node*)new (C, 2) SinDNode(arg));
1393 break;
1394 case vmIntrinsics::_dcos:
1395 trig = _gvn.transform((Node*)new (C, 2) CosDNode(arg));
1396 break;
1397 case vmIntrinsics::_dtan:
1398 trig = _gvn.transform((Node*)new (C, 2) TanDNode(arg));
1399 break;
1400 default:
1401 assert(false, "bad intrinsic was passed in");
1402 return false;
1403 }
1405 // Rounding required? Check for argument reduction!
1406 if( Matcher::strict_fp_requires_explicit_rounding ) {
1408 static const double pi_4 = 0.7853981633974483;
1409 static const double neg_pi_4 = -0.7853981633974483;
1410 // pi/2 in 80-bit extended precision
1411 // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};
1412 // -pi/2 in 80-bit extended precision
1413 // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};
1414 // Cutoff value for using this argument reduction technique
1415 //static const double pi_2_minus_epsilon = 1.564660403643354;
1416 //static const double neg_pi_2_plus_epsilon = -1.564660403643354;
1418 // Pseudocode for sin:
1419 // if (x <= Math.PI / 4.0) {
1420 // if (x >= -Math.PI / 4.0) return fsin(x);
1421 // if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);
1422 // } else {
1423 // if (x <= Math.PI / 2.0) return fcos(x - Math.PI / 2.0);
1424 // }
1425 // return StrictMath.sin(x);
1427 // Pseudocode for cos:
1428 // if (x <= Math.PI / 4.0) {
1429 // if (x >= -Math.PI / 4.0) return fcos(x);
1430 // if (x >= -Math.PI / 2.0) return fsin(x + Math.PI / 2.0);
1431 // } else {
1432 // if (x <= Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);
1433 // }
1434 // return StrictMath.cos(x);
1436 // Actually, sticking in an 80-bit Intel value into C2 will be tough; it
1437 // requires a special machine instruction to load it. Instead we'll try
1438 // the 'easy' case. If we really need the extra range +/- PI/2 we'll
1439 // probably do the math inside the SIN encoding.
1441 // Make the merge point
1442 RegionNode *r = new (C, 3) RegionNode(3);
1443 Node *phi = new (C, 3) PhiNode(r,Type::DOUBLE);
1445 // Flatten arg so we need only 1 test
1446 Node *abs = _gvn.transform(new (C, 2) AbsDNode(arg));
1447 // Node for PI/4 constant
1448 Node *pi4 = makecon(TypeD::make(pi_4));
1449 // Check PI/4 : abs(arg)
1450 Node *cmp = _gvn.transform(new (C, 3) CmpDNode(pi4,abs));
1451 // Check: If PI/4 < abs(arg) then go slow
1452 Node *bol = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::lt ) );
1453 // Branch either way
1454 IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
1455 set_control(opt_iff(r,iff));
1457 // Set fast path result
1458 phi->init_req(2,trig);
1460 // Slow path - non-blocking leaf call
1461 Node* call = NULL;
1462 switch (id) {
1463 case vmIntrinsics::_dsin:
1464 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1465 CAST_FROM_FN_PTR(address, SharedRuntime::dsin),
1466 "Sin", NULL, arg, top());
1467 break;
1468 case vmIntrinsics::_dcos:
1469 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1470 CAST_FROM_FN_PTR(address, SharedRuntime::dcos),
1471 "Cos", NULL, arg, top());
1472 break;
1473 case vmIntrinsics::_dtan:
1474 call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),
1475 CAST_FROM_FN_PTR(address, SharedRuntime::dtan),
1476 "Tan", NULL, arg, top());
1477 break;
1478 }
1479 assert(control()->in(0) == call, "");
1480 Node* slow_result = _gvn.transform(new (C, 1) ProjNode(call,TypeFunc::Parms));
1481 r->init_req(1,control());
1482 phi->init_req(1,slow_result);
1484 // Post-merge
1485 set_control(_gvn.transform(r));
1486 record_for_igvn(r);
1487 trig = _gvn.transform(phi);
1489 C->set_has_split_ifs(true); // Has chance for split-if optimization
1490 }
1491 // Push result back on JVM stack
1492 push_pair(trig);
1493 return true;
1494 }
1496 //------------------------------inline_sqrt-------------------------------------
1497 // Inline square root instruction, if possible.
1498 bool LibraryCallKit::inline_sqrt(vmIntrinsics::ID id) {
1499 assert(id == vmIntrinsics::_dsqrt, "Not square root");
1500 _sp += arg_size(); // restore stack pointer
1501 push_pair(_gvn.transform(new (C, 2) SqrtDNode(0, pop_math_arg())));
1502 return true;
1503 }
1505 //------------------------------inline_abs-------------------------------------
1506 // Inline absolute value instruction, if possible.
1507 bool LibraryCallKit::inline_abs(vmIntrinsics::ID id) {
1508 assert(id == vmIntrinsics::_dabs, "Not absolute value");
1509 _sp += arg_size(); // restore stack pointer
1510 push_pair(_gvn.transform(new (C, 2) AbsDNode(pop_math_arg())));
1511 return true;
1512 }
1514 //------------------------------inline_exp-------------------------------------
1515 // Inline exp instructions, if possible. The Intel hardware only misses
1516 // really odd corner cases (+/- Infinity). Just uncommon-trap them.
1517 bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) {
1518 assert(id == vmIntrinsics::_dexp, "Not exp");
1520 // If this inlining ever returned NaN in the past, we do not intrinsify it
1521 // every again. NaN results requires StrictMath.exp handling.
1522 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
1524 // Do not intrinsify on older platforms which lack cmove.
1525 if (ConditionalMoveLimit == 0) return false;
1527 _sp += arg_size(); // restore stack pointer
1528 Node *x = pop_math_arg();
1529 Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x));
1531 //-------------------
1532 //result=(result.isNaN())? StrictMath::exp():result;
1533 // Check: If isNaN() by checking result!=result? then go to Strict Math
1534 Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));
1535 // Build the boolean node
1536 Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );
1538 { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1539 // End the current control-flow path
1540 push_pair(x);
1541 // Math.exp intrinsic returned a NaN, which requires StrictMath.exp
1542 // to handle. Recompile without intrinsifying Math.exp
1543 uncommon_trap(Deoptimization::Reason_intrinsic,
1544 Deoptimization::Action_make_not_entrant);
1545 }
1547 C->set_has_split_ifs(true); // Has chance for split-if optimization
1549 push_pair(result);
1551 return true;
1552 }
1554 //------------------------------inline_pow-------------------------------------
1555 // Inline power instructions, if possible.
1556 bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {
1557 assert(id == vmIntrinsics::_dpow, "Not pow");
1559 // If this inlining ever returned NaN in the past, we do not intrinsify it
1560 // every again. NaN results requires StrictMath.pow handling.
1561 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
1563 // Do not intrinsify on older platforms which lack cmove.
1564 if (ConditionalMoveLimit == 0) return false;
1566 // Pseudocode for pow
1567 // if (x <= 0.0) {
1568 // if ((double)((int)y)==y) { // if y is int
1569 // result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y)
1570 // } else {
1571 // result = NaN;
1572 // }
1573 // } else {
1574 // result = DPow(x,y);
1575 // }
1576 // if (result != result)? {
1577 // uncommon_trap();
1578 // }
1579 // return result;
1581 _sp += arg_size(); // restore stack pointer
1582 Node* y = pop_math_arg();
1583 Node* x = pop_math_arg();
1585 Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) );
1587 // Short form: if not top-level (i.e., Math.pow but inlining Math.pow
1588 // inside of something) then skip the fancy tests and just check for
1589 // NaN result.
1590 Node *result = NULL;
1591 if( jvms()->depth() >= 1 ) {
1592 result = fast_result;
1593 } else {
1595 // Set the merge point for If node with condition of (x <= 0.0)
1596 // There are four possible paths to region node and phi node
1597 RegionNode *r = new (C, 4) RegionNode(4);
1598 Node *phi = new (C, 4) PhiNode(r, Type::DOUBLE);
1600 // Build the first if node: if (x <= 0.0)
1601 // Node for 0 constant
1602 Node *zeronode = makecon(TypeD::ZERO);
1603 // Check x:0
1604 Node *cmp = _gvn.transform(new (C, 3) CmpDNode(x, zeronode));
1605 // Check: If (x<=0) then go complex path
1606 Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) );
1607 // Branch either way
1608 IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1609 Node *opt_test = _gvn.transform(if1);
1610 //assert( opt_test->is_If(), "Expect an IfNode");
1611 IfNode *opt_if1 = (IfNode*)opt_test;
1612 // Fast path taken; set region slot 3
1613 Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) );
1614 r->init_req(3,fast_taken); // Capture fast-control
1616 // Fast path not-taken, i.e. slow path
1617 Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) );
1619 // Set fast path result
1620 Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) );
1621 phi->init_req(3, fast_result);
1623 // Complex path
1624 // Build the second if node (if y is int)
1625 // Node for (int)y
1626 Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y));
1627 // Node for (double)((int) y)
1628 Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty));
1629 // Check (double)((int) y) : y
1630 Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y));
1631 // Check if (y isn't int) then go to slow path
1633 Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) );
1634 // Branch either way
1635 IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);
1636 Node *slow_path = opt_iff(r,if2); // Set region path 2
1638 // Calculate DPow(abs(x), y)*(1 & (int)y)
1639 // Node for constant 1
1640 Node *conone = intcon(1);
1641 // 1& (int)y
1642 Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) );
1643 // zero node
1644 Node *conzero = intcon(0);
1645 // Check (1&(int)y)==0?
1646 Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero));
1647 // Check if (1&(int)y)!=0?, if so the result is negative
1648 Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) );
1649 // abs(x)
1650 Node *absx=_gvn.transform( new (C, 2) AbsDNode(x));
1651 // abs(x)^y
1652 Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) );
1653 // -abs(x)^y
1654 Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy));
1655 // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y)
1656 Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));
1657 // Set complex path fast result
1658 phi->init_req(2, signresult);
1660 static const jlong nan_bits = CONST64(0x7ff8000000000000);
1661 Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN
1662 r->init_req(1,slow_path);
1663 phi->init_req(1,slow_result);
1665 // Post merge
1666 set_control(_gvn.transform(r));
1667 record_for_igvn(r);
1668 result=_gvn.transform(phi);
1669 }
1671 //-------------------
1672 //result=(result.isNaN())? uncommon_trap():result;
1673 // Check: If isNaN() by checking result!=result? then go to Strict Math
1674 Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));
1675 // Build the boolean node
1676 Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );
1678 { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);
1679 // End the current control-flow path
1680 push_pair(x);
1681 push_pair(y);
1682 // Math.pow intrinsic returned a NaN, which requires StrictMath.pow
1683 // to handle. Recompile without intrinsifying Math.pow.
1684 uncommon_trap(Deoptimization::Reason_intrinsic,
1685 Deoptimization::Action_make_not_entrant);
1686 }
1688 C->set_has_split_ifs(true); // Has chance for split-if optimization
1690 push_pair(result);
1692 return true;
1693 }
1695 //------------------------------inline_trans-------------------------------------
1696 // Inline transcendental instructions, if possible. The Intel hardware gets
1697 // these right, no funny corner cases missed.
1698 bool LibraryCallKit::inline_trans(vmIntrinsics::ID id) {
1699 _sp += arg_size(); // restore stack pointer
1700 Node* arg = pop_math_arg();
1701 Node* trans = NULL;
1703 switch (id) {
1704 case vmIntrinsics::_dlog:
1705 trans = _gvn.transform((Node*)new (C, 2) LogDNode(arg));
1706 break;
1707 case vmIntrinsics::_dlog10:
1708 trans = _gvn.transform((Node*)new (C, 2) Log10DNode(arg));
1709 break;
1710 default:
1711 assert(false, "bad intrinsic was passed in");
1712 return false;
1713 }
1715 // Push result back on JVM stack
1716 push_pair(trans);
1717 return true;
1718 }
1720 //------------------------------runtime_math-----------------------------
1721 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {
1722 Node* a = NULL;
1723 Node* b = NULL;
1725 assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),
1726 "must be (DD)D or (D)D type");
1728 // Inputs
1729 _sp += arg_size(); // restore stack pointer
1730 if (call_type == OptoRuntime::Math_DD_D_Type()) {
1731 b = pop_math_arg();
1732 }
1733 a = pop_math_arg();
1735 const TypePtr* no_memory_effects = NULL;
1736 Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,
1737 no_memory_effects,
1738 a, top(), b, b ? top() : NULL);
1739 Node* value = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+0));
1740 #ifdef ASSERT
1741 Node* value_top = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+1));
1742 assert(value_top == top(), "second value must be top");
1743 #endif
1745 push_pair(value);
1746 return true;
1747 }
1749 //------------------------------inline_math_native-----------------------------
1750 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {
1751 switch (id) {
1752 // These intrinsics are not properly supported on all hardware
1753 case vmIntrinsics::_dcos: return Matcher::has_match_rule(Op_CosD) ? inline_trig(id) :
1754 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dcos), "COS");
1755 case vmIntrinsics::_dsin: return Matcher::has_match_rule(Op_SinD) ? inline_trig(id) :
1756 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dsin), "SIN");
1757 case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) :
1758 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dtan), "TAN");
1760 case vmIntrinsics::_dlog: return Matcher::has_match_rule(Op_LogD) ? inline_trans(id) :
1761 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog), "LOG");
1762 case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_trans(id) :
1763 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), "LOG10");
1765 // These intrinsics are supported on all hardware
1766 case vmIntrinsics::_dsqrt: return Matcher::has_match_rule(Op_SqrtD) ? inline_sqrt(id) : false;
1767 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_abs(id) : false;
1769 // These intrinsics don't work on X86. The ad implementation doesn't
1770 // handle NaN's properly. Instead of returning infinity, the ad
1771 // implementation returns a NaN on overflow. See bug: 6304089
1772 // Once the ad implementations are fixed, change the code below
1773 // to match the intrinsics above
1775 case vmIntrinsics::_dexp: return
1776 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");
1777 case vmIntrinsics::_dpow: return
1778 runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");
1780 // These intrinsics are not yet correctly implemented
1781 case vmIntrinsics::_datan2:
1782 return false;
1784 default:
1785 ShouldNotReachHere();
1786 return false;
1787 }
1788 }
1790 static bool is_simple_name(Node* n) {
1791 return (n->req() == 1 // constant
1792 || (n->is_Type() && n->as_Type()->type()->singleton())
1793 || n->is_Proj() // parameter or return value
1794 || n->is_Phi() // local of some sort
1795 );
1796 }
1798 //----------------------------inline_min_max-----------------------------------
1799 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {
1800 push(generate_min_max(id, argument(0), argument(1)));
1802 return true;
1803 }
1805 Node*
1806 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {
1807 // These are the candidate return value:
1808 Node* xvalue = x0;
1809 Node* yvalue = y0;
1811 if (xvalue == yvalue) {
1812 return xvalue;
1813 }
1815 bool want_max = (id == vmIntrinsics::_max);
1817 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();
1818 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();
1819 if (txvalue == NULL || tyvalue == NULL) return top();
1820 // This is not really necessary, but it is consistent with a
1821 // hypothetical MaxINode::Value method:
1822 int widen = MAX2(txvalue->_widen, tyvalue->_widen);
1824 // %%% This folding logic should (ideally) be in a different place.
1825 // Some should be inside IfNode, and there to be a more reliable
1826 // transformation of ?: style patterns into cmoves. We also want
1827 // more powerful optimizations around cmove and min/max.
1829 // Try to find a dominating comparison of these guys.
1830 // It can simplify the index computation for Arrays.copyOf
1831 // and similar uses of System.arraycopy.
1832 // First, compute the normalized version of CmpI(x, y).
1833 int cmp_op = Op_CmpI;
1834 Node* xkey = xvalue;
1835 Node* ykey = yvalue;
1836 Node* ideal_cmpxy = _gvn.transform( new(C, 3) CmpINode(xkey, ykey) );
1837 if (ideal_cmpxy->is_Cmp()) {
1838 // E.g., if we have CmpI(length - offset, count),
1839 // it might idealize to CmpI(length, count + offset)
1840 cmp_op = ideal_cmpxy->Opcode();
1841 xkey = ideal_cmpxy->in(1);
1842 ykey = ideal_cmpxy->in(2);
1843 }
1845 // Start by locating any relevant comparisons.
1846 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;
1847 Node* cmpxy = NULL;
1848 Node* cmpyx = NULL;
1849 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {
1850 Node* cmp = start_from->fast_out(k);
1851 if (cmp->outcnt() > 0 && // must have prior uses
1852 cmp->in(0) == NULL && // must be context-independent
1853 cmp->Opcode() == cmp_op) { // right kind of compare
1854 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp;
1855 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp;
1856 }
1857 }
1859 const int NCMPS = 2;
1860 Node* cmps[NCMPS] = { cmpxy, cmpyx };
1861 int cmpn;
1862 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1863 if (cmps[cmpn] != NULL) break; // find a result
1864 }
1865 if (cmpn < NCMPS) {
1866 // Look for a dominating test that tells us the min and max.
1867 int depth = 0; // Limit search depth for speed
1868 Node* dom = control();
1869 for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {
1870 if (++depth >= 100) break;
1871 Node* ifproj = dom;
1872 if (!ifproj->is_Proj()) continue;
1873 Node* iff = ifproj->in(0);
1874 if (!iff->is_If()) continue;
1875 Node* bol = iff->in(1);
1876 if (!bol->is_Bool()) continue;
1877 Node* cmp = bol->in(1);
1878 if (cmp == NULL) continue;
1879 for (cmpn = 0; cmpn < NCMPS; cmpn++)
1880 if (cmps[cmpn] == cmp) break;
1881 if (cmpn == NCMPS) continue;
1882 BoolTest::mask btest = bol->as_Bool()->_test._test;
1883 if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate();
1884 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
1885 // At this point, we know that 'x btest y' is true.
1886 switch (btest) {
1887 case BoolTest::eq:
1888 // They are proven equal, so we can collapse the min/max.
1889 // Either value is the answer. Choose the simpler.
1890 if (is_simple_name(yvalue) && !is_simple_name(xvalue))
1891 return yvalue;
1892 return xvalue;
1893 case BoolTest::lt: // x < y
1894 case BoolTest::le: // x <= y
1895 return (want_max ? yvalue : xvalue);
1896 case BoolTest::gt: // x > y
1897 case BoolTest::ge: // x >= y
1898 return (want_max ? xvalue : yvalue);
1899 }
1900 }
1901 }
1903 // We failed to find a dominating test.
1904 // Let's pick a test that might GVN with prior tests.
1905 Node* best_bol = NULL;
1906 BoolTest::mask best_btest = BoolTest::illegal;
1907 for (cmpn = 0; cmpn < NCMPS; cmpn++) {
1908 Node* cmp = cmps[cmpn];
1909 if (cmp == NULL) continue;
1910 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {
1911 Node* bol = cmp->fast_out(j);
1912 if (!bol->is_Bool()) continue;
1913 BoolTest::mask btest = bol->as_Bool()->_test._test;
1914 if (btest == BoolTest::eq || btest == BoolTest::ne) continue;
1915 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute();
1916 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {
1917 best_bol = bol->as_Bool();
1918 best_btest = btest;
1919 }
1920 }
1921 }
1923 Node* answer_if_true = NULL;
1924 Node* answer_if_false = NULL;
1925 switch (best_btest) {
1926 default:
1927 if (cmpxy == NULL)
1928 cmpxy = ideal_cmpxy;
1929 best_bol = _gvn.transform( new(C, 2) BoolNode(cmpxy, BoolTest::lt) );
1930 // and fall through:
1931 case BoolTest::lt: // x < y
1932 case BoolTest::le: // x <= y
1933 answer_if_true = (want_max ? yvalue : xvalue);
1934 answer_if_false = (want_max ? xvalue : yvalue);
1935 break;
1936 case BoolTest::gt: // x > y
1937 case BoolTest::ge: // x >= y
1938 answer_if_true = (want_max ? xvalue : yvalue);
1939 answer_if_false = (want_max ? yvalue : xvalue);
1940 break;
1941 }
1943 jint hi, lo;
1944 if (want_max) {
1945 // We can sharpen the minimum.
1946 hi = MAX2(txvalue->_hi, tyvalue->_hi);
1947 lo = MAX2(txvalue->_lo, tyvalue->_lo);
1948 } else {
1949 // We can sharpen the maximum.
1950 hi = MIN2(txvalue->_hi, tyvalue->_hi);
1951 lo = MIN2(txvalue->_lo, tyvalue->_lo);
1952 }
1954 // Use a flow-free graph structure, to avoid creating excess control edges
1955 // which could hinder other optimizations.
1956 // Since Math.min/max is often used with arraycopy, we want
1957 // tightly_coupled_allocation to be able to see beyond min/max expressions.
1958 Node* cmov = CMoveNode::make(C, NULL, best_bol,
1959 answer_if_false, answer_if_true,
1960 TypeInt::make(lo, hi, widen));
1962 return _gvn.transform(cmov);
1964 /*
1965 // This is not as desirable as it may seem, since Min and Max
1966 // nodes do not have a full set of optimizations.
1967 // And they would interfere, anyway, with 'if' optimizations
1968 // and with CMoveI canonical forms.
1969 switch (id) {
1970 case vmIntrinsics::_min:
1971 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;
1972 case vmIntrinsics::_max:
1973 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;
1974 default:
1975 ShouldNotReachHere();
1976 }
1977 */
1978 }
1980 inline int
1981 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {
1982 const TypePtr* base_type = TypePtr::NULL_PTR;
1983 if (base != NULL) base_type = _gvn.type(base)->isa_ptr();
1984 if (base_type == NULL) {
1985 // Unknown type.
1986 return Type::AnyPtr;
1987 } else if (base_type == TypePtr::NULL_PTR) {
1988 // Since this is a NULL+long form, we have to switch to a rawptr.
1989 base = _gvn.transform( new (C, 2) CastX2PNode(offset) );
1990 offset = MakeConX(0);
1991 return Type::RawPtr;
1992 } else if (base_type->base() == Type::RawPtr) {
1993 return Type::RawPtr;
1994 } else if (base_type->isa_oopptr()) {
1995 // Base is never null => always a heap address.
1996 if (base_type->ptr() == TypePtr::NotNull) {
1997 return Type::OopPtr;
1998 }
1999 // Offset is small => always a heap address.
2000 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();
2001 if (offset_type != NULL &&
2002 base_type->offset() == 0 && // (should always be?)
2003 offset_type->_lo >= 0 &&
2004 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {
2005 return Type::OopPtr;
2006 }
2007 // Otherwise, it might either be oop+off or NULL+addr.
2008 return Type::AnyPtr;
2009 } else {
2010 // No information:
2011 return Type::AnyPtr;
2012 }
2013 }
2015 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {
2016 int kind = classify_unsafe_addr(base, offset);
2017 if (kind == Type::RawPtr) {
2018 return basic_plus_adr(top(), base, offset);
2019 } else {
2020 return basic_plus_adr(base, offset);
2021 }
2022 }
2024 //-------------------inline_numberOfLeadingZeros_int/long-----------------------
2025 // inline int Integer.numberOfLeadingZeros(int)
2026 // inline int Long.numberOfLeadingZeros(long)
2027 bool LibraryCallKit::inline_numberOfLeadingZeros(vmIntrinsics::ID id) {
2028 assert(id == vmIntrinsics::_numberOfLeadingZeros_i || id == vmIntrinsics::_numberOfLeadingZeros_l, "not numberOfLeadingZeros");
2029 if (id == vmIntrinsics::_numberOfLeadingZeros_i && !Matcher::match_rule_supported(Op_CountLeadingZerosI)) return false;
2030 if (id == vmIntrinsics::_numberOfLeadingZeros_l && !Matcher::match_rule_supported(Op_CountLeadingZerosL)) return false;
2031 _sp += arg_size(); // restore stack pointer
2032 switch (id) {
2033 case vmIntrinsics::_numberOfLeadingZeros_i:
2034 push(_gvn.transform(new (C, 2) CountLeadingZerosINode(pop())));
2035 break;
2036 case vmIntrinsics::_numberOfLeadingZeros_l:
2037 push(_gvn.transform(new (C, 2) CountLeadingZerosLNode(pop_pair())));
2038 break;
2039 default:
2040 ShouldNotReachHere();
2041 }
2042 return true;
2043 }
2045 //-------------------inline_numberOfTrailingZeros_int/long----------------------
2046 // inline int Integer.numberOfTrailingZeros(int)
2047 // inline int Long.numberOfTrailingZeros(long)
2048 bool LibraryCallKit::inline_numberOfTrailingZeros(vmIntrinsics::ID id) {
2049 assert(id == vmIntrinsics::_numberOfTrailingZeros_i || id == vmIntrinsics::_numberOfTrailingZeros_l, "not numberOfTrailingZeros");
2050 if (id == vmIntrinsics::_numberOfTrailingZeros_i && !Matcher::match_rule_supported(Op_CountTrailingZerosI)) return false;
2051 if (id == vmIntrinsics::_numberOfTrailingZeros_l && !Matcher::match_rule_supported(Op_CountTrailingZerosL)) return false;
2052 _sp += arg_size(); // restore stack pointer
2053 switch (id) {
2054 case vmIntrinsics::_numberOfTrailingZeros_i:
2055 push(_gvn.transform(new (C, 2) CountTrailingZerosINode(pop())));
2056 break;
2057 case vmIntrinsics::_numberOfTrailingZeros_l:
2058 push(_gvn.transform(new (C, 2) CountTrailingZerosLNode(pop_pair())));
2059 break;
2060 default:
2061 ShouldNotReachHere();
2062 }
2063 return true;
2064 }
2066 //----------------------------inline_bitCount_int/long-----------------------
2067 // inline int Integer.bitCount(int)
2068 // inline int Long.bitCount(long)
2069 bool LibraryCallKit::inline_bitCount(vmIntrinsics::ID id) {
2070 assert(id == vmIntrinsics::_bitCount_i || id == vmIntrinsics::_bitCount_l, "not bitCount");
2071 if (id == vmIntrinsics::_bitCount_i && !Matcher::has_match_rule(Op_PopCountI)) return false;
2072 if (id == vmIntrinsics::_bitCount_l && !Matcher::has_match_rule(Op_PopCountL)) return false;
2073 _sp += arg_size(); // restore stack pointer
2074 switch (id) {
2075 case vmIntrinsics::_bitCount_i:
2076 push(_gvn.transform(new (C, 2) PopCountINode(pop())));
2077 break;
2078 case vmIntrinsics::_bitCount_l:
2079 push(_gvn.transform(new (C, 2) PopCountLNode(pop_pair())));
2080 break;
2081 default:
2082 ShouldNotReachHere();
2083 }
2084 return true;
2085 }
2087 //----------------------------inline_reverseBytes_int/long/char/short-------------------
2088 // inline Integer.reverseBytes(int)
2089 // inline Long.reverseBytes(long)
2090 // inline Character.reverseBytes(char)
2091 // inline Short.reverseBytes(short)
2092 bool LibraryCallKit::inline_reverseBytes(vmIntrinsics::ID id) {
2093 assert(id == vmIntrinsics::_reverseBytes_i || id == vmIntrinsics::_reverseBytes_l ||
2094 id == vmIntrinsics::_reverseBytes_c || id == vmIntrinsics::_reverseBytes_s,
2095 "not reverse Bytes");
2096 if (id == vmIntrinsics::_reverseBytes_i && !Matcher::has_match_rule(Op_ReverseBytesI)) return false;
2097 if (id == vmIntrinsics::_reverseBytes_l && !Matcher::has_match_rule(Op_ReverseBytesL)) return false;
2098 if (id == vmIntrinsics::_reverseBytes_c && !Matcher::has_match_rule(Op_ReverseBytesUS)) return false;
2099 if (id == vmIntrinsics::_reverseBytes_s && !Matcher::has_match_rule(Op_ReverseBytesS)) return false;
2100 _sp += arg_size(); // restore stack pointer
2101 switch (id) {
2102 case vmIntrinsics::_reverseBytes_i:
2103 push(_gvn.transform(new (C, 2) ReverseBytesINode(0, pop())));
2104 break;
2105 case vmIntrinsics::_reverseBytes_l:
2106 push_pair(_gvn.transform(new (C, 2) ReverseBytesLNode(0, pop_pair())));
2107 break;
2108 case vmIntrinsics::_reverseBytes_c:
2109 push(_gvn.transform(new (C, 2) ReverseBytesUSNode(0, pop())));
2110 break;
2111 case vmIntrinsics::_reverseBytes_s:
2112 push(_gvn.transform(new (C, 2) ReverseBytesSNode(0, pop())));
2113 break;
2114 default:
2115 ;
2116 }
2117 return true;
2118 }
2120 //----------------------------inline_unsafe_access----------------------------
2122 const static BasicType T_ADDRESS_HOLDER = T_LONG;
2124 // Helper that guards and inserts a G1 pre-barrier.
2125 void LibraryCallKit::insert_g1_pre_barrier(Node* base_oop, Node* offset, Node* pre_val) {
2126 assert(UseG1GC, "should not call this otherwise");
2128 // We could be accessing the referent field of a reference object. If so, when G1
2129 // is enabled, we need to log the value in the referent field in an SATB buffer.
2130 // This routine performs some compile time filters and generates suitable
2131 // runtime filters that guard the pre-barrier code.
2133 // Some compile time checks.
2135 // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
2136 const TypeX* otype = offset->find_intptr_t_type();
2137 if (otype != NULL && otype->is_con() &&
2138 otype->get_con() != java_lang_ref_Reference::referent_offset) {
2139 // Constant offset but not the reference_offset so just return
2140 return;
2141 }
2143 // We only need to generate the runtime guards for instances.
2144 const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();
2145 if (btype != NULL) {
2146 if (btype->isa_aryptr()) {
2147 // Array type so nothing to do
2148 return;
2149 }
2151 const TypeInstPtr* itype = btype->isa_instptr();
2152 if (itype != NULL) {
2153 // Can the klass of base_oop be statically determined
2154 // to be _not_ a sub-class of Reference?
2155 ciKlass* klass = itype->klass();
2156 if (klass->is_subtype_of(env()->Reference_klass()) &&
2157 !env()->Reference_klass()->is_subtype_of(klass)) {
2158 return;
2159 }
2160 }
2161 }
2163 // The compile time filters did not reject base_oop/offset so
2164 // we need to generate the following runtime filters
2165 //
2166 // if (offset == java_lang_ref_Reference::_reference_offset) {
2167 // if (base != null) {
2168 // if (klass(base)->reference_type() != REF_NONE)) {
2169 // pre_barrier(_, pre_val, ...);
2170 // }
2171 // }
2172 // }
2174 float likely = PROB_LIKELY(0.999);
2175 float unlikely = PROB_UNLIKELY(0.999);
2177 IdealKit ideal(gvn(), control(), merged_memory());
2178 #define __ ideal.
2180 const int reference_type_offset = instanceKlass::reference_type_offset_in_bytes() +
2181 sizeof(oopDesc);
2183 Node* referent_off = __ ConI(java_lang_ref_Reference::referent_offset);
2185 __ if_then(offset, BoolTest::eq, referent_off, unlikely); {
2186 __ if_then(base_oop, BoolTest::ne, null(), likely); {
2188 // Update graphKit memory and control from IdealKit.
2189 set_all_memory(__ merged_memory());
2190 set_control(__ ctrl());
2192 Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));
2193 Node* is_instof = gen_instanceof(base_oop, ref_klass_con);
2195 // Update IdealKit memory and control from graphKit.
2196 __ set_all_memory(merged_memory());
2197 __ set_ctrl(control());
2199 Node* one = __ ConI(1);
2201 __ if_then(is_instof, BoolTest::eq, one, unlikely); {
2203 // Update graphKit from IdeakKit.
2204 set_all_memory(__ merged_memory());
2205 set_control(__ ctrl());
2207 // Use the pre-barrier to record the value in the referent field
2208 pre_barrier(false /* do_load */,
2209 __ ctrl(),
2210 NULL /* obj */, NULL /* adr */, -1 /* alias_idx */, NULL /* val */, NULL /* val_type */,
2211 pre_val /* pre_val */,
2212 T_OBJECT);
2214 // Update IdealKit from graphKit.
2215 __ set_all_memory(merged_memory());
2216 __ set_ctrl(control());
2218 } __ end_if(); // _ref_type != ref_none
2219 } __ end_if(); // base != NULL
2220 } __ end_if(); // offset == referent_offset
2222 // Final sync IdealKit and GraphKit.
2223 sync_kit(ideal);
2224 #undef __
2225 }
2228 // Interpret Unsafe.fieldOffset cookies correctly:
2229 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);
2231 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) {
2232 if (callee()->is_static()) return false; // caller must have the capability!
2234 #ifndef PRODUCT
2235 {
2236 ResourceMark rm;
2237 // Check the signatures.
2238 ciSignature* sig = signature();
2239 #ifdef ASSERT
2240 if (!is_store) {
2241 // Object getObject(Object base, int/long offset), etc.
2242 BasicType rtype = sig->return_type()->basic_type();
2243 if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())
2244 rtype = T_ADDRESS; // it is really a C void*
2245 assert(rtype == type, "getter must return the expected value");
2246 if (!is_native_ptr) {
2247 assert(sig->count() == 2, "oop getter has 2 arguments");
2248 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");
2249 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");
2250 } else {
2251 assert(sig->count() == 1, "native getter has 1 argument");
2252 assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");
2253 }
2254 } else {
2255 // void putObject(Object base, int/long offset, Object x), etc.
2256 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");
2257 if (!is_native_ptr) {
2258 assert(sig->count() == 3, "oop putter has 3 arguments");
2259 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");
2260 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");
2261 } else {
2262 assert(sig->count() == 2, "native putter has 2 arguments");
2263 assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");
2264 }
2265 BasicType vtype = sig->type_at(sig->count()-1)->basic_type();
2266 if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())
2267 vtype = T_ADDRESS; // it is really a C void*
2268 assert(vtype == type, "putter must accept the expected value");
2269 }
2270 #endif // ASSERT
2271 }
2272 #endif //PRODUCT
2274 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2276 int type_words = type2size[ (type == T_ADDRESS) ? T_LONG : type ];
2278 // Argument words: "this" plus (oop/offset) or (lo/hi) args plus maybe 1 or 2 value words
2279 int nargs = 1 + (is_native_ptr ? 2 : 3) + (is_store ? type_words : 0);
2281 debug_only(int saved_sp = _sp);
2282 _sp += nargs;
2284 Node* val;
2285 debug_only(val = (Node*)(uintptr_t)-1);
2288 if (is_store) {
2289 // Get the value being stored. (Pop it first; it was pushed last.)
2290 switch (type) {
2291 case T_DOUBLE:
2292 case T_LONG:
2293 case T_ADDRESS:
2294 val = pop_pair();
2295 break;
2296 default:
2297 val = pop();
2298 }
2299 }
2301 // Build address expression. See the code in inline_unsafe_prefetch.
2302 Node *adr;
2303 Node *heap_base_oop = top();
2304 Node* offset = top();
2306 if (!is_native_ptr) {
2307 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2308 offset = pop_pair();
2309 // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2310 Node* base = pop();
2311 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2312 // to be plain byte offsets, which are also the same as those accepted
2313 // by oopDesc::field_base.
2314 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2315 "fieldOffset must be byte-scaled");
2316 // 32-bit machines ignore the high half!
2317 offset = ConvL2X(offset);
2318 adr = make_unsafe_address(base, offset);
2319 heap_base_oop = base;
2320 } else {
2321 Node* ptr = pop_pair();
2322 // Adjust Java long to machine word:
2323 ptr = ConvL2X(ptr);
2324 adr = make_unsafe_address(NULL, ptr);
2325 }
2327 // Pop receiver last: it was pushed first.
2328 Node *receiver = pop();
2330 assert(saved_sp == _sp, "must have correct argument count");
2332 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2334 // First guess at the value type.
2335 const Type *value_type = Type::get_const_basic_type(type);
2337 // Try to categorize the address. If it comes up as TypeJavaPtr::BOTTOM,
2338 // there was not enough information to nail it down.
2339 Compile::AliasType* alias_type = C->alias_type(adr_type);
2340 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2342 // We will need memory barriers unless we can determine a unique
2343 // alias category for this reference. (Note: If for some reason
2344 // the barriers get omitted and the unsafe reference begins to "pollute"
2345 // the alias analysis of the rest of the graph, either Compile::can_alias
2346 // or Compile::must_alias will throw a diagnostic assert.)
2347 bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);
2349 // If we are reading the value of the referent field of a Reference
2350 // object (either by using Unsafe directly or through reflection)
2351 // then, if G1 is enabled, we need to record the referent in an
2352 // SATB log buffer using the pre-barrier mechanism.
2353 bool need_read_barrier = UseG1GC && !is_native_ptr && !is_store &&
2354 offset != top() && heap_base_oop != top();
2356 if (!is_store && type == T_OBJECT) {
2357 // Attempt to infer a sharper value type from the offset and base type.
2358 ciKlass* sharpened_klass = NULL;
2360 // See if it is an instance field, with an object type.
2361 if (alias_type->field() != NULL) {
2362 assert(!is_native_ptr, "native pointer op cannot use a java address");
2363 if (alias_type->field()->type()->is_klass()) {
2364 sharpened_klass = alias_type->field()->type()->as_klass();
2365 }
2366 }
2368 // See if it is a narrow oop array.
2369 if (adr_type->isa_aryptr()) {
2370 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {
2371 const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();
2372 if (elem_type != NULL) {
2373 sharpened_klass = elem_type->klass();
2374 }
2375 }
2376 }
2378 if (sharpened_klass != NULL) {
2379 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);
2381 // Sharpen the value type.
2382 value_type = tjp;
2384 #ifndef PRODUCT
2385 if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {
2386 tty->print(" from base type: "); adr_type->dump();
2387 tty->print(" sharpened value: "); value_type->dump();
2388 }
2389 #endif
2390 }
2391 }
2393 // Null check on self without removing any arguments. The argument
2394 // null check technically happens in the wrong place, which can lead to
2395 // invalid stack traces when the primitive is inlined into a method
2396 // which handles NullPointerExceptions.
2397 _sp += nargs;
2398 do_null_check(receiver, T_OBJECT);
2399 _sp -= nargs;
2400 if (stopped()) {
2401 return true;
2402 }
2403 // Heap pointers get a null-check from the interpreter,
2404 // as a courtesy. However, this is not guaranteed by Unsafe,
2405 // and it is not possible to fully distinguish unintended nulls
2406 // from intended ones in this API.
2408 if (is_volatile) {
2409 // We need to emit leading and trailing CPU membars (see below) in
2410 // addition to memory membars when is_volatile. This is a little
2411 // too strong, but avoids the need to insert per-alias-type
2412 // volatile membars (for stores; compare Parse::do_put_xxx), which
2413 // we cannot do effectively here because we probably only have a
2414 // rough approximation of type.
2415 need_mem_bar = true;
2416 // For Stores, place a memory ordering barrier now.
2417 if (is_store)
2418 insert_mem_bar(Op_MemBarRelease);
2419 }
2421 // Memory barrier to prevent normal and 'unsafe' accesses from
2422 // bypassing each other. Happens after null checks, so the
2423 // exception paths do not take memory state from the memory barrier,
2424 // so there's no problems making a strong assert about mixing users
2425 // of safe & unsafe memory. Otherwise fails in a CTW of rt.jar
2426 // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.
2427 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2429 if (!is_store) {
2430 Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile);
2431 // load value and push onto stack
2432 switch (type) {
2433 case T_BOOLEAN:
2434 case T_CHAR:
2435 case T_BYTE:
2436 case T_SHORT:
2437 case T_INT:
2438 case T_FLOAT:
2439 push(p);
2440 break;
2441 case T_OBJECT:
2442 if (need_read_barrier) {
2443 insert_g1_pre_barrier(heap_base_oop, offset, p);
2444 }
2445 push(p);
2446 break;
2447 case T_ADDRESS:
2448 // Cast to an int type.
2449 p = _gvn.transform( new (C, 2) CastP2XNode(NULL,p) );
2450 p = ConvX2L(p);
2451 push_pair(p);
2452 break;
2453 case T_DOUBLE:
2454 case T_LONG:
2455 push_pair( p );
2456 break;
2457 default: ShouldNotReachHere();
2458 }
2459 } else {
2460 // place effect of store into memory
2461 switch (type) {
2462 case T_DOUBLE:
2463 val = dstore_rounding(val);
2464 break;
2465 case T_ADDRESS:
2466 // Repackage the long as a pointer.
2467 val = ConvL2X(val);
2468 val = _gvn.transform( new (C, 2) CastX2PNode(val) );
2469 break;
2470 }
2472 if (type != T_OBJECT ) {
2473 (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);
2474 } else {
2475 // Possibly an oop being stored to Java heap or native memory
2476 if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {
2477 // oop to Java heap.
2478 (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2479 } else {
2480 // We can't tell at compile time if we are storing in the Java heap or outside
2481 // of it. So we need to emit code to conditionally do the proper type of
2482 // store.
2484 IdealKit ideal(gvn(), control(), merged_memory());
2485 #define __ ideal.
2486 // QQQ who knows what probability is here??
2487 __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {
2488 // Sync IdealKit and graphKit.
2489 set_all_memory( __ merged_memory());
2490 set_control(__ ctrl());
2491 Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);
2492 // Update IdealKit memory.
2493 __ set_all_memory(merged_memory());
2494 __ set_ctrl(control());
2495 } __ else_(); {
2496 __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile);
2497 } __ end_if();
2498 // Final sync IdealKit and GraphKit.
2499 sync_kit(ideal);
2500 #undef __
2501 }
2502 }
2503 }
2505 if (is_volatile) {
2506 if (!is_store)
2507 insert_mem_bar(Op_MemBarAcquire);
2508 else
2509 insert_mem_bar(Op_MemBarVolatile);
2510 }
2512 if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);
2514 return true;
2515 }
2517 //----------------------------inline_unsafe_prefetch----------------------------
2519 bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {
2520 #ifndef PRODUCT
2521 {
2522 ResourceMark rm;
2523 // Check the signatures.
2524 ciSignature* sig = signature();
2525 #ifdef ASSERT
2526 // Object getObject(Object base, int/long offset), etc.
2527 BasicType rtype = sig->return_type()->basic_type();
2528 if (!is_native_ptr) {
2529 assert(sig->count() == 2, "oop prefetch has 2 arguments");
2530 assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");
2531 assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");
2532 } else {
2533 assert(sig->count() == 1, "native prefetch has 1 argument");
2534 assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");
2535 }
2536 #endif // ASSERT
2537 }
2538 #endif // !PRODUCT
2540 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2542 // Argument words: "this" if not static, plus (oop/offset) or (lo/hi) args
2543 int nargs = (is_static ? 0 : 1) + (is_native_ptr ? 2 : 3);
2545 debug_only(int saved_sp = _sp);
2546 _sp += nargs;
2548 // Build address expression. See the code in inline_unsafe_access.
2549 Node *adr;
2550 if (!is_native_ptr) {
2551 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset
2552 Node* offset = pop_pair();
2553 // The base is either a Java object or a value produced by Unsafe.staticFieldBase
2554 Node* base = pop();
2555 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2556 // to be plain byte offsets, which are also the same as those accepted
2557 // by oopDesc::field_base.
2558 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
2559 "fieldOffset must be byte-scaled");
2560 // 32-bit machines ignore the high half!
2561 offset = ConvL2X(offset);
2562 adr = make_unsafe_address(base, offset);
2563 } else {
2564 Node* ptr = pop_pair();
2565 // Adjust Java long to machine word:
2566 ptr = ConvL2X(ptr);
2567 adr = make_unsafe_address(NULL, ptr);
2568 }
2570 if (is_static) {
2571 assert(saved_sp == _sp, "must have correct argument count");
2572 } else {
2573 // Pop receiver last: it was pushed first.
2574 Node *receiver = pop();
2575 assert(saved_sp == _sp, "must have correct argument count");
2577 // Null check on self without removing any arguments. The argument
2578 // null check technically happens in the wrong place, which can lead to
2579 // invalid stack traces when the primitive is inlined into a method
2580 // which handles NullPointerExceptions.
2581 _sp += nargs;
2582 do_null_check(receiver, T_OBJECT);
2583 _sp -= nargs;
2584 if (stopped()) {
2585 return true;
2586 }
2587 }
2589 // Generate the read or write prefetch
2590 Node *prefetch;
2591 if (is_store) {
2592 prefetch = new (C, 3) PrefetchWriteNode(i_o(), adr);
2593 } else {
2594 prefetch = new (C, 3) PrefetchReadNode(i_o(), adr);
2595 }
2596 prefetch->init_req(0, control());
2597 set_i_o(_gvn.transform(prefetch));
2599 return true;
2600 }
2602 //----------------------------inline_unsafe_CAS----------------------------
2604 bool LibraryCallKit::inline_unsafe_CAS(BasicType type) {
2605 // This basic scheme here is the same as inline_unsafe_access, but
2606 // differs in enough details that combining them would make the code
2607 // overly confusing. (This is a true fact! I originally combined
2608 // them, but even I was confused by it!) As much code/comments as
2609 // possible are retained from inline_unsafe_access though to make
2610 // the correspondences clearer. - dl
2612 if (callee()->is_static()) return false; // caller must have the capability!
2614 #ifndef PRODUCT
2615 {
2616 ResourceMark rm;
2617 // Check the signatures.
2618 ciSignature* sig = signature();
2619 #ifdef ASSERT
2620 BasicType rtype = sig->return_type()->basic_type();
2621 assert(rtype == T_BOOLEAN, "CAS must return boolean");
2622 assert(sig->count() == 4, "CAS has 4 arguments");
2623 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");
2624 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");
2625 #endif // ASSERT
2626 }
2627 #endif //PRODUCT
2629 // number of stack slots per value argument (1 or 2)
2630 int type_words = type2size[type];
2632 // Cannot inline wide CAS on machines that don't support it natively
2633 if (type2aelembytes(type) > BytesPerInt && !VM_Version::supports_cx8())
2634 return false;
2636 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2638 // Argument words: "this" plus oop plus offset plus oldvalue plus newvalue;
2639 int nargs = 1 + 1 + 2 + type_words + type_words;
2641 // pop arguments: newval, oldval, offset, base, and receiver
2642 debug_only(int saved_sp = _sp);
2643 _sp += nargs;
2644 Node* newval = (type_words == 1) ? pop() : pop_pair();
2645 Node* oldval = (type_words == 1) ? pop() : pop_pair();
2646 Node *offset = pop_pair();
2647 Node *base = pop();
2648 Node *receiver = pop();
2649 assert(saved_sp == _sp, "must have correct argument count");
2651 // Null check receiver.
2652 _sp += nargs;
2653 do_null_check(receiver, T_OBJECT);
2654 _sp -= nargs;
2655 if (stopped()) {
2656 return true;
2657 }
2659 // Build field offset expression.
2660 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset
2661 // to be plain byte offsets, which are also the same as those accepted
2662 // by oopDesc::field_base.
2663 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2664 // 32-bit machines ignore the high half of long offsets
2665 offset = ConvL2X(offset);
2666 Node* adr = make_unsafe_address(base, offset);
2667 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2669 // (Unlike inline_unsafe_access, there seems no point in trying
2670 // to refine types. Just use the coarse types here.
2671 const Type *value_type = Type::get_const_basic_type(type);
2672 Compile::AliasType* alias_type = C->alias_type(adr_type);
2673 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");
2674 int alias_idx = C->get_alias_index(adr_type);
2676 // Memory-model-wise, a CAS acts like a little synchronized block,
2677 // so needs barriers on each side. These don't translate into
2678 // actual barriers on most machines, but we still need rest of
2679 // compiler to respect ordering.
2681 insert_mem_bar(Op_MemBarRelease);
2682 insert_mem_bar(Op_MemBarCPUOrder);
2684 // 4984716: MemBars must be inserted before this
2685 // memory node in order to avoid a false
2686 // dependency which will confuse the scheduler.
2687 Node *mem = memory(alias_idx);
2689 // For now, we handle only those cases that actually exist: ints,
2690 // longs, and Object. Adding others should be straightforward.
2691 Node* cas;
2692 switch(type) {
2693 case T_INT:
2694 cas = _gvn.transform(new (C, 5) CompareAndSwapINode(control(), mem, adr, newval, oldval));
2695 break;
2696 case T_LONG:
2697 cas = _gvn.transform(new (C, 5) CompareAndSwapLNode(control(), mem, adr, newval, oldval));
2698 break;
2699 case T_OBJECT:
2700 // reference stores need a store barrier.
2701 // (They don't if CAS fails, but it isn't worth checking.)
2702 pre_barrier(true /* do_load*/,
2703 control(), base, adr, alias_idx, newval, value_type->make_oopptr(),
2704 NULL /* pre_val*/,
2705 T_OBJECT);
2706 #ifdef _LP64
2707 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2708 Node *newval_enc = _gvn.transform(new (C, 2) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));
2709 Node *oldval_enc = _gvn.transform(new (C, 2) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));
2710 cas = _gvn.transform(new (C, 5) CompareAndSwapNNode(control(), mem, adr,
2711 newval_enc, oldval_enc));
2712 } else
2713 #endif
2714 {
2715 cas = _gvn.transform(new (C, 5) CompareAndSwapPNode(control(), mem, adr, newval, oldval));
2716 }
2717 post_barrier(control(), cas, base, adr, alias_idx, newval, T_OBJECT, true);
2718 break;
2719 default:
2720 ShouldNotReachHere();
2721 break;
2722 }
2724 // SCMemProjNodes represent the memory state of CAS. Their main
2725 // role is to prevent CAS nodes from being optimized away when their
2726 // results aren't used.
2727 Node* proj = _gvn.transform( new (C, 1) SCMemProjNode(cas));
2728 set_memory(proj, alias_idx);
2730 // Add the trailing membar surrounding the access
2731 insert_mem_bar(Op_MemBarCPUOrder);
2732 insert_mem_bar(Op_MemBarAcquire);
2734 push(cas);
2735 return true;
2736 }
2738 bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {
2739 // This is another variant of inline_unsafe_access, differing in
2740 // that it always issues store-store ("release") barrier and ensures
2741 // store-atomicity (which only matters for "long").
2743 if (callee()->is_static()) return false; // caller must have the capability!
2745 #ifndef PRODUCT
2746 {
2747 ResourceMark rm;
2748 // Check the signatures.
2749 ciSignature* sig = signature();
2750 #ifdef ASSERT
2751 BasicType rtype = sig->return_type()->basic_type();
2752 assert(rtype == T_VOID, "must return void");
2753 assert(sig->count() == 3, "has 3 arguments");
2754 assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");
2755 assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");
2756 #endif // ASSERT
2757 }
2758 #endif //PRODUCT
2760 // number of stack slots per value argument (1 or 2)
2761 int type_words = type2size[type];
2763 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
2765 // Argument words: "this" plus oop plus offset plus value;
2766 int nargs = 1 + 1 + 2 + type_words;
2768 // pop arguments: val, offset, base, and receiver
2769 debug_only(int saved_sp = _sp);
2770 _sp += nargs;
2771 Node* val = (type_words == 1) ? pop() : pop_pair();
2772 Node *offset = pop_pair();
2773 Node *base = pop();
2774 Node *receiver = pop();
2775 assert(saved_sp == _sp, "must have correct argument count");
2777 // Null check receiver.
2778 _sp += nargs;
2779 do_null_check(receiver, T_OBJECT);
2780 _sp -= nargs;
2781 if (stopped()) {
2782 return true;
2783 }
2785 // Build field offset expression.
2786 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");
2787 // 32-bit machines ignore the high half of long offsets
2788 offset = ConvL2X(offset);
2789 Node* adr = make_unsafe_address(base, offset);
2790 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();
2791 const Type *value_type = Type::get_const_basic_type(type);
2792 Compile::AliasType* alias_type = C->alias_type(adr_type);
2794 insert_mem_bar(Op_MemBarRelease);
2795 insert_mem_bar(Op_MemBarCPUOrder);
2796 // Ensure that the store is atomic for longs:
2797 bool require_atomic_access = true;
2798 Node* store;
2799 if (type == T_OBJECT) // reference stores need a store barrier.
2800 store = store_oop_to_unknown(control(), base, adr, adr_type, val, type);
2801 else {
2802 store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access);
2803 }
2804 insert_mem_bar(Op_MemBarCPUOrder);
2805 return true;
2806 }
2808 bool LibraryCallKit::inline_unsafe_allocate() {
2809 if (callee()->is_static()) return false; // caller must have the capability!
2810 int nargs = 1 + 1;
2811 assert(signature()->size() == nargs-1, "alloc has 1 argument");
2812 null_check_receiver(callee()); // check then ignore argument(0)
2813 _sp += nargs; // set original stack for use by uncommon_trap
2814 Node* cls = do_null_check(argument(1), T_OBJECT);
2815 _sp -= nargs;
2816 if (stopped()) return true;
2818 Node* kls = load_klass_from_mirror(cls, false, nargs, NULL, 0);
2819 _sp += nargs; // set original stack for use by uncommon_trap
2820 kls = do_null_check(kls, T_OBJECT);
2821 _sp -= nargs;
2822 if (stopped()) return true; // argument was like int.class
2824 // Note: The argument might still be an illegal value like
2825 // Serializable.class or Object[].class. The runtime will handle it.
2826 // But we must make an explicit check for initialization.
2827 Node* insp = basic_plus_adr(kls, instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc));
2828 Node* inst = make_load(NULL, insp, TypeInt::INT, T_INT);
2829 Node* bits = intcon(instanceKlass::fully_initialized);
2830 Node* test = _gvn.transform( new (C, 3) SubINode(inst, bits) );
2831 // The 'test' is non-zero if we need to take a slow path.
2833 Node* obj = new_instance(kls, test);
2834 push(obj);
2836 return true;
2837 }
2839 //------------------------inline_native_time_funcs--------------
2840 // inline code for System.currentTimeMillis() and System.nanoTime()
2841 // these have the same type and signature
2842 bool LibraryCallKit::inline_native_time_funcs(bool isNano) {
2843 address funcAddr = isNano ? CAST_FROM_FN_PTR(address, os::javaTimeNanos) :
2844 CAST_FROM_FN_PTR(address, os::javaTimeMillis);
2845 const char * funcName = isNano ? "nanoTime" : "currentTimeMillis";
2846 const TypeFunc *tf = OptoRuntime::current_time_millis_Type();
2847 const TypePtr* no_memory_effects = NULL;
2848 Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects);
2849 Node* value = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms+0));
2850 #ifdef ASSERT
2851 Node* value_top = _gvn.transform(new (C, 1) ProjNode(time, TypeFunc::Parms + 1));
2852 assert(value_top == top(), "second value must be top");
2853 #endif
2854 push_pair(value);
2855 return true;
2856 }
2858 //------------------------inline_native_currentThread------------------
2859 bool LibraryCallKit::inline_native_currentThread() {
2860 Node* junk = NULL;
2861 push(generate_current_thread(junk));
2862 return true;
2863 }
2865 //------------------------inline_native_isInterrupted------------------
2866 bool LibraryCallKit::inline_native_isInterrupted() {
2867 const int nargs = 1+1; // receiver + boolean
2868 assert(nargs == arg_size(), "sanity");
2869 // Add a fast path to t.isInterrupted(clear_int):
2870 // (t == Thread.current() && (!TLS._osthread._interrupted || !clear_int))
2871 // ? TLS._osthread._interrupted : /*slow path:*/ t.isInterrupted(clear_int)
2872 // So, in the common case that the interrupt bit is false,
2873 // we avoid making a call into the VM. Even if the interrupt bit
2874 // is true, if the clear_int argument is false, we avoid the VM call.
2875 // However, if the receiver is not currentThread, we must call the VM,
2876 // because there must be some locking done around the operation.
2878 // We only go to the fast case code if we pass two guards.
2879 // Paths which do not pass are accumulated in the slow_region.
2880 RegionNode* slow_region = new (C, 1) RegionNode(1);
2881 record_for_igvn(slow_region);
2882 RegionNode* result_rgn = new (C, 4) RegionNode(1+3); // fast1, fast2, slow
2883 PhiNode* result_val = new (C, 4) PhiNode(result_rgn, TypeInt::BOOL);
2884 enum { no_int_result_path = 1,
2885 no_clear_result_path = 2,
2886 slow_result_path = 3
2887 };
2889 // (a) Receiving thread must be the current thread.
2890 Node* rec_thr = argument(0);
2891 Node* tls_ptr = NULL;
2892 Node* cur_thr = generate_current_thread(tls_ptr);
2893 Node* cmp_thr = _gvn.transform( new (C, 3) CmpPNode(cur_thr, rec_thr) );
2894 Node* bol_thr = _gvn.transform( new (C, 2) BoolNode(cmp_thr, BoolTest::ne) );
2896 bool known_current_thread = (_gvn.type(bol_thr) == TypeInt::ZERO);
2897 if (!known_current_thread)
2898 generate_slow_guard(bol_thr, slow_region);
2900 // (b) Interrupt bit on TLS must be false.
2901 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset()));
2902 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS);
2903 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset()));
2904 // Set the control input on the field _interrupted read to prevent it floating up.
2905 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT);
2906 Node* cmp_bit = _gvn.transform( new (C, 3) CmpINode(int_bit, intcon(0)) );
2907 Node* bol_bit = _gvn.transform( new (C, 2) BoolNode(cmp_bit, BoolTest::ne) );
2909 IfNode* iff_bit = create_and_map_if(control(), bol_bit, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN);
2911 // First fast path: if (!TLS._interrupted) return false;
2912 Node* false_bit = _gvn.transform( new (C, 1) IfFalseNode(iff_bit) );
2913 result_rgn->init_req(no_int_result_path, false_bit);
2914 result_val->init_req(no_int_result_path, intcon(0));
2916 // drop through to next case
2917 set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_bit)) );
2919 // (c) Or, if interrupt bit is set and clear_int is false, use 2nd fast path.
2920 Node* clr_arg = argument(1);
2921 Node* cmp_arg = _gvn.transform( new (C, 3) CmpINode(clr_arg, intcon(0)) );
2922 Node* bol_arg = _gvn.transform( new (C, 2) BoolNode(cmp_arg, BoolTest::ne) );
2923 IfNode* iff_arg = create_and_map_if(control(), bol_arg, PROB_FAIR, COUNT_UNKNOWN);
2925 // Second fast path: ... else if (!clear_int) return true;
2926 Node* false_arg = _gvn.transform( new (C, 1) IfFalseNode(iff_arg) );
2927 result_rgn->init_req(no_clear_result_path, false_arg);
2928 result_val->init_req(no_clear_result_path, intcon(1));
2930 // drop through to next case
2931 set_control( _gvn.transform(new (C, 1) IfTrueNode(iff_arg)) );
2933 // (d) Otherwise, go to the slow path.
2934 slow_region->add_req(control());
2935 set_control( _gvn.transform(slow_region) );
2937 if (stopped()) {
2938 // There is no slow path.
2939 result_rgn->init_req(slow_result_path, top());
2940 result_val->init_req(slow_result_path, top());
2941 } else {
2942 // non-virtual because it is a private non-static
2943 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted);
2945 Node* slow_val = set_results_for_java_call(slow_call);
2946 // this->control() comes from set_results_for_java_call
2948 // If we know that the result of the slow call will be true, tell the optimizer!
2949 if (known_current_thread) slow_val = intcon(1);
2951 Node* fast_io = slow_call->in(TypeFunc::I_O);
2952 Node* fast_mem = slow_call->in(TypeFunc::Memory);
2953 // These two phis are pre-filled with copies of of the fast IO and Memory
2954 Node* io_phi = PhiNode::make(result_rgn, fast_io, Type::ABIO);
2955 Node* mem_phi = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM);
2957 result_rgn->init_req(slow_result_path, control());
2958 io_phi ->init_req(slow_result_path, i_o());
2959 mem_phi ->init_req(slow_result_path, reset_memory());
2960 result_val->init_req(slow_result_path, slow_val);
2962 set_all_memory( _gvn.transform(mem_phi) );
2963 set_i_o( _gvn.transform(io_phi) );
2964 }
2966 push_result(result_rgn, result_val);
2967 C->set_has_split_ifs(true); // Has chance for split-if optimization
2969 return true;
2970 }
2972 //---------------------------load_mirror_from_klass----------------------------
2973 // Given a klass oop, load its java mirror (a java.lang.Class oop).
2974 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
2975 Node* p = basic_plus_adr(klass, Klass::java_mirror_offset_in_bytes() + sizeof(oopDesc));
2976 return make_load(NULL, p, TypeInstPtr::MIRROR, T_OBJECT);
2977 }
2979 //-----------------------load_klass_from_mirror_common-------------------------
2980 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
2981 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
2982 // and branch to the given path on the region.
2983 // If never_see_null, take an uncommon trap on null, so we can optimistically
2984 // compile for the non-null case.
2985 // If the region is NULL, force never_see_null = true.
2986 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror,
2987 bool never_see_null,
2988 int nargs,
2989 RegionNode* region,
2990 int null_path,
2991 int offset) {
2992 if (region == NULL) never_see_null = true;
2993 Node* p = basic_plus_adr(mirror, offset);
2994 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
2995 Node* kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type) );
2996 _sp += nargs; // any deopt will start just before call to enclosing method
2997 Node* null_ctl = top();
2998 kls = null_check_oop(kls, &null_ctl, never_see_null);
2999 if (region != NULL) {
3000 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class).
3001 region->init_req(null_path, null_ctl);
3002 } else {
3003 assert(null_ctl == top(), "no loose ends");
3004 }
3005 _sp -= nargs;
3006 return kls;
3007 }
3009 //--------------------(inline_native_Class_query helpers)---------------------
3010 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE, JVM_ACC_HAS_FINALIZER.
3011 // Fall through if (mods & mask) == bits, take the guard otherwise.
3012 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) {
3013 // Branch around if the given klass has the given modifier bit set.
3014 // Like generate_guard, adds a new path onto the region.
3015 Node* modp = basic_plus_adr(kls, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc));
3016 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT);
3017 Node* mask = intcon(modifier_mask);
3018 Node* bits = intcon(modifier_bits);
3019 Node* mbit = _gvn.transform( new (C, 3) AndINode(mods, mask) );
3020 Node* cmp = _gvn.transform( new (C, 3) CmpINode(mbit, bits) );
3021 Node* bol = _gvn.transform( new (C, 2) BoolNode(cmp, BoolTest::ne) );
3022 return generate_fair_guard(bol, region);
3023 }
3024 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) {
3025 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region);
3026 }
3028 //-------------------------inline_native_Class_query-------------------
3029 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) {
3030 int nargs = 1+0; // just the Class mirror, in most cases
3031 const Type* return_type = TypeInt::BOOL;
3032 Node* prim_return_value = top(); // what happens if it's a primitive class?
3033 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3034 bool expect_prim = false; // most of these guys expect to work on refs
3036 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT };
3038 switch (id) {
3039 case vmIntrinsics::_isInstance:
3040 nargs = 1+1; // the Class mirror, plus the object getting queried about
3041 // nothing is an instance of a primitive type
3042 prim_return_value = intcon(0);
3043 break;
3044 case vmIntrinsics::_getModifiers:
3045 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3046 assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line");
3047 return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin);
3048 break;
3049 case vmIntrinsics::_isInterface:
3050 prim_return_value = intcon(0);
3051 break;
3052 case vmIntrinsics::_isArray:
3053 prim_return_value = intcon(0);
3054 expect_prim = true; // cf. ObjectStreamClass.getClassSignature
3055 break;
3056 case vmIntrinsics::_isPrimitive:
3057 prim_return_value = intcon(1);
3058 expect_prim = true; // obviously
3059 break;
3060 case vmIntrinsics::_getSuperclass:
3061 prim_return_value = null();
3062 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3063 break;
3064 case vmIntrinsics::_getComponentType:
3065 prim_return_value = null();
3066 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR);
3067 break;
3068 case vmIntrinsics::_getClassAccessFlags:
3069 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC);
3070 return_type = TypeInt::INT; // not bool! 6297094
3071 break;
3072 default:
3073 ShouldNotReachHere();
3074 }
3076 Node* mirror = argument(0);
3077 Node* obj = (nargs <= 1)? top(): argument(1);
3079 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr();
3080 if (mirror_con == NULL) return false; // cannot happen?
3082 #ifndef PRODUCT
3083 if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {
3084 ciType* k = mirror_con->java_mirror_type();
3085 if (k) {
3086 tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id()));
3087 k->print_name();
3088 tty->cr();
3089 }
3090 }
3091 #endif
3093 // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive).
3094 RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT);
3095 record_for_igvn(region);
3096 PhiNode* phi = new (C, PATH_LIMIT) PhiNode(region, return_type);
3098 // The mirror will never be null of Reflection.getClassAccessFlags, however
3099 // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE
3100 // if it is. See bug 4774291.
3102 // For Reflection.getClassAccessFlags(), the null check occurs in
3103 // the wrong place; see inline_unsafe_access(), above, for a similar
3104 // situation.
3105 _sp += nargs; // set original stack for use by uncommon_trap
3106 mirror = do_null_check(mirror, T_OBJECT);
3107 _sp -= nargs;
3108 // If mirror or obj is dead, only null-path is taken.
3109 if (stopped()) return true;
3111 if (expect_prim) never_see_null = false; // expect nulls (meaning prims)
3113 // Now load the mirror's klass metaobject, and null-check it.
3114 // Side-effects region with the control path if the klass is null.
3115 Node* kls = load_klass_from_mirror(mirror, never_see_null, nargs,
3116 region, _prim_path);
3117 // If kls is null, we have a primitive mirror.
3118 phi->init_req(_prim_path, prim_return_value);
3119 if (stopped()) { push_result(region, phi); return true; }
3121 Node* p; // handy temp
3122 Node* null_ctl;
3124 // Now that we have the non-null klass, we can perform the real query.
3125 // For constant classes, the query will constant-fold in LoadNode::Value.
3126 Node* query_value = top();
3127 switch (id) {
3128 case vmIntrinsics::_isInstance:
3129 // nothing is an instance of a primitive type
3130 _sp += nargs; // gen_instanceof might do an uncommon trap
3131 query_value = gen_instanceof(obj, kls);
3132 _sp -= nargs;
3133 break;
3135 case vmIntrinsics::_getModifiers:
3136 p = basic_plus_adr(kls, Klass::modifier_flags_offset_in_bytes() + sizeof(oopDesc));
3137 query_value = make_load(NULL, p, TypeInt::INT, T_INT);
3138 break;
3140 case vmIntrinsics::_isInterface:
3141 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3142 if (generate_interface_guard(kls, region) != NULL)
3143 // A guard was added. If the guard is taken, it was an interface.
3144 phi->add_req(intcon(1));
3145 // If we fall through, it's a plain class.
3146 query_value = intcon(0);
3147 break;
3149 case vmIntrinsics::_isArray:
3150 // (To verify this code sequence, check the asserts in JVM_IsArrayClass.)
3151 if (generate_array_guard(kls, region) != NULL)
3152 // A guard was added. If the guard is taken, it was an array.
3153 phi->add_req(intcon(1));
3154 // If we fall through, it's a plain class.
3155 query_value = intcon(0);
3156 break;
3158 case vmIntrinsics::_isPrimitive:
3159 query_value = intcon(0); // "normal" path produces false
3160 break;
3162 case vmIntrinsics::_getSuperclass:
3163 // The rules here are somewhat unfortunate, but we can still do better
3164 // with random logic than with a JNI call.
3165 // Interfaces store null or Object as _super, but must report null.
3166 // Arrays store an intermediate super as _super, but must report Object.
3167 // Other types can report the actual _super.
3168 // (To verify this code sequence, check the asserts in JVM_IsInterface.)
3169 if (generate_interface_guard(kls, region) != NULL)
3170 // A guard was added. If the guard is taken, it was an interface.
3171 phi->add_req(null());
3172 if (generate_array_guard(kls, region) != NULL)
3173 // A guard was added. If the guard is taken, it was an array.
3174 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror())));
3175 // If we fall through, it's a plain class. Get its _super.
3176 p = basic_plus_adr(kls, Klass::super_offset_in_bytes() + sizeof(oopDesc));
3177 kls = _gvn.transform( LoadKlassNode::make(_gvn, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeKlassPtr::OBJECT_OR_NULL) );
3178 null_ctl = top();
3179 kls = null_check_oop(kls, &null_ctl);
3180 if (null_ctl != top()) {
3181 // If the guard is taken, Object.superClass is null (both klass and mirror).
3182 region->add_req(null_ctl);
3183 phi ->add_req(null());
3184 }
3185 if (!stopped()) {
3186 query_value = load_mirror_from_klass(kls);
3187 }
3188 break;
3190 case vmIntrinsics::_getComponentType:
3191 if (generate_array_guard(kls, region) != NULL) {
3192 // Be sure to pin the oop load to the guard edge just created:
3193 Node* is_array_ctrl = region->in(region->req()-1);
3194 Node* cma = basic_plus_adr(kls, in_bytes(arrayKlass::component_mirror_offset()) + sizeof(oopDesc));
3195 Node* cmo = make_load(is_array_ctrl, cma, TypeInstPtr::MIRROR, T_OBJECT);
3196 phi->add_req(cmo);
3197 }
3198 query_value = null(); // non-array case is null
3199 break;
3201 case vmIntrinsics::_getClassAccessFlags:
3202 p = basic_plus_adr(kls, Klass::access_flags_offset_in_bytes() + sizeof(oopDesc));
3203 query_value = make_load(NULL, p, TypeInt::INT, T_INT);
3204 break;
3206 default:
3207 ShouldNotReachHere();
3208 }
3210 // Fall-through is the normal case of a query to a real class.
3211 phi->init_req(1, query_value);
3212 region->init_req(1, control());
3214 push_result(region, phi);
3215 C->set_has_split_ifs(true); // Has chance for split-if optimization
3217 return true;
3218 }
3220 //--------------------------inline_native_subtype_check------------------------
3221 // This intrinsic takes the JNI calls out of the heart of
3222 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc.
3223 bool LibraryCallKit::inline_native_subtype_check() {
3224 int nargs = 1+1; // the Class mirror, plus the other class getting examined
3226 // Pull both arguments off the stack.
3227 Node* args[2]; // two java.lang.Class mirrors: superc, subc
3228 args[0] = argument(0);
3229 args[1] = argument(1);
3230 Node* klasses[2]; // corresponding Klasses: superk, subk
3231 klasses[0] = klasses[1] = top();
3233 enum {
3234 // A full decision tree on {superc is prim, subc is prim}:
3235 _prim_0_path = 1, // {P,N} => false
3236 // {P,P} & superc!=subc => false
3237 _prim_same_path, // {P,P} & superc==subc => true
3238 _prim_1_path, // {N,P} => false
3239 _ref_subtype_path, // {N,N} & subtype check wins => true
3240 _both_ref_path, // {N,N} & subtype check loses => false
3241 PATH_LIMIT
3242 };
3244 RegionNode* region = new (C, PATH_LIMIT) RegionNode(PATH_LIMIT);
3245 Node* phi = new (C, PATH_LIMIT) PhiNode(region, TypeInt::BOOL);
3246 record_for_igvn(region);
3248 const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads
3249 const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL;
3250 int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
3252 // First null-check both mirrors and load each mirror's klass metaobject.
3253 int which_arg;
3254 for (which_arg = 0; which_arg <= 1; which_arg++) {
3255 Node* arg = args[which_arg];
3256 _sp += nargs; // set original stack for use by uncommon_trap
3257 arg = do_null_check(arg, T_OBJECT);
3258 _sp -= nargs;
3259 if (stopped()) break;
3260 args[which_arg] = _gvn.transform(arg);
3262 Node* p = basic_plus_adr(arg, class_klass_offset);
3263 Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type);
3264 klasses[which_arg] = _gvn.transform(kls);
3265 }
3267 // Having loaded both klasses, test each for null.
3268 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3269 for (which_arg = 0; which_arg <= 1; which_arg++) {
3270 Node* kls = klasses[which_arg];
3271 Node* null_ctl = top();
3272 _sp += nargs; // set original stack for use by uncommon_trap
3273 kls = null_check_oop(kls, &null_ctl, never_see_null);
3274 _sp -= nargs;
3275 int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path);
3276 region->init_req(prim_path, null_ctl);
3277 if (stopped()) break;
3278 klasses[which_arg] = kls;
3279 }
3281 if (!stopped()) {
3282 // now we have two reference types, in klasses[0..1]
3283 Node* subk = klasses[1]; // the argument to isAssignableFrom
3284 Node* superk = klasses[0]; // the receiver
3285 region->set_req(_both_ref_path, gen_subtype_check(subk, superk));
3286 // now we have a successful reference subtype check
3287 region->set_req(_ref_subtype_path, control());
3288 }
3290 // If both operands are primitive (both klasses null), then
3291 // we must return true when they are identical primitives.
3292 // It is convenient to test this after the first null klass check.
3293 set_control(region->in(_prim_0_path)); // go back to first null check
3294 if (!stopped()) {
3295 // Since superc is primitive, make a guard for the superc==subc case.
3296 Node* cmp_eq = _gvn.transform( new (C, 3) CmpPNode(args[0], args[1]) );
3297 Node* bol_eq = _gvn.transform( new (C, 2) BoolNode(cmp_eq, BoolTest::eq) );
3298 generate_guard(bol_eq, region, PROB_FAIR);
3299 if (region->req() == PATH_LIMIT+1) {
3300 // A guard was added. If the added guard is taken, superc==subc.
3301 region->swap_edges(PATH_LIMIT, _prim_same_path);
3302 region->del_req(PATH_LIMIT);
3303 }
3304 region->set_req(_prim_0_path, control()); // Not equal after all.
3305 }
3307 // these are the only paths that produce 'true':
3308 phi->set_req(_prim_same_path, intcon(1));
3309 phi->set_req(_ref_subtype_path, intcon(1));
3311 // pull together the cases:
3312 assert(region->req() == PATH_LIMIT, "sane region");
3313 for (uint i = 1; i < region->req(); i++) {
3314 Node* ctl = region->in(i);
3315 if (ctl == NULL || ctl == top()) {
3316 region->set_req(i, top());
3317 phi ->set_req(i, top());
3318 } else if (phi->in(i) == NULL) {
3319 phi->set_req(i, intcon(0)); // all other paths produce 'false'
3320 }
3321 }
3323 set_control(_gvn.transform(region));
3324 push(_gvn.transform(phi));
3326 return true;
3327 }
3329 //---------------------generate_array_guard_common------------------------
3330 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
3331 bool obj_array, bool not_array) {
3332 // If obj_array/non_array==false/false:
3333 // Branch around if the given klass is in fact an array (either obj or prim).
3334 // If obj_array/non_array==false/true:
3335 // Branch around if the given klass is not an array klass of any kind.
3336 // If obj_array/non_array==true/true:
3337 // Branch around if the kls is not an oop array (kls is int[], String, etc.)
3338 // If obj_array/non_array==true/false:
3339 // Branch around if the kls is an oop array (Object[] or subtype)
3340 //
3341 // Like generate_guard, adds a new path onto the region.
3342 jint layout_con = 0;
3343 Node* layout_val = get_layout_helper(kls, layout_con);
3344 if (layout_val == NULL) {
3345 bool query = (obj_array
3346 ? Klass::layout_helper_is_objArray(layout_con)
3347 : Klass::layout_helper_is_javaArray(layout_con));
3348 if (query == not_array) {
3349 return NULL; // never a branch
3350 } else { // always a branch
3351 Node* always_branch = control();
3352 if (region != NULL)
3353 region->add_req(always_branch);
3354 set_control(top());
3355 return always_branch;
3356 }
3357 }
3358 // Now test the correct condition.
3359 jint nval = (obj_array
3360 ? ((jint)Klass::_lh_array_tag_type_value
3361 << Klass::_lh_array_tag_shift)
3362 : Klass::_lh_neutral_value);
3363 Node* cmp = _gvn.transform( new(C, 3) CmpINode(layout_val, intcon(nval)) );
3364 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array
3365 // invert the test if we are looking for a non-array
3366 if (not_array) btest = BoolTest(btest).negate();
3367 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, btest) );
3368 return generate_fair_guard(bol, region);
3369 }
3372 //-----------------------inline_native_newArray--------------------------
3373 bool LibraryCallKit::inline_native_newArray() {
3374 int nargs = 2;
3375 Node* mirror = argument(0);
3376 Node* count_val = argument(1);
3378 _sp += nargs; // set original stack for use by uncommon_trap
3379 mirror = do_null_check(mirror, T_OBJECT);
3380 _sp -= nargs;
3381 // If mirror or obj is dead, only null-path is taken.
3382 if (stopped()) return true;
3384 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT };
3385 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
3386 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg,
3387 TypeInstPtr::NOTNULL);
3388 PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO);
3389 PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY,
3390 TypePtr::BOTTOM);
3392 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check);
3393 Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null,
3394 nargs,
3395 result_reg, _slow_path);
3396 Node* normal_ctl = control();
3397 Node* no_array_ctl = result_reg->in(_slow_path);
3399 // Generate code for the slow case. We make a call to newArray().
3400 set_control(no_array_ctl);
3401 if (!stopped()) {
3402 // Either the input type is void.class, or else the
3403 // array klass has not yet been cached. Either the
3404 // ensuing call will throw an exception, or else it
3405 // will cache the array klass for next time.
3406 PreserveJVMState pjvms(this);
3407 CallJavaNode* slow_call = generate_method_call_static(vmIntrinsics::_newArray);
3408 Node* slow_result = set_results_for_java_call(slow_call);
3409 // this->control() comes from set_results_for_java_call
3410 result_reg->set_req(_slow_path, control());
3411 result_val->set_req(_slow_path, slow_result);
3412 result_io ->set_req(_slow_path, i_o());
3413 result_mem->set_req(_slow_path, reset_memory());
3414 }
3416 set_control(normal_ctl);
3417 if (!stopped()) {
3418 // Normal case: The array type has been cached in the java.lang.Class.
3419 // The following call works fine even if the array type is polymorphic.
3420 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3421 Node* obj = new_array(klass_node, count_val, nargs);
3422 result_reg->init_req(_normal_path, control());
3423 result_val->init_req(_normal_path, obj);
3424 result_io ->init_req(_normal_path, i_o());
3425 result_mem->init_req(_normal_path, reset_memory());
3426 }
3428 // Return the combined state.
3429 set_i_o( _gvn.transform(result_io) );
3430 set_all_memory( _gvn.transform(result_mem) );
3431 push_result(result_reg, result_val);
3432 C->set_has_split_ifs(true); // Has chance for split-if optimization
3434 return true;
3435 }
3437 //----------------------inline_native_getLength--------------------------
3438 bool LibraryCallKit::inline_native_getLength() {
3439 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3441 int nargs = 1;
3442 Node* array = argument(0);
3444 _sp += nargs; // set original stack for use by uncommon_trap
3445 array = do_null_check(array, T_OBJECT);
3446 _sp -= nargs;
3448 // If array is dead, only null-path is taken.
3449 if (stopped()) return true;
3451 // Deoptimize if it is a non-array.
3452 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL);
3454 if (non_array != NULL) {
3455 PreserveJVMState pjvms(this);
3456 set_control(non_array);
3457 _sp += nargs; // push the arguments back on the stack
3458 uncommon_trap(Deoptimization::Reason_intrinsic,
3459 Deoptimization::Action_maybe_recompile);
3460 }
3462 // If control is dead, only non-array-path is taken.
3463 if (stopped()) return true;
3465 // The works fine even if the array type is polymorphic.
3466 // It could be a dynamic mix of int[], boolean[], Object[], etc.
3467 push( load_array_length(array) );
3469 C->set_has_split_ifs(true); // Has chance for split-if optimization
3471 return true;
3472 }
3474 //------------------------inline_array_copyOf----------------------------
3475 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) {
3476 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false;
3478 // Restore the stack and pop off the arguments.
3479 int nargs = 3 + (is_copyOfRange? 1: 0);
3480 Node* original = argument(0);
3481 Node* start = is_copyOfRange? argument(1): intcon(0);
3482 Node* end = is_copyOfRange? argument(2): argument(1);
3483 Node* array_type_mirror = is_copyOfRange? argument(3): argument(2);
3485 Node* newcopy;
3487 //set the original stack and the reexecute bit for the interpreter to reexecute
3488 //the bytecode that invokes Arrays.copyOf if deoptimization happens
3489 { PreserveReexecuteState preexecs(this);
3490 _sp += nargs;
3491 jvms()->set_should_reexecute(true);
3493 array_type_mirror = do_null_check(array_type_mirror, T_OBJECT);
3494 original = do_null_check(original, T_OBJECT);
3496 // Check if a null path was taken unconditionally.
3497 if (stopped()) return true;
3499 Node* orig_length = load_array_length(original);
3501 Node* klass_node = load_klass_from_mirror(array_type_mirror, false, 0,
3502 NULL, 0);
3503 klass_node = do_null_check(klass_node, T_OBJECT);
3505 RegionNode* bailout = new (C, 1) RegionNode(1);
3506 record_for_igvn(bailout);
3508 // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc.
3509 // Bail out if that is so.
3510 Node* not_objArray = generate_non_objArray_guard(klass_node, bailout);
3511 if (not_objArray != NULL) {
3512 // Improve the klass node's type from the new optimistic assumption:
3513 ciKlass* ak = ciArrayKlass::make(env()->Object_klass());
3514 const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
3515 Node* cast = new (C, 2) CastPPNode(klass_node, akls);
3516 cast->init_req(0, control());
3517 klass_node = _gvn.transform(cast);
3518 }
3520 // Bail out if either start or end is negative.
3521 generate_negative_guard(start, bailout, &start);
3522 generate_negative_guard(end, bailout, &end);
3524 Node* length = end;
3525 if (_gvn.type(start) != TypeInt::ZERO) {
3526 length = _gvn.transform( new (C, 3) SubINode(end, start) );
3527 }
3529 // Bail out if length is negative.
3530 // ...Not needed, since the new_array will throw the right exception.
3531 //generate_negative_guard(length, bailout, &length);
3533 if (bailout->req() > 1) {
3534 PreserveJVMState pjvms(this);
3535 set_control( _gvn.transform(bailout) );
3536 uncommon_trap(Deoptimization::Reason_intrinsic,
3537 Deoptimization::Action_maybe_recompile);
3538 }
3540 if (!stopped()) {
3542 // How many elements will we copy from the original?
3543 // The answer is MinI(orig_length - start, length).
3544 Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) );
3545 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length);
3547 const bool raw_mem_only = true;
3548 newcopy = new_array(klass_node, length, 0, raw_mem_only);
3550 // Generate a direct call to the right arraycopy function(s).
3551 // We know the copy is disjoint but we might not know if the
3552 // oop stores need checking.
3553 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class).
3554 // This will fail a store-check if x contains any non-nulls.
3555 bool disjoint_bases = true;
3556 bool length_never_negative = true;
3557 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
3558 original, start, newcopy, intcon(0), moved,
3559 disjoint_bases, length_never_negative);
3560 }
3561 } //original reexecute and sp are set back here
3563 if(!stopped()) {
3564 push(newcopy);
3565 }
3567 C->set_has_split_ifs(true); // Has chance for split-if optimization
3569 return true;
3570 }
3573 //----------------------generate_virtual_guard---------------------------
3574 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call.
3575 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass,
3576 RegionNode* slow_region) {
3577 ciMethod* method = callee();
3578 int vtable_index = method->vtable_index();
3579 // Get the methodOop out of the appropriate vtable entry.
3580 int entry_offset = (instanceKlass::vtable_start_offset() +
3581 vtable_index*vtableEntry::size()) * wordSize +
3582 vtableEntry::method_offset_in_bytes();
3583 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset);
3584 Node* target_call = make_load(NULL, entry_addr, TypeInstPtr::NOTNULL, T_OBJECT);
3586 // Compare the target method with the expected method (e.g., Object.hashCode).
3587 const TypeInstPtr* native_call_addr = TypeInstPtr::make(method);
3589 Node* native_call = makecon(native_call_addr);
3590 Node* chk_native = _gvn.transform( new(C, 3) CmpPNode(target_call, native_call) );
3591 Node* test_native = _gvn.transform( new(C, 2) BoolNode(chk_native, BoolTest::ne) );
3593 return generate_slow_guard(test_native, slow_region);
3594 }
3596 //-----------------------generate_method_call----------------------------
3597 // Use generate_method_call to make a slow-call to the real
3598 // method if the fast path fails. An alternative would be to
3599 // use a stub like OptoRuntime::slow_arraycopy_Java.
3600 // This only works for expanding the current library call,
3601 // not another intrinsic. (E.g., don't use this for making an
3602 // arraycopy call inside of the copyOf intrinsic.)
3603 CallJavaNode*
3604 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) {
3605 // When compiling the intrinsic method itself, do not use this technique.
3606 guarantee(callee() != C->method(), "cannot make slow-call to self");
3608 ciMethod* method = callee();
3609 // ensure the JVMS we have will be correct for this call
3610 guarantee(method_id == method->intrinsic_id(), "must match");
3612 const TypeFunc* tf = TypeFunc::make(method);
3613 int tfdc = tf->domain()->cnt();
3614 CallJavaNode* slow_call;
3615 if (is_static) {
3616 assert(!is_virtual, "");
3617 slow_call = new(C, tfdc) CallStaticJavaNode(tf,
3618 SharedRuntime::get_resolve_static_call_stub(),
3619 method, bci());
3620 } else if (is_virtual) {
3621 null_check_receiver(method);
3622 int vtable_index = methodOopDesc::invalid_vtable_index;
3623 if (UseInlineCaches) {
3624 // Suppress the vtable call
3625 } else {
3626 // hashCode and clone are not a miranda methods,
3627 // so the vtable index is fixed.
3628 // No need to use the linkResolver to get it.
3629 vtable_index = method->vtable_index();
3630 }
3631 slow_call = new(C, tfdc) CallDynamicJavaNode(tf,
3632 SharedRuntime::get_resolve_virtual_call_stub(),
3633 method, vtable_index, bci());
3634 } else { // neither virtual nor static: opt_virtual
3635 null_check_receiver(method);
3636 slow_call = new(C, tfdc) CallStaticJavaNode(tf,
3637 SharedRuntime::get_resolve_opt_virtual_call_stub(),
3638 method, bci());
3639 slow_call->set_optimized_virtual(true);
3640 }
3641 set_arguments_for_java_call(slow_call);
3642 set_edges_for_java_call(slow_call);
3643 return slow_call;
3644 }
3647 //------------------------------inline_native_hashcode--------------------
3648 // Build special case code for calls to hashCode on an object.
3649 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) {
3650 assert(is_static == callee()->is_static(), "correct intrinsic selection");
3651 assert(!(is_virtual && is_static), "either virtual, special, or static");
3653 enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT };
3655 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
3656 PhiNode* result_val = new(C, PATH_LIMIT) PhiNode(result_reg,
3657 TypeInt::INT);
3658 PhiNode* result_io = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO);
3659 PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY,
3660 TypePtr::BOTTOM);
3661 Node* obj = NULL;
3662 if (!is_static) {
3663 // Check for hashing null object
3664 obj = null_check_receiver(callee());
3665 if (stopped()) return true; // unconditionally null
3666 result_reg->init_req(_null_path, top());
3667 result_val->init_req(_null_path, top());
3668 } else {
3669 // Do a null check, and return zero if null.
3670 // System.identityHashCode(null) == 0
3671 obj = argument(0);
3672 Node* null_ctl = top();
3673 obj = null_check_oop(obj, &null_ctl);
3674 result_reg->init_req(_null_path, null_ctl);
3675 result_val->init_req(_null_path, _gvn.intcon(0));
3676 }
3678 // Unconditionally null? Then return right away.
3679 if (stopped()) {
3680 set_control( result_reg->in(_null_path) );
3681 if (!stopped())
3682 push( result_val ->in(_null_path) );
3683 return true;
3684 }
3686 // After null check, get the object's klass.
3687 Node* obj_klass = load_object_klass(obj);
3689 // This call may be virtual (invokevirtual) or bound (invokespecial).
3690 // For each case we generate slightly different code.
3692 // We only go to the fast case code if we pass a number of guards. The
3693 // paths which do not pass are accumulated in the slow_region.
3694 RegionNode* slow_region = new (C, 1) RegionNode(1);
3695 record_for_igvn(slow_region);
3697 // If this is a virtual call, we generate a funny guard. We pull out
3698 // the vtable entry corresponding to hashCode() from the target object.
3699 // If the target method which we are calling happens to be the native
3700 // Object hashCode() method, we pass the guard. We do not need this
3701 // guard for non-virtual calls -- the caller is known to be the native
3702 // Object hashCode().
3703 if (is_virtual) {
3704 generate_virtual_guard(obj_klass, slow_region);
3705 }
3707 // Get the header out of the object, use LoadMarkNode when available
3708 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes());
3709 Node* header = make_load(control(), header_addr, TypeX_X, TypeX_X->basic_type());
3711 // Test the header to see if it is unlocked.
3712 Node *lock_mask = _gvn.MakeConX(markOopDesc::biased_lock_mask_in_place);
3713 Node *lmasked_header = _gvn.transform( new (C, 3) AndXNode(header, lock_mask) );
3714 Node *unlocked_val = _gvn.MakeConX(markOopDesc::unlocked_value);
3715 Node *chk_unlocked = _gvn.transform( new (C, 3) CmpXNode( lmasked_header, unlocked_val));
3716 Node *test_unlocked = _gvn.transform( new (C, 2) BoolNode( chk_unlocked, BoolTest::ne) );
3718 generate_slow_guard(test_unlocked, slow_region);
3720 // Get the hash value and check to see that it has been properly assigned.
3721 // We depend on hash_mask being at most 32 bits and avoid the use of
3722 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
3723 // vm: see markOop.hpp.
3724 Node *hash_mask = _gvn.intcon(markOopDesc::hash_mask);
3725 Node *hash_shift = _gvn.intcon(markOopDesc::hash_shift);
3726 Node *hshifted_header= _gvn.transform( new (C, 3) URShiftXNode(header, hash_shift) );
3727 // This hack lets the hash bits live anywhere in the mark object now, as long
3728 // as the shift drops the relevant bits into the low 32 bits. Note that
3729 // Java spec says that HashCode is an int so there's no point in capturing
3730 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build).
3731 hshifted_header = ConvX2I(hshifted_header);
3732 Node *hash_val = _gvn.transform( new (C, 3) AndINode(hshifted_header, hash_mask) );
3734 Node *no_hash_val = _gvn.intcon(markOopDesc::no_hash);
3735 Node *chk_assigned = _gvn.transform( new (C, 3) CmpINode( hash_val, no_hash_val));
3736 Node *test_assigned = _gvn.transform( new (C, 2) BoolNode( chk_assigned, BoolTest::eq) );
3738 generate_slow_guard(test_assigned, slow_region);
3740 Node* init_mem = reset_memory();
3741 // fill in the rest of the null path:
3742 result_io ->init_req(_null_path, i_o());
3743 result_mem->init_req(_null_path, init_mem);
3745 result_val->init_req(_fast_path, hash_val);
3746 result_reg->init_req(_fast_path, control());
3747 result_io ->init_req(_fast_path, i_o());
3748 result_mem->init_req(_fast_path, init_mem);
3750 // Generate code for the slow case. We make a call to hashCode().
3751 set_control(_gvn.transform(slow_region));
3752 if (!stopped()) {
3753 // No need for PreserveJVMState, because we're using up the present state.
3754 set_all_memory(init_mem);
3755 vmIntrinsics::ID hashCode_id = vmIntrinsics::_hashCode;
3756 if (is_static) hashCode_id = vmIntrinsics::_identityHashCode;
3757 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static);
3758 Node* slow_result = set_results_for_java_call(slow_call);
3759 // this->control() comes from set_results_for_java_call
3760 result_reg->init_req(_slow_path, control());
3761 result_val->init_req(_slow_path, slow_result);
3762 result_io ->set_req(_slow_path, i_o());
3763 result_mem ->set_req(_slow_path, reset_memory());
3764 }
3766 // Return the combined state.
3767 set_i_o( _gvn.transform(result_io) );
3768 set_all_memory( _gvn.transform(result_mem) );
3769 push_result(result_reg, result_val);
3771 return true;
3772 }
3774 //---------------------------inline_native_getClass----------------------------
3775 // Build special case code for calls to getClass on an object.
3776 bool LibraryCallKit::inline_native_getClass() {
3777 Node* obj = null_check_receiver(callee());
3778 if (stopped()) return true;
3779 push( load_mirror_from_klass(load_object_klass(obj)) );
3780 return true;
3781 }
3783 //-----------------inline_native_Reflection_getCallerClass---------------------
3784 // In the presence of deep enough inlining, getCallerClass() becomes a no-op.
3785 //
3786 // NOTE that this code must perform the same logic as
3787 // vframeStream::security_get_caller_frame in that it must skip
3788 // Method.invoke() and auxiliary frames.
3793 bool LibraryCallKit::inline_native_Reflection_getCallerClass() {
3794 ciMethod* method = callee();
3796 #ifndef PRODUCT
3797 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3798 tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass");
3799 }
3800 #endif
3802 debug_only(int saved_sp = _sp);
3804 // Argument words: (int depth)
3805 int nargs = 1;
3807 _sp += nargs;
3808 Node* caller_depth_node = pop();
3810 assert(saved_sp == _sp, "must have correct argument count");
3812 // The depth value must be a constant in order for the runtime call
3813 // to be eliminated.
3814 const TypeInt* caller_depth_type = _gvn.type(caller_depth_node)->isa_int();
3815 if (caller_depth_type == NULL || !caller_depth_type->is_con()) {
3816 #ifndef PRODUCT
3817 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3818 tty->print_cr(" Bailing out because caller depth was not a constant");
3819 }
3820 #endif
3821 return false;
3822 }
3823 // Note that the JVM state at this point does not include the
3824 // getCallerClass() frame which we are trying to inline. The
3825 // semantics of getCallerClass(), however, are that the "first"
3826 // frame is the getCallerClass() frame, so we subtract one from the
3827 // requested depth before continuing. We don't inline requests of
3828 // getCallerClass(0).
3829 int caller_depth = caller_depth_type->get_con() - 1;
3830 if (caller_depth < 0) {
3831 #ifndef PRODUCT
3832 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3833 tty->print_cr(" Bailing out because caller depth was %d", caller_depth);
3834 }
3835 #endif
3836 return false;
3837 }
3839 if (!jvms()->has_method()) {
3840 #ifndef PRODUCT
3841 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3842 tty->print_cr(" Bailing out because intrinsic was inlined at top level");
3843 }
3844 #endif
3845 return false;
3846 }
3847 int _depth = jvms()->depth(); // cache call chain depth
3849 // Walk back up the JVM state to find the caller at the required
3850 // depth. NOTE that this code must perform the same logic as
3851 // vframeStream::security_get_caller_frame in that it must skip
3852 // Method.invoke() and auxiliary frames. Note also that depth is
3853 // 1-based (1 is the bottom of the inlining).
3854 int inlining_depth = _depth;
3855 JVMState* caller_jvms = NULL;
3857 if (inlining_depth > 0) {
3858 caller_jvms = jvms();
3859 assert(caller_jvms = jvms()->of_depth(inlining_depth), "inlining_depth == our depth");
3860 do {
3861 // The following if-tests should be performed in this order
3862 if (is_method_invoke_or_aux_frame(caller_jvms)) {
3863 // Skip a Method.invoke() or auxiliary frame
3864 } else if (caller_depth > 0) {
3865 // Skip real frame
3866 --caller_depth;
3867 } else {
3868 // We're done: reached desired caller after skipping.
3869 break;
3870 }
3871 caller_jvms = caller_jvms->caller();
3872 --inlining_depth;
3873 } while (inlining_depth > 0);
3874 }
3876 if (inlining_depth == 0) {
3877 #ifndef PRODUCT
3878 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3879 tty->print_cr(" Bailing out because caller depth (%d) exceeded inlining depth (%d)", caller_depth_type->get_con(), _depth);
3880 tty->print_cr(" JVM state at this point:");
3881 for (int i = _depth; i >= 1; i--) {
3882 tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8());
3883 }
3884 }
3885 #endif
3886 return false; // Reached end of inlining
3887 }
3889 // Acquire method holder as java.lang.Class
3890 ciInstanceKlass* caller_klass = caller_jvms->method()->holder();
3891 ciInstance* caller_mirror = caller_klass->java_mirror();
3892 // Push this as a constant
3893 push(makecon(TypeInstPtr::make(caller_mirror)));
3894 #ifndef PRODUCT
3895 if ((PrintIntrinsics || PrintInlining || PrintOptoInlining) && Verbose) {
3896 tty->print_cr(" Succeeded: caller = %s.%s, caller depth = %d, depth = %d", caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), caller_depth_type->get_con(), _depth);
3897 tty->print_cr(" JVM state at this point:");
3898 for (int i = _depth; i >= 1; i--) {
3899 tty->print_cr(" %d) %s", i, jvms()->of_depth(i)->method()->name()->as_utf8());
3900 }
3901 }
3902 #endif
3903 return true;
3904 }
3906 // Helper routine for above
3907 bool LibraryCallKit::is_method_invoke_or_aux_frame(JVMState* jvms) {
3908 ciMethod* method = jvms->method();
3910 // Is this the Method.invoke method itself?
3911 if (method->intrinsic_id() == vmIntrinsics::_invoke)
3912 return true;
3914 // Is this a helper, defined somewhere underneath MethodAccessorImpl.
3915 ciKlass* k = method->holder();
3916 if (k->is_instance_klass()) {
3917 ciInstanceKlass* ik = k->as_instance_klass();
3918 for (; ik != NULL; ik = ik->super()) {
3919 if (ik->name() == ciSymbol::sun_reflect_MethodAccessorImpl() &&
3920 ik == env()->find_system_klass(ik->name())) {
3921 return true;
3922 }
3923 }
3924 }
3925 else if (method->is_method_handle_adapter()) {
3926 // This is an internal adapter frame from the MethodHandleCompiler -- skip it
3927 return true;
3928 }
3930 return false;
3931 }
3933 static int value_field_offset = -1; // offset of the "value" field of AtomicLongCSImpl. This is needed by
3934 // inline_native_AtomicLong_attemptUpdate() but it has no way of
3935 // computing it since there is no lookup field by name function in the
3936 // CI interface. This is computed and set by inline_native_AtomicLong_get().
3937 // Using a static variable here is safe even if we have multiple compilation
3938 // threads because the offset is constant. At worst the same offset will be
3939 // computed and stored multiple
3941 bool LibraryCallKit::inline_native_AtomicLong_get() {
3942 // Restore the stack and pop off the argument
3943 _sp+=1;
3944 Node *obj = pop();
3946 // get the offset of the "value" field. Since the CI interfaces
3947 // does not provide a way to look up a field by name, we scan the bytecodes
3948 // to get the field index. We expect the first 2 instructions of the method
3949 // to be:
3950 // 0 aload_0
3951 // 1 getfield "value"
3952 ciMethod* method = callee();
3953 if (value_field_offset == -1)
3954 {
3955 ciField* value_field;
3956 ciBytecodeStream iter(method);
3957 Bytecodes::Code bc = iter.next();
3959 if ((bc != Bytecodes::_aload_0) &&
3960 ((bc != Bytecodes::_aload) || (iter.get_index() != 0)))
3961 return false;
3962 bc = iter.next();
3963 if (bc != Bytecodes::_getfield)
3964 return false;
3965 bool ignore;
3966 value_field = iter.get_field(ignore);
3967 value_field_offset = value_field->offset_in_bytes();
3968 }
3970 // Null check without removing any arguments.
3971 _sp++;
3972 obj = do_null_check(obj, T_OBJECT);
3973 _sp--;
3974 // Check for locking null object
3975 if (stopped()) return true;
3977 Node *adr = basic_plus_adr(obj, obj, value_field_offset);
3978 const TypePtr *adr_type = _gvn.type(adr)->is_ptr();
3979 int alias_idx = C->get_alias_index(adr_type);
3981 Node *result = _gvn.transform(new (C, 3) LoadLLockedNode(control(), memory(alias_idx), adr));
3983 push_pair(result);
3985 return true;
3986 }
3988 bool LibraryCallKit::inline_native_AtomicLong_attemptUpdate() {
3989 // Restore the stack and pop off the arguments
3990 _sp+=5;
3991 Node *newVal = pop_pair();
3992 Node *oldVal = pop_pair();
3993 Node *obj = pop();
3995 // we need the offset of the "value" field which was computed when
3996 // inlining the get() method. Give up if we don't have it.
3997 if (value_field_offset == -1)
3998 return false;
4000 // Null check without removing any arguments.
4001 _sp+=5;
4002 obj = do_null_check(obj, T_OBJECT);
4003 _sp-=5;
4004 // Check for locking null object
4005 if (stopped()) return true;
4007 Node *adr = basic_plus_adr(obj, obj, value_field_offset);
4008 const TypePtr *adr_type = _gvn.type(adr)->is_ptr();
4009 int alias_idx = C->get_alias_index(adr_type);
4011 Node *cas = _gvn.transform(new (C, 5) StoreLConditionalNode(control(), memory(alias_idx), adr, newVal, oldVal));
4012 Node *store_proj = _gvn.transform( new (C, 1) SCMemProjNode(cas));
4013 set_memory(store_proj, alias_idx);
4014 Node *bol = _gvn.transform( new (C, 2) BoolNode( cas, BoolTest::eq ) );
4016 Node *result;
4017 // CMove node is not used to be able fold a possible check code
4018 // after attemptUpdate() call. This code could be transformed
4019 // into CMove node by loop optimizations.
4020 {
4021 RegionNode *r = new (C, 3) RegionNode(3);
4022 result = new (C, 3) PhiNode(r, TypeInt::BOOL);
4024 Node *iff = create_and_xform_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN);
4025 Node *iftrue = opt_iff(r, iff);
4026 r->init_req(1, iftrue);
4027 result->init_req(1, intcon(1));
4028 result->init_req(2, intcon(0));
4030 set_control(_gvn.transform(r));
4031 record_for_igvn(r);
4033 C->set_has_split_ifs(true); // Has chance for split-if optimization
4034 }
4036 push(_gvn.transform(result));
4037 return true;
4038 }
4040 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) {
4041 // restore the arguments
4042 _sp += arg_size();
4044 switch (id) {
4045 case vmIntrinsics::_floatToRawIntBits:
4046 push(_gvn.transform( new (C, 2) MoveF2INode(pop())));
4047 break;
4049 case vmIntrinsics::_intBitsToFloat:
4050 push(_gvn.transform( new (C, 2) MoveI2FNode(pop())));
4051 break;
4053 case vmIntrinsics::_doubleToRawLongBits:
4054 push_pair(_gvn.transform( new (C, 2) MoveD2LNode(pop_pair())));
4055 break;
4057 case vmIntrinsics::_longBitsToDouble:
4058 push_pair(_gvn.transform( new (C, 2) MoveL2DNode(pop_pair())));
4059 break;
4061 case vmIntrinsics::_doubleToLongBits: {
4062 Node* value = pop_pair();
4064 // two paths (plus control) merge in a wood
4065 RegionNode *r = new (C, 3) RegionNode(3);
4066 Node *phi = new (C, 3) PhiNode(r, TypeLong::LONG);
4068 Node *cmpisnan = _gvn.transform( new (C, 3) CmpDNode(value, value));
4069 // Build the boolean node
4070 Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) );
4072 // Branch either way.
4073 // NaN case is less traveled, which makes all the difference.
4074 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4075 Node *opt_isnan = _gvn.transform(ifisnan);
4076 assert( opt_isnan->is_If(), "Expect an IfNode");
4077 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4078 Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) );
4080 set_control(iftrue);
4082 static const jlong nan_bits = CONST64(0x7ff8000000000000);
4083 Node *slow_result = longcon(nan_bits); // return NaN
4084 phi->init_req(1, _gvn.transform( slow_result ));
4085 r->init_req(1, iftrue);
4087 // Else fall through
4088 Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) );
4089 set_control(iffalse);
4091 phi->init_req(2, _gvn.transform( new (C, 2) MoveD2LNode(value)));
4092 r->init_req(2, iffalse);
4094 // Post merge
4095 set_control(_gvn.transform(r));
4096 record_for_igvn(r);
4098 Node* result = _gvn.transform(phi);
4099 assert(result->bottom_type()->isa_long(), "must be");
4100 push_pair(result);
4102 C->set_has_split_ifs(true); // Has chance for split-if optimization
4104 break;
4105 }
4107 case vmIntrinsics::_floatToIntBits: {
4108 Node* value = pop();
4110 // two paths (plus control) merge in a wood
4111 RegionNode *r = new (C, 3) RegionNode(3);
4112 Node *phi = new (C, 3) PhiNode(r, TypeInt::INT);
4114 Node *cmpisnan = _gvn.transform( new (C, 3) CmpFNode(value, value));
4115 // Build the boolean node
4116 Node *bolisnan = _gvn.transform( new (C, 2) BoolNode( cmpisnan, BoolTest::ne ) );
4118 // Branch either way.
4119 // NaN case is less traveled, which makes all the difference.
4120 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);
4121 Node *opt_isnan = _gvn.transform(ifisnan);
4122 assert( opt_isnan->is_If(), "Expect an IfNode");
4123 IfNode *opt_ifisnan = (IfNode*)opt_isnan;
4124 Node *iftrue = _gvn.transform( new (C, 1) IfTrueNode(opt_ifisnan) );
4126 set_control(iftrue);
4128 static const jint nan_bits = 0x7fc00000;
4129 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN
4130 phi->init_req(1, _gvn.transform( slow_result ));
4131 r->init_req(1, iftrue);
4133 // Else fall through
4134 Node *iffalse = _gvn.transform( new (C, 1) IfFalseNode(opt_ifisnan) );
4135 set_control(iffalse);
4137 phi->init_req(2, _gvn.transform( new (C, 2) MoveF2INode(value)));
4138 r->init_req(2, iffalse);
4140 // Post merge
4141 set_control(_gvn.transform(r));
4142 record_for_igvn(r);
4144 Node* result = _gvn.transform(phi);
4145 assert(result->bottom_type()->isa_int(), "must be");
4146 push(result);
4148 C->set_has_split_ifs(true); // Has chance for split-if optimization
4150 break;
4151 }
4153 default:
4154 ShouldNotReachHere();
4155 }
4157 return true;
4158 }
4160 #ifdef _LP64
4161 #define XTOP ,top() /*additional argument*/
4162 #else //_LP64
4163 #define XTOP /*no additional argument*/
4164 #endif //_LP64
4166 //----------------------inline_unsafe_copyMemory-------------------------
4167 bool LibraryCallKit::inline_unsafe_copyMemory() {
4168 if (callee()->is_static()) return false; // caller must have the capability!
4169 int nargs = 1 + 5 + 3; // 5 args: (src: ptr,off, dst: ptr,off, size)
4170 assert(signature()->size() == nargs-1, "copy has 5 arguments");
4171 null_check_receiver(callee()); // check then ignore argument(0)
4172 if (stopped()) return true;
4174 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe".
4176 Node* src_ptr = argument(1);
4177 Node* src_off = ConvL2X(argument(2));
4178 assert(argument(3)->is_top(), "2nd half of long");
4179 Node* dst_ptr = argument(4);
4180 Node* dst_off = ConvL2X(argument(5));
4181 assert(argument(6)->is_top(), "2nd half of long");
4182 Node* size = ConvL2X(argument(7));
4183 assert(argument(8)->is_top(), "2nd half of long");
4185 assert(Unsafe_field_offset_to_byte_offset(11) == 11,
4186 "fieldOffset must be byte-scaled");
4188 Node* src = make_unsafe_address(src_ptr, src_off);
4189 Node* dst = make_unsafe_address(dst_ptr, dst_off);
4191 // Conservatively insert a memory barrier on all memory slices.
4192 // Do not let writes of the copy source or destination float below the copy.
4193 insert_mem_bar(Op_MemBarCPUOrder);
4195 // Call it. Note that the length argument is not scaled.
4196 make_runtime_call(RC_LEAF|RC_NO_FP,
4197 OptoRuntime::fast_arraycopy_Type(),
4198 StubRoutines::unsafe_arraycopy(),
4199 "unsafe_arraycopy",
4200 TypeRawPtr::BOTTOM,
4201 src, dst, size XTOP);
4203 // Do not let reads of the copy destination float above the copy.
4204 insert_mem_bar(Op_MemBarCPUOrder);
4206 return true;
4207 }
4209 //------------------------clone_coping-----------------------------------
4210 // Helper function for inline_native_clone.
4211 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) {
4212 assert(obj_size != NULL, "");
4213 Node* raw_obj = alloc_obj->in(1);
4214 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), "");
4216 if (ReduceBulkZeroing) {
4217 // We will be completely responsible for initializing this object -
4218 // mark Initialize node as complete.
4219 AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
4220 // The object was just allocated - there should be no any stores!
4221 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), "");
4222 }
4224 // Copy the fastest available way.
4225 // TODO: generate fields copies for small objects instead.
4226 Node* src = obj;
4227 Node* dest = alloc_obj;
4228 Node* size = _gvn.transform(obj_size);
4230 // Exclude the header but include array length to copy by 8 bytes words.
4231 // Can't use base_offset_in_bytes(bt) since basic type is unknown.
4232 int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() :
4233 instanceOopDesc::base_offset_in_bytes();
4234 // base_off:
4235 // 8 - 32-bit VM
4236 // 12 - 64-bit VM, compressed oops
4237 // 16 - 64-bit VM, normal oops
4238 if (base_off % BytesPerLong != 0) {
4239 assert(UseCompressedOops, "");
4240 if (is_array) {
4241 // Exclude length to copy by 8 bytes words.
4242 base_off += sizeof(int);
4243 } else {
4244 // Include klass to copy by 8 bytes words.
4245 base_off = instanceOopDesc::klass_offset_in_bytes();
4246 }
4247 assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment");
4248 }
4249 src = basic_plus_adr(src, base_off);
4250 dest = basic_plus_adr(dest, base_off);
4252 // Compute the length also, if needed:
4253 Node* countx = size;
4254 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) );
4255 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
4257 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4258 bool disjoint_bases = true;
4259 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
4260 src, NULL, dest, NULL, countx,
4261 /*dest_uninitialized*/true);
4263 // If necessary, emit some card marks afterwards. (Non-arrays only.)
4264 if (card_mark) {
4265 assert(!is_array, "");
4266 // Put in store barrier for any and all oops we are sticking
4267 // into this object. (We could avoid this if we could prove
4268 // that the object type contains no oop fields at all.)
4269 Node* no_particular_value = NULL;
4270 Node* no_particular_field = NULL;
4271 int raw_adr_idx = Compile::AliasIdxRaw;
4272 post_barrier(control(),
4273 memory(raw_adr_type),
4274 alloc_obj,
4275 no_particular_field,
4276 raw_adr_idx,
4277 no_particular_value,
4278 T_OBJECT,
4279 false);
4280 }
4282 // Do not let reads from the cloned object float above the arraycopy.
4283 insert_mem_bar(Op_MemBarCPUOrder);
4284 }
4286 //------------------------inline_native_clone----------------------------
4287 // Here are the simple edge cases:
4288 // null receiver => normal trap
4289 // virtual and clone was overridden => slow path to out-of-line clone
4290 // not cloneable or finalizer => slow path to out-of-line Object.clone
4291 //
4292 // The general case has two steps, allocation and copying.
4293 // Allocation has two cases, and uses GraphKit::new_instance or new_array.
4294 //
4295 // Copying also has two cases, oop arrays and everything else.
4296 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy).
4297 // Everything else uses the tight inline loop supplied by CopyArrayNode.
4298 //
4299 // These steps fold up nicely if and when the cloned object's klass
4300 // can be sharply typed as an object array, a type array, or an instance.
4301 //
4302 bool LibraryCallKit::inline_native_clone(bool is_virtual) {
4303 int nargs = 1;
4304 PhiNode* result_val;
4306 //set the original stack and the reexecute bit for the interpreter to reexecute
4307 //the bytecode that invokes Object.clone if deoptimization happens
4308 { PreserveReexecuteState preexecs(this);
4309 jvms()->set_should_reexecute(true);
4311 //null_check_receiver will adjust _sp (push and pop)
4312 Node* obj = null_check_receiver(callee());
4313 if (stopped()) return true;
4315 _sp += nargs;
4317 Node* obj_klass = load_object_klass(obj);
4318 const TypeKlassPtr* tklass = _gvn.type(obj_klass)->isa_klassptr();
4319 const TypeOopPtr* toop = ((tklass != NULL)
4320 ? tklass->as_instance_type()
4321 : TypeInstPtr::NOTNULL);
4323 // Conservatively insert a memory barrier on all memory slices.
4324 // Do not let writes into the original float below the clone.
4325 insert_mem_bar(Op_MemBarCPUOrder);
4327 // paths into result_reg:
4328 enum {
4329 _slow_path = 1, // out-of-line call to clone method (virtual or not)
4330 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy
4331 _array_path, // plain array allocation, plus arrayof_long_arraycopy
4332 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy
4333 PATH_LIMIT
4334 };
4335 RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
4336 result_val = new(C, PATH_LIMIT) PhiNode(result_reg,
4337 TypeInstPtr::NOTNULL);
4338 PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_reg, Type::ABIO);
4339 PhiNode* result_mem = new(C, PATH_LIMIT) PhiNode(result_reg, Type::MEMORY,
4340 TypePtr::BOTTOM);
4341 record_for_igvn(result_reg);
4343 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
4344 int raw_adr_idx = Compile::AliasIdxRaw;
4345 const bool raw_mem_only = true;
4348 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL);
4349 if (array_ctl != NULL) {
4350 // It's an array.
4351 PreserveJVMState pjvms(this);
4352 set_control(array_ctl);
4353 Node* obj_length = load_array_length(obj);
4354 Node* obj_size = NULL;
4355 Node* alloc_obj = new_array(obj_klass, obj_length, 0,
4356 raw_mem_only, &obj_size);
4358 if (!use_ReduceInitialCardMarks()) {
4359 // If it is an oop array, it requires very special treatment,
4360 // because card marking is required on each card of the array.
4361 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL);
4362 if (is_obja != NULL) {
4363 PreserveJVMState pjvms2(this);
4364 set_control(is_obja);
4365 // Generate a direct call to the right arraycopy function(s).
4366 bool disjoint_bases = true;
4367 bool length_never_negative = true;
4368 generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT,
4369 obj, intcon(0), alloc_obj, intcon(0),
4370 obj_length,
4371 disjoint_bases, length_never_negative);
4372 result_reg->init_req(_objArray_path, control());
4373 result_val->init_req(_objArray_path, alloc_obj);
4374 result_i_o ->set_req(_objArray_path, i_o());
4375 result_mem ->set_req(_objArray_path, reset_memory());
4376 }
4377 }
4378 // Otherwise, there are no card marks to worry about.
4379 // (We can dispense with card marks if we know the allocation
4380 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks
4381 // causes the non-eden paths to take compensating steps to
4382 // simulate a fresh allocation, so that no further
4383 // card marks are required in compiled code to initialize
4384 // the object.)
4386 if (!stopped()) {
4387 copy_to_clone(obj, alloc_obj, obj_size, true, false);
4389 // Present the results of the copy.
4390 result_reg->init_req(_array_path, control());
4391 result_val->init_req(_array_path, alloc_obj);
4392 result_i_o ->set_req(_array_path, i_o());
4393 result_mem ->set_req(_array_path, reset_memory());
4394 }
4395 }
4397 // We only go to the instance fast case code if we pass a number of guards.
4398 // The paths which do not pass are accumulated in the slow_region.
4399 RegionNode* slow_region = new (C, 1) RegionNode(1);
4400 record_for_igvn(slow_region);
4401 if (!stopped()) {
4402 // It's an instance (we did array above). Make the slow-path tests.
4403 // If this is a virtual call, we generate a funny guard. We grab
4404 // the vtable entry corresponding to clone() from the target object.
4405 // If the target method which we are calling happens to be the
4406 // Object clone() method, we pass the guard. We do not need this
4407 // guard for non-virtual calls; the caller is known to be the native
4408 // Object clone().
4409 if (is_virtual) {
4410 generate_virtual_guard(obj_klass, slow_region);
4411 }
4413 // The object must be cloneable and must not have a finalizer.
4414 // Both of these conditions may be checked in a single test.
4415 // We could optimize the cloneable test further, but we don't care.
4416 generate_access_flags_guard(obj_klass,
4417 // Test both conditions:
4418 JVM_ACC_IS_CLONEABLE | JVM_ACC_HAS_FINALIZER,
4419 // Must be cloneable but not finalizer:
4420 JVM_ACC_IS_CLONEABLE,
4421 slow_region);
4422 }
4424 if (!stopped()) {
4425 // It's an instance, and it passed the slow-path tests.
4426 PreserveJVMState pjvms(this);
4427 Node* obj_size = NULL;
4428 Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size);
4430 copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks());
4432 // Present the results of the slow call.
4433 result_reg->init_req(_instance_path, control());
4434 result_val->init_req(_instance_path, alloc_obj);
4435 result_i_o ->set_req(_instance_path, i_o());
4436 result_mem ->set_req(_instance_path, reset_memory());
4437 }
4439 // Generate code for the slow case. We make a call to clone().
4440 set_control(_gvn.transform(slow_region));
4441 if (!stopped()) {
4442 PreserveJVMState pjvms(this);
4443 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual);
4444 Node* slow_result = set_results_for_java_call(slow_call);
4445 // this->control() comes from set_results_for_java_call
4446 result_reg->init_req(_slow_path, control());
4447 result_val->init_req(_slow_path, slow_result);
4448 result_i_o ->set_req(_slow_path, i_o());
4449 result_mem ->set_req(_slow_path, reset_memory());
4450 }
4452 // Return the combined state.
4453 set_control( _gvn.transform(result_reg) );
4454 set_i_o( _gvn.transform(result_i_o) );
4455 set_all_memory( _gvn.transform(result_mem) );
4456 } //original reexecute and sp are set back here
4458 push(_gvn.transform(result_val));
4460 return true;
4461 }
4464 // constants for computing the copy function
4465 enum {
4466 COPYFUNC_UNALIGNED = 0,
4467 COPYFUNC_ALIGNED = 1, // src, dest aligned to HeapWordSize
4468 COPYFUNC_CONJOINT = 0,
4469 COPYFUNC_DISJOINT = 2 // src != dest, or transfer can descend
4470 };
4472 // Note: The condition "disjoint" applies also for overlapping copies
4473 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
4474 static address
4475 select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
4476 int selector =
4477 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
4478 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
4480 #define RETURN_STUB(xxx_arraycopy) { \
4481 name = #xxx_arraycopy; \
4482 return StubRoutines::xxx_arraycopy(); }
4484 #define RETURN_STUB_PARM(xxx_arraycopy, parm) { \
4485 name = #xxx_arraycopy; \
4486 return StubRoutines::xxx_arraycopy(parm); }
4488 switch (t) {
4489 case T_BYTE:
4490 case T_BOOLEAN:
4491 switch (selector) {
4492 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_arraycopy);
4493 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_arraycopy);
4494 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jbyte_disjoint_arraycopy);
4495 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jbyte_disjoint_arraycopy);
4496 }
4497 case T_CHAR:
4498 case T_SHORT:
4499 switch (selector) {
4500 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_arraycopy);
4501 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_arraycopy);
4502 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jshort_disjoint_arraycopy);
4503 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jshort_disjoint_arraycopy);
4504 }
4505 case T_INT:
4506 case T_FLOAT:
4507 switch (selector) {
4508 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_arraycopy);
4509 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_arraycopy);
4510 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jint_disjoint_arraycopy);
4511 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jint_disjoint_arraycopy);
4512 }
4513 case T_DOUBLE:
4514 case T_LONG:
4515 switch (selector) {
4516 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_arraycopy);
4517 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_arraycopy);
4518 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(jlong_disjoint_arraycopy);
4519 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
4520 }
4521 case T_ARRAY:
4522 case T_OBJECT:
4523 switch (selector) {
4524 case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
4525 case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
4526 case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
4527 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
4528 }
4529 default:
4530 ShouldNotReachHere();
4531 return NULL;
4532 }
4534 #undef RETURN_STUB
4535 #undef RETURN_STUB_PARM
4536 }
4538 //------------------------------basictype2arraycopy----------------------------
4539 address LibraryCallKit::basictype2arraycopy(BasicType t,
4540 Node* src_offset,
4541 Node* dest_offset,
4542 bool disjoint_bases,
4543 const char* &name,
4544 bool dest_uninitialized) {
4545 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
4546 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
4548 bool aligned = false;
4549 bool disjoint = disjoint_bases;
4551 // if the offsets are the same, we can treat the memory regions as
4552 // disjoint, because either the memory regions are in different arrays,
4553 // or they are identical (which we can treat as disjoint.) We can also
4554 // treat a copy with a destination index less that the source index
4555 // as disjoint since a low->high copy will work correctly in this case.
4556 if (src_offset_inttype != NULL && src_offset_inttype->is_con() &&
4557 dest_offset_inttype != NULL && dest_offset_inttype->is_con()) {
4558 // both indices are constants
4559 int s_offs = src_offset_inttype->get_con();
4560 int d_offs = dest_offset_inttype->get_con();
4561 int element_size = type2aelembytes(t);
4562 aligned = ((arrayOopDesc::base_offset_in_bytes(t) + s_offs * element_size) % HeapWordSize == 0) &&
4563 ((arrayOopDesc::base_offset_in_bytes(t) + d_offs * element_size) % HeapWordSize == 0);
4564 if (s_offs >= d_offs) disjoint = true;
4565 } else if (src_offset == dest_offset && src_offset != NULL) {
4566 // This can occur if the offsets are identical non-constants.
4567 disjoint = true;
4568 }
4570 return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
4571 }
4574 //------------------------------inline_arraycopy-----------------------
4575 bool LibraryCallKit::inline_arraycopy() {
4576 // Restore the stack and pop off the arguments.
4577 int nargs = 5; // 2 oops, 3 ints, no size_t or long
4578 assert(callee()->signature()->size() == nargs, "copy has 5 arguments");
4580 Node *src = argument(0);
4581 Node *src_offset = argument(1);
4582 Node *dest = argument(2);
4583 Node *dest_offset = argument(3);
4584 Node *length = argument(4);
4586 // Compile time checks. If any of these checks cannot be verified at compile time,
4587 // we do not make a fast path for this call. Instead, we let the call remain as it
4588 // is. The checks we choose to mandate at compile time are:
4589 //
4590 // (1) src and dest are arrays.
4591 const Type* src_type = src->Value(&_gvn);
4592 const Type* dest_type = dest->Value(&_gvn);
4593 const TypeAryPtr* top_src = src_type->isa_aryptr();
4594 const TypeAryPtr* top_dest = dest_type->isa_aryptr();
4595 if (top_src == NULL || top_src->klass() == NULL ||
4596 top_dest == NULL || top_dest->klass() == NULL) {
4597 // Conservatively insert a memory barrier on all memory slices.
4598 // Do not let writes into the source float below the arraycopy.
4599 insert_mem_bar(Op_MemBarCPUOrder);
4601 // Call StubRoutines::generic_arraycopy stub.
4602 generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT,
4603 src, src_offset, dest, dest_offset, length);
4605 // Do not let reads from the destination float above the arraycopy.
4606 // Since we cannot type the arrays, we don't know which slices
4607 // might be affected. We could restrict this barrier only to those
4608 // memory slices which pertain to array elements--but don't bother.
4609 if (!InsertMemBarAfterArraycopy)
4610 // (If InsertMemBarAfterArraycopy, there is already one in place.)
4611 insert_mem_bar(Op_MemBarCPUOrder);
4612 return true;
4613 }
4615 // (2) src and dest arrays must have elements of the same BasicType
4616 // Figure out the size and type of the elements we will be copying.
4617 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type();
4618 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type();
4619 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
4620 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
4622 if (src_elem != dest_elem || dest_elem == T_VOID) {
4623 // The component types are not the same or are not recognized. Punt.
4624 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
4625 generate_slow_arraycopy(TypePtr::BOTTOM,
4626 src, src_offset, dest, dest_offset, length,
4627 /*dest_uninitialized*/false);
4628 return true;
4629 }
4631 //---------------------------------------------------------------------------
4632 // We will make a fast path for this call to arraycopy.
4634 // We have the following tests left to perform:
4635 //
4636 // (3) src and dest must not be null.
4637 // (4) src_offset must not be negative.
4638 // (5) dest_offset must not be negative.
4639 // (6) length must not be negative.
4640 // (7) src_offset + length must not exceed length of src.
4641 // (8) dest_offset + length must not exceed length of dest.
4642 // (9) each element of an oop array must be assignable
4644 RegionNode* slow_region = new (C, 1) RegionNode(1);
4645 record_for_igvn(slow_region);
4647 // (3) operands must not be null
4648 // We currently perform our null checks with the do_null_check routine.
4649 // This means that the null exceptions will be reported in the caller
4650 // rather than (correctly) reported inside of the native arraycopy call.
4651 // This should be corrected, given time. We do our null check with the
4652 // stack pointer restored.
4653 _sp += nargs;
4654 src = do_null_check(src, T_ARRAY);
4655 dest = do_null_check(dest, T_ARRAY);
4656 _sp -= nargs;
4658 // (4) src_offset must not be negative.
4659 generate_negative_guard(src_offset, slow_region);
4661 // (5) dest_offset must not be negative.
4662 generate_negative_guard(dest_offset, slow_region);
4664 // (6) length must not be negative (moved to generate_arraycopy()).
4665 // generate_negative_guard(length, slow_region);
4667 // (7) src_offset + length must not exceed length of src.
4668 generate_limit_guard(src_offset, length,
4669 load_array_length(src),
4670 slow_region);
4672 // (8) dest_offset + length must not exceed length of dest.
4673 generate_limit_guard(dest_offset, length,
4674 load_array_length(dest),
4675 slow_region);
4677 // (9) each element of an oop array must be assignable
4678 // The generate_arraycopy subroutine checks this.
4680 // This is where the memory effects are placed:
4681 const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem);
4682 generate_arraycopy(adr_type, dest_elem,
4683 src, src_offset, dest, dest_offset, length,
4684 false, false, slow_region);
4686 return true;
4687 }
4689 //-----------------------------generate_arraycopy----------------------
4690 // Generate an optimized call to arraycopy.
4691 // Caller must guard against non-arrays.
4692 // Caller must determine a common array basic-type for both arrays.
4693 // Caller must validate offsets against array bounds.
4694 // The slow_region has already collected guard failure paths
4695 // (such as out of bounds length or non-conformable array types).
4696 // The generated code has this shape, in general:
4697 //
4698 // if (length == 0) return // via zero_path
4699 // slowval = -1
4700 // if (types unknown) {
4701 // slowval = call generic copy loop
4702 // if (slowval == 0) return // via checked_path
4703 // } else if (indexes in bounds) {
4704 // if ((is object array) && !(array type check)) {
4705 // slowval = call checked copy loop
4706 // if (slowval == 0) return // via checked_path
4707 // } else {
4708 // call bulk copy loop
4709 // return // via fast_path
4710 // }
4711 // }
4712 // // adjust params for remaining work:
4713 // if (slowval != -1) {
4714 // n = -1^slowval; src_offset += n; dest_offset += n; length -= n
4715 // }
4716 // slow_region:
4717 // call slow arraycopy(src, src_offset, dest, dest_offset, length)
4718 // return // via slow_call_path
4719 //
4720 // This routine is used from several intrinsics: System.arraycopy,
4721 // Object.clone (the array subcase), and Arrays.copyOf[Range].
4722 //
4723 void
4724 LibraryCallKit::generate_arraycopy(const TypePtr* adr_type,
4725 BasicType basic_elem_type,
4726 Node* src, Node* src_offset,
4727 Node* dest, Node* dest_offset,
4728 Node* copy_length,
4729 bool disjoint_bases,
4730 bool length_never_negative,
4731 RegionNode* slow_region) {
4733 if (slow_region == NULL) {
4734 slow_region = new(C,1) RegionNode(1);
4735 record_for_igvn(slow_region);
4736 }
4738 Node* original_dest = dest;
4739 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
4740 bool dest_uninitialized = false;
4742 // See if this is the initialization of a newly-allocated array.
4743 // If so, we will take responsibility here for initializing it to zero.
4744 // (Note: Because tightly_coupled_allocation performs checks on the
4745 // out-edges of the dest, we need to avoid making derived pointers
4746 // from it until we have checked its uses.)
4747 if (ReduceBulkZeroing
4748 && !ZeroTLAB // pointless if already zeroed
4749 && basic_elem_type != T_CONFLICT // avoid corner case
4750 && !_gvn.eqv_uncast(src, dest)
4751 && ((alloc = tightly_coupled_allocation(dest, slow_region))
4752 != NULL)
4753 && _gvn.find_int_con(alloc->in(AllocateNode::ALength), 1) > 0
4754 && alloc->maybe_set_complete(&_gvn)) {
4755 // "You break it, you buy it."
4756 InitializeNode* init = alloc->initialization();
4757 assert(init->is_complete(), "we just did this");
4758 assert(dest->is_CheckCastPP(), "sanity");
4759 assert(dest->in(0)->in(0) == init, "dest pinned");
4760 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
4761 // From this point on, every exit path is responsible for
4762 // initializing any non-copied parts of the object to zero.
4763 // Also, if this flag is set we make sure that arraycopy interacts properly
4764 // with G1, eliding pre-barriers. See CR 6627983.
4765 dest_uninitialized = true;
4766 } else {
4767 // No zeroing elimination here.
4768 alloc = NULL;
4769 //original_dest = dest;
4770 //dest_uninitialized = false;
4771 }
4773 // Results are placed here:
4774 enum { fast_path = 1, // normal void-returning assembly stub
4775 checked_path = 2, // special assembly stub with cleanup
4776 slow_call_path = 3, // something went wrong; call the VM
4777 zero_path = 4, // bypass when length of copy is zero
4778 bcopy_path = 5, // copy primitive array by 64-bit blocks
4779 PATH_LIMIT = 6
4780 };
4781 RegionNode* result_region = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT);
4782 PhiNode* result_i_o = new(C, PATH_LIMIT) PhiNode(result_region, Type::ABIO);
4783 PhiNode* result_memory = new(C, PATH_LIMIT) PhiNode(result_region, Type::MEMORY, adr_type);
4784 record_for_igvn(result_region);
4785 _gvn.set_type_bottom(result_i_o);
4786 _gvn.set_type_bottom(result_memory);
4787 assert(adr_type != TypePtr::BOTTOM, "must be RawMem or a T[] slice");
4789 // The slow_control path:
4790 Node* slow_control;
4791 Node* slow_i_o = i_o();
4792 Node* slow_mem = memory(adr_type);
4793 debug_only(slow_control = (Node*) badAddress);
4795 // Checked control path:
4796 Node* checked_control = top();
4797 Node* checked_mem = NULL;
4798 Node* checked_i_o = NULL;
4799 Node* checked_value = NULL;
4801 if (basic_elem_type == T_CONFLICT) {
4802 assert(!dest_uninitialized, "");
4803 Node* cv = generate_generic_arraycopy(adr_type,
4804 src, src_offset, dest, dest_offset,
4805 copy_length, dest_uninitialized);
4806 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4807 checked_control = control();
4808 checked_i_o = i_o();
4809 checked_mem = memory(adr_type);
4810 checked_value = cv;
4811 set_control(top()); // no fast path
4812 }
4814 Node* not_pos = generate_nonpositive_guard(copy_length, length_never_negative);
4815 if (not_pos != NULL) {
4816 PreserveJVMState pjvms(this);
4817 set_control(not_pos);
4819 // (6) length must not be negative.
4820 if (!length_never_negative) {
4821 generate_negative_guard(copy_length, slow_region);
4822 }
4824 // copy_length is 0.
4825 if (!stopped() && dest_uninitialized) {
4826 Node* dest_length = alloc->in(AllocateNode::ALength);
4827 if (_gvn.eqv_uncast(copy_length, dest_length)
4828 || _gvn.find_int_con(dest_length, 1) <= 0) {
4829 // There is no zeroing to do. No need for a secondary raw memory barrier.
4830 } else {
4831 // Clear the whole thing since there are no source elements to copy.
4832 generate_clear_array(adr_type, dest, basic_elem_type,
4833 intcon(0), NULL,
4834 alloc->in(AllocateNode::AllocSize));
4835 // Use a secondary InitializeNode as raw memory barrier.
4836 // Currently it is needed only on this path since other
4837 // paths have stub or runtime calls as raw memory barriers.
4838 InitializeNode* init = insert_mem_bar_volatile(Op_Initialize,
4839 Compile::AliasIdxRaw,
4840 top())->as_Initialize();
4841 init->set_complete(&_gvn); // (there is no corresponding AllocateNode)
4842 }
4843 }
4845 // Present the results of the fast call.
4846 result_region->init_req(zero_path, control());
4847 result_i_o ->init_req(zero_path, i_o());
4848 result_memory->init_req(zero_path, memory(adr_type));
4849 }
4851 if (!stopped() && dest_uninitialized) {
4852 // We have to initialize the *uncopied* part of the array to zero.
4853 // The copy destination is the slice dest[off..off+len]. The other slices
4854 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
4855 Node* dest_size = alloc->in(AllocateNode::AllocSize);
4856 Node* dest_length = alloc->in(AllocateNode::ALength);
4857 Node* dest_tail = _gvn.transform( new(C,3) AddINode(dest_offset,
4858 copy_length) );
4860 // If there is a head section that needs zeroing, do it now.
4861 if (find_int_con(dest_offset, -1) != 0) {
4862 generate_clear_array(adr_type, dest, basic_elem_type,
4863 intcon(0), dest_offset,
4864 NULL);
4865 }
4867 // Next, perform a dynamic check on the tail length.
4868 // It is often zero, and we can win big if we prove this.
4869 // There are two wins: Avoid generating the ClearArray
4870 // with its attendant messy index arithmetic, and upgrade
4871 // the copy to a more hardware-friendly word size of 64 bits.
4872 Node* tail_ctl = NULL;
4873 if (!stopped() && !_gvn.eqv_uncast(dest_tail, dest_length)) {
4874 Node* cmp_lt = _gvn.transform( new(C,3) CmpINode(dest_tail, dest_length) );
4875 Node* bol_lt = _gvn.transform( new(C,2) BoolNode(cmp_lt, BoolTest::lt) );
4876 tail_ctl = generate_slow_guard(bol_lt, NULL);
4877 assert(tail_ctl != NULL || !stopped(), "must be an outcome");
4878 }
4880 // At this point, let's assume there is no tail.
4881 if (!stopped() && alloc != NULL && basic_elem_type != T_OBJECT) {
4882 // There is no tail. Try an upgrade to a 64-bit copy.
4883 bool didit = false;
4884 { PreserveJVMState pjvms(this);
4885 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
4886 src, src_offset, dest, dest_offset,
4887 dest_size, dest_uninitialized);
4888 if (didit) {
4889 // Present the results of the block-copying fast call.
4890 result_region->init_req(bcopy_path, control());
4891 result_i_o ->init_req(bcopy_path, i_o());
4892 result_memory->init_req(bcopy_path, memory(adr_type));
4893 }
4894 }
4895 if (didit)
4896 set_control(top()); // no regular fast path
4897 }
4899 // Clear the tail, if any.
4900 if (tail_ctl != NULL) {
4901 Node* notail_ctl = stopped() ? NULL : control();
4902 set_control(tail_ctl);
4903 if (notail_ctl == NULL) {
4904 generate_clear_array(adr_type, dest, basic_elem_type,
4905 dest_tail, NULL,
4906 dest_size);
4907 } else {
4908 // Make a local merge.
4909 Node* done_ctl = new(C,3) RegionNode(3);
4910 Node* done_mem = new(C,3) PhiNode(done_ctl, Type::MEMORY, adr_type);
4911 done_ctl->init_req(1, notail_ctl);
4912 done_mem->init_req(1, memory(adr_type));
4913 generate_clear_array(adr_type, dest, basic_elem_type,
4914 dest_tail, NULL,
4915 dest_size);
4916 done_ctl->init_req(2, control());
4917 done_mem->init_req(2, memory(adr_type));
4918 set_control( _gvn.transform(done_ctl) );
4919 set_memory( _gvn.transform(done_mem), adr_type );
4920 }
4921 }
4922 }
4924 BasicType copy_type = basic_elem_type;
4925 assert(basic_elem_type != T_ARRAY, "caller must fix this");
4926 if (!stopped() && copy_type == T_OBJECT) {
4927 // If src and dest have compatible element types, we can copy bits.
4928 // Types S[] and D[] are compatible if D is a supertype of S.
4929 //
4930 // If they are not, we will use checked_oop_disjoint_arraycopy,
4931 // which performs a fast optimistic per-oop check, and backs off
4932 // further to JVM_ArrayCopy on the first per-oop check that fails.
4933 // (Actually, we don't move raw bits only; the GC requires card marks.)
4935 // Get the klassOop for both src and dest
4936 Node* src_klass = load_object_klass(src);
4937 Node* dest_klass = load_object_klass(dest);
4939 // Generate the subtype check.
4940 // This might fold up statically, or then again it might not.
4941 //
4942 // Non-static example: Copying List<String>.elements to a new String[].
4943 // The backing store for a List<String> is always an Object[],
4944 // but its elements are always type String, if the generic types
4945 // are correct at the source level.
4946 //
4947 // Test S[] against D[], not S against D, because (probably)
4948 // the secondary supertype cache is less busy for S[] than S.
4949 // This usually only matters when D is an interface.
4950 Node* not_subtype_ctrl = gen_subtype_check(src_klass, dest_klass);
4951 // Plug failing path into checked_oop_disjoint_arraycopy
4952 if (not_subtype_ctrl != top()) {
4953 PreserveJVMState pjvms(this);
4954 set_control(not_subtype_ctrl);
4955 // (At this point we can assume disjoint_bases, since types differ.)
4956 int ek_offset = objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc);
4957 Node* p1 = basic_plus_adr(dest_klass, ek_offset);
4958 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
4959 Node* dest_elem_klass = _gvn.transform(n1);
4960 Node* cv = generate_checkcast_arraycopy(adr_type,
4961 dest_elem_klass,
4962 src, src_offset, dest, dest_offset,
4963 ConvI2X(copy_length), dest_uninitialized);
4964 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
4965 checked_control = control();
4966 checked_i_o = i_o();
4967 checked_mem = memory(adr_type);
4968 checked_value = cv;
4969 }
4970 // At this point we know we do not need type checks on oop stores.
4972 // Let's see if we need card marks:
4973 if (alloc != NULL && use_ReduceInitialCardMarks()) {
4974 // If we do not need card marks, copy using the jint or jlong stub.
4975 copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT);
4976 assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type),
4977 "sizes agree");
4978 }
4979 }
4981 if (!stopped()) {
4982 // Generate the fast path, if possible.
4983 PreserveJVMState pjvms(this);
4984 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
4985 src, src_offset, dest, dest_offset,
4986 ConvI2X(copy_length), dest_uninitialized);
4988 // Present the results of the fast call.
4989 result_region->init_req(fast_path, control());
4990 result_i_o ->init_req(fast_path, i_o());
4991 result_memory->init_req(fast_path, memory(adr_type));
4992 }
4994 // Here are all the slow paths up to this point, in one bundle:
4995 slow_control = top();
4996 if (slow_region != NULL)
4997 slow_control = _gvn.transform(slow_region);
4998 debug_only(slow_region = (RegionNode*)badAddress);
5000 set_control(checked_control);
5001 if (!stopped()) {
5002 // Clean up after the checked call.
5003 // The returned value is either 0 or -1^K,
5004 // where K = number of partially transferred array elements.
5005 Node* cmp = _gvn.transform( new(C, 3) CmpINode(checked_value, intcon(0)) );
5006 Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::eq) );
5007 IfNode* iff = create_and_map_if(control(), bol, PROB_MAX, COUNT_UNKNOWN);
5009 // If it is 0, we are done, so transfer to the end.
5010 Node* checks_done = _gvn.transform( new(C, 1) IfTrueNode(iff) );
5011 result_region->init_req(checked_path, checks_done);
5012 result_i_o ->init_req(checked_path, checked_i_o);
5013 result_memory->init_req(checked_path, checked_mem);
5015 // If it is not zero, merge into the slow call.
5016 set_control( _gvn.transform( new(C, 1) IfFalseNode(iff) ));
5017 RegionNode* slow_reg2 = new(C, 3) RegionNode(3);
5018 PhiNode* slow_i_o2 = new(C, 3) PhiNode(slow_reg2, Type::ABIO);
5019 PhiNode* slow_mem2 = new(C, 3) PhiNode(slow_reg2, Type::MEMORY, adr_type);
5020 record_for_igvn(slow_reg2);
5021 slow_reg2 ->init_req(1, slow_control);
5022 slow_i_o2 ->init_req(1, slow_i_o);
5023 slow_mem2 ->init_req(1, slow_mem);
5024 slow_reg2 ->init_req(2, control());
5025 slow_i_o2 ->init_req(2, checked_i_o);
5026 slow_mem2 ->init_req(2, checked_mem);
5028 slow_control = _gvn.transform(slow_reg2);
5029 slow_i_o = _gvn.transform(slow_i_o2);
5030 slow_mem = _gvn.transform(slow_mem2);
5032 if (alloc != NULL) {
5033 // We'll restart from the very beginning, after zeroing the whole thing.
5034 // This can cause double writes, but that's OK since dest is brand new.
5035 // So we ignore the low 31 bits of the value returned from the stub.
5036 } else {
5037 // We must continue the copy exactly where it failed, or else
5038 // another thread might see the wrong number of writes to dest.
5039 Node* checked_offset = _gvn.transform( new(C, 3) XorINode(checked_value, intcon(-1)) );
5040 Node* slow_offset = new(C, 3) PhiNode(slow_reg2, TypeInt::INT);
5041 slow_offset->init_req(1, intcon(0));
5042 slow_offset->init_req(2, checked_offset);
5043 slow_offset = _gvn.transform(slow_offset);
5045 // Adjust the arguments by the conditionally incoming offset.
5046 Node* src_off_plus = _gvn.transform( new(C, 3) AddINode(src_offset, slow_offset) );
5047 Node* dest_off_plus = _gvn.transform( new(C, 3) AddINode(dest_offset, slow_offset) );
5048 Node* length_minus = _gvn.transform( new(C, 3) SubINode(copy_length, slow_offset) );
5050 // Tweak the node variables to adjust the code produced below:
5051 src_offset = src_off_plus;
5052 dest_offset = dest_off_plus;
5053 copy_length = length_minus;
5054 }
5055 }
5057 set_control(slow_control);
5058 if (!stopped()) {
5059 // Generate the slow path, if needed.
5060 PreserveJVMState pjvms(this); // replace_in_map may trash the map
5062 set_memory(slow_mem, adr_type);
5063 set_i_o(slow_i_o);
5065 if (dest_uninitialized) {
5066 generate_clear_array(adr_type, dest, basic_elem_type,
5067 intcon(0), NULL,
5068 alloc->in(AllocateNode::AllocSize));
5069 }
5071 generate_slow_arraycopy(adr_type,
5072 src, src_offset, dest, dest_offset,
5073 copy_length, /*dest_uninitialized*/false);
5075 result_region->init_req(slow_call_path, control());
5076 result_i_o ->init_req(slow_call_path, i_o());
5077 result_memory->init_req(slow_call_path, memory(adr_type));
5078 }
5080 // Remove unused edges.
5081 for (uint i = 1; i < result_region->req(); i++) {
5082 if (result_region->in(i) == NULL)
5083 result_region->init_req(i, top());
5084 }
5086 // Finished; return the combined state.
5087 set_control( _gvn.transform(result_region) );
5088 set_i_o( _gvn.transform(result_i_o) );
5089 set_memory( _gvn.transform(result_memory), adr_type );
5091 // The memory edges above are precise in order to model effects around
5092 // array copies accurately to allow value numbering of field loads around
5093 // arraycopy. Such field loads, both before and after, are common in Java
5094 // collections and similar classes involving header/array data structures.
5095 //
5096 // But with low number of register or when some registers are used or killed
5097 // by arraycopy calls it causes registers spilling on stack. See 6544710.
5098 // The next memory barrier is added to avoid it. If the arraycopy can be
5099 // optimized away (which it can, sometimes) then we can manually remove
5100 // the membar also.
5101 //
5102 // Do not let reads from the cloned object float above the arraycopy.
5103 if (InsertMemBarAfterArraycopy || alloc != NULL)
5104 insert_mem_bar(Op_MemBarCPUOrder);
5105 }
5108 // Helper function which determines if an arraycopy immediately follows
5109 // an allocation, with no intervening tests or other escapes for the object.
5110 AllocateArrayNode*
5111 LibraryCallKit::tightly_coupled_allocation(Node* ptr,
5112 RegionNode* slow_region) {
5113 if (stopped()) return NULL; // no fast path
5114 if (C->AliasLevel() == 0) return NULL; // no MergeMems around
5116 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn);
5117 if (alloc == NULL) return NULL;
5119 Node* rawmem = memory(Compile::AliasIdxRaw);
5120 // Is the allocation's memory state untouched?
5121 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) {
5122 // Bail out if there have been raw-memory effects since the allocation.
5123 // (Example: There might have been a call or safepoint.)
5124 return NULL;
5125 }
5126 rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw);
5127 if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) {
5128 return NULL;
5129 }
5131 // There must be no unexpected observers of this allocation.
5132 for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) {
5133 Node* obs = ptr->fast_out(i);
5134 if (obs != this->map()) {
5135 return NULL;
5136 }
5137 }
5139 // This arraycopy must unconditionally follow the allocation of the ptr.
5140 Node* alloc_ctl = ptr->in(0);
5141 assert(just_allocated_object(alloc_ctl) == ptr, "most recent allo");
5143 Node* ctl = control();
5144 while (ctl != alloc_ctl) {
5145 // There may be guards which feed into the slow_region.
5146 // Any other control flow means that we might not get a chance
5147 // to finish initializing the allocated object.
5148 if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) {
5149 IfNode* iff = ctl->in(0)->as_If();
5150 Node* not_ctl = iff->proj_out(1 - ctl->as_Proj()->_con);
5151 assert(not_ctl != NULL && not_ctl != ctl, "found alternate");
5152 if (slow_region != NULL && slow_region->find_edge(not_ctl) >= 1) {
5153 ctl = iff->in(0); // This test feeds the known slow_region.
5154 continue;
5155 }
5156 // One more try: Various low-level checks bottom out in
5157 // uncommon traps. If the debug-info of the trap omits
5158 // any reference to the allocation, as we've already
5159 // observed, then there can be no objection to the trap.
5160 bool found_trap = false;
5161 for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) {
5162 Node* obs = not_ctl->fast_out(j);
5163 if (obs->in(0) == not_ctl && obs->is_Call() &&
5164 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) {
5165 found_trap = true; break;
5166 }
5167 }
5168 if (found_trap) {
5169 ctl = iff->in(0); // This test feeds a harmless uncommon trap.
5170 continue;
5171 }
5172 }
5173 return NULL;
5174 }
5176 // If we get this far, we have an allocation which immediately
5177 // precedes the arraycopy, and we can take over zeroing the new object.
5178 // The arraycopy will finish the initialization, and provide
5179 // a new control state to which we will anchor the destination pointer.
5181 return alloc;
5182 }
5184 // Helper for initialization of arrays, creating a ClearArray.
5185 // It writes zero bits in [start..end), within the body of an array object.
5186 // The memory effects are all chained onto the 'adr_type' alias category.
5187 //
5188 // Since the object is otherwise uninitialized, we are free
5189 // to put a little "slop" around the edges of the cleared area,
5190 // as long as it does not go back into the array's header,
5191 // or beyond the array end within the heap.
5192 //
5193 // The lower edge can be rounded down to the nearest jint and the
5194 // upper edge can be rounded up to the nearest MinObjAlignmentInBytes.
5195 //
5196 // Arguments:
5197 // adr_type memory slice where writes are generated
5198 // dest oop of the destination array
5199 // basic_elem_type element type of the destination
5200 // slice_idx array index of first element to store
5201 // slice_len number of elements to store (or NULL)
5202 // dest_size total size in bytes of the array object
5203 //
5204 // Exactly one of slice_len or dest_size must be non-NULL.
5205 // If dest_size is non-NULL, zeroing extends to the end of the object.
5206 // If slice_len is non-NULL, the slice_idx value must be a constant.
5207 void
5208 LibraryCallKit::generate_clear_array(const TypePtr* adr_type,
5209 Node* dest,
5210 BasicType basic_elem_type,
5211 Node* slice_idx,
5212 Node* slice_len,
5213 Node* dest_size) {
5214 // one or the other but not both of slice_len and dest_size:
5215 assert((slice_len != NULL? 1: 0) + (dest_size != NULL? 1: 0) == 1, "");
5216 if (slice_len == NULL) slice_len = top();
5217 if (dest_size == NULL) dest_size = top();
5219 // operate on this memory slice:
5220 Node* mem = memory(adr_type); // memory slice to operate on
5222 // scaling and rounding of indexes:
5223 int scale = exact_log2(type2aelembytes(basic_elem_type));
5224 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5225 int clear_low = (-1 << scale) & (BytesPerInt - 1);
5226 int bump_bit = (-1 << scale) & BytesPerInt;
5228 // determine constant starts and ends
5229 const intptr_t BIG_NEG = -128;
5230 assert(BIG_NEG + 2*abase < 0, "neg enough");
5231 intptr_t slice_idx_con = (intptr_t) find_int_con(slice_idx, BIG_NEG);
5232 intptr_t slice_len_con = (intptr_t) find_int_con(slice_len, BIG_NEG);
5233 if (slice_len_con == 0) {
5234 return; // nothing to do here
5235 }
5236 intptr_t start_con = (abase + (slice_idx_con << scale)) & ~clear_low;
5237 intptr_t end_con = find_intptr_t_con(dest_size, -1);
5238 if (slice_idx_con >= 0 && slice_len_con >= 0) {
5239 assert(end_con < 0, "not two cons");
5240 end_con = round_to(abase + ((slice_idx_con + slice_len_con) << scale),
5241 BytesPerLong);
5242 }
5244 if (start_con >= 0 && end_con >= 0) {
5245 // Constant start and end. Simple.
5246 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5247 start_con, end_con, &_gvn);
5248 } else if (start_con >= 0 && dest_size != top()) {
5249 // Constant start, pre-rounded end after the tail of the array.
5250 Node* end = dest_size;
5251 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5252 start_con, end, &_gvn);
5253 } else if (start_con >= 0 && slice_len != top()) {
5254 // Constant start, non-constant end. End needs rounding up.
5255 // End offset = round_up(abase + ((slice_idx_con + slice_len) << scale), 8)
5256 intptr_t end_base = abase + (slice_idx_con << scale);
5257 int end_round = (-1 << scale) & (BytesPerLong - 1);
5258 Node* end = ConvI2X(slice_len);
5259 if (scale != 0)
5260 end = _gvn.transform( new(C,3) LShiftXNode(end, intcon(scale) ));
5261 end_base += end_round;
5262 end = _gvn.transform( new(C,3) AddXNode(end, MakeConX(end_base)) );
5263 end = _gvn.transform( new(C,3) AndXNode(end, MakeConX(~end_round)) );
5264 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5265 start_con, end, &_gvn);
5266 } else if (start_con < 0 && dest_size != top()) {
5267 // Non-constant start, pre-rounded end after the tail of the array.
5268 // This is almost certainly a "round-to-end" operation.
5269 Node* start = slice_idx;
5270 start = ConvI2X(start);
5271 if (scale != 0)
5272 start = _gvn.transform( new(C,3) LShiftXNode( start, intcon(scale) ));
5273 start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(abase)) );
5274 if ((bump_bit | clear_low) != 0) {
5275 int to_clear = (bump_bit | clear_low);
5276 // Align up mod 8, then store a jint zero unconditionally
5277 // just before the mod-8 boundary.
5278 if (((abase + bump_bit) & ~to_clear) - bump_bit
5279 < arrayOopDesc::length_offset_in_bytes() + BytesPerInt) {
5280 bump_bit = 0;
5281 assert((abase & to_clear) == 0, "array base must be long-aligned");
5282 } else {
5283 // Bump 'start' up to (or past) the next jint boundary:
5284 start = _gvn.transform( new(C,3) AddXNode(start, MakeConX(bump_bit)) );
5285 assert((abase & clear_low) == 0, "array base must be int-aligned");
5286 }
5287 // Round bumped 'start' down to jlong boundary in body of array.
5288 start = _gvn.transform( new(C,3) AndXNode(start, MakeConX(~to_clear)) );
5289 if (bump_bit != 0) {
5290 // Store a zero to the immediately preceding jint:
5291 Node* x1 = _gvn.transform( new(C,3) AddXNode(start, MakeConX(-bump_bit)) );
5292 Node* p1 = basic_plus_adr(dest, x1);
5293 mem = StoreNode::make(_gvn, control(), mem, p1, adr_type, intcon(0), T_INT);
5294 mem = _gvn.transform(mem);
5295 }
5296 }
5297 Node* end = dest_size; // pre-rounded
5298 mem = ClearArrayNode::clear_memory(control(), mem, dest,
5299 start, end, &_gvn);
5300 } else {
5301 // Non-constant start, unrounded non-constant end.
5302 // (Nobody zeroes a random midsection of an array using this routine.)
5303 ShouldNotReachHere(); // fix caller
5304 }
5306 // Done.
5307 set_memory(mem, adr_type);
5308 }
5311 bool
5312 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
5313 BasicType basic_elem_type,
5314 AllocateNode* alloc,
5315 Node* src, Node* src_offset,
5316 Node* dest, Node* dest_offset,
5317 Node* dest_size, bool dest_uninitialized) {
5318 // See if there is an advantage from block transfer.
5319 int scale = exact_log2(type2aelembytes(basic_elem_type));
5320 if (scale >= LogBytesPerLong)
5321 return false; // it is already a block transfer
5323 // Look at the alignment of the starting offsets.
5324 int abase = arrayOopDesc::base_offset_in_bytes(basic_elem_type);
5325 const intptr_t BIG_NEG = -128;
5326 assert(BIG_NEG + 2*abase < 0, "neg enough");
5328 intptr_t src_off = abase + ((intptr_t) find_int_con(src_offset, -1) << scale);
5329 intptr_t dest_off = abase + ((intptr_t) find_int_con(dest_offset, -1) << scale);
5330 if (src_off < 0 || dest_off < 0)
5331 // At present, we can only understand constants.
5332 return false;
5334 if (((src_off | dest_off) & (BytesPerLong-1)) != 0) {
5335 // Non-aligned; too bad.
5336 // One more chance: Pick off an initial 32-bit word.
5337 // This is a common case, since abase can be odd mod 8.
5338 if (((src_off | dest_off) & (BytesPerLong-1)) == BytesPerInt &&
5339 ((src_off ^ dest_off) & (BytesPerLong-1)) == 0) {
5340 Node* sptr = basic_plus_adr(src, src_off);
5341 Node* dptr = basic_plus_adr(dest, dest_off);
5342 Node* sval = make_load(control(), sptr, TypeInt::INT, T_INT, adr_type);
5343 store_to_memory(control(), dptr, sval, T_INT, adr_type);
5344 src_off += BytesPerInt;
5345 dest_off += BytesPerInt;
5346 } else {
5347 return false;
5348 }
5349 }
5350 assert(src_off % BytesPerLong == 0, "");
5351 assert(dest_off % BytesPerLong == 0, "");
5353 // Do this copy by giant steps.
5354 Node* sptr = basic_plus_adr(src, src_off);
5355 Node* dptr = basic_plus_adr(dest, dest_off);
5356 Node* countx = dest_size;
5357 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
5358 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
5360 bool disjoint_bases = true; // since alloc != NULL
5361 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
5362 sptr, NULL, dptr, NULL, countx, dest_uninitialized);
5364 return true;
5365 }
5368 // Helper function; generates code for the slow case.
5369 // We make a call to a runtime method which emulates the native method,
5370 // but without the native wrapper overhead.
5371 void
5372 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
5373 Node* src, Node* src_offset,
5374 Node* dest, Node* dest_offset,
5375 Node* copy_length, bool dest_uninitialized) {
5376 assert(!dest_uninitialized, "Invariant");
5377 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
5378 OptoRuntime::slow_arraycopy_Type(),
5379 OptoRuntime::slow_arraycopy_Java(),
5380 "slow_arraycopy", adr_type,
5381 src, src_offset, dest, dest_offset,
5382 copy_length);
5384 // Handle exceptions thrown by this fellow:
5385 make_slow_call_ex(call, env()->Throwable_klass(), false);
5386 }
5388 // Helper function; generates code for cases requiring runtime checks.
5389 Node*
5390 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
5391 Node* dest_elem_klass,
5392 Node* src, Node* src_offset,
5393 Node* dest, Node* dest_offset,
5394 Node* copy_length, bool dest_uninitialized) {
5395 if (stopped()) return NULL;
5397 address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
5398 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5399 return NULL;
5400 }
5402 // Pick out the parameters required to perform a store-check
5403 // for the target array. This is an optimistic check. It will
5404 // look in each non-null element's class, at the desired klass's
5405 // super_check_offset, for the desired klass.
5406 int sco_offset = Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc);
5407 Node* p3 = basic_plus_adr(dest_elem_klass, sco_offset);
5408 Node* n3 = new(C, 3) LoadINode(NULL, memory(p3), p3, _gvn.type(p3)->is_ptr());
5409 Node* check_offset = ConvI2X(_gvn.transform(n3));
5410 Node* check_value = dest_elem_klass;
5412 Node* src_start = array_element_address(src, src_offset, T_OBJECT);
5413 Node* dest_start = array_element_address(dest, dest_offset, T_OBJECT);
5415 // (We know the arrays are never conjoint, because their types differ.)
5416 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5417 OptoRuntime::checkcast_arraycopy_Type(),
5418 copyfunc_addr, "checkcast_arraycopy", adr_type,
5419 // five arguments, of which two are
5420 // intptr_t (jlong in LP64)
5421 src_start, dest_start,
5422 copy_length XTOP,
5423 check_offset XTOP,
5424 check_value);
5426 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5427 }
5430 // Helper function; generates code for cases requiring runtime checks.
5431 Node*
5432 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
5433 Node* src, Node* src_offset,
5434 Node* dest, Node* dest_offset,
5435 Node* copy_length, bool dest_uninitialized) {
5436 assert(!dest_uninitialized, "Invariant");
5437 if (stopped()) return NULL;
5438 address copyfunc_addr = StubRoutines::generic_arraycopy();
5439 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
5440 return NULL;
5441 }
5443 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP,
5444 OptoRuntime::generic_arraycopy_Type(),
5445 copyfunc_addr, "generic_arraycopy", adr_type,
5446 src, src_offset, dest, dest_offset, copy_length);
5448 return _gvn.transform(new (C, 1) ProjNode(call, TypeFunc::Parms));
5449 }
5451 // Helper function; generates the fast out-of-line call to an arraycopy stub.
5452 void
5453 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
5454 BasicType basic_elem_type,
5455 bool disjoint_bases,
5456 Node* src, Node* src_offset,
5457 Node* dest, Node* dest_offset,
5458 Node* copy_length, bool dest_uninitialized) {
5459 if (stopped()) return; // nothing to do
5461 Node* src_start = src;
5462 Node* dest_start = dest;
5463 if (src_offset != NULL || dest_offset != NULL) {
5464 assert(src_offset != NULL && dest_offset != NULL, "");
5465 src_start = array_element_address(src, src_offset, basic_elem_type);
5466 dest_start = array_element_address(dest, dest_offset, basic_elem_type);
5467 }
5469 // Figure out which arraycopy runtime method to call.
5470 const char* copyfunc_name = "arraycopy";
5471 address copyfunc_addr =
5472 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
5473 disjoint_bases, copyfunc_name, dest_uninitialized);
5475 // Call it. Note that the count_ix value is not scaled to a byte-size.
5476 make_runtime_call(RC_LEAF|RC_NO_FP,
5477 OptoRuntime::fast_arraycopy_Type(),
5478 copyfunc_addr, copyfunc_name, adr_type,
5479 src_start, dest_start, copy_length XTOP);
5480 }
5482 //----------------------------inline_reference_get----------------------------
5484 bool LibraryCallKit::inline_reference_get() {
5485 const int nargs = 1; // self
5487 guarantee(java_lang_ref_Reference::referent_offset > 0,
5488 "should have already been set");
5490 int referent_offset = java_lang_ref_Reference::referent_offset;
5492 // Restore the stack and pop off the argument
5493 _sp += nargs;
5494 Node *reference_obj = pop();
5496 // Null check on self without removing any arguments.
5497 _sp += nargs;
5498 reference_obj = do_null_check(reference_obj, T_OBJECT);
5499 _sp -= nargs;;
5501 if (stopped()) return true;
5503 Node *adr = basic_plus_adr(reference_obj, reference_obj, referent_offset);
5505 ciInstanceKlass* klass = env()->Object_klass();
5506 const TypeOopPtr* object_type = TypeOopPtr::make_from_klass(klass);
5508 Node* no_ctrl = NULL;
5509 Node *result = make_load(no_ctrl, adr, object_type, T_OBJECT);
5511 // Use the pre-barrier to record the value in the referent field
5512 pre_barrier(false /* do_load */,
5513 control(),
5514 NULL /* obj */, NULL /* adr */, -1 /* alias_idx */, NULL /* val */, NULL /* val_type */,
5515 result /* pre_val */,
5516 T_OBJECT);
5518 push(result);
5519 return true;
5520 }