280 Node* dest, Node* dest_offset, |
280 Node* dest, Node* dest_offset, |
281 Node* copy_length, bool dest_uninitialized); |
281 Node* copy_length, bool dest_uninitialized); |
282 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind; |
282 typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind; |
283 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind); |
283 bool inline_unsafe_load_store(BasicType type, LoadStoreKind kind); |
284 bool inline_unsafe_ordered_store(BasicType type); |
284 bool inline_unsafe_ordered_store(BasicType type); |
|
285 bool inline_unsafe_fence(vmIntrinsics::ID id); |
285 bool inline_fp_conversions(vmIntrinsics::ID id); |
286 bool inline_fp_conversions(vmIntrinsics::ID id); |
286 bool inline_number_methods(vmIntrinsics::ID id); |
287 bool inline_number_methods(vmIntrinsics::ID id); |
287 bool inline_reference_get(); |
288 bool inline_reference_get(); |
288 bool inline_aescrypt_Block(vmIntrinsics::ID id); |
289 bool inline_aescrypt_Block(vmIntrinsics::ID id); |
289 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); |
290 bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id); |
332 case vmIntrinsics::_getAndAddInt: |
333 case vmIntrinsics::_getAndAddInt: |
333 case vmIntrinsics::_getAndAddLong: |
334 case vmIntrinsics::_getAndAddLong: |
334 case vmIntrinsics::_getAndSetInt: |
335 case vmIntrinsics::_getAndSetInt: |
335 case vmIntrinsics::_getAndSetLong: |
336 case vmIntrinsics::_getAndSetLong: |
336 case vmIntrinsics::_getAndSetObject: |
337 case vmIntrinsics::_getAndSetObject: |
|
338 case vmIntrinsics::_loadFence: |
|
339 case vmIntrinsics::_storeFence: |
|
340 case vmIntrinsics::_fullFence: |
337 break; // InlineNatives does not control String.compareTo |
341 break; // InlineNatives does not control String.compareTo |
338 case vmIntrinsics::_Reference_get: |
342 case vmIntrinsics::_Reference_get: |
339 break; // InlineNatives does not control Reference.get |
343 break; // InlineNatives does not control Reference.get |
340 default: |
344 default: |
341 return NULL; |
345 return NULL; |
534 const int bci = kit.bci(); |
538 const int bci = kit.bci(); |
535 |
539 |
536 // Try to inline the intrinsic. |
540 // Try to inline the intrinsic. |
537 if (kit.try_to_inline()) { |
541 if (kit.try_to_inline()) { |
538 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
542 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
539 CompileTask::print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"); |
543 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"); |
540 } |
544 } |
541 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
545 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
542 if (C->log()) { |
546 if (C->log()) { |
543 C->log()->elem("intrinsic id='%s'%s nodes='%d'", |
547 C->log()->elem("intrinsic id='%s'%s nodes='%d'", |
544 vmIntrinsics::name_at(intrinsic_id()), |
548 vmIntrinsics::name_at(intrinsic_id()), |
553 // The intrinsic bailed out |
557 // The intrinsic bailed out |
554 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
558 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
555 if (jvms->has_method()) { |
559 if (jvms->has_method()) { |
556 // Not a root compile. |
560 // Not a root compile. |
557 const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)"; |
561 const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)"; |
558 CompileTask::print_inlining(callee, jvms->depth() - 1, bci, msg); |
562 C->print_inlining(callee, jvms->depth() - 1, bci, msg); |
559 } else { |
563 } else { |
560 // Root compile |
564 // Root compile |
561 tty->print("Did not generate intrinsic %s%s at bci:%d in", |
565 tty->print("Did not generate intrinsic %s%s at bci:%d in", |
562 vmIntrinsics::name_at(intrinsic_id()), |
566 vmIntrinsics::name_at(intrinsic_id()), |
563 (is_virtual() ? " (virtual)" : ""), bci); |
567 (is_virtual() ? " (virtual)" : ""), bci); |
583 const int bci = kit.bci(); |
587 const int bci = kit.bci(); |
584 |
588 |
585 Node* slow_ctl = kit.try_to_predicate(); |
589 Node* slow_ctl = kit.try_to_predicate(); |
586 if (!kit.failing()) { |
590 if (!kit.failing()) { |
587 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
591 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
588 CompileTask::print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"); |
592 C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)"); |
589 } |
593 } |
590 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
594 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); |
591 if (C->log()) { |
595 if (C->log()) { |
592 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'", |
596 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'", |
593 vmIntrinsics::name_at(intrinsic_id()), |
597 vmIntrinsics::name_at(intrinsic_id()), |
600 // The intrinsic bailed out |
604 // The intrinsic bailed out |
601 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
605 if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) { |
602 if (jvms->has_method()) { |
606 if (jvms->has_method()) { |
603 // Not a root compile. |
607 // Not a root compile. |
604 const char* msg = "failed to generate predicate for intrinsic"; |
608 const char* msg = "failed to generate predicate for intrinsic"; |
605 CompileTask::print_inlining(kit.callee(), jvms->depth() - 1, bci, msg); |
609 C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg); |
606 } else { |
610 } else { |
607 // Root compile |
611 // Root compile |
608 tty->print("Did not generate predicate for intrinsic %s%s at bci:%d in", |
612 C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in", |
609 vmIntrinsics::name_at(intrinsic_id()), |
613 vmIntrinsics::name_at(intrinsic_id()), |
610 (is_virtual() ? " (virtual)" : ""), bci); |
614 (is_virtual() ? " (virtual)" : ""), bci); |
611 } |
615 } |
612 } |
616 } |
613 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); |
617 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); |
614 return NULL; |
618 return NULL; |
615 } |
619 } |
729 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_xadd); |
733 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_xadd); |
730 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_xadd); |
734 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_xadd); |
731 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_xchg); |
735 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_xchg); |
732 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_xchg); |
736 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_xchg); |
733 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_xchg); |
737 case vmIntrinsics::_getAndSetObject: return inline_unsafe_load_store(T_OBJECT, LS_xchg); |
|
738 |
|
739 case vmIntrinsics::_loadFence: |
|
740 case vmIntrinsics::_storeFence: |
|
741 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id()); |
734 |
742 |
735 case vmIntrinsics::_currentThread: return inline_native_currentThread(); |
743 case vmIntrinsics::_currentThread: return inline_native_currentThread(); |
736 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted(); |
744 case vmIntrinsics::_isInterrupted: return inline_native_isInterrupted(); |
737 |
745 |
738 #ifdef TRACE_HAVE_INTRINSICS |
746 #ifdef TRACE_HAVE_INTRINSICS |
2838 } |
2846 } |
2839 insert_mem_bar(Op_MemBarCPUOrder); |
2847 insert_mem_bar(Op_MemBarCPUOrder); |
2840 return true; |
2848 return true; |
2841 } |
2849 } |
2842 |
2850 |
|
2851 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) { |
|
2852 // Regardless of form, don't allow previous ld/st to move down, |
|
2853 // then issue acquire, release, or volatile mem_bar. |
|
2854 insert_mem_bar(Op_MemBarCPUOrder); |
|
2855 switch(id) { |
|
2856 case vmIntrinsics::_loadFence: |
|
2857 insert_mem_bar(Op_MemBarAcquire); |
|
2858 return true; |
|
2859 case vmIntrinsics::_storeFence: |
|
2860 insert_mem_bar(Op_MemBarRelease); |
|
2861 return true; |
|
2862 case vmIntrinsics::_fullFence: |
|
2863 insert_mem_bar(Op_MemBarVolatile); |
|
2864 return true; |
|
2865 default: |
|
2866 fatal_unexpected_iid(id); |
|
2867 return false; |
|
2868 } |
|
2869 } |
|
2870 |
2843 //----------------------------inline_unsafe_allocate--------------------------- |
2871 //----------------------------inline_unsafe_allocate--------------------------- |
2844 // public native Object sun.mics.Unsafe.allocateInstance(Class<?> cls); |
2872 // public native Object sun.mics.Unsafe.allocateInstance(Class<?> cls); |
2845 bool LibraryCallKit::inline_unsafe_allocate() { |
2873 bool LibraryCallKit::inline_unsafe_allocate() { |
2846 if (callee()->is_static()) return false; // caller must have the capability! |
2874 if (callee()->is_static()) return false; // caller must have the capability! |
2847 |
2875 |
2950 // However, if the receiver is not currentThread, we must call the VM, |
2978 // However, if the receiver is not currentThread, we must call the VM, |
2951 // because there must be some locking done around the operation. |
2979 // because there must be some locking done around the operation. |
2952 |
2980 |
2953 // We only go to the fast case code if we pass two guards. |
2981 // We only go to the fast case code if we pass two guards. |
2954 // Paths which do not pass are accumulated in the slow_region. |
2982 // Paths which do not pass are accumulated in the slow_region. |
|
2983 |
|
2984 enum { |
|
2985 no_int_result_path = 1, // t == Thread.current() && !TLS._osthread._interrupted |
|
2986 no_clear_result_path = 2, // t == Thread.current() && TLS._osthread._interrupted && !clear_int |
|
2987 slow_result_path = 3, // slow path: t.isInterrupted(clear_int) |
|
2988 PATH_LIMIT |
|
2989 }; |
|
2990 |
|
2991 // Ensure that it's not possible to move the load of TLS._osthread._interrupted flag |
|
2992 // out of the function. |
|
2993 insert_mem_bar(Op_MemBarCPUOrder); |
|
2994 |
|
2995 RegionNode* result_rgn = new (C) RegionNode(PATH_LIMIT); |
|
2996 PhiNode* result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL); |
|
2997 |
2955 RegionNode* slow_region = new (C) RegionNode(1); |
2998 RegionNode* slow_region = new (C) RegionNode(1); |
2956 record_for_igvn(slow_region); |
2999 record_for_igvn(slow_region); |
2957 RegionNode* result_rgn = new (C) RegionNode(1+3); // fast1, fast2, slow |
|
2958 PhiNode* result_val = new (C) PhiNode(result_rgn, TypeInt::BOOL); |
|
2959 enum { no_int_result_path = 1, |
|
2960 no_clear_result_path = 2, |
|
2961 slow_result_path = 3 |
|
2962 }; |
|
2963 |
3000 |
2964 // (a) Receiving thread must be the current thread. |
3001 // (a) Receiving thread must be the current thread. |
2965 Node* rec_thr = argument(0); |
3002 Node* rec_thr = argument(0); |
2966 Node* tls_ptr = NULL; |
3003 Node* tls_ptr = NULL; |
2967 Node* cur_thr = generate_current_thread(tls_ptr); |
3004 Node* cur_thr = generate_current_thread(tls_ptr); |
2968 Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) ); |
3005 Node* cmp_thr = _gvn.transform( new (C) CmpPNode(cur_thr, rec_thr) ); |
2969 Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) ); |
3006 Node* bol_thr = _gvn.transform( new (C) BoolNode(cmp_thr, BoolTest::ne) ); |
2970 |
3007 |
2971 bool known_current_thread = (_gvn.type(bol_thr) == TypeInt::ZERO); |
3008 generate_slow_guard(bol_thr, slow_region); |
2972 if (!known_current_thread) |
|
2973 generate_slow_guard(bol_thr, slow_region); |
|
2974 |
3009 |
2975 // (b) Interrupt bit on TLS must be false. |
3010 // (b) Interrupt bit on TLS must be false. |
2976 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset())); |
3011 Node* p = basic_plus_adr(top()/*!oop*/, tls_ptr, in_bytes(JavaThread::osthread_offset())); |
2977 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS); |
3012 Node* osthread = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS); |
2978 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset())); |
3013 p = basic_plus_adr(top()/*!oop*/, osthread, in_bytes(OSThread::interrupted_offset())); |
|
3014 |
2979 // Set the control input on the field _interrupted read to prevent it floating up. |
3015 // Set the control input on the field _interrupted read to prevent it floating up. |
2980 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT); |
3016 Node* int_bit = make_load(control(), p, TypeInt::BOOL, T_INT); |
2981 Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) ); |
3017 Node* cmp_bit = _gvn.transform( new (C) CmpINode(int_bit, intcon(0)) ); |
2982 Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) ); |
3018 Node* bol_bit = _gvn.transform( new (C) BoolNode(cmp_bit, BoolTest::ne) ); |
2983 |
3019 |
3018 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted); |
3054 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_isInterrupted); |
3019 |
3055 |
3020 Node* slow_val = set_results_for_java_call(slow_call); |
3056 Node* slow_val = set_results_for_java_call(slow_call); |
3021 // this->control() comes from set_results_for_java_call |
3057 // this->control() comes from set_results_for_java_call |
3022 |
3058 |
3023 // If we know that the result of the slow call will be true, tell the optimizer! |
|
3024 if (known_current_thread) slow_val = intcon(1); |
|
3025 |
|
3026 Node* fast_io = slow_call->in(TypeFunc::I_O); |
3059 Node* fast_io = slow_call->in(TypeFunc::I_O); |
3027 Node* fast_mem = slow_call->in(TypeFunc::Memory); |
3060 Node* fast_mem = slow_call->in(TypeFunc::Memory); |
|
3061 |
3028 // These two phis are pre-filled with copies of of the fast IO and Memory |
3062 // These two phis are pre-filled with copies of of the fast IO and Memory |
3029 Node* io_phi = PhiNode::make(result_rgn, fast_io, Type::ABIO); |
3063 PhiNode* result_mem = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM); |
3030 Node* mem_phi = PhiNode::make(result_rgn, fast_mem, Type::MEMORY, TypePtr::BOTTOM); |
3064 PhiNode* result_io = PhiNode::make(result_rgn, fast_io, Type::ABIO); |
3031 |
3065 |
3032 result_rgn->init_req(slow_result_path, control()); |
3066 result_rgn->init_req(slow_result_path, control()); |
3033 io_phi ->init_req(slow_result_path, i_o()); |
3067 result_io ->init_req(slow_result_path, i_o()); |
3034 mem_phi ->init_req(slow_result_path, reset_memory()); |
3068 result_mem->init_req(slow_result_path, reset_memory()); |
3035 result_val->init_req(slow_result_path, slow_val); |
3069 result_val->init_req(slow_result_path, slow_val); |
3036 |
3070 |
3037 set_all_memory( _gvn.transform(mem_phi) ); |
3071 set_all_memory(_gvn.transform(result_mem)); |
3038 set_i_o( _gvn.transform(io_phi) ); |
3072 set_i_o( _gvn.transform(result_io)); |
3039 } |
3073 } |
3040 |
3074 |
3041 C->set_has_split_ifs(true); // Has chance for split-if optimization |
3075 C->set_has_split_ifs(true); // Has chance for split-if optimization |
3042 set_result(result_rgn, result_val); |
3076 set_result(result_rgn, result_val); |
3043 return true; |
3077 return true; |
3317 int which_arg; |
3351 int which_arg; |
3318 for (which_arg = 0; which_arg <= 1; which_arg++) { |
3352 for (which_arg = 0; which_arg <= 1; which_arg++) { |
3319 Node* arg = args[which_arg]; |
3353 Node* arg = args[which_arg]; |
3320 arg = null_check(arg); |
3354 arg = null_check(arg); |
3321 if (stopped()) break; |
3355 if (stopped()) break; |
3322 args[which_arg] = _gvn.transform(arg); |
3356 args[which_arg] = arg; |
3323 |
3357 |
3324 Node* p = basic_plus_adr(arg, class_klass_offset); |
3358 Node* p = basic_plus_adr(arg, class_klass_offset); |
3325 Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type); |
3359 Node* kls = LoadKlassNode::make(_gvn, immutable_memory(), p, adr_type, kls_type); |
3326 klasses[which_arg] = _gvn.transform(kls); |
3360 klasses[which_arg] = _gvn.transform(kls); |
3327 } |
3361 } |