Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "precompiled.hpp"

 #include "classfile/systemDictionary.hpp"

 #include "classfile/vmSymbols.hpp"

 #include "compiler/compileBroker.hpp"

 #include "compiler/compileLog.hpp"

 #include "oops/objArrayKlass.hpp"

 #include "opto/addnode.hpp"

 #include "opto/callGenerator.hpp"

 #include "opto/cfgnode.hpp"

 #include "opto/idealKit.hpp"

 #include "opto/mulnode.hpp"

 #include "opto/parse.hpp"

 #include "opto/runtime.hpp"

 #include "opto/subnode.hpp"

 #include "prims/nativeLookup.hpp"

 #include "runtime/sharedRuntime.hpp"

 #include "trace/traceMacros.hpp"

 class LibraryIntrinsic : public InlineCallGenerator {

   // Extend the set of intrinsics known to the runtime:

  public:

  private:

   bool             _is_virtual;

   bool             _is_predicted;

   vmIntrinsics::ID _intrinsic_id;

  public:

   LibraryIntrinsic(ciMethod* m, bool is_virtual, bool is_predicted, vmIntrinsics::ID id)

     : InlineCallGenerator(m),

       _is_virtual(is_virtual),

       _is_predicted(is_predicted),

       _intrinsic_id(id)

   {

   }

   virtual bool is_intrinsic() const { return true; }

   virtual bool is_virtual()   const { return _is_virtual; }

   virtual bool is_predicted()   const { return _is_predicted; }

   virtual JVMState* generate(JVMState* jvms);

   virtual Node* generate_predicate(JVMState* jvms);

   vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }

 };

 // Local helper class for LibraryIntrinsic:

 class LibraryCallKit : public GraphKit {

  private:

   LibraryIntrinsic* _intrinsic;     // the library intrinsic being called

   Node*             _result;        // the result node, if any

   int               _reexecute_sp;  // the stack pointer when bytecode needs to be reexecuted

   const TypeOopPtr* sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr = false);

  public:

   LibraryCallKit(JVMState* jvms, LibraryIntrinsic* intrinsic)

     : GraphKit(jvms),

       _intrinsic(intrinsic),

       _result(NULL)

   {

     // Check if this is a root compile.  In that case we don't have a caller.

     if (!jvms->has_method()) {

       _reexecute_sp = sp();

     } else {

       // Find out how many arguments the interpreter needs when deoptimizing

       // and save the stack pointer value so it can used by uncommon_trap.

       // We find the argument count by looking at the declared signature.

       bool ignored_will_link;

       ciSignature* declared_signature = NULL;

       ciMethod* ignored_callee = caller()->get_method_at_bci(bci(), ignored_will_link, &declared_signature);

       const int nargs = declared_signature->arg_size_for_bc(caller()->java_code_at_bci(bci()));

       _reexecute_sp = sp() + nargs;  // "push" arguments back on stack

     }

   }

   virtual LibraryCallKit* is_LibraryCallKit() const { return (LibraryCallKit*)this; }

   ciMethod*         caller()    const    { return jvms()->method(); }

   int               bci()       const    { return jvms()->bci(); }

   LibraryIntrinsic* intrinsic() const    { return _intrinsic; }

   vmIntrinsics::ID  intrinsic_id() const { return _intrinsic->intrinsic_id(); }

   ciMethod*         callee()    const    { return _intrinsic->method(); }

   bool try_to_inline();

   Node* try_to_predicate();

   void push_result() {

     // Push the result onto the stack.

     if (!stopped() && result() != NULL) {

       BasicType bt = result()->bottom_type()->basic_type();

       push_node(bt, result());

     }

   }

  private:

   void fatal_unexpected_iid(vmIntrinsics::ID iid) {

     fatal(err_msg_res("unexpected intrinsic %d: %s", iid, vmIntrinsics::name_at(iid)));

   }

   void  set_result(Node* n) { assert(_result == NULL, "only set once"); _result = n; }

   void  set_result(RegionNode* region, PhiNode* value);

   Node*     result() { return _result; }

   virtual int reexecute_sp() { return _reexecute_sp; }

   // Helper functions to inline natives

   Node* generate_guard(Node* test, RegionNode* region, float true_prob);

   Node* generate_slow_guard(Node* test, RegionNode* region);

   Node* generate_fair_guard(Node* test, RegionNode* region);

   Node* generate_negative_guard(Node* index, RegionNode* region,

                                 // resulting CastII of index:

                                 Node* *pos_index = NULL);

   Node* generate_nonpositive_guard(Node* index, bool never_negative,

                                    // resulting CastII of index:

                                    Node* *pos_index = NULL);

   Node* generate_limit_guard(Node* offset, Node* subseq_length,

                              Node* array_length,

                              RegionNode* region);

   Node* generate_current_thread(Node* &tls_output);

   address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,

                               bool disjoint_bases, const char* &name, bool dest_uninitialized);

   Node* load_mirror_from_klass(Node* klass);

   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,

                                       RegionNode* region, int null_path,

                                       int offset);

   Node* load_klass_from_mirror(Node* mirror, bool never_see_null,

                                RegionNode* region, int null_path) {

     int offset = java_lang_Class::klass_offset_in_bytes();

     return load_klass_from_mirror_common(mirror, never_see_null,

                                          region, null_path,

                                          offset);

   }

   Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,

                                      RegionNode* region, int null_path) {

     int offset = java_lang_Class::array_klass_offset_in_bytes();

     return load_klass_from_mirror_common(mirror, never_see_null,

                                          region, null_path,

                                          offset);

   }

   Node* generate_access_flags_guard(Node* kls,

                                     int modifier_mask, int modifier_bits,

                                     RegionNode* region);

   Node* generate_interface_guard(Node* kls, RegionNode* region);

   Node* generate_array_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, false, false);

   }

   Node* generate_non_array_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, false, true);

   }

   Node* generate_objArray_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, true, false);

   }

   Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, true, true);

   }

   Node* generate_array_guard_common(Node* kls, RegionNode* region,

                                     bool obj_array, bool not_array);

   Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);

   CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,

                                      bool is_virtual = false, bool is_static = false);

   CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {

     return generate_method_call(method_id, false, true);

   }

   CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {

     return generate_method_call(method_id, true, false);

   }

   Node * load_field_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, bool is_exact, bool is_static);

   Node* make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2);

   Node* make_string_method_node(int opcode, Node* str1, Node* str2);

   bool inline_string_compareTo();

   bool inline_string_indexOf();

   Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);

   bool inline_string_equals();

   Node* round_double_node(Node* n);

   bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);

   bool inline_math_native(vmIntrinsics::ID id);

   bool inline_trig(vmIntrinsics::ID id);

   bool inline_math(vmIntrinsics::ID id);

   bool inline_exp();

   bool inline_pow();

   void finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName);

   bool inline_min_max(vmIntrinsics::ID id);

   Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);

   // This returns Type::AnyPtr, RawPtr, or OopPtr.

   int classify_unsafe_addr(Node* &base, Node* &offset);

   Node* make_unsafe_address(Node* base, Node* offset);

   // Helper for inline_unsafe_access.

   // Generates the guards that check whether the result of

   // Unsafe.getObject should be recorded in an SATB log buffer.

   void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar);

   bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile);

   bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);

   static bool klass_needs_init_guard(Node* kls);

   bool inline_unsafe_allocate();

   bool inline_unsafe_copyMemory();

   bool inline_native_currentThread();

 #ifdef TRACE_HAVE_INTRINSICS

   bool inline_native_classID();

   bool inline_native_threadID();

 #endif

   bool inline_native_time_funcs(address method, const char* funcName);

   bool inline_native_isInterrupted();

   bool inline_native_Class_query(vmIntrinsics::ID id);

   bool inline_native_subtype_check();

   bool inline_native_newArray();

   bool inline_native_getLength();

   bool inline_array_copyOf(bool is_copyOfRange);

   bool inline_array_equals();

   void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);

   bool inline_native_clone(bool is_virtual);

   bool inline_native_Reflection_getCallerClass();

   // Helper function for inlining native object hash method

   bool inline_native_hashcode(bool is_virtual, bool is_static);

   bool inline_native_getClass();

   // Helper functions for inlining arraycopy

   bool inline_arraycopy();

   void generate_arraycopy(const TypePtr* adr_type,

                           BasicType basic_elem_type,

                           Node* src,  Node* src_offset,

                           Node* dest, Node* dest_offset,

                           Node* copy_length,

                           bool disjoint_bases = false,

                           bool length_never_negative = false,

                           RegionNode* slow_region = NULL);

   AllocateArrayNode* tightly_coupled_allocation(Node* ptr,

                                                 RegionNode* slow_region);

   void generate_clear_array(const TypePtr* adr_type,

                             Node* dest,

                             BasicType basic_elem_type,

                             Node* slice_off,

                             Node* slice_len,

                             Node* slice_end);

   bool generate_block_arraycopy(const TypePtr* adr_type,

                                 BasicType basic_elem_type,

                                 AllocateNode* alloc,

                                 Node* src,  Node* src_offset,

                                 Node* dest, Node* dest_offset,

                                 Node* dest_size, bool dest_uninitialized);

   void generate_slow_arraycopy(const TypePtr* adr_type,

                                Node* src,  Node* src_offset,

                                Node* dest, Node* dest_offset,

                                Node* copy_length, bool dest_uninitialized);

   Node* generate_checkcast_arraycopy(const TypePtr* adr_type,

                                      Node* dest_elem_klass,

                                      Node* src,  Node* src_offset,

                                      Node* dest, Node* dest_offset,

                                      Node* copy_length, bool dest_uninitialized);

   Node* generate_generic_arraycopy(const TypePtr* adr_type,

                                    Node* src,  Node* src_offset,

                                    Node* dest, Node* dest_offset,

                                    Node* copy_length, bool dest_uninitialized);

   void generate_unchecked_arraycopy(const TypePtr* adr_type,

                                     BasicType basic_elem_type,

                                     bool disjoint_bases,

                                     Node* src,  Node* src_offset,

                                     Node* dest, Node* dest_offset,

                                     Node* copy_length, bool dest_uninitialized);

   typedef enum { LS_xadd, LS_xchg, LS_cmpxchg } LoadStoreKind;

   bool inline_unsafe_load_store(BasicType type,  LoadStoreKind kind);

   bool inline_unsafe_ordered_store(BasicType type);

   bool inline_unsafe_fence(vmIntrinsics::ID id);

   bool inline_fp_conversions(vmIntrinsics::ID id);

   bool inline_number_methods(vmIntrinsics::ID id);

   bool inline_reference_get();

   bool inline_aescrypt_Block(vmIntrinsics::ID id);

   bool inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id);

   Node* inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting);

   Node* get_key_start_from_aescrypt_object(Node* aescrypt_object);

   bool inline_encodeISOArray();

   bool inline_updateCRC32();

   bool inline_updateBytesCRC32();

   bool inline_updateByteBufferCRC32();

 };

 //---------------------------make_vm_intrinsic----------------------------

 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {

   vmIntrinsics::ID id = m->intrinsic_id();

   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");

   if (DisableIntrinsic[0] != '\0'

       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {

     // disabled by a user request on the command line:

     // example: -XX:DisableIntrinsic=_hashCode,_getClass

     return NULL;

   }

   if (!m->is_loaded()) {

     // do not attempt to inline unloaded methods

     return NULL;

   }

   // Only a few intrinsics implement a virtual dispatch.

   // They are expensive calls which are also frequently overridden.

   if (is_virtual) {

     switch (id) {

     case vmIntrinsics::_hashCode:

     case vmIntrinsics::_clone:

       // OK, Object.hashCode and Object.clone intrinsics come in both flavors

       break;

     default:

       return NULL;

     }

   }

   // -XX:-InlineNatives disables nearly all intrinsics:

   if (!InlineNatives) {

     switch (id) {

     case vmIntrinsics::_indexOf:

     case vmIntrinsics::_compareTo:

     case vmIntrinsics::_equals:

     case vmIntrinsics::_equalsC:

     case vmIntrinsics::_getAndAddInt:

     case vmIntrinsics::_getAndAddLong:

     case vmIntrinsics::_getAndSetInt:

     case vmIntrinsics::_getAndSetLong:

     case vmIntrinsics::_getAndSetObject:

     case vmIntrinsics::_loadFence:

     case vmIntrinsics::_storeFence:

     case vmIntrinsics::_fullFence:

       break;  // InlineNatives does not control String.compareTo

     case vmIntrinsics::_Reference_get:

       break;  // InlineNatives does not control Reference.get

     default:

       return NULL;

     }

   }

   bool is_predicted = false;

   switch (id) {

   case vmIntrinsics::_compareTo:

     if (!SpecialStringCompareTo)  return NULL;

     if (!Matcher::match_rule_supported(Op_StrComp))  return NULL;

     break;

   case vmIntrinsics::_indexOf:

     if (!SpecialStringIndexOf)  return NULL;

     break;

   case vmIntrinsics::_equals:

     if (!SpecialStringEquals)  return NULL;

     if (!Matcher::match_rule_supported(Op_StrEquals))  return NULL;

     break;

   case vmIntrinsics::_equalsC:

     if (!SpecialArraysEquals)  return NULL;

     if (!Matcher::match_rule_supported(Op_AryEq))  return NULL;

     break;

   case vmIntrinsics::_arraycopy:

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_copyMemory:

     if (StubRoutines::unsafe_arraycopy() == NULL)  return NULL;

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_hashCode:

     if (!InlineObjectHash)  return NULL;

     break;

   case vmIntrinsics::_clone:

   case vmIntrinsics::_copyOf:

   case vmIntrinsics::_copyOfRange:

     if (!InlineObjectCopy)  return NULL;

     // These also use the arraycopy intrinsic mechanism:

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_encodeISOArray:

     if (!SpecialEncodeISOArray)  return NULL;

     if (!Matcher::match_rule_supported(Op_EncodeISOArray))  return NULL;

     break;

   case vmIntrinsics::_checkIndex:

     // We do not intrinsify this.  The optimizer does fine with it.

     return NULL;

   case vmIntrinsics::_getCallerClass:

     if (!UseNewReflection)  return NULL;

     if (!InlineReflectionGetCallerClass)  return NULL;

     if (SystemDictionary::reflect_CallerSensitive_klass() == NULL)  return NULL;

     break;

   case vmIntrinsics::_bitCount_i:

     if (!Matcher::match_rule_supported(Op_PopCountI)) return NULL;

     break;

   case vmIntrinsics::_bitCount_l:

     if (!Matcher::match_rule_supported(Op_PopCountL)) return NULL;

     break;

   case vmIntrinsics::_numberOfLeadingZeros_i:

     if (!Matcher::match_rule_supported(Op_CountLeadingZerosI)) return NULL;

     break;

   case vmIntrinsics::_numberOfLeadingZeros_l:

     if (!Matcher::match_rule_supported(Op_CountLeadingZerosL)) return NULL;

     break;

   case vmIntrinsics::_numberOfTrailingZeros_i:

     if (!Matcher::match_rule_supported(Op_CountTrailingZerosI)) return NULL;

     break;

   case vmIntrinsics::_numberOfTrailingZeros_l:

     if (!Matcher::match_rule_supported(Op_CountTrailingZerosL)) return NULL;

     break;

   case vmIntrinsics::_reverseBytes_c:

     if (!Matcher::match_rule_supported(Op_ReverseBytesUS)) return NULL;

     break;

   case vmIntrinsics::_reverseBytes_s:

     if (!Matcher::match_rule_supported(Op_ReverseBytesS))  return NULL;

     break;

   case vmIntrinsics::_reverseBytes_i:

     if (!Matcher::match_rule_supported(Op_ReverseBytesI))  return NULL;

     break;

   case vmIntrinsics::_reverseBytes_l:

     if (!Matcher::match_rule_supported(Op_ReverseBytesL))  return NULL;

     break;

   case vmIntrinsics::_Reference_get:

     // Use the intrinsic version of Reference.get() so that the value in

     // the referent field can be registered by the G1 pre-barrier code.

     // Also add memory barrier to prevent commoning reads from this field

     // across safepoint since GC can change it value.

     break;

   case vmIntrinsics::_compareAndSwapObject:

 #ifdef _LP64

     if (!UseCompressedOops && !Matcher::match_rule_supported(Op_CompareAndSwapP)) return NULL;

 #endif

     break;

   case vmIntrinsics::_compareAndSwapLong:

     if (!Matcher::match_rule_supported(Op_CompareAndSwapL)) return NULL;

     break;

   case vmIntrinsics::_getAndAddInt:

     if (!Matcher::match_rule_supported(Op_GetAndAddI)) return NULL;

     break;

   case vmIntrinsics::_getAndAddLong:

     if (!Matcher::match_rule_supported(Op_GetAndAddL)) return NULL;

     break;

   case vmIntrinsics::_getAndSetInt:

     if (!Matcher::match_rule_supported(Op_GetAndSetI)) return NULL;

     break;

   case vmIntrinsics::_getAndSetLong:

     if (!Matcher::match_rule_supported(Op_GetAndSetL)) return NULL;

     break;

   case vmIntrinsics::_getAndSetObject:

 #ifdef _LP64

     if (!UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;

     if (UseCompressedOops && !Matcher::match_rule_supported(Op_GetAndSetN)) return NULL;

     break;

 #else

     if (!Matcher::match_rule_supported(Op_GetAndSetP)) return NULL;

     break;

 #endif

   case vmIntrinsics::_aescrypt_encryptBlock:

   case vmIntrinsics::_aescrypt_decryptBlock:

     if (!UseAESIntrinsics) return NULL;

     break;

   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:

   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:

     if (!UseAESIntrinsics) return NULL;

     // these two require the predicated logic

     is_predicted = true;

     break;

   case vmIntrinsics::_updateCRC32:

   case vmIntrinsics::_updateBytesCRC32:

   case vmIntrinsics::_updateByteBufferCRC32:

     if (!UseCRC32Intrinsics) return NULL;

     break;

  default:

     assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");

     assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");

     break;

   }

   // -XX:-InlineClassNatives disables natives from the Class class.

   // The flag applies to all reflective calls, notably Array.newArray

   // (visible to Java programmers as Array.newInstance).

   if (m->holder()->name() == ciSymbol::java_lang_Class() ||

       m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {

     if (!InlineClassNatives)  return NULL;

   }

   // -XX:-InlineThreadNatives disables natives from the Thread class.

   if (m->holder()->name() == ciSymbol::java_lang_Thread()) {

     if (!InlineThreadNatives)  return NULL;

   }

   // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.

   if (m->holder()->name() == ciSymbol::java_lang_Math() ||

       m->holder()->name() == ciSymbol::java_lang_Float() ||

       m->holder()->name() == ciSymbol::java_lang_Double()) {

     if (!InlineMathNatives)  return NULL;

   }

   // -XX:-InlineUnsafeOps disables natives from the Unsafe class.

   if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {

     if (!InlineUnsafeOps)  return NULL;

   }

   return new LibraryIntrinsic(m, is_virtual, is_predicted, (vmIntrinsics::ID) id);

 }

 //----------------------register_library_intrinsics-----------------------

 // Initialize this file's data structures, for each Compile instance.

 void Compile::register_library_intrinsics() {

   // Nothing to do here.

 }

 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {

   LibraryCallKit kit(jvms, this);

   Compile* C = kit.C;

   int nodes = C->unique();

 #ifndef PRODUCT

   if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {

     char buf[1000];

     const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));

     tty->print_cr("Intrinsic %s", str);

   }

 #endif

   ciMethod* callee = kit.callee();

   const int bci    = kit.bci();

   // Try to inline the intrinsic.

   if (kit.try_to_inline()) {

     if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {

       C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");

     }

     C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);

     if (C->log()) {

       C->log()->elem("intrinsic id='%s'%s nodes='%d'",

                      vmIntrinsics::name_at(intrinsic_id()),

                      (is_virtual() ? " virtual='1'" : ""),

                      C->unique() - nodes);

     }

     // Push the result from the inlined method onto the stack.

     kit.push_result();

     return kit.transfer_exceptions_into_jvms();

   }

   // The intrinsic bailed out

   if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {

     if (jvms->has_method()) {

       // Not a root compile.

       const char* msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)";

       C->print_inlining(callee, jvms->depth() - 1, bci, msg);

     } else {

       // Root compile

       tty->print("Did not generate intrinsic %s%s at bci:%d in",

                vmIntrinsics::name_at(intrinsic_id()),

                (is_virtual() ? " (virtual)" : ""), bci);

     }

   }

   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);

   return NULL;

 }

 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms) {

   LibraryCallKit kit(jvms, this);

   Compile* C = kit.C;

   int nodes = C->unique();

 #ifndef PRODUCT

   assert(is_predicted(), "sanity");

   if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {

     char buf[1000];

     const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));

     tty->print_cr("Predicate for intrinsic %s", str);

   }

 #endif

   ciMethod* callee = kit.callee();

   const int bci    = kit.bci();

   Node* slow_ctl = kit.try_to_predicate();

   if (!kit.failing()) {

     if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {

       C->print_inlining(callee, jvms->depth() - 1, bci, is_virtual() ? "(intrinsic, virtual)" : "(intrinsic)");

     }

     C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);

     if (C->log()) {

       C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'",

                      vmIntrinsics::name_at(intrinsic_id()),

                      (is_virtual() ? " virtual='1'" : ""),

                      C->unique() - nodes);

     }

     return slow_ctl; // Could be NULL if the check folds.

   }

   // The intrinsic bailed out

   if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {

     if (jvms->has_method()) {

       // Not a root compile.

       const char* msg = "failed to generate predicate for intrinsic";

       C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg);

     } else {

       // Root compile

       C->print_inlining_stream()->print("Did not generate predicate for intrinsic %s%s at bci:%d in",

                                         vmIntrinsics::name_at(intrinsic_id()),

                                         (is_virtual() ? " (virtual)" : ""), bci);

     }

   }

   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);

   return NULL;

 }

 bool LibraryCallKit::try_to_inline() {

   // Handle symbolic names for otherwise undistinguished boolean switches:

   const bool is_store       = true;

   const bool is_native_ptr  = true;

   const bool is_static      = true;

   const bool is_volatile    = true;

   if (!jvms()->has_method()) {

     // Root JVMState has a null method.

     assert(map()->memory()->Opcode() == Op_Parm, "");

     // Insert the memory aliasing node

     set_all_memory(reset_memory());

   }

   assert(merged_memory(), "");

   switch (intrinsic_id()) {

   case vmIntrinsics::_hashCode:                 return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);

   case vmIntrinsics::_identityHashCode:         return inline_native_hashcode(/*!virtual*/ false,         is_static);

   case vmIntrinsics::_getClass:                 return inline_native_getClass();

   case vmIntrinsics::_dsin:

   case vmIntrinsics::_dcos:

   case vmIntrinsics::_dtan:

   case vmIntrinsics::_dabs:

   case vmIntrinsics::_datan2:

   case vmIntrinsics::_dsqrt:

   case vmIntrinsics::_dexp:

   case vmIntrinsics::_dlog:

   case vmIntrinsics::_dlog10:

   case vmIntrinsics::_dpow:                     return inline_math_native(intrinsic_id());

   case vmIntrinsics::_min:

   case vmIntrinsics::_max:                      return inline_min_max(intrinsic_id());

   case vmIntrinsics::_arraycopy:                return inline_arraycopy();

   case vmIntrinsics::_compareTo:                return inline_string_compareTo();

   case vmIntrinsics::_indexOf:                  return inline_string_indexOf();

   case vmIntrinsics::_equals:                   return inline_string_equals();

   case vmIntrinsics::_getObject:                return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,  !is_volatile);

   case vmIntrinsics::_getBoolean:               return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, !is_volatile);

   case vmIntrinsics::_getByte:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,    !is_volatile);

   case vmIntrinsics::_getShort:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,   !is_volatile);

   case vmIntrinsics::_getChar:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,    !is_volatile);

   case vmIntrinsics::_getInt:                   return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,     !is_volatile);

   case vmIntrinsics::_getLong:                  return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,    !is_volatile);

   case vmIntrinsics::_getFloat:                 return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,   !is_volatile);

   case vmIntrinsics::_getDouble:                return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);

   case vmIntrinsics::_putObject:                return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,  !is_volatile);

   case vmIntrinsics::_putBoolean:               return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN, !is_volatile);

   case vmIntrinsics::_putByte:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,    !is_volatile);

   case vmIntrinsics::_putShort:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,   !is_volatile);

   case vmIntrinsics::_putChar:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,    !is_volatile);

   case vmIntrinsics::_putInt:                   return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,     !is_volatile);

   case vmIntrinsics::_putLong:                  return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,    !is_volatile);

   case vmIntrinsics::_putFloat:                 return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,   !is_volatile);

   case vmIntrinsics::_putDouble:                return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);

   case vmIntrinsics::_getByte_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_BYTE,    !is_volatile);

   case vmIntrinsics::_getShort_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_SHORT,   !is_volatile);

   case vmIntrinsics::_getChar_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_CHAR,    !is_volatile);

   case vmIntrinsics::_getInt_raw:               return inline_unsafe_access( is_native_ptr, !is_store, T_INT,     !is_volatile);

   case vmIntrinsics::_getLong_raw:              return inline_unsafe_access( is_native_ptr, !is_store, T_LONG,    !is_volatile);

   case vmIntrinsics::_getFloat_raw:             return inline_unsafe_access( is_native_ptr, !is_store, T_FLOAT,   !is_volatile);

   case vmIntrinsics::_getDouble_raw:            return inline_unsafe_access( is_native_ptr, !is_store, T_DOUBLE,  !is_volatile);

   case vmIntrinsics::_getAddress_raw:           return inline_unsafe_access( is_native_ptr, !is_store, T_ADDRESS, !is_volatile);

   case vmIntrinsics::_putByte_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_BYTE,    !is_volatile);

   case vmIntrinsics::_putShort_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_SHORT,   !is_volatile);

   case vmIntrinsics::_putChar_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_CHAR,    !is_volatile);

   case vmIntrinsics::_putInt_raw:               return inline_unsafe_access( is_native_ptr,  is_store, T_INT,     !is_volatile);

   case vmIntrinsics::_putLong_raw:              return inline_unsafe_access( is_native_ptr,  is_store, T_LONG,    !is_volatile);

   case vmIntrinsics::_putFloat_raw:             return inline_unsafe_access( is_native_ptr,  is_store, T_FLOAT,   !is_volatile);

   case vmIntrinsics::_putDouble_raw:            return inline_unsafe_access( is_native_ptr,  is_store, T_DOUBLE,  !is_volatile);

   case vmIntrinsics::_putAddress_raw:           return inline_unsafe_access( is_native_ptr,  is_store, T_ADDRESS, !is_volatile);

   case vmIntrinsics::_getObjectVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT,   is_volatile);

   case vmIntrinsics::_getBooleanVolatile:       return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN,  is_volatile);

   case vmIntrinsics::_getByteVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE,     is_volatile);

   case vmIntrinsics::_getShortVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT,    is_volatile);

   case vmIntrinsics::_getCharVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR,     is_volatile);

   case vmIntrinsics::_getIntVolatile:           return inline_unsafe_access(!is_native_ptr, !is_store, T_INT,      is_volatile);

   case vmIntrinsics::_getLongVolatile:          return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG,     is_volatile);

   case vmIntrinsics::_getFloatVolatile:         return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT,    is_volatile);

   case vmIntrinsics::_getDoubleVolatile:        return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE,   is_volatile);

   case vmIntrinsics::_putObjectVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_OBJECT,   is_volatile);

   case vmIntrinsics::_putBooleanVolatile:       return inline_unsafe_access(!is_native_ptr,  is_store, T_BOOLEAN,  is_volatile);

   case vmIntrinsics::_putByteVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_BYTE,     is_volatile);

   case vmIntrinsics::_putShortVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_SHORT,    is_volatile);

   case vmIntrinsics::_putCharVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_CHAR,     is_volatile);

   case vmIntrinsics::_putIntVolatile:           return inline_unsafe_access(!is_native_ptr,  is_store, T_INT,      is_volatile);

   case vmIntrinsics::_putLongVolatile:          return inline_unsafe_access(!is_native_ptr,  is_store, T_LONG,     is_volatile);

   case vmIntrinsics::_putFloatVolatile:         return inline_unsafe_access(!is_native_ptr,  is_store, T_FLOAT,    is_volatile);

   case vmIntrinsics::_putDoubleVolatile:        return inline_unsafe_access(!is_native_ptr,  is_store, T_DOUBLE,   is_volatile);

   case vmIntrinsics::_prefetchRead:             return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);

   case vmIntrinsics::_prefetchWrite:            return inline_unsafe_prefetch(!is_native_ptr,  is_store, !is_static);

   case vmIntrinsics::_prefetchReadStatic:       return inline_unsafe_prefetch(!is_native_ptr, !is_store,  is_static);

   case vmIntrinsics::_prefetchWriteStatic:      return inline_unsafe_prefetch(!is_native_ptr,  is_store,  is_static);

   case vmIntrinsics::_compareAndSwapObject:     return inline_unsafe_load_store(T_OBJECT, LS_cmpxchg);

   case vmIntrinsics::_compareAndSwapInt:        return inline_unsafe_load_store(T_INT,    LS_cmpxchg);

   case vmIntrinsics::_compareAndSwapLong:       return inline_unsafe_load_store(T_LONG,   LS_cmpxchg);

   case vmIntrinsics::_putOrderedObject:         return inline_unsafe_ordered_store(T_OBJECT);

   case vmIntrinsics::_putOrderedInt:            return inline_unsafe_ordered_store(T_INT);

   case vmIntrinsics::_putOrderedLong:           return inline_unsafe_ordered_store(T_LONG);

   case vmIntrinsics::_getAndAddInt:             return inline_unsafe_load_store(T_INT,    LS_xadd);

   case vmIntrinsics::_getAndAddLong:            return inline_unsafe_load_store(T_LONG,   LS_xadd);

   case vmIntrinsics::_getAndSetInt:             return inline_unsafe_load_store(T_INT,    LS_xchg);

   case vmIntrinsics::_getAndSetLong:            return inline_unsafe_load_store(T_LONG,   LS_xchg);

   case vmIntrinsics::_getAndSetObject:          return inline_unsafe_load_store(T_OBJECT, LS_xchg);

   case vmIntrinsics::_loadFence:

   case vmIntrinsics::_storeFence:

   case vmIntrinsics::_fullFence:                return inline_unsafe_fence(intrinsic_id());

   case vmIntrinsics::_currentThread:            return inline_native_currentThread();

   case vmIntrinsics::_isInterrupted:            return inline_native_isInterrupted();

 #ifdef TRACE_HAVE_INTRINSICS

   case vmIntrinsics::_classID:                  return inline_native_classID();

   case vmIntrinsics::_threadID:                 return inline_native_threadID();

   case vmIntrinsics::_counterTime:              return inline_native_time_funcs(CAST_FROM_FN_PTR(address, TRACE_TIME_METHOD), "counterTime");

 #endif

   case vmIntrinsics::_currentTimeMillis:        return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");

   case vmIntrinsics::_nanoTime:                 return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");

   case vmIntrinsics::_allocateInstance:         return inline_unsafe_allocate();

   case vmIntrinsics::_copyMemory:               return inline_unsafe_copyMemory();

   case vmIntrinsics::_newArray:                 return inline_native_newArray();

   case vmIntrinsics::_getLength:                return inline_native_getLength();

   case vmIntrinsics::_copyOf:                   return inline_array_copyOf(false);

   case vmIntrinsics::_copyOfRange:              return inline_array_copyOf(true);

   case vmIntrinsics::_equalsC:                  return inline_array_equals();

   case vmIntrinsics::_clone:                    return inline_native_clone(intrinsic()->is_virtual());

   case vmIntrinsics::_isAssignableFrom:         return inline_native_subtype_check();

   case vmIntrinsics::_isInstance:

   case vmIntrinsics::_getModifiers:

   case vmIntrinsics::_isInterface:

   case vmIntrinsics::_isArray:

   case vmIntrinsics::_isPrimitive:

   case vmIntrinsics::_getSuperclass:

   case vmIntrinsics::_getComponentType:

   case vmIntrinsics::_getClassAccessFlags:      return inline_native_Class_query(intrinsic_id());

   case vmIntrinsics::_floatToRawIntBits:

   case vmIntrinsics::_floatToIntBits:

   case vmIntrinsics::_intBitsToFloat:

   case vmIntrinsics::_doubleToRawLongBits:

   case vmIntrinsics::_doubleToLongBits:

   case vmIntrinsics::_longBitsToDouble:         return inline_fp_conversions(intrinsic_id());

   case vmIntrinsics::_numberOfLeadingZeros_i:

   case vmIntrinsics::_numberOfLeadingZeros_l:

   case vmIntrinsics::_numberOfTrailingZeros_i:

   case vmIntrinsics::_numberOfTrailingZeros_l:

   case vmIntrinsics::_bitCount_i:

   case vmIntrinsics::_bitCount_l:

   case vmIntrinsics::_reverseBytes_i:

   case vmIntrinsics::_reverseBytes_l:

   case vmIntrinsics::_reverseBytes_s:

   case vmIntrinsics::_reverseBytes_c:           return inline_number_methods(intrinsic_id());

   case vmIntrinsics::_getCallerClass:           return inline_native_Reflection_getCallerClass();

   case vmIntrinsics::_Reference_get:            return inline_reference_get();

   case vmIntrinsics::_aescrypt_encryptBlock:

   case vmIntrinsics::_aescrypt_decryptBlock:    return inline_aescrypt_Block(intrinsic_id());

   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:

   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:

     return inline_cipherBlockChaining_AESCrypt(intrinsic_id());

   case vmIntrinsics::_encodeISOArray:

     return inline_encodeISOArray();

   case vmIntrinsics::_updateCRC32:

     return inline_updateCRC32();

   case vmIntrinsics::_updateBytesCRC32:

     return inline_updateBytesCRC32();

   case vmIntrinsics::_updateByteBufferCRC32:

     return inline_updateByteBufferCRC32();

   default:

     // If you get here, it may be that someone has added a new intrinsic

     // to the list in vmSymbols.hpp without implementing it here.

 #ifndef PRODUCT

     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {

       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",

                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());

     }

 #endif

     return false;

   }

 }

 Node* LibraryCallKit::try_to_predicate() {

   if (!jvms()->has_method()) {

     // Root JVMState has a null method.

     assert(map()->memory()->Opcode() == Op_Parm, "");

     // Insert the memory aliasing node

     set_all_memory(reset_memory());

   }

   assert(merged_memory(), "");

   switch (intrinsic_id()) {

   case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt:

     return inline_cipherBlockChaining_AESCrypt_predicate(false);

   case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt:

     return inline_cipherBlockChaining_AESCrypt_predicate(true);

   default:

     // If you get here, it may be that someone has added a new intrinsic

     // to the list in vmSymbols.hpp without implementing it here.

 #ifndef PRODUCT

     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {

       tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)",

                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());

     }

 #endif

     Node* slow_ctl = control();

     set_control(top()); // No fast path instrinsic

     return slow_ctl;

   }

 }

 //------------------------------set_result-------------------------------

 // Helper function for finishing intrinsics.

 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) {

   record_for_igvn(region);

   set_control(_gvn.transform(region));

   set_result( _gvn.transform(value));

   assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity");

 }

 //------------------------------generate_guard---------------------------

 // Helper function for generating guarded fast-slow graph structures.

 // The given 'test', if true, guards a slow path.  If the test fails

 // then a fast path can be taken.  (We generally hope it fails.)

 // In all cases, GraphKit::control() is updated to the fast path.

 // The returned value represents the control for the slow path.

 // The return value is never 'top'; it is either a valid control

 // or NULL if it is obvious that the slow path can never be taken.

 // Also, if region and the slow control are not NULL, the slow edge

 // is appended to the region.

 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {

   if (stopped()) {

     // Already short circuited.

     return NULL;

   }

   // Build an if node and its projections.

   // If test is true we take the slow path, which we assume is uncommon.

   if (_gvn.type(test) == TypeInt::ZERO) {

     // The slow branch is never taken.  No need to build this guard.

     return NULL;

   }

   IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);

   Node* if_slow = _gvn.transform(new (C) IfTrueNode(iff));

   if (if_slow == top()) {

     // The slow branch is never taken.  No need to build this guard.

     return NULL;

   }

   if (region != NULL)

     region->add_req(if_slow);

   Node* if_fast = _gvn.transform(new (C) IfFalseNode(iff));

   set_control(if_fast);

   return if_slow;

 }

 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {

   return generate_guard(test, region, PROB_UNLIKELY_MAG(3));

 }

 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {

   return generate_guard(test, region, PROB_FAIR);

 }

 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,

                                                      Node* *pos_index) {

   if (stopped())

     return NULL;                // already stopped

   if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]

     return NULL;                // index is already adequately typed

   Node* cmp_lt = _gvn.transform(new (C) CmpINode(index, intcon(0)));

   Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));

   Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);

   if (is_neg != NULL && pos_index != NULL) {

     // Emulate effect of Parse::adjust_map_after_if.

     Node* ccast = new (C) CastIINode(index, TypeInt::POS);

     ccast->set_req(0, control());

     (*pos_index) = _gvn.transform(ccast);

   }

   return is_neg;

 }

 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,

                                                         Node* *pos_index) {

   if (stopped())

     return NULL;                // already stopped

   if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]

     return NULL;                // index is already adequately typed

   Node* cmp_le = _gvn.transform(new (C) CmpINode(index, intcon(0)));

   BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);

   Node* bol_le = _gvn.transform(new (C) BoolNode(cmp_le, le_or_eq));

   Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);

   if (is_notp != NULL && pos_index != NULL) {

     // Emulate effect of Parse::adjust_map_after_if.

     Node* ccast = new (C) CastIINode(index, TypeInt::POS1);

     ccast->set_req(0, control());

     (*pos_index) = _gvn.transform(ccast);

   }

   return is_notp;

 }

 // Make sure that 'position' is a valid limit index, in [0..length].

 // There are two equivalent plans for checking this:

 //   A. (offset + copyLength)  unsigned<=  arrayLength

 //   B. offset  <=  (arrayLength - copyLength)

 // We require that all of the values above, except for the sum and

 // difference, are already known to be non-negative.

 // Plan A is robust in the face of overflow, if offset and copyLength

 // are both hugely positive.

 //

 // Plan B is less direct and intuitive, but it does not overflow at

 // all, since the difference of two non-negatives is always

 // representable.  Whenever Java methods must perform the equivalent

 // check they generally use Plan B instead of Plan A.

 // For the moment we use Plan A.

 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,

                                                   Node* subseq_length,

                                                   Node* array_length,

                                                   RegionNode* region) {

   if (stopped())

     return NULL;                // already stopped

   bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;

   if (zero_offset && subseq_length->eqv_uncast(array_length))

     return NULL;                // common case of whole-array copy

   Node* last = subseq_length;

   if (!zero_offset)             // last += offset

     last = _gvn.transform(new (C) AddINode(last, offset));

   Node* cmp_lt = _gvn.transform(new (C) CmpUNode(array_length, last));

   Node* bol_lt = _gvn.transform(new (C) BoolNode(cmp_lt, BoolTest::lt));

   Node* is_over = generate_guard(bol_lt, region, PROB_MIN);

   return is_over;

 }

 //--------------------------generate_current_thread--------------------

 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {

   ciKlass*    thread_klass = env()->Thread_klass();

   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);

   Node* thread = _gvn.transform(new (C) ThreadLocalNode());

   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));

   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);

   tls_output = thread;

   return threadObj;

 }

 //------------------------------make_string_method_node------------------------

 // Helper method for String intrinsic functions. This version is called

 // with str1 and str2 pointing to String object nodes.

 //

 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* str2) {

   Node* no_ctrl = NULL;

   // Get start addr of string

   Node* str1_value   = load_String_value(no_ctrl, str1);

   Node* str1_offset  = load_String_offset(no_ctrl, str1);

   Node* str1_start   = array_element_address(str1_value, str1_offset, T_CHAR);

   // Get length of string 1

   Node* str1_len  = load_String_length(no_ctrl, str1);

   Node* str2_value   = load_String_value(no_ctrl, str2);

   Node* str2_offset  = load_String_offset(no_ctrl, str2);

   Node* str2_start   = array_element_address(str2_value, str2_offset, T_CHAR);

   Node* str2_len = NULL;

   Node* result = NULL;

   switch (opcode) {

   case Op_StrIndexOf:

     // Get length of string 2

     str2_len = load_String_length(no_ctrl, str2);

     result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),

                                  str1_start, str1_len, str2_start, str2_len);

     break;

   case Op_StrComp:

     // Get length of string 2

     str2_len = load_String_length(no_ctrl, str2);

     result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),

                                  str1_start, str1_len, str2_start, str2_len);

     break;

   case Op_StrEquals:

     result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),

                                str1_start, str2_start, str1_len);

     break;

   default:

     ShouldNotReachHere();

     return NULL;

   }

   // All these intrinsics have checks.

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   return _gvn.transform(result);

 }

 // Helper method for String intrinsic functions. This version is called

 // with str1 and str2 pointing to char[] nodes, with cnt1 and cnt2 pointing

 // to Int nodes containing the lenghts of str1 and str2.

 //

 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2) {

   Node* result = NULL;

   switch (opcode) {

   case Op_StrIndexOf:

     result = new (C) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),

                                  str1_start, cnt1, str2_start, cnt2);

     break;

   case Op_StrComp:

     result = new (C) StrCompNode(control(), memory(TypeAryPtr::CHARS),

                                  str1_start, cnt1, str2_start, cnt2);

     break;

   case Op_StrEquals:

     result = new (C) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),

                                  str1_start, str2_start, cnt1);

     break;

   default:

     ShouldNotReachHere();

     return NULL;

   }

   // All these intrinsics have checks.

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   return _gvn.transform(result);

 }

 //------------------------------inline_string_compareTo------------------------

 // public int java.lang.String.compareTo(String anotherString);

 bool LibraryCallKit::inline_string_compareTo() {

   Node* receiver = null_check(argument(0));

   Node* arg      = null_check(argument(1));

   if (stopped()) {

     return true;

   }

   set_result(make_string_method_node(Op_StrComp, receiver, arg));

   return true;

 }

 //------------------------------inline_string_equals------------------------

 bool LibraryCallKit::inline_string_equals() {

   Node* receiver = null_check_receiver();

   // NOTE: Do not null check argument for String.equals() because spec

   // allows to specify NULL as argument.

   Node* argument = this->argument(1);

   if (stopped()) {

     return true;

   }

   // paths (plus control) merge

   RegionNode* region = new (C) RegionNode(5);

   Node* phi = new (C) PhiNode(region, TypeInt::BOOL);

   // does source == target string?

   Node* cmp = _gvn.transform(new (C) CmpPNode(receiver, argument));

   Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq));

   Node* if_eq = generate_slow_guard(bol, NULL);

   if (if_eq != NULL) {

     // receiver == argument

     phi->init_req(2, intcon(1));

     region->init_req(2, if_eq);

   }

   // get String klass for instanceOf

   ciInstanceKlass* klass = env()->String_klass();

   if (!stopped()) {

     Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));

     Node* cmp  = _gvn.transform(new (C) CmpINode(inst, intcon(1)));

     Node* bol  = _gvn.transform(new (C) BoolNode(cmp, BoolTest::ne));

     Node* inst_false = generate_guard(bol, NULL, PROB_MIN);

     //instanceOf == true, fallthrough

     if (inst_false != NULL) {

       phi->init_req(3, intcon(0));

       region->init_req(3, inst_false);

     }

   }

   if (!stopped()) {

     const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

     // Properly cast the argument to String

     argument = _gvn.transform(new (C) CheckCastPPNode(control(), argument, string_type));

     // This path is taken only when argument's type is String:NotNull.

     argument = cast_not_null(argument, false);

     Node* no_ctrl = NULL;

     // Get start addr of receiver

     Node* receiver_val    = load_String_value(no_ctrl, receiver);

     Node* receiver_offset = load_String_offset(no_ctrl, receiver);

     Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR);

     // Get length of receiver

     Node* receiver_cnt  = load_String_length(no_ctrl, receiver);

     // Get start addr of argument

     Node* argument_val    = load_String_value(no_ctrl, argument);

     Node* argument_offset = load_String_offset(no_ctrl, argument);

     Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR);

     // Get length of argument

     Node* argument_cnt  = load_String_length(no_ctrl, argument);

     // Check for receiver count != argument count

     Node* cmp = _gvn.transform(new(C) CmpINode(receiver_cnt, argument_cnt));

     Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::ne));

     Node* if_ne = generate_slow_guard(bol, NULL);

     if (if_ne != NULL) {

       phi->init_req(4, intcon(0));

       region->init_req(4, if_ne);

     }

     // Check for count == 0 is done by assembler code for StrEquals.

     if (!stopped()) {

       Node* equals = make_string_method_node(Op_StrEquals, receiver_start, receiver_cnt, argument_start, argument_cnt);

       phi->init_req(1, equals);

       region->init_req(1, control());

     }

   }

   // post merge

   set_control(_gvn.transform(region));

   record_for_igvn(region);

   set_result(_gvn.transform(phi));

   return true;

 }

 //------------------------------inline_array_equals----------------------------

 bool LibraryCallKit::inline_array_equals() {

   Node* arg1 = argument(0);

   Node* arg2 = argument(1);

   set_result(_gvn.transform(new (C) AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2)));

   return true;

 }

 // Java version of String.indexOf(constant string)

 // class StringDecl {

 //   StringDecl(char[] ca) {

 //     offset = 0;

 //     count = ca.length;

 //     value = ca;

 //   }

 //   int offset;

 //   int count;

 //   char[] value;

 // }

 //

 // static int string_indexOf_J(StringDecl string_object, char[] target_object,

 //                             int targetOffset, int cache_i, int md2) {

 //   int cache = cache_i;

 //   int sourceOffset = string_object.offset;

 //   int sourceCount = string_object.count;

 //   int targetCount = target_object.length;

 //

 //   int targetCountLess1 = targetCount - 1;

 //   int sourceEnd = sourceOffset + sourceCount - targetCountLess1;

 //

 //   char[] source = string_object.value;

 //   char[] target = target_object;

 //   int lastChar = target[targetCountLess1];

 //

 //  outer_loop:

 //   for (int i = sourceOffset; i < sourceEnd; ) {

 //     int src = source[i + targetCountLess1];

 //     if (src == lastChar) {

 //       // With random strings and a 4-character alphabet,

 //       // reverse matching at this point sets up 0.8% fewer

 //       // frames, but (paradoxically) makes 0.3% more probes.

 //       // Since those probes are nearer the lastChar probe,

 //       // there is may be a net D$ win with reverse matching.

 //       // But, reversing loop inhibits unroll of inner loop

 //       // for unknown reason.  So, does running outer loop from

 //       // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)

 //       for (int j = 0; j < targetCountLess1; j++) {

 //         if (target[targetOffset + j] != source[i+j]) {

 //           if ((cache & (1 << source[i+j])) == 0) {

 //             if (md2 < j+1) {

 //               i += j+1;

 //               continue outer_loop;

 //             }

 //           }

 //           i += md2;

 //           continue outer_loop;

 //         }

 //       }

 //       return i - sourceOffset;

 //     }

 //     if ((cache & (1 << src)) == 0) {

 //       i += targetCountLess1;

 //     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.

 //     i++;

 //   }

 //   return -1;

 // }

 //------------------------------string_indexOf------------------------

 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,

                                      jint cache_i, jint md2_i) {

   Node* no_ctrl  = NULL;

   float likely   = PROB_LIKELY(0.9);

   float unlikely = PROB_UNLIKELY(0.9);

   const int nargs = 0; // no arguments to push back for uncommon trap in predicate

   Node* source        = load_String_value(no_ctrl, string_object);

   Node* sourceOffset  = load_String_offset(no_ctrl, string_object);

   Node* sourceCount   = load_String_length(no_ctrl, string_object);

   Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true)));

   jint target_length = target_array->length();

   const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));

   const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);

   IdealKit kit(this, false, true);

 #define __ kit.

   Node* zero             = __ ConI(0);

   Node* one              = __ ConI(1);

   Node* cache            = __ ConI(cache_i);

   Node* md2              = __ ConI(md2_i);

   Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));

   Node* targetCount      = __ ConI(target_length);

   Node* targetCountLess1 = __ ConI(target_length - 1);

   Node* targetOffset     = __ ConI(targetOffset_i);

   Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);

   IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();

   Node* outer_loop = __ make_label(2 /* goto */);

   Node* return_    = __ make_label(1);

   __ set(rtn,__ ConI(-1));

   __ loop(this, nargs, i, sourceOffset, BoolTest::lt, sourceEnd); {

        Node* i2  = __ AddI(__ value(i), targetCountLess1);

        // pin to prohibit loading of "next iteration" value which may SEGV (rare)

        Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);

        __ if_then(src, BoolTest::eq, lastChar, unlikely); {

          __ loop(this, nargs, j, zero, BoolTest::lt, targetCountLess1); {

               Node* tpj = __ AddI(targetOffset, __ value(j));

               Node* targ = load_array_element(no_ctrl, target, tpj, target_type);

               Node* ipj  = __ AddI(__ value(i), __ value(j));

               Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);

               __ if_then(targ, BoolTest::ne, src2); {

                 __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {

                   __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {

                     __ increment(i, __ AddI(__ value(j), one));

                     __ goto_(outer_loop);

                   } __ end_if(); __ dead(j);

                 }__ end_if(); __ dead(j);

                 __ increment(i, md2);

                 __ goto_(outer_loop);

               }__ end_if();

               __ increment(j, one);

          }__ end_loop(); __ dead(j);

          __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);

          __ goto_(return_);

        }__ end_if();

        __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {

          __ increment(i, targetCountLess1);

        }__ end_if();

        __ increment(i, one);

        __ bind(outer_loop);

   }__ end_loop(); __ dead(i);

   __ bind(return_);

   // Final sync IdealKit and GraphKit.

   final_sync(kit);

   Node* result = __ value(rtn);

 #undef __

   C->set_has_loops(true);

   return result;

 }

 //------------------------------inline_string_indexOf------------------------

 bool LibraryCallKit::inline_string_indexOf() {

   Node* receiver = argument(0);

   Node* arg      = argument(1);

   Node* result;

   // Disable the use of pcmpestri until it can be guaranteed that

   // the load doesn't cross into the uncommited space.

   if (Matcher::has_match_rule(Op_StrIndexOf) &&

       UseSSE42Intrinsics) {

     // Generate SSE4.2 version of indexOf

     // We currently only have match rules that use SSE4.2

     receiver = null_check(receiver);

     arg      = null_check(arg);

     if (stopped()) {

       return true;

     }

     ciInstanceKlass* str_klass = env()->String_klass();

     const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(str_klass);

     // Make the merge point

     RegionNode* result_rgn = new (C) RegionNode(4);

     Node*       result_phi = new (C) PhiNode(result_rgn, TypeInt::INT);

     Node* no_ctrl  = NULL;

     // Get start addr of source string

     Node* source = load_String_value(no_ctrl, receiver);

     Node* source_offset = load_String_offset(no_ctrl, receiver);

     Node* source_start = array_element_address(source, source_offset, T_CHAR);

     // Get length of source string

     Node* source_cnt  = load_String_length(no_ctrl, receiver);

     // Get start addr of substring

     Node* substr = load_String_value(no_ctrl, arg);

     Node* substr_offset = load_String_offset(no_ctrl, arg);

     Node* substr_start = array_element_address(substr, substr_offset, T_CHAR);

     // Get length of source string

     Node* substr_cnt  = load_String_length(no_ctrl, arg);

     // Check for substr count > string count

     Node* cmp = _gvn.transform(new(C) CmpINode(substr_cnt, source_cnt));

     Node* bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::gt));

     Node* if_gt = generate_slow_guard(bol, NULL);

     if (if_gt != NULL) {

       result_phi->init_req(2, intcon(-1));

       result_rgn->init_req(2, if_gt);

     }

     if (!stopped()) {

       // Check for substr count == 0

       cmp = _gvn.transform(new(C) CmpINode(substr_cnt, intcon(0)));

       bol = _gvn.transform(new(C) BoolNode(cmp, BoolTest::eq));

       Node* if_zero = generate_slow_guard(bol, NULL);

       if (if_zero != NULL) {

         result_phi->init_req(3, intcon(0));

         result_rgn->init_req(3, if_zero);

       }

     }

     if (!stopped()) {

       result = make_string_method_node(Op_StrIndexOf, source_start, source_cnt, substr_start, substr_cnt);

       result_phi->init_req(1, result);

       result_rgn->init_req(1, control());

     }

     set_control(_gvn.transform(result_rgn));

     record_for_igvn(result_rgn);

     result = _gvn.transform(result_phi);

   } else { // Use LibraryCallKit::string_indexOf

     // don't intrinsify if argument isn't a constant string.

     if (!arg->is_Con()) {

      return false;

     }

     const TypeOopPtr* str_type = _gvn.type(arg)->isa_oopptr();

     if (str_type == NULL) {

       return false;

     }

     ciInstanceKlass* klass = env()->String_klass();

     ciObject* str_const = str_type->const_oop();

     if (str_const == NULL || str_const->klass() != klass) {

       return false;

     }

     ciInstance* str = str_const->as_instance();

     assert(str != NULL, "must be instance");

     ciObject* v = str->field_value_by_offset(java_lang_String::value_offset_in_bytes()).as_object();

     ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array

     int o;

     int c;

     if (java_lang_String::has_offset_field()) {

       o = str->field_value_by_offset(java_lang_String::offset_offset_in_bytes()).as_int();

       c = str->field_value_by_offset(java_lang_String::count_offset_in_bytes()).as_int();

     } else {

       o = 0;

       c = pat->length();

     }

     // constant strings have no offset and count == length which

     // simplifies the resulting code somewhat so lets optimize for that.

     if (o != 0 || c != pat->length()) {

      return false;

     }

     receiver = null_check(receiver, T_OBJECT);

     // NOTE: No null check on the argument is needed since it's a constant String oop.

     if (stopped()) {

       return true;

     }

     // The null string as a pattern always returns 0 (match at beginning of string)

     if (c == 0) {

       set_result(intcon(0));

       return true;

     }

     // Generate default indexOf

     jchar lastChar = pat->char_at(o + (c - 1));

     int cache = 0;

     int i;

     for (i = 0; i < c - 1; i++) {

       assert(i < pat->length(), "out of range");

       cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));

     }

     int md2 = c;

     for (i = 0; i < c - 1; i++) {

       assert(i < pat->length(), "out of range");

       if (pat->char_at(o + i) == lastChar) {

         md2 = (c - 1) - i;

       }

     }

     result = string_indexOf(receiver, pat, o, cache, md2);

   }

   set_result(result);

   return true;

 }

 //--------------------------round_double_node--------------------------------

 // Round a double node if necessary.

 Node* LibraryCallKit::round_double_node(Node* n) {

   if (Matcher::strict_fp_requires_explicit_rounding && UseSSE <= 1)

     n = _gvn.transform(new (C) RoundDoubleNode(0, n));

   return n;

 }

 //------------------------------inline_math-----------------------------------

 // public static double Math.abs(double)

 // public static double Math.sqrt(double)

 // public static double Math.log(double)

 // public static double Math.log10(double)

 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) {

   Node* arg = round_double_node(argument(0));

   Node* n;

   switch (id) {

   case vmIntrinsics::_dabs:   n = new (C) AbsDNode(                arg);  break;

   case vmIntrinsics::_dsqrt:  n = new (C) SqrtDNode(C, control(),  arg);  break;

   case vmIntrinsics::_dlog:   n = new (C) LogDNode(C, control(),   arg);  break;

   case vmIntrinsics::_dlog10: n = new (C) Log10DNode(C, control(), arg);  break;

   default:  fatal_unexpected_iid(id);  break;

   }

   set_result(_gvn.transform(n));

   return true;

 }

 //------------------------------inline_trig----------------------------------

 // Inline sin/cos/tan instructions, if possible.  If rounding is required, do

 // argument reduction which will turn into a fast/slow diamond.

 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {

   Node* arg = round_double_node(argument(0));

   Node* n = NULL;

   switch (id) {

   case vmIntrinsics::_dsin:  n = new (C) SinDNode(C, control(), arg);  break;

   case vmIntrinsics::_dcos:  n = new (C) CosDNode(C, control(), arg);  break;

   case vmIntrinsics::_dtan:  n = new (C) TanDNode(C, control(), arg);  break;

   default:  fatal_unexpected_iid(id);  break;

   }

   n = _gvn.transform(n);

   // Rounding required?  Check for argument reduction!

   if (Matcher::strict_fp_requires_explicit_rounding) {

     static const double     pi_4 =  0.7853981633974483;

     static const double neg_pi_4 = -0.7853981633974483;

     // pi/2 in 80-bit extended precision

     // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};

     // -pi/2 in 80-bit extended precision

     // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};

     // Cutoff value for using this argument reduction technique

     //static const double    pi_2_minus_epsilon =  1.564660403643354;

     //static const double neg_pi_2_plus_epsilon = -1.564660403643354;

     // Pseudocode for sin:

     // if (x <= Math.PI / 4.0) {

     //   if (x >= -Math.PI / 4.0) return  fsin(x);

     //   if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);

     // } else {

     //   if (x <=  Math.PI / 2.0) return  fcos(x - Math.PI / 2.0);

     // }

     // return StrictMath.sin(x);

     // Pseudocode for cos:

     // if (x <= Math.PI / 4.0) {

     //   if (x >= -Math.PI / 4.0) return  fcos(x);

     //   if (x >= -Math.PI / 2.0) return  fsin(x + Math.PI / 2.0);

     // } else {

     //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);

     // }

     // return StrictMath.cos(x);

     // Actually, sticking in an 80-bit Intel value into C2 will be tough; it

     // requires a special machine instruction to load it.  Instead we'll try

     // the 'easy' case.  If we really need the extra range +/- PI/2 we'll

     // probably do the math inside the SIN encoding.

     // Make the merge point

     RegionNode* r = new (C) RegionNode(3);

     Node* phi = new (C) PhiNode(r, Type::DOUBLE);

     // Flatten arg so we need only 1 test

     Node *abs = _gvn.transform(new (C) AbsDNode(arg));

     // Node for PI/4 constant

     Node *pi4 = makecon(TypeD::make(pi_4));

     // Check PI/4 : abs(arg)

     Node *cmp = _gvn.transform(new (C) CmpDNode(pi4,abs));

     // Check: If PI/4 < abs(arg) then go slow

     Node *bol = _gvn.transform(new (C) BoolNode( cmp, BoolTest::lt ));

     // Branch either way

     IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);

     set_control(opt_iff(r,iff));

     // Set fast path result

     phi->init_req(2, n);

     // Slow path - non-blocking leaf call

     Node* call = NULL;

     switch (id) {

     case vmIntrinsics::_dsin:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dsin),

                                "Sin", NULL, arg, top());

       break;

     case vmIntrinsics::_dcos:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dcos),

                                "Cos", NULL, arg, top());

       break;

     case vmIntrinsics::_dtan:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dtan),

                                "Tan", NULL, arg, top());

       break;

     }

     assert(control()->in(0) == call, "");

     Node* slow_result = _gvn.transform(new (C) ProjNode(call, TypeFunc::Parms));

     r->init_req(1, control());

     phi->init_req(1, slow_result);

     // Post-merge

     set_control(_gvn.transform(r));

     record_for_igvn(r);

     n = _gvn.transform(phi);

     C->set_has_split_ifs(true); // Has chance for split-if optimization

   }

   set_result(n);

   return true;

 }

 void LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) {

   //-------------------

   //result=(result.isNaN())? funcAddr():result;

   // Check: If isNaN() by checking result!=result? then either trap

   // or go to runtime

   Node* cmpisnan = _gvn.transform(new (C) CmpDNode(result, result));

   // Build the boolean node

   Node* bolisnum = _gvn.transform(new (C) BoolNode(cmpisnan, BoolTest::eq));

   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {

     { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);

       // The pow or exp intrinsic returned a NaN, which requires a call

       // to the runtime.  Recompile with the runtime call.

       uncommon_trap(Deoptimization::Reason_intrinsic,

                     Deoptimization::Action_make_not_entrant);

     }

     set_result(result);

   } else {

     // If this inlining ever returned NaN in the past, we compile a call

     // to the runtime to properly handle corner cases

     IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);

     Node* if_slow = _gvn.transform(new (C) IfFalseNode(iff));

     Node* if_fast = _gvn.transform(new (C) IfTrueNode(iff));

     if (!if_slow->is_top()) {

       RegionNode* result_region = new (C) RegionNode(3);

       PhiNode*    result_val = new (C) PhiNode(result_region, Type::DOUBLE);

       result_region->init_req(1, if_fast);

       result_val->init_req(1, result);

       set_control(if_slow);

       const TypePtr* no_memory_effects = NULL;

       Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,

                                    no_memory_effects,

                                    x, top(), y, y ? top() : NULL);

       Node* value = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+0));

 #ifdef ASSERT

       Node* value_top = _gvn.transform(new (C) ProjNode(rt, TypeFunc::Parms+1));

       assert(value_top == top(), "second value must be top");

 #endif

       result_region->init_req(2, control());

       result_val->init_req(2, value);

       set_result(result_region, result_val);

     } else {

       set_result(result);

     }

   }

 }

 //------------------------------inline_exp-------------------------------------

 // Inline exp instructions, if possible.  The Intel hardware only misses

 // really odd corner cases (+/- Infinity).  Just uncommon-trap them.

 bool LibraryCallKit::inline_exp() {

   Node* arg = round_double_node(argument(0));

   Node* n   = _gvn.transform(new (C) ExpDNode(C, control(), arg));

   finish_pow_exp(n, arg, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   return true;

 }

 //------------------------------inline_pow-------------------------------------

 // Inline power instructions, if possible.

 bool LibraryCallKit::inline_pow() {

   // Pseudocode for pow

   // if (x <= 0.0) {

   //   long longy = (long)y;

   //   if ((double)longy == y) { // if y is long

   //     if (y + 1 == y) longy = 0; // huge number: even

   //     result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y);

   //   } else {

   //     result = NaN;

   //   }

   // } else {

   //   result = DPow(x,y);

   // }

   // if (result != result)?  {

   //   result = uncommon_trap() or runtime_call();

   // }

   // return result;

   Node* x = round_double_node(argument(0));

   Node* y = round_double_node(argument(2));

   Node* result = NULL;

   if (!too_many_traps(Deoptimization::Reason_intrinsic)) {

     // Short form: skip the fancy tests and just check for NaN result.

     result = _gvn.transform(new (C) PowDNode(C, control(), x, y));

   } else {

     // If this inlining ever returned NaN in the past, include all

     // checks + call to the runtime.

     // Set the merge point for If node with condition of (x <= 0.0)

     // There are four possible paths to region node and phi node

     RegionNode *r = new (C) RegionNode(4);

     Node *phi = new (C) PhiNode(r, Type::DOUBLE);

     // Build the first if node: if (x <= 0.0)

     // Node for 0 constant

     Node *zeronode = makecon(TypeD::ZERO);

     // Check x:0

     Node *cmp = _gvn.transform(new (C) CmpDNode(x, zeronode));

     // Check: If (x<=0) then go complex path

     Node *bol1 = _gvn.transform(new (C) BoolNode( cmp, BoolTest::le ));

     // Branch either way

     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);

     // Fast path taken; set region slot 3

     Node *fast_taken = _gvn.transform(new (C) IfFalseNode(if1));

     r->init_req(3,fast_taken); // Capture fast-control

     // Fast path not-taken, i.e. slow path

     Node *complex_path = _gvn.transform(new (C) IfTrueNode(if1));

     // Set fast path result

     Node *fast_result = _gvn.transform(new (C) PowDNode(C, control(), x, y));

     phi->init_req(3, fast_result);

     // Complex path

     // Build the second if node (if y is long)

     // Node for (long)y

     Node *longy = _gvn.transform(new (C) ConvD2LNode(y));

     // Node for (double)((long) y)

     Node *doublelongy= _gvn.transform(new (C) ConvL2DNode(longy));

     // Check (double)((long) y) : y

     Node *cmplongy= _gvn.transform(new (C) CmpDNode(doublelongy, y));

     // Check if (y isn't long) then go to slow path

     Node *bol2 = _gvn.transform(new (C) BoolNode( cmplongy, BoolTest::ne ));

     // Branch either way

     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);

     Node* ylong_path = _gvn.transform(new (C) IfFalseNode(if2));

     Node *slow_path = _gvn.transform(new (C) IfTrueNode(if2));

     // Calculate DPow(abs(x), y)*(1 & (long)y)

     // Node for constant 1

     Node *conone = longcon(1);

     // 1& (long)y

     Node *signnode= _gvn.transform(new (C) AndLNode(conone, longy));

     // A huge number is always even. Detect a huge number by checking

     // if y + 1 == y and set integer to be tested for parity to 0.

     // Required for corner case:

     // (long)9.223372036854776E18 = max_jlong

     // (double)(long)9.223372036854776E18 = 9.223372036854776E18

     // max_jlong is odd but 9.223372036854776E18 is even

     Node* yplus1 = _gvn.transform(new (C) AddDNode(y, makecon(TypeD::make(1))));

     Node *cmpyplus1= _gvn.transform(new (C) CmpDNode(yplus1, y));

     Node *bolyplus1 = _gvn.transform(new (C) BoolNode( cmpyplus1, BoolTest::eq ));

     Node* correctedsign = NULL;

     if (ConditionalMoveLimit != 0) {

       correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG));

     } else {

       IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN);

       RegionNode *r = new (C) RegionNode(3);

       Node *phi = new (C) PhiNode(r, TypeLong::LONG);

       r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyplus1)));

       r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyplus1)));

       phi->init_req(1, signnode);

       phi->init_req(2, longcon(0));

       correctedsign = _gvn.transform(phi);

       ylong_path = _gvn.transform(r);

       record_for_igvn(r);

     }

     // zero node

     Node *conzero = longcon(0);

     // Check (1&(long)y)==0?

     Node *cmpeq1 = _gvn.transform(new (C) CmpLNode(correctedsign, conzero));

     // Check if (1&(long)y)!=0?, if so the result is negative

     Node *bol3 = _gvn.transform(new (C) BoolNode( cmpeq1, BoolTest::ne ));

     // abs(x)

     Node *absx=_gvn.transform(new (C) AbsDNode(x));

     // abs(x)^y

     Node *absxpowy = _gvn.transform(new (C) PowDNode(C, control(), absx, y));

     // -abs(x)^y

     Node *negabsxpowy = _gvn.transform(new (C) NegDNode (absxpowy));

     // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y)

     Node *signresult = NULL;

     if (ConditionalMoveLimit != 0) {

       signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));

     } else {

       IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN);

       RegionNode *r = new (C) RegionNode(3);

       Node *phi = new (C) PhiNode(r, Type::DOUBLE);

       r->init_req(1, _gvn.transform(new (C) IfFalseNode(ifyeven)));

       r->init_req(2, _gvn.transform(new (C) IfTrueNode(ifyeven)));

       phi->init_req(1, absxpowy);

       phi->init_req(2, negabsxpowy);

       signresult = _gvn.transform(phi);

       ylong_path = _gvn.transform(r);

       record_for_igvn(r);

     }

     // Set complex path fast result

     r->init_req(2, ylong_path);

     phi->init_req(2, signresult);

     static const jlong nan_bits = CONST64(0x7ff8000000000000);

     Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN

     r->init_req(1,slow_path);

     phi->init_req(1,slow_result);

     // Post merge

     set_control(_gvn.transform(r));

     record_for_igvn(r);

     result = _gvn.transform(phi);

   }

   finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   return true;

 }

 //------------------------------runtime_math-----------------------------

 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {

   assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),

          "must be (DD)D or (D)D type");

   // Inputs

   Node* a = round_double_node(argument(0));

   Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL;

   const TypePtr* no_memory_effects = NULL;

   Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,

                                  no_memory_effects,

                                  a, top(), b, b ? top() : NULL);

   Node* value = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+0));

 #ifdef ASSERT

   Node* value_top = _gvn.transform(new (C) ProjNode(trig, TypeFunc::Parms+1));

   assert(value_top == top(), "second value must be top");

 #endif

   set_result(value);

   return true;

 }

 //------------------------------inline_math_native-----------------------------

 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {

 #define FN_PTR(f) CAST_FROM_FN_PTR(address, f)

   switch (id) {

     // These intrinsics are not properly supported on all hardware

   case vmIntrinsics::_dcos:   return Matcher::has_match_rule(Op_CosD)   ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos),   "COS");

   case vmIntrinsics::_dsin:   return Matcher::has_match_rule(Op_SinD)   ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin),   "SIN");

   case vmIntrinsics::_dtan:   return Matcher::has_match_rule(Op_TanD)   ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan),   "TAN");

   case vmIntrinsics::_dlog:   return Matcher::has_match_rule(Op_LogD)   ? inline_math(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog),   "LOG");

   case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_math(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10");

     // These intrinsics are supported on all hardware

   case vmIntrinsics::_dsqrt:  return Matcher::has_match_rule(Op_SqrtD)  ? inline_math(id) : false;

   case vmIntrinsics::_dabs:   return Matcher::has_match_rule(Op_AbsD)   ? inline_math(id) : false;

   case vmIntrinsics::_dexp:   return Matcher::has_match_rule(Op_ExpD)   ? inline_exp()    :

     runtime_math(OptoRuntime::Math_D_D_Type(),  FN_PTR(SharedRuntime::dexp),  "EXP");

   case vmIntrinsics::_dpow:   return Matcher::has_match_rule(Op_PowD)   ? inline_pow()    :

     runtime_math(OptoRuntime::Math_DD_D_Type(), FN_PTR(SharedRuntime::dpow),  "POW");

 #undef FN_PTR

    // These intrinsics are not yet correctly implemented

   case vmIntrinsics::_datan2:

     return false;

   default:

     fatal_unexpected_iid(id);

     return false;

   }

 }

 static bool is_simple_name(Node* n) {

   return (n->req() == 1         // constant

           || (n->is_Type() && n->as_Type()->type()->singleton())

           || n->is_Proj()       // parameter or return value

           || n->is_Phi()        // local of some sort

           );

 }

 //----------------------------inline_min_max-----------------------------------

 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {

   set_result(generate_min_max(id, argument(0), argument(1)));

   return true;

 }

 Node*

 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {

   // These are the candidate return value:

   Node* xvalue = x0;

   Node* yvalue = y0;

   if (xvalue == yvalue) {

     return xvalue;

   }

   bool want_max = (id == vmIntrinsics::_max);

   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();

   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();

   if (txvalue == NULL || tyvalue == NULL)  return top();

   // This is not really necessary, but it is consistent with a

   // hypothetical MaxINode::Value method:

   int widen = MAX2(txvalue->_widen, tyvalue->_widen);

   // %%% This folding logic should (ideally) be in a different place.

   // Some should be inside IfNode, and there to be a more reliable

   // transformation of ?: style patterns into cmoves.  We also want

   // more powerful optimizations around cmove and min/max.

   // Try to find a dominating comparison of these guys.

   // It can simplify the index computation for Arrays.copyOf

   // and similar uses of System.arraycopy.

   // First, compute the normalized version of CmpI(x, y).

   int   cmp_op = Op_CmpI;

   Node* xkey = xvalue;

   Node* ykey = yvalue;

   Node* ideal_cmpxy = _gvn.transform(new(C) CmpINode(xkey, ykey));

   if (ideal_cmpxy->is_Cmp()) {

     // E.g., if we have CmpI(length - offset, count),

     // it might idealize to CmpI(length, count + offset)

     cmp_op = ideal_cmpxy->Opcode();

     xkey = ideal_cmpxy->in(1);

     ykey = ideal_cmpxy->in(2);

   }

   // Start by locating any relevant comparisons.

   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;

   Node* cmpxy = NULL;

   Node* cmpyx = NULL;

   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {

     Node* cmp = start_from->fast_out(k);

     if (cmp->outcnt() > 0 &&            // must have prior uses

         cmp->in(0) == NULL &&           // must be context-independent

         cmp->Opcode() == cmp_op) {      // right kind of compare

       if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;

       if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;

     }

   }

   const int NCMPS = 2;

   Node* cmps[NCMPS] = { cmpxy, cmpyx };

   int cmpn;

   for (cmpn = 0; cmpn < NCMPS; cmpn++) {

     if (cmps[cmpn] != NULL)  break;     // find a result

   }

   if (cmpn < NCMPS) {

     // Look for a dominating test that tells us the min and max.

     int depth = 0;                // Limit search depth for speed

     Node* dom = control();

     for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {

       if (++depth >= 100)  break;

       Node* ifproj = dom;

       if (!ifproj->is_Proj())  continue;

       Node* iff = ifproj->in(0);

       if (!iff->is_If())  continue;

       Node* bol = iff->in(1);

       if (!bol->is_Bool())  continue;

       Node* cmp = bol->in(1);

       if (cmp == NULL)  continue;

       for (cmpn = 0; cmpn < NCMPS; cmpn++)

         if (cmps[cmpn] == cmp)  break;

       if (cmpn == NCMPS)  continue;

       BoolTest::mask btest = bol->as_Bool()->_test._test;

       if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();

       if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();

       // At this point, we know that 'x btest y' is true.

       switch (btest) {

       case BoolTest::eq:

         // They are proven equal, so we can collapse the min/max.

         // Either value is the answer.  Choose the simpler.

         if (is_simple_name(yvalue) && !is_simple_name(xvalue))

           return yvalue;

         return xvalue;

       case BoolTest::lt:          // x < y

       case BoolTest::le:          // x <= y

         return (want_max ? yvalue : xvalue);

       case BoolTest::gt:          // x > y

       case BoolTest::ge:          // x >= y

         return (want_max ? xvalue : yvalue);

       }

     }

   }

   // We failed to find a dominating test.

   // Let's pick a test that might GVN with prior tests.

   Node*          best_bol   = NULL;

   BoolTest::mask best_btest = BoolTest::illegal;

   for (cmpn = 0; cmpn < NCMPS; cmpn++) {

     Node* cmp = cmps[cmpn];

     if (cmp == NULL)  continue;

     for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {

       Node* bol = cmp->fast_out(j);

       if (!bol->is_Bool())  continue;

       BoolTest::mask btest = bol->as_Bool()->_test._test;

       if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;

       if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();

       if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {

         best_bol   = bol->as_Bool();

         best_btest = btest;

       }

     }

   }

   Node* answer_if_true  = NULL;

   Node* answer_if_false = NULL;

   switch (best_btest) {

   default:

     if (cmpxy == NULL)

       cmpxy = ideal_cmpxy;

     best_bol = _gvn.transform(new(C) BoolNode(cmpxy, BoolTest::lt));

     // and fall through:

   case BoolTest::lt:          // x < y

   case BoolTest::le:          // x <= y

     answer_if_true  = (want_max ? yvalue : xvalue);

     answer_if_false = (want_max ? xvalue : yvalue);

     break;

   case BoolTest::gt:          // x > y

   case BoolTest::ge:          // x >= y

     answer_if_true  = (want_max ? xvalue : yvalue);

     answer_if_false = (want_max ? yvalue : xvalue);

     break;

   }

   jint hi, lo;

   if (want_max) {

     // We can sharpen the minimum.

     hi = MAX2(txvalue->_hi, tyvalue->_hi);

     lo = MAX2(txvalue->_lo, tyvalue->_lo);

   } else {

     // We can sharpen the maximum.

     hi = MIN2(txvalue->_hi, tyvalue->_hi);

     lo = MIN2(txvalue->_lo, tyvalue->_lo);

   }

   // Use a flow-free graph structure, to avoid creating excess control edges

   // which could hinder other optimizations.

   // Since Math.min/max is often used with arraycopy, we want

   // tightly_coupled_allocation to be able to see beyond min/max expressions.

   Node* cmov = CMoveNode::make(C, NULL, best_bol,

                                answer_if_false, answer_if_true,

                                TypeInt::make(lo, hi, widen));

   return _gvn.transform(cmov);

   /*

   // This is not as desirable as it may seem, since Min and Max

   // nodes do not have a full set of optimizations.

   // And they would interfere, anyway, with 'if' optimizations

   // and with CMoveI canonical forms.

   switch (id) {

   case vmIntrinsics::_min:

     result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;

   case vmIntrinsics::_max:

     result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;

   default:

     ShouldNotReachHere();

   }

   */

 }

 inline int

 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {

   const TypePtr* base_type = TypePtr::NULL_PTR;

   if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();

   if (base_type == NULL) {

     // Unknown type.

     return Type::AnyPtr;

   } else if (base_type == TypePtr::NULL_PTR) {

     // Since this is a NULL+long form, we have to switch to a rawptr.

     base   = _gvn.transform(new (C) CastX2PNode(offset));

     offset = MakeConX(0);

     return Type::RawPtr;

   } else if (base_type->base() == Type::RawPtr) {

     return Type::RawPtr;

   } else if (base_type->isa_oopptr()) {

     // Base is never null => always a heap address.

     if (base_type->ptr() == TypePtr::NotNull) {

       return Type::OopPtr;

     }

     // Offset is small => always a heap address.

     const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();

     if (offset_type != NULL &&

         base_type->offset() == 0 &&     // (should always be?)

         offset_type->_lo >= 0 &&

         !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {

       return Type::OopPtr;

     }

     // Otherwise, it might either be oop+off or NULL+addr.

     return Type::AnyPtr;

   } else {

     // No information:

     return Type::AnyPtr;

   }

 }

 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {

   int kind = classify_unsafe_addr(base, offset);

   if (kind == Type::RawPtr) {

     return basic_plus_adr(top(), base, offset);

   } else {

     return basic_plus_adr(base, offset);

   }

 }

 //--------------------------inline_number_methods-----------------------------

 // inline int     Integer.numberOfLeadingZeros(int)

 // inline int        Long.numberOfLeadingZeros(long)

 //

 // inline int     Integer.numberOfTrailingZeros(int)

 // inline int        Long.numberOfTrailingZeros(long)

 //

 // inline int     Integer.bitCount(int)

 // inline int        Long.bitCount(long)

 //

 // inline char  Character.reverseBytes(char)

 // inline short     Short.reverseBytes(short)

 // inline int     Integer.reverseBytes(int)

 // inline long       Long.reverseBytes(long)

 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) {

   Node* arg = argument(0);

   Node* n;

   switch (id) {

   case vmIntrinsics::_numberOfLeadingZeros_i:   n = new (C) CountLeadingZerosINode( arg);  break;

   case vmIntrinsics::_numberOfLeadingZeros_l:   n = new (C) CountLeadingZerosLNode( arg);  break;

   case vmIntrinsics::_numberOfTrailingZeros_i:  n = new (C) CountTrailingZerosINode(arg);  break;

   case vmIntrinsics::_numberOfTrailingZeros_l:  n = new (C) CountTrailingZerosLNode(arg);  break;

   case vmIntrinsics::_bitCount_i:               n = new (C) PopCountINode(          arg);  break;

   case vmIntrinsics::_bitCount_l:               n = new (C) PopCountLNode(          arg);  break;

   case vmIntrinsics::_reverseBytes_c:           n = new (C) ReverseBytesUSNode(0,   arg);  break;

   case vmIntrinsics::_reverseBytes_s:           n = new (C) ReverseBytesSNode( 0,   arg);  break;

   case vmIntrinsics::_reverseBytes_i:           n = new (C) ReverseBytesINode( 0,   arg);  break;

   case vmIntrinsics::_reverseBytes_l:           n = new (C) ReverseBytesLNode( 0,   arg);  break;

   default:  fatal_unexpected_iid(id);  break;

   }

   set_result(_gvn.transform(n));

   return true;

 }

 //----------------------------inline_unsafe_access----------------------------

 const static BasicType T_ADDRESS_HOLDER = T_LONG;

 // Helper that guards and inserts a pre-barrier.

 void LibraryCallKit::insert_pre_barrier(Node* base_oop, Node* offset,

                                         Node* pre_val, bool need_mem_bar) {

   // We could be accessing the referent field of a reference object. If so, when G1

   // is enabled, we need to log the value in the referent field in an SATB buffer.

   // This routine performs some compile time filters and generates suitable

   // runtime filters that guard the pre-barrier code.

   // Also add memory barrier for non volatile load from the referent field

   // to prevent commoning of loads across safepoint.

   if (!UseG1GC && !need_mem_bar)

     return;

   // Some compile time checks.

   // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?

   const TypeX* otype = offset->find_intptr_t_type();

   if (otype != NULL && otype->is_con() &&

       otype->get_con() != java_lang_ref_Reference::referent_offset) {

     // Constant offset but not the reference_offset so just return

     return;

   }

   // We only need to generate the runtime guards for instances.

   const TypeOopPtr* btype = base_oop->bottom_type()->isa_oopptr();

   if (btype != NULL) {

     if (btype->isa_aryptr()) {

       // Array type so nothing to do

       return;

     }

     const TypeInstPtr* itype = btype->isa_instptr();

     if (itype != NULL) {

       // Can the klass of base_oop be statically determined to be

       // _not_ a sub-class of Reference and _not_ Object?

       ciKlass* klass = itype->klass();

       if ( klass->is_loaded() &&

           !klass->is_subtype_of(env()->Reference_klass()) &&

           !env()->Object_klass()->is_subtype_of(klass)) {

         return;

       }

     }

   }

   // The compile time filters did not reject base_oop/offset so

   // we need to generate the following runtime filters

   //

   // if (offset == java_lang_ref_Reference::_reference_offset) {

   //   if (instance_of(base, java.lang.ref.Reference)) {

   //     pre_barrier(_, pre_val, ...);

   //   }

   // }

   float likely   = PROB_LIKELY(  0.999);

   float unlikely = PROB_UNLIKELY(0.999);

   IdealKit ideal(this);

 #define __ ideal.

   Node* referent_off = __ ConX(java_lang_ref_Reference::referent_offset);

   __ if_then(offset, BoolTest::eq, referent_off, unlikely); {

       // Update graphKit memory and control from IdealKit.

       sync_kit(ideal);

       Node* ref_klass_con = makecon(TypeKlassPtr::make(env()->Reference_klass()));

       Node* is_instof = gen_instanceof(base_oop, ref_klass_con);

       // Update IdealKit memory and control from graphKit.

       __ sync_kit(this);

       Node* one = __ ConI(1);

       // is_instof == 0 if base_oop == NULL

       __ if_then(is_instof, BoolTest::eq, one, unlikely); {

         // Update graphKit from IdeakKit.

         sync_kit(ideal);

         // Use the pre-barrier to record the value in the referent field

         pre_barrier(false /* do_load */,

                     __ ctrl(),

                     NULL /* obj */, NULL /* adr */, max_juint /* alias_idx */, NULL /* val */, NULL /* val_type */,

                     pre_val /* pre_val */,

                     T_OBJECT);

         if (need_mem_bar) {

           // Add memory barrier to prevent commoning reads from this field

           // across safepoint since GC can change its value.

           insert_mem_bar(Op_MemBarCPUOrder);

         }

         // Update IdealKit from graphKit.

         __ sync_kit(this);

       } __ end_if(); // _ref_type != ref_none

   } __ end_if(); // offset == referent_offset

   // Final sync IdealKit and GraphKit.

   final_sync(ideal);

 #undef __

 }

 // Interpret Unsafe.fieldOffset cookies correctly:

 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);

 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type, bool is_native_ptr) {

   // Attempt to infer a sharper value type from the offset and base type.

   ciKlass* sharpened_klass = NULL;

   // See if it is an instance field, with an object type.

   if (alias_type->field() != NULL) {

     assert(!is_native_ptr, "native pointer op cannot use a java address");

     if (alias_type->field()->type()->is_klass()) {

       sharpened_klass = alias_type->field()->type()->as_klass();

     }

   }

   // See if it is a narrow oop array.

   if (adr_type->isa_aryptr()) {

     if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {

       const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();

       if (elem_type != NULL) {

         sharpened_klass = elem_type->klass();

       }

     }

   }

   // The sharpened class might be unloaded if there is no class loader

   // contraint in place.

   if (sharpened_klass != NULL && sharpened_klass->is_loaded()) {

     const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);

 #ifndef PRODUCT

     if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {

       tty->print("  from base type: ");  adr_type->dump();

       tty->print("  sharpened value: ");  tjp->dump();

     }

 #endif

     // Sharpen the value type.

     return tjp;

   }

   return NULL;

 }

 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) {

   if (callee()->is_static())  return false;  // caller must have the capability!

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = callee()->signature();

 #ifdef ASSERT

     if (!is_store) {

       // Object getObject(Object base, int/long offset), etc.

       BasicType rtype = sig->return_type()->basic_type();

       if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())

           rtype = T_ADDRESS;  // it is really a C void*

       assert(rtype == type, "getter must return the expected value");

       if (!is_native_ptr) {

         assert(sig->count() == 2, "oop getter has 2 arguments");

         assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");

         assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");

       } else {

         assert(sig->count() == 1, "native getter has 1 argument");

         assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");

       }

     } else {

       // void putObject(Object base, int/long offset, Object x), etc.

       assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");

       if (!is_native_ptr) {

         assert(sig->count() == 3, "oop putter has 3 arguments");

         assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");

         assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");

       } else {

         assert(sig->count() == 2, "native putter has 2 arguments");

         assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");

       }

       BasicType vtype = sig->type_at(sig->count()-1)->basic_type();

       if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())

         vtype = T_ADDRESS;  // it is really a C void*

       assert(vtype == type, "putter must accept the expected value");

     }

 #endif // ASSERT

  }

 #endif //PRODUCT

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   Node* receiver = argument(0);  // type: oop

   // Build address expression.  See the code in inline_unsafe_prefetch.

   Node* adr;

   Node* heap_base_oop = top();

   Node* offset = top();

   Node* val;

   if (!is_native_ptr) {

     // The base is either a Java object or a value produced by Unsafe.staticFieldBase

     Node* base = argument(1);  // type: oop

     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset

     offset = argument(2);  // type: long

     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset

     // to be plain byte offsets, which are also the same as those accepted

     // by oopDesc::field_base.

     assert(Unsafe_field_offset_to_byte_offset(11) == 11,

            "fieldOffset must be byte-scaled");

     // 32-bit machines ignore the high half!

     offset = ConvL2X(offset);

     adr = make_unsafe_address(base, offset);

     heap_base_oop = base;

     val = is_store ? argument(4) : NULL;

   } else {

     Node* ptr = argument(1);  // type: long

     ptr = ConvL2X(ptr);  // adjust Java long to machine word

     adr = make_unsafe_address(NULL, ptr);

     val = is_store ? argument(3) : NULL;

   }

   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();

   // First guess at the value type.

   const Type *value_type = Type::get_const_basic_type(type);

   // Try to categorize the address.  If it comes up as TypeJavaPtr::BOTTOM,

   // there was not enough information to nail it down.

   Compile::AliasType* alias_type = C->alias_type(adr_type);

   assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");

   // We will need memory barriers unless we can determine a unique

   // alias category for this reference.  (Note:  If for some reason

   // the barriers get omitted and the unsafe reference begins to "pollute"

   // the alias analysis of the rest of the graph, either Compile::can_alias

   // or Compile::must_alias will throw a diagnostic assert.)

   bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);

   // If we are reading the value of the referent field of a Reference

   // object (either by using Unsafe directly or through reflection)

   // then, if G1 is enabled, we need to record the referent in an

   // SATB log buffer using the pre-barrier mechanism.

   // Also we need to add memory barrier to prevent commoning reads

   // from this field across safepoint since GC can change its value.

   bool need_read_barrier = !is_native_ptr && !is_store &&

                            offset != top() && heap_base_oop != top();

   if (!is_store && type == T_OBJECT) {

     const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type, is_native_ptr);

     if (tjp != NULL) {

       value_type = tjp;

     }

   }

   receiver = null_check(receiver);

   if (stopped()) {

     return true;

   }

   // Heap pointers get a null-check from the interpreter,

   // as a courtesy.  However, this is not guaranteed by Unsafe,

   // and it is not possible to fully distinguish unintended nulls

   // from intended ones in this API.

   if (is_volatile) {

     // We need to emit leading and trailing CPU membars (see below) in

     // addition to memory membars when is_volatile. This is a little

     // too strong, but avoids the need to insert per-alias-type

     // volatile membars (for stores; compare Parse::do_put_xxx), which

     // we cannot do effectively here because we probably only have a

     // rough approximation of type.

     need_mem_bar = true;

     // For Stores, place a memory ordering barrier now.

     if (is_store)

       insert_mem_bar(Op_MemBarRelease);

   }

   // Memory barrier to prevent normal and 'unsafe' accesses from

   // bypassing each other.  Happens after null checks, so the

   // exception paths do not take memory state from the memory barrier,

   // so there's no problems making a strong assert about mixing users

   // of safe & unsafe memory.  Otherwise fails in a CTW of rt.jar

   // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.

   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);

   if (!is_store) {

     Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile);

     // load value

     switch (type) {

     case T_BOOLEAN:

     case T_CHAR:

     case T_BYTE:

     case T_SHORT:

     case T_INT:

     case T_LONG:

     case T_FLOAT:

     case T_DOUBLE:

       break;

     case T_OBJECT:

       if (need_read_barrier) {

         insert_pre_barrier(heap_base_oop, offset, p, !(is_volatile || need_mem_bar));

       }

       break;

     case T_ADDRESS:

       // Cast to an int type.

       p = _gvn.transform(new (C) CastP2XNode(NULL, p));

       p = ConvX2L(p);

       break;

     default:

       fatal(err_msg_res("unexpected type %d: %s", type, type2name(type)));

       break;

     }

     // The load node has the control of the preceding MemBarCPUOrder.  All

     // following nodes will have the control of the MemBarCPUOrder inserted at

     // the end of this method.  So, pushing the load onto the stack at a later

     // point is fine.

     set_result(p);

   } else {

     // place effect of store into memory

     switch (type) {

     case T_DOUBLE:

       val = dstore_rounding(val);

       break;

     case T_ADDRESS:

       // Repackage the long as a pointer.

       val = ConvL2X(val);

       val = _gvn.transform(new (C) CastX2PNode(val));

       break;

     }

     if (type != T_OBJECT ) {

       (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);

     } else {

       // Possibly an oop being stored to Java heap or native memory

       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {

         // oop to Java heap.

         (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);

       } else {

         // We can't tell at compile time if we are storing in the Java heap or outside

         // of it. So we need to emit code to conditionally do the proper type of

         // store.

         IdealKit ideal(this);

 #define __ ideal.

         // QQQ who knows what probability is here??

         __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {

           // Sync IdealKit and graphKit.

           sync_kit(ideal);

           Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);

           // Update IdealKit memory.

           __ sync_kit(this);

         } __ else_(); {

           __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile);

         } __ end_if();

         // Final sync IdealKit and GraphKit.

         final_sync(ideal);

 #undef __

       }

     }

   }

   if (is_volatile) {

     if (!is_store)

       insert_mem_bar(Op_MemBarAcquire);

     else

       insert_mem_bar(Op_MemBarVolatile);

   }

   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);

   return true;

 }

 //----------------------------inline_unsafe_prefetch----------------------------

 bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = callee()->signature();

 #ifdef ASSERT

     // Object getObject(Object base, int/long offset), etc.

     BasicType rtype = sig->return_type()->basic_type();

     if (!is_native_ptr) {

       assert(sig->count() == 2, "oop prefetch has 2 arguments");

       assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");

       assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");

     } else {

       assert(sig->count() == 1, "native prefetch has 1 argument");

       assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");

     }

 #endif // ASSERT

   }

 #endif // !PRODUCT

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   const int idx = is_static ? 0 : 1;

   if (!is_static) {

     null_check_receiver();

     if (stopped()) {

       return true;

     }

   }

   // Build address expression.  See the code in inline_unsafe_access.

   Node *adr;

   if (!is_native_ptr) {

     // The base is either a Java object or a value produced by Unsafe.staticFieldBase

     Node* base   = argument(idx + 0);  // type: oop

     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset

     Node* offset = argument(idx + 1);  // type: long

     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset

     // to be plain byte offsets, which are also the same as those accepted

     // by oopDesc::field_base.

     assert(Unsafe_field_offset_to_byte_offset(11) == 11,

            "fieldOffset must be byte-scaled");

     // 32-bit machines ignore the high half!

     offset = ConvL2X(offset);

     adr = make_unsafe_address(base, offset);

   } else {

     Node* ptr = argument(idx + 0);  // type: long

     ptr = ConvL2X(ptr);  // adjust Java long to machine word

     adr = make_unsafe_address(NULL, ptr);

   }

   // Generate the read or write prefetch

   Node *prefetch;

   if (is_store) {

     prefetch = new (C) PrefetchWriteNode(i_o(), adr);

   } else {

     prefetch = new (C) PrefetchReadNode(i_o(), adr);

   }

   prefetch->init_req(0, control());

   set_i_o(_gvn.transform(prefetch));

   return true;

 }

 //----------------------------inline_unsafe_load_store----------------------------

 // This method serves a couple of different customers (depending on LoadStoreKind):

 //

 // LS_cmpxchg:

 //   public final native boolean compareAndSwapObject(Object o, long offset, Object expected, Object x);

 //   public final native boolean compareAndSwapInt(   Object o, long offset, int    expected, int    x);

 //   public final native boolean compareAndSwapLong(  Object o, long offset, long   expected, long   x);

 //

 // LS_xadd:

 //   public int  getAndAddInt( Object o, long offset, int  delta)

 //   public long getAndAddLong(Object o, long offset, long delta)

 //

 // LS_xchg:

 //   int    getAndSet(Object o, long offset, int    newValue)

 //   long   getAndSet(Object o, long offset, long   newValue)

 //   Object getAndSet(Object o, long offset, Object newValue)

 //

 bool LibraryCallKit::inline_unsafe_load_store(BasicType type, LoadStoreKind kind) {

   // This basic scheme here is the same as inline_unsafe_access, but

   // differs in enough details that combining them would make the code

   // overly confusing.  (This is a true fact! I originally combined

   // them, but even I was confused by it!) As much code/comments as

   // possible are retained from inline_unsafe_access though to make

   // the correspondences clearer. - dl

   if (callee()->is_static())  return false;  // caller must have the capability!

 #ifndef PRODUCT

   BasicType rtype;

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = callee()->signature();

     rtype = sig->return_type()->basic_type();

     if (kind == LS_xadd || kind == LS_xchg) {

       // Check the signatures.

 #ifdef ASSERT

       assert(rtype == type, "get and set must return the expected type");

       assert(sig->count() == 3, "get and set has 3 arguments");

       assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object");

       assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long");

       assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta");

 #endif // ASSERT

     } else if (kind == LS_cmpxchg) {

       // Check the signatures.