Mercurial > jdk8-mips64-public > hotspot / file revision

 /*

  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.

  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

  *

  * This code is free software; you can redistribute it and/or modify it

  * under the terms of the GNU General Public License version 2 only, as

  * published by the Free Software Foundation.

  *

  * This code is distributed in the hope that it will be useful, but WITHOUT

  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

  * version 2 for more details (a copy is included in the LICENSE file that

  * accompanied this code).

  *

  * You should have received a copy of the GNU General Public License version

  * 2 along with this work; if not, write to the Free Software Foundation,

  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

  *

  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

  * or visit www.oracle.com if you need additional information or have any

  * questions.

  *

  */

 #include "incls/_precompiled.incl"

 #include "incls/_library_call.cpp.incl"

 class LibraryIntrinsic : public InlineCallGenerator {

   // Extend the set of intrinsics known to the runtime:

  public:

  private:

   bool             _is_virtual;

   vmIntrinsics::ID _intrinsic_id;

  public:

   LibraryIntrinsic(ciMethod* m, bool is_virtual, vmIntrinsics::ID id)

     : InlineCallGenerator(m),

       _is_virtual(is_virtual),

       _intrinsic_id(id)

   {

   }

   virtual bool is_intrinsic() const { return true; }

   virtual bool is_virtual()   const { return _is_virtual; }

   virtual JVMState* generate(JVMState* jvms);

   vmIntrinsics::ID intrinsic_id() const { return _intrinsic_id; }

 };

 // Local helper class for LibraryIntrinsic:

 class LibraryCallKit : public GraphKit {

  private:

   LibraryIntrinsic* _intrinsic;   // the library intrinsic being called

  public:

   LibraryCallKit(JVMState* caller, LibraryIntrinsic* intrinsic)

     : GraphKit(caller),

       _intrinsic(intrinsic)

   {

   }

   ciMethod*         caller()    const    { return jvms()->method(); }

   int               bci()       const    { return jvms()->bci(); }

   LibraryIntrinsic* intrinsic() const    { return _intrinsic; }

   vmIntrinsics::ID  intrinsic_id() const { return _intrinsic->intrinsic_id(); }

   ciMethod*         callee()    const    { return _intrinsic->method(); }

   ciSignature*      signature() const    { return callee()->signature(); }

   int               arg_size()  const    { return callee()->arg_size(); }

   bool try_to_inline();

   // Helper functions to inline natives

   void push_result(RegionNode* region, PhiNode* value);

   Node* generate_guard(Node* test, RegionNode* region, float true_prob);

   Node* generate_slow_guard(Node* test, RegionNode* region);

   Node* generate_fair_guard(Node* test, RegionNode* region);

   Node* generate_negative_guard(Node* index, RegionNode* region,

                                 // resulting CastII of index:

                                 Node* *pos_index = NULL);

   Node* generate_nonpositive_guard(Node* index, bool never_negative,

                                    // resulting CastII of index:

                                    Node* *pos_index = NULL);

   Node* generate_limit_guard(Node* offset, Node* subseq_length,

                              Node* array_length,

                              RegionNode* region);

   Node* generate_current_thread(Node* &tls_output);

   address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,

                               bool disjoint_bases, const char* &name);

   Node* load_mirror_from_klass(Node* klass);

   Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,

                                       int nargs,

                                       RegionNode* region, int null_path,

                                       int offset);

   Node* load_klass_from_mirror(Node* mirror, bool never_see_null, int nargs,

                                RegionNode* region, int null_path) {

     int offset = java_lang_Class::klass_offset_in_bytes();

     return load_klass_from_mirror_common(mirror, never_see_null, nargs,

                                          region, null_path,

                                          offset);

   }

   Node* load_array_klass_from_mirror(Node* mirror, bool never_see_null,

                                      int nargs,

                                      RegionNode* region, int null_path) {

     int offset = java_lang_Class::array_klass_offset_in_bytes();

     return load_klass_from_mirror_common(mirror, never_see_null, nargs,

                                          region, null_path,

                                          offset);

   }

   Node* generate_access_flags_guard(Node* kls,

                                     int modifier_mask, int modifier_bits,

                                     RegionNode* region);

   Node* generate_interface_guard(Node* kls, RegionNode* region);

   Node* generate_array_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, false, false);

   }

   Node* generate_non_array_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, false, true);

   }

   Node* generate_objArray_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, true, false);

   }

   Node* generate_non_objArray_guard(Node* kls, RegionNode* region) {

     return generate_array_guard_common(kls, region, true, true);

   }

   Node* generate_array_guard_common(Node* kls, RegionNode* region,

                                     bool obj_array, bool not_array);

   Node* generate_virtual_guard(Node* obj_klass, RegionNode* slow_region);

   CallJavaNode* generate_method_call(vmIntrinsics::ID method_id,

                                      bool is_virtual = false, bool is_static = false);

   CallJavaNode* generate_method_call_static(vmIntrinsics::ID method_id) {

     return generate_method_call(method_id, false, true);

   }

   CallJavaNode* generate_method_call_virtual(vmIntrinsics::ID method_id) {

     return generate_method_call(method_id, true, false);

   }

   Node* make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2);

   bool inline_string_compareTo();

   bool inline_string_indexOf();

   Node* string_indexOf(Node* string_object, ciTypeArray* target_array, jint offset, jint cache_i, jint md2_i);

   bool inline_string_equals();

   Node* pop_math_arg();

   bool runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName);

   bool inline_math_native(vmIntrinsics::ID id);

   bool inline_trig(vmIntrinsics::ID id);

   bool inline_trans(vmIntrinsics::ID id);

   bool inline_abs(vmIntrinsics::ID id);

   bool inline_sqrt(vmIntrinsics::ID id);

   bool inline_pow(vmIntrinsics::ID id);

   bool inline_exp(vmIntrinsics::ID id);

   bool inline_min_max(vmIntrinsics::ID id);

   Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y);

   // This returns Type::AnyPtr, RawPtr, or OopPtr.

   int classify_unsafe_addr(Node* &base, Node* &offset);

   Node* make_unsafe_address(Node* base, Node* offset);

   bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile);

   bool inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static);

   bool inline_unsafe_allocate();

   bool inline_unsafe_copyMemory();

   bool inline_native_currentThread();

   bool inline_native_time_funcs(bool isNano);

   bool inline_native_isInterrupted();

   bool inline_native_Class_query(vmIntrinsics::ID id);

   bool inline_native_subtype_check();

   bool inline_native_newArray();

   bool inline_native_getLength();

   bool inline_array_copyOf(bool is_copyOfRange);

   bool inline_array_equals();

   void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark);

   bool inline_native_clone(bool is_virtual);

   bool inline_native_Reflection_getCallerClass();

   bool inline_native_AtomicLong_get();

   bool inline_native_AtomicLong_attemptUpdate();

   bool is_method_invoke_or_aux_frame(JVMState* jvms);

   // Helper function for inlining native object hash method

   bool inline_native_hashcode(bool is_virtual, bool is_static);

   bool inline_native_getClass();

   // Helper functions for inlining arraycopy

   bool inline_arraycopy();

   void generate_arraycopy(const TypePtr* adr_type,

                           BasicType basic_elem_type,

                           Node* src,  Node* src_offset,

                           Node* dest, Node* dest_offset,

                           Node* copy_length,

                           bool disjoint_bases = false,

                           bool length_never_negative = false,

                           RegionNode* slow_region = NULL);

   AllocateArrayNode* tightly_coupled_allocation(Node* ptr,

                                                 RegionNode* slow_region);

   void generate_clear_array(const TypePtr* adr_type,

                             Node* dest,

                             BasicType basic_elem_type,

                             Node* slice_off,

                             Node* slice_len,

                             Node* slice_end);

   bool generate_block_arraycopy(const TypePtr* adr_type,

                                 BasicType basic_elem_type,

                                 AllocateNode* alloc,

                                 Node* src,  Node* src_offset,

                                 Node* dest, Node* dest_offset,

                                 Node* dest_size);

   void generate_slow_arraycopy(const TypePtr* adr_type,

                                Node* src,  Node* src_offset,

                                Node* dest, Node* dest_offset,

                                Node* copy_length);

   Node* generate_checkcast_arraycopy(const TypePtr* adr_type,

                                      Node* dest_elem_klass,

                                      Node* src,  Node* src_offset,

                                      Node* dest, Node* dest_offset,

                                      Node* copy_length);

   Node* generate_generic_arraycopy(const TypePtr* adr_type,

                                    Node* src,  Node* src_offset,

                                    Node* dest, Node* dest_offset,

                                    Node* copy_length);

   void generate_unchecked_arraycopy(const TypePtr* adr_type,

                                     BasicType basic_elem_type,

                                     bool disjoint_bases,

                                     Node* src,  Node* src_offset,

                                     Node* dest, Node* dest_offset,

                                     Node* copy_length);

   bool inline_unsafe_CAS(BasicType type);

   bool inline_unsafe_ordered_store(BasicType type);

   bool inline_fp_conversions(vmIntrinsics::ID id);

   bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);

   bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);

   bool inline_bitCount(vmIntrinsics::ID id);

   bool inline_reverseBytes(vmIntrinsics::ID id);

 };

 //---------------------------make_vm_intrinsic----------------------------

 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {

   vmIntrinsics::ID id = m->intrinsic_id();

   assert(id != vmIntrinsics::_none, "must be a VM intrinsic");

   if (DisableIntrinsic[0] != '\0'

       && strstr(DisableIntrinsic, vmIntrinsics::name_at(id)) != NULL) {

     // disabled by a user request on the command line:

     // example: -XX:DisableIntrinsic=_hashCode,_getClass

     return NULL;

   }

   if (!m->is_loaded()) {

     // do not attempt to inline unloaded methods

     return NULL;

   }

   // Only a few intrinsics implement a virtual dispatch.

   // They are expensive calls which are also frequently overridden.

   if (is_virtual) {

     switch (id) {

     case vmIntrinsics::_hashCode:

     case vmIntrinsics::_clone:

       // OK, Object.hashCode and Object.clone intrinsics come in both flavors

       break;

     default:

       return NULL;

     }

   }

   // -XX:-InlineNatives disables nearly all intrinsics:

   if (!InlineNatives) {

     switch (id) {

     case vmIntrinsics::_indexOf:

     case vmIntrinsics::_compareTo:

     case vmIntrinsics::_equals:

     case vmIntrinsics::_equalsC:

       break;  // InlineNatives does not control String.compareTo

     default:

       return NULL;

     }

   }

   switch (id) {

   case vmIntrinsics::_compareTo:

     if (!SpecialStringCompareTo)  return NULL;

     break;

   case vmIntrinsics::_indexOf:

     if (!SpecialStringIndexOf)  return NULL;

     break;

   case vmIntrinsics::_equals:

     if (!SpecialStringEquals)  return NULL;

     break;

   case vmIntrinsics::_equalsC:

     if (!SpecialArraysEquals)  return NULL;

     break;

   case vmIntrinsics::_arraycopy:

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_copyMemory:

     if (StubRoutines::unsafe_arraycopy() == NULL)  return NULL;

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_hashCode:

     if (!InlineObjectHash)  return NULL;

     break;

   case vmIntrinsics::_clone:

   case vmIntrinsics::_copyOf:

   case vmIntrinsics::_copyOfRange:

     if (!InlineObjectCopy)  return NULL;

     // These also use the arraycopy intrinsic mechanism:

     if (!InlineArrayCopy)  return NULL;

     break;

   case vmIntrinsics::_checkIndex:

     // We do not intrinsify this.  The optimizer does fine with it.

     return NULL;

   case vmIntrinsics::_get_AtomicLong:

   case vmIntrinsics::_attemptUpdate:

     if (!InlineAtomicLong)  return NULL;

     break;

   case vmIntrinsics::_getCallerClass:

     if (!UseNewReflection)  return NULL;

     if (!InlineReflectionGetCallerClass)  return NULL;

     if (!JDK_Version::is_gte_jdk14x_version())  return NULL;

     break;

   case vmIntrinsics::_bitCount_i:

   case vmIntrinsics::_bitCount_l:

     if (!UsePopCountInstruction)  return NULL;

     break;

  default:

     assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility");

     assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?");

     break;

   }

   // -XX:-InlineClassNatives disables natives from the Class class.

   // The flag applies to all reflective calls, notably Array.newArray

   // (visible to Java programmers as Array.newInstance).

   if (m->holder()->name() == ciSymbol::java_lang_Class() ||

       m->holder()->name() == ciSymbol::java_lang_reflect_Array()) {

     if (!InlineClassNatives)  return NULL;

   }

   // -XX:-InlineThreadNatives disables natives from the Thread class.

   if (m->holder()->name() == ciSymbol::java_lang_Thread()) {

     if (!InlineThreadNatives)  return NULL;

   }

   // -XX:-InlineMathNatives disables natives from the Math,Float and Double classes.

   if (m->holder()->name() == ciSymbol::java_lang_Math() ||

       m->holder()->name() == ciSymbol::java_lang_Float() ||

       m->holder()->name() == ciSymbol::java_lang_Double()) {

     if (!InlineMathNatives)  return NULL;

   }

   // -XX:-InlineUnsafeOps disables natives from the Unsafe class.

   if (m->holder()->name() == ciSymbol::sun_misc_Unsafe()) {

     if (!InlineUnsafeOps)  return NULL;

   }

   return new LibraryIntrinsic(m, is_virtual, (vmIntrinsics::ID) id);

 }

 //----------------------register_library_intrinsics-----------------------

 // Initialize this file's data structures, for each Compile instance.

 void Compile::register_library_intrinsics() {

   // Nothing to do here.

 }

 JVMState* LibraryIntrinsic::generate(JVMState* jvms) {

   LibraryCallKit kit(jvms, this);

   Compile* C = kit.C;

   int nodes = C->unique();

 #ifndef PRODUCT

   if ((PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) && Verbose) {

     char buf[1000];

     const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf));

     tty->print_cr("Intrinsic %s", str);

   }

 #endif

   if (kit.try_to_inline()) {

     if (PrintIntrinsics || PrintInlining NOT_PRODUCT( || PrintOptoInlining) ) {

       tty->print("Inlining intrinsic %s%s at bci:%d in",

                  vmIntrinsics::name_at(intrinsic_id()),

                  (is_virtual() ? " (virtual)" : ""), kit.bci());

       kit.caller()->print_short_name(tty);

       tty->print_cr(" (%d bytes)", kit.caller()->code_size());

     }

     C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked);

     if (C->log()) {

       C->log()->elem("intrinsic id='%s'%s nodes='%d'",

                      vmIntrinsics::name_at(intrinsic_id()),

                      (is_virtual() ? " virtual='1'" : ""),

                      C->unique() - nodes);

     }

     return kit.transfer_exceptions_into_jvms();

   }

   if (PrintIntrinsics) {

     tty->print("Did not inline intrinsic %s%s at bci:%d in",

                vmIntrinsics::name_at(intrinsic_id()),

                (is_virtual() ? " (virtual)" : ""), kit.bci());

     kit.caller()->print_short_name(tty);

     tty->print_cr(" (%d bytes)", kit.caller()->code_size());

   }

   C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed);

   return NULL;

 }

 bool LibraryCallKit::try_to_inline() {

   // Handle symbolic names for otherwise undistinguished boolean switches:

   const bool is_store       = true;

   const bool is_native_ptr  = true;

   const bool is_static      = true;

   switch (intrinsic_id()) {

   case vmIntrinsics::_hashCode:

     return inline_native_hashcode(intrinsic()->is_virtual(), !is_static);

   case vmIntrinsics::_identityHashCode:

     return inline_native_hashcode(/*!virtual*/ false, is_static);

   case vmIntrinsics::_getClass:

     return inline_native_getClass();

   case vmIntrinsics::_dsin:

   case vmIntrinsics::_dcos:

   case vmIntrinsics::_dtan:

   case vmIntrinsics::_dabs:

   case vmIntrinsics::_datan2:

   case vmIntrinsics::_dsqrt:

   case vmIntrinsics::_dexp:

   case vmIntrinsics::_dlog:

   case vmIntrinsics::_dlog10:

   case vmIntrinsics::_dpow:

     return inline_math_native(intrinsic_id());

   case vmIntrinsics::_min:

   case vmIntrinsics::_max:

     return inline_min_max(intrinsic_id());

   case vmIntrinsics::_arraycopy:

     return inline_arraycopy();

   case vmIntrinsics::_compareTo:

     return inline_string_compareTo();

   case vmIntrinsics::_indexOf:

     return inline_string_indexOf();

   case vmIntrinsics::_equals:

     return inline_string_equals();

   case vmIntrinsics::_getObject:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, false);

   case vmIntrinsics::_getBoolean:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, false);

   case vmIntrinsics::_getByte:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, false);

   case vmIntrinsics::_getShort:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, false);

   case vmIntrinsics::_getChar:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, false);

   case vmIntrinsics::_getInt:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, false);

   case vmIntrinsics::_getLong:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, false);

   case vmIntrinsics::_getFloat:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, false);

   case vmIntrinsics::_getDouble:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, false);

   case vmIntrinsics::_putObject:

     return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, false);

   case vmIntrinsics::_putBoolean:

     return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, false);

   case vmIntrinsics::_putByte:

     return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, false);

   case vmIntrinsics::_putShort:

     return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, false);

   case vmIntrinsics::_putChar:

     return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, false);

   case vmIntrinsics::_putInt:

     return inline_unsafe_access(!is_native_ptr, is_store, T_INT, false);

   case vmIntrinsics::_putLong:

     return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, false);

   case vmIntrinsics::_putFloat:

     return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, false);

   case vmIntrinsics::_putDouble:

     return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, false);

   case vmIntrinsics::_getByte_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_BYTE, false);

   case vmIntrinsics::_getShort_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_SHORT, false);

   case vmIntrinsics::_getChar_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_CHAR, false);

   case vmIntrinsics::_getInt_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_INT, false);

   case vmIntrinsics::_getLong_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_LONG, false);

   case vmIntrinsics::_getFloat_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_FLOAT, false);

   case vmIntrinsics::_getDouble_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_DOUBLE, false);

   case vmIntrinsics::_getAddress_raw:

     return inline_unsafe_access(is_native_ptr, !is_store, T_ADDRESS, false);

   case vmIntrinsics::_putByte_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_BYTE, false);

   case vmIntrinsics::_putShort_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_SHORT, false);

   case vmIntrinsics::_putChar_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_CHAR, false);

   case vmIntrinsics::_putInt_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_INT, false);

   case vmIntrinsics::_putLong_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_LONG, false);

   case vmIntrinsics::_putFloat_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_FLOAT, false);

   case vmIntrinsics::_putDouble_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_DOUBLE, false);

   case vmIntrinsics::_putAddress_raw:

     return inline_unsafe_access(is_native_ptr, is_store, T_ADDRESS, false);

   case vmIntrinsics::_getObjectVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_OBJECT, true);

   case vmIntrinsics::_getBooleanVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_BOOLEAN, true);

   case vmIntrinsics::_getByteVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_BYTE, true);

   case vmIntrinsics::_getShortVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_SHORT, true);

   case vmIntrinsics::_getCharVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_CHAR, true);

   case vmIntrinsics::_getIntVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_INT, true);

   case vmIntrinsics::_getLongVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_LONG, true);

   case vmIntrinsics::_getFloatVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_FLOAT, true);

   case vmIntrinsics::_getDoubleVolatile:

     return inline_unsafe_access(!is_native_ptr, !is_store, T_DOUBLE, true);

   case vmIntrinsics::_putObjectVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_OBJECT, true);

   case vmIntrinsics::_putBooleanVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_BOOLEAN, true);

   case vmIntrinsics::_putByteVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_BYTE, true);

   case vmIntrinsics::_putShortVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_SHORT, true);

   case vmIntrinsics::_putCharVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_CHAR, true);

   case vmIntrinsics::_putIntVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_INT, true);

   case vmIntrinsics::_putLongVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_LONG, true);

   case vmIntrinsics::_putFloatVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_FLOAT, true);

   case vmIntrinsics::_putDoubleVolatile:

     return inline_unsafe_access(!is_native_ptr, is_store, T_DOUBLE, true);

   case vmIntrinsics::_prefetchRead:

     return inline_unsafe_prefetch(!is_native_ptr, !is_store, !is_static);

   case vmIntrinsics::_prefetchWrite:

     return inline_unsafe_prefetch(!is_native_ptr, is_store, !is_static);

   case vmIntrinsics::_prefetchReadStatic:

     return inline_unsafe_prefetch(!is_native_ptr, !is_store, is_static);

   case vmIntrinsics::_prefetchWriteStatic:

     return inline_unsafe_prefetch(!is_native_ptr, is_store, is_static);

   case vmIntrinsics::_compareAndSwapObject:

     return inline_unsafe_CAS(T_OBJECT);

   case vmIntrinsics::_compareAndSwapInt:

     return inline_unsafe_CAS(T_INT);

   case vmIntrinsics::_compareAndSwapLong:

     return inline_unsafe_CAS(T_LONG);

   case vmIntrinsics::_putOrderedObject:

     return inline_unsafe_ordered_store(T_OBJECT);

   case vmIntrinsics::_putOrderedInt:

     return inline_unsafe_ordered_store(T_INT);

   case vmIntrinsics::_putOrderedLong:

     return inline_unsafe_ordered_store(T_LONG);

   case vmIntrinsics::_currentThread:

     return inline_native_currentThread();

   case vmIntrinsics::_isInterrupted:

     return inline_native_isInterrupted();

   case vmIntrinsics::_currentTimeMillis:

     return inline_native_time_funcs(false);

   case vmIntrinsics::_nanoTime:

     return inline_native_time_funcs(true);

   case vmIntrinsics::_allocateInstance:

     return inline_unsafe_allocate();

   case vmIntrinsics::_copyMemory:

     return inline_unsafe_copyMemory();

   case vmIntrinsics::_newArray:

     return inline_native_newArray();

   case vmIntrinsics::_getLength:

     return inline_native_getLength();

   case vmIntrinsics::_copyOf:

     return inline_array_copyOf(false);

   case vmIntrinsics::_copyOfRange:

     return inline_array_copyOf(true);

   case vmIntrinsics::_equalsC:

     return inline_array_equals();

   case vmIntrinsics::_clone:

     return inline_native_clone(intrinsic()->is_virtual());

   case vmIntrinsics::_isAssignableFrom:

     return inline_native_subtype_check();

   case vmIntrinsics::_isInstance:

   case vmIntrinsics::_getModifiers:

   case vmIntrinsics::_isInterface:

   case vmIntrinsics::_isArray:

   case vmIntrinsics::_isPrimitive:

   case vmIntrinsics::_getSuperclass:

   case vmIntrinsics::_getComponentType:

   case vmIntrinsics::_getClassAccessFlags:

     return inline_native_Class_query(intrinsic_id());

   case vmIntrinsics::_floatToRawIntBits:

   case vmIntrinsics::_floatToIntBits:

   case vmIntrinsics::_intBitsToFloat:

   case vmIntrinsics::_doubleToRawLongBits:

   case vmIntrinsics::_doubleToLongBits:

   case vmIntrinsics::_longBitsToDouble:

     return inline_fp_conversions(intrinsic_id());

   case vmIntrinsics::_numberOfLeadingZeros_i:

   case vmIntrinsics::_numberOfLeadingZeros_l:

     return inline_numberOfLeadingZeros(intrinsic_id());

   case vmIntrinsics::_numberOfTrailingZeros_i:

   case vmIntrinsics::_numberOfTrailingZeros_l:

     return inline_numberOfTrailingZeros(intrinsic_id());

   case vmIntrinsics::_bitCount_i:

   case vmIntrinsics::_bitCount_l:

     return inline_bitCount(intrinsic_id());

   case vmIntrinsics::_reverseBytes_i:

   case vmIntrinsics::_reverseBytes_l:

   case vmIntrinsics::_reverseBytes_s:

   case vmIntrinsics::_reverseBytes_c:

     return inline_reverseBytes((vmIntrinsics::ID) intrinsic_id());

   case vmIntrinsics::_get_AtomicLong:

     return inline_native_AtomicLong_get();

   case vmIntrinsics::_attemptUpdate:

     return inline_native_AtomicLong_attemptUpdate();

   case vmIntrinsics::_getCallerClass:

     return inline_native_Reflection_getCallerClass();

   default:

     // If you get here, it may be that someone has added a new intrinsic

     // to the list in vmSymbols.hpp without implementing it here.

 #ifndef PRODUCT

     if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) {

       tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)",

                     vmIntrinsics::name_at(intrinsic_id()), intrinsic_id());

     }

 #endif

     return false;

   }

 }

 //------------------------------push_result------------------------------

 // Helper function for finishing intrinsics.

 void LibraryCallKit::push_result(RegionNode* region, PhiNode* value) {

   record_for_igvn(region);

   set_control(_gvn.transform(region));

   BasicType value_type = value->type()->basic_type();

   push_node(value_type, _gvn.transform(value));

 }

 //------------------------------generate_guard---------------------------

 // Helper function for generating guarded fast-slow graph structures.

 // The given 'test', if true, guards a slow path.  If the test fails

 // then a fast path can be taken.  (We generally hope it fails.)

 // In all cases, GraphKit::control() is updated to the fast path.

 // The returned value represents the control for the slow path.

 // The return value is never 'top'; it is either a valid control

 // or NULL if it is obvious that the slow path can never be taken.

 // Also, if region and the slow control are not NULL, the slow edge

 // is appended to the region.

 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) {

   if (stopped()) {

     // Already short circuited.

     return NULL;

   }

   // Build an if node and its projections.

   // If test is true we take the slow path, which we assume is uncommon.

   if (_gvn.type(test) == TypeInt::ZERO) {

     // The slow branch is never taken.  No need to build this guard.

     return NULL;

   }

   IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN);

   Node* if_slow = _gvn.transform( new (C, 1) IfTrueNode(iff) );

   if (if_slow == top()) {

     // The slow branch is never taken.  No need to build this guard.

     return NULL;

   }

   if (region != NULL)

     region->add_req(if_slow);

   Node* if_fast = _gvn.transform( new (C, 1) IfFalseNode(iff) );

   set_control(if_fast);

   return if_slow;

 }

 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) {

   return generate_guard(test, region, PROB_UNLIKELY_MAG(3));

 }

 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) {

   return generate_guard(test, region, PROB_FAIR);

 }

 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region,

                                                      Node* *pos_index) {

   if (stopped())

     return NULL;                // already stopped

   if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint]

     return NULL;                // index is already adequately typed

   Node* cmp_lt = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );

   Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );

   Node* is_neg = generate_guard(bol_lt, region, PROB_MIN);

   if (is_neg != NULL && pos_index != NULL) {

     // Emulate effect of Parse::adjust_map_after_if.

     Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS);

     ccast->set_req(0, control());

     (*pos_index) = _gvn.transform(ccast);

   }

   return is_neg;

 }

 inline Node* LibraryCallKit::generate_nonpositive_guard(Node* index, bool never_negative,

                                                         Node* *pos_index) {

   if (stopped())

     return NULL;                // already stopped

   if (_gvn.type(index)->higher_equal(TypeInt::POS1)) // [1,maxint]

     return NULL;                // index is already adequately typed

   Node* cmp_le = _gvn.transform( new (C, 3) CmpINode(index, intcon(0)) );

   BoolTest::mask le_or_eq = (never_negative ? BoolTest::eq : BoolTest::le);

   Node* bol_le = _gvn.transform( new (C, 2) BoolNode(cmp_le, le_or_eq) );

   Node* is_notp = generate_guard(bol_le, NULL, PROB_MIN);

   if (is_notp != NULL && pos_index != NULL) {

     // Emulate effect of Parse::adjust_map_after_if.

     Node* ccast = new (C, 2) CastIINode(index, TypeInt::POS1);

     ccast->set_req(0, control());

     (*pos_index) = _gvn.transform(ccast);

   }

   return is_notp;

 }

 // Make sure that 'position' is a valid limit index, in [0..length].

 // There are two equivalent plans for checking this:

 //   A. (offset + copyLength)  unsigned<=  arrayLength

 //   B. offset  <=  (arrayLength - copyLength)

 // We require that all of the values above, except for the sum and

 // difference, are already known to be non-negative.

 // Plan A is robust in the face of overflow, if offset and copyLength

 // are both hugely positive.

 //

 // Plan B is less direct and intuitive, but it does not overflow at

 // all, since the difference of two non-negatives is always

 // representable.  Whenever Java methods must perform the equivalent

 // check they generally use Plan B instead of Plan A.

 // For the moment we use Plan A.

 inline Node* LibraryCallKit::generate_limit_guard(Node* offset,

                                                   Node* subseq_length,

                                                   Node* array_length,

                                                   RegionNode* region) {

   if (stopped())

     return NULL;                // already stopped

   bool zero_offset = _gvn.type(offset) == TypeInt::ZERO;

   if (zero_offset && _gvn.eqv_uncast(subseq_length, array_length))

     return NULL;                // common case of whole-array copy

   Node* last = subseq_length;

   if (!zero_offset)             // last += offset

     last = _gvn.transform( new (C, 3) AddINode(last, offset));

   Node* cmp_lt = _gvn.transform( new (C, 3) CmpUNode(array_length, last) );

   Node* bol_lt = _gvn.transform( new (C, 2) BoolNode(cmp_lt, BoolTest::lt) );

   Node* is_over = generate_guard(bol_lt, region, PROB_MIN);

   return is_over;

 }

 //--------------------------generate_current_thread--------------------

 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) {

   ciKlass*    thread_klass = env()->Thread_klass();

   const Type* thread_type  = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull);

   Node* thread = _gvn.transform(new (C, 1) ThreadLocalNode());

   Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset()));

   Node* threadObj = make_load(NULL, p, thread_type, T_OBJECT);

   tls_output = thread;

   return threadObj;

 }

 //------------------------------make_string_method_node------------------------

 // Helper method for String intrinsic finctions.

 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* cnt1, Node* str2, Node* cnt2) {

   const int value_offset  = java_lang_String::value_offset_in_bytes();

   const int count_offset  = java_lang_String::count_offset_in_bytes();

   const int offset_offset = java_lang_String::offset_offset_in_bytes();

   Node* no_ctrl = NULL;

   ciInstanceKlass* klass = env()->String_klass();

   const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

   const TypeAryPtr* value_type =

         TypeAryPtr::make(TypePtr::NotNull,

                          TypeAry::make(TypeInt::CHAR,TypeInt::POS),

                          ciTypeArrayKlass::make(T_CHAR), true, 0);

   // Get start addr of string and substring

   Node* str1_valuea  = basic_plus_adr(str1, str1, value_offset);

   Node* str1_value   = make_load(no_ctrl, str1_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset));

   Node* str1_offseta = basic_plus_adr(str1, str1, offset_offset);

   Node* str1_offset  = make_load(no_ctrl, str1_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));

   Node* str1_start   = array_element_address(str1_value, str1_offset, T_CHAR);

   // Pin loads from String::equals() argument since it could be NULL.

   Node* str2_ctrl = (opcode == Op_StrEquals) ? control() : no_ctrl;

   Node* str2_valuea  = basic_plus_adr(str2, str2, value_offset);

   Node* str2_value   = make_load(str2_ctrl, str2_valuea, value_type, T_OBJECT, string_type->add_offset(value_offset));

   Node* str2_offseta = basic_plus_adr(str2, str2, offset_offset);

   Node* str2_offset  = make_load(str2_ctrl, str2_offseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));

   Node* str2_start   = array_element_address(str2_value, str2_offset, T_CHAR);

   Node* result = NULL;

   switch (opcode) {

   case Op_StrIndexOf:

     result = new (C, 6) StrIndexOfNode(control(), memory(TypeAryPtr::CHARS),

                                        str1_start, cnt1, str2_start, cnt2);

     break;

   case Op_StrComp:

     result = new (C, 6) StrCompNode(control(), memory(TypeAryPtr::CHARS),

                                     str1_start, cnt1, str2_start, cnt2);

     break;

   case Op_StrEquals:

     result = new (C, 5) StrEqualsNode(control(), memory(TypeAryPtr::CHARS),

                                       str1_start, str2_start, cnt1);

     break;

   default:

     ShouldNotReachHere();

     return NULL;

   }

   // All these intrinsics have checks.

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   return _gvn.transform(result);

 }

 //------------------------------inline_string_compareTo------------------------

 bool LibraryCallKit::inline_string_compareTo() {

   if (!Matcher::has_match_rule(Op_StrComp)) return false;

   const int value_offset = java_lang_String::value_offset_in_bytes();

   const int count_offset = java_lang_String::count_offset_in_bytes();

   const int offset_offset = java_lang_String::offset_offset_in_bytes();

   _sp += 2;

   Node *argument = pop();  // pop non-receiver first:  it was pushed second

   Node *receiver = pop();

   // Null check on self without removing any arguments.  The argument

   // null check technically happens in the wrong place, which can lead to

   // invalid stack traces when string compare is inlined into a method

   // which handles NullPointerExceptions.

   _sp += 2;

   receiver = do_null_check(receiver, T_OBJECT);

   argument = do_null_check(argument, T_OBJECT);

   _sp -= 2;

   if (stopped()) {

     return true;

   }

   ciInstanceKlass* klass = env()->String_klass();

   const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

   Node* no_ctrl = NULL;

   // Get counts for string and argument

   Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset);

   Node* receiver_cnt  = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

   Node* argument_cnta = basic_plus_adr(argument, argument, count_offset);

   Node* argument_cnt  = make_load(no_ctrl, argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

   Node* compare = make_string_method_node(Op_StrComp, receiver, receiver_cnt, argument, argument_cnt);

   push(compare);

   return true;

 }

 //------------------------------inline_string_equals------------------------

 bool LibraryCallKit::inline_string_equals() {

   if (!Matcher::has_match_rule(Op_StrEquals)) return false;

   const int value_offset = java_lang_String::value_offset_in_bytes();

   const int count_offset = java_lang_String::count_offset_in_bytes();

   const int offset_offset = java_lang_String::offset_offset_in_bytes();

   int nargs = 2;

   _sp += nargs;

   Node* argument = pop();  // pop non-receiver first:  it was pushed second

   Node* receiver = pop();

   // Null check on self without removing any arguments.  The argument

   // null check technically happens in the wrong place, which can lead to

   // invalid stack traces when string compare is inlined into a method

   // which handles NullPointerExceptions.

   _sp += nargs;

   receiver = do_null_check(receiver, T_OBJECT);

   //should not do null check for argument for String.equals(), because spec

   //allows to specify NULL as argument.

   _sp -= nargs;

   if (stopped()) {

     return true;

   }

   // paths (plus control) merge

   RegionNode* region = new (C, 5) RegionNode(5);

   Node* phi = new (C, 5) PhiNode(region, TypeInt::BOOL);

   // does source == target string?

   Node* cmp = _gvn.transform(new (C, 3) CmpPNode(receiver, argument));

   Node* bol = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::eq));

   Node* if_eq = generate_slow_guard(bol, NULL);

   if (if_eq != NULL) {

     // receiver == argument

     phi->init_req(2, intcon(1));

     region->init_req(2, if_eq);

   }

   // get String klass for instanceOf

   ciInstanceKlass* klass = env()->String_klass();

   if (!stopped()) {

     _sp += nargs;          // gen_instanceof might do an uncommon trap

     Node* inst = gen_instanceof(argument, makecon(TypeKlassPtr::make(klass)));

     _sp -= nargs;

     Node* cmp  = _gvn.transform(new (C, 3) CmpINode(inst, intcon(1)));

     Node* bol  = _gvn.transform(new (C, 2) BoolNode(cmp, BoolTest::ne));

     Node* inst_false = generate_guard(bol, NULL, PROB_MIN);

     //instanceOf == true, fallthrough

     if (inst_false != NULL) {

       phi->init_req(3, intcon(0));

       region->init_req(3, inst_false);

     }

   }

   const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

   Node* no_ctrl = NULL;

   Node* receiver_cnt;

   Node* argument_cnt;

   if (!stopped()) {

     // Properly cast the argument to String

     argument = _gvn.transform(new (C, 2) CheckCastPPNode(control(), argument, string_type));

     // Get counts for string and argument

     Node* receiver_cnta = basic_plus_adr(receiver, receiver, count_offset);

     receiver_cnt  = make_load(no_ctrl, receiver_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

     // Pin load from argument string since it could be NULL.

     Node* argument_cnta = basic_plus_adr(argument, argument, count_offset);

     argument_cnt  = make_load(control(), argument_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

     // Check for receiver count != argument count

     Node* cmp = _gvn.transform( new(C, 3) CmpINode(receiver_cnt, argument_cnt) );

     Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::ne) );

     Node* if_ne = generate_slow_guard(bol, NULL);

     if (if_ne != NULL) {

       phi->init_req(4, intcon(0));

       region->init_req(4, if_ne);

     }

   }

   // Check for count == 0 is done by mach node StrEquals.

   if (!stopped()) {

     Node* equals = make_string_method_node(Op_StrEquals, receiver, receiver_cnt, argument, argument_cnt);

     phi->init_req(1, equals);

     region->init_req(1, control());

   }

   // post merge

   set_control(_gvn.transform(region));

   record_for_igvn(region);

   push(_gvn.transform(phi));

   return true;

 }

 //------------------------------inline_array_equals----------------------------

 bool LibraryCallKit::inline_array_equals() {

   if (!Matcher::has_match_rule(Op_AryEq)) return false;

   _sp += 2;

   Node *argument2 = pop();

   Node *argument1 = pop();

   Node* equals =

     _gvn.transform(new (C, 4) AryEqNode(control(), memory(TypeAryPtr::CHARS),

                                         argument1, argument2) );

   push(equals);

   return true;

 }

 // Java version of String.indexOf(constant string)

 // class StringDecl {

 //   StringDecl(char[] ca) {

 //     offset = 0;

 //     count = ca.length;

 //     value = ca;

 //   }

 //   int offset;

 //   int count;

 //   char[] value;

 // }

 //

 // static int string_indexOf_J(StringDecl string_object, char[] target_object,

 //                             int targetOffset, int cache_i, int md2) {

 //   int cache = cache_i;

 //   int sourceOffset = string_object.offset;

 //   int sourceCount = string_object.count;

 //   int targetCount = target_object.length;

 //

 //   int targetCountLess1 = targetCount - 1;

 //   int sourceEnd = sourceOffset + sourceCount - targetCountLess1;

 //

 //   char[] source = string_object.value;

 //   char[] target = target_object;

 //   int lastChar = target[targetCountLess1];

 //

 //  outer_loop:

 //   for (int i = sourceOffset; i < sourceEnd; ) {

 //     int src = source[i + targetCountLess1];

 //     if (src == lastChar) {

 //       // With random strings and a 4-character alphabet,

 //       // reverse matching at this point sets up 0.8% fewer

 //       // frames, but (paradoxically) makes 0.3% more probes.

 //       // Since those probes are nearer the lastChar probe,

 //       // there is may be a net D$ win with reverse matching.

 //       // But, reversing loop inhibits unroll of inner loop

 //       // for unknown reason.  So, does running outer loop from

 //       // (sourceOffset - targetCountLess1) to (sourceOffset + sourceCount)

 //       for (int j = 0; j < targetCountLess1; j++) {

 //         if (target[targetOffset + j] != source[i+j]) {

 //           if ((cache & (1 << source[i+j])) == 0) {

 //             if (md2 < j+1) {

 //               i += j+1;

 //               continue outer_loop;

 //             }

 //           }

 //           i += md2;

 //           continue outer_loop;

 //         }

 //       }

 //       return i - sourceOffset;

 //     }

 //     if ((cache & (1 << src)) == 0) {

 //       i += targetCountLess1;

 //     } // using "i += targetCount;" and an "else i++;" causes a jump to jump.

 //     i++;

 //   }

 //   return -1;

 // }

 //------------------------------string_indexOf------------------------

 Node* LibraryCallKit::string_indexOf(Node* string_object, ciTypeArray* target_array, jint targetOffset_i,

                                      jint cache_i, jint md2_i) {

   Node* no_ctrl  = NULL;

   float likely   = PROB_LIKELY(0.9);

   float unlikely = PROB_UNLIKELY(0.9);

   const int value_offset  = java_lang_String::value_offset_in_bytes();

   const int count_offset  = java_lang_String::count_offset_in_bytes();

   const int offset_offset = java_lang_String::offset_offset_in_bytes();

   ciInstanceKlass* klass = env()->String_klass();

   const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

   const TypeAryPtr*  source_type = TypeAryPtr::make(TypePtr::NotNull, TypeAry::make(TypeInt::CHAR,TypeInt::POS), ciTypeArrayKlass::make(T_CHAR), true, 0);

   Node* sourceOffseta = basic_plus_adr(string_object, string_object, offset_offset);

   Node* sourceOffset  = make_load(no_ctrl, sourceOffseta, TypeInt::INT, T_INT, string_type->add_offset(offset_offset));

   Node* sourceCounta  = basic_plus_adr(string_object, string_object, count_offset);

   Node* sourceCount   = make_load(no_ctrl, sourceCounta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

   Node* sourcea       = basic_plus_adr(string_object, string_object, value_offset);

   Node* source        = make_load(no_ctrl, sourcea, source_type, T_OBJECT, string_type->add_offset(value_offset));

   Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array)) );

   jint target_length = target_array->length();

   const TypeAry* target_array_type = TypeAry::make(TypeInt::CHAR, TypeInt::make(0, target_length, Type::WidenMin));

   const TypeAryPtr* target_type = TypeAryPtr::make(TypePtr::BotPTR, target_array_type, target_array->klass(), true, Type::OffsetBot);

   IdealKit kit(gvn(), control(), merged_memory(), false, true);

 #define __ kit.

   Node* zero             = __ ConI(0);

   Node* one              = __ ConI(1);

   Node* cache            = __ ConI(cache_i);

   Node* md2              = __ ConI(md2_i);

   Node* lastChar         = __ ConI(target_array->char_at(target_length - 1));

   Node* targetCount      = __ ConI(target_length);

   Node* targetCountLess1 = __ ConI(target_length - 1);

   Node* targetOffset     = __ ConI(targetOffset_i);

   Node* sourceEnd        = __ SubI(__ AddI(sourceOffset, sourceCount), targetCountLess1);

   IdealVariable rtn(kit), i(kit), j(kit); __ declarations_done();

   Node* outer_loop = __ make_label(2 /* goto */);

   Node* return_    = __ make_label(1);

   __ set(rtn,__ ConI(-1));

   __ loop(i, sourceOffset, BoolTest::lt, sourceEnd); {

        Node* i2  = __ AddI(__ value(i), targetCountLess1);

        // pin to prohibit loading of "next iteration" value which may SEGV (rare)

        Node* src = load_array_element(__ ctrl(), source, i2, TypeAryPtr::CHARS);

        __ if_then(src, BoolTest::eq, lastChar, unlikely); {

          __ loop(j, zero, BoolTest::lt, targetCountLess1); {

               Node* tpj = __ AddI(targetOffset, __ value(j));

               Node* targ = load_array_element(no_ctrl, target, tpj, target_type);

               Node* ipj  = __ AddI(__ value(i), __ value(j));

               Node* src2 = load_array_element(no_ctrl, source, ipj, TypeAryPtr::CHARS);

               __ if_then(targ, BoolTest::ne, src2); {

                 __ if_then(__ AndI(cache, __ LShiftI(one, src2)), BoolTest::eq, zero); {

                   __ if_then(md2, BoolTest::lt, __ AddI(__ value(j), one)); {

                     __ increment(i, __ AddI(__ value(j), one));

                     __ goto_(outer_loop);

                   } __ end_if(); __ dead(j);

                 }__ end_if(); __ dead(j);

                 __ increment(i, md2);

                 __ goto_(outer_loop);

               }__ end_if();

               __ increment(j, one);

          }__ end_loop(); __ dead(j);

          __ set(rtn, __ SubI(__ value(i), sourceOffset)); __ dead(i);

          __ goto_(return_);

        }__ end_if();

        __ if_then(__ AndI(cache, __ LShiftI(one, src)), BoolTest::eq, zero, likely); {

          __ increment(i, targetCountLess1);

        }__ end_if();

        __ increment(i, one);

        __ bind(outer_loop);

   }__ end_loop(); __ dead(i);

   __ bind(return_);

   // Final sync IdealKit and GraphKit.

   sync_kit(kit);

   Node* result = __ value(rtn);

 #undef __

   C->set_has_loops(true);

   return result;

 }

 //------------------------------inline_string_indexOf------------------------

 bool LibraryCallKit::inline_string_indexOf() {

   const int value_offset  = java_lang_String::value_offset_in_bytes();

   const int count_offset  = java_lang_String::count_offset_in_bytes();

   const int offset_offset = java_lang_String::offset_offset_in_bytes();

   _sp += 2;

   Node *argument = pop();  // pop non-receiver first:  it was pushed second

   Node *receiver = pop();

   Node* result;

   // Disable the use of pcmpestri until it can be guaranteed that

   // the load doesn't cross into the uncommited space.

   if (false && Matcher::has_match_rule(Op_StrIndexOf) &&

       UseSSE42Intrinsics) {

     // Generate SSE4.2 version of indexOf

     // We currently only have match rules that use SSE4.2

     // Null check on self without removing any arguments.  The argument

     // null check technically happens in the wrong place, which can lead to

     // invalid stack traces when string compare is inlined into a method

     // which handles NullPointerExceptions.

     _sp += 2;

     receiver = do_null_check(receiver, T_OBJECT);

     argument = do_null_check(argument, T_OBJECT);

     _sp -= 2;

     if (stopped()) {

       return true;

     }

     // Make the merge point

     RegionNode* result_rgn = new (C, 3) RegionNode(3);

     Node*       result_phi = new (C, 3) PhiNode(result_rgn, TypeInt::INT);

     Node* no_ctrl  = NULL;

     ciInstanceKlass* klass = env()->String_klass();

     const TypeOopPtr* string_type = TypeOopPtr::make_from_klass(klass);

     // Get counts for string and substr

     Node* source_cnta = basic_plus_adr(receiver, receiver, count_offset);

     Node* source_cnt  = make_load(no_ctrl, source_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

     Node* substr_cnta = basic_plus_adr(argument, argument, count_offset);

     Node* substr_cnt  = make_load(no_ctrl, substr_cnta, TypeInt::INT, T_INT, string_type->add_offset(count_offset));

     // Check for substr count > string count

     Node* cmp = _gvn.transform( new(C, 3) CmpINode(substr_cnt, source_cnt) );

     Node* bol = _gvn.transform( new(C, 2) BoolNode(cmp, BoolTest::gt) );

     Node* if_gt = generate_slow_guard(bol, NULL);

     if (if_gt != NULL) {

       result_phi->init_req(2, intcon(-1));

       result_rgn->init_req(2, if_gt);

     }

     if (!stopped()) {

       result = make_string_method_node(Op_StrIndexOf, receiver, source_cnt, argument, substr_cnt);

       result_phi->init_req(1, result);

       result_rgn->init_req(1, control());

     }

     set_control(_gvn.transform(result_rgn));

     record_for_igvn(result_rgn);

     result = _gvn.transform(result_phi);

   } else { //Use LibraryCallKit::string_indexOf

     // don't intrinsify is argument isn't a constant string.

     if (!argument->is_Con()) {

      return false;

     }

     const TypeOopPtr* str_type = _gvn.type(argument)->isa_oopptr();

     if (str_type == NULL) {

       return false;

     }

     ciInstanceKlass* klass = env()->String_klass();

     ciObject* str_const = str_type->const_oop();

     if (str_const == NULL || str_const->klass() != klass) {

       return false;

     }

     ciInstance* str = str_const->as_instance();

     assert(str != NULL, "must be instance");

     ciObject* v = str->field_value_by_offset(value_offset).as_object();

     int       o = str->field_value_by_offset(offset_offset).as_int();

     int       c = str->field_value_by_offset(count_offset).as_int();

     ciTypeArray* pat = v->as_type_array(); // pattern (argument) character array

     // constant strings have no offset and count == length which

     // simplifies the resulting code somewhat so lets optimize for that.

     if (o != 0 || c != pat->length()) {

      return false;

     }

     // Null check on self without removing any arguments.  The argument

     // null check technically happens in the wrong place, which can lead to

     // invalid stack traces when string compare is inlined into a method

     // which handles NullPointerExceptions.

     _sp += 2;

     receiver = do_null_check(receiver, T_OBJECT);

     // No null check on the argument is needed since it's a constant String oop.

     _sp -= 2;

     if (stopped()) {

      return true;

     }

     // The null string as a pattern always returns 0 (match at beginning of string)

     if (c == 0) {

       push(intcon(0));

       return true;

     }

     // Generate default indexOf

     jchar lastChar = pat->char_at(o + (c - 1));

     int cache = 0;

     int i;

     for (i = 0; i < c - 1; i++) {

       assert(i < pat->length(), "out of range");

       cache |= (1 << (pat->char_at(o + i) & (sizeof(cache) * BitsPerByte - 1)));

     }

     int md2 = c;

     for (i = 0; i < c - 1; i++) {

       assert(i < pat->length(), "out of range");

       if (pat->char_at(o + i) == lastChar) {

         md2 = (c - 1) - i;

       }

     }

     result = string_indexOf(receiver, pat, o, cache, md2);

   }

   push(result);

   return true;

 }

 //--------------------------pop_math_arg--------------------------------

 // Pop a double argument to a math function from the stack

 // rounding it if necessary.

 Node * LibraryCallKit::pop_math_arg() {

   Node *arg = pop_pair();

   if( Matcher::strict_fp_requires_explicit_rounding && UseSSE<=1 )

     arg = _gvn.transform( new (C, 2) RoundDoubleNode(0, arg) );

   return arg;

 }

 //------------------------------inline_trig----------------------------------

 // Inline sin/cos/tan instructions, if possible.  If rounding is required, do

 // argument reduction which will turn into a fast/slow diamond.

 bool LibraryCallKit::inline_trig(vmIntrinsics::ID id) {

   _sp += arg_size();            // restore stack pointer

   Node* arg = pop_math_arg();

   Node* trig = NULL;

   switch (id) {

   case vmIntrinsics::_dsin:

     trig = _gvn.transform((Node*)new (C, 2) SinDNode(arg));

     break;

   case vmIntrinsics::_dcos:

     trig = _gvn.transform((Node*)new (C, 2) CosDNode(arg));

     break;

   case vmIntrinsics::_dtan:

     trig = _gvn.transform((Node*)new (C, 2) TanDNode(arg));

     break;

   default:

     assert(false, "bad intrinsic was passed in");

     return false;

   }

   // Rounding required?  Check for argument reduction!

   if( Matcher::strict_fp_requires_explicit_rounding ) {

     static const double     pi_4 =  0.7853981633974483;

     static const double neg_pi_4 = -0.7853981633974483;

     // pi/2 in 80-bit extended precision

     // static const unsigned char pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0x3f,0x00,0x00,0x00,0x00,0x00,0x00};

     // -pi/2 in 80-bit extended precision

     // static const unsigned char neg_pi_2_bits_x[] = {0x35,0xc2,0x68,0x21,0xa2,0xda,0x0f,0xc9,0xff,0xbf,0x00,0x00,0x00,0x00,0x00,0x00};

     // Cutoff value for using this argument reduction technique

     //static const double    pi_2_minus_epsilon =  1.564660403643354;

     //static const double neg_pi_2_plus_epsilon = -1.564660403643354;

     // Pseudocode for sin:

     // if (x <= Math.PI / 4.0) {

     //   if (x >= -Math.PI / 4.0) return  fsin(x);

     //   if (x >= -Math.PI / 2.0) return -fcos(x + Math.PI / 2.0);

     // } else {

     //   if (x <=  Math.PI / 2.0) return  fcos(x - Math.PI / 2.0);

     // }

     // return StrictMath.sin(x);

     // Pseudocode for cos:

     // if (x <= Math.PI / 4.0) {

     //   if (x >= -Math.PI / 4.0) return  fcos(x);

     //   if (x >= -Math.PI / 2.0) return  fsin(x + Math.PI / 2.0);

     // } else {

     //   if (x <=  Math.PI / 2.0) return -fsin(x - Math.PI / 2.0);

     // }

     // return StrictMath.cos(x);

     // Actually, sticking in an 80-bit Intel value into C2 will be tough; it

     // requires a special machine instruction to load it.  Instead we'll try

     // the 'easy' case.  If we really need the extra range +/- PI/2 we'll

     // probably do the math inside the SIN encoding.

     // Make the merge point

     RegionNode *r = new (C, 3) RegionNode(3);

     Node *phi = new (C, 3) PhiNode(r,Type::DOUBLE);

     // Flatten arg so we need only 1 test

     Node *abs = _gvn.transform(new (C, 2) AbsDNode(arg));

     // Node for PI/4 constant

     Node *pi4 = makecon(TypeD::make(pi_4));

     // Check PI/4 : abs(arg)

     Node *cmp = _gvn.transform(new (C, 3) CmpDNode(pi4,abs));

     // Check: If PI/4 < abs(arg) then go slow

     Node *bol = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::lt ) );

     // Branch either way

     IfNode *iff = create_and_xform_if(control(),bol, PROB_STATIC_FREQUENT, COUNT_UNKNOWN);

     set_control(opt_iff(r,iff));

     // Set fast path result

     phi->init_req(2,trig);

     // Slow path - non-blocking leaf call

     Node* call = NULL;

     switch (id) {

     case vmIntrinsics::_dsin:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dsin),

                                "Sin", NULL, arg, top());

       break;

     case vmIntrinsics::_dcos:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dcos),

                                "Cos", NULL, arg, top());

       break;

     case vmIntrinsics::_dtan:

       call = make_runtime_call(RC_LEAF, OptoRuntime::Math_D_D_Type(),

                                CAST_FROM_FN_PTR(address, SharedRuntime::dtan),

                                "Tan", NULL, arg, top());

       break;

     }

     assert(control()->in(0) == call, "");

     Node* slow_result = _gvn.transform(new (C, 1) ProjNode(call,TypeFunc::Parms));

     r->init_req(1,control());

     phi->init_req(1,slow_result);

     // Post-merge

     set_control(_gvn.transform(r));

     record_for_igvn(r);

     trig = _gvn.transform(phi);

     C->set_has_split_ifs(true); // Has chance for split-if optimization

   }

   // Push result back on JVM stack

   push_pair(trig);

   return true;

 }

 //------------------------------inline_sqrt-------------------------------------

 // Inline square root instruction, if possible.

 bool LibraryCallKit::inline_sqrt(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_dsqrt, "Not square root");

   _sp += arg_size();        // restore stack pointer

   push_pair(_gvn.transform(new (C, 2) SqrtDNode(0, pop_math_arg())));

   return true;

 }

 //------------------------------inline_abs-------------------------------------

 // Inline absolute value instruction, if possible.

 bool LibraryCallKit::inline_abs(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_dabs, "Not absolute value");

   _sp += arg_size();        // restore stack pointer

   push_pair(_gvn.transform(new (C, 2) AbsDNode(pop_math_arg())));

   return true;

 }

 //------------------------------inline_exp-------------------------------------

 // Inline exp instructions, if possible.  The Intel hardware only misses

 // really odd corner cases (+/- Infinity).  Just uncommon-trap them.

 bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_dexp, "Not exp");

   // If this inlining ever returned NaN in the past, we do not intrinsify it

   // every again.  NaN results requires StrictMath.exp handling.

   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;

   // Do not intrinsify on older platforms which lack cmove.

   if (ConditionalMoveLimit == 0)  return false;

   _sp += arg_size();        // restore stack pointer

   Node *x = pop_math_arg();

   Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x));

   //-------------------

   //result=(result.isNaN())? StrictMath::exp():result;

   // Check: If isNaN() by checking result!=result? then go to Strict Math

   Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));

   // Build the boolean node

   Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );

   { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);

     // End the current control-flow path

     push_pair(x);

     // Math.exp intrinsic returned a NaN, which requires StrictMath.exp

     // to handle.  Recompile without intrinsifying Math.exp

     uncommon_trap(Deoptimization::Reason_intrinsic,

                   Deoptimization::Action_make_not_entrant);

   }

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   push_pair(result);

   return true;

 }

 //------------------------------inline_pow-------------------------------------

 // Inline power instructions, if possible.

 bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_dpow, "Not pow");

   // If this inlining ever returned NaN in the past, we do not intrinsify it

   // every again.  NaN results requires StrictMath.pow handling.

   if (too_many_traps(Deoptimization::Reason_intrinsic))  return false;

   // Do not intrinsify on older platforms which lack cmove.

   if (ConditionalMoveLimit == 0)  return false;

   // Pseudocode for pow

   // if (x <= 0.0) {

   //   if ((double)((int)y)==y) { // if y is int

   //     result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y)

   //   } else {

   //     result = NaN;

   //   }

   // } else {

   //   result = DPow(x,y);

   // }

   // if (result != result)?  {

   //   uncommon_trap();

   // }

   // return result;

   _sp += arg_size();        // restore stack pointer

   Node* y = pop_math_arg();

   Node* x = pop_math_arg();

   Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) );

   // Short form: if not top-level (i.e., Math.pow but inlining Math.pow

   // inside of something) then skip the fancy tests and just check for

   // NaN result.

   Node *result = NULL;

   if( jvms()->depth() >= 1 ) {

     result = fast_result;

   } else {

     // Set the merge point for If node with condition of (x <= 0.0)

     // There are four possible paths to region node and phi node

     RegionNode *r = new (C, 4) RegionNode(4);

     Node *phi = new (C, 4) PhiNode(r, Type::DOUBLE);

     // Build the first if node: if (x <= 0.0)

     // Node for 0 constant

     Node *zeronode = makecon(TypeD::ZERO);

     // Check x:0

     Node *cmp = _gvn.transform(new (C, 3) CmpDNode(x, zeronode));

     // Check: If (x<=0) then go complex path

     Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) );

     // Branch either way

     IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);

     Node *opt_test = _gvn.transform(if1);

     //assert( opt_test->is_If(), "Expect an IfNode");

     IfNode *opt_if1 = (IfNode*)opt_test;

     // Fast path taken; set region slot 3

     Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) );

     r->init_req(3,fast_taken); // Capture fast-control

     // Fast path not-taken, i.e. slow path

     Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) );

     // Set fast path result

     Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) );

     phi->init_req(3, fast_result);

     // Complex path

     // Build the second if node (if y is int)

     // Node for (int)y

     Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y));

     // Node for (double)((int) y)

     Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty));

     // Check (double)((int) y) : y

     Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y));

     // Check if (y isn't int) then go to slow path

     Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) );

     // Branch either way

     IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN);

     Node *slow_path = opt_iff(r,if2); // Set region path 2

     // Calculate DPow(abs(x), y)*(1 & (int)y)

     // Node for constant 1

     Node *conone = intcon(1);

     // 1& (int)y

     Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) );

     // zero node

     Node *conzero = intcon(0);

     // Check (1&(int)y)==0?

     Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero));

     // Check if (1&(int)y)!=0?, if so the result is negative

     Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) );

     // abs(x)

     Node *absx=_gvn.transform( new (C, 2) AbsDNode(x));

     // abs(x)^y

     Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) );

     // -abs(x)^y

     Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy));

     // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y)

     Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE));

     // Set complex path fast result

     phi->init_req(2, signresult);

     static const jlong nan_bits = CONST64(0x7ff8000000000000);

     Node *slow_result = makecon(TypeD::make(*(double*)&nan_bits)); // return NaN

     r->init_req(1,slow_path);

     phi->init_req(1,slow_result);

     // Post merge

     set_control(_gvn.transform(r));

     record_for_igvn(r);

     result=_gvn.transform(phi);

   }

   //-------------------

   //result=(result.isNaN())? uncommon_trap():result;

   // Check: If isNaN() by checking result!=result? then go to Strict Math

   Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result));

   // Build the boolean node

   Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) );

   { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT);

     // End the current control-flow path

     push_pair(x);

     push_pair(y);

     // Math.pow intrinsic returned a NaN, which requires StrictMath.pow

     // to handle.  Recompile without intrinsifying Math.pow.

     uncommon_trap(Deoptimization::Reason_intrinsic,

                   Deoptimization::Action_make_not_entrant);

   }

   C->set_has_split_ifs(true); // Has chance for split-if optimization

   push_pair(result);

   return true;

 }

 //------------------------------inline_trans-------------------------------------

 // Inline transcendental instructions, if possible.  The Intel hardware gets

 // these right, no funny corner cases missed.

 bool LibraryCallKit::inline_trans(vmIntrinsics::ID id) {

   _sp += arg_size();        // restore stack pointer

   Node* arg = pop_math_arg();

   Node* trans = NULL;

   switch (id) {

   case vmIntrinsics::_dlog:

     trans = _gvn.transform((Node*)new (C, 2) LogDNode(arg));

     break;

   case vmIntrinsics::_dlog10:

     trans = _gvn.transform((Node*)new (C, 2) Log10DNode(arg));

     break;

   default:

     assert(false, "bad intrinsic was passed in");

     return false;

   }

   // Push result back on JVM stack

   push_pair(trans);

   return true;

 }

 //------------------------------runtime_math-----------------------------

 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) {

   Node* a = NULL;

   Node* b = NULL;

   assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(),

          "must be (DD)D or (D)D type");

   // Inputs

   _sp += arg_size();        // restore stack pointer

   if (call_type == OptoRuntime::Math_DD_D_Type()) {

     b = pop_math_arg();

   }

   a = pop_math_arg();

   const TypePtr* no_memory_effects = NULL;

   Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName,

                                  no_memory_effects,

                                  a, top(), b, b ? top() : NULL);

   Node* value = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+0));

 #ifdef ASSERT

   Node* value_top = _gvn.transform(new (C, 1) ProjNode(trig, TypeFunc::Parms+1));

   assert(value_top == top(), "second value must be top");

 #endif

   push_pair(value);

   return true;

 }

 //------------------------------inline_math_native-----------------------------

 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) {

   switch (id) {

     // These intrinsics are not properly supported on all hardware

   case vmIntrinsics::_dcos: return Matcher::has_match_rule(Op_CosD) ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dcos), "COS");

   case vmIntrinsics::_dsin: return Matcher::has_match_rule(Op_SinD) ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dsin), "SIN");

   case vmIntrinsics::_dtan: return Matcher::has_match_rule(Op_TanD) ? inline_trig(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dtan), "TAN");

   case vmIntrinsics::_dlog:   return Matcher::has_match_rule(Op_LogD) ? inline_trans(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog), "LOG");

   case vmIntrinsics::_dlog10: return Matcher::has_match_rule(Op_Log10D) ? inline_trans(id) :

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), "LOG10");

     // These intrinsics are supported on all hardware

   case vmIntrinsics::_dsqrt: return Matcher::has_match_rule(Op_SqrtD) ? inline_sqrt(id) : false;

   case vmIntrinsics::_dabs:  return Matcher::has_match_rule(Op_AbsD)  ? inline_abs(id)  : false;

     // These intrinsics don't work on X86.  The ad implementation doesn't

     // handle NaN's properly.  Instead of returning infinity, the ad

     // implementation returns a NaN on overflow. See bug: 6304089

     // Once the ad implementations are fixed, change the code below

     // to match the intrinsics above

   case vmIntrinsics::_dexp:  return

     runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP");

   case vmIntrinsics::_dpow:  return

     runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW");

    // These intrinsics are not yet correctly implemented

   case vmIntrinsics::_datan2:

     return false;

   default:

     ShouldNotReachHere();

     return false;

   }

 }

 static bool is_simple_name(Node* n) {

   return (n->req() == 1         // constant

           || (n->is_Type() && n->as_Type()->type()->singleton())

           || n->is_Proj()       // parameter or return value

           || n->is_Phi()        // local of some sort

           );

 }

 //----------------------------inline_min_max-----------------------------------

 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) {

   push(generate_min_max(id, argument(0), argument(1)));

   return true;

 }

 Node*

 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) {

   // These are the candidate return value:

   Node* xvalue = x0;

   Node* yvalue = y0;

   if (xvalue == yvalue) {

     return xvalue;

   }

   bool want_max = (id == vmIntrinsics::_max);

   const TypeInt* txvalue = _gvn.type(xvalue)->isa_int();

   const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int();

   if (txvalue == NULL || tyvalue == NULL)  return top();

   // This is not really necessary, but it is consistent with a

   // hypothetical MaxINode::Value method:

   int widen = MAX2(txvalue->_widen, tyvalue->_widen);

   // %%% This folding logic should (ideally) be in a different place.

   // Some should be inside IfNode, and there to be a more reliable

   // transformation of ?: style patterns into cmoves.  We also want

   // more powerful optimizations around cmove and min/max.

   // Try to find a dominating comparison of these guys.

   // It can simplify the index computation for Arrays.copyOf

   // and similar uses of System.arraycopy.

   // First, compute the normalized version of CmpI(x, y).

   int   cmp_op = Op_CmpI;

   Node* xkey = xvalue;

   Node* ykey = yvalue;

   Node* ideal_cmpxy = _gvn.transform( new(C, 3) CmpINode(xkey, ykey) );

   if (ideal_cmpxy->is_Cmp()) {

     // E.g., if we have CmpI(length - offset, count),

     // it might idealize to CmpI(length, count + offset)

     cmp_op = ideal_cmpxy->Opcode();

     xkey = ideal_cmpxy->in(1);

     ykey = ideal_cmpxy->in(2);

   }

   // Start by locating any relevant comparisons.

   Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey;

   Node* cmpxy = NULL;

   Node* cmpyx = NULL;

   for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) {

     Node* cmp = start_from->fast_out(k);

     if (cmp->outcnt() > 0 &&            // must have prior uses

         cmp->in(0) == NULL &&           // must be context-independent

         cmp->Opcode() == cmp_op) {      // right kind of compare

       if (cmp->in(1) == xkey && cmp->in(2) == ykey)  cmpxy = cmp;

       if (cmp->in(1) == ykey && cmp->in(2) == xkey)  cmpyx = cmp;

     }

   }

   const int NCMPS = 2;

   Node* cmps[NCMPS] = { cmpxy, cmpyx };

   int cmpn;

   for (cmpn = 0; cmpn < NCMPS; cmpn++) {

     if (cmps[cmpn] != NULL)  break;     // find a result

   }

   if (cmpn < NCMPS) {

     // Look for a dominating test that tells us the min and max.

     int depth = 0;                // Limit search depth for speed

     Node* dom = control();

     for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) {

       if (++depth >= 100)  break;

       Node* ifproj = dom;

       if (!ifproj->is_Proj())  continue;

       Node* iff = ifproj->in(0);

       if (!iff->is_If())  continue;

       Node* bol = iff->in(1);

       if (!bol->is_Bool())  continue;

       Node* cmp = bol->in(1);

       if (cmp == NULL)  continue;

       for (cmpn = 0; cmpn < NCMPS; cmpn++)

         if (cmps[cmpn] == cmp)  break;

       if (cmpn == NCMPS)  continue;

       BoolTest::mask btest = bol->as_Bool()->_test._test;

       if (ifproj->is_IfFalse())  btest = BoolTest(btest).negate();

       if (cmp->in(1) == ykey)    btest = BoolTest(btest).commute();

       // At this point, we know that 'x btest y' is true.

       switch (btest) {

       case BoolTest::eq:

         // They are proven equal, so we can collapse the min/max.

         // Either value is the answer.  Choose the simpler.

         if (is_simple_name(yvalue) && !is_simple_name(xvalue))

           return yvalue;

         return xvalue;

       case BoolTest::lt:          // x < y

       case BoolTest::le:          // x <= y

         return (want_max ? yvalue : xvalue);

       case BoolTest::gt:          // x > y

       case BoolTest::ge:          // x >= y

         return (want_max ? xvalue : yvalue);

       }

     }

   }

   // We failed to find a dominating test.

   // Let's pick a test that might GVN with prior tests.

   Node*          best_bol   = NULL;

   BoolTest::mask best_btest = BoolTest::illegal;

   for (cmpn = 0; cmpn < NCMPS; cmpn++) {

     Node* cmp = cmps[cmpn];

     if (cmp == NULL)  continue;

     for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) {

       Node* bol = cmp->fast_out(j);

       if (!bol->is_Bool())  continue;

       BoolTest::mask btest = bol->as_Bool()->_test._test;

       if (btest == BoolTest::eq || btest == BoolTest::ne)  continue;

       if (cmp->in(1) == ykey)   btest = BoolTest(btest).commute();

       if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) {

         best_bol   = bol->as_Bool();

         best_btest = btest;

       }

     }

   }

   Node* answer_if_true  = NULL;

   Node* answer_if_false = NULL;

   switch (best_btest) {

   default:

     if (cmpxy == NULL)

       cmpxy = ideal_cmpxy;

     best_bol = _gvn.transform( new(C, 2) BoolNode(cmpxy, BoolTest::lt) );

     // and fall through:

   case BoolTest::lt:          // x < y

   case BoolTest::le:          // x <= y

     answer_if_true  = (want_max ? yvalue : xvalue);

     answer_if_false = (want_max ? xvalue : yvalue);

     break;

   case BoolTest::gt:          // x > y

   case BoolTest::ge:          // x >= y

     answer_if_true  = (want_max ? xvalue : yvalue);

     answer_if_false = (want_max ? yvalue : xvalue);

     break;

   }

   jint hi, lo;

   if (want_max) {

     // We can sharpen the minimum.

     hi = MAX2(txvalue->_hi, tyvalue->_hi);

     lo = MAX2(txvalue->_lo, tyvalue->_lo);

   } else {

     // We can sharpen the maximum.

     hi = MIN2(txvalue->_hi, tyvalue->_hi);

     lo = MIN2(txvalue->_lo, tyvalue->_lo);

   }

   // Use a flow-free graph structure, to avoid creating excess control edges

   // which could hinder other optimizations.

   // Since Math.min/max is often used with arraycopy, we want

   // tightly_coupled_allocation to be able to see beyond min/max expressions.

   Node* cmov = CMoveNode::make(C, NULL, best_bol,

                                answer_if_false, answer_if_true,

                                TypeInt::make(lo, hi, widen));

   return _gvn.transform(cmov);

   /*

   // This is not as desirable as it may seem, since Min and Max

   // nodes do not have a full set of optimizations.

   // And they would interfere, anyway, with 'if' optimizations

   // and with CMoveI canonical forms.

   switch (id) {

   case vmIntrinsics::_min:

     result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break;

   case vmIntrinsics::_max:

     result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break;

   default:

     ShouldNotReachHere();

   }

   */

 }

 inline int

 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset) {

   const TypePtr* base_type = TypePtr::NULL_PTR;

   if (base != NULL)  base_type = _gvn.type(base)->isa_ptr();

   if (base_type == NULL) {

     // Unknown type.

     return Type::AnyPtr;

   } else if (base_type == TypePtr::NULL_PTR) {

     // Since this is a NULL+long form, we have to switch to a rawptr.

     base   = _gvn.transform( new (C, 2) CastX2PNode(offset) );

     offset = MakeConX(0);

     return Type::RawPtr;

   } else if (base_type->base() == Type::RawPtr) {

     return Type::RawPtr;

   } else if (base_type->isa_oopptr()) {

     // Base is never null => always a heap address.

     if (base_type->ptr() == TypePtr::NotNull) {

       return Type::OopPtr;

     }

     // Offset is small => always a heap address.

     const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t();

     if (offset_type != NULL &&

         base_type->offset() == 0 &&     // (should always be?)

         offset_type->_lo >= 0 &&

         !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) {

       return Type::OopPtr;

     }

     // Otherwise, it might either be oop+off or NULL+addr.

     return Type::AnyPtr;

   } else {

     // No information:

     return Type::AnyPtr;

   }

 }

 inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) {

   int kind = classify_unsafe_addr(base, offset);

   if (kind == Type::RawPtr) {

     return basic_plus_adr(top(), base, offset);

   } else {

     return basic_plus_adr(base, offset);

   }

 }

 //-------------------inline_numberOfLeadingZeros_int/long-----------------------

 // inline int Integer.numberOfLeadingZeros(int)

 // inline int Long.numberOfLeadingZeros(long)

 bool LibraryCallKit::inline_numberOfLeadingZeros(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_numberOfLeadingZeros_i || id == vmIntrinsics::_numberOfLeadingZeros_l, "not numberOfLeadingZeros");

   if (id == vmIntrinsics::_numberOfLeadingZeros_i && !Matcher::match_rule_supported(Op_CountLeadingZerosI)) return false;

   if (id == vmIntrinsics::_numberOfLeadingZeros_l && !Matcher::match_rule_supported(Op_CountLeadingZerosL)) return false;

   _sp += arg_size();  // restore stack pointer

   switch (id) {

   case vmIntrinsics::_numberOfLeadingZeros_i:

     push(_gvn.transform(new (C, 2) CountLeadingZerosINode(pop())));

     break;

   case vmIntrinsics::_numberOfLeadingZeros_l:

     push(_gvn.transform(new (C, 2) CountLeadingZerosLNode(pop_pair())));

     break;

   default:

     ShouldNotReachHere();

   }

   return true;

 }

 //-------------------inline_numberOfTrailingZeros_int/long----------------------

 // inline int Integer.numberOfTrailingZeros(int)

 // inline int Long.numberOfTrailingZeros(long)

 bool LibraryCallKit::inline_numberOfTrailingZeros(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_numberOfTrailingZeros_i || id == vmIntrinsics::_numberOfTrailingZeros_l, "not numberOfTrailingZeros");

   if (id == vmIntrinsics::_numberOfTrailingZeros_i && !Matcher::match_rule_supported(Op_CountTrailingZerosI)) return false;

   if (id == vmIntrinsics::_numberOfTrailingZeros_l && !Matcher::match_rule_supported(Op_CountTrailingZerosL)) return false;

   _sp += arg_size();  // restore stack pointer

   switch (id) {

   case vmIntrinsics::_numberOfTrailingZeros_i:

     push(_gvn.transform(new (C, 2) CountTrailingZerosINode(pop())));

     break;

   case vmIntrinsics::_numberOfTrailingZeros_l:

     push(_gvn.transform(new (C, 2) CountTrailingZerosLNode(pop_pair())));

     break;

   default:

     ShouldNotReachHere();

   }

   return true;

 }

 //----------------------------inline_bitCount_int/long-----------------------

 // inline int Integer.bitCount(int)

 // inline int Long.bitCount(long)

 bool LibraryCallKit::inline_bitCount(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_bitCount_i || id == vmIntrinsics::_bitCount_l, "not bitCount");

   if (id == vmIntrinsics::_bitCount_i && !Matcher::has_match_rule(Op_PopCountI)) return false;

   if (id == vmIntrinsics::_bitCount_l && !Matcher::has_match_rule(Op_PopCountL)) return false;

   _sp += arg_size();  // restore stack pointer

   switch (id) {

   case vmIntrinsics::_bitCount_i:

     push(_gvn.transform(new (C, 2) PopCountINode(pop())));

     break;

   case vmIntrinsics::_bitCount_l:

     push(_gvn.transform(new (C, 2) PopCountLNode(pop_pair())));

     break;

   default:

     ShouldNotReachHere();

   }

   return true;

 }

 //----------------------------inline_reverseBytes_int/long/char/short-------------------

 // inline Integer.reverseBytes(int)

 // inline Long.reverseBytes(long)

 // inline Character.reverseBytes(char)

 // inline Short.reverseBytes(short)

 bool LibraryCallKit::inline_reverseBytes(vmIntrinsics::ID id) {

   assert(id == vmIntrinsics::_reverseBytes_i || id == vmIntrinsics::_reverseBytes_l ||

          id == vmIntrinsics::_reverseBytes_c || id == vmIntrinsics::_reverseBytes_s,

          "not reverse Bytes");

   if (id == vmIntrinsics::_reverseBytes_i && !Matcher::has_match_rule(Op_ReverseBytesI))  return false;

   if (id == vmIntrinsics::_reverseBytes_l && !Matcher::has_match_rule(Op_ReverseBytesL))  return false;

   if (id == vmIntrinsics::_reverseBytes_c && !Matcher::has_match_rule(Op_ReverseBytesUS)) return false;

   if (id == vmIntrinsics::_reverseBytes_s && !Matcher::has_match_rule(Op_ReverseBytesS))  return false;

   _sp += arg_size();        // restore stack pointer

   switch (id) {

   case vmIntrinsics::_reverseBytes_i:

     push(_gvn.transform(new (C, 2) ReverseBytesINode(0, pop())));

     break;

   case vmIntrinsics::_reverseBytes_l:

     push_pair(_gvn.transform(new (C, 2) ReverseBytesLNode(0, pop_pair())));

     break;

   case vmIntrinsics::_reverseBytes_c:

     push(_gvn.transform(new (C, 2) ReverseBytesUSNode(0, pop())));

     break;

   case vmIntrinsics::_reverseBytes_s:

     push(_gvn.transform(new (C, 2) ReverseBytesSNode(0, pop())));

     break;

   default:

     ;

   }

   return true;

 }

 //----------------------------inline_unsafe_access----------------------------

 const static BasicType T_ADDRESS_HOLDER = T_LONG;

 // Interpret Unsafe.fieldOffset cookies correctly:

 extern jlong Unsafe_field_offset_to_byte_offset(jlong field_offset);

 bool LibraryCallKit::inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile) {

   if (callee()->is_static())  return false;  // caller must have the capability!

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = signature();

 #ifdef ASSERT

     if (!is_store) {

       // Object getObject(Object base, int/long offset), etc.

       BasicType rtype = sig->return_type()->basic_type();

       if (rtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::getAddress_name())

           rtype = T_ADDRESS;  // it is really a C void*

       assert(rtype == type, "getter must return the expected value");

       if (!is_native_ptr) {

         assert(sig->count() == 2, "oop getter has 2 arguments");

         assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object");

         assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct");

       } else {

         assert(sig->count() == 1, "native getter has 1 argument");

         assert(sig->type_at(0)->basic_type() == T_LONG, "getter base is long");

       }

     } else {

       // void putObject(Object base, int/long offset, Object x), etc.

       assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value");

       if (!is_native_ptr) {

         assert(sig->count() == 3, "oop putter has 3 arguments");

         assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object");

         assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct");

       } else {

         assert(sig->count() == 2, "native putter has 2 arguments");

         assert(sig->type_at(0)->basic_type() == T_LONG, "putter base is long");

       }

       BasicType vtype = sig->type_at(sig->count()-1)->basic_type();

       if (vtype == T_ADDRESS_HOLDER && callee()->name() == ciSymbol::putAddress_name())

         vtype = T_ADDRESS;  // it is really a C void*

       assert(vtype == type, "putter must accept the expected value");

     }

 #endif // ASSERT

  }

 #endif //PRODUCT

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   int type_words = type2size[ (type == T_ADDRESS) ? T_LONG : type ];

   // Argument words:  "this" plus (oop/offset) or (lo/hi) args plus maybe 1 or 2 value words

   int nargs = 1 + (is_native_ptr ? 2 : 3) + (is_store ? type_words : 0);

   debug_only(int saved_sp = _sp);

   _sp += nargs;

   Node* val;

   debug_only(val = (Node*)(uintptr_t)-1);

   if (is_store) {

     // Get the value being stored.  (Pop it first; it was pushed last.)

     switch (type) {

     case T_DOUBLE:

     case T_LONG:

     case T_ADDRESS:

       val = pop_pair();

       break;

     default:

       val = pop();

     }

   }

   // Build address expression.  See the code in inline_unsafe_prefetch.

   Node *adr;

   Node *heap_base_oop = top();

   if (!is_native_ptr) {

     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset

     Node* offset = pop_pair();

     // The base is either a Java object or a value produced by Unsafe.staticFieldBase

     Node* base   = pop();

     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset

     // to be plain byte offsets, which are also the same as those accepted

     // by oopDesc::field_base.

     assert(Unsafe_field_offset_to_byte_offset(11) == 11,

            "fieldOffset must be byte-scaled");

     // 32-bit machines ignore the high half!

     offset = ConvL2X(offset);

     adr = make_unsafe_address(base, offset);

     heap_base_oop = base;

   } else {

     Node* ptr = pop_pair();

     // Adjust Java long to machine word:

     ptr = ConvL2X(ptr);

     adr = make_unsafe_address(NULL, ptr);

   }

   // Pop receiver last:  it was pushed first.

   Node *receiver = pop();

   assert(saved_sp == _sp, "must have correct argument count");

   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();

   // First guess at the value type.

   const Type *value_type = Type::get_const_basic_type(type);

   // Try to categorize the address.  If it comes up as TypeJavaPtr::BOTTOM,

   // there was not enough information to nail it down.

   Compile::AliasType* alias_type = C->alias_type(adr_type);

   assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");

   // We will need memory barriers unless we can determine a unique

   // alias category for this reference.  (Note:  If for some reason

   // the barriers get omitted and the unsafe reference begins to "pollute"

   // the alias analysis of the rest of the graph, either Compile::can_alias

   // or Compile::must_alias will throw a diagnostic assert.)

   bool need_mem_bar = (alias_type->adr_type() == TypeOopPtr::BOTTOM);

   if (!is_store && type == T_OBJECT) {

     // Attempt to infer a sharper value type from the offset and base type.

     ciKlass* sharpened_klass = NULL;

     // See if it is an instance field, with an object type.

     if (alias_type->field() != NULL) {

       assert(!is_native_ptr, "native pointer op cannot use a java address");

       if (alias_type->field()->type()->is_klass()) {

         sharpened_klass = alias_type->field()->type()->as_klass();

       }

     }

     // See if it is a narrow oop array.

     if (adr_type->isa_aryptr()) {

       if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) {

         const TypeOopPtr *elem_type = adr_type->is_aryptr()->elem()->isa_oopptr();

         if (elem_type != NULL) {

           sharpened_klass = elem_type->klass();

         }

       }

     }

     if (sharpened_klass != NULL) {

       const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass);

       // Sharpen the value type.

       value_type = tjp;

 #ifndef PRODUCT

       if (PrintIntrinsics || PrintInlining || PrintOptoInlining) {

         tty->print("  from base type:  ");   adr_type->dump();

         tty->print("  sharpened value: "); value_type->dump();

       }

 #endif

     }

   }

   // Null check on self without removing any arguments.  The argument

   // null check technically happens in the wrong place, which can lead to

   // invalid stack traces when the primitive is inlined into a method

   // which handles NullPointerExceptions.

   _sp += nargs;

   do_null_check(receiver, T_OBJECT);

   _sp -= nargs;

   if (stopped()) {

     return true;

   }

   // Heap pointers get a null-check from the interpreter,

   // as a courtesy.  However, this is not guaranteed by Unsafe,

   // and it is not possible to fully distinguish unintended nulls

   // from intended ones in this API.

   if (is_volatile) {

     // We need to emit leading and trailing CPU membars (see below) in

     // addition to memory membars when is_volatile. This is a little

     // too strong, but avoids the need to insert per-alias-type

     // volatile membars (for stores; compare Parse::do_put_xxx), which

     // we cannot do effectively here because we probably only have a

     // rough approximation of type.

     need_mem_bar = true;

     // For Stores, place a memory ordering barrier now.

     if (is_store)

       insert_mem_bar(Op_MemBarRelease);

   }

   // Memory barrier to prevent normal and 'unsafe' accesses from

   // bypassing each other.  Happens after null checks, so the

   // exception paths do not take memory state from the memory barrier,

   // so there's no problems making a strong assert about mixing users

   // of safe & unsafe memory.  Otherwise fails in a CTW of rt.jar

   // around 5701, class sun/reflect/UnsafeBooleanFieldAccessorImpl.

   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);

   if (!is_store) {

     Node* p = make_load(control(), adr, value_type, type, adr_type, is_volatile);

     // load value and push onto stack

     switch (type) {

     case T_BOOLEAN:

     case T_CHAR:

     case T_BYTE:

     case T_SHORT:

     case T_INT:

     case T_FLOAT:

     case T_OBJECT:

       push( p );

       break;

     case T_ADDRESS:

       // Cast to an int type.

       p = _gvn.transform( new (C, 2) CastP2XNode(NULL,p) );

       p = ConvX2L(p);

       push_pair(p);

       break;

     case T_DOUBLE:

     case T_LONG:

       push_pair( p );

       break;

     default: ShouldNotReachHere();

     }

   } else {

     // place effect of store into memory

     switch (type) {

     case T_DOUBLE:

       val = dstore_rounding(val);

       break;

     case T_ADDRESS:

       // Repackage the long as a pointer.

       val = ConvL2X(val);

       val = _gvn.transform( new (C, 2) CastX2PNode(val) );

       break;

     }

     if (type != T_OBJECT ) {

       (void) store_to_memory(control(), adr, val, type, adr_type, is_volatile);

     } else {

       // Possibly an oop being stored to Java heap or native memory

       if (!TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop))) {

         // oop to Java heap.

         (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);

       } else {

         // We can't tell at compile time if we are storing in the Java heap or outside

         // of it. So we need to emit code to conditionally do the proper type of

         // store.

         IdealKit ideal(gvn(), control(),  merged_memory());

 #define __ ideal.

         // QQQ who knows what probability is here??

         __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); {

           // Sync IdealKit and graphKit.

           set_all_memory( __ merged_memory());

           set_control(__ ctrl());

           Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type);

           // Update IdealKit memory.

           __ set_all_memory(merged_memory());

           __ set_ctrl(control());

         } __ else_(); {

           __ store(__ ctrl(), adr, val, type, alias_type->index(), is_volatile);

         } __ end_if();

         // Final sync IdealKit and GraphKit.

         sync_kit(ideal);

 #undef __

       }

     }

   }

   if (is_volatile) {

     if (!is_store)

       insert_mem_bar(Op_MemBarAcquire);

     else

       insert_mem_bar(Op_MemBarVolatile);

   }

   if (need_mem_bar) insert_mem_bar(Op_MemBarCPUOrder);

   return true;

 }

 //----------------------------inline_unsafe_prefetch----------------------------

 bool LibraryCallKit::inline_unsafe_prefetch(bool is_native_ptr, bool is_store, bool is_static) {

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = signature();

 #ifdef ASSERT

     // Object getObject(Object base, int/long offset), etc.

     BasicType rtype = sig->return_type()->basic_type();

     if (!is_native_ptr) {

       assert(sig->count() == 2, "oop prefetch has 2 arguments");

       assert(sig->type_at(0)->basic_type() == T_OBJECT, "prefetch base is object");

       assert(sig->type_at(1)->basic_type() == T_LONG, "prefetcha offset is correct");

     } else {

       assert(sig->count() == 1, "native prefetch has 1 argument");

       assert(sig->type_at(0)->basic_type() == T_LONG, "prefetch base is long");

     }

 #endif // ASSERT

   }

 #endif // !PRODUCT

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   // Argument words:  "this" if not static, plus (oop/offset) or (lo/hi) args

   int nargs = (is_static ? 0 : 1) + (is_native_ptr ? 2 : 3);

   debug_only(int saved_sp = _sp);

   _sp += nargs;

   // Build address expression.  See the code in inline_unsafe_access.

   Node *adr;

   if (!is_native_ptr) {

     // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset

     Node* offset = pop_pair();

     // The base is either a Java object or a value produced by Unsafe.staticFieldBase

     Node* base   = pop();

     // We currently rely on the cookies produced by Unsafe.xxxFieldOffset

     // to be plain byte offsets, which are also the same as those accepted

     // by oopDesc::field_base.

     assert(Unsafe_field_offset_to_byte_offset(11) == 11,

            "fieldOffset must be byte-scaled");

     // 32-bit machines ignore the high half!

     offset = ConvL2X(offset);

     adr = make_unsafe_address(base, offset);

   } else {

     Node* ptr = pop_pair();

     // Adjust Java long to machine word:

     ptr = ConvL2X(ptr);

     adr = make_unsafe_address(NULL, ptr);

   }

   if (is_static) {

     assert(saved_sp == _sp, "must have correct argument count");

   } else {

     // Pop receiver last:  it was pushed first.

     Node *receiver = pop();

     assert(saved_sp == _sp, "must have correct argument count");

     // Null check on self without removing any arguments.  The argument

     // null check technically happens in the wrong place, which can lead to

     // invalid stack traces when the primitive is inlined into a method

     // which handles NullPointerExceptions.

     _sp += nargs;

     do_null_check(receiver, T_OBJECT);

     _sp -= nargs;

     if (stopped()) {

       return true;

     }

   }

   // Generate the read or write prefetch

   Node *prefetch;

   if (is_store) {

     prefetch = new (C, 3) PrefetchWriteNode(i_o(), adr);

   } else {

     prefetch = new (C, 3) PrefetchReadNode(i_o(), adr);

   }

   prefetch->init_req(0, control());

   set_i_o(_gvn.transform(prefetch));

   return true;

 }

 //----------------------------inline_unsafe_CAS----------------------------

 bool LibraryCallKit::inline_unsafe_CAS(BasicType type) {

   // This basic scheme here is the same as inline_unsafe_access, but

   // differs in enough details that combining them would make the code

   // overly confusing.  (This is a true fact! I originally combined

   // them, but even I was confused by it!) As much code/comments as

   // possible are retained from inline_unsafe_access though to make

   // the correspondences clearer. - dl

   if (callee()->is_static())  return false;  // caller must have the capability!

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = signature();

 #ifdef ASSERT

     BasicType rtype = sig->return_type()->basic_type();

     assert(rtype == T_BOOLEAN, "CAS must return boolean");

     assert(sig->count() == 4, "CAS has 4 arguments");

     assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object");

     assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long");

 #endif // ASSERT

   }

 #endif //PRODUCT

   // number of stack slots per value argument (1 or 2)

   int type_words = type2size[type];

   // Cannot inline wide CAS on machines that don't support it natively

   if (type2aelembytes(type) > BytesPerInt && !VM_Version::supports_cx8())

     return false;

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   // Argument words:  "this" plus oop plus offset plus oldvalue plus newvalue;

   int nargs = 1 + 1 + 2  + type_words + type_words;

   // pop arguments: newval, oldval, offset, base, and receiver

   debug_only(int saved_sp = _sp);

   _sp += nargs;

   Node* newval   = (type_words == 1) ? pop() : pop_pair();

   Node* oldval   = (type_words == 1) ? pop() : pop_pair();

   Node *offset   = pop_pair();

   Node *base     = pop();

   Node *receiver = pop();

   assert(saved_sp == _sp, "must have correct argument count");

   //  Null check receiver.

   _sp += nargs;

   do_null_check(receiver, T_OBJECT);

   _sp -= nargs;

   if (stopped()) {

     return true;

   }

   // Build field offset expression.

   // We currently rely on the cookies produced by Unsafe.xxxFieldOffset

   // to be plain byte offsets, which are also the same as those accepted

   // by oopDesc::field_base.

   assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");

   // 32-bit machines ignore the high half of long offsets

   offset = ConvL2X(offset);

   Node* adr = make_unsafe_address(base, offset);

   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();

   // (Unlike inline_unsafe_access, there seems no point in trying

   // to refine types. Just use the coarse types here.

   const Type *value_type = Type::get_const_basic_type(type);

   Compile::AliasType* alias_type = C->alias_type(adr_type);

   assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here");

   int alias_idx = C->get_alias_index(adr_type);

   // Memory-model-wise, a CAS acts like a little synchronized block,

   // so needs barriers on each side.  These don't translate into

   // actual barriers on most machines, but we still need rest of

   // compiler to respect ordering.

   insert_mem_bar(Op_MemBarRelease);

   insert_mem_bar(Op_MemBarCPUOrder);

   // 4984716: MemBars must be inserted before this

   //          memory node in order to avoid a false

   //          dependency which will confuse the scheduler.

   Node *mem = memory(alias_idx);

   // For now, we handle only those cases that actually exist: ints,

   // longs, and Object. Adding others should be straightforward.

   Node* cas;

   switch(type) {

   case T_INT:

     cas = _gvn.transform(new (C, 5) CompareAndSwapINode(control(), mem, adr, newval, oldval));

     break;

   case T_LONG:

     cas = _gvn.transform(new (C, 5) CompareAndSwapLNode(control(), mem, adr, newval, oldval));

     break;

   case T_OBJECT:

      // reference stores need a store barrier.

     // (They don't if CAS fails, but it isn't worth checking.)

     pre_barrier(control(), base, adr, alias_idx, newval, value_type->make_oopptr(), T_OBJECT);

 #ifdef _LP64

     if (adr->bottom_type()->is_ptr_to_narrowoop()) {

       Node *newval_enc = _gvn.transform(new (C, 2) EncodePNode(newval, newval->bottom_type()->make_narrowoop()));

       Node *oldval_enc = _gvn.transform(new (C, 2) EncodePNode(oldval, oldval->bottom_type()->make_narrowoop()));

       cas = _gvn.transform(new (C, 5) CompareAndSwapNNode(control(), mem, adr,

                                                           newval_enc, oldval_enc));

     } else

 #endif

     {

       cas = _gvn.transform(new (C, 5) CompareAndSwapPNode(control(), mem, adr, newval, oldval));

     }

     post_barrier(control(), cas, base, adr, alias_idx, newval, T_OBJECT, true);

     break;

   default:

     ShouldNotReachHere();

     break;

   }

   // SCMemProjNodes represent the memory state of CAS. Their main

   // role is to prevent CAS nodes from being optimized away when their

   // results aren't used.

   Node* proj = _gvn.transform( new (C, 1) SCMemProjNode(cas));

   set_memory(proj, alias_idx);

   // Add the trailing membar surrounding the access

   insert_mem_bar(Op_MemBarCPUOrder);

   insert_mem_bar(Op_MemBarAcquire);

   push(cas);

   return true;

 }

 bool LibraryCallKit::inline_unsafe_ordered_store(BasicType type) {

   // This is another variant of inline_unsafe_access, differing in

   // that it always issues store-store ("release") barrier and ensures

   // store-atomicity (which only matters for "long").

   if (callee()->is_static())  return false;  // caller must have the capability!

 #ifndef PRODUCT

   {

     ResourceMark rm;

     // Check the signatures.

     ciSignature* sig = signature();

 #ifdef ASSERT

     BasicType rtype = sig->return_type()->basic_type();

     assert(rtype == T_VOID, "must return void");

     assert(sig->count() == 3, "has 3 arguments");

     assert(sig->type_at(0)->basic_type() == T_OBJECT, "base is object");

     assert(sig->type_at(1)->basic_type() == T_LONG, "offset is long");

 #endif // ASSERT

   }

 #endif //PRODUCT

   // number of stack slots per value argument (1 or 2)

   int type_words = type2size[type];

   C->set_has_unsafe_access(true);  // Mark eventual nmethod as "unsafe".

   // Argument words:  "this" plus oop plus offset plus value;

   int nargs = 1 + 1 + 2 + type_words;

   // pop arguments: val, offset, base, and receiver

   debug_only(int saved_sp = _sp);

   _sp += nargs;

   Node* val      = (type_words == 1) ? pop() : pop_pair();

   Node *offset   = pop_pair();

   Node *base     = pop();

   Node *receiver = pop();

   assert(saved_sp == _sp, "must have correct argument count");

   //  Null check receiver.

   _sp += nargs;

   do_null_check(receiver, T_OBJECT);

   _sp -= nargs;

   if (stopped()) {

     return true;

   }

   // Build field offset expression.

   assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled");

   // 32-bit machines ignore the high half of long offsets

   offset = ConvL2X(offset);

   Node* adr = make_unsafe_address(base, offset);

   const TypePtr *adr_type = _gvn.type(adr)->isa_ptr();

   const Type *value_type = Type::get_const_basic_type(type);

   Compile::AliasType* alias_type = C->alias_type(adr_type);

   insert_mem_bar(Op_MemBarRelease);

   insert_mem_bar(Op_MemBarCPUOrder);

   // Ensure that the store is atomic for longs:

   bool require_atomic_access = true;

   Node* store;

   if (type == T_OBJECT) // reference stores need a store barrier.

     store = store_oop_to_unknown(control(), base, adr, adr_type, val, type);

   else {

     store = store_to_memory(control(), adr, val, type, adr_type, require_atomic_access);

   }

   insert_mem_bar(Op_MemBarCPUOrder);

   return true;

 }

 bool LibraryCallKit::inline_unsafe_allocate() {

   if (callee()->is_static())  return false;  // caller must have the capability!

   int nargs = 1 + 1;

   assert(signature()->size() == nargs-1, "alloc has 1 argument");

   null_check_receiver(callee());  // check then ignore argument(0)

   _sp += nargs;  // set original stack for use by uncommon_trap

   Node* cls = do_null_check(argument(1), T_OBJECT);

   _sp -= nargs;

   if (stopped())  return true;

   Node* kls = load_klass_from_mirror(cls, false, nargs, NULL, 0);

   _sp += nargs;  // set original stack for use by uncommon_trap

   kls = do_null_check(kls, T_OBJECT);

   _sp -= nargs;

   if (stopped())  return true;  // argument was like int.class

   // Note:  The argument might still be an illegal value like

   // Serializable.class or Object[].class.   The runtime will handle it.

   // But we must make an explicit check for initialization.

   Node* insp = basic_plus_adr(kls, instanceKlass::init_state_offset_in_bytes() + sizeof(oopDesc));

   Node* inst = make_load(NULL, insp, TypeInt::INT, T_INT);

   Node* bits = intcon(instanceKlass::fully_initialized);

   Node* test = _gvn.transform( new (C, 3) SubINode(inst, bits) );

   // The 'test' is non-zero if we need to take a slow path.

   Node* obj = new_instance(kls, test);