jdk8-mips64-public/hotspot: src/cpu/x86/vm/stubGenerator_x86

7087445: Improve platform independence of JSR292 shared code
Summary: changes necessary for some JSR292 ports
Reviewed-by: jrose, dholmes

     1 /*

     2  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    25 #include "precompiled.hpp"

    26 #include "asm/assembler.hpp"

    27 #include "assembler_x86.inline.hpp"

    28 #include "interpreter/interpreter.hpp"

    29 #include "nativeInst_x86.hpp"

    30 #include "oops/instanceOop.hpp"

    31 #include "oops/methodOop.hpp"

    32 #include "oops/objArrayKlass.hpp"

    33 #include "oops/oop.inline.hpp"

    34 #include "prims/methodHandles.hpp"

    35 #include "runtime/frame.inline.hpp"

    36 #include "runtime/handles.inline.hpp"

    37 #include "runtime/sharedRuntime.hpp"

    38 #include "runtime/stubCodeGenerator.hpp"

    39 #include "runtime/stubRoutines.hpp"

    40 #include "utilities/top.hpp"

    41 #ifdef TARGET_OS_FAMILY_linux

    42 # include "thread_linux.inline.hpp"

    43 #endif

    44 #ifdef TARGET_OS_FAMILY_solaris

    45 # include "thread_solaris.inline.hpp"

    46 #endif

    47 #ifdef TARGET_OS_FAMILY_windows

    48 # include "thread_windows.inline.hpp"

    49 #endif

    50 #ifdef COMPILER2

    51 #include "opto/runtime.hpp"

    52 #endif

    54 // Declaration and definition of StubGenerator (no .hpp file).

    55 // For a more detailed description of the stub routine structure

    56 // see the comment in stubRoutines.hpp

    58 #define __ _masm->

    59 #define a__ ((Assembler*)_masm)->

    61 #ifdef PRODUCT

    62 #define BLOCK_COMMENT(str) /* nothing */

    63 #else

    64 #define BLOCK_COMMENT(str) __ block_comment(str)

    65 #endif

    67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

    69 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions

    70 const int FPU_CNTRL_WRD_MASK = 0xFFFF;

    72 // -------------------------------------------------------------------------------------------------------------------------

    73 // Stub Code definitions

    75 static address handle_unsafe_access() {

    76   JavaThread* thread = JavaThread::current();

    77   address pc  = thread->saved_exception_pc();

    78   // pc is the instruction which we must emulate

    79   // doing a no-op is fine:  return garbage from the load

    80   // therefore, compute npc

    81   address npc = Assembler::locate_next_instruction(pc);

    83   // request an async exception

    84   thread->set_pending_unsafe_access_error();

    86   // return address of next instruction to execute

    87   return npc;

    88 }

    90 class StubGenerator: public StubCodeGenerator {

    91  private:

    93 #ifdef PRODUCT

    94 #define inc_counter_np(counter) (0)

    95 #else

    96   void inc_counter_np_(int& counter) {

    97     __ incrementl(ExternalAddress((address)&counter));

    98   }

    99 #define inc_counter_np(counter) \

   100   BLOCK_COMMENT("inc_counter " #counter); \

   101   inc_counter_np_(counter);

   102 #endif //PRODUCT

   104   void inc_copy_counter_np(BasicType t) {

   105 #ifndef PRODUCT

   106     switch (t) {

   107     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;

   108     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;

   109     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;

   110     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;

   111     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;

   112     }

   113     ShouldNotReachHere();

   114 #endif //PRODUCT

   115   }

   117   //------------------------------------------------------------------------------------------------------------------------

   118   // Call stubs are used to call Java from C

   119   //

   120   //    [ return_from_Java     ] <--- rsp

   121   //    [ argument word n      ]

   122   //      ...

   123   // -N [ argument word 1      ]

   124   // -7 [ Possible padding for stack alignment ]

   125   // -6 [ Possible padding for stack alignment ]

   126   // -5 [ Possible padding for stack alignment ]

   127   // -4 [ mxcsr save           ] <--- rsp_after_call

   128   // -3 [ saved rbx,            ]

   129   // -2 [ saved rsi            ]

   130   // -1 [ saved rdi            ]

   131   //  0 [ saved rbp,            ] <--- rbp,

   132   //  1 [ return address       ]

   133   //  2 [ ptr. to call wrapper ]

   134   //  3 [ result               ]

   135   //  4 [ result_type          ]

   136   //  5 [ method               ]

   137   //  6 [ entry_point          ]

   138   //  7 [ parameters           ]

   139   //  8 [ parameter_size       ]

   140   //  9 [ thread               ]

   143   address generate_call_stub(address& return_address) {

   144     StubCodeMark mark(this, "StubRoutines", "call_stub");

   145     address start = __ pc();

   147     // stub code parameters / addresses

   148     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");

   149     bool  sse_save = false;

   150     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!

   151     const int     locals_count_in_bytes  (4*wordSize);

   152     const Address mxcsr_save    (rbp, -4 * wordSize);

   153     const Address saved_rbx     (rbp, -3 * wordSize);

   154     const Address saved_rsi     (rbp, -2 * wordSize);

   155     const Address saved_rdi     (rbp, -1 * wordSize);

   156     const Address result        (rbp,  3 * wordSize);

   157     const Address result_type   (rbp,  4 * wordSize);

   158     const Address method        (rbp,  5 * wordSize);

   159     const Address entry_point   (rbp,  6 * wordSize);

   160     const Address parameters    (rbp,  7 * wordSize);

   161     const Address parameter_size(rbp,  8 * wordSize);

   162     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!

   163     sse_save =  UseSSE > 0;

   165     // stub code

   166     __ enter();

   167     __ movptr(rcx, parameter_size);              // parameter counter

   168     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes

   169     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves

   170     __ subptr(rsp, rcx);

   171     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack

   173     // save rdi, rsi, & rbx, according to C calling conventions

   174     __ movptr(saved_rdi, rdi);

   175     __ movptr(saved_rsi, rsi);

   176     __ movptr(saved_rbx, rbx);

   177     // save and initialize %mxcsr

   178     if (sse_save) {

   179       Label skip_ldmx;

   180       __ stmxcsr(mxcsr_save);

   181       __ movl(rax, mxcsr_save);

   182       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits

   183       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

   184       __ cmp32(rax, mxcsr_std);

   185       __ jcc(Assembler::equal, skip_ldmx);

   186       __ ldmxcsr(mxcsr_std);

   187       __ bind(skip_ldmx);

   188     }

   190     // make sure the control word is correct.

   191     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

   193 #ifdef ASSERT

   194     // make sure we have no pending exceptions

   195     { Label L;

   196       __ movptr(rcx, thread);

   197       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

   198       __ jcc(Assembler::equal, L);

   199       __ stop("StubRoutines::call_stub: entered with pending exception");

   200       __ bind(L);

   201     }

   202 #endif

   204     // pass parameters if any

   205     BLOCK_COMMENT("pass parameters if any");

   206     Label parameters_done;

   207     __ movl(rcx, parameter_size);  // parameter counter

   208     __ testl(rcx, rcx);

   209     __ jcc(Assembler::zero, parameters_done);

   211     // parameter passing loop

   213     Label loop;

   214     // Copy Java parameters in reverse order (receiver last)

   215     // Note that the argument order is inverted in the process

   216     // source is rdx[rcx: N-1..0]

   217     // dest   is rsp[rbx: 0..N-1]

   219     __ movptr(rdx, parameters);          // parameter pointer

   220     __ xorptr(rbx, rbx);

   222     __ BIND(loop);

   224     // get parameter

   225     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));

   226     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),

   227                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter

   228     __ increment(rbx);

   229     __ decrement(rcx);

   230     __ jcc(Assembler::notZero, loop);

   232     // call Java function

   233     __ BIND(parameters_done);

   234     __ movptr(rbx, method);           // get methodOop

   235     __ movptr(rax, entry_point);      // get entry_point

   236     __ mov(rsi, rsp);                 // set sender sp

   237     BLOCK_COMMENT("call Java function");

   238     __ call(rax);

   240     BLOCK_COMMENT("call_stub_return_address:");

   241     return_address = __ pc();

   243 #ifdef COMPILER2

   244     {

   245       Label L_skip;

   246       if (UseSSE >= 2) {

   247         __ verify_FPU(0, "call_stub_return");

   248       } else {

   249         for (int i = 1; i < 8; i++) {

   250           __ ffree(i);

   251         }

   253         // UseSSE <= 1 so double result should be left on TOS

   254         __ movl(rsi, result_type);

   255         __ cmpl(rsi, T_DOUBLE);

   256         __ jcc(Assembler::equal, L_skip);

   257         if (UseSSE == 0) {

   258           // UseSSE == 0 so float result should be left on TOS

   259           __ cmpl(rsi, T_FLOAT);

   260           __ jcc(Assembler::equal, L_skip);

   261         }

   262         __ ffree(0);

   263       }

   264       __ BIND(L_skip);

   265     }

   266 #endif // COMPILER2

   268     // store result depending on type

   269     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)

   270     __ movptr(rdi, result);

   271     Label is_long, is_float, is_double, exit;

   272     __ movl(rsi, result_type);

   273     __ cmpl(rsi, T_LONG);

   274     __ jcc(Assembler::equal, is_long);

   275     __ cmpl(rsi, T_FLOAT);

   276     __ jcc(Assembler::equal, is_float);

   277     __ cmpl(rsi, T_DOUBLE);

   278     __ jcc(Assembler::equal, is_double);

   280     // handle T_INT case

   281     __ movl(Address(rdi, 0), rax);

   282     __ BIND(exit);

   284     // check that FPU stack is empty

   285     __ verify_FPU(0, "generate_call_stub");

   287     // pop parameters

   288     __ lea(rsp, rsp_after_call);

   290     // restore %mxcsr

   291     if (sse_save) {

   292       __ ldmxcsr(mxcsr_save);

   293     }

   295     // restore rdi, rsi and rbx,

   296     __ movptr(rbx, saved_rbx);

   297     __ movptr(rsi, saved_rsi);

   298     __ movptr(rdi, saved_rdi);

   299     __ addptr(rsp, 4*wordSize);

   301     // return

   302     __ pop(rbp);

   303     __ ret(0);

   305     // handle return types different from T_INT

   306     __ BIND(is_long);

   307     __ movl(Address(rdi, 0 * wordSize), rax);

   308     __ movl(Address(rdi, 1 * wordSize), rdx);

   309     __ jmp(exit);

   311     __ BIND(is_float);

   312     // interpreter uses xmm0 for return values

   313     if (UseSSE >= 1) {

   314       __ movflt(Address(rdi, 0), xmm0);

   315     } else {

   316       __ fstp_s(Address(rdi, 0));

   317     }

   318     __ jmp(exit);

   320     __ BIND(is_double);

   321     // interpreter uses xmm0 for return values

   322     if (UseSSE >= 2) {

   323       __ movdbl(Address(rdi, 0), xmm0);

   324     } else {

   325       __ fstp_d(Address(rdi, 0));

   326     }

   327     __ jmp(exit);

   329     return start;

   330   }

   333   //------------------------------------------------------------------------------------------------------------------------

   334   // Return point for a Java call if there's an exception thrown in Java code.

   335   // The exception is caught and transformed into a pending exception stored in

   336   // JavaThread that can be tested from within the VM.

   337   //

   338   // Note: Usually the parameters are removed by the callee. In case of an exception

   339   //       crossing an activation frame boundary, that is not the case if the callee

   340   //       is compiled code => need to setup the rsp.

   341   //

   342   // rax,: exception oop

   344   address generate_catch_exception() {

   345     StubCodeMark mark(this, "StubRoutines", "catch_exception");

   346     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!

   347     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!

   348     address start = __ pc();

   350     // get thread directly

   351     __ movptr(rcx, thread);

   352 #ifdef ASSERT

   353     // verify that threads correspond

   354     { Label L;

   355       __ get_thread(rbx);

   356       __ cmpptr(rbx, rcx);

   357       __ jcc(Assembler::equal, L);

   358       __ stop("StubRoutines::catch_exception: threads must correspond");

   359       __ bind(L);

   360     }

   361 #endif

   362     // set pending exception

   363     __ verify_oop(rax);

   364     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );

   365     __ lea(Address(rcx, Thread::exception_file_offset   ()),

   366            ExternalAddress((address)__FILE__));

   367     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );

   368     // complete return to VM

   369     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");

   370     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));

   372     return start;

   373   }

   376   //------------------------------------------------------------------------------------------------------------------------

   377   // Continuation point for runtime calls returning with a pending exception.

   378   // The pending exception check happened in the runtime or native call stub.

   379   // The pending exception in Thread is converted into a Java-level exception.

   380   //

   381   // Contract with Java-level exception handlers:

   382   // rax: exception

   383   // rdx: throwing pc

   384   //

   385   // NOTE: At entry of this stub, exception-pc must be on stack !!

   387   address generate_forward_exception() {

   388     StubCodeMark mark(this, "StubRoutines", "forward exception");

   389     address start = __ pc();

   390     const Register thread = rcx;

   392     // other registers used in this stub

   393     const Register exception_oop = rax;

   394     const Register handler_addr  = rbx;

   395     const Register exception_pc  = rdx;

   397     // Upon entry, the sp points to the return address returning into Java

   398     // (interpreted or compiled) code; i.e., the return address becomes the

   399     // throwing pc.

   400     //

   401     // Arguments pushed before the runtime call are still on the stack but

   402     // the exception handler will reset the stack pointer -> ignore them.

   403     // A potential result in registers can be ignored as well.

   405 #ifdef ASSERT

   406     // make sure this code is only executed if there is a pending exception

   407     { Label L;

   408       __ get_thread(thread);

   409       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

   410       __ jcc(Assembler::notEqual, L);

   411       __ stop("StubRoutines::forward exception: no pending exception (1)");

   412       __ bind(L);

   413     }

   414 #endif

   416     // compute exception handler into rbx,

   417     __ get_thread(thread);

   418     __ movptr(exception_pc, Address(rsp, 0));

   419     BLOCK_COMMENT("call exception_handler_for_return_address");

   420     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);

   421     __ mov(handler_addr, rax);

   423     // setup rax & rdx, remove return address & clear pending exception

   424     __ get_thread(thread);

   425     __ pop(exception_pc);

   426     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));

   427     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);

   429 #ifdef ASSERT

   430     // make sure exception is set

   431     { Label L;

   432       __ testptr(exception_oop, exception_oop);

   433       __ jcc(Assembler::notEqual, L);

   434       __ stop("StubRoutines::forward exception: no pending exception (2)");

   435       __ bind(L);

   436     }

   437 #endif

   439     // Verify that there is really a valid exception in RAX.

   440     __ verify_oop(exception_oop);

   442     // continue at exception handler (return address removed)

   443     // rax: exception

   444     // rbx: exception handler

   445     // rdx: throwing pc

   446     __ jmp(handler_addr);

   448     return start;

   449   }

   452   //----------------------------------------------------------------------------------------------------

   453   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)

   454   //

   455   // xchg exists as far back as 8086, lock needed for MP only

   456   // Stack layout immediately after call:

   457   //

   458   // 0 [ret addr ] <--- rsp

   459   // 1 [  ex     ]

   460   // 2 [  dest   ]

   461   //

   462   // Result:   *dest <- ex, return (old *dest)

   463   //

   464   // Note: win32 does not currently use this code

   466   address generate_atomic_xchg() {

   467     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");

   468     address start = __ pc();

   470     __ push(rdx);

   471     Address exchange(rsp, 2 * wordSize);

   472     Address dest_addr(rsp, 3 * wordSize);

   473     __ movl(rax, exchange);

   474     __ movptr(rdx, dest_addr);

   475     __ xchgl(rax, Address(rdx, 0));

   476     __ pop(rdx);

   477     __ ret(0);

   479     return start;

   480   }

   482   //----------------------------------------------------------------------------------------------------

   483   // Support for void verify_mxcsr()

   484   //

   485   // This routine is used with -Xcheck:jni to verify that native

   486   // JNI code does not return to Java code without restoring the

   487   // MXCSR register to our expected state.

   490   address generate_verify_mxcsr() {

   491     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");

   492     address start = __ pc();

   494     const Address mxcsr_save(rsp, 0);

   496     if (CheckJNICalls && UseSSE > 0 ) {

   497       Label ok_ret;

   498       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

   499       __ push(rax);

   500       __ subptr(rsp, wordSize);      // allocate a temp location

   501       __ stmxcsr(mxcsr_save);

   502       __ movl(rax, mxcsr_save);

   503       __ andl(rax, MXCSR_MASK);

   504       __ cmp32(rax, mxcsr_std);

   505       __ jcc(Assembler::equal, ok_ret);

   507       __ warn("MXCSR changed by native JNI code.");

   509       __ ldmxcsr(mxcsr_std);

   511       __ bind(ok_ret);

   512       __ addptr(rsp, wordSize);

   513       __ pop(rax);

   514     }

   516     __ ret(0);

   518     return start;

   519   }

   522   //---------------------------------------------------------------------------

   523   // Support for void verify_fpu_cntrl_wrd()

   524   //

   525   // This routine is used with -Xcheck:jni to verify that native

   526   // JNI code does not return to Java code without restoring the

   527   // FP control word to our expected state.

   529   address generate_verify_fpu_cntrl_wrd() {

   530     StubCodeMark mark(this, "StubRoutines", "verify_spcw");

   531     address start = __ pc();

   533     const Address fpu_cntrl_wrd_save(rsp, 0);

   535     if (CheckJNICalls) {

   536       Label ok_ret;

   537       __ push(rax);

   538       __ subptr(rsp, wordSize);      // allocate a temp location

   539       __ fnstcw(fpu_cntrl_wrd_save);

   540       __ movl(rax, fpu_cntrl_wrd_save);

   541       __ andl(rax, FPU_CNTRL_WRD_MASK);

   542       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());

   543       __ cmp32(rax, fpu_std);

   544       __ jcc(Assembler::equal, ok_ret);

   546       __ warn("Floating point control word changed by native JNI code.");

   548       __ fldcw(fpu_std);

   550       __ bind(ok_ret);

   551       __ addptr(rsp, wordSize);

   552       __ pop(rax);

   553     }

   555     __ ret(0);

   557     return start;

   558   }

   560   //---------------------------------------------------------------------------

   561   // Wrapper for slow-case handling of double-to-integer conversion

   562   // d2i or f2i fast case failed either because it is nan or because

   563   // of under/overflow.

   564   // Input:  FPU TOS: float value

   565   // Output: rax, (rdx): integer (long) result

   567   address generate_d2i_wrapper(BasicType t, address fcn) {

   568     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");

   569     address start = __ pc();

   571   // Capture info about frame layout

   572   enum layout { FPUState_off         = 0,

   573                 rbp_off              = FPUStateSizeInWords,

   574                 rdi_off,

   575                 rsi_off,

   576                 rcx_off,

   577                 rbx_off,

   578                 saved_argument_off,

   579                 saved_argument_off2, // 2nd half of double

   580                 framesize

   581   };

   583   assert(FPUStateSizeInWords == 27, "update stack layout");

   585     // Save outgoing argument to stack across push_FPU_state()

   586     __ subptr(rsp, wordSize * 2);

   587     __ fstp_d(Address(rsp, 0));

   589     // Save CPU & FPU state

   590     __ push(rbx);

   591     __ push(rcx);

   592     __ push(rsi);

   593     __ push(rdi);

   594     __ push(rbp);

   595     __ push_FPU_state();

   597     // push_FPU_state() resets the FP top of stack

   598     // Load original double into FP top of stack

   599     __ fld_d(Address(rsp, saved_argument_off * wordSize));

   600     // Store double into stack as outgoing argument

   601     __ subptr(rsp, wordSize*2);

   602     __ fst_d(Address(rsp, 0));

   604     // Prepare FPU for doing math in C-land

   605     __ empty_FPU_stack();

   606     // Call the C code to massage the double.  Result in EAX

   607     if (t == T_INT)

   608       { BLOCK_COMMENT("SharedRuntime::d2i"); }

   609     else if (t == T_LONG)

   610       { BLOCK_COMMENT("SharedRuntime::d2l"); }

   611     __ call_VM_leaf( fcn, 2 );

   613     // Restore CPU & FPU state

   614     __ pop_FPU_state();

   615     __ pop(rbp);

   616     __ pop(rdi);

   617     __ pop(rsi);

   618     __ pop(rcx);

   619     __ pop(rbx);

   620     __ addptr(rsp, wordSize * 2);

   622     __ ret(0);

   624     return start;

   625   }

   628   //---------------------------------------------------------------------------

   629   // The following routine generates a subroutine to throw an asynchronous

   630   // UnknownError when an unsafe access gets a fault that could not be

   631   // reasonably prevented by the programmer.  (Example: SIGBUS/OBJERR.)

   632   address generate_handler_for_unsafe_access() {

   633     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");

   634     address start = __ pc();

   636     __ push(0);                       // hole for return address-to-be

   637     __ pusha();                       // push registers

   638     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);

   639     BLOCK_COMMENT("call handle_unsafe_access");

   640     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));

   641     __ movptr(next_pc, rax);          // stuff next address

   642     __ popa();

   643     __ ret(0);                        // jump to next address

   645     return start;

   646   }

   649   //----------------------------------------------------------------------------------------------------

   650   // Non-destructive plausibility checks for oops

   652   address generate_verify_oop() {

   653     StubCodeMark mark(this, "StubRoutines", "verify_oop");

   654     address start = __ pc();

   656     // Incoming arguments on stack after saving rax,:

   657     //

   658     // [tos    ]: saved rdx

   659     // [tos + 1]: saved EFLAGS

   660     // [tos + 2]: return address

   661     // [tos + 3]: char* error message

   662     // [tos + 4]: oop   object to verify

   663     // [tos + 5]: saved rax, - saved by caller and bashed

   665     Label exit, error;

   666     __ pushf();

   667     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));

   668     __ push(rdx);                                // save rdx

   669     // make sure object is 'reasonable'

   670     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object

   671     __ testptr(rax, rax);

   672     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok

   674     // Check if the oop is in the right area of memory

   675     const int oop_mask = Universe::verify_oop_mask();

   676     const int oop_bits = Universe::verify_oop_bits();

   677     __ mov(rdx, rax);

   678     __ andptr(rdx, oop_mask);

   679     __ cmpptr(rdx, oop_bits);

   680     __ jcc(Assembler::notZero, error);

   682     // make sure klass is 'reasonable'

   683     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass

   684     __ testptr(rax, rax);

   685     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken

   687     // Check if the klass is in the right area of memory

   688     const int klass_mask = Universe::verify_klass_mask();

   689     const int klass_bits = Universe::verify_klass_bits();

   690     __ mov(rdx, rax);

   691     __ andptr(rdx, klass_mask);

   692     __ cmpptr(rdx, klass_bits);

   693     __ jcc(Assembler::notZero, error);

   695     // make sure klass' klass is 'reasonable'

   696     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass

   697     __ testptr(rax, rax);

   698     __ jcc(Assembler::zero, error);              // if klass' klass is NULL it is broken

   700     __ mov(rdx, rax);

   701     __ andptr(rdx, klass_mask);

   702     __ cmpptr(rdx, klass_bits);

   703     __ jcc(Assembler::notZero, error);           // if klass not in right area

   704                                                  // of memory it is broken too.

   706     // return if everything seems ok

   707     __ bind(exit);

   708     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back

   709     __ pop(rdx);                                 // restore rdx

   710     __ popf();                                   // restore EFLAGS

   711     __ ret(3 * wordSize);                        // pop arguments

   713     // handle errors

   714     __ bind(error);

   715     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back

   716     __ pop(rdx);                                 // get saved rdx back

   717     __ popf();                                   // get saved EFLAGS off stack -- will be ignored

   718     __ pusha();                                  // push registers (eip = return address & msg are already pushed)

   719     BLOCK_COMMENT("call MacroAssembler::debug");

   720     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));

   721     __ popa();

   722     __ ret(3 * wordSize);                        // pop arguments

   723     return start;

   724   }

   726   //

   727   //  Generate pre-barrier for array stores

   728   //

   729   //  Input:

   730   //     start   -  starting address

   731   //     count   -  element count

   732   void  gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {

   733     assert_different_registers(start, count);

   734     BarrierSet* bs = Universe::heap()->barrier_set();

   735     switch (bs->kind()) {

   736       case BarrierSet::G1SATBCT:

   737       case BarrierSet::G1SATBCTLogging:

   738         // With G1, don't generate the call if we statically know that the target in uninitialized

   739         if (!uninitialized_target) {

   740            __ pusha();                      // push registers

   741            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),

   742                            start, count);

   743            __ popa();

   744          }

   745         break;

   746       case BarrierSet::CardTableModRef:

   747       case BarrierSet::CardTableExtension:

   748       case BarrierSet::ModRef:

   749         break;

   750       default      :

   751         ShouldNotReachHere();

   753     }

   754   }

   757   //

   758   // Generate a post-barrier for an array store

   759   //

   760   //     start    -  starting address

   761   //     count    -  element count

   762   //

   763   //  The two input registers are overwritten.

   764   //

   765   void  gen_write_ref_array_post_barrier(Register start, Register count) {

   766     BarrierSet* bs = Universe::heap()->barrier_set();

   767     assert_different_registers(start, count);

   768     switch (bs->kind()) {

   769       case BarrierSet::G1SATBCT:

   770       case BarrierSet::G1SATBCTLogging:

   771         {

   772           __ pusha();                      // push registers

   773           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),

   774                           start, count);

   775           __ popa();

   776         }

   777         break;

   779       case BarrierSet::CardTableModRef:

   780       case BarrierSet::CardTableExtension:

   781         {

   782           CardTableModRefBS* ct = (CardTableModRefBS*)bs;

   783           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");

   785           Label L_loop;

   786           const Register end = count;  // elements count; end == start+count-1

   787           assert_different_registers(start, end);

   789           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));

   790           __ shrptr(start, CardTableModRefBS::card_shift);

   791           __ shrptr(end,   CardTableModRefBS::card_shift);

   792           __ subptr(end, start); // end --> count

   793         __ BIND(L_loop);

   794           intptr_t disp = (intptr_t) ct->byte_map_base;

   795           Address cardtable(start, count, Address::times_1, disp);

   796           __ movb(cardtable, 0);

   797           __ decrement(count);

   798           __ jcc(Assembler::greaterEqual, L_loop);

   799         }

   800         break;

   801       case BarrierSet::ModRef:

   802         break;

   803       default      :

   804         ShouldNotReachHere();

   806     }

   807   }

   810   // Copy 64 bytes chunks

   811   //

   812   // Inputs:

   813   //   from        - source array address

   814   //   to_from     - destination array address - from

   815   //   qword_count - 8-bytes element count, negative

   816   //

   817   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {

   818     assert( UseSSE >= 2, "supported cpu only" );

   819     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;

   820     // Copy 64-byte chunks

   821     __ jmpb(L_copy_64_bytes);

   822     __ align(OptoLoopAlignment);

   823   __ BIND(L_copy_64_bytes_loop);

   825     if(UseUnalignedLoadStores) {

   826       __ movdqu(xmm0, Address(from, 0));

   827       __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);

   828       __ movdqu(xmm1, Address(from, 16));

   829       __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);

   830       __ movdqu(xmm2, Address(from, 32));

   831       __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);

   832       __ movdqu(xmm3, Address(from, 48));

   833       __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);

   835     } else {

   836       __ movq(xmm0, Address(from, 0));

   837       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);

   838       __ movq(xmm1, Address(from, 8));

   839       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);

   840       __ movq(xmm2, Address(from, 16));

   841       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);

   842       __ movq(xmm3, Address(from, 24));

   843       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);

   844       __ movq(xmm4, Address(from, 32));

   845       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);

   846       __ movq(xmm5, Address(from, 40));

   847       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);

   848       __ movq(xmm6, Address(from, 48));

   849       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);

   850       __ movq(xmm7, Address(from, 56));

   851       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);

   852     }

   854     __ addl(from, 64);

   855   __ BIND(L_copy_64_bytes);

   856     __ subl(qword_count, 8);

   857     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);

   858     __ addl(qword_count, 8);

   859     __ jccb(Assembler::zero, L_exit);

   860     //

   861     // length is too short, just copy qwords

   862     //

   863   __ BIND(L_copy_8_bytes);

   864     __ movq(xmm0, Address(from, 0));

   865     __ movq(Address(from, to_from, Address::times_1), xmm0);

   866     __ addl(from, 8);

   867     __ decrement(qword_count);

   868     __ jcc(Assembler::greater, L_copy_8_bytes);

   869   __ BIND(L_exit);

   870   }

   872   // Copy 64 bytes chunks

   873   //

   874   // Inputs:

   875   //   from        - source array address

   876   //   to_from     - destination array address - from

   877   //   qword_count - 8-bytes element count, negative

   878   //

   879   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {

   880     assert( VM_Version::supports_mmx(), "supported cpu only" );

   881     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;

   882     // Copy 64-byte chunks

   883     __ jmpb(L_copy_64_bytes);

   884     __ align(OptoLoopAlignment);

   885   __ BIND(L_copy_64_bytes_loop);

   886     __ movq(mmx0, Address(from, 0));

   887     __ movq(mmx1, Address(from, 8));

   888     __ movq(mmx2, Address(from, 16));

   889     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);

   890     __ movq(mmx3, Address(from, 24));

   891     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);

   892     __ movq(mmx4, Address(from, 32));

   893     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);

   894     __ movq(mmx5, Address(from, 40));

   895     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);

   896     __ movq(mmx6, Address(from, 48));

   897     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);

   898     __ movq(mmx7, Address(from, 56));

   899     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);

   900     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);

   901     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);

   902     __ addptr(from, 64);

   903   __ BIND(L_copy_64_bytes);

   904     __ subl(qword_count, 8);

   905     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);

   906     __ addl(qword_count, 8);

   907     __ jccb(Assembler::zero, L_exit);

   908     //

   909     // length is too short, just copy qwords

   910     //

   911   __ BIND(L_copy_8_bytes);

   912     __ movq(mmx0, Address(from, 0));

   913     __ movq(Address(from, to_from, Address::times_1), mmx0);

   914     __ addptr(from, 8);

   915     __ decrement(qword_count);

   916     __ jcc(Assembler::greater, L_copy_8_bytes);

   917   __ BIND(L_exit);

   918     __ emms();

   919   }

   921   address generate_disjoint_copy(BasicType t, bool aligned,

   922                                  Address::ScaleFactor sf,

   923                                  address* entry, const char *name,

   924                                  bool dest_uninitialized = false) {

   925     __ align(CodeEntryAlignment);

   926     StubCodeMark mark(this, "StubRoutines", name);

   927     address start = __ pc();

   929     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;

   930     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;

   932     int shift = Address::times_ptr - sf;

   934     const Register from     = rsi;  // source array address

   935     const Register to       = rdi;  // destination array address

   936     const Register count    = rcx;  // elements count

   937     const Register to_from  = to;   // (to - from)

   938     const Register saved_to = rdx;  // saved destination array address

   940     __ enter(); // required for proper stackwalking of RuntimeStub frame

   941     __ push(rsi);

   942     __ push(rdi);

   943     __ movptr(from , Address(rsp, 12+ 4));

   944     __ movptr(to   , Address(rsp, 12+ 8));

   945     __ movl(count, Address(rsp, 12+ 12));

   947     if (entry != NULL) {

   948       *entry = __ pc(); // Entry point from conjoint arraycopy stub.

   949       BLOCK_COMMENT("Entry:");

   950     }

   952     if (t == T_OBJECT) {

   953       __ testl(count, count);

   954       __ jcc(Assembler::zero, L_0_count);

   955       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);

   956       __ mov(saved_to, to);          // save 'to'

   957     }

   959     __ subptr(to, from); // to --> to_from

   960     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

   961     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp

   962     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {

   963       // align source address at 4 bytes address boundary

   964       if (t == T_BYTE) {

   965         // One byte misalignment happens only for byte arrays

   966         __ testl(from, 1);

   967         __ jccb(Assembler::zero, L_skip_align1);

   968         __ movb(rax, Address(from, 0));

   969         __ movb(Address(from, to_from, Address::times_1, 0), rax);

   970         __ increment(from);

   971         __ decrement(count);

   972       __ BIND(L_skip_align1);

   973       }

   974       // Two bytes misalignment happens only for byte and short (char) arrays

   975       __ testl(from, 2);

   976       __ jccb(Assembler::zero, L_skip_align2);

   977       __ movw(rax, Address(from, 0));

   978       __ movw(Address(from, to_from, Address::times_1, 0), rax);

   979       __ addptr(from, 2);

   980       __ subl(count, 1<<(shift-1));

   981     __ BIND(L_skip_align2);

   982     }

   983     if (!VM_Version::supports_mmx()) {

   984       __ mov(rax, count);      // save 'count'

   985       __ shrl(count, shift); // bytes count

   986       __ addptr(to_from, from);// restore 'to'

   987       __ rep_mov();

   988       __ subptr(to_from, from);// restore 'to_from'

   989       __ mov(count, rax);      // restore 'count'

   990       __ jmpb(L_copy_2_bytes); // all dwords were copied

   991     } else {

   992       if (!UseUnalignedLoadStores) {

   993         // align to 8 bytes, we know we are 4 byte aligned to start

   994         __ testptr(from, 4);

   995         __ jccb(Assembler::zero, L_copy_64_bytes);

   996         __ movl(rax, Address(from, 0));

   997         __ movl(Address(from, to_from, Address::times_1, 0), rax);

   998         __ addptr(from, 4);

   999         __ subl(count, 1<<shift);

  1000       }

  1001     __ BIND(L_copy_64_bytes);

  1002       __ mov(rax, count);

  1003       __ shrl(rax, shift+1);  // 8 bytes chunk count

  1004       //

  1005       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop

  1006       //

  1007       if (UseXMMForArrayCopy) {

  1008         xmm_copy_forward(from, to_from, rax);

  1009       } else {

  1010         mmx_copy_forward(from, to_from, rax);

  1011       }

  1012     }

  1013     // copy tailing dword

  1014   __ BIND(L_copy_4_bytes);

  1015     __ testl(count, 1<<shift);

  1016     __ jccb(Assembler::zero, L_copy_2_bytes);

  1017     __ movl(rax, Address(from, 0));

  1018     __ movl(Address(from, to_from, Address::times_1, 0), rax);

  1019     if (t == T_BYTE || t == T_SHORT) {

  1020       __ addptr(from, 4);

  1021     __ BIND(L_copy_2_bytes);

  1022       // copy tailing word

  1023       __ testl(count, 1<<(shift-1));

  1024       __ jccb(Assembler::zero, L_copy_byte);

  1025       __ movw(rax, Address(from, 0));

  1026       __ movw(Address(from, to_from, Address::times_1, 0), rax);

  1027       if (t == T_BYTE) {

  1028         __ addptr(from, 2);

  1029       __ BIND(L_copy_byte);

  1030         // copy tailing byte

  1031         __ testl(count, 1);

  1032         __ jccb(Assembler::zero, L_exit);

  1033         __ movb(rax, Address(from, 0));

  1034         __ movb(Address(from, to_from, Address::times_1, 0), rax);

  1035       __ BIND(L_exit);

  1036       } else {

  1037       __ BIND(L_copy_byte);

  1038       }

  1039     } else {

  1040     __ BIND(L_copy_2_bytes);

  1041     }

  1043     if (t == T_OBJECT) {

  1044       __ movl(count, Address(rsp, 12+12)); // reread 'count'

  1045       __ mov(to, saved_to); // restore 'to'

  1046       gen_write_ref_array_post_barrier(to, count);

  1047     __ BIND(L_0_count);

  1048     }

  1049     inc_copy_counter_np(t);

  1050     __ pop(rdi);

  1051     __ pop(rsi);

  1052     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1053     __ xorptr(rax, rax); // return 0

  1054     __ ret(0);

  1055     return start;

  1056   }

  1059   address generate_fill(BasicType t, bool aligned, const char *name) {

  1060     __ align(CodeEntryAlignment);

  1061     StubCodeMark mark(this, "StubRoutines", name);

  1062     address start = __ pc();

  1064     BLOCK_COMMENT("Entry:");

  1066     const Register to       = rdi;  // source array address

  1067     const Register value    = rdx;  // value

  1068     const Register count    = rsi;  // elements count

  1070     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1071     __ push(rsi);

  1072     __ push(rdi);

  1073     __ movptr(to   , Address(rsp, 12+ 4));

  1074     __ movl(value, Address(rsp, 12+ 8));

  1075     __ movl(count, Address(rsp, 12+ 12));

  1077     __ generate_fill(t, aligned, to, value, count, rax, xmm0);

  1079     __ pop(rdi);

  1080     __ pop(rsi);

  1081     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1082     __ ret(0);

  1083     return start;

  1084   }

  1086   address generate_conjoint_copy(BasicType t, bool aligned,

  1087                                  Address::ScaleFactor sf,

  1088                                  address nooverlap_target,

  1089                                  address* entry, const char *name,

  1090                                  bool dest_uninitialized = false) {

  1091     __ align(CodeEntryAlignment);

  1092     StubCodeMark mark(this, "StubRoutines", name);

  1093     address start = __ pc();

  1095     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;

  1096     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;

  1098     int shift = Address::times_ptr - sf;

  1100     const Register src   = rax;  // source array address

  1101     const Register dst   = rdx;  // destination array address

  1102     const Register from  = rsi;  // source array address

  1103     const Register to    = rdi;  // destination array address

  1104     const Register count = rcx;  // elements count

  1105     const Register end   = rax;  // array end address

  1107     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1108     __ push(rsi);

  1109     __ push(rdi);

  1110     __ movptr(src  , Address(rsp, 12+ 4));   // from

  1111     __ movptr(dst  , Address(rsp, 12+ 8));   // to

  1112     __ movl2ptr(count, Address(rsp, 12+12)); // count

  1114     if (entry != NULL) {

  1115       *entry = __ pc(); // Entry point from generic arraycopy stub.

  1116       BLOCK_COMMENT("Entry:");

  1117     }

  1119     // nooverlap_target expects arguments in rsi and rdi.

  1120     __ mov(from, src);

  1121     __ mov(to  , dst);

  1123     // arrays overlap test: dispatch to disjoint stub if necessary.

  1124     RuntimeAddress nooverlap(nooverlap_target);

  1125     __ cmpptr(dst, src);

  1126     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size

  1127     __ jump_cc(Assembler::belowEqual, nooverlap);

  1128     __ cmpptr(dst, end);

  1129     __ jump_cc(Assembler::aboveEqual, nooverlap);

  1131     if (t == T_OBJECT) {

  1132       __ testl(count, count);

  1133       __ jcc(Assembler::zero, L_0_count);

  1134       gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);

  1135     }

  1137     // copy from high to low

  1138     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

  1139     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp

  1140     if (t == T_BYTE || t == T_SHORT) {

  1141       // Align the end of destination array at 4 bytes address boundary

  1142       __ lea(end, Address(dst, count, sf, 0));

  1143       if (t == T_BYTE) {

  1144         // One byte misalignment happens only for byte arrays

  1145         __ testl(end, 1);

  1146         __ jccb(Assembler::zero, L_skip_align1);

  1147         __ decrement(count);

  1148         __ movb(rdx, Address(from, count, sf, 0));

  1149         __ movb(Address(to, count, sf, 0), rdx);

  1150       __ BIND(L_skip_align1);

  1151       }

  1152       // Two bytes misalignment happens only for byte and short (char) arrays

  1153       __ testl(end, 2);

  1154       __ jccb(Assembler::zero, L_skip_align2);

  1155       __ subptr(count, 1<<(shift-1));

  1156       __ movw(rdx, Address(from, count, sf, 0));

  1157       __ movw(Address(to, count, sf, 0), rdx);

  1158     __ BIND(L_skip_align2);

  1159       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

  1160       __ jcc(Assembler::below, L_copy_4_bytes);

  1161     }

  1163     if (!VM_Version::supports_mmx()) {

  1164       __ std();

  1165       __ mov(rax, count); // Save 'count'

  1166       __ mov(rdx, to);    // Save 'to'

  1167       __ lea(rsi, Address(from, count, sf, -4));

  1168       __ lea(rdi, Address(to  , count, sf, -4));

  1169       __ shrptr(count, shift); // bytes count

  1170       __ rep_mov();

  1171       __ cld();

  1172       __ mov(count, rax); // restore 'count'

  1173       __ andl(count, (1<<shift)-1);      // mask the number of rest elements

  1174       __ movptr(from, Address(rsp, 12+4)); // reread 'from'

  1175       __ mov(to, rdx);   // restore 'to'

  1176       __ jmpb(L_copy_2_bytes); // all dword were copied

  1177    } else {

  1178       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.

  1179       __ testptr(end, 4);

  1180       __ jccb(Assembler::zero, L_copy_8_bytes);

  1181       __ subl(count, 1<<shift);

  1182       __ movl(rdx, Address(from, count, sf, 0));

  1183       __ movl(Address(to, count, sf, 0), rdx);

  1184       __ jmpb(L_copy_8_bytes);

  1186       __ align(OptoLoopAlignment);

  1187       // Move 8 bytes

  1188     __ BIND(L_copy_8_bytes_loop);

  1189       if (UseXMMForArrayCopy) {

  1190         __ movq(xmm0, Address(from, count, sf, 0));

  1191         __ movq(Address(to, count, sf, 0), xmm0);

  1192       } else {

  1193         __ movq(mmx0, Address(from, count, sf, 0));

  1194         __ movq(Address(to, count, sf, 0), mmx0);

  1195       }

  1196     __ BIND(L_copy_8_bytes);

  1197       __ subl(count, 2<<shift);

  1198       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1199       __ addl(count, 2<<shift);

  1200       if (!UseXMMForArrayCopy) {

  1201         __ emms();

  1202       }

  1203     }

  1204   __ BIND(L_copy_4_bytes);

  1205     // copy prefix qword

  1206     __ testl(count, 1<<shift);

  1207     __ jccb(Assembler::zero, L_copy_2_bytes);

  1208     __ movl(rdx, Address(from, count, sf, -4));

  1209     __ movl(Address(to, count, sf, -4), rdx);

  1211     if (t == T_BYTE || t == T_SHORT) {

  1212         __ subl(count, (1<<shift));

  1213       __ BIND(L_copy_2_bytes);

  1214         // copy prefix dword

  1215         __ testl(count, 1<<(shift-1));

  1216         __ jccb(Assembler::zero, L_copy_byte);

  1217         __ movw(rdx, Address(from, count, sf, -2));

  1218         __ movw(Address(to, count, sf, -2), rdx);

  1219         if (t == T_BYTE) {

  1220           __ subl(count, 1<<(shift-1));

  1221         __ BIND(L_copy_byte);

  1222           // copy prefix byte

  1223           __ testl(count, 1);

  1224           __ jccb(Assembler::zero, L_exit);

  1225           __ movb(rdx, Address(from, 0));

  1226           __ movb(Address(to, 0), rdx);

  1227         __ BIND(L_exit);

  1228         } else {

  1229         __ BIND(L_copy_byte);

  1230         }

  1231     } else {

  1232     __ BIND(L_copy_2_bytes);

  1233     }

  1234     if (t == T_OBJECT) {

  1235       __ movl2ptr(count, Address(rsp, 12+12)); // reread count

  1236       gen_write_ref_array_post_barrier(to, count);

  1237     __ BIND(L_0_count);

  1238     }

  1239     inc_copy_counter_np(t);

  1240     __ pop(rdi);

  1241     __ pop(rsi);

  1242     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1243     __ xorptr(rax, rax); // return 0

  1244     __ ret(0);

  1245     return start;

  1246   }

  1249   address generate_disjoint_long_copy(address* entry, const char *name) {

  1250     __ align(CodeEntryAlignment);

  1251     StubCodeMark mark(this, "StubRoutines", name);

  1252     address start = __ pc();

  1254     Label L_copy_8_bytes, L_copy_8_bytes_loop;

  1255     const Register from       = rax;  // source array address

  1256     const Register to         = rdx;  // destination array address

  1257     const Register count      = rcx;  // elements count

  1258     const Register to_from    = rdx;  // (to - from)

  1260     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1261     __ movptr(from , Address(rsp, 8+0));       // from

  1262     __ movptr(to   , Address(rsp, 8+4));       // to

  1263     __ movl2ptr(count, Address(rsp, 8+8));     // count

  1265     *entry = __ pc(); // Entry point from conjoint arraycopy stub.

  1266     BLOCK_COMMENT("Entry:");

  1268     __ subptr(to, from); // to --> to_from

  1269     if (VM_Version::supports_mmx()) {

  1270       if (UseXMMForArrayCopy) {

  1271         xmm_copy_forward(from, to_from, count);

  1272       } else {

  1273         mmx_copy_forward(from, to_from, count);

  1274       }

  1275     } else {

  1276       __ jmpb(L_copy_8_bytes);

  1277       __ align(OptoLoopAlignment);

  1278     __ BIND(L_copy_8_bytes_loop);

  1279       __ fild_d(Address(from, 0));

  1280       __ fistp_d(Address(from, to_from, Address::times_1));

  1281       __ addptr(from, 8);

  1282     __ BIND(L_copy_8_bytes);

  1283       __ decrement(count);

  1284       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1285     }

  1286     inc_copy_counter_np(T_LONG);

  1287     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1288     __ xorptr(rax, rax); // return 0

  1289     __ ret(0);

  1290     return start;

  1291   }

  1293   address generate_conjoint_long_copy(address nooverlap_target,

  1294                                       address* entry, const char *name) {

  1295     __ align(CodeEntryAlignment);

  1296     StubCodeMark mark(this, "StubRoutines", name);

  1297     address start = __ pc();

  1299     Label L_copy_8_bytes, L_copy_8_bytes_loop;

  1300     const Register from       = rax;  // source array address

  1301     const Register to         = rdx;  // destination array address

  1302     const Register count      = rcx;  // elements count

  1303     const Register end_from   = rax;  // source array end address

  1305     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1306     __ movptr(from , Address(rsp, 8+0));       // from

  1307     __ movptr(to   , Address(rsp, 8+4));       // to

  1308     __ movl2ptr(count, Address(rsp, 8+8));     // count

  1310     *entry = __ pc(); // Entry point from generic arraycopy stub.

  1311     BLOCK_COMMENT("Entry:");

  1313     // arrays overlap test

  1314     __ cmpptr(to, from);

  1315     RuntimeAddress nooverlap(nooverlap_target);

  1316     __ jump_cc(Assembler::belowEqual, nooverlap);

  1317     __ lea(end_from, Address(from, count, Address::times_8, 0));

  1318     __ cmpptr(to, end_from);

  1319     __ movptr(from, Address(rsp, 8));  // from

  1320     __ jump_cc(Assembler::aboveEqual, nooverlap);

  1322     __ jmpb(L_copy_8_bytes);

  1324     __ align(OptoLoopAlignment);

  1325   __ BIND(L_copy_8_bytes_loop);

  1326     if (VM_Version::supports_mmx()) {

  1327       if (UseXMMForArrayCopy) {

  1328         __ movq(xmm0, Address(from, count, Address::times_8));

  1329         __ movq(Address(to, count, Address::times_8), xmm0);

  1330       } else {

  1331         __ movq(mmx0, Address(from, count, Address::times_8));

  1332         __ movq(Address(to, count, Address::times_8), mmx0);

  1333       }

  1334     } else {

  1335       __ fild_d(Address(from, count, Address::times_8));

  1336       __ fistp_d(Address(to, count, Address::times_8));

  1337     }

  1338   __ BIND(L_copy_8_bytes);

  1339     __ decrement(count);

  1340     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1342     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {

  1343       __ emms();

  1344     }

  1345     inc_copy_counter_np(T_LONG);

  1346     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1347     __ xorptr(rax, rax); // return 0

  1348     __ ret(0);

  1349     return start;

  1350   }

  1353   // Helper for generating a dynamic type check.

  1354   // The sub_klass must be one of {rbx, rdx, rsi}.

  1355   // The temp is killed.

  1356   void generate_type_check(Register sub_klass,

  1357                            Address& super_check_offset_addr,

  1358                            Address& super_klass_addr,

  1359                            Register temp,

  1360                            Label* L_success, Label* L_failure) {

  1361     BLOCK_COMMENT("type_check:");

  1363     Label L_fallthrough;

  1364 #define LOCAL_JCC(assembler_con, label_ptr)                             \

  1365     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \

  1366     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/

  1368     // The following is a strange variation of the fast path which requires

  1369     // one less register, because needed values are on the argument stack.

  1370     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,

  1371     //                                  L_success, L_failure, NULL);

  1372     assert_different_registers(sub_klass, temp);

  1374     int sc_offset = (klassOopDesc::header_size() * HeapWordSize +

  1375                      Klass::secondary_super_cache_offset_in_bytes());

  1377     // if the pointers are equal, we are done (e.g., String[] elements)

  1378     __ cmpptr(sub_klass, super_klass_addr);

  1379     LOCAL_JCC(Assembler::equal, L_success);

  1381     // check the supertype display:

  1382     __ movl2ptr(temp, super_check_offset_addr);

  1383     Address super_check_addr(sub_klass, temp, Address::times_1, 0);

  1384     __ movptr(temp, super_check_addr); // load displayed supertype

  1385     __ cmpptr(temp, super_klass_addr); // test the super type

  1386     LOCAL_JCC(Assembler::equal, L_success);

  1388     // if it was a primary super, we can just fail immediately

  1389     __ cmpl(super_check_offset_addr, sc_offset);

  1390     LOCAL_JCC(Assembler::notEqual, L_failure);

  1392     // The repne_scan instruction uses fixed registers, which will get spilled.

  1393     // We happen to know this works best when super_klass is in rax.

  1394     Register super_klass = temp;

  1395     __ movptr(super_klass, super_klass_addr);

  1396     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,

  1397                                      L_success, L_failure);

  1399     __ bind(L_fallthrough);

  1401     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }

  1402     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }

  1404 #undef LOCAL_JCC

  1405   }

  1407   //

  1408   //  Generate checkcasting array copy stub

  1409   //

  1410   //  Input:

  1411   //    4(rsp)   - source array address

  1412   //    8(rsp)   - destination array address

  1413   //   12(rsp)   - element count, can be zero

  1414   //   16(rsp)   - size_t ckoff (super_check_offset)

  1415   //   20(rsp)   - oop ckval (super_klass)

  1416   //

  1417   //  Output:

  1418   //    rax, ==  0  -  success

  1419   //    rax, == -1^K - failure, where K is partial transfer count

  1420   //

  1421   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {

  1422     __ align(CodeEntryAlignment);

  1423     StubCodeMark mark(this, "StubRoutines", name);

  1424     address start = __ pc();

  1426     Label L_load_element, L_store_element, L_do_card_marks, L_done;

  1428     // register use:

  1429     //  rax, rdx, rcx -- loop control (end_from, end_to, count)

  1430     //  rdi, rsi      -- element access (oop, klass)

  1431     //  rbx,           -- temp

  1432     const Register from       = rax;    // source array address

  1433     const Register to         = rdx;    // destination array address

  1434     const Register length     = rcx;    // elements count

  1435     const Register elem       = rdi;    // each oop copied

  1436     const Register elem_klass = rsi;    // each elem._klass (sub_klass)

  1437     const Register temp       = rbx;    // lone remaining temp

  1439     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1441     __ push(rsi);

  1442     __ push(rdi);

  1443     __ push(rbx);

  1445     Address   from_arg(rsp, 16+ 4);     // from

  1446     Address     to_arg(rsp, 16+ 8);     // to

  1447     Address length_arg(rsp, 16+12);     // elements count

  1448     Address  ckoff_arg(rsp, 16+16);     // super_check_offset

  1449     Address  ckval_arg(rsp, 16+20);     // super_klass

  1451     // Load up:

  1452     __ movptr(from,     from_arg);

  1453     __ movptr(to,         to_arg);

  1454     __ movl2ptr(length, length_arg);

  1456     if (entry != NULL) {

  1457       *entry = __ pc(); // Entry point from generic arraycopy stub.

  1458       BLOCK_COMMENT("Entry:");

  1459     }

  1461     //---------------------------------------------------------------

  1462     // Assembler stub will be used for this call to arraycopy

  1463     // if the two arrays are subtypes of Object[] but the

  1464     // destination array type is not equal to or a supertype

  1465     // of the source type.  Each element must be separately

  1466     // checked.

  1468     // Loop-invariant addresses.  They are exclusive end pointers.

  1469     Address end_from_addr(from, length, Address::times_ptr, 0);

  1470     Address   end_to_addr(to,   length, Address::times_ptr, 0);

  1472     Register end_from = from;           // re-use

  1473     Register end_to   = to;             // re-use

  1474     Register count    = length;         // re-use

  1476     // Loop-variant addresses.  They assume post-incremented count < 0.

  1477     Address from_element_addr(end_from, count, Address::times_ptr, 0);

  1478     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);

  1479     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());

  1481     // Copy from low to high addresses, indexed from the end of each array.

  1482     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);

  1483     __ lea(end_from, end_from_addr);

  1484     __ lea(end_to,   end_to_addr);

  1485     assert(length == count, "");        // else fix next line:

  1486     __ negptr(count);                   // negate and test the length

  1487     __ jccb(Assembler::notZero, L_load_element);

  1489     // Empty array:  Nothing to do.

  1490     __ xorptr(rax, rax);                  // return 0 on (trivial) success

  1491     __ jmp(L_done);

  1493     // ======== begin loop ========

  1494     // (Loop is rotated; its entry is L_load_element.)

  1495     // Loop control:

  1496     //   for (count = -count; count != 0; count++)

  1497     // Base pointers src, dst are biased by 8*count,to last element.

  1498     __ align(OptoLoopAlignment);

  1500     __ BIND(L_store_element);

  1501     __ movptr(to_element_addr, elem);     // store the oop

  1502     __ increment(count);                // increment the count toward zero

  1503     __ jccb(Assembler::zero, L_do_card_marks);

  1505     // ======== loop entry is here ========

  1506     __ BIND(L_load_element);

  1507     __ movptr(elem, from_element_addr);   // load the oop

  1508     __ testptr(elem, elem);

  1509     __ jccb(Assembler::zero, L_store_element);

  1511     // (Could do a trick here:  Remember last successful non-null

  1512     // element stored and make a quick oop equality check on it.)

  1514     __ movptr(elem_klass, elem_klass_addr); // query the object klass

  1515     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,

  1516                         &L_store_element, NULL);

  1517       // (On fall-through, we have failed the element type check.)

  1518     // ======== end loop ========

  1520     // It was a real error; we must depend on the caller to finish the job.

  1521     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.

  1522     // Emit GC store barriers for the oops we have copied (length_arg + count),

  1523     // and report their number to the caller.

  1524     __ addl(count, length_arg);         // transfers = (length - remaining)

  1525     __ movl2ptr(rax, count);            // save the value

  1526     __ notptr(rax);                     // report (-1^K) to caller

  1527     __ movptr(to, to_arg);              // reload

  1528     assert_different_registers(to, count, rax);

  1529     gen_write_ref_array_post_barrier(to, count);

  1530     __ jmpb(L_done);

  1532     // Come here on success only.

  1533     __ BIND(L_do_card_marks);

  1534     __ movl2ptr(count, length_arg);

  1535     __ movptr(to, to_arg);                // reload

  1536     gen_write_ref_array_post_barrier(to, count);

  1537     __ xorptr(rax, rax);                  // return 0 on success

  1539     // Common exit point (success or failure).

  1540     __ BIND(L_done);

  1541     __ pop(rbx);

  1542     __ pop(rdi);

  1543     __ pop(rsi);

  1544     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);

  1545     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1546     __ ret(0);

  1548     return start;

  1549   }

  1551   //

  1552   //  Generate 'unsafe' array copy stub

  1553   //  Though just as safe as the other stubs, it takes an unscaled

  1554   //  size_t argument instead of an element count.

  1555   //

  1556   //  Input:

  1557   //    4(rsp)   - source array address

  1558   //    8(rsp)   - destination array address

  1559   //   12(rsp)   - byte count, can be zero

  1560   //

  1561   //  Output:

  1562   //    rax, ==  0  -  success

  1563   //    rax, == -1  -  need to call System.arraycopy

  1564   //

  1565   // Examines the alignment of the operands and dispatches

  1566   // to a long, int, short, or byte copy loop.

  1567   //

  1568   address generate_unsafe_copy(const char *name,

  1569                                address byte_copy_entry,

  1570                                address short_copy_entry,

  1571                                address int_copy_entry,

  1572                                address long_copy_entry) {

  1574     Label L_long_aligned, L_int_aligned, L_short_aligned;

  1576     __ align(CodeEntryAlignment);

  1577     StubCodeMark mark(this, "StubRoutines", name);

  1578     address start = __ pc();

  1580     const Register from       = rax;  // source array address

  1581     const Register to         = rdx;  // destination array address

  1582     const Register count      = rcx;  // elements count

  1584     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1585     __ push(rsi);

  1586     __ push(rdi);

  1587     Address  from_arg(rsp, 12+ 4);      // from

  1588     Address    to_arg(rsp, 12+ 8);      // to

  1589     Address count_arg(rsp, 12+12);      // byte count

  1591     // Load up:

  1592     __ movptr(from ,  from_arg);

  1593     __ movptr(to   ,    to_arg);

  1594     __ movl2ptr(count, count_arg);

  1596     // bump this on entry, not on exit:

  1597     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);

  1599     const Register bits = rsi;

  1600     __ mov(bits, from);

  1601     __ orptr(bits, to);

  1602     __ orptr(bits, count);

  1604     __ testl(bits, BytesPerLong-1);

  1605     __ jccb(Assembler::zero, L_long_aligned);

  1607     __ testl(bits, BytesPerInt-1);

  1608     __ jccb(Assembler::zero, L_int_aligned);

  1610     __ testl(bits, BytesPerShort-1);

  1611     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));

  1613     __ BIND(L_short_aligned);

  1614     __ shrptr(count, LogBytesPerShort); // size => short_count

  1615     __ movl(count_arg, count);          // update 'count'

  1616     __ jump(RuntimeAddress(short_copy_entry));

  1618     __ BIND(L_int_aligned);

  1619     __ shrptr(count, LogBytesPerInt); // size => int_count

  1620     __ movl(count_arg, count);          // update 'count'

  1621     __ jump(RuntimeAddress(int_copy_entry));

  1623     __ BIND(L_long_aligned);

  1624     __ shrptr(count, LogBytesPerLong); // size => qword_count

  1625     __ movl(count_arg, count);          // update 'count'

  1626     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.

  1627     __ pop(rsi);

  1628     __ jump(RuntimeAddress(long_copy_entry));

  1630     return start;

  1631   }

  1634   // Perform range checks on the proposed arraycopy.

  1635   // Smashes src_pos and dst_pos.  (Uses them up for temps.)

  1636   void arraycopy_range_checks(Register src,

  1637                               Register src_pos,

  1638                               Register dst,

  1639                               Register dst_pos,

  1640                               Address& length,

  1641                               Label& L_failed) {

  1642     BLOCK_COMMENT("arraycopy_range_checks:");

  1643     const Register src_end = src_pos;   // source array end position

  1644     const Register dst_end = dst_pos;   // destination array end position

  1645     __ addl(src_end, length); // src_pos + length

  1646     __ addl(dst_end, length); // dst_pos + length

  1648     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;

  1649     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));

  1650     __ jcc(Assembler::above, L_failed);

  1652     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;

  1653     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));

  1654     __ jcc(Assembler::above, L_failed);

  1656     BLOCK_COMMENT("arraycopy_range_checks done");

  1657   }

  1660   //

  1661   //  Generate generic array copy stubs

  1662   //

  1663   //  Input:

  1664   //     4(rsp)    -  src oop

  1665   //     8(rsp)    -  src_pos

  1666   //    12(rsp)    -  dst oop

  1667   //    16(rsp)    -  dst_pos

  1668   //    20(rsp)    -  element count

  1669   //

  1670   //  Output:

  1671   //    rax, ==  0  -  success

  1672   //    rax, == -1^K - failure, where K is partial transfer count

  1673   //

  1674   address generate_generic_copy(const char *name,

  1675                                 address entry_jbyte_arraycopy,

  1676                                 address entry_jshort_arraycopy,

  1677                                 address entry_jint_arraycopy,

  1678                                 address entry_oop_arraycopy,

  1679                                 address entry_jlong_arraycopy,

  1680                                 address entry_checkcast_arraycopy) {

  1681     Label L_failed, L_failed_0, L_objArray;

  1683     { int modulus = CodeEntryAlignment;

  1684       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)

  1685       int advance = target - (__ offset() % modulus);

  1686       if (advance < 0)  advance += modulus;

  1687       if (advance > 0)  __ nop(advance);

  1688     }

  1689     StubCodeMark mark(this, "StubRoutines", name);

  1691     // Short-hop target to L_failed.  Makes for denser prologue code.

  1692     __ BIND(L_failed_0);

  1693     __ jmp(L_failed);

  1694     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");

  1696     __ align(CodeEntryAlignment);

  1697     address start = __ pc();

  1699     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1700     __ push(rsi);

  1701     __ push(rdi);

  1703     // bump this on entry, not on exit:

  1704     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);

  1706     // Input values

  1707     Address SRC     (rsp, 12+ 4);

  1708     Address SRC_POS (rsp, 12+ 8);

  1709     Address DST     (rsp, 12+12);

  1710     Address DST_POS (rsp, 12+16);

  1711     Address LENGTH  (rsp, 12+20);

  1713     //-----------------------------------------------------------------------

  1714     // Assembler stub will be used for this call to arraycopy

  1715     // if the following conditions are met:

  1716     //

  1717     // (1) src and dst must not be null.

  1718     // (2) src_pos must not be negative.

  1719     // (3) dst_pos must not be negative.

  1720     // (4) length  must not be negative.

  1721     // (5) src klass and dst klass should be the same and not NULL.

  1722     // (6) src and dst should be arrays.

  1723     // (7) src_pos + length must not exceed length of src.

  1724     // (8) dst_pos + length must not exceed length of dst.

  1725     //

  1727     const Register src     = rax;       // source array oop

  1728     const Register src_pos = rsi;

  1729     const Register dst     = rdx;       // destination array oop

  1730     const Register dst_pos = rdi;

  1731     const Register length  = rcx;       // transfer count

  1733     //  if (src == NULL) return -1;

  1734     __ movptr(src, SRC);      // src oop

  1735     __ testptr(src, src);

  1736     __ jccb(Assembler::zero, L_failed_0);

  1738     //  if (src_pos < 0) return -1;

  1739     __ movl2ptr(src_pos, SRC_POS);  // src_pos

  1740     __ testl(src_pos, src_pos);

  1741     __ jccb(Assembler::negative, L_failed_0);

  1743     //  if (dst == NULL) return -1;

  1744     __ movptr(dst, DST);      // dst oop

  1745     __ testptr(dst, dst);

  1746     __ jccb(Assembler::zero, L_failed_0);

  1748     //  if (dst_pos < 0) return -1;

  1749     __ movl2ptr(dst_pos, DST_POS);  // dst_pos

  1750     __ testl(dst_pos, dst_pos);

  1751     __ jccb(Assembler::negative, L_failed_0);

  1753     //  if (length < 0) return -1;

  1754     __ movl2ptr(length, LENGTH);   // length

  1755     __ testl(length, length);

  1756     __ jccb(Assembler::negative, L_failed_0);

  1758     //  if (src->klass() == NULL) return -1;

  1759     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());

  1760     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());

  1761     const Register rcx_src_klass = rcx;    // array klass

  1762     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));

  1764 #ifdef ASSERT

  1765     //  assert(src->klass() != NULL);

  1766     BLOCK_COMMENT("assert klasses not null");

  1767     { Label L1, L2;

  1768       __ testptr(rcx_src_klass, rcx_src_klass);

  1769       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL

  1770       __ bind(L1);

  1771       __ stop("broken null klass");

  1772       __ bind(L2);

  1773       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);

  1774       __ jccb(Assembler::equal, L1);      // this would be broken also

  1775       BLOCK_COMMENT("assert done");

  1776     }

  1777 #endif //ASSERT

  1779     // Load layout helper (32-bits)

  1780     //

  1781     //  |array_tag|     | header_size | element_type |     |log2_element_size|

  1782     // 32        30    24            16              8     2                 0

  1783     //

  1784     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0

  1785     //

  1787     int lh_offset = klassOopDesc::header_size() * HeapWordSize +

  1788                     Klass::layout_helper_offset_in_bytes();

  1789     Address src_klass_lh_addr(rcx_src_klass, lh_offset);

  1791     // Handle objArrays completely differently...

  1792     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);

  1793     __ cmpl(src_klass_lh_addr, objArray_lh);

  1794     __ jcc(Assembler::equal, L_objArray);

  1796     //  if (src->klass() != dst->klass()) return -1;

  1797     __ cmpptr(rcx_src_klass, dst_klass_addr);

  1798     __ jccb(Assembler::notEqual, L_failed_0);

  1800     const Register rcx_lh = rcx;  // layout helper

  1801     assert(rcx_lh == rcx_src_klass, "known alias");

  1802     __ movl(rcx_lh, src_klass_lh_addr);

  1804     //  if (!src->is_Array()) return -1;

  1805     __ cmpl(rcx_lh, Klass::_lh_neutral_value);

  1806     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp

  1808     // At this point, it is known to be a typeArray (array_tag 0x3).

  1809 #ifdef ASSERT

  1810     { Label L;

  1811       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));

  1812       __ jcc(Assembler::greaterEqual, L); // signed cmp

  1813       __ stop("must be a primitive array");

  1814       __ bind(L);

  1815     }

  1816 #endif

  1818     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);

  1819     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1821     // typeArrayKlass

  1822     //

  1823     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);

  1824     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);

  1825     //

  1826     const Register rsi_offset = rsi; // array offset

  1827     const Register src_array  = src; // src array offset

  1828     const Register dst_array  = dst; // dst array offset

  1829     const Register rdi_elsize = rdi; // log2 element size

  1831     __ mov(rsi_offset, rcx_lh);

  1832     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);

  1833     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset

  1834     __ addptr(src_array, rsi_offset);  // src array offset

  1835     __ addptr(dst_array, rsi_offset);  // dst array offset

  1836     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize

  1838     // next registers should be set before the jump to corresponding stub

  1839     const Register from       = src; // source array address

  1840     const Register to         = dst; // destination array address

  1841     const Register count      = rcx; // elements count

  1842     // some of them should be duplicated on stack

  1843 #define FROM   Address(rsp, 12+ 4)

  1844 #define TO     Address(rsp, 12+ 8)   // Not used now

  1845 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy

  1847     BLOCK_COMMENT("scale indexes to element size");

  1848     __ movl2ptr(rsi, SRC_POS);  // src_pos

  1849     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)

  1850     assert(src_array == from, "");

  1851     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize

  1852     __ movl2ptr(rdi, DST_POS);  // dst_pos

  1853     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)

  1854     assert(dst_array == to, "");

  1855     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize

  1856     __ movptr(FROM, from);      // src_addr

  1857     __ mov(rdi_elsize, rcx_lh); // log2 elsize

  1858     __ movl2ptr(count, LENGTH); // elements count

  1860     BLOCK_COMMENT("choose copy loop based on element size");

  1861     __ cmpl(rdi_elsize, 0);

  1863     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));

  1864     __ cmpl(rdi_elsize, LogBytesPerShort);

  1865     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));

  1866     __ cmpl(rdi_elsize, LogBytesPerInt);

  1867     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));

  1868 #ifdef ASSERT

  1869     __ cmpl(rdi_elsize, LogBytesPerLong);

  1870     __ jccb(Assembler::notEqual, L_failed);

  1871 #endif

  1872     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.

  1873     __ pop(rsi);

  1874     __ jump(RuntimeAddress(entry_jlong_arraycopy));

  1876   __ BIND(L_failed);

  1877     __ xorptr(rax, rax);

  1878     __ notptr(rax); // return -1

  1879     __ pop(rdi);

  1880     __ pop(rsi);

  1881     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1882     __ ret(0);

  1884     // objArrayKlass

  1885   __ BIND(L_objArray);

  1886     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]

  1888     Label L_plain_copy, L_checkcast_copy;

  1889     //  test array classes for subtyping

  1890     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality

  1891     __ jccb(Assembler::notEqual, L_checkcast_copy);

  1893     // Identically typed arrays can be copied without element-wise checks.

  1894     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);

  1895     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1897   __ BIND(L_plain_copy);

  1898     __ movl2ptr(count, LENGTH); // elements count

  1899     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos

  1900     __ lea(from, Address(src, src_pos, Address::times_ptr,

  1901                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr

  1902     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos

  1903     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,

  1904                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr

  1905     __ movptr(FROM,  from);   // src_addr

  1906     __ movptr(TO,    to);     // dst_addr

  1907     __ movl(COUNT, count);  // count

  1908     __ jump(RuntimeAddress(entry_oop_arraycopy));

  1910   __ BIND(L_checkcast_copy);

  1911     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]

  1912     {

  1913       // Handy offsets:

  1914       int  ek_offset = (klassOopDesc::header_size() * HeapWordSize +

  1915                         objArrayKlass::element_klass_offset_in_bytes());

  1916       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +

  1917                         Klass::super_check_offset_offset_in_bytes());

  1919       Register rsi_dst_klass = rsi;

  1920       Register rdi_temp      = rdi;

  1921       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");

  1922       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");

  1923       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);

  1925       // Before looking at dst.length, make sure dst is also an objArray.

  1926       __ movptr(rsi_dst_klass, dst_klass_addr);

  1927       __ cmpl(dst_klass_lh_addr, objArray_lh);

  1928       __ jccb(Assembler::notEqual, L_failed);

  1930       // It is safe to examine both src.length and dst.length.

  1931       __ movl2ptr(src_pos, SRC_POS);        // reload rsi

  1932       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1933       // (Now src_pos and dst_pos are killed, but not src and dst.)

  1935       // We'll need this temp (don't forget to pop it after the type check).

  1936       __ push(rbx);

  1937       Register rbx_src_klass = rbx;

  1939       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx

  1940       __ movptr(rsi_dst_klass, dst_klass_addr);

  1941       Address super_check_offset_addr(rsi_dst_klass, sco_offset);

  1942       Label L_fail_array_check;

  1943       generate_type_check(rbx_src_klass,

  1944                           super_check_offset_addr, dst_klass_addr,

  1945                           rdi_temp, NULL, &L_fail_array_check);

  1946       // (On fall-through, we have passed the array type check.)

  1947       __ pop(rbx);

  1948       __ jmp(L_plain_copy);

  1950       __ BIND(L_fail_array_check);

  1951       // Reshuffle arguments so we can call checkcast_arraycopy:

  1953       // match initial saves for checkcast_arraycopy

  1954       // push(rsi);    // already done; see above

  1955       // push(rdi);    // already done; see above

  1956       // push(rbx);    // already done; see above

  1958       // Marshal outgoing arguments now, freeing registers.

  1959       Address   from_arg(rsp, 16+ 4);   // from

  1960       Address     to_arg(rsp, 16+ 8);   // to

  1961       Address length_arg(rsp, 16+12);   // elements count

  1962       Address  ckoff_arg(rsp, 16+16);   // super_check_offset

  1963       Address  ckval_arg(rsp, 16+20);   // super_klass

  1965       Address SRC_POS_arg(rsp, 16+ 8);

  1966       Address DST_POS_arg(rsp, 16+16);

  1967       Address  LENGTH_arg(rsp, 16+20);

  1968       // push rbx, changed the incoming offsets (why not just use rbp,??)

  1969       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");

  1971       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));

  1972       __ movl2ptr(length, LENGTH_arg);    // reload elements count

  1973       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos

  1974       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos

  1976       __ movptr(ckval_arg, rbx);          // destination element type

  1977       __ movl(rbx, Address(rbx, sco_offset));

  1978       __ movl(ckoff_arg, rbx);          // corresponding class check offset

  1980       __ movl(length_arg, length);      // outgoing length argument

  1982       __ lea(from, Address(src, src_pos, Address::times_ptr,

  1983                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

  1984       __ movptr(from_arg, from);

  1986       __ lea(to, Address(dst, dst_pos, Address::times_ptr,

  1987                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

  1988       __ movptr(to_arg, to);

  1989       __ jump(RuntimeAddress(entry_checkcast_arraycopy));

  1990     }

  1992     return start;

  1993   }

  1995   void generate_arraycopy_stubs() {

  1996     address entry;

  1997     address entry_jbyte_arraycopy;

  1998     address entry_jshort_arraycopy;

  1999     address entry_jint_arraycopy;

  2000     address entry_oop_arraycopy;

  2001     address entry_jlong_arraycopy;

  2002     address entry_checkcast_arraycopy;

  2004     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =

  2005         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,

  2006                                "arrayof_jbyte_disjoint_arraycopy");

  2007     StubRoutines::_arrayof_jbyte_arraycopy =

  2008         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,

  2009                                NULL, "arrayof_jbyte_arraycopy");

  2010     StubRoutines::_jbyte_disjoint_arraycopy =

  2011         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,

  2012                                "jbyte_disjoint_arraycopy");

  2013     StubRoutines::_jbyte_arraycopy =

  2014         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,

  2015                                &entry_jbyte_arraycopy, "jbyte_arraycopy");

  2017     StubRoutines::_arrayof_jshort_disjoint_arraycopy =

  2018         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,

  2019                                "arrayof_jshort_disjoint_arraycopy");

  2020     StubRoutines::_arrayof_jshort_arraycopy =

  2021         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,

  2022                                NULL, "arrayof_jshort_arraycopy");

  2023     StubRoutines::_jshort_disjoint_arraycopy =

  2024         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,

  2025                                "jshort_disjoint_arraycopy");

  2026     StubRoutines::_jshort_arraycopy =

  2027         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,

  2028                                &entry_jshort_arraycopy, "jshort_arraycopy");

  2030     // Next arrays are always aligned on 4 bytes at least.

  2031     StubRoutines::_jint_disjoint_arraycopy =

  2032         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,

  2033                                "jint_disjoint_arraycopy");

  2034     StubRoutines::_jint_arraycopy =

  2035         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,

  2036                                &entry_jint_arraycopy, "jint_arraycopy");

  2038     StubRoutines::_oop_disjoint_arraycopy =

  2039         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,

  2040                                "oop_disjoint_arraycopy");

  2041     StubRoutines::_oop_arraycopy =

  2042         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,

  2043                                &entry_oop_arraycopy, "oop_arraycopy");

  2045     StubRoutines::_oop_disjoint_arraycopy_uninit =

  2046         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,

  2047                                "oop_disjoint_arraycopy_uninit",

  2048                                /*dest_uninitialized*/true);

  2049     StubRoutines::_oop_arraycopy_uninit =

  2050         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,

  2051                                NULL, "oop_arraycopy_uninit",

  2052                                /*dest_uninitialized*/true);

  2054     StubRoutines::_jlong_disjoint_arraycopy =

  2055         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");

  2056     StubRoutines::_jlong_arraycopy =

  2057         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,

  2058                                     "jlong_arraycopy");

  2060     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");

  2061     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");

  2062     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");

  2063     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");

  2064     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");

  2065     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");

  2067     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;

  2068     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;

  2069     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;

  2070     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;

  2072     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;

  2073     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;

  2074     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;

  2075     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;

  2077     StubRoutines::_checkcast_arraycopy =

  2078         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);

  2079     StubRoutines::_checkcast_arraycopy_uninit =

  2080         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);

  2082     StubRoutines::_unsafe_arraycopy =

  2083         generate_unsafe_copy("unsafe_arraycopy",

  2084                                entry_jbyte_arraycopy,

  2085                                entry_jshort_arraycopy,

  2086                                entry_jint_arraycopy,

  2087                                entry_jlong_arraycopy);

  2089     StubRoutines::_generic_arraycopy =

  2090         generate_generic_copy("generic_arraycopy",

  2091                                entry_jbyte_arraycopy,

  2092                                entry_jshort_arraycopy,

  2093                                entry_jint_arraycopy,

  2094                                entry_oop_arraycopy,

  2095                                entry_jlong_arraycopy,

  2096                                entry_checkcast_arraycopy);

  2097   }

  2099   void generate_math_stubs() {

  2100     {

  2101       StubCodeMark mark(this, "StubRoutines", "log");

  2102       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();

  2104       __ fld_d(Address(rsp, 4));

  2105       __ flog();

  2106       __ ret(0);

  2107     }

  2108     {

  2109       StubCodeMark mark(this, "StubRoutines", "log10");

  2110       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();

  2112       __ fld_d(Address(rsp, 4));

  2113       __ flog10();

  2114       __ ret(0);

  2115     }

  2116     {

  2117       StubCodeMark mark(this, "StubRoutines", "sin");

  2118       StubRoutines::_intrinsic_sin = (double (*)(double))  __ pc();

  2120       __ fld_d(Address(rsp, 4));

  2121       __ trigfunc('s');

  2122       __ ret(0);

  2123     }

  2124     {

  2125       StubCodeMark mark(this, "StubRoutines", "cos");

  2126       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();

  2128       __ fld_d(Address(rsp, 4));

  2129       __ trigfunc('c');

  2130       __ ret(0);

  2131     }

  2132     {

  2133       StubCodeMark mark(this, "StubRoutines", "tan");

  2134       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();

  2136       __ fld_d(Address(rsp, 4));

  2137       __ trigfunc('t');

  2138       __ ret(0);

  2139     }

  2141     // The intrinsic version of these seem to return the same value as

  2142     // the strict version.

  2143     StubRoutines::_intrinsic_exp = SharedRuntime::dexp;

  2144     StubRoutines::_intrinsic_pow = SharedRuntime::dpow;

  2145   }

  2147  public:

  2148   // Information about frame layout at time of blocking runtime call.

  2149   // Note that we only have to preserve callee-saved registers since

  2150   // the compilers are responsible for supplying a continuation point

  2151   // if they expect all registers to be preserved.

  2152   enum layout {

  2153     thread_off,    // last_java_sp

  2154     arg1_off,

  2155     arg2_off,

  2156     rbp_off,       // callee saved register

  2157     ret_pc,

  2158     framesize

  2159   };

  2161  private:

  2163 #undef  __

  2164 #define __ masm->

  2166   //------------------------------------------------------------------------------------------------------------------------

  2167   // Continuation point for throwing of implicit exceptions that are not handled in

  2168   // the current activation. Fabricates an exception oop and initiates normal

  2169   // exception dispatching in this frame.

  2170   //

  2171   // Previously the compiler (c2) allowed for callee save registers on Java calls.

  2172   // This is no longer true after adapter frames were removed but could possibly

  2173   // be brought back in the future if the interpreter code was reworked and it

  2174   // was deemed worthwhile. The comment below was left to describe what must

  2175   // happen here if callee saves were resurrected. As it stands now this stub

  2176   // could actually be a vanilla BufferBlob and have now oopMap at all.

  2177   // Since it doesn't make much difference we've chosen to leave it the

  2178   // way it was in the callee save days and keep the comment.

  2180   // If we need to preserve callee-saved values we need a callee-saved oop map and

  2181   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.

  2182   // If the compiler needs all registers to be preserved between the fault

  2183   // point and the exception handler then it must assume responsibility for that in

  2184   // AbstractCompiler::continuation_for_implicit_null_exception or

  2185   // continuation_for_implicit_division_by_zero_exception. All other implicit

  2186   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are

  2187   // either at call sites or otherwise assume that stack unwinding will be initiated,

  2188   // so caller saved registers were assumed volatile in the compiler.

  2189   address generate_throw_exception(const char* name, address runtime_entry,

  2190                                    bool restore_saved_exception_pc, Register arg1 = noreg, Register arg2 = noreg) {

  2192     int insts_size = 256;

  2193     int locs_size  = 32;

  2195     CodeBuffer code(name, insts_size, locs_size);

  2196     OopMapSet* oop_maps  = new OopMapSet();

  2197     MacroAssembler* masm = new MacroAssembler(&code);

  2199     address start = __ pc();

  2201     // This is an inlined and slightly modified version of call_VM

  2202     // which has the ability to fetch the return PC out of

  2203     // thread-local storage and also sets up last_Java_sp slightly

  2204     // differently than the real call_VM

  2205     Register java_thread = rbx;

  2206     __ get_thread(java_thread);

  2207     if (restore_saved_exception_pc) {

  2208       __ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));

  2209       __ push(rax);

  2210     }

  2212     __ enter(); // required for proper stackwalking of RuntimeStub frame

  2214     // pc and rbp, already pushed

  2215     __ subptr(rsp, (framesize-2) * wordSize); // prolog

  2217     // Frame is now completed as far as size and linkage.

  2219     int frame_complete = __ pc() - start;

  2221     // push java thread (becomes first argument of C function)

  2222     __ movptr(Address(rsp, thread_off * wordSize), java_thread);

  2223     if (arg1 != noreg) {

  2224       __ movptr(Address(rsp, arg1_off * wordSize), arg1);

  2225     }

  2226     if (arg2 != noreg) {

  2227       assert(arg1 != noreg, "missing reg arg");

  2228       __ movptr(Address(rsp, arg2_off * wordSize), arg2);

  2229     }

  2231     // Set up last_Java_sp and last_Java_fp

  2232     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);

  2234     // Call runtime

  2235     BLOCK_COMMENT("call runtime_entry");

  2236     __ call(RuntimeAddress(runtime_entry));

  2237     // Generate oop map

  2238     OopMap* map =  new OopMap(framesize, 0);

  2239     oop_maps->add_gc_map(__ pc() - start, map);

  2241     // restore the thread (cannot use the pushed argument since arguments

  2242     // may be overwritten by C code generated by an optimizing compiler);

  2243     // however can use the register value directly if it is callee saved.

  2244     __ get_thread(java_thread);

  2246     __ reset_last_Java_frame(java_thread, true, false);

  2248     __ leave(); // required for proper stackwalking of RuntimeStub frame

  2250     // check for pending exceptions

  2251 #ifdef ASSERT

  2252     Label L;

  2253     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

  2254     __ jcc(Assembler::notEqual, L);

  2255     __ should_not_reach_here();

  2256     __ bind(L);

  2257 #endif /* ASSERT */

  2258     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));

  2261     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);

  2262     return stub->entry_point();

  2263   }

  2266   void create_control_words() {

  2267     // Round to nearest, 53-bit mode, exceptions masked

  2268     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;

  2269     // Round to zero, 53-bit mode, exception mased

  2270     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;

  2271     // Round to nearest, 24-bit mode, exceptions masked

  2272     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;

  2273     // Round to nearest, 64-bit mode, exceptions masked

  2274     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;

  2275     // Round to nearest, 64-bit mode, exceptions masked

  2276     StubRoutines::_mxcsr_std           = 0x1F80;

  2277     // Note: the following two constants are 80-bit values

  2278     //       layout is critical for correct loading by FPU.

  2279     // Bias for strict fp multiply/divide

  2280     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000

  2281     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;

  2282     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;

  2283     // Un-Bias for strict fp multiply/divide

  2284     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000

  2285     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;

  2286     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;

  2287   }

  2289   //---------------------------------------------------------------------------

  2290   // Initialization

  2292   void generate_initial() {

  2293     // Generates all stubs and initializes the entry points

  2295     //------------------------------------------------------------------------------------------------------------------------

  2296     // entry points that exist in all platforms

  2297     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than

  2298     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.

  2299     StubRoutines::_forward_exception_entry      = generate_forward_exception();

  2301     StubRoutines::_call_stub_entry              =

  2302       generate_call_stub(StubRoutines::_call_stub_return_address);

  2303     // is referenced by megamorphic call

  2304     StubRoutines::_catch_exception_entry        = generate_catch_exception();

  2306     // These are currently used by Solaris/Intel

  2307     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();

  2309     StubRoutines::_handler_for_unsafe_access_entry =

  2310       generate_handler_for_unsafe_access();

  2312     // platform dependent

  2313     create_control_words();

  2315     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();

  2316     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();

  2317     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,

  2318                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));

  2319     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,

  2320                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));

  2322     // Build this early so it's available for the interpreter

  2323     StubRoutines::_throw_WrongMethodTypeException_entry =

  2324       generate_throw_exception("WrongMethodTypeException throw_exception",

  2325                                CAST_FROM_FN_PTR(address, SharedRuntime::throw_WrongMethodTypeException),

  2326                                false, rax, rcx);

  2327   }

  2330   void generate_all() {

  2331     // Generates all stubs and initializes the entry points

  2333     // These entry points require SharedInfo::stack0 to be set up in non-core builds

  2334     // and need to be relocatable, so they each fabricate a RuntimeStub internally.

  2335     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError),  false);

  2336     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError),  false);

  2337     StubRoutines::_throw_ArithmeticException_entry         = generate_throw_exception("ArithmeticException throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException),  true);

  2338     StubRoutines::_throw_NullPointerException_entry        = generate_throw_exception("NullPointerException throw_exception",         CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);

  2339     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);

  2340     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",           CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError),   false);

  2342     //------------------------------------------------------------------------------------------------------------------------

  2343     // entry points that are platform specific

  2345     // support for verify_oop (must happen after universe_init)

  2346     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();

  2348     // arraycopy stubs used by compilers

  2349     generate_arraycopy_stubs();

  2351     generate_math_stubs();

  2352   }

  2355  public:

  2356   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {

  2357     if (all) {

  2358       generate_all();

  2359     } else {

  2360       generate_initial();

  2361     }

  2362   }

  2363 }; // end class declaration

  2366 void StubGenerator_generate(CodeBuffer* code, bool all) {

  2367   StubGenerator g(code, all);

  2368 }

src/cpu/x86/vm/stubGenerator_x86_32.cpp@5432047c7db7

src/cpu/x86/vm/stubGenerator_x86_32.cpp

Mercurial > jdk8-mips64-public > hotspot / file revision

src/cpu/x86/vm/stubGenerator_x86_32.cpp@5432047c7db7

src/cpu/x86/vm/stubGenerator_x86_32.cpp