src/cpu/x86/vm/stubGenerator_x86_64.cpp

Mon, 15 Jul 2013 10:28:32 -0700

author
kvn
date
Mon, 15 Jul 2013 10:28:32 -0700
changeset 5439
0f57ccdb9084
parent 5353
b800986664f4
child 5440
46a90f83df31
permissions
-rw-r--r--

8020433: Crash when using -XX:+RestoreMXCSROnJNICalls
Summary: remove StubRoutines::x86::_mxcsr_std and use StubRoutines::_mxcsr_std
Reviewed-by: jrose

     1 /*
     2  * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
     4  *
     5  * This code is free software; you can redistribute it and/or modify it
     6  * under the terms of the GNU General Public License version 2 only, as
     7  * published by the Free Software Foundation.
     8  *
     9  * This code is distributed in the hope that it will be useful, but WITHOUT
    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    12  * version 2 for more details (a copy is included in the LICENSE file that
    13  * accompanied this code).
    14  *
    15  * You should have received a copy of the GNU General Public License version
    16  * 2 along with this work; if not, write to the Free Software Foundation,
    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
    18  *
    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
    20  * or visit www.oracle.com if you need additional information or have any
    21  * questions.
    22  *
    23  */
    25 #include "precompiled.hpp"
    26 #include "asm/macroAssembler.hpp"
    27 #include "asm/macroAssembler.inline.hpp"
    28 #include "interpreter/interpreter.hpp"
    29 #include "nativeInst_x86.hpp"
    30 #include "oops/instanceOop.hpp"
    31 #include "oops/method.hpp"
    32 #include "oops/objArrayKlass.hpp"
    33 #include "oops/oop.inline.hpp"
    34 #include "prims/methodHandles.hpp"
    35 #include "runtime/frame.inline.hpp"
    36 #include "runtime/handles.inline.hpp"
    37 #include "runtime/sharedRuntime.hpp"
    38 #include "runtime/stubCodeGenerator.hpp"
    39 #include "runtime/stubRoutines.hpp"
    40 #include "runtime/thread.inline.hpp"
    41 #include "utilities/top.hpp"
    42 #ifdef COMPILER2
    43 #include "opto/runtime.hpp"
    44 #endif
    46 // Declaration and definition of StubGenerator (no .hpp file).
    47 // For a more detailed description of the stub routine structure
    48 // see the comment in stubRoutines.hpp
    50 #define __ _masm->
    51 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
    52 #define a__ ((Assembler*)_masm)->
    54 #ifdef PRODUCT
    55 #define BLOCK_COMMENT(str) /* nothing */
    56 #else
    57 #define BLOCK_COMMENT(str) __ block_comment(str)
    58 #endif
    60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
    61 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
    63 // Stub Code definitions
    65 static address handle_unsafe_access() {
    66   JavaThread* thread = JavaThread::current();
    67   address pc = thread->saved_exception_pc();
    68   // pc is the instruction which we must emulate
    69   // doing a no-op is fine:  return garbage from the load
    70   // therefore, compute npc
    71   address npc = Assembler::locate_next_instruction(pc);
    73   // request an async exception
    74   thread->set_pending_unsafe_access_error();
    76   // return address of next instruction to execute
    77   return npc;
    78 }
    80 class StubGenerator: public StubCodeGenerator {
    81  private:
    83 #ifdef PRODUCT
    84 #define inc_counter_np(counter) ((void)0)
    85 #else
    86   void inc_counter_np_(int& counter) {
    87     // This can destroy rscratch1 if counter is far from the code cache
    88     __ incrementl(ExternalAddress((address)&counter));
    89   }
    90 #define inc_counter_np(counter) \
    91   BLOCK_COMMENT("inc_counter " #counter); \
    92   inc_counter_np_(counter);
    93 #endif
    95   // Call stubs are used to call Java from C
    96   //
    97   // Linux Arguments:
    98   //    c_rarg0:   call wrapper address                   address
    99   //    c_rarg1:   result                                 address
   100   //    c_rarg2:   result type                            BasicType
   101   //    c_rarg3:   method                                 Method*
   102   //    c_rarg4:   (interpreter) entry point              address
   103   //    c_rarg5:   parameters                             intptr_t*
   104   //    16(rbp): parameter size (in words)              int
   105   //    24(rbp): thread                                 Thread*
   106   //
   107   //     [ return_from_Java     ] <--- rsp
   108   //     [ argument word n      ]
   109   //      ...
   110   // -12 [ argument word 1      ]
   111   // -11 [ saved r15            ] <--- rsp_after_call
   112   // -10 [ saved r14            ]
   113   //  -9 [ saved r13            ]
   114   //  -8 [ saved r12            ]
   115   //  -7 [ saved rbx            ]
   116   //  -6 [ call wrapper         ]
   117   //  -5 [ result               ]
   118   //  -4 [ result type          ]
   119   //  -3 [ method               ]
   120   //  -2 [ entry point          ]
   121   //  -1 [ parameters           ]
   122   //   0 [ saved rbp            ] <--- rbp
   123   //   1 [ return address       ]
   124   //   2 [ parameter size       ]
   125   //   3 [ thread               ]
   126   //
   127   // Windows Arguments:
   128   //    c_rarg0:   call wrapper address                   address
   129   //    c_rarg1:   result                                 address
   130   //    c_rarg2:   result type                            BasicType
   131   //    c_rarg3:   method                                 Method*
   132   //    48(rbp): (interpreter) entry point              address
   133   //    56(rbp): parameters                             intptr_t*
   134   //    64(rbp): parameter size (in words)              int
   135   //    72(rbp): thread                                 Thread*
   136   //
   137   //     [ return_from_Java     ] <--- rsp
   138   //     [ argument word n      ]
   139   //      ...
   140   // -28 [ argument word 1      ]
   141   // -27 [ saved xmm15          ] <--- rsp_after_call
   142   //     [ saved xmm7-xmm14     ]
   143   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
   144   //  -7 [ saved r15            ]
   145   //  -6 [ saved r14            ]
   146   //  -5 [ saved r13            ]
   147   //  -4 [ saved r12            ]
   148   //  -3 [ saved rdi            ]
   149   //  -2 [ saved rsi            ]
   150   //  -1 [ saved rbx            ]
   151   //   0 [ saved rbp            ] <--- rbp
   152   //   1 [ return address       ]
   153   //   2 [ call wrapper         ]
   154   //   3 [ result               ]
   155   //   4 [ result type          ]
   156   //   5 [ method               ]
   157   //   6 [ entry point          ]
   158   //   7 [ parameters           ]
   159   //   8 [ parameter size       ]
   160   //   9 [ thread               ]
   161   //
   162   //    Windows reserves the callers stack space for arguments 1-4.
   163   //    We spill c_rarg0-c_rarg3 to this space.
   165   // Call stub stack layout word offsets from rbp
   166   enum call_stub_layout {
   167 #ifdef _WIN64
   168     xmm_save_first     = 6,  // save from xmm6
   169     xmm_save_last      = 15, // to xmm15
   170     xmm_save_base      = -9,
   171     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
   172     r15_off            = -7,
   173     r14_off            = -6,
   174     r13_off            = -5,
   175     r12_off            = -4,
   176     rdi_off            = -3,
   177     rsi_off            = -2,
   178     rbx_off            = -1,
   179     rbp_off            =  0,
   180     retaddr_off        =  1,
   181     call_wrapper_off   =  2,
   182     result_off         =  3,
   183     result_type_off    =  4,
   184     method_off         =  5,
   185     entry_point_off    =  6,
   186     parameters_off     =  7,
   187     parameter_size_off =  8,
   188     thread_off         =  9
   189 #else
   190     rsp_after_call_off = -12,
   191     mxcsr_off          = rsp_after_call_off,
   192     r15_off            = -11,
   193     r14_off            = -10,
   194     r13_off            = -9,
   195     r12_off            = -8,
   196     rbx_off            = -7,
   197     call_wrapper_off   = -6,
   198     result_off         = -5,
   199     result_type_off    = -4,
   200     method_off         = -3,
   201     entry_point_off    = -2,
   202     parameters_off     = -1,
   203     rbp_off            =  0,
   204     retaddr_off        =  1,
   205     parameter_size_off =  2,
   206     thread_off         =  3
   207 #endif
   208   };
   210 #ifdef _WIN64
   211   Address xmm_save(int reg) {
   212     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
   213     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
   214   }
   215 #endif
   217   address generate_call_stub(address& return_address) {
   218     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
   219            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
   220            "adjust this code");
   221     StubCodeMark mark(this, "StubRoutines", "call_stub");
   222     address start = __ pc();
   224     // same as in generate_catch_exception()!
   225     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
   227     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
   228     const Address result        (rbp, result_off         * wordSize);
   229     const Address result_type   (rbp, result_type_off    * wordSize);
   230     const Address method        (rbp, method_off         * wordSize);
   231     const Address entry_point   (rbp, entry_point_off    * wordSize);
   232     const Address parameters    (rbp, parameters_off     * wordSize);
   233     const Address parameter_size(rbp, parameter_size_off * wordSize);
   235     // same as in generate_catch_exception()!
   236     const Address thread        (rbp, thread_off         * wordSize);
   238     const Address r15_save(rbp, r15_off * wordSize);
   239     const Address r14_save(rbp, r14_off * wordSize);
   240     const Address r13_save(rbp, r13_off * wordSize);
   241     const Address r12_save(rbp, r12_off * wordSize);
   242     const Address rbx_save(rbp, rbx_off * wordSize);
   244     // stub code
   245     __ enter();
   246     __ subptr(rsp, -rsp_after_call_off * wordSize);
   248     // save register parameters
   249 #ifndef _WIN64
   250     __ movptr(parameters,   c_rarg5); // parameters
   251     __ movptr(entry_point,  c_rarg4); // entry_point
   252 #endif
   254     __ movptr(method,       c_rarg3); // method
   255     __ movl(result_type,  c_rarg2);   // result type
   256     __ movptr(result,       c_rarg1); // result
   257     __ movptr(call_wrapper, c_rarg0); // call wrapper
   259     // save regs belonging to calling function
   260     __ movptr(rbx_save, rbx);
   261     __ movptr(r12_save, r12);
   262     __ movptr(r13_save, r13);
   263     __ movptr(r14_save, r14);
   264     __ movptr(r15_save, r15);
   265 #ifdef _WIN64
   266     for (int i = 6; i <= 15; i++) {
   267       __ movdqu(xmm_save(i), as_XMMRegister(i));
   268     }
   270     const Address rdi_save(rbp, rdi_off * wordSize);
   271     const Address rsi_save(rbp, rsi_off * wordSize);
   273     __ movptr(rsi_save, rsi);
   274     __ movptr(rdi_save, rdi);
   275 #else
   276     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
   277     {
   278       Label skip_ldmx;
   279       __ stmxcsr(mxcsr_save);
   280       __ movl(rax, mxcsr_save);
   281       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
   282       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
   283       __ cmp32(rax, mxcsr_std);
   284       __ jcc(Assembler::equal, skip_ldmx);
   285       __ ldmxcsr(mxcsr_std);
   286       __ bind(skip_ldmx);
   287     }
   288 #endif
   290     // Load up thread register
   291     __ movptr(r15_thread, thread);
   292     __ reinit_heapbase();
   294 #ifdef ASSERT
   295     // make sure we have no pending exceptions
   296     {
   297       Label L;
   298       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
   299       __ jcc(Assembler::equal, L);
   300       __ stop("StubRoutines::call_stub: entered with pending exception");
   301       __ bind(L);
   302     }
   303 #endif
   305     // pass parameters if any
   306     BLOCK_COMMENT("pass parameters if any");
   307     Label parameters_done;
   308     __ movl(c_rarg3, parameter_size);
   309     __ testl(c_rarg3, c_rarg3);
   310     __ jcc(Assembler::zero, parameters_done);
   312     Label loop;
   313     __ movptr(c_rarg2, parameters);       // parameter pointer
   314     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
   315     __ BIND(loop);
   316     __ movptr(rax, Address(c_rarg2, 0));// get parameter
   317     __ addptr(c_rarg2, wordSize);       // advance to next parameter
   318     __ decrementl(c_rarg1);             // decrement counter
   319     __ push(rax);                       // pass parameter
   320     __ jcc(Assembler::notZero, loop);
   322     // call Java function
   323     __ BIND(parameters_done);
   324     __ movptr(rbx, method);             // get Method*
   325     __ movptr(c_rarg1, entry_point);    // get entry_point
   326     __ mov(r13, rsp);                   // set sender sp
   327     BLOCK_COMMENT("call Java function");
   328     __ call(c_rarg1);
   330     BLOCK_COMMENT("call_stub_return_address:");
   331     return_address = __ pc();
   333     // store result depending on type (everything that is not
   334     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
   335     __ movptr(c_rarg0, result);
   336     Label is_long, is_float, is_double, exit;
   337     __ movl(c_rarg1, result_type);
   338     __ cmpl(c_rarg1, T_OBJECT);
   339     __ jcc(Assembler::equal, is_long);
   340     __ cmpl(c_rarg1, T_LONG);
   341     __ jcc(Assembler::equal, is_long);
   342     __ cmpl(c_rarg1, T_FLOAT);
   343     __ jcc(Assembler::equal, is_float);
   344     __ cmpl(c_rarg1, T_DOUBLE);
   345     __ jcc(Assembler::equal, is_double);
   347     // handle T_INT case
   348     __ movl(Address(c_rarg0, 0), rax);
   350     __ BIND(exit);
   352     // pop parameters
   353     __ lea(rsp, rsp_after_call);
   355 #ifdef ASSERT
   356     // verify that threads correspond
   357     {
   358       Label L, S;
   359       __ cmpptr(r15_thread, thread);
   360       __ jcc(Assembler::notEqual, S);
   361       __ get_thread(rbx);
   362       __ cmpptr(r15_thread, rbx);
   363       __ jcc(Assembler::equal, L);
   364       __ bind(S);
   365       __ jcc(Assembler::equal, L);
   366       __ stop("StubRoutines::call_stub: threads must correspond");
   367       __ bind(L);
   368     }
   369 #endif
   371     // restore regs belonging to calling function
   372 #ifdef _WIN64
   373     for (int i = 15; i >= 6; i--) {
   374       __ movdqu(as_XMMRegister(i), xmm_save(i));
   375     }
   376 #endif
   377     __ movptr(r15, r15_save);
   378     __ movptr(r14, r14_save);
   379     __ movptr(r13, r13_save);
   380     __ movptr(r12, r12_save);
   381     __ movptr(rbx, rbx_save);
   383 #ifdef _WIN64
   384     __ movptr(rdi, rdi_save);
   385     __ movptr(rsi, rsi_save);
   386 #else
   387     __ ldmxcsr(mxcsr_save);
   388 #endif
   390     // restore rsp
   391     __ addptr(rsp, -rsp_after_call_off * wordSize);
   393     // return
   394     __ pop(rbp);
   395     __ ret(0);
   397     // handle return types different from T_INT
   398     __ BIND(is_long);
   399     __ movq(Address(c_rarg0, 0), rax);
   400     __ jmp(exit);
   402     __ BIND(is_float);
   403     __ movflt(Address(c_rarg0, 0), xmm0);
   404     __ jmp(exit);
   406     __ BIND(is_double);
   407     __ movdbl(Address(c_rarg0, 0), xmm0);
   408     __ jmp(exit);
   410     return start;
   411   }
   413   // Return point for a Java call if there's an exception thrown in
   414   // Java code.  The exception is caught and transformed into a
   415   // pending exception stored in JavaThread that can be tested from
   416   // within the VM.
   417   //
   418   // Note: Usually the parameters are removed by the callee. In case
   419   // of an exception crossing an activation frame boundary, that is
   420   // not the case if the callee is compiled code => need to setup the
   421   // rsp.
   422   //
   423   // rax: exception oop
   425   address generate_catch_exception() {
   426     StubCodeMark mark(this, "StubRoutines", "catch_exception");
   427     address start = __ pc();
   429     // same as in generate_call_stub():
   430     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
   431     const Address thread        (rbp, thread_off         * wordSize);
   433 #ifdef ASSERT
   434     // verify that threads correspond
   435     {
   436       Label L, S;
   437       __ cmpptr(r15_thread, thread);
   438       __ jcc(Assembler::notEqual, S);
   439       __ get_thread(rbx);
   440       __ cmpptr(r15_thread, rbx);
   441       __ jcc(Assembler::equal, L);
   442       __ bind(S);
   443       __ stop("StubRoutines::catch_exception: threads must correspond");
   444       __ bind(L);
   445     }
   446 #endif
   448     // set pending exception
   449     __ verify_oop(rax);
   451     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
   452     __ lea(rscratch1, ExternalAddress((address)__FILE__));
   453     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
   454     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
   456     // complete return to VM
   457     assert(StubRoutines::_call_stub_return_address != NULL,
   458            "_call_stub_return_address must have been generated before");
   459     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
   461     return start;
   462   }
   464   // Continuation point for runtime calls returning with a pending
   465   // exception.  The pending exception check happened in the runtime
   466   // or native call stub.  The pending exception in Thread is
   467   // converted into a Java-level exception.
   468   //
   469   // Contract with Java-level exception handlers:
   470   // rax: exception
   471   // rdx: throwing pc
   472   //
   473   // NOTE: At entry of this stub, exception-pc must be on stack !!
   475   address generate_forward_exception() {
   476     StubCodeMark mark(this, "StubRoutines", "forward exception");
   477     address start = __ pc();
   479     // Upon entry, the sp points to the return address returning into
   480     // Java (interpreted or compiled) code; i.e., the return address
   481     // becomes the throwing pc.
   482     //
   483     // Arguments pushed before the runtime call are still on the stack
   484     // but the exception handler will reset the stack pointer ->
   485     // ignore them.  A potential result in registers can be ignored as
   486     // well.
   488 #ifdef ASSERT
   489     // make sure this code is only executed if there is a pending exception
   490     {
   491       Label L;
   492       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
   493       __ jcc(Assembler::notEqual, L);
   494       __ stop("StubRoutines::forward exception: no pending exception (1)");
   495       __ bind(L);
   496     }
   497 #endif
   499     // compute exception handler into rbx
   500     __ movptr(c_rarg0, Address(rsp, 0));
   501     BLOCK_COMMENT("call exception_handler_for_return_address");
   502     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
   503                          SharedRuntime::exception_handler_for_return_address),
   504                     r15_thread, c_rarg0);
   505     __ mov(rbx, rax);
   507     // setup rax & rdx, remove return address & clear pending exception
   508     __ pop(rdx);
   509     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
   510     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
   512 #ifdef ASSERT
   513     // make sure exception is set
   514     {
   515       Label L;
   516       __ testptr(rax, rax);
   517       __ jcc(Assembler::notEqual, L);
   518       __ stop("StubRoutines::forward exception: no pending exception (2)");
   519       __ bind(L);
   520     }
   521 #endif
   523     // continue at exception handler (return address removed)
   524     // rax: exception
   525     // rbx: exception handler
   526     // rdx: throwing pc
   527     __ verify_oop(rax);
   528     __ jmp(rbx);
   530     return start;
   531   }
   533   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
   534   //
   535   // Arguments :
   536   //    c_rarg0: exchange_value
   537   //    c_rarg0: dest
   538   //
   539   // Result:
   540   //    *dest <- ex, return (orig *dest)
   541   address generate_atomic_xchg() {
   542     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
   543     address start = __ pc();
   545     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
   546     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
   547     __ ret(0);
   549     return start;
   550   }
   552   // Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
   553   //
   554   // Arguments :
   555   //    c_rarg0: exchange_value
   556   //    c_rarg1: dest
   557   //
   558   // Result:
   559   //    *dest <- ex, return (orig *dest)
   560   address generate_atomic_xchg_ptr() {
   561     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
   562     address start = __ pc();
   564     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
   565     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
   566     __ ret(0);
   568     return start;
   569   }
   571   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
   572   //                                         jint compare_value)
   573   //
   574   // Arguments :
   575   //    c_rarg0: exchange_value
   576   //    c_rarg1: dest
   577   //    c_rarg2: compare_value
   578   //
   579   // Result:
   580   //    if ( compare_value == *dest ) {
   581   //       *dest = exchange_value
   582   //       return compare_value;
   583   //    else
   584   //       return *dest;
   585   address generate_atomic_cmpxchg() {
   586     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
   587     address start = __ pc();
   589     __ movl(rax, c_rarg2);
   590    if ( os::is_MP() ) __ lock();
   591     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
   592     __ ret(0);
   594     return start;
   595   }
   597   // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
   598   //                                             volatile jlong* dest,
   599   //                                             jlong compare_value)
   600   // Arguments :
   601   //    c_rarg0: exchange_value
   602   //    c_rarg1: dest
   603   //    c_rarg2: compare_value
   604   //
   605   // Result:
   606   //    if ( compare_value == *dest ) {
   607   //       *dest = exchange_value
   608   //       return compare_value;
   609   //    else
   610   //       return *dest;
   611   address generate_atomic_cmpxchg_long() {
   612     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
   613     address start = __ pc();
   615     __ movq(rax, c_rarg2);
   616    if ( os::is_MP() ) __ lock();
   617     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
   618     __ ret(0);
   620     return start;
   621   }
   623   // Support for jint atomic::add(jint add_value, volatile jint* dest)
   624   //
   625   // Arguments :
   626   //    c_rarg0: add_value
   627   //    c_rarg1: dest
   628   //
   629   // Result:
   630   //    *dest += add_value
   631   //    return *dest;
   632   address generate_atomic_add() {
   633     StubCodeMark mark(this, "StubRoutines", "atomic_add");
   634     address start = __ pc();
   636     __ movl(rax, c_rarg0);
   637    if ( os::is_MP() ) __ lock();
   638     __ xaddl(Address(c_rarg1, 0), c_rarg0);
   639     __ addl(rax, c_rarg0);
   640     __ ret(0);
   642     return start;
   643   }
   645   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
   646   //
   647   // Arguments :
   648   //    c_rarg0: add_value
   649   //    c_rarg1: dest
   650   //
   651   // Result:
   652   //    *dest += add_value
   653   //    return *dest;
   654   address generate_atomic_add_ptr() {
   655     StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
   656     address start = __ pc();
   658     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
   659    if ( os::is_MP() ) __ lock();
   660     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
   661     __ addptr(rax, c_rarg0);
   662     __ ret(0);
   664     return start;
   665   }
   667   // Support for intptr_t OrderAccess::fence()
   668   //
   669   // Arguments :
   670   //
   671   // Result:
   672   address generate_orderaccess_fence() {
   673     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
   674     address start = __ pc();
   675     __ membar(Assembler::StoreLoad);
   676     __ ret(0);
   678     return start;
   679   }
   681   // Support for intptr_t get_previous_fp()
   682   //
   683   // This routine is used to find the previous frame pointer for the
   684   // caller (current_frame_guess). This is used as part of debugging
   685   // ps() is seemingly lost trying to find frames.
   686   // This code assumes that caller current_frame_guess) has a frame.
   687   address generate_get_previous_fp() {
   688     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
   689     const Address old_fp(rbp, 0);
   690     const Address older_fp(rax, 0);
   691     address start = __ pc();
   693     __ enter();
   694     __ movptr(rax, old_fp); // callers fp
   695     __ movptr(rax, older_fp); // the frame for ps()
   696     __ pop(rbp);
   697     __ ret(0);
   699     return start;
   700   }
   702   // Support for intptr_t get_previous_sp()
   703   //
   704   // This routine is used to find the previous stack pointer for the
   705   // caller.
   706   address generate_get_previous_sp() {
   707     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
   708     address start = __ pc();
   710     __ movptr(rax, rsp);
   711     __ addptr(rax, 8); // return address is at the top of the stack.
   712     __ ret(0);
   714     return start;
   715   }
   717   //----------------------------------------------------------------------------------------------------
   718   // Support for void verify_mxcsr()
   719   //
   720   // This routine is used with -Xcheck:jni to verify that native
   721   // JNI code does not return to Java code without restoring the
   722   // MXCSR register to our expected state.
   724   address generate_verify_mxcsr() {
   725     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
   726     address start = __ pc();
   728     const Address mxcsr_save(rsp, 0);
   730     if (CheckJNICalls) {
   731       Label ok_ret;
   732       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
   733       __ push(rax);
   734       __ subptr(rsp, wordSize);      // allocate a temp location
   735       __ stmxcsr(mxcsr_save);
   736       __ movl(rax, mxcsr_save);
   737       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
   738       __ cmp32(rax, mxcsr_std);
   739       __ jcc(Assembler::equal, ok_ret);
   741       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
   743       __ ldmxcsr(mxcsr_std);
   745       __ bind(ok_ret);
   746       __ addptr(rsp, wordSize);
   747       __ pop(rax);
   748     }
   750     __ ret(0);
   752     return start;
   753   }
   755   address generate_f2i_fixup() {
   756     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
   757     Address inout(rsp, 5 * wordSize); // return address + 4 saves
   759     address start = __ pc();
   761     Label L;
   763     __ push(rax);
   764     __ push(c_rarg3);
   765     __ push(c_rarg2);
   766     __ push(c_rarg1);
   768     __ movl(rax, 0x7f800000);
   769     __ xorl(c_rarg3, c_rarg3);
   770     __ movl(c_rarg2, inout);
   771     __ movl(c_rarg1, c_rarg2);
   772     __ andl(c_rarg1, 0x7fffffff);
   773     __ cmpl(rax, c_rarg1); // NaN? -> 0
   774     __ jcc(Assembler::negative, L);
   775     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
   776     __ movl(c_rarg3, 0x80000000);
   777     __ movl(rax, 0x7fffffff);
   778     __ cmovl(Assembler::positive, c_rarg3, rax);
   780     __ bind(L);
   781     __ movptr(inout, c_rarg3);
   783     __ pop(c_rarg1);
   784     __ pop(c_rarg2);
   785     __ pop(c_rarg3);
   786     __ pop(rax);
   788     __ ret(0);
   790     return start;
   791   }
   793   address generate_f2l_fixup() {
   794     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
   795     Address inout(rsp, 5 * wordSize); // return address + 4 saves
   796     address start = __ pc();
   798     Label L;
   800     __ push(rax);
   801     __ push(c_rarg3);
   802     __ push(c_rarg2);
   803     __ push(c_rarg1);
   805     __ movl(rax, 0x7f800000);
   806     __ xorl(c_rarg3, c_rarg3);
   807     __ movl(c_rarg2, inout);
   808     __ movl(c_rarg1, c_rarg2);
   809     __ andl(c_rarg1, 0x7fffffff);
   810     __ cmpl(rax, c_rarg1); // NaN? -> 0
   811     __ jcc(Assembler::negative, L);
   812     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
   813     __ mov64(c_rarg3, 0x8000000000000000);
   814     __ mov64(rax, 0x7fffffffffffffff);
   815     __ cmov(Assembler::positive, c_rarg3, rax);
   817     __ bind(L);
   818     __ movptr(inout, c_rarg3);
   820     __ pop(c_rarg1);
   821     __ pop(c_rarg2);
   822     __ pop(c_rarg3);
   823     __ pop(rax);
   825     __ ret(0);
   827     return start;
   828   }
   830   address generate_d2i_fixup() {
   831     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
   832     Address inout(rsp, 6 * wordSize); // return address + 5 saves
   834     address start = __ pc();
   836     Label L;
   838     __ push(rax);
   839     __ push(c_rarg3);
   840     __ push(c_rarg2);
   841     __ push(c_rarg1);
   842     __ push(c_rarg0);
   844     __ movl(rax, 0x7ff00000);
   845     __ movq(c_rarg2, inout);
   846     __ movl(c_rarg3, c_rarg2);
   847     __ mov(c_rarg1, c_rarg2);
   848     __ mov(c_rarg0, c_rarg2);
   849     __ negl(c_rarg3);
   850     __ shrptr(c_rarg1, 0x20);
   851     __ orl(c_rarg3, c_rarg2);
   852     __ andl(c_rarg1, 0x7fffffff);
   853     __ xorl(c_rarg2, c_rarg2);
   854     __ shrl(c_rarg3, 0x1f);
   855     __ orl(c_rarg1, c_rarg3);
   856     __ cmpl(rax, c_rarg1);
   857     __ jcc(Assembler::negative, L); // NaN -> 0
   858     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
   859     __ movl(c_rarg2, 0x80000000);
   860     __ movl(rax, 0x7fffffff);
   861     __ cmov(Assembler::positive, c_rarg2, rax);
   863     __ bind(L);
   864     __ movptr(inout, c_rarg2);
   866     __ pop(c_rarg0);
   867     __ pop(c_rarg1);
   868     __ pop(c_rarg2);
   869     __ pop(c_rarg3);
   870     __ pop(rax);
   872     __ ret(0);
   874     return start;
   875   }
   877   address generate_d2l_fixup() {
   878     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
   879     Address inout(rsp, 6 * wordSize); // return address + 5 saves
   881     address start = __ pc();
   883     Label L;
   885     __ push(rax);
   886     __ push(c_rarg3);
   887     __ push(c_rarg2);
   888     __ push(c_rarg1);
   889     __ push(c_rarg0);
   891     __ movl(rax, 0x7ff00000);
   892     __ movq(c_rarg2, inout);
   893     __ movl(c_rarg3, c_rarg2);
   894     __ mov(c_rarg1, c_rarg2);
   895     __ mov(c_rarg0, c_rarg2);
   896     __ negl(c_rarg3);
   897     __ shrptr(c_rarg1, 0x20);
   898     __ orl(c_rarg3, c_rarg2);
   899     __ andl(c_rarg1, 0x7fffffff);
   900     __ xorl(c_rarg2, c_rarg2);
   901     __ shrl(c_rarg3, 0x1f);
   902     __ orl(c_rarg1, c_rarg3);
   903     __ cmpl(rax, c_rarg1);
   904     __ jcc(Assembler::negative, L); // NaN -> 0
   905     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
   906     __ mov64(c_rarg2, 0x8000000000000000);
   907     __ mov64(rax, 0x7fffffffffffffff);
   908     __ cmovq(Assembler::positive, c_rarg2, rax);
   910     __ bind(L);
   911     __ movq(inout, c_rarg2);
   913     __ pop(c_rarg0);
   914     __ pop(c_rarg1);
   915     __ pop(c_rarg2);
   916     __ pop(c_rarg3);
   917     __ pop(rax);
   919     __ ret(0);
   921     return start;
   922   }
   924   address generate_fp_mask(const char *stub_name, int64_t mask) {
   925     __ align(CodeEntryAlignment);
   926     StubCodeMark mark(this, "StubRoutines", stub_name);
   927     address start = __ pc();
   929     __ emit_data64( mask, relocInfo::none );
   930     __ emit_data64( mask, relocInfo::none );
   932     return start;
   933   }
   935   // The following routine generates a subroutine to throw an
   936   // asynchronous UnknownError when an unsafe access gets a fault that
   937   // could not be reasonably prevented by the programmer.  (Example:
   938   // SIGBUS/OBJERR.)
   939   address generate_handler_for_unsafe_access() {
   940     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
   941     address start = __ pc();
   943     __ push(0);                       // hole for return address-to-be
   944     __ pusha();                       // push registers
   945     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
   947     // FIXME: this probably needs alignment logic
   949     __ subptr(rsp, frame::arg_reg_save_area_bytes);
   950     BLOCK_COMMENT("call handle_unsafe_access");
   951     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
   952     __ addptr(rsp, frame::arg_reg_save_area_bytes);
   954     __ movptr(next_pc, rax);          // stuff next address
   955     __ popa();
   956     __ ret(0);                        // jump to next address
   958     return start;
   959   }
   961   // Non-destructive plausibility checks for oops
   962   //
   963   // Arguments:
   964   //    all args on stack!
   965   //
   966   // Stack after saving c_rarg3:
   967   //    [tos + 0]: saved c_rarg3
   968   //    [tos + 1]: saved c_rarg2
   969   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
   970   //    [tos + 3]: saved flags
   971   //    [tos + 4]: return address
   972   //  * [tos + 5]: error message (char*)
   973   //  * [tos + 6]: object to verify (oop)
   974   //  * [tos + 7]: saved rax - saved by caller and bashed
   975   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
   976   //  * = popped on exit
   977   address generate_verify_oop() {
   978     StubCodeMark mark(this, "StubRoutines", "verify_oop");
   979     address start = __ pc();
   981     Label exit, error;
   983     __ pushf();
   984     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
   986     __ push(r12);
   988     // save c_rarg2 and c_rarg3
   989     __ push(c_rarg2);
   990     __ push(c_rarg3);
   992     enum {
   993            // After previous pushes.
   994            oop_to_verify = 6 * wordSize,
   995            saved_rax     = 7 * wordSize,
   996            saved_r10     = 8 * wordSize,
   998            // Before the call to MacroAssembler::debug(), see below.
   999            return_addr   = 16 * wordSize,
  1000            error_msg     = 17 * wordSize
  1001     };
  1003     // get object
  1004     __ movptr(rax, Address(rsp, oop_to_verify));
  1006     // make sure object is 'reasonable'
  1007     __ testptr(rax, rax);
  1008     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
  1009     // Check if the oop is in the right area of memory
  1010     __ movptr(c_rarg2, rax);
  1011     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
  1012     __ andptr(c_rarg2, c_rarg3);
  1013     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
  1014     __ cmpptr(c_rarg2, c_rarg3);
  1015     __ jcc(Assembler::notZero, error);
  1017     // set r12 to heapbase for load_klass()
  1018     __ reinit_heapbase();
  1020     // make sure klass is 'reasonable', which is not zero.
  1021     __ load_klass(rax, rax);  // get klass
  1022     __ testptr(rax, rax);
  1023     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
  1024     // TODO: Future assert that klass is lower 4g memory for UseCompressedKlassPointers
  1026     // return if everything seems ok
  1027     __ bind(exit);
  1028     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
  1029     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
  1030     __ pop(c_rarg3);                             // restore c_rarg3
  1031     __ pop(c_rarg2);                             // restore c_rarg2
  1032     __ pop(r12);                                 // restore r12
  1033     __ popf();                                   // restore flags
  1034     __ ret(4 * wordSize);                        // pop caller saved stuff
  1036     // handle errors
  1037     __ bind(error);
  1038     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
  1039     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
  1040     __ pop(c_rarg3);                             // get saved c_rarg3 back
  1041     __ pop(c_rarg2);                             // get saved c_rarg2 back
  1042     __ pop(r12);                                 // get saved r12 back
  1043     __ popf();                                   // get saved flags off stack --
  1044                                                  // will be ignored
  1046     __ pusha();                                  // push registers
  1047                                                  // (rip is already
  1048                                                  // already pushed)
  1049     // debug(char* msg, int64_t pc, int64_t regs[])
  1050     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
  1051     // pushed all the registers, so now the stack looks like:
  1052     //     [tos +  0] 16 saved registers
  1053     //     [tos + 16] return address
  1054     //   * [tos + 17] error message (char*)
  1055     //   * [tos + 18] object to verify (oop)
  1056     //   * [tos + 19] saved rax - saved by caller and bashed
  1057     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
  1058     //   * = popped on exit
  1060     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
  1061     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
  1062     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
  1063     __ mov(r12, rsp);                               // remember rsp
  1064     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
  1065     __ andptr(rsp, -16);                            // align stack as required by ABI
  1066     BLOCK_COMMENT("call MacroAssembler::debug");
  1067     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
  1068     __ mov(rsp, r12);                               // restore rsp
  1069     __ popa();                                      // pop registers (includes r12)
  1070     __ ret(4 * wordSize);                           // pop caller saved stuff
  1072     return start;
  1075   //
  1076   // Verify that a register contains clean 32-bits positive value
  1077   // (high 32-bits are 0) so it could be used in 64-bits shifts.
  1078   //
  1079   //  Input:
  1080   //    Rint  -  32-bits value
  1081   //    Rtmp  -  scratch
  1082   //
  1083   void assert_clean_int(Register Rint, Register Rtmp) {
  1084 #ifdef ASSERT
  1085     Label L;
  1086     assert_different_registers(Rtmp, Rint);
  1087     __ movslq(Rtmp, Rint);
  1088     __ cmpq(Rtmp, Rint);
  1089     __ jcc(Assembler::equal, L);
  1090     __ stop("high 32-bits of int value are not 0");
  1091     __ bind(L);
  1092 #endif
  1095   //  Generate overlap test for array copy stubs
  1096   //
  1097   //  Input:
  1098   //     c_rarg0 - from
  1099   //     c_rarg1 - to
  1100   //     c_rarg2 - element count
  1101   //
  1102   //  Output:
  1103   //     rax   - &from[element count - 1]
  1104   //
  1105   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
  1106     assert(no_overlap_target != NULL, "must be generated");
  1107     array_overlap_test(no_overlap_target, NULL, sf);
  1109   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
  1110     array_overlap_test(NULL, &L_no_overlap, sf);
  1112   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
  1113     const Register from     = c_rarg0;
  1114     const Register to       = c_rarg1;
  1115     const Register count    = c_rarg2;
  1116     const Register end_from = rax;
  1118     __ cmpptr(to, from);
  1119     __ lea(end_from, Address(from, count, sf, 0));
  1120     if (NOLp == NULL) {
  1121       ExternalAddress no_overlap(no_overlap_target);
  1122       __ jump_cc(Assembler::belowEqual, no_overlap);
  1123       __ cmpptr(to, end_from);
  1124       __ jump_cc(Assembler::aboveEqual, no_overlap);
  1125     } else {
  1126       __ jcc(Assembler::belowEqual, (*NOLp));
  1127       __ cmpptr(to, end_from);
  1128       __ jcc(Assembler::aboveEqual, (*NOLp));
  1132   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
  1133   //
  1134   // Outputs:
  1135   //    rdi - rcx
  1136   //    rsi - rdx
  1137   //    rdx - r8
  1138   //    rcx - r9
  1139   //
  1140   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
  1141   // are non-volatile.  r9 and r10 should not be used by the caller.
  1142   //
  1143   void setup_arg_regs(int nargs = 3) {
  1144     const Register saved_rdi = r9;
  1145     const Register saved_rsi = r10;
  1146     assert(nargs == 3 || nargs == 4, "else fix");
  1147 #ifdef _WIN64
  1148     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
  1149            "unexpected argument registers");
  1150     if (nargs >= 4)
  1151       __ mov(rax, r9);  // r9 is also saved_rdi
  1152     __ movptr(saved_rdi, rdi);
  1153     __ movptr(saved_rsi, rsi);
  1154     __ mov(rdi, rcx); // c_rarg0
  1155     __ mov(rsi, rdx); // c_rarg1
  1156     __ mov(rdx, r8);  // c_rarg2
  1157     if (nargs >= 4)
  1158       __ mov(rcx, rax); // c_rarg3 (via rax)
  1159 #else
  1160     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
  1161            "unexpected argument registers");
  1162 #endif
  1165   void restore_arg_regs() {
  1166     const Register saved_rdi = r9;
  1167     const Register saved_rsi = r10;
  1168 #ifdef _WIN64
  1169     __ movptr(rdi, saved_rdi);
  1170     __ movptr(rsi, saved_rsi);
  1171 #endif
  1174   // Generate code for an array write pre barrier
  1175   //
  1176   //     addr    -  starting address
  1177   //     count   -  element count
  1178   //     tmp     - scratch register
  1179   //
  1180   //     Destroy no registers!
  1181   //
  1182   void  gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
  1183     BarrierSet* bs = Universe::heap()->barrier_set();
  1184     switch (bs->kind()) {
  1185       case BarrierSet::G1SATBCT:
  1186       case BarrierSet::G1SATBCTLogging:
  1187         // With G1, don't generate the call if we statically know that the target in uninitialized
  1188         if (!dest_uninitialized) {
  1189            __ pusha();                      // push registers
  1190            if (count == c_rarg0) {
  1191              if (addr == c_rarg1) {
  1192                // exactly backwards!!
  1193                __ xchgptr(c_rarg1, c_rarg0);
  1194              } else {
  1195                __ movptr(c_rarg1, count);
  1196                __ movptr(c_rarg0, addr);
  1198            } else {
  1199              __ movptr(c_rarg0, addr);
  1200              __ movptr(c_rarg1, count);
  1202            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
  1203            __ popa();
  1205          break;
  1206       case BarrierSet::CardTableModRef:
  1207       case BarrierSet::CardTableExtension:
  1208       case BarrierSet::ModRef:
  1209         break;
  1210       default:
  1211         ShouldNotReachHere();
  1216   //
  1217   // Generate code for an array write post barrier
  1218   //
  1219   //  Input:
  1220   //     start    - register containing starting address of destination array
  1221   //     count    - elements count
  1222   //     scratch  - scratch register
  1223   //
  1224   //  The input registers are overwritten.
  1225   //
  1226   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
  1227     assert_different_registers(start, count, scratch);
  1228     BarrierSet* bs = Universe::heap()->barrier_set();
  1229     switch (bs->kind()) {
  1230       case BarrierSet::G1SATBCT:
  1231       case BarrierSet::G1SATBCTLogging:
  1233           __ pusha();             // push registers (overkill)
  1234           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
  1235             assert_different_registers(c_rarg1, start);
  1236             __ mov(c_rarg1, count);
  1237             __ mov(c_rarg0, start);
  1238           } else {
  1239             assert_different_registers(c_rarg0, count);
  1240             __ mov(c_rarg0, start);
  1241             __ mov(c_rarg1, count);
  1243           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
  1244           __ popa();
  1246         break;
  1247       case BarrierSet::CardTableModRef:
  1248       case BarrierSet::CardTableExtension:
  1250           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
  1251           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
  1253           Label L_loop;
  1254           const Register end = count;
  1256           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
  1257           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
  1258           __ shrptr(start, CardTableModRefBS::card_shift);
  1259           __ shrptr(end,   CardTableModRefBS::card_shift);
  1260           __ subptr(end, start); // end --> cards count
  1262           int64_t disp = (int64_t) ct->byte_map_base;
  1263           __ mov64(scratch, disp);
  1264           __ addptr(start, scratch);
  1265         __ BIND(L_loop);
  1266           __ movb(Address(start, count, Address::times_1), 0);
  1267           __ decrement(count);
  1268           __ jcc(Assembler::greaterEqual, L_loop);
  1270         break;
  1271       default:
  1272         ShouldNotReachHere();
  1278   // Copy big chunks forward
  1279   //
  1280   // Inputs:
  1281   //   end_from     - source arrays end address
  1282   //   end_to       - destination array end address
  1283   //   qword_count  - 64-bits element count, negative
  1284   //   to           - scratch
  1285   //   L_copy_bytes - entry label
  1286   //   L_copy_8_bytes  - exit  label
  1287   //
  1288   void copy_bytes_forward(Register end_from, Register end_to,
  1289                              Register qword_count, Register to,
  1290                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
  1291     DEBUG_ONLY(__ stop("enter at entry label, not here"));
  1292     Label L_loop;
  1293     __ align(OptoLoopAlignment);
  1294     if (UseUnalignedLoadStores) {
  1295       Label L_end;
  1296       // Copy 64-bytes per iteration
  1297       __ BIND(L_loop);
  1298       if (UseAVX >= 2) {
  1299         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
  1300         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
  1301         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
  1302         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
  1303       } else {
  1304         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
  1305         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
  1306         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
  1307         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
  1308         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
  1309         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
  1310         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
  1311         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
  1313       __ BIND(L_copy_bytes);
  1314       __ addptr(qword_count, 8);
  1315       __ jcc(Assembler::lessEqual, L_loop);
  1316       __ subptr(qword_count, 4);  // sub(8) and add(4)
  1317       __ jccb(Assembler::greater, L_end);
  1318       // Copy trailing 32 bytes
  1319       if (UseAVX >= 2) {
  1320         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
  1321         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
  1322       } else {
  1323         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
  1324         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
  1325         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
  1326         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
  1328       __ addptr(qword_count, 4);
  1329       __ BIND(L_end);
  1330       if (UseAVX >= 2) {
  1331         // clean upper bits of YMM registers
  1332         __ vzeroupper();
  1334     } else {
  1335       // Copy 32-bytes per iteration
  1336       __ BIND(L_loop);
  1337       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
  1338       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
  1339       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
  1340       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
  1341       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
  1342       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
  1343       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
  1344       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
  1346       __ BIND(L_copy_bytes);
  1347       __ addptr(qword_count, 4);
  1348       __ jcc(Assembler::lessEqual, L_loop);
  1350     __ subptr(qword_count, 4);
  1351     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
  1354   // Copy big chunks backward
  1355   //
  1356   // Inputs:
  1357   //   from         - source arrays address
  1358   //   dest         - destination array address
  1359   //   qword_count  - 64-bits element count
  1360   //   to           - scratch
  1361   //   L_copy_bytes - entry label
  1362   //   L_copy_8_bytes  - exit  label
  1363   //
  1364   void copy_bytes_backward(Register from, Register dest,
  1365                               Register qword_count, Register to,
  1366                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
  1367     DEBUG_ONLY(__ stop("enter at entry label, not here"));
  1368     Label L_loop;
  1369     __ align(OptoLoopAlignment);
  1370     if (UseUnalignedLoadStores) {
  1371       Label L_end;
  1372       // Copy 64-bytes per iteration
  1373       __ BIND(L_loop);
  1374       if (UseAVX >= 2) {
  1375         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
  1376         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
  1377         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
  1378         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
  1379       } else {
  1380         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
  1381         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
  1382         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
  1383         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
  1384         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
  1385         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
  1386         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
  1387         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
  1389       __ BIND(L_copy_bytes);
  1390       __ subptr(qword_count, 8);
  1391       __ jcc(Assembler::greaterEqual, L_loop);
  1393       __ addptr(qword_count, 4);  // add(8) and sub(4)
  1394       __ jccb(Assembler::less, L_end);
  1395       // Copy trailing 32 bytes
  1396       if (UseAVX >= 2) {
  1397         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
  1398         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
  1399       } else {
  1400         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
  1401         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
  1402         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
  1403         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
  1405       __ subptr(qword_count, 4);
  1406       __ BIND(L_end);
  1407       if (UseAVX >= 2) {
  1408         // clean upper bits of YMM registers
  1409         __ vzeroupper();
  1411     } else {
  1412       // Copy 32-bytes per iteration
  1413       __ BIND(L_loop);
  1414       __ movq(to, Address(from, qword_count, Address::times_8, 24));
  1415       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
  1416       __ movq(to, Address(from, qword_count, Address::times_8, 16));
  1417       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
  1418       __ movq(to, Address(from, qword_count, Address::times_8,  8));
  1419       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
  1420       __ movq(to, Address(from, qword_count, Address::times_8,  0));
  1421       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
  1423       __ BIND(L_copy_bytes);
  1424       __ subptr(qword_count, 4);
  1425       __ jcc(Assembler::greaterEqual, L_loop);
  1427     __ addptr(qword_count, 4);
  1428     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
  1432   // Arguments:
  1433   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1434   //             ignored
  1435   //   name    - stub name string
  1436   //
  1437   // Inputs:
  1438   //   c_rarg0   - source array address
  1439   //   c_rarg1   - destination array address
  1440   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1441   //
  1442   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
  1443   // we let the hardware handle it.  The one to eight bytes within words,
  1444   // dwords or qwords that span cache line boundaries will still be loaded
  1445   // and stored atomically.
  1446   //
  1447   // Side Effects:
  1448   //   disjoint_byte_copy_entry is set to the no-overlap entry point
  1449   //   used by generate_conjoint_byte_copy().
  1450   //
  1451   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
  1452     __ align(CodeEntryAlignment);
  1453     StubCodeMark mark(this, "StubRoutines", name);
  1454     address start = __ pc();
  1456     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
  1457     Label L_copy_byte, L_exit;
  1458     const Register from        = rdi;  // source array address
  1459     const Register to          = rsi;  // destination array address
  1460     const Register count       = rdx;  // elements count
  1461     const Register byte_count  = rcx;
  1462     const Register qword_count = count;
  1463     const Register end_from    = from; // source array end address
  1464     const Register end_to      = to;   // destination array end address
  1465     // End pointers are inclusive, and if count is not zero they point
  1466     // to the last unit copied:  end_to[0] := end_from[0]
  1468     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1469     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1471     if (entry != NULL) {
  1472       *entry = __ pc();
  1473        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1474       BLOCK_COMMENT("Entry:");
  1477     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1478                       // r9 and r10 may be used to save non-volatile registers
  1480     // 'from', 'to' and 'count' are now valid
  1481     __ movptr(byte_count, count);
  1482     __ shrptr(count, 3); // count => qword_count
  1484     // Copy from low to high addresses.  Use 'to' as scratch.
  1485     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
  1486     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
  1487     __ negptr(qword_count); // make the count negative
  1488     __ jmp(L_copy_bytes);
  1490     // Copy trailing qwords
  1491   __ BIND(L_copy_8_bytes);
  1492     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
  1493     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
  1494     __ increment(qword_count);
  1495     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1497     // Check for and copy trailing dword
  1498   __ BIND(L_copy_4_bytes);
  1499     __ testl(byte_count, 4);
  1500     __ jccb(Assembler::zero, L_copy_2_bytes);
  1501     __ movl(rax, Address(end_from, 8));
  1502     __ movl(Address(end_to, 8), rax);
  1504     __ addptr(end_from, 4);
  1505     __ addptr(end_to, 4);
  1507     // Check for and copy trailing word
  1508   __ BIND(L_copy_2_bytes);
  1509     __ testl(byte_count, 2);
  1510     __ jccb(Assembler::zero, L_copy_byte);
  1511     __ movw(rax, Address(end_from, 8));
  1512     __ movw(Address(end_to, 8), rax);
  1514     __ addptr(end_from, 2);
  1515     __ addptr(end_to, 2);
  1517     // Check for and copy trailing byte
  1518   __ BIND(L_copy_byte);
  1519     __ testl(byte_count, 1);
  1520     __ jccb(Assembler::zero, L_exit);
  1521     __ movb(rax, Address(end_from, 8));
  1522     __ movb(Address(end_to, 8), rax);
  1524   __ BIND(L_exit);
  1525     restore_arg_regs();
  1526     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
  1527     __ xorptr(rax, rax); // return 0
  1528     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1529     __ ret(0);
  1531     // Copy in multi-bytes chunks
  1532     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  1533     __ jmp(L_copy_4_bytes);
  1535     return start;
  1538   // Arguments:
  1539   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1540   //             ignored
  1541   //   name    - stub name string
  1542   //
  1543   // Inputs:
  1544   //   c_rarg0   - source array address
  1545   //   c_rarg1   - destination array address
  1546   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1547   //
  1548   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
  1549   // we let the hardware handle it.  The one to eight bytes within words,
  1550   // dwords or qwords that span cache line boundaries will still be loaded
  1551   // and stored atomically.
  1552   //
  1553   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
  1554                                       address* entry, const char *name) {
  1555     __ align(CodeEntryAlignment);
  1556     StubCodeMark mark(this, "StubRoutines", name);
  1557     address start = __ pc();
  1559     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
  1560     const Register from        = rdi;  // source array address
  1561     const Register to          = rsi;  // destination array address
  1562     const Register count       = rdx;  // elements count
  1563     const Register byte_count  = rcx;
  1564     const Register qword_count = count;
  1566     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1567     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1569     if (entry != NULL) {
  1570       *entry = __ pc();
  1571       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1572       BLOCK_COMMENT("Entry:");
  1575     array_overlap_test(nooverlap_target, Address::times_1);
  1576     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1577                       // r9 and r10 may be used to save non-volatile registers
  1579     // 'from', 'to' and 'count' are now valid
  1580     __ movptr(byte_count, count);
  1581     __ shrptr(count, 3);   // count => qword_count
  1583     // Copy from high to low addresses.
  1585     // Check for and copy trailing byte
  1586     __ testl(byte_count, 1);
  1587     __ jcc(Assembler::zero, L_copy_2_bytes);
  1588     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
  1589     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
  1590     __ decrement(byte_count); // Adjust for possible trailing word
  1592     // Check for and copy trailing word
  1593   __ BIND(L_copy_2_bytes);
  1594     __ testl(byte_count, 2);
  1595     __ jcc(Assembler::zero, L_copy_4_bytes);
  1596     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
  1597     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
  1599     // Check for and copy trailing dword
  1600   __ BIND(L_copy_4_bytes);
  1601     __ testl(byte_count, 4);
  1602     __ jcc(Assembler::zero, L_copy_bytes);
  1603     __ movl(rax, Address(from, qword_count, Address::times_8));
  1604     __ movl(Address(to, qword_count, Address::times_8), rax);
  1605     __ jmp(L_copy_bytes);
  1607     // Copy trailing qwords
  1608   __ BIND(L_copy_8_bytes);
  1609     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
  1610     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
  1611     __ decrement(qword_count);
  1612     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1614     restore_arg_regs();
  1615     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
  1616     __ xorptr(rax, rax); // return 0
  1617     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1618     __ ret(0);
  1620     // Copy in multi-bytes chunks
  1621     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  1623     restore_arg_regs();
  1624     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
  1625     __ xorptr(rax, rax); // return 0
  1626     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1627     __ ret(0);
  1629     return start;
  1632   // Arguments:
  1633   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1634   //             ignored
  1635   //   name    - stub name string
  1636   //
  1637   // Inputs:
  1638   //   c_rarg0   - source array address
  1639   //   c_rarg1   - destination array address
  1640   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1641   //
  1642   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
  1643   // let the hardware handle it.  The two or four words within dwords
  1644   // or qwords that span cache line boundaries will still be loaded
  1645   // and stored atomically.
  1646   //
  1647   // Side Effects:
  1648   //   disjoint_short_copy_entry is set to the no-overlap entry point
  1649   //   used by generate_conjoint_short_copy().
  1650   //
  1651   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
  1652     __ align(CodeEntryAlignment);
  1653     StubCodeMark mark(this, "StubRoutines", name);
  1654     address start = __ pc();
  1656     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
  1657     const Register from        = rdi;  // source array address
  1658     const Register to          = rsi;  // destination array address
  1659     const Register count       = rdx;  // elements count
  1660     const Register word_count  = rcx;
  1661     const Register qword_count = count;
  1662     const Register end_from    = from; // source array end address
  1663     const Register end_to      = to;   // destination array end address
  1664     // End pointers are inclusive, and if count is not zero they point
  1665     // to the last unit copied:  end_to[0] := end_from[0]
  1667     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1668     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1670     if (entry != NULL) {
  1671       *entry = __ pc();
  1672       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1673       BLOCK_COMMENT("Entry:");
  1676     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1677                       // r9 and r10 may be used to save non-volatile registers
  1679     // 'from', 'to' and 'count' are now valid
  1680     __ movptr(word_count, count);
  1681     __ shrptr(count, 2); // count => qword_count
  1683     // Copy from low to high addresses.  Use 'to' as scratch.
  1684     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
  1685     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
  1686     __ negptr(qword_count);
  1687     __ jmp(L_copy_bytes);
  1689     // Copy trailing qwords
  1690   __ BIND(L_copy_8_bytes);
  1691     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
  1692     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
  1693     __ increment(qword_count);
  1694     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1696     // Original 'dest' is trashed, so we can't use it as a
  1697     // base register for a possible trailing word copy
  1699     // Check for and copy trailing dword
  1700   __ BIND(L_copy_4_bytes);
  1701     __ testl(word_count, 2);
  1702     __ jccb(Assembler::zero, L_copy_2_bytes);
  1703     __ movl(rax, Address(end_from, 8));
  1704     __ movl(Address(end_to, 8), rax);
  1706     __ addptr(end_from, 4);
  1707     __ addptr(end_to, 4);
  1709     // Check for and copy trailing word
  1710   __ BIND(L_copy_2_bytes);
  1711     __ testl(word_count, 1);
  1712     __ jccb(Assembler::zero, L_exit);
  1713     __ movw(rax, Address(end_from, 8));
  1714     __ movw(Address(end_to, 8), rax);
  1716   __ BIND(L_exit);
  1717     restore_arg_regs();
  1718     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
  1719     __ xorptr(rax, rax); // return 0
  1720     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1721     __ ret(0);
  1723     // Copy in multi-bytes chunks
  1724     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  1725     __ jmp(L_copy_4_bytes);
  1727     return start;
  1730   address generate_fill(BasicType t, bool aligned, const char *name) {
  1731     __ align(CodeEntryAlignment);
  1732     StubCodeMark mark(this, "StubRoutines", name);
  1733     address start = __ pc();
  1735     BLOCK_COMMENT("Entry:");
  1737     const Register to       = c_rarg0;  // source array address
  1738     const Register value    = c_rarg1;  // value
  1739     const Register count    = c_rarg2;  // elements count
  1741     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1743     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
  1745     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1746     __ ret(0);
  1747     return start;
  1750   // Arguments:
  1751   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1752   //             ignored
  1753   //   name    - stub name string
  1754   //
  1755   // Inputs:
  1756   //   c_rarg0   - source array address
  1757   //   c_rarg1   - destination array address
  1758   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1759   //
  1760   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
  1761   // let the hardware handle it.  The two or four words within dwords
  1762   // or qwords that span cache line boundaries will still be loaded
  1763   // and stored atomically.
  1764   //
  1765   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
  1766                                        address *entry, const char *name) {
  1767     __ align(CodeEntryAlignment);
  1768     StubCodeMark mark(this, "StubRoutines", name);
  1769     address start = __ pc();
  1771     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
  1772     const Register from        = rdi;  // source array address
  1773     const Register to          = rsi;  // destination array address
  1774     const Register count       = rdx;  // elements count
  1775     const Register word_count  = rcx;
  1776     const Register qword_count = count;
  1778     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1779     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1781     if (entry != NULL) {
  1782       *entry = __ pc();
  1783       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1784       BLOCK_COMMENT("Entry:");
  1787     array_overlap_test(nooverlap_target, Address::times_2);
  1788     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1789                       // r9 and r10 may be used to save non-volatile registers
  1791     // 'from', 'to' and 'count' are now valid
  1792     __ movptr(word_count, count);
  1793     __ shrptr(count, 2); // count => qword_count
  1795     // Copy from high to low addresses.  Use 'to' as scratch.
  1797     // Check for and copy trailing word
  1798     __ testl(word_count, 1);
  1799     __ jccb(Assembler::zero, L_copy_4_bytes);
  1800     __ movw(rax, Address(from, word_count, Address::times_2, -2));
  1801     __ movw(Address(to, word_count, Address::times_2, -2), rax);
  1803     // Check for and copy trailing dword
  1804   __ BIND(L_copy_4_bytes);
  1805     __ testl(word_count, 2);
  1806     __ jcc(Assembler::zero, L_copy_bytes);
  1807     __ movl(rax, Address(from, qword_count, Address::times_8));
  1808     __ movl(Address(to, qword_count, Address::times_8), rax);
  1809     __ jmp(L_copy_bytes);
  1811     // Copy trailing qwords
  1812   __ BIND(L_copy_8_bytes);
  1813     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
  1814     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
  1815     __ decrement(qword_count);
  1816     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1818     restore_arg_regs();
  1819     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
  1820     __ xorptr(rax, rax); // return 0
  1821     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1822     __ ret(0);
  1824     // Copy in multi-bytes chunks
  1825     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  1827     restore_arg_regs();
  1828     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
  1829     __ xorptr(rax, rax); // return 0
  1830     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1831     __ ret(0);
  1833     return start;
  1836   // Arguments:
  1837   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1838   //             ignored
  1839   //   is_oop  - true => oop array, so generate store check code
  1840   //   name    - stub name string
  1841   //
  1842   // Inputs:
  1843   //   c_rarg0   - source array address
  1844   //   c_rarg1   - destination array address
  1845   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1846   //
  1847   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
  1848   // the hardware handle it.  The two dwords within qwords that span
  1849   // cache line boundaries will still be loaded and stored atomicly.
  1850   //
  1851   // Side Effects:
  1852   //   disjoint_int_copy_entry is set to the no-overlap entry point
  1853   //   used by generate_conjoint_int_oop_copy().
  1854   //
  1855   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
  1856                                          const char *name, bool dest_uninitialized = false) {
  1857     __ align(CodeEntryAlignment);
  1858     StubCodeMark mark(this, "StubRoutines", name);
  1859     address start = __ pc();
  1861     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
  1862     const Register from        = rdi;  // source array address
  1863     const Register to          = rsi;  // destination array address
  1864     const Register count       = rdx;  // elements count
  1865     const Register dword_count = rcx;
  1866     const Register qword_count = count;
  1867     const Register end_from    = from; // source array end address
  1868     const Register end_to      = to;   // destination array end address
  1869     const Register saved_to    = r11;  // saved destination array address
  1870     // End pointers are inclusive, and if count is not zero they point
  1871     // to the last unit copied:  end_to[0] := end_from[0]
  1873     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1874     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1876     if (entry != NULL) {
  1877       *entry = __ pc();
  1878       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1879       BLOCK_COMMENT("Entry:");
  1882     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1883                       // r9 and r10 may be used to save non-volatile registers
  1884     if (is_oop) {
  1885       __ movq(saved_to, to);
  1886       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
  1889     // 'from', 'to' and 'count' are now valid
  1890     __ movptr(dword_count, count);
  1891     __ shrptr(count, 1); // count => qword_count
  1893     // Copy from low to high addresses.  Use 'to' as scratch.
  1894     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
  1895     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
  1896     __ negptr(qword_count);
  1897     __ jmp(L_copy_bytes);
  1899     // Copy trailing qwords
  1900   __ BIND(L_copy_8_bytes);
  1901     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
  1902     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
  1903     __ increment(qword_count);
  1904     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1906     // Check for and copy trailing dword
  1907   __ BIND(L_copy_4_bytes);
  1908     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
  1909     __ jccb(Assembler::zero, L_exit);
  1910     __ movl(rax, Address(end_from, 8));
  1911     __ movl(Address(end_to, 8), rax);
  1913   __ BIND(L_exit);
  1914     if (is_oop) {
  1915       gen_write_ref_array_post_barrier(saved_to, dword_count, rax);
  1917     restore_arg_regs();
  1918     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
  1919     __ xorptr(rax, rax); // return 0
  1920     __ leave(); // required for proper stackwalking of RuntimeStub frame
  1921     __ ret(0);
  1923     // Copy in multi-bytes chunks
  1924     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  1925     __ jmp(L_copy_4_bytes);
  1927     return start;
  1930   // Arguments:
  1931   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
  1932   //             ignored
  1933   //   is_oop  - true => oop array, so generate store check code
  1934   //   name    - stub name string
  1935   //
  1936   // Inputs:
  1937   //   c_rarg0   - source array address
  1938   //   c_rarg1   - destination array address
  1939   //   c_rarg2   - element count, treated as ssize_t, can be zero
  1940   //
  1941   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
  1942   // the hardware handle it.  The two dwords within qwords that span
  1943   // cache line boundaries will still be loaded and stored atomicly.
  1944   //
  1945   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
  1946                                          address *entry, const char *name,
  1947                                          bool dest_uninitialized = false) {
  1948     __ align(CodeEntryAlignment);
  1949     StubCodeMark mark(this, "StubRoutines", name);
  1950     address start = __ pc();
  1952     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
  1953     const Register from        = rdi;  // source array address
  1954     const Register to          = rsi;  // destination array address
  1955     const Register count       = rdx;  // elements count
  1956     const Register dword_count = rcx;
  1957     const Register qword_count = count;
  1959     __ enter(); // required for proper stackwalking of RuntimeStub frame
  1960     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  1962     if (entry != NULL) {
  1963       *entry = __ pc();
  1964        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  1965       BLOCK_COMMENT("Entry:");
  1968     array_overlap_test(nooverlap_target, Address::times_4);
  1969     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  1970                       // r9 and r10 may be used to save non-volatile registers
  1972     if (is_oop) {
  1973       // no registers are destroyed by this call
  1974       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
  1977     assert_clean_int(count, rax); // Make sure 'count' is clean int.
  1978     // 'from', 'to' and 'count' are now valid
  1979     __ movptr(dword_count, count);
  1980     __ shrptr(count, 1); // count => qword_count
  1982     // Copy from high to low addresses.  Use 'to' as scratch.
  1984     // Check for and copy trailing dword
  1985     __ testl(dword_count, 1);
  1986     __ jcc(Assembler::zero, L_copy_bytes);
  1987     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
  1988     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
  1989     __ jmp(L_copy_bytes);
  1991     // Copy trailing qwords
  1992   __ BIND(L_copy_8_bytes);
  1993     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
  1994     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
  1995     __ decrement(qword_count);
  1996     __ jcc(Assembler::notZero, L_copy_8_bytes);
  1998     if (is_oop) {
  1999       __ jmp(L_exit);
  2001     restore_arg_regs();
  2002     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
  2003     __ xorptr(rax, rax); // return 0
  2004     __ leave(); // required for proper stackwalking of RuntimeStub frame
  2005     __ ret(0);
  2007     // Copy in multi-bytes chunks
  2008     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  2010   __ BIND(L_exit);
  2011     if (is_oop) {
  2012       gen_write_ref_array_post_barrier(to, dword_count, rax);
  2014     restore_arg_regs();
  2015     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
  2016     __ xorptr(rax, rax); // return 0
  2017     __ leave(); // required for proper stackwalking of RuntimeStub frame
  2018     __ ret(0);
  2020     return start;
  2023   // Arguments:
  2024   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
  2025   //             ignored
  2026   //   is_oop  - true => oop array, so generate store check code
  2027   //   name    - stub name string
  2028   //
  2029   // Inputs:
  2030   //   c_rarg0   - source array address
  2031   //   c_rarg1   - destination array address
  2032   //   c_rarg2   - element count, treated as ssize_t, can be zero
  2033   //
  2034  // Side Effects:
  2035   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
  2036   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
  2037   //
  2038   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
  2039                                           const char *name, bool dest_uninitialized = false) {
  2040     __ align(CodeEntryAlignment);
  2041     StubCodeMark mark(this, "StubRoutines", name);
  2042     address start = __ pc();
  2044     Label L_copy_bytes, L_copy_8_bytes, L_exit;
  2045     const Register from        = rdi;  // source array address
  2046     const Register to          = rsi;  // destination array address
  2047     const Register qword_count = rdx;  // elements count
  2048     const Register end_from    = from; // source array end address
  2049     const Register end_to      = rcx;  // destination array end address
  2050     const Register saved_to    = to;
  2051     const Register saved_count = r11;
  2052     // End pointers are inclusive, and if count is not zero they point
  2053     // to the last unit copied:  end_to[0] := end_from[0]
  2055     __ enter(); // required for proper stackwalking of RuntimeStub frame
  2056     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
  2057     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  2059     if (entry != NULL) {
  2060       *entry = __ pc();
  2061       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  2062       BLOCK_COMMENT("Entry:");
  2065     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  2066                       // r9 and r10 may be used to save non-volatile registers
  2067     // 'from', 'to' and 'qword_count' are now valid
  2068     if (is_oop) {
  2069       // Save to and count for store barrier
  2070       __ movptr(saved_count, qword_count);
  2071       // no registers are destroyed by this call
  2072       gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);
  2075     // Copy from low to high addresses.  Use 'to' as scratch.
  2076     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
  2077     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
  2078     __ negptr(qword_count);
  2079     __ jmp(L_copy_bytes);
  2081     // Copy trailing qwords
  2082   __ BIND(L_copy_8_bytes);
  2083     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
  2084     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
  2085     __ increment(qword_count);
  2086     __ jcc(Assembler::notZero, L_copy_8_bytes);
  2088     if (is_oop) {
  2089       __ jmp(L_exit);
  2090     } else {
  2091       restore_arg_regs();
  2092       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
  2093       __ xorptr(rax, rax); // return 0
  2094       __ leave(); // required for proper stackwalking of RuntimeStub frame
  2095       __ ret(0);
  2098     // Copy in multi-bytes chunks
  2099     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  2101     if (is_oop) {
  2102     __ BIND(L_exit);
  2103       gen_write_ref_array_post_barrier(saved_to, saved_count, rax);
  2105     restore_arg_regs();
  2106     if (is_oop) {
  2107       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
  2108     } else {
  2109       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
  2111     __ xorptr(rax, rax); // return 0
  2112     __ leave(); // required for proper stackwalking of RuntimeStub frame
  2113     __ ret(0);
  2115     return start;
  2118   // Arguments:
  2119   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
  2120   //             ignored
  2121   //   is_oop  - true => oop array, so generate store check code
  2122   //   name    - stub name string
  2123   //
  2124   // Inputs:
  2125   //   c_rarg0   - source array address
  2126   //   c_rarg1   - destination array address
  2127   //   c_rarg2   - element count, treated as ssize_t, can be zero
  2128   //
  2129   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
  2130                                           address nooverlap_target, address *entry,
  2131                                           const char *name, bool dest_uninitialized = false) {
  2132     __ align(CodeEntryAlignment);
  2133     StubCodeMark mark(this, "StubRoutines", name);
  2134     address start = __ pc();
  2136     Label L_copy_bytes, L_copy_8_bytes, L_exit;
  2137     const Register from        = rdi;  // source array address
  2138     const Register to          = rsi;  // destination array address
  2139     const Register qword_count = rdx;  // elements count
  2140     const Register saved_count = rcx;
  2142     __ enter(); // required for proper stackwalking of RuntimeStub frame
  2143     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
  2145     if (entry != NULL) {
  2146       *entry = __ pc();
  2147       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
  2148       BLOCK_COMMENT("Entry:");
  2151     array_overlap_test(nooverlap_target, Address::times_8);
  2152     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
  2153                       // r9 and r10 may be used to save non-volatile registers
  2154     // 'from', 'to' and 'qword_count' are now valid
  2155     if (is_oop) {
  2156       // Save to and count for store barrier
  2157       __ movptr(saved_count, qword_count);
  2158       // No registers are destroyed by this call
  2159       gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);
  2162     __ jmp(L_copy_bytes);
  2164     // Copy trailing qwords
  2165   __ BIND(L_copy_8_bytes);
  2166     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
  2167     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
  2168     __ decrement(qword_count);
  2169     __ jcc(Assembler::notZero, L_copy_8_bytes);
  2171     if (is_oop) {
  2172       __ jmp(L_exit);
  2173     } else {
  2174       restore_arg_regs();
  2175       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
  2176       __ xorptr(rax, rax); // return 0
  2177       __ leave(); // required for proper stackwalking of RuntimeStub frame
  2178       __ ret(0);
  2181     // Copy in multi-bytes chunks
  2182     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
  2184     if (is_oop) {
  2185     __ BIND(L_exit);
  2186       gen_write_ref_array_post_barrier(to, saved_count, rax);
  2188     restore_arg_regs();
  2189     if (is_oop) {
  2190       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
  2191     } else {
  2192       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
  2194     __ xorptr(rax, rax); // return 0
  2195     __ leave(); // required for proper stackwalking of RuntimeStub frame
  2196     __ ret(0);
  2198     return start;
  2202   // Helper for generating a dynamic type check.
  2203   // Smashes no registers.
  2204   void generate_type_check(Register sub_klass,
  2205                            Register super_check_offset,
  2206                            Register super_klass,
  2207                            Label& L_success) {
  2208     assert_different_registers(sub_klass, super_check_offset, super_klass);
  2210     BLOCK_COMMENT("type_check:");
  2212     Label L_miss;
  2214     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
  2215                                      super_check_offset);
  2216     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
  2218     // Fall through on failure!
  2219     __ BIND(L_miss);
  2222   //
  2223   //  Generate checkcasting array copy stub
  2224   //
  2225   //  Input:
  2226   //    c_rarg0   - source array address
  2227   //    c_rarg1   - destination array address
  2228   //    c_rarg2   - element count, treated as ssize_t, can be zero
  2229   //    c_rarg3   - size_t ckoff (super_check_offset)
  2230   // not Win64
  2231   //    c_rarg4   - oop ckval (super_klass)
  2232   // Win64
  2233   //    rsp+40    - oop ckval (super_klass)
  2234   //
  2235   //  Output:
  2236   //    rax ==  0  -  success
  2237   //    rax == -1^K - failure, where K is partial transfer count
  2238   //
  2239   address generate_checkcast_copy(const char *name, address *entry,
  2240                                   bool dest_uninitialized = false) {
  2242     Label L_load_element, L_store_element, L_do_card_marks, L_done;
  2244     // Input registers (after setup_arg_regs)
  2245     const Register from        = rdi;   // source array address
  2246     const Register to          = rsi;   // destination array address
  2247     const Register length      = rdx;   // elements count
  2248     const Register ckoff       = rcx;   // super_check_offset
  2249     const Register ckval       = r8;    // super_klass
  2251     // Registers used as temps (r13, r14 are save-on-entry)
  2252     const Register end_from    = from;  // source array end address
  2253     const Register end_to      = r13;   // destination array end address
  2254     const Register count       = rdx;   // -(count_remaining)
  2255     const Register r14_length  = r14;   // saved copy of length
  2256     // End pointers are inclusive, and if length is not zero they point
  2257     // to the last unit copied:  end_to[0] := end_from[0]
  2259     const Register rax_oop    = rax;    // actual oop copied
  2260     const Register r11_klass  = r11;    // oop._klass
  2262     //---------------------------------------------------------------
  2263     // Assembler stub will be used for this call to arraycopy
  2264     // if the two arrays are subtypes of Object[] but the
  2265     // destination array type is not equal to or a supertype
  2266     // of the source type.  Each element must be separately
  2267     // checked.
  2269     __ align(CodeEntryAlignment);
  2270     StubCodeMark mark(this, "StubRoutines", name);
  2271     address start = __ pc();
  2273     __ enter(); // required for proper stackwalking of RuntimeStub frame
  2275 #ifdef ASSERT
  2276     // caller guarantees that the arrays really are different
  2277     // otherwise, we would have to make conjoint checks
  2278     { Label L;
  2279       array_overlap_test(L, TIMES_OOP);
  2280       __ stop("checkcast_copy within a single array");
  2281       __ bind(L);
  2283 #endif //ASSERT
  2285     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
  2286                        // ckoff => rcx, ckval => r8
  2287                        // r9 and r10 may be used to save non-volatile registers
  2288 #ifdef _WIN64
  2289     // last argument (#4) is on stack on Win64
  2290     __ movptr(ckval, Address(rsp, 6 * wordSize));
  2291 #endif
  2293     // Caller of this entry point must set up the argument registers.
  2294     if (entry != NULL) {
  2295       *entry = __ pc();
  2296       BLOCK_COMMENT("Entry:");
  2299     // allocate spill slots for r13, r14
  2300     enum {
  2301       saved_r13_offset,
  2302       saved_r14_offset,
  2303       saved_rbp_offset
  2304     };
  2305     __ subptr(rsp, saved_rbp_offset * wordSize);
  2306     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
  2307     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
  2309     // check that int operands are properly extended to size_t
  2310     assert_clean_int(length, rax);
  2311     assert_clean_int(ckoff, rax);
  2313 #ifdef ASSERT
  2314     BLOCK_COMMENT("assert consistent ckoff/ckval");
  2315     // The ckoff and ckval must be mutually consistent,
  2316     // even though caller generates both.
  2317     { Label L;
  2318       int sco_offset = in_bytes(Klass::super_check_offset_offset());
  2319       __ cmpl(ckoff, Address(ckval, sco_offset));
  2320       __ jcc(Assembler::equal, L);
  2321       __ stop("super_check_offset inconsistent");
  2322       __ bind(L);
  2324 #endif //ASSERT
  2326     // Loop-invariant addresses.  They are exclusive end pointers.
  2327     Address end_from_addr(from, length, TIMES_OOP, 0);
  2328     Address   end_to_addr(to,   length, TIMES_OOP, 0);
  2329     // Loop-variant addresses.  They assume post-incremented count < 0.
  2330     Address from_element_addr(end_from, count, TIMES_OOP, 0);
  2331     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
  2333     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
  2335     // Copy from low to high addresses, indexed from the end of each array.
  2336     __ lea(end_from, end_from_addr);
  2337     __ lea(end_to,   end_to_addr);
  2338     __ movptr(r14_length, length);        // save a copy of the length
  2339     assert(length == count, "");          // else fix next line:
  2340     __ negptr(count);                     // negate and test the length
  2341     __ jcc(Assembler::notZero, L_load_element);
  2343     // Empty array:  Nothing to do.
  2344     __ xorptr(rax, rax);                  // return 0 on (trivial) success
  2345     __ jmp(L_done);
  2347     // ======== begin loop ========
  2348     // (Loop is rotated; its entry is L_load_element.)
  2349     // Loop control:
  2350     //   for (count = -count; count != 0; count++)
  2351     // Base pointers src, dst are biased by 8*(count-1),to last element.
  2352     __ align(OptoLoopAlignment);
  2354     __ BIND(L_store_element);
  2355     __ store_heap_oop(to_element_addr, rax_oop);  // store the oop
  2356     __ increment(count);               // increment the count toward zero
  2357     __ jcc(Assembler::zero, L_do_card_marks);
  2359     // ======== loop entry is here ========
  2360     __ BIND(L_load_element);
  2361     __ load_heap_oop(rax_oop, from_element_addr); // load the oop
  2362     __ testptr(rax_oop, rax_oop);
  2363     __ jcc(Assembler::zero, L_store_element);
  2365     __ load_klass(r11_klass, rax_oop);// query the object klass
  2366     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
  2367     // ======== end loop ========
  2369     // It was a real error; we must depend on the caller to finish the job.
  2370     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
  2371     // Emit GC store barriers for the oops we have copied (r14 + rdx),
  2372     // and report their number to the caller.
  2373     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
  2374     Label L_post_barrier;
  2375     __ addptr(r14_length, count);     // K = (original - remaining) oops
  2376     __ movptr(rax, r14_length);       // save the value
  2377     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
  2378     __ jccb(Assembler::notZero, L_post_barrier);
  2379     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
  2381     // Come here on success only.
  2382     __ BIND(L_do_card_marks);
  2383     __ xorptr(rax, rax);              // return 0 on success
  2385     __ BIND(L_post_barrier);
  2386     gen_write_ref_array_post_barrier(to, r14_length, rscratch1);
  2388     // Common exit point (success or failure).
  2389     __ BIND(L_done);
  2390     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
  2391     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
  2392     restore_arg_regs();
  2393     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
  2394     __ leave(); // required for proper stackwalking of RuntimeStub frame
  2395     __ ret(0);
  2397     return start;
  2400   //
  2401   //  Generate 'unsafe' array copy stub
  2402   //  Though just as safe as the other stubs, it takes an unscaled
  2403   //  size_t argument instead of an element count.
  2404   //
  2405   //  Input:
  2406   //    c_rarg0   - source array address
  2407   //    c_rarg1   - destination array address
  2408   //    c_rarg2   - byte count, treated as ssize_t, can be zero
  2409   //
  2410   // Examines the alignment of the operands and dispatches
  2411   // to a long, int, short, or byte copy loop.
  2412   //
  2413   address generate_unsafe_copy(const char *name,
  2414                                address byte_copy_entry, address short_copy_entry,
  2415                                address int_copy_entry, address long_copy_entry) {
  2417     Label L_long_aligned, L_int_aligned, L_short_aligned;
  2419     // Input registers (before setup_arg_regs)
  2420     const Register from        = c_rarg0;  // source array address
  2421     const Register to          = c_rarg1;  // destination array address
  2422     const Register size        = c_rarg2;  // byte count (size_t)
  2424     // Register used as a temp
  2425     const Register bits        = rax;      // test copy of low bits
  2427     __ align(CodeEntryAlignment);
  2428     StubCodeMark mark(this, "StubRoutines", name);
  2429     address start = __ pc();
  2431     __ enter(); // required for proper stackwalking of RuntimeStub frame
  2433     // bump this on entry, not on exit:
  2434     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
  2436     __ mov(bits, from);
  2437     __ orptr(bits, to);
  2438     __ orptr(bits, size);
  2440     __ testb(bits, BytesPerLong-1);
  2441     __ jccb(Assembler::zero, L_long_aligned);
  2443     __ testb(bits, BytesPerInt-1);
  2444     __ jccb(Assembler::zero, L_int_aligned);
  2446     __ testb(bits, BytesPerShort-1);
  2447     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
  2449     __ BIND(L_short_aligned);
  2450     __ shrptr(size, LogBytesPerShort); // size => short_count
  2451     __ jump(RuntimeAddress(short_copy_entry));
  2453     __ BIND(L_int_aligned);
  2454     __ shrptr(size, LogBytesPerInt); // size => int_count
  2455     __ jump(RuntimeAddress(int_copy_entry));
  2457     __ BIND(L_long_aligned);
  2458     __ shrptr(size, LogBytesPerLong); // size => qword_count
  2459     __ jump(RuntimeAddress(long_copy_entry));
  2461     return start;
  2464   // Perform range checks on the proposed arraycopy.
  2465   // Kills temp, but nothing else.
  2466   // Also, clean the sign bits of src_pos and dst_pos.
  2467   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
  2468                               Register src_pos, // source position (c_rarg1)
  2469                               Register dst,     // destination array oo (c_rarg2)
  2470                               Register dst_pos, // destination position (c_rarg3)
  2471                               Register length,
  2472                               Register temp,
  2473                               Label& L_failed) {
  2474     BLOCK_COMMENT("arraycopy_range_checks:");
  2476     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
  2477     __ movl(temp, length);
  2478     __ addl(temp, src_pos);             // src_pos + length
  2479     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
  2480     __ jcc(Assembler::above, L_failed);
  2482     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
  2483     __ movl(temp, length);
  2484     __ addl(temp, dst_pos);             // dst_pos + length
  2485     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
  2486     __ jcc(Assembler::above, L_failed);
  2488     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
  2489     // Move with sign extension can be used since they are positive.
  2490     __ movslq(src_pos, src_pos);
  2491     __ movslq(dst_pos, dst_pos);
  2493     BLOCK_COMMENT("arraycopy_range_checks done");
  2496   //
  2497   //  Generate generic array copy stubs
  2498   //
  2499   //  Input:
  2500   //    c_rarg0    -  src oop
  2501   //    c_rarg1    -  src_pos (32-bits)
  2502   //    c_rarg2    -  dst oop
  2503   //    c_rarg3    -  dst_pos (32-bits)
  2504   // not Win64
  2505   //    c_rarg4    -  element count (32-bits)
  2506   // Win64
  2507   //    rsp+40     -  element count (32-bits)
  2508   //
  2509   //  Output:
  2510   //    rax ==  0  -  success
  2511   //    rax == -1^K - failure, where K is partial transfer count
  2512   //
  2513   address generate_generic_copy(const char *name,
  2514                                 address byte_copy_entry, address short_copy_entry,
  2515                                 address int_copy_entry, address oop_copy_entry,
  2516                                 address long_copy_entry, address checkcast_copy_entry) {
  2518     Label L_failed, L_failed_0, L_objArray;
  2519     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
  2521     // Input registers
  2522     const Register src        = c_rarg0;  // source array oop
  2523     const Register src_pos    = c_rarg1;  // source position
  2524     const Register dst        = c_rarg2;  // destination array oop
  2525     const Register dst_pos    = c_rarg3;  // destination position
  2526 #ifndef _WIN64
  2527     const Register length     = c_rarg4;
  2528 #else
  2529     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
  2530 #endif
  2532     { int modulus = CodeEntryAlignment;
  2533       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
  2534       int advance = target - (__ offset() % modulus);
  2535       if (advance < 0)  advance += modulus;
  2536       if (advance > 0)  __ nop(advance);
  2538     StubCodeMark mark(this, "StubRoutines", name);
  2540     // Short-hop target to L_failed.  Makes for denser prologue code.
  2541     __ BIND(L_failed_0);
  2542     __ jmp(L_failed);
  2543     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
  2545     __ align(CodeEntryAlignment);
  2546     address start = __ pc();
  2548     __ enter(); // required for proper stackwalking of RuntimeStub frame
  2550     // bump this on entry, not on exit:
  2551     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
  2553     //-----------------------------------------------------------------------
  2554     // Assembler stub will be used for this call to arraycopy
  2555     // if the following conditions are met:
  2556     //
  2557     // (1) src and dst must not be null.
  2558     // (2) src_pos must not be negative.
  2559     // (3) dst_pos must not be negative.
  2560     // (4) length  must not be negative.
  2561     // (5) src klass and dst klass should be the same and not NULL.
  2562     // (6) src and dst should be arrays.
  2563     // (7) src_pos + length must not exceed length of src.
  2564     // (8) dst_pos + length must not exceed length of dst.
  2565     //
  2567     //  if (src == NULL) return -1;
  2568     __ testptr(src, src);         // src oop
  2569     size_t j1off = __ offset();
  2570     __ jccb(Assembler::zero, L_failed_0);
  2572     //  if (src_pos < 0) return -1;
  2573     __ testl(src_pos, src_pos); // src_pos (32-bits)
  2574     __ jccb(Assembler::negative, L_failed_0);
  2576     //  if (dst == NULL) return -1;
  2577     __ testptr(dst, dst);         // dst oop
  2578     __ jccb(Assembler::zero, L_failed_0);
  2580     //  if (dst_pos < 0) return -1;
  2581     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
  2582     size_t j4off = __ offset();
  2583     __ jccb(Assembler::negative, L_failed_0);
  2585     // The first four tests are very dense code,
  2586     // but not quite dense enough to put four
  2587     // jumps in a 16-byte instruction fetch buffer.
  2588     // That's good, because some branch predicters
  2589     // do not like jumps so close together.
  2590     // Make sure of this.
  2591     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
  2593     // registers used as temp
  2594     const Register r11_length    = r11; // elements count to copy
  2595     const Register r10_src_klass = r10; // array klass
  2597     //  if (length < 0) return -1;
  2598     __ movl(r11_length, length);        // length (elements count, 32-bits value)
  2599     __ testl(r11_length, r11_length);
  2600     __ jccb(Assembler::negative, L_failed_0);
  2602     __ load_klass(r10_src_klass, src);
  2603 #ifdef ASSERT
  2604     //  assert(src->klass() != NULL);
  2606       BLOCK_COMMENT("assert klasses not null {");
  2607       Label L1, L2;
  2608       __ testptr(r10_src_klass, r10_src_klass);
  2609       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
  2610       __ bind(L1);
  2611       __ stop("broken null klass");
  2612       __ bind(L2);
  2613       __ load_klass(rax, dst);
  2614       __ cmpq(rax, 0);
  2615       __ jcc(Assembler::equal, L1);     // this would be broken also
  2616       BLOCK_COMMENT("} assert klasses not null done");
  2618 #endif
  2620     // Load layout helper (32-bits)
  2621     //
  2622     //  |array_tag|     | header_size | element_type |     |log2_element_size|
  2623     // 32        30    24            16              8     2                 0
  2624     //
  2625     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
  2626     //
  2628     const int lh_offset = in_bytes(Klass::layout_helper_offset());
  2630     // Handle objArrays completely differently...
  2631     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
  2632     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
  2633     __ jcc(Assembler::equal, L_objArray);
  2635     //  if (src->klass() != dst->klass()) return -1;
  2636     __ load_klass(rax, dst);
  2637     __ cmpq(r10_src_klass, rax);
  2638     __ jcc(Assembler::notEqual, L_failed);
  2640     const Register rax_lh = rax;  // layout helper
  2641     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
  2643     //  if (!src->is_Array()) return -1;
  2644     __ cmpl(rax_lh, Klass::_lh_neutral_value);
  2645     __ jcc(Assembler::greaterEqual, L_failed);
  2647     // At this point, it is known to be a typeArray (array_tag 0x3).
  2648 #ifdef ASSERT
  2650       BLOCK_COMMENT("assert primitive array {");
  2651       Label L;
  2652       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
  2653       __ jcc(Assembler::greaterEqual, L);
  2654       __ stop("must be a primitive array");
  2655       __ bind(L);
  2656       BLOCK_COMMENT("} assert primitive array done");
  2658 #endif
  2660     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
  2661                            r10, L_failed);
  2663     // TypeArrayKlass
  2664     //
  2665     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
  2666     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
  2667     //
  2669     const Register r10_offset = r10;    // array offset
  2670     const Register rax_elsize = rax_lh; // element size
  2672     __ movl(r10_offset, rax_lh);
  2673     __ shrl(r10_offset, Klass::_lh_header_size_shift);
  2674     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
  2675     __ addptr(src, r10_offset);           // src array offset
  2676     __ addptr(dst, r10_offset);           // dst array offset
  2677     BLOCK_COMMENT("choose copy loop based on element size");
  2678     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
  2680     // next registers should be set before the jump to corresponding stub
  2681     const Register from     = c_rarg0;  // source array address
  2682     const Register to       = c_rarg1;  // destination array address
  2683     const Register count    = c_rarg2;  // elements count
  2685     // 'from', 'to', 'count' registers should be set in such order
  2686     // since they are the same as 'src', 'src_pos', 'dst'.
  2688   __ BIND(L_copy_bytes);
  2689     __ cmpl(rax_elsize, 0);
  2690     __ jccb(Assembler::notEqual, L_copy_shorts);
  2691     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
  2692     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
  2693     __ movl2ptr(count, r11_length); // length
  2694     __ jump(RuntimeAddress(byte_copy_entry));
  2696   __ BIND(L_copy_shorts);
  2697     __ cmpl(rax_elsize, LogBytesPerShort);
  2698     __ jccb(Assembler::notEqual, L_copy_ints);
  2699     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
  2700     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
  2701     __ movl2ptr(count, r11_length); // length
  2702     __ jump(RuntimeAddress(short_copy_entry));
  2704   __ BIND(L_copy_ints);
  2705     __ cmpl(rax_elsize, LogBytesPerInt);
  2706     __ jccb(Assembler::notEqual, L_copy_longs);
  2707     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
  2708     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
  2709     __ movl2ptr(count, r11_length); // length
  2710     __ jump(RuntimeAddress(int_copy_entry));
  2712   __ BIND(L_copy_longs);
  2713 #ifdef ASSERT
  2715       BLOCK_COMMENT("assert long copy {");
  2716       Label L;
  2717       __ cmpl(rax_elsize, LogBytesPerLong);
  2718       __ jcc(Assembler::equal, L);
  2719       __ stop("must be long copy, but elsize is wrong");
  2720       __ bind(L);
  2721       BLOCK_COMMENT("} assert long copy done");
  2723 #endif
  2724     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
  2725     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
  2726     __ movl2ptr(count, r11_length); // length
  2727     __ jump(RuntimeAddress(long_copy_entry));
  2729     // ObjArrayKlass
  2730   __ BIND(L_objArray);
  2731     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
  2733     Label L_plain_copy, L_checkcast_copy;
  2734     //  test array classes for subtyping
  2735     __ load_klass(rax, dst);
  2736     __ cmpq(r10_src_klass, rax); // usual case is exact equality
  2737     __ jcc(Assembler::notEqual, L_checkcast_copy);
  2739     // Identically typed arrays can be copied without element-wise checks.
  2740     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
  2741                            r10, L_failed);
  2743     __ lea(from, Address(src, src_pos, TIMES_OOP,
  2744                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
  2745     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
  2746                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
  2747     __ movl2ptr(count, r11_length); // length
  2748   __ BIND(L_plain_copy);
  2749     __ jump(RuntimeAddress(oop_copy_entry));
  2751   __ BIND(L_checkcast_copy);
  2752     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
  2754       // Before looking at dst.length, make sure dst is also an objArray.
  2755       __ cmpl(Address(rax, lh_offset), objArray_lh);
  2756       __ jcc(Assembler::notEqual, L_failed);
  2758       // It is safe to examine both src.length and dst.length.
  2759       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
  2760                              rax, L_failed);
  2762       const Register r11_dst_klass = r11;
  2763       __ load_klass(r11_dst_klass, dst); // reload
  2765       // Marshal the base address arguments now, freeing registers.
  2766       __ lea(from, Address(src, src_pos, TIMES_OOP,
  2767                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
  2768       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
  2769                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
  2770       __ movl(count, length);           // length (reloaded)
  2771       Register sco_temp = c_rarg3;      // this register is free now
  2772       assert_different_registers(from, to, count, sco_temp,
  2773                                  r11_dst_klass, r10_src_klass);
  2774       assert_clean_int(count, sco_temp);
  2776       // Generate the type check.
  2777       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
  2778       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
  2779       assert_clean_int(sco_temp, rax);
  2780       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
  2782       // Fetch destination element klass from the ObjArrayKlass header.
  2783       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
  2784       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
  2785       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
  2786       assert_clean_int(sco_temp, rax);
  2788       // the checkcast_copy loop needs two extra arguments:
  2789       assert(c_rarg3 == sco_temp, "#3 already in place");
  2790       // Set up arguments for checkcast_copy_entry.
  2791       setup_arg_regs(4);
  2792       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
  2793       __ jump(RuntimeAddress(checkcast_copy_entry));
  2796   __ BIND(L_failed);
  2797     __ xorptr(rax, rax);
  2798     __ notptr(rax); // return -1
  2799     __ leave();   // required for proper stackwalking of RuntimeStub frame
  2800     __ ret(0);
  2802     return start;
  2805   void generate_arraycopy_stubs() {
  2806     address entry;
  2807     address entry_jbyte_arraycopy;
  2808     address entry_jshort_arraycopy;
  2809     address entry_jint_arraycopy;
  2810     address entry_oop_arraycopy;
  2811     address entry_jlong_arraycopy;
  2812     address entry_checkcast_arraycopy;
  2814     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
  2815                                                                            "jbyte_disjoint_arraycopy");
  2816     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
  2817                                                                            "jbyte_arraycopy");
  2819     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
  2820                                                                             "jshort_disjoint_arraycopy");
  2821     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
  2822                                                                             "jshort_arraycopy");
  2824     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
  2825                                                                               "jint_disjoint_arraycopy");
  2826     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
  2827                                                                               &entry_jint_arraycopy, "jint_arraycopy");
  2829     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
  2830                                                                                "jlong_disjoint_arraycopy");
  2831     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
  2832                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
  2835     if (UseCompressedOops) {
  2836       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
  2837                                                                               "oop_disjoint_arraycopy");
  2838       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
  2839                                                                               &entry_oop_arraycopy, "oop_arraycopy");
  2840       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
  2841                                                                                      "oop_disjoint_arraycopy_uninit",
  2842                                                                                      /*dest_uninitialized*/true);
  2843       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
  2844                                                                                      NULL, "oop_arraycopy_uninit",
  2845                                                                                      /*dest_uninitialized*/true);
  2846     } else {
  2847       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
  2848                                                                                "oop_disjoint_arraycopy");
  2849       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
  2850                                                                                &entry_oop_arraycopy, "oop_arraycopy");
  2851       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
  2852                                                                                       "oop_disjoint_arraycopy_uninit",
  2853                                                                                       /*dest_uninitialized*/true);
  2854       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
  2855                                                                                       NULL, "oop_arraycopy_uninit",
  2856                                                                                       /*dest_uninitialized*/true);
  2859     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
  2860     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
  2861                                                                         /*dest_uninitialized*/true);
  2863     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
  2864                                                               entry_jbyte_arraycopy,
  2865                                                               entry_jshort_arraycopy,
  2866                                                               entry_jint_arraycopy,
  2867                                                               entry_jlong_arraycopy);
  2868     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
  2869                                                                entry_jbyte_arraycopy,
  2870                                                                entry_jshort_arraycopy,
  2871                                                                entry_jint_arraycopy,
  2872                                                                entry_oop_arraycopy,
  2873                                                                entry_jlong_arraycopy,
  2874                                                                entry_checkcast_arraycopy);
  2876     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
  2877     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
  2878     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
  2879     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
  2880     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
  2881     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
  2883     // We don't generate specialized code for HeapWord-aligned source
  2884     // arrays, so just use the code we've already generated
  2885     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
  2886     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
  2888     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
  2889     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
  2891     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
  2892     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
  2894     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
  2895     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
  2897     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
  2898     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
  2900     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
  2901     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
  2904   void generate_math_stubs() {
  2906       StubCodeMark mark(this, "StubRoutines", "log");
  2907       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
  2909       __ subq(rsp, 8);
  2910       __ movdbl(Address(rsp, 0), xmm0);
  2911       __ fld_d(Address(rsp, 0));
  2912       __ flog();
  2913       __ fstp_d(Address(rsp, 0));
  2914       __ movdbl(xmm0, Address(rsp, 0));
  2915       __ addq(rsp, 8);
  2916       __ ret(0);
  2919       StubCodeMark mark(this, "StubRoutines", "log10");
  2920       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
  2922       __ subq(rsp, 8);
  2923       __ movdbl(Address(rsp, 0), xmm0);
  2924       __ fld_d(Address(rsp, 0));
  2925       __ flog10();
  2926       __ fstp_d(Address(rsp, 0));
  2927       __ movdbl(xmm0, Address(rsp, 0));
  2928       __ addq(rsp, 8);
  2929       __ ret(0);
  2932       StubCodeMark mark(this, "StubRoutines", "sin");
  2933       StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
  2935       __ subq(rsp, 8);
  2936       __ movdbl(Address(rsp, 0), xmm0);
  2937       __ fld_d(Address(rsp, 0));
  2938       __ trigfunc('s');
  2939       __ fstp_d(Address(rsp, 0));
  2940       __ movdbl(xmm0, Address(rsp, 0));
  2941       __ addq(rsp, 8);
  2942       __ ret(0);
  2945       StubCodeMark mark(this, "StubRoutines", "cos");
  2946       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
  2948       __ subq(rsp, 8);
  2949       __ movdbl(Address(rsp, 0), xmm0);
  2950       __ fld_d(Address(rsp, 0));
  2951       __ trigfunc('c');
  2952       __ fstp_d(Address(rsp, 0));
  2953       __ movdbl(xmm0, Address(rsp, 0));
  2954       __ addq(rsp, 8);
  2955       __ ret(0);
  2958       StubCodeMark mark(this, "StubRoutines", "tan");
  2959       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
  2961       __ subq(rsp, 8);
  2962       __ movdbl(Address(rsp, 0), xmm0);
  2963       __ fld_d(Address(rsp, 0));
  2964       __ trigfunc('t');
  2965       __ fstp_d(Address(rsp, 0));
  2966       __ movdbl(xmm0, Address(rsp, 0));
  2967       __ addq(rsp, 8);
  2968       __ ret(0);
  2971       StubCodeMark mark(this, "StubRoutines", "exp");
  2972       StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
  2974       __ subq(rsp, 8);
  2975       __ movdbl(Address(rsp, 0), xmm0);
  2976       __ fld_d(Address(rsp, 0));
  2977       __ exp_with_fallback(0);
  2978       __ fstp_d(Address(rsp, 0));
  2979       __ movdbl(xmm0, Address(rsp, 0));
  2980       __ addq(rsp, 8);
  2981       __ ret(0);
  2984       StubCodeMark mark(this, "StubRoutines", "pow");
  2985       StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
  2987       __ subq(rsp, 8);
  2988       __ movdbl(Address(rsp, 0), xmm1);
  2989       __ fld_d(Address(rsp, 0));
  2990       __ movdbl(Address(rsp, 0), xmm0);
  2991       __ fld_d(Address(rsp, 0));
  2992       __ pow_with_fallback(0);
  2993       __ fstp_d(Address(rsp, 0));
  2994       __ movdbl(xmm0, Address(rsp, 0));
  2995       __ addq(rsp, 8);
  2996       __ ret(0);
  3000   // AES intrinsic stubs
  3001   enum {AESBlockSize = 16};
  3003   address generate_key_shuffle_mask() {
  3004     __ align(16);
  3005     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
  3006     address start = __ pc();
  3007     __ emit_data64( 0x0405060700010203, relocInfo::none );
  3008     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
  3009     return start;
  3012   // Utility routine for loading a 128-bit key word in little endian format
  3013   // can optionally specify that the shuffle mask is already in an xmmregister
  3014   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
  3015     __ movdqu(xmmdst, Address(key, offset));
  3016     if (xmm_shuf_mask != NULL) {
  3017       __ pshufb(xmmdst, xmm_shuf_mask);
  3018     } else {
  3019       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
  3023   // Arguments:
  3024   //
  3025   // Inputs:
  3026   //   c_rarg0   - source byte array address
  3027   //   c_rarg1   - destination byte array address
  3028   //   c_rarg2   - K (key) in little endian int array
  3029   //
  3030   address generate_aescrypt_encryptBlock() {
  3031     assert(UseAES, "need AES instructions and misaligned SSE support");
  3032     __ align(CodeEntryAlignment);
  3033     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
  3034     Label L_doLast;
  3035     address start = __ pc();
  3037     const Register from        = c_rarg0;  // source array address
  3038     const Register to          = c_rarg1;  // destination array address
  3039     const Register key         = c_rarg2;  // key array address
  3040     const Register keylen      = rax;
  3042     const XMMRegister xmm_result = xmm0;
  3043     const XMMRegister xmm_key_shuf_mask = xmm1;
  3044     // On win64 xmm6-xmm15 must be preserved so don't use them.
  3045     const XMMRegister xmm_temp1  = xmm2;
  3046     const XMMRegister xmm_temp2  = xmm3;
  3047     const XMMRegister xmm_temp3  = xmm4;
  3048     const XMMRegister xmm_temp4  = xmm5;
  3050     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3052     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
  3053     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
  3055     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
  3056     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
  3058     // For encryption, the java expanded key ordering is just what we need
  3059     // we don't know if the key is aligned, hence not using load-execute form
  3061     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
  3062     __ pxor(xmm_result, xmm_temp1);
  3064     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
  3065     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
  3066     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
  3067     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
  3069     __ aesenc(xmm_result, xmm_temp1);
  3070     __ aesenc(xmm_result, xmm_temp2);
  3071     __ aesenc(xmm_result, xmm_temp3);
  3072     __ aesenc(xmm_result, xmm_temp4);
  3074     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
  3075     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
  3076     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
  3077     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
  3079     __ aesenc(xmm_result, xmm_temp1);
  3080     __ aesenc(xmm_result, xmm_temp2);
  3081     __ aesenc(xmm_result, xmm_temp3);
  3082     __ aesenc(xmm_result, xmm_temp4);
  3084     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
  3085     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
  3087     __ cmpl(keylen, 44);
  3088     __ jccb(Assembler::equal, L_doLast);
  3090     __ aesenc(xmm_result, xmm_temp1);
  3091     __ aesenc(xmm_result, xmm_temp2);
  3093     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
  3094     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
  3096     __ cmpl(keylen, 52);
  3097     __ jccb(Assembler::equal, L_doLast);
  3099     __ aesenc(xmm_result, xmm_temp1);
  3100     __ aesenc(xmm_result, xmm_temp2);
  3102     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
  3103     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
  3105     __ BIND(L_doLast);
  3106     __ aesenc(xmm_result, xmm_temp1);
  3107     __ aesenclast(xmm_result, xmm_temp2);
  3108     __ movdqu(Address(to, 0), xmm_result);        // store the result
  3109     __ xorptr(rax, rax); // return 0
  3110     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3111     __ ret(0);
  3113     return start;
  3117   // Arguments:
  3118   //
  3119   // Inputs:
  3120   //   c_rarg0   - source byte array address
  3121   //   c_rarg1   - destination byte array address
  3122   //   c_rarg2   - K (key) in little endian int array
  3123   //
  3124   address generate_aescrypt_decryptBlock() {
  3125     assert(UseAES, "need AES instructions and misaligned SSE support");
  3126     __ align(CodeEntryAlignment);
  3127     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
  3128     Label L_doLast;
  3129     address start = __ pc();
  3131     const Register from        = c_rarg0;  // source array address
  3132     const Register to          = c_rarg1;  // destination array address
  3133     const Register key         = c_rarg2;  // key array address
  3134     const Register keylen      = rax;
  3136     const XMMRegister xmm_result = xmm0;
  3137     const XMMRegister xmm_key_shuf_mask = xmm1;
  3138     // On win64 xmm6-xmm15 must be preserved so don't use them.
  3139     const XMMRegister xmm_temp1  = xmm2;
  3140     const XMMRegister xmm_temp2  = xmm3;
  3141     const XMMRegister xmm_temp3  = xmm4;
  3142     const XMMRegister xmm_temp4  = xmm5;
  3144     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3146     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
  3147     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
  3149     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
  3150     __ movdqu(xmm_result, Address(from, 0));
  3152     // for decryption java expanded key ordering is rotated one position from what we want
  3153     // so we start from 0x10 here and hit 0x00 last
  3154     // we don't know if the key is aligned, hence not using load-execute form
  3155     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
  3156     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
  3157     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
  3158     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
  3160     __ pxor  (xmm_result, xmm_temp1);
  3161     __ aesdec(xmm_result, xmm_temp2);
  3162     __ aesdec(xmm_result, xmm_temp3);
  3163     __ aesdec(xmm_result, xmm_temp4);
  3165     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
  3166     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
  3167     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
  3168     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
  3170     __ aesdec(xmm_result, xmm_temp1);
  3171     __ aesdec(xmm_result, xmm_temp2);
  3172     __ aesdec(xmm_result, xmm_temp3);
  3173     __ aesdec(xmm_result, xmm_temp4);
  3175     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
  3176     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
  3177     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
  3179     __ cmpl(keylen, 44);
  3180     __ jccb(Assembler::equal, L_doLast);
  3182     __ aesdec(xmm_result, xmm_temp1);
  3183     __ aesdec(xmm_result, xmm_temp2);
  3185     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
  3186     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
  3188     __ cmpl(keylen, 52);
  3189     __ jccb(Assembler::equal, L_doLast);
  3191     __ aesdec(xmm_result, xmm_temp1);
  3192     __ aesdec(xmm_result, xmm_temp2);
  3194     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
  3195     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
  3197     __ BIND(L_doLast);
  3198     __ aesdec(xmm_result, xmm_temp1);
  3199     __ aesdec(xmm_result, xmm_temp2);
  3201     // for decryption the aesdeclast operation is always on key+0x00
  3202     __ aesdeclast(xmm_result, xmm_temp3);
  3203     __ movdqu(Address(to, 0), xmm_result);  // store the result
  3204     __ xorptr(rax, rax); // return 0
  3205     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3206     __ ret(0);
  3208     return start;
  3212   // Arguments:
  3213   //
  3214   // Inputs:
  3215   //   c_rarg0   - source byte array address
  3216   //   c_rarg1   - destination byte array address
  3217   //   c_rarg2   - K (key) in little endian int array
  3218   //   c_rarg3   - r vector byte array address
  3219   //   c_rarg4   - input length
  3220   //
  3221   address generate_cipherBlockChaining_encryptAESCrypt() {
  3222     assert(UseAES, "need AES instructions and misaligned SSE support");
  3223     __ align(CodeEntryAlignment);
  3224     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
  3225     address start = __ pc();
  3227     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
  3228     const Register from        = c_rarg0;  // source array address
  3229     const Register to          = c_rarg1;  // destination array address
  3230     const Register key         = c_rarg2;  // key array address
  3231     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
  3232                                            // and left with the results of the last encryption block
  3233 #ifndef _WIN64
  3234     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
  3235 #else
  3236     const Address  len_mem(rsp, 6 * wordSize);  // length is on stack on Win64
  3237     const Register len_reg     = r10;      // pick the first volatile windows register
  3238 #endif
  3239     const Register pos         = rax;
  3241     // xmm register assignments for the loops below
  3242     const XMMRegister xmm_result = xmm0;
  3243     const XMMRegister xmm_temp   = xmm1;
  3244     // keys 0-10 preloaded into xmm2-xmm12
  3245     const int XMM_REG_NUM_KEY_FIRST = 2;
  3246     const int XMM_REG_NUM_KEY_LAST  = 15;
  3247     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
  3248     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
  3249     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
  3250     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
  3251     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
  3253     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3255 #ifdef _WIN64
  3256     // on win64, fill len_reg from stack position
  3257     __ movl(len_reg, len_mem);
  3258     // save the xmm registers which must be preserved 6-15
  3259     __ subptr(rsp, -rsp_after_call_off * wordSize);
  3260     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
  3261       __ movdqu(xmm_save(i), as_XMMRegister(i));
  3263 #endif
  3265     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
  3266     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
  3267     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
  3268     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
  3269       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
  3270       offset += 0x10;
  3272     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
  3274     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
  3275     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
  3276     __ cmpl(rax, 44);
  3277     __ jcc(Assembler::notEqual, L_key_192_256);
  3279     // 128 bit code follows here
  3280     __ movptr(pos, 0);
  3281     __ align(OptoLoopAlignment);
  3283     __ BIND(L_loopTop_128);
  3284     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
  3285     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
  3286     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
  3287     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
  3288       __ aesenc(xmm_result, as_XMMRegister(rnum));
  3290     __ aesenclast(xmm_result, xmm_key10);
  3291     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
  3292     // no need to store r to memory until we exit
  3293     __ addptr(pos, AESBlockSize);
  3294     __ subptr(len_reg, AESBlockSize);
  3295     __ jcc(Assembler::notEqual, L_loopTop_128);
  3297     __ BIND(L_exit);
  3298     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
  3300 #ifdef _WIN64
  3301     // restore xmm regs belonging to calling function
  3302     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
  3303       __ movdqu(as_XMMRegister(i), xmm_save(i));
  3305 #endif
  3306     __ movl(rax, 0); // return 0 (why?)
  3307     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3308     __ ret(0);
  3310     __ BIND(L_key_192_256);
  3311     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
  3312     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
  3313     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
  3314     __ cmpl(rax, 52);
  3315     __ jcc(Assembler::notEqual, L_key_256);
  3317     // 192-bit code follows here (could be changed to use more xmm registers)
  3318     __ movptr(pos, 0);
  3319     __ align(OptoLoopAlignment);
  3321     __ BIND(L_loopTop_192);
  3322     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
  3323     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
  3324     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
  3325     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
  3326       __ aesenc(xmm_result, as_XMMRegister(rnum));
  3328     __ aesenclast(xmm_result, xmm_key12);
  3329     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
  3330     // no need to store r to memory until we exit
  3331     __ addptr(pos, AESBlockSize);
  3332     __ subptr(len_reg, AESBlockSize);
  3333     __ jcc(Assembler::notEqual, L_loopTop_192);
  3334     __ jmp(L_exit);
  3336     __ BIND(L_key_256);
  3337     // 256-bit code follows here (could be changed to use more xmm registers)
  3338     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
  3339     __ movptr(pos, 0);
  3340     __ align(OptoLoopAlignment);
  3342     __ BIND(L_loopTop_256);
  3343     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
  3344     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
  3345     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
  3346     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
  3347       __ aesenc(xmm_result, as_XMMRegister(rnum));
  3349     load_key(xmm_temp, key, 0xe0);
  3350     __ aesenclast(xmm_result, xmm_temp);
  3351     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
  3352     // no need to store r to memory until we exit
  3353     __ addptr(pos, AESBlockSize);
  3354     __ subptr(len_reg, AESBlockSize);
  3355     __ jcc(Assembler::notEqual, L_loopTop_256);
  3356     __ jmp(L_exit);
  3358     return start;
  3363   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
  3364   // to hide instruction latency
  3365   //
  3366   // Arguments:
  3367   //
  3368   // Inputs:
  3369   //   c_rarg0   - source byte array address
  3370   //   c_rarg1   - destination byte array address
  3371   //   c_rarg2   - K (key) in little endian int array
  3372   //   c_rarg3   - r vector byte array address
  3373   //   c_rarg4   - input length
  3374   //
  3376   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
  3377     assert(UseAES, "need AES instructions and misaligned SSE support");
  3378     __ align(CodeEntryAlignment);
  3379     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
  3380     address start = __ pc();
  3382     Label L_exit, L_key_192_256, L_key_256;
  3383     Label L_singleBlock_loopTop_128, L_multiBlock_loopTop_128;
  3384     Label L_singleBlock_loopTop_192, L_singleBlock_loopTop_256;
  3385     const Register from        = c_rarg0;  // source array address
  3386     const Register to          = c_rarg1;  // destination array address
  3387     const Register key         = c_rarg2;  // key array address
  3388     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
  3389                                            // and left with the results of the last encryption block
  3390 #ifndef _WIN64
  3391     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
  3392 #else
  3393     const Address  len_mem(rsp, 6 * wordSize);  // length is on stack on Win64
  3394     const Register len_reg     = r10;      // pick the first volatile windows register
  3395 #endif
  3396     const Register pos         = rax;
  3398     // keys 0-10 preloaded into xmm2-xmm12
  3399     const int XMM_REG_NUM_KEY_FIRST = 5;
  3400     const int XMM_REG_NUM_KEY_LAST  = 15;
  3401     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
  3402     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
  3404     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3406 #ifdef _WIN64
  3407     // on win64, fill len_reg from stack position
  3408     __ movl(len_reg, len_mem);
  3409     // save the xmm registers which must be preserved 6-15
  3410     __ subptr(rsp, -rsp_after_call_off * wordSize);
  3411     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
  3412       __ movdqu(xmm_save(i), as_XMMRegister(i));
  3414 #endif
  3415     // the java expanded key ordering is rotated one position from what we want
  3416     // so we start from 0x10 here and hit 0x00 last
  3417     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
  3418     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
  3419     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
  3420     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
  3421       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
  3422       offset += 0x10;
  3424     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
  3426     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
  3428     // registers holding the four results in the parallelized loop
  3429     const XMMRegister xmm_result0 = xmm0;
  3430     const XMMRegister xmm_result1 = xmm2;
  3431     const XMMRegister xmm_result2 = xmm3;
  3432     const XMMRegister xmm_result3 = xmm4;
  3434     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
  3436     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
  3437     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
  3438     __ cmpl(rax, 44);
  3439     __ jcc(Assembler::notEqual, L_key_192_256);
  3442     // 128-bit code follows here, parallelized
  3443     __ movptr(pos, 0);
  3444     __ align(OptoLoopAlignment);
  3445     __ BIND(L_multiBlock_loopTop_128);
  3446     __ cmpptr(len_reg, 4*AESBlockSize);           // see if at least 4 blocks left
  3447     __ jcc(Assembler::less, L_singleBlock_loopTop_128);
  3449     __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0*AESBlockSize));   // get next 4 blocks into xmmresult registers
  3450     __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1*AESBlockSize));
  3451     __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2*AESBlockSize));
  3452     __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3*AESBlockSize));
  3454 #define DoFour(opc, src_reg)                    \
  3455     __ opc(xmm_result0, src_reg);               \
  3456     __ opc(xmm_result1, src_reg);               \
  3457     __ opc(xmm_result2, src_reg);               \
  3458     __ opc(xmm_result3, src_reg);
  3460     DoFour(pxor, xmm_key_first);
  3461     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
  3462       DoFour(aesdec, as_XMMRegister(rnum));
  3464     DoFour(aesdeclast, xmm_key_last);
  3465     // for each result, xor with the r vector of previous cipher block
  3466     __ pxor(xmm_result0, xmm_prev_block_cipher);
  3467     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0*AESBlockSize));
  3468     __ pxor(xmm_result1, xmm_prev_block_cipher);
  3469     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1*AESBlockSize));
  3470     __ pxor(xmm_result2, xmm_prev_block_cipher);
  3471     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2*AESBlockSize));
  3472     __ pxor(xmm_result3, xmm_prev_block_cipher);
  3473     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3*AESBlockSize));   // this will carry over to next set of blocks
  3475     __ movdqu(Address(to, pos, Address::times_1, 0*AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
  3476     __ movdqu(Address(to, pos, Address::times_1, 1*AESBlockSize), xmm_result1);
  3477     __ movdqu(Address(to, pos, Address::times_1, 2*AESBlockSize), xmm_result2);
  3478     __ movdqu(Address(to, pos, Address::times_1, 3*AESBlockSize), xmm_result3);
  3480     __ addptr(pos, 4*AESBlockSize);
  3481     __ subptr(len_reg, 4*AESBlockSize);
  3482     __ jmp(L_multiBlock_loopTop_128);
  3484     // registers used in the non-parallelized loops
  3485     // xmm register assignments for the loops below
  3486     const XMMRegister xmm_result = xmm0;
  3487     const XMMRegister xmm_prev_block_cipher_save = xmm2;
  3488     const XMMRegister xmm_key11 = xmm3;
  3489     const XMMRegister xmm_key12 = xmm4;
  3490     const XMMRegister xmm_temp  = xmm4;
  3492     __ align(OptoLoopAlignment);
  3493     __ BIND(L_singleBlock_loopTop_128);
  3494     __ cmpptr(len_reg, 0);           // any blocks left??
  3495     __ jcc(Assembler::equal, L_exit);
  3496     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
  3497     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
  3498     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
  3499     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
  3500       __ aesdec(xmm_result, as_XMMRegister(rnum));
  3502     __ aesdeclast(xmm_result, xmm_key_last);
  3503     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
  3504     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
  3505     // no need to store r to memory until we exit
  3506     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);              // set up next r vector with cipher input from this block
  3508     __ addptr(pos, AESBlockSize);
  3509     __ subptr(len_reg, AESBlockSize);
  3510     __ jmp(L_singleBlock_loopTop_128);
  3513     __ BIND(L_exit);
  3514     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
  3515 #ifdef _WIN64
  3516     // restore regs belonging to calling function
  3517     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
  3518       __ movdqu(as_XMMRegister(i), xmm_save(i));
  3520 #endif
  3521     __ movl(rax, 0); // return 0 (why?)
  3522     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3523     __ ret(0);
  3526     __ BIND(L_key_192_256);
  3527     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
  3528     load_key(xmm_key11, key, 0xb0);
  3529     __ cmpl(rax, 52);
  3530     __ jcc(Assembler::notEqual, L_key_256);
  3532     // 192-bit code follows here (could be optimized to use parallelism)
  3533     load_key(xmm_key12, key, 0xc0);     // 192-bit key goes up to c0
  3534     __ movptr(pos, 0);
  3535     __ align(OptoLoopAlignment);
  3537     __ BIND(L_singleBlock_loopTop_192);
  3538     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
  3539     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
  3540     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
  3541     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
  3542       __ aesdec(xmm_result, as_XMMRegister(rnum));
  3544     __ aesdec(xmm_result, xmm_key11);
  3545     __ aesdec(xmm_result, xmm_key12);
  3546     __ aesdeclast(xmm_result, xmm_key_last);                    // xmm15 always came from key+0
  3547     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
  3548     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
  3549     // no need to store r to memory until we exit
  3550     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
  3551     __ addptr(pos, AESBlockSize);
  3552     __ subptr(len_reg, AESBlockSize);
  3553     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
  3554     __ jmp(L_exit);
  3556     __ BIND(L_key_256);
  3557     // 256-bit code follows here (could be optimized to use parallelism)
  3558     __ movptr(pos, 0);
  3559     __ align(OptoLoopAlignment);
  3561     __ BIND(L_singleBlock_loopTop_256);
  3562     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
  3563     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
  3564     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
  3565     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
  3566       __ aesdec(xmm_result, as_XMMRegister(rnum));
  3568     __ aesdec(xmm_result, xmm_key11);
  3569     load_key(xmm_temp, key, 0xc0);
  3570     __ aesdec(xmm_result, xmm_temp);
  3571     load_key(xmm_temp, key, 0xd0);
  3572     __ aesdec(xmm_result, xmm_temp);
  3573     load_key(xmm_temp, key, 0xe0);     // 256-bit key goes up to e0
  3574     __ aesdec(xmm_result, xmm_temp);
  3575     __ aesdeclast(xmm_result, xmm_key_last);          // xmm15 came from key+0
  3576     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
  3577     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
  3578     // no need to store r to memory until we exit
  3579     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
  3580     __ addptr(pos, AESBlockSize);
  3581     __ subptr(len_reg, AESBlockSize);
  3582     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
  3583     __ jmp(L_exit);
  3585     return start;
  3588   /**
  3589    *  Arguments:
  3591    * Inputs:
  3592    *   c_rarg0   - int crc
  3593    *   c_rarg1   - byte* buf
  3594    *   c_rarg2   - int length
  3596    * Ouput:
  3597    *       rax   - int crc result
  3598    */
  3599   address generate_updateBytesCRC32() {
  3600     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
  3602     __ align(CodeEntryAlignment);
  3603     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
  3605     address start = __ pc();
  3606     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
  3607     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
  3608     // rscratch1: r10
  3609     const Register crc   = c_rarg0;  // crc
  3610     const Register buf   = c_rarg1;  // source java byte array address
  3611     const Register len   = c_rarg2;  // length
  3612     const Register table = c_rarg3;  // crc_table address (reuse register)
  3613     const Register tmp   = r11;
  3614     assert_different_registers(crc, buf, len, table, tmp, rax);
  3616     BLOCK_COMMENT("Entry:");
  3617     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3619     __ kernel_crc32(crc, buf, len, table, tmp);
  3621     __ movl(rax, crc);
  3622     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3623     __ ret(0);
  3625     return start;
  3628 #undef __
  3629 #define __ masm->
  3631   // Continuation point for throwing of implicit exceptions that are
  3632   // not handled in the current activation. Fabricates an exception
  3633   // oop and initiates normal exception dispatching in this
  3634   // frame. Since we need to preserve callee-saved values (currently
  3635   // only for C2, but done for C1 as well) we need a callee-saved oop
  3636   // map and therefore have to make these stubs into RuntimeStubs
  3637   // rather than BufferBlobs.  If the compiler needs all registers to
  3638   // be preserved between the fault point and the exception handler
  3639   // then it must assume responsibility for that in
  3640   // AbstractCompiler::continuation_for_implicit_null_exception or
  3641   // continuation_for_implicit_division_by_zero_exception. All other
  3642   // implicit exceptions (e.g., NullPointerException or
  3643   // AbstractMethodError on entry) are either at call sites or
  3644   // otherwise assume that stack unwinding will be initiated, so
  3645   // caller saved registers were assumed volatile in the compiler.
  3646   address generate_throw_exception(const char* name,
  3647                                    address runtime_entry,
  3648                                    Register arg1 = noreg,
  3649                                    Register arg2 = noreg) {
  3650     // Information about frame layout at time of blocking runtime call.
  3651     // Note that we only have to preserve callee-saved registers since
  3652     // the compilers are responsible for supplying a continuation point
  3653     // if they expect all registers to be preserved.
  3654     enum layout {
  3655       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
  3656       rbp_off2,
  3657       return_off,
  3658       return_off2,
  3659       framesize // inclusive of return address
  3660     };
  3662     int insts_size = 512;
  3663     int locs_size  = 64;
  3665     CodeBuffer code(name, insts_size, locs_size);
  3666     OopMapSet* oop_maps  = new OopMapSet();
  3667     MacroAssembler* masm = new MacroAssembler(&code);
  3669     address start = __ pc();
  3671     // This is an inlined and slightly modified version of call_VM
  3672     // which has the ability to fetch the return PC out of
  3673     // thread-local storage and also sets up last_Java_sp slightly
  3674     // differently than the real call_VM
  3676     __ enter(); // required for proper stackwalking of RuntimeStub frame
  3678     assert(is_even(framesize/2), "sp not 16-byte aligned");
  3680     // return address and rbp are already in place
  3681     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
  3683     int frame_complete = __ pc() - start;
  3685     // Set up last_Java_sp and last_Java_fp
  3686     address the_pc = __ pc();
  3687     __ set_last_Java_frame(rsp, rbp, the_pc);
  3688     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
  3690     // Call runtime
  3691     if (arg1 != noreg) {
  3692       assert(arg2 != c_rarg1, "clobbered");
  3693       __ movptr(c_rarg1, arg1);
  3695     if (arg2 != noreg) {
  3696       __ movptr(c_rarg2, arg2);
  3698     __ movptr(c_rarg0, r15_thread);
  3699     BLOCK_COMMENT("call runtime_entry");
  3700     __ call(RuntimeAddress(runtime_entry));
  3702     // Generate oop map
  3703     OopMap* map = new OopMap(framesize, 0);
  3705     oop_maps->add_gc_map(the_pc - start, map);
  3707     __ reset_last_Java_frame(true, true);
  3709     __ leave(); // required for proper stackwalking of RuntimeStub frame
  3711     // check for pending exceptions
  3712 #ifdef ASSERT
  3713     Label L;
  3714     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
  3715             (int32_t) NULL_WORD);
  3716     __ jcc(Assembler::notEqual, L);
  3717     __ should_not_reach_here();
  3718     __ bind(L);
  3719 #endif // ASSERT
  3720     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
  3723     // codeBlob framesize is in words (not VMRegImpl::slot_size)
  3724     RuntimeStub* stub =
  3725       RuntimeStub::new_runtime_stub(name,
  3726                                     &code,
  3727                                     frame_complete,
  3728                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
  3729                                     oop_maps, false);
  3730     return stub->entry_point();
  3733   void create_control_words() {
  3734     // Round to nearest, 53-bit mode, exceptions masked
  3735     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
  3736     // Round to zero, 53-bit mode, exception mased
  3737     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
  3738     // Round to nearest, 24-bit mode, exceptions masked
  3739     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
  3740     // Round to nearest, 64-bit mode, exceptions masked
  3741     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
  3742     // Round to nearest, 64-bit mode, exceptions masked
  3743     StubRoutines::_mxcsr_std           = 0x1F80;
  3744     // Note: the following two constants are 80-bit values
  3745     //       layout is critical for correct loading by FPU.
  3746     // Bias for strict fp multiply/divide
  3747     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
  3748     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
  3749     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
  3750     // Un-Bias for strict fp multiply/divide
  3751     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
  3752     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
  3753     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
  3756   // Initialization
  3757   void generate_initial() {
  3758     // Generates all stubs and initializes the entry points
  3760     // This platform-specific settings are needed by generate_call_stub()
  3761     create_control_words();
  3763     // entry points that exist in all platforms Note: This is code
  3764     // that could be shared among different platforms - however the
  3765     // benefit seems to be smaller than the disadvantage of having a
  3766     // much more complicated generator structure. See also comment in
  3767     // stubRoutines.hpp.
  3769     StubRoutines::_forward_exception_entry = generate_forward_exception();
  3771     StubRoutines::_call_stub_entry =
  3772       generate_call_stub(StubRoutines::_call_stub_return_address);
  3774     // is referenced by megamorphic call
  3775     StubRoutines::_catch_exception_entry = generate_catch_exception();
  3777     // atomic calls
  3778     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
  3779     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
  3780     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
  3781     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
  3782     StubRoutines::_atomic_add_entry          = generate_atomic_add();
  3783     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();
  3784     StubRoutines::_fence_entry               = generate_orderaccess_fence();
  3786     StubRoutines::_handler_for_unsafe_access_entry =
  3787       generate_handler_for_unsafe_access();
  3789     // platform dependent
  3790     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
  3791     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
  3793     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
  3795     // Build this early so it's available for the interpreter.
  3796     StubRoutines::_throw_StackOverflowError_entry =
  3797       generate_throw_exception("StackOverflowError throw_exception",
  3798                                CAST_FROM_FN_PTR(address,
  3799                                                 SharedRuntime::
  3800                                                 throw_StackOverflowError));
  3801     if (UseCRC32Intrinsics) {
  3802       // set table address before stub generation which use it
  3803       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
  3804       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
  3808   void generate_all() {
  3809     // Generates all stubs and initializes the entry points
  3811     // These entry points require SharedInfo::stack0 to be set up in
  3812     // non-core builds and need to be relocatable, so they each
  3813     // fabricate a RuntimeStub internally.
  3814     StubRoutines::_throw_AbstractMethodError_entry =
  3815       generate_throw_exception("AbstractMethodError throw_exception",
  3816                                CAST_FROM_FN_PTR(address,
  3817                                                 SharedRuntime::
  3818                                                 throw_AbstractMethodError));
  3820     StubRoutines::_throw_IncompatibleClassChangeError_entry =
  3821       generate_throw_exception("IncompatibleClassChangeError throw_exception",
  3822                                CAST_FROM_FN_PTR(address,
  3823                                                 SharedRuntime::
  3824                                                 throw_IncompatibleClassChangeError));
  3826     StubRoutines::_throw_NullPointerException_at_call_entry =
  3827       generate_throw_exception("NullPointerException at call throw_exception",
  3828                                CAST_FROM_FN_PTR(address,
  3829                                                 SharedRuntime::
  3830                                                 throw_NullPointerException_at_call));
  3832     // entry points that are platform specific
  3833     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
  3834     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
  3835     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
  3836     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
  3838     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
  3839     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
  3840     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
  3841     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
  3843     // support for verify_oop (must happen after universe_init)
  3844     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
  3846     // arraycopy stubs used by compilers
  3847     generate_arraycopy_stubs();
  3849     generate_math_stubs();
  3851     // don't bother generating these AES intrinsic stubs unless global flag is set
  3852     if (UseAESIntrinsics) {
  3853       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
  3855       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
  3856       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
  3857       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
  3858       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
  3862  public:
  3863   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
  3864     if (all) {
  3865       generate_all();
  3866     } else {
  3867       generate_initial();
  3870 }; // end class declaration
  3872 void StubGenerator_generate(CodeBuffer* code, bool all) {
  3873   StubGenerator g(code, all);

mercurial