jdk8-mips64-public/hotspot: src/cpu/x86/vm/stubGenerator_x86

     1 /*

     2  * Copyright 1999-2007 Sun Microsystems, Inc.  All Rights Reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,

    20  * CA 95054 USA or visit www.sun.com if you need additional information or

    21  * have any questions.

    22  *

    23  */

    25 #include "incls/_precompiled.incl"

    26 #include "incls/_stubGenerator_x86_32.cpp.incl"

    28 // Declaration and definition of StubGenerator (no .hpp file).

    29 // For a more detailed description of the stub routine structure

    30 // see the comment in stubRoutines.hpp

    32 #define __ _masm->

    33 #define a__ ((Assembler*)_masm)->

    35 #ifdef PRODUCT

    36 #define BLOCK_COMMENT(str) /* nothing */

    37 #else

    38 #define BLOCK_COMMENT(str) __ block_comment(str)

    39 #endif

    41 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")

    43 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions

    44 const int FPU_CNTRL_WRD_MASK = 0xFFFF;

    46 // -------------------------------------------------------------------------------------------------------------------------

    47 // Stub Code definitions

    49 static address handle_unsafe_access() {

    50   JavaThread* thread = JavaThread::current();

    51   address pc  = thread->saved_exception_pc();

    52   // pc is the instruction which we must emulate

    53   // doing a no-op is fine:  return garbage from the load

    54   // therefore, compute npc

    55   address npc = Assembler::locate_next_instruction(pc);

    57   // request an async exception

    58   thread->set_pending_unsafe_access_error();

    60   // return address of next instruction to execute

    61   return npc;

    62 }

    64 class StubGenerator: public StubCodeGenerator {

    65  private:

    67 #ifdef PRODUCT

    68 #define inc_counter_np(counter) (0)

    69 #else

    70   void inc_counter_np_(int& counter) {

    71     __ incrementl(ExternalAddress((address)&counter));

    72   }

    73 #define inc_counter_np(counter) \

    74   BLOCK_COMMENT("inc_counter " #counter); \

    75   inc_counter_np_(counter);

    76 #endif //PRODUCT

    78   void inc_copy_counter_np(BasicType t) {

    79 #ifndef PRODUCT

    80     switch (t) {

    81     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;

    82     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;

    83     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;

    84     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;

    85     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;

    86     }

    87     ShouldNotReachHere();

    88 #endif //PRODUCT

    89   }

    91   //------------------------------------------------------------------------------------------------------------------------

    92   // Call stubs are used to call Java from C

    93   //

    94   //    [ return_from_Java     ] <--- rsp

    95   //    [ argument word n      ]

    96   //      ...

    97   // -N [ argument word 1      ]

    98   // -7 [ Possible padding for stack alignment ]

    99   // -6 [ Possible padding for stack alignment ]

   100   // -5 [ Possible padding for stack alignment ]

   101   // -4 [ mxcsr save           ] <--- rsp_after_call

   102   // -3 [ saved rbx,            ]

   103   // -2 [ saved rsi            ]

   104   // -1 [ saved rdi            ]

   105   //  0 [ saved rbp,            ] <--- rbp,

   106   //  1 [ return address       ]

   107   //  2 [ ptr. to call wrapper ]

   108   //  3 [ result               ]

   109   //  4 [ result_type          ]

   110   //  5 [ method               ]

   111   //  6 [ entry_point          ]

   112   //  7 [ parameters           ]

   113   //  8 [ parameter_size       ]

   114   //  9 [ thread               ]

   117   address generate_call_stub(address& return_address) {

   118     StubCodeMark mark(this, "StubRoutines", "call_stub");

   119     address start = __ pc();

   121     // stub code parameters / addresses

   122     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");

   123     bool  sse_save = false;

   124     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!

   125     const int     locals_count_in_bytes  (4*wordSize);

   126     const Address mxcsr_save    (rbp, -4 * wordSize);

   127     const Address saved_rbx     (rbp, -3 * wordSize);

   128     const Address saved_rsi     (rbp, -2 * wordSize);

   129     const Address saved_rdi     (rbp, -1 * wordSize);

   130     const Address result        (rbp,  3 * wordSize);

   131     const Address result_type   (rbp,  4 * wordSize);

   132     const Address method        (rbp,  5 * wordSize);

   133     const Address entry_point   (rbp,  6 * wordSize);

   134     const Address parameters    (rbp,  7 * wordSize);

   135     const Address parameter_size(rbp,  8 * wordSize);

   136     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!

   137     sse_save =  UseSSE > 0;

   139     // stub code

   140     __ enter();

   141     __ movptr(rcx, parameter_size);              // parameter counter

   142     __ shlptr(rcx, Interpreter::logStackElementSize()); // convert parameter count to bytes

   143     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves

   144     __ subptr(rsp, rcx);

   145     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack

   147     // save rdi, rsi, & rbx, according to C calling conventions

   148     __ movptr(saved_rdi, rdi);

   149     __ movptr(saved_rsi, rsi);

   150     __ movptr(saved_rbx, rbx);

   151     // save and initialize %mxcsr

   152     if (sse_save) {

   153       Label skip_ldmx;

   154       __ stmxcsr(mxcsr_save);

   155       __ movl(rax, mxcsr_save);

   156       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits

   157       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

   158       __ cmp32(rax, mxcsr_std);

   159       __ jcc(Assembler::equal, skip_ldmx);

   160       __ ldmxcsr(mxcsr_std);

   161       __ bind(skip_ldmx);

   162     }

   164     // make sure the control word is correct.

   165     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));

   167 #ifdef ASSERT

   168     // make sure we have no pending exceptions

   169     { Label L;

   170       __ movptr(rcx, thread);

   171       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

   172       __ jcc(Assembler::equal, L);

   173       __ stop("StubRoutines::call_stub: entered with pending exception");

   174       __ bind(L);

   175     }

   176 #endif

   178     // pass parameters if any

   179     BLOCK_COMMENT("pass parameters if any");

   180     Label parameters_done;

   181     __ movl(rcx, parameter_size);  // parameter counter

   182     __ testl(rcx, rcx);

   183     __ jcc(Assembler::zero, parameters_done);

   185     // parameter passing loop

   187     Label loop;

   188     // Copy Java parameters in reverse order (receiver last)

   189     // Note that the argument order is inverted in the process

   190     // source is rdx[rcx: N-1..0]

   191     // dest   is rsp[rbx: 0..N-1]

   193     __ movptr(rdx, parameters);          // parameter pointer

   194     __ xorptr(rbx, rbx);

   196     __ BIND(loop);

   197     if (TaggedStackInterpreter) {

   198       __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(),

   199                       -2*wordSize));                          // get tag

   200       __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),

   201                       Interpreter::expr_tag_offset_in_bytes(0)), rax);     // store tag

   202     }

   204     // get parameter

   205     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));

   206     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),

   207                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter

   208     __ increment(rbx);

   209     __ decrement(rcx);

   210     __ jcc(Assembler::notZero, loop);

   212     // call Java function

   213     __ BIND(parameters_done);

   214     __ movptr(rbx, method);           // get methodOop

   215     __ movptr(rax, entry_point);      // get entry_point

   216     __ mov(rsi, rsp);                 // set sender sp

   217     BLOCK_COMMENT("call Java function");

   218     __ call(rax);

   220     BLOCK_COMMENT("call_stub_return_address:");

   221     return_address = __ pc();

   223     Label common_return;

   225     __ BIND(common_return);

   227     // store result depending on type

   228     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)

   229     __ movptr(rdi, result);

   230     Label is_long, is_float, is_double, exit;

   231     __ movl(rsi, result_type);

   232     __ cmpl(rsi, T_LONG);

   233     __ jcc(Assembler::equal, is_long);

   234     __ cmpl(rsi, T_FLOAT);

   235     __ jcc(Assembler::equal, is_float);

   236     __ cmpl(rsi, T_DOUBLE);

   237     __ jcc(Assembler::equal, is_double);

   239     // handle T_INT case

   240     __ movl(Address(rdi, 0), rax);

   241     __ BIND(exit);

   243     // check that FPU stack is empty

   244     __ verify_FPU(0, "generate_call_stub");

   246     // pop parameters

   247     __ lea(rsp, rsp_after_call);

   249     // restore %mxcsr

   250     if (sse_save) {

   251       __ ldmxcsr(mxcsr_save);

   252     }

   254     // restore rdi, rsi and rbx,

   255     __ movptr(rbx, saved_rbx);

   256     __ movptr(rsi, saved_rsi);

   257     __ movptr(rdi, saved_rdi);

   258     __ addptr(rsp, 4*wordSize);

   260     // return

   261     __ pop(rbp);

   262     __ ret(0);

   264     // handle return types different from T_INT

   265     __ BIND(is_long);

   266     __ movl(Address(rdi, 0 * wordSize), rax);

   267     __ movl(Address(rdi, 1 * wordSize), rdx);

   268     __ jmp(exit);

   270     __ BIND(is_float);

   271     // interpreter uses xmm0 for return values

   272     if (UseSSE >= 1) {

   273       __ movflt(Address(rdi, 0), xmm0);

   274     } else {

   275       __ fstp_s(Address(rdi, 0));

   276     }

   277     __ jmp(exit);

   279     __ BIND(is_double);

   280     // interpreter uses xmm0 for return values

   281     if (UseSSE >= 2) {

   282       __ movdbl(Address(rdi, 0), xmm0);

   283     } else {

   284       __ fstp_d(Address(rdi, 0));

   285     }

   286     __ jmp(exit);

   288     // If we call compiled code directly from the call stub we will

   289     // need to adjust the return back to the call stub to a specialized

   290     // piece of code that can handle compiled results and cleaning the fpu

   291     // stack. compiled code will be set to return here instead of the

   292     // return above that handles interpreter returns.

   294     BLOCK_COMMENT("call_stub_compiled_return:");

   295     StubRoutines::x86::set_call_stub_compiled_return( __ pc());

   297 #ifdef COMPILER2

   298     if (UseSSE >= 2) {

   299       __ verify_FPU(0, "call_stub_compiled_return");

   300     } else {

   301       for (int i = 1; i < 8; i++) {

   302         __ ffree(i);

   303       }

   305       // UseSSE <= 1 so double result should be left on TOS

   306       __ movl(rsi, result_type);

   307       __ cmpl(rsi, T_DOUBLE);

   308       __ jcc(Assembler::equal, common_return);

   309       if (UseSSE == 0) {

   310         // UseSSE == 0 so float result should be left on TOS

   311         __ cmpl(rsi, T_FLOAT);

   312         __ jcc(Assembler::equal, common_return);

   313       }

   314       __ ffree(0);

   315     }

   316 #endif /* COMPILER2 */

   317     __ jmp(common_return);

   319     return start;

   320   }

   323   //------------------------------------------------------------------------------------------------------------------------

   324   // Return point for a Java call if there's an exception thrown in Java code.

   325   // The exception is caught and transformed into a pending exception stored in

   326   // JavaThread that can be tested from within the VM.

   327   //

   328   // Note: Usually the parameters are removed by the callee. In case of an exception

   329   //       crossing an activation frame boundary, that is not the case if the callee

   330   //       is compiled code => need to setup the rsp.

   331   //

   332   // rax,: exception oop

   334   address generate_catch_exception() {

   335     StubCodeMark mark(this, "StubRoutines", "catch_exception");

   336     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!

   337     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!

   338     address start = __ pc();

   340     // get thread directly

   341     __ movptr(rcx, thread);

   342 #ifdef ASSERT

   343     // verify that threads correspond

   344     { Label L;

   345       __ get_thread(rbx);

   346       __ cmpptr(rbx, rcx);

   347       __ jcc(Assembler::equal, L);

   348       __ stop("StubRoutines::catch_exception: threads must correspond");

   349       __ bind(L);

   350     }

   351 #endif

   352     // set pending exception

   353     __ verify_oop(rax);

   354     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );

   355     __ lea(Address(rcx, Thread::exception_file_offset   ()),

   356            ExternalAddress((address)__FILE__));

   357     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );

   358     // complete return to VM

   359     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");

   360     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));

   362     return start;

   363   }

   366   //------------------------------------------------------------------------------------------------------------------------

   367   // Continuation point for runtime calls returning with a pending exception.

   368   // The pending exception check happened in the runtime or native call stub.

   369   // The pending exception in Thread is converted into a Java-level exception.

   370   //

   371   // Contract with Java-level exception handlers:

   372   // rax,: exception

   373   // rdx: throwing pc

   374   //

   375   // NOTE: At entry of this stub, exception-pc must be on stack !!

   377   address generate_forward_exception() {

   378     StubCodeMark mark(this, "StubRoutines", "forward exception");

   379     address start = __ pc();

   381     // Upon entry, the sp points to the return address returning into Java

   382     // (interpreted or compiled) code; i.e., the return address becomes the

   383     // throwing pc.

   384     //

   385     // Arguments pushed before the runtime call are still on the stack but

   386     // the exception handler will reset the stack pointer -> ignore them.

   387     // A potential result in registers can be ignored as well.

   389 #ifdef ASSERT

   390     // make sure this code is only executed if there is a pending exception

   391     { Label L;

   392       __ get_thread(rcx);

   393       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

   394       __ jcc(Assembler::notEqual, L);

   395       __ stop("StubRoutines::forward exception: no pending exception (1)");

   396       __ bind(L);

   397     }

   398 #endif

   400     // compute exception handler into rbx,

   401     __ movptr(rax, Address(rsp, 0));

   402     BLOCK_COMMENT("call exception_handler_for_return_address");

   403     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rax);

   404     __ mov(rbx, rax);

   406     // setup rax, & rdx, remove return address & clear pending exception

   407     __ get_thread(rcx);

   408     __ pop(rdx);

   409     __ movptr(rax, Address(rcx, Thread::pending_exception_offset()));

   410     __ movptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

   412 #ifdef ASSERT

   413     // make sure exception is set

   414     { Label L;

   415       __ testptr(rax, rax);

   416       __ jcc(Assembler::notEqual, L);

   417       __ stop("StubRoutines::forward exception: no pending exception (2)");

   418       __ bind(L);

   419     }

   420 #endif

   422     // continue at exception handler (return address removed)

   423     // rax,: exception

   424     // rbx,: exception handler

   425     // rdx: throwing pc

   426     __ verify_oop(rax);

   427     __ jmp(rbx);

   429     return start;

   430   }

   433   //----------------------------------------------------------------------------------------------------

   434   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)

   435   //

   436   // xchg exists as far back as 8086, lock needed for MP only

   437   // Stack layout immediately after call:

   438   //

   439   // 0 [ret addr ] <--- rsp

   440   // 1 [  ex     ]

   441   // 2 [  dest   ]

   442   //

   443   // Result:   *dest <- ex, return (old *dest)

   444   //

   445   // Note: win32 does not currently use this code

   447   address generate_atomic_xchg() {

   448     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");

   449     address start = __ pc();

   451     __ push(rdx);

   452     Address exchange(rsp, 2 * wordSize);

   453     Address dest_addr(rsp, 3 * wordSize);

   454     __ movl(rax, exchange);

   455     __ movptr(rdx, dest_addr);

   456     __ xchgl(rax, Address(rdx, 0));

   457     __ pop(rdx);

   458     __ ret(0);

   460     return start;

   461   }

   463   //----------------------------------------------------------------------------------------------------

   464   // Support for void verify_mxcsr()

   465   //

   466   // This routine is used with -Xcheck:jni to verify that native

   467   // JNI code does not return to Java code without restoring the

   468   // MXCSR register to our expected state.

   471   address generate_verify_mxcsr() {

   472     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");

   473     address start = __ pc();

   475     const Address mxcsr_save(rsp, 0);

   477     if (CheckJNICalls && UseSSE > 0 ) {

   478       Label ok_ret;

   479       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());

   480       __ push(rax);

   481       __ subptr(rsp, wordSize);      // allocate a temp location

   482       __ stmxcsr(mxcsr_save);

   483       __ movl(rax, mxcsr_save);

   484       __ andl(rax, MXCSR_MASK);

   485       __ cmp32(rax, mxcsr_std);

   486       __ jcc(Assembler::equal, ok_ret);

   488       __ warn("MXCSR changed by native JNI code.");

   490       __ ldmxcsr(mxcsr_std);

   492       __ bind(ok_ret);

   493       __ addptr(rsp, wordSize);

   494       __ pop(rax);

   495     }

   497     __ ret(0);

   499     return start;

   500   }

   503   //---------------------------------------------------------------------------

   504   // Support for void verify_fpu_cntrl_wrd()

   505   //

   506   // This routine is used with -Xcheck:jni to verify that native

   507   // JNI code does not return to Java code without restoring the

   508   // FP control word to our expected state.

   510   address generate_verify_fpu_cntrl_wrd() {

   511     StubCodeMark mark(this, "StubRoutines", "verify_spcw");

   512     address start = __ pc();

   514     const Address fpu_cntrl_wrd_save(rsp, 0);

   516     if (CheckJNICalls) {

   517       Label ok_ret;

   518       __ push(rax);

   519       __ subptr(rsp, wordSize);      // allocate a temp location

   520       __ fnstcw(fpu_cntrl_wrd_save);

   521       __ movl(rax, fpu_cntrl_wrd_save);

   522       __ andl(rax, FPU_CNTRL_WRD_MASK);

   523       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());

   524       __ cmp32(rax, fpu_std);

   525       __ jcc(Assembler::equal, ok_ret);

   527       __ warn("Floating point control word changed by native JNI code.");

   529       __ fldcw(fpu_std);

   531       __ bind(ok_ret);

   532       __ addptr(rsp, wordSize);

   533       __ pop(rax);

   534     }

   536     __ ret(0);

   538     return start;

   539   }

   541   //---------------------------------------------------------------------------

   542   // Wrapper for slow-case handling of double-to-integer conversion

   543   // d2i or f2i fast case failed either because it is nan or because

   544   // of under/overflow.

   545   // Input:  FPU TOS: float value

   546   // Output: rax, (rdx): integer (long) result

   548   address generate_d2i_wrapper(BasicType t, address fcn) {

   549     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");

   550     address start = __ pc();

   552   // Capture info about frame layout

   553   enum layout { FPUState_off         = 0,

   554                 rbp_off              = FPUStateSizeInWords,

   555                 rdi_off,

   556                 rsi_off,

   557                 rcx_off,

   558                 rbx_off,

   559                 saved_argument_off,

   560                 saved_argument_off2, // 2nd half of double

   561                 framesize

   562   };

   564   assert(FPUStateSizeInWords == 27, "update stack layout");

   566     // Save outgoing argument to stack across push_FPU_state()

   567     __ subptr(rsp, wordSize * 2);

   568     __ fstp_d(Address(rsp, 0));

   570     // Save CPU & FPU state

   571     __ push(rbx);

   572     __ push(rcx);

   573     __ push(rsi);

   574     __ push(rdi);

   575     __ push(rbp);

   576     __ push_FPU_state();

   578     // push_FPU_state() resets the FP top of stack

   579     // Load original double into FP top of stack

   580     __ fld_d(Address(rsp, saved_argument_off * wordSize));

   581     // Store double into stack as outgoing argument

   582     __ subptr(rsp, wordSize*2);

   583     __ fst_d(Address(rsp, 0));

   585     // Prepare FPU for doing math in C-land

   586     __ empty_FPU_stack();

   587     // Call the C code to massage the double.  Result in EAX

   588     if (t == T_INT)

   589       { BLOCK_COMMENT("SharedRuntime::d2i"); }

   590     else if (t == T_LONG)

   591       { BLOCK_COMMENT("SharedRuntime::d2l"); }

   592     __ call_VM_leaf( fcn, 2 );

   594     // Restore CPU & FPU state

   595     __ pop_FPU_state();

   596     __ pop(rbp);

   597     __ pop(rdi);

   598     __ pop(rsi);

   599     __ pop(rcx);

   600     __ pop(rbx);

   601     __ addptr(rsp, wordSize * 2);

   603     __ ret(0);

   605     return start;

   606   }

   609   //---------------------------------------------------------------------------

   610   // The following routine generates a subroutine to throw an asynchronous

   611   // UnknownError when an unsafe access gets a fault that could not be

   612   // reasonably prevented by the programmer.  (Example: SIGBUS/OBJERR.)

   613   address generate_handler_for_unsafe_access() {

   614     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");

   615     address start = __ pc();

   617     __ push(0);                       // hole for return address-to-be

   618     __ pusha();                       // push registers

   619     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);

   620     BLOCK_COMMENT("call handle_unsafe_access");

   621     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));

   622     __ movptr(next_pc, rax);          // stuff next address

   623     __ popa();

   624     __ ret(0);                        // jump to next address

   626     return start;

   627   }

   630   //----------------------------------------------------------------------------------------------------

   631   // Non-destructive plausibility checks for oops

   633   address generate_verify_oop() {

   634     StubCodeMark mark(this, "StubRoutines", "verify_oop");

   635     address start = __ pc();

   637     // Incoming arguments on stack after saving rax,:

   638     //

   639     // [tos    ]: saved rdx

   640     // [tos + 1]: saved EFLAGS

   641     // [tos + 2]: return address

   642     // [tos + 3]: char* error message

   643     // [tos + 4]: oop   object to verify

   644     // [tos + 5]: saved rax, - saved by caller and bashed

   646     Label exit, error;

   647     __ pushf();

   648     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));

   649     __ push(rdx);                                // save rdx

   650     // make sure object is 'reasonable'

   651     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object

   652     __ testptr(rax, rax);

   653     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok

   655     // Check if the oop is in the right area of memory

   656     const int oop_mask = Universe::verify_oop_mask();

   657     const int oop_bits = Universe::verify_oop_bits();

   658     __ mov(rdx, rax);

   659     __ andptr(rdx, oop_mask);

   660     __ cmpptr(rdx, oop_bits);

   661     __ jcc(Assembler::notZero, error);

   663     // make sure klass is 'reasonable'

   664     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass

   665     __ testptr(rax, rax);

   666     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken

   668     // Check if the klass is in the right area of memory

   669     const int klass_mask = Universe::verify_klass_mask();

   670     const int klass_bits = Universe::verify_klass_bits();

   671     __ mov(rdx, rax);

   672     __ andptr(rdx, klass_mask);

   673     __ cmpptr(rdx, klass_bits);

   674     __ jcc(Assembler::notZero, error);

   676     // make sure klass' klass is 'reasonable'

   677     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass

   678     __ testptr(rax, rax);

   679     __ jcc(Assembler::zero, error);              // if klass' klass is NULL it is broken

   681     __ mov(rdx, rax);

   682     __ andptr(rdx, klass_mask);

   683     __ cmpptr(rdx, klass_bits);

   684     __ jcc(Assembler::notZero, error);           // if klass not in right area

   685                                                  // of memory it is broken too.

   687     // return if everything seems ok

   688     __ bind(exit);

   689     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back

   690     __ pop(rdx);                                 // restore rdx

   691     __ popf();                                   // restore EFLAGS

   692     __ ret(3 * wordSize);                        // pop arguments

   694     // handle errors

   695     __ bind(error);

   696     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back

   697     __ pop(rdx);                                 // get saved rdx back

   698     __ popf();                                   // get saved EFLAGS off stack -- will be ignored

   699     __ pusha();                                  // push registers (eip = return address & msg are already pushed)

   700     BLOCK_COMMENT("call MacroAssembler::debug");

   701     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));

   702     __ popa();

   703     __ ret(3 * wordSize);                        // pop arguments

   704     return start;

   705   }

   707   //

   708   //  Generate pre-barrier for array stores

   709   //

   710   //  Input:

   711   //     start   -  starting address

   712   //     end     -  element count

   713   void  gen_write_ref_array_pre_barrier(Register start, Register count) {

   714     assert_different_registers(start, count);

   715     BarrierSet* bs = Universe::heap()->barrier_set();

   716     switch (bs->kind()) {

   717       case BarrierSet::G1SATBCT:

   718       case BarrierSet::G1SATBCTLogging:

   719         {

   720           __ pusha();                      // push registers

   721           __ push(count);

   722           __ push(start);

   723           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)));

   724           __ addptr(rsp, 2*wordSize);

   725           __ popa();

   726         }

   727         break;

   728       case BarrierSet::CardTableModRef:

   729       case BarrierSet::CardTableExtension:

   730       case BarrierSet::ModRef:

   731         break;

   732       default      :

   733         ShouldNotReachHere();

   735     }

   736   }

   739   //

   740   // Generate a post-barrier for an array store

   741   //

   742   //     start    -  starting address

   743   //     count    -  element count

   744   //

   745   //  The two input registers are overwritten.

   746   //

   747   void  gen_write_ref_array_post_barrier(Register start, Register count) {

   748     BarrierSet* bs = Universe::heap()->barrier_set();

   749     assert_different_registers(start, count);

   750     switch (bs->kind()) {

   751       case BarrierSet::G1SATBCT:

   752       case BarrierSet::G1SATBCTLogging:

   753         {

   754           __ pusha();                      // push registers

   755           __ push(count);

   756           __ push(start);

   757           __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)));

   758           __ addptr(rsp, 2*wordSize);

   759           __ popa();

   761         }

   762         break;

   764       case BarrierSet::CardTableModRef:

   765       case BarrierSet::CardTableExtension:

   766         {

   767           CardTableModRefBS* ct = (CardTableModRefBS*)bs;

   768           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");

   770           Label L_loop;

   771           const Register end = count;  // elements count; end == start+count-1

   772           assert_different_registers(start, end);

   774           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));

   775           __ shrptr(start, CardTableModRefBS::card_shift);

   776           __ shrptr(end,   CardTableModRefBS::card_shift);

   777           __ subptr(end, start); // end --> count

   778         __ BIND(L_loop);

   779           intptr_t disp = (intptr_t) ct->byte_map_base;

   780           Address cardtable(start, count, Address::times_1, disp);

   781           __ movb(cardtable, 0);

   782           __ decrement(count);

   783           __ jcc(Assembler::greaterEqual, L_loop);

   784         }

   785         break;

   786       case BarrierSet::ModRef:

   787         break;

   788       default      :

   789         ShouldNotReachHere();

   791     }

   792   }

   794   // Copy 64 bytes chunks

   795   //

   796   // Inputs:

   797   //   from        - source array address

   798   //   to_from     - destination array address - from

   799   //   qword_count - 8-bytes element count, negative

   800   //

   801   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {

   802     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;

   803     // Copy 64-byte chunks

   804     __ jmpb(L_copy_64_bytes);

   805     __ align(16);

   806   __ BIND(L_copy_64_bytes_loop);

   807     __ movq(mmx0, Address(from, 0));

   808     __ movq(mmx1, Address(from, 8));

   809     __ movq(mmx2, Address(from, 16));

   810     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);

   811     __ movq(mmx3, Address(from, 24));

   812     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);

   813     __ movq(mmx4, Address(from, 32));

   814     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);

   815     __ movq(mmx5, Address(from, 40));

   816     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);

   817     __ movq(mmx6, Address(from, 48));

   818     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);

   819     __ movq(mmx7, Address(from, 56));

   820     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);

   821     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);

   822     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);

   823     __ addptr(from, 64);

   824   __ BIND(L_copy_64_bytes);

   825     __ subl(qword_count, 8);

   826     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);

   827     __ addl(qword_count, 8);

   828     __ jccb(Assembler::zero, L_exit);

   829     //

   830     // length is too short, just copy qwords

   831     //

   832   __ BIND(L_copy_8_bytes);

   833     __ movq(mmx0, Address(from, 0));

   834     __ movq(Address(from, to_from, Address::times_1), mmx0);

   835     __ addptr(from, 8);

   836     __ decrement(qword_count);

   837     __ jcc(Assembler::greater, L_copy_8_bytes);

   838   __ BIND(L_exit);

   839     __ emms();

   840   }

   842   address generate_disjoint_copy(BasicType t, bool aligned,

   843                                  Address::ScaleFactor sf,

   844                                  address* entry, const char *name) {

   845     __ align(CodeEntryAlignment);

   846     StubCodeMark mark(this, "StubRoutines", name);

   847     address start = __ pc();

   849     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;

   850     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;

   852     int shift = Address::times_ptr - sf;

   854     const Register from     = rsi;  // source array address

   855     const Register to       = rdi;  // destination array address

   856     const Register count    = rcx;  // elements count

   857     const Register to_from  = to;   // (to - from)

   858     const Register saved_to = rdx;  // saved destination array address

   860     __ enter(); // required for proper stackwalking of RuntimeStub frame

   861     __ push(rsi);

   862     __ push(rdi);

   863     __ movptr(from , Address(rsp, 12+ 4));

   864     __ movptr(to   , Address(rsp, 12+ 8));

   865     __ movl(count, Address(rsp, 12+ 12));

   866     if (t == T_OBJECT) {

   867       __ testl(count, count);

   868       __ jcc(Assembler::zero, L_0_count);

   869       gen_write_ref_array_pre_barrier(to, count);

   870       __ mov(saved_to, to);          // save 'to'

   871     }

   873     *entry = __ pc(); // Entry point from conjoint arraycopy stub.

   874     BLOCK_COMMENT("Entry:");

   876     __ subptr(to, from); // to --> to_from

   877     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

   878     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp

   879     if (!aligned && (t == T_BYTE || t == T_SHORT)) {

   880       // align source address at 4 bytes address boundary

   881       if (t == T_BYTE) {

   882         // One byte misalignment happens only for byte arrays

   883         __ testl(from, 1);

   884         __ jccb(Assembler::zero, L_skip_align1);

   885         __ movb(rax, Address(from, 0));

   886         __ movb(Address(from, to_from, Address::times_1, 0), rax);

   887         __ increment(from);

   888         __ decrement(count);

   889       __ BIND(L_skip_align1);

   890       }

   891       // Two bytes misalignment happens only for byte and short (char) arrays

   892       __ testl(from, 2);

   893       __ jccb(Assembler::zero, L_skip_align2);

   894       __ movw(rax, Address(from, 0));

   895       __ movw(Address(from, to_from, Address::times_1, 0), rax);

   896       __ addptr(from, 2);

   897       __ subl(count, 1<<(shift-1));

   898     __ BIND(L_skip_align2);

   899     }

   900     if (!VM_Version::supports_mmx()) {

   901       __ mov(rax, count);      // save 'count'

   902       __ shrl(count, shift); // bytes count

   903       __ addptr(to_from, from);// restore 'to'

   904       __ rep_mov();

   905       __ subptr(to_from, from);// restore 'to_from'

   906       __ mov(count, rax);      // restore 'count'

   907       __ jmpb(L_copy_2_bytes); // all dwords were copied

   908     } else {

   909       // align to 8 bytes, we know we are 4 byte aligned to start

   910       __ testptr(from, 4);

   911       __ jccb(Assembler::zero, L_copy_64_bytes);

   912       __ movl(rax, Address(from, 0));

   913       __ movl(Address(from, to_from, Address::times_1, 0), rax);

   914       __ addptr(from, 4);

   915       __ subl(count, 1<<shift);

   916     __ BIND(L_copy_64_bytes);

   917       __ mov(rax, count);

   918       __ shrl(rax, shift+1);  // 8 bytes chunk count

   919       //

   920       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop

   921       //

   922       mmx_copy_forward(from, to_from, rax);

   923     }

   924     // copy tailing dword

   925   __ BIND(L_copy_4_bytes);

   926     __ testl(count, 1<<shift);

   927     __ jccb(Assembler::zero, L_copy_2_bytes);

   928     __ movl(rax, Address(from, 0));

   929     __ movl(Address(from, to_from, Address::times_1, 0), rax);

   930     if (t == T_BYTE || t == T_SHORT) {

   931       __ addptr(from, 4);

   932     __ BIND(L_copy_2_bytes);

   933       // copy tailing word

   934       __ testl(count, 1<<(shift-1));

   935       __ jccb(Assembler::zero, L_copy_byte);

   936       __ movw(rax, Address(from, 0));

   937       __ movw(Address(from, to_from, Address::times_1, 0), rax);

   938       if (t == T_BYTE) {

   939         __ addptr(from, 2);

   940       __ BIND(L_copy_byte);

   941         // copy tailing byte

   942         __ testl(count, 1);

   943         __ jccb(Assembler::zero, L_exit);

   944         __ movb(rax, Address(from, 0));

   945         __ movb(Address(from, to_from, Address::times_1, 0), rax);

   946       __ BIND(L_exit);

   947       } else {

   948       __ BIND(L_copy_byte);

   949       }

   950     } else {

   951     __ BIND(L_copy_2_bytes);

   952     }

   954     if (t == T_OBJECT) {

   955       __ movl(count, Address(rsp, 12+12)); // reread 'count'

   956       __ mov(to, saved_to); // restore 'to'

   957       gen_write_ref_array_post_barrier(to, count);

   958     __ BIND(L_0_count);

   959     }

   960     inc_copy_counter_np(t);

   961     __ pop(rdi);

   962     __ pop(rsi);

   963     __ leave(); // required for proper stackwalking of RuntimeStub frame

   964     __ xorptr(rax, rax); // return 0

   965     __ ret(0);

   966     return start;

   967   }

   970   address generate_conjoint_copy(BasicType t, bool aligned,

   971                                  Address::ScaleFactor sf,

   972                                  address nooverlap_target,

   973                                  address* entry, const char *name) {

   974     __ align(CodeEntryAlignment);

   975     StubCodeMark mark(this, "StubRoutines", name);

   976     address start = __ pc();

   978     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;

   979     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;

   981     int shift = Address::times_ptr - sf;

   983     const Register src   = rax;  // source array address

   984     const Register dst   = rdx;  // destination array address

   985     const Register from  = rsi;  // source array address

   986     const Register to    = rdi;  // destination array address

   987     const Register count = rcx;  // elements count

   988     const Register end   = rax;  // array end address

   990     __ enter(); // required for proper stackwalking of RuntimeStub frame

   991     __ push(rsi);

   992     __ push(rdi);

   993     __ movptr(src  , Address(rsp, 12+ 4));   // from

   994     __ movptr(dst  , Address(rsp, 12+ 8));   // to

   995     __ movl2ptr(count, Address(rsp, 12+12)); // count

   996     if (t == T_OBJECT) {

   997        gen_write_ref_array_pre_barrier(dst, count);

   998     }

  1000     if (entry != NULL) {

  1001       *entry = __ pc(); // Entry point from generic arraycopy stub.

  1002       BLOCK_COMMENT("Entry:");

  1003     }

  1005     if (t == T_OBJECT) {

  1006       __ testl(count, count);

  1007       __ jcc(Assembler::zero, L_0_count);

  1008     }

  1009     __ mov(from, src);

  1010     __ mov(to  , dst);

  1012     // arrays overlap test

  1013     RuntimeAddress nooverlap(nooverlap_target);

  1014     __ cmpptr(dst, src);

  1015     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size

  1016     __ jump_cc(Assembler::belowEqual, nooverlap);

  1017     __ cmpptr(dst, end);

  1018     __ jump_cc(Assembler::aboveEqual, nooverlap);

  1020     // copy from high to low

  1021     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

  1022     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp

  1023     if (t == T_BYTE || t == T_SHORT) {

  1024       // Align the end of destination array at 4 bytes address boundary

  1025       __ lea(end, Address(dst, count, sf, 0));

  1026       if (t == T_BYTE) {

  1027         // One byte misalignment happens only for byte arrays

  1028         __ testl(end, 1);

  1029         __ jccb(Assembler::zero, L_skip_align1);

  1030         __ decrement(count);

  1031         __ movb(rdx, Address(from, count, sf, 0));

  1032         __ movb(Address(to, count, sf, 0), rdx);

  1033       __ BIND(L_skip_align1);

  1034       }

  1035       // Two bytes misalignment happens only for byte and short (char) arrays

  1036       __ testl(end, 2);

  1037       __ jccb(Assembler::zero, L_skip_align2);

  1038       __ subptr(count, 1<<(shift-1));

  1039       __ movw(rdx, Address(from, count, sf, 0));

  1040       __ movw(Address(to, count, sf, 0), rdx);

  1041     __ BIND(L_skip_align2);

  1042       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element

  1043       __ jcc(Assembler::below, L_copy_4_bytes);

  1044     }

  1046     if (!VM_Version::supports_mmx()) {

  1047       __ std();

  1048       __ mov(rax, count); // Save 'count'

  1049       __ mov(rdx, to);    // Save 'to'

  1050       __ lea(rsi, Address(from, count, sf, -4));

  1051       __ lea(rdi, Address(to  , count, sf, -4));

  1052       __ shrptr(count, shift); // bytes count

  1053       __ rep_mov();

  1054       __ cld();

  1055       __ mov(count, rax); // restore 'count'

  1056       __ andl(count, (1<<shift)-1);      // mask the number of rest elements

  1057       __ movptr(from, Address(rsp, 12+4)); // reread 'from'

  1058       __ mov(to, rdx);   // restore 'to'

  1059       __ jmpb(L_copy_2_bytes); // all dword were copied

  1060    } else {

  1061       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.

  1062       __ testptr(end, 4);

  1063       __ jccb(Assembler::zero, L_copy_8_bytes);

  1064       __ subl(count, 1<<shift);

  1065       __ movl(rdx, Address(from, count, sf, 0));

  1066       __ movl(Address(to, count, sf, 0), rdx);

  1067       __ jmpb(L_copy_8_bytes);

  1069       __ align(16);

  1070       // Move 8 bytes

  1071     __ BIND(L_copy_8_bytes_loop);

  1072       __ movq(mmx0, Address(from, count, sf, 0));

  1073       __ movq(Address(to, count, sf, 0), mmx0);

  1074     __ BIND(L_copy_8_bytes);

  1075       __ subl(count, 2<<shift);

  1076       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1077       __ addl(count, 2<<shift);

  1078       __ emms();

  1079     }

  1080   __ BIND(L_copy_4_bytes);

  1081     // copy prefix qword

  1082     __ testl(count, 1<<shift);

  1083     __ jccb(Assembler::zero, L_copy_2_bytes);

  1084     __ movl(rdx, Address(from, count, sf, -4));

  1085     __ movl(Address(to, count, sf, -4), rdx);

  1087     if (t == T_BYTE || t == T_SHORT) {

  1088         __ subl(count, (1<<shift));

  1089       __ BIND(L_copy_2_bytes);

  1090         // copy prefix dword

  1091         __ testl(count, 1<<(shift-1));

  1092         __ jccb(Assembler::zero, L_copy_byte);

  1093         __ movw(rdx, Address(from, count, sf, -2));

  1094         __ movw(Address(to, count, sf, -2), rdx);

  1095         if (t == T_BYTE) {

  1096           __ subl(count, 1<<(shift-1));

  1097         __ BIND(L_copy_byte);

  1098           // copy prefix byte

  1099           __ testl(count, 1);

  1100           __ jccb(Assembler::zero, L_exit);

  1101           __ movb(rdx, Address(from, 0));

  1102           __ movb(Address(to, 0), rdx);

  1103         __ BIND(L_exit);

  1104         } else {

  1105         __ BIND(L_copy_byte);

  1106         }

  1107     } else {

  1108     __ BIND(L_copy_2_bytes);

  1109     }

  1110     if (t == T_OBJECT) {

  1111       __ movl2ptr(count, Address(rsp, 12+12)); // reread count

  1112       gen_write_ref_array_post_barrier(to, count);

  1113     __ BIND(L_0_count);

  1114     }

  1115     inc_copy_counter_np(t);

  1116     __ pop(rdi);

  1117     __ pop(rsi);

  1118     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1119     __ xorptr(rax, rax); // return 0

  1120     __ ret(0);

  1121     return start;

  1122   }

  1125   address generate_disjoint_long_copy(address* entry, const char *name) {

  1126     __ align(CodeEntryAlignment);

  1127     StubCodeMark mark(this, "StubRoutines", name);

  1128     address start = __ pc();

  1130     Label L_copy_8_bytes, L_copy_8_bytes_loop;

  1131     const Register from       = rax;  // source array address

  1132     const Register to         = rdx;  // destination array address

  1133     const Register count      = rcx;  // elements count

  1134     const Register to_from    = rdx;  // (to - from)

  1136     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1137     __ movptr(from , Address(rsp, 8+0));       // from

  1138     __ movptr(to   , Address(rsp, 8+4));       // to

  1139     __ movl2ptr(count, Address(rsp, 8+8));     // count

  1141     *entry = __ pc(); // Entry point from conjoint arraycopy stub.

  1142     BLOCK_COMMENT("Entry:");

  1144     __ subptr(to, from); // to --> to_from

  1145     if (VM_Version::supports_mmx()) {

  1146       mmx_copy_forward(from, to_from, count);

  1147     } else {

  1148       __ jmpb(L_copy_8_bytes);

  1149       __ align(16);

  1150     __ BIND(L_copy_8_bytes_loop);

  1151       __ fild_d(Address(from, 0));

  1152       __ fistp_d(Address(from, to_from, Address::times_1));

  1153       __ addptr(from, 8);

  1154     __ BIND(L_copy_8_bytes);

  1155       __ decrement(count);

  1156       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1157     }

  1158     inc_copy_counter_np(T_LONG);

  1159     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1160     __ xorptr(rax, rax); // return 0

  1161     __ ret(0);

  1162     return start;

  1163   }

  1165   address generate_conjoint_long_copy(address nooverlap_target,

  1166                                       address* entry, const char *name) {

  1167     __ align(CodeEntryAlignment);

  1168     StubCodeMark mark(this, "StubRoutines", name);

  1169     address start = __ pc();

  1171     Label L_copy_8_bytes, L_copy_8_bytes_loop;

  1172     const Register from       = rax;  // source array address

  1173     const Register to         = rdx;  // destination array address

  1174     const Register count      = rcx;  // elements count

  1175     const Register end_from   = rax;  // source array end address

  1177     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1178     __ movptr(from , Address(rsp, 8+0));       // from

  1179     __ movptr(to   , Address(rsp, 8+4));       // to

  1180     __ movl2ptr(count, Address(rsp, 8+8));     // count

  1182     *entry = __ pc(); // Entry point from generic arraycopy stub.

  1183     BLOCK_COMMENT("Entry:");

  1185     // arrays overlap test

  1186     __ cmpptr(to, from);

  1187     RuntimeAddress nooverlap(nooverlap_target);

  1188     __ jump_cc(Assembler::belowEqual, nooverlap);

  1189     __ lea(end_from, Address(from, count, Address::times_8, 0));

  1190     __ cmpptr(to, end_from);

  1191     __ movptr(from, Address(rsp, 8));  // from

  1192     __ jump_cc(Assembler::aboveEqual, nooverlap);

  1194     __ jmpb(L_copy_8_bytes);

  1196     __ align(16);

  1197   __ BIND(L_copy_8_bytes_loop);

  1198     if (VM_Version::supports_mmx()) {

  1199       __ movq(mmx0, Address(from, count, Address::times_8));

  1200       __ movq(Address(to, count, Address::times_8), mmx0);

  1201     } else {

  1202       __ fild_d(Address(from, count, Address::times_8));

  1203       __ fistp_d(Address(to, count, Address::times_8));

  1204     }

  1205   __ BIND(L_copy_8_bytes);

  1206     __ decrement(count);

  1207     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);

  1209     if (VM_Version::supports_mmx()) {

  1210       __ emms();

  1211     }

  1212     inc_copy_counter_np(T_LONG);

  1213     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1214     __ xorptr(rax, rax); // return 0

  1215     __ ret(0);

  1216     return start;

  1217   }

  1220   // Helper for generating a dynamic type check.

  1221   // The sub_klass must be one of {rbx, rdx, rsi}.

  1222   // The temp is killed.

  1223   void generate_type_check(Register sub_klass,

  1224                            Address& super_check_offset_addr,

  1225                            Address& super_klass_addr,

  1226                            Register temp,

  1227                            Label* L_success_ptr, Label* L_failure_ptr) {

  1228     BLOCK_COMMENT("type_check:");

  1230     Label L_fallthrough;

  1231     bool fall_through_on_success = (L_success_ptr == NULL);

  1232     if (fall_through_on_success) {

  1233       L_success_ptr = &L_fallthrough;

  1234     } else {

  1235       L_failure_ptr = &L_fallthrough;

  1236     }

  1237     Label& L_success = *L_success_ptr;

  1238     Label& L_failure = *L_failure_ptr;

  1240     assert_different_registers(sub_klass, temp);

  1242     // a couple of useful fields in sub_klass:

  1243     int ss_offset = (klassOopDesc::header_size() * HeapWordSize +

  1244                      Klass::secondary_supers_offset_in_bytes());

  1245     int sc_offset = (klassOopDesc::header_size() * HeapWordSize +

  1246                      Klass::secondary_super_cache_offset_in_bytes());

  1247     Address secondary_supers_addr(sub_klass, ss_offset);

  1248     Address super_cache_addr(     sub_klass, sc_offset);

  1250     // if the pointers are equal, we are done (e.g., String[] elements)

  1251     __ cmpptr(sub_klass, super_klass_addr);

  1252     __ jcc(Assembler::equal, L_success);

  1254     // check the supertype display:

  1255     __ movl2ptr(temp, super_check_offset_addr);

  1256     Address super_check_addr(sub_klass, temp, Address::times_1, 0);

  1257     __ movptr(temp, super_check_addr); // load displayed supertype

  1258     __ cmpptr(temp, super_klass_addr); // test the super type

  1259     __ jcc(Assembler::equal, L_success);

  1261     // if it was a primary super, we can just fail immediately

  1262     __ cmpl(super_check_offset_addr, sc_offset);

  1263     __ jcc(Assembler::notEqual, L_failure);

  1265     // Now do a linear scan of the secondary super-klass chain.

  1266     // This code is rarely used, so simplicity is a virtue here.

  1267     inc_counter_np(SharedRuntime::_partial_subtype_ctr);

  1268     {

  1269       // The repne_scan instruction uses fixed registers, which we must spill.

  1270       // (We need a couple more temps in any case.)

  1271       __ push(rax);

  1272       __ push(rcx);

  1273       __ push(rdi);

  1274       assert_different_registers(sub_klass, rax, rcx, rdi);

  1276       __ movptr(rdi, secondary_supers_addr);

  1277       // Load the array length.

  1278       __ movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes()));

  1279       // Skip to start of data.

  1280       __ addptr(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT));

  1281       // Scan rcx words at [edi] for occurance of rax,

  1282       // Set NZ/Z based on last compare

  1283       __ movptr(rax, super_klass_addr);

  1284       __ repne_scan();

  1286       // Unspill the temp. registers:

  1287       __ pop(rdi);

  1288       __ pop(rcx);

  1289       __ pop(rax);

  1290     }

  1291     __ jcc(Assembler::notEqual, L_failure);

  1293     // Success.  Cache the super we found and proceed in triumph.

  1294     __ movptr(temp, super_klass_addr); // note: rax, is dead

  1295     __ movptr(super_cache_addr, temp);

  1297     if (!fall_through_on_success)

  1298       __ jmp(L_success);

  1300     // Fall through on failure!

  1301     __ bind(L_fallthrough);

  1302   }

  1304   //

  1305   //  Generate checkcasting array copy stub

  1306   //

  1307   //  Input:

  1308   //    4(rsp)   - source array address

  1309   //    8(rsp)   - destination array address

  1310   //   12(rsp)   - element count, can be zero

  1311   //   16(rsp)   - size_t ckoff (super_check_offset)

  1312   //   20(rsp)   - oop ckval (super_klass)

  1313   //

  1314   //  Output:

  1315   //    rax, ==  0  -  success

  1316   //    rax, == -1^K - failure, where K is partial transfer count

  1317   //

  1318   address generate_checkcast_copy(const char *name, address* entry) {

  1319     __ align(CodeEntryAlignment);

  1320     StubCodeMark mark(this, "StubRoutines", name);

  1321     address start = __ pc();

  1323     Label L_load_element, L_store_element, L_do_card_marks, L_done;

  1325     // register use:

  1326     //  rax, rdx, rcx -- loop control (end_from, end_to, count)

  1327     //  rdi, rsi      -- element access (oop, klass)

  1328     //  rbx,           -- temp

  1329     const Register from       = rax;    // source array address

  1330     const Register to         = rdx;    // destination array address

  1331     const Register length     = rcx;    // elements count

  1332     const Register elem       = rdi;    // each oop copied

  1333     const Register elem_klass = rsi;    // each elem._klass (sub_klass)

  1334     const Register temp       = rbx;    // lone remaining temp

  1336     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1338     __ push(rsi);

  1339     __ push(rdi);

  1340     __ push(rbx);

  1342     Address   from_arg(rsp, 16+ 4);     // from

  1343     Address     to_arg(rsp, 16+ 8);     // to

  1344     Address length_arg(rsp, 16+12);     // elements count

  1345     Address  ckoff_arg(rsp, 16+16);     // super_check_offset

  1346     Address  ckval_arg(rsp, 16+20);     // super_klass

  1348     // Load up:

  1349     __ movptr(from,     from_arg);

  1350     __ movptr(to,         to_arg);

  1351     __ movl2ptr(length, length_arg);

  1353     *entry = __ pc(); // Entry point from generic arraycopy stub.

  1354     BLOCK_COMMENT("Entry:");

  1356     //---------------------------------------------------------------

  1357     // Assembler stub will be used for this call to arraycopy

  1358     // if the two arrays are subtypes of Object[] but the

  1359     // destination array type is not equal to or a supertype

  1360     // of the source type.  Each element must be separately

  1361     // checked.

  1363     // Loop-invariant addresses.  They are exclusive end pointers.

  1364     Address end_from_addr(from, length, Address::times_ptr, 0);

  1365     Address   end_to_addr(to,   length, Address::times_ptr, 0);

  1367     Register end_from = from;           // re-use

  1368     Register end_to   = to;             // re-use

  1369     Register count    = length;         // re-use

  1371     // Loop-variant addresses.  They assume post-incremented count < 0.

  1372     Address from_element_addr(end_from, count, Address::times_ptr, 0);

  1373     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);

  1374     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());

  1376     // Copy from low to high addresses, indexed from the end of each array.

  1377     gen_write_ref_array_pre_barrier(to, count);

  1378     __ lea(end_from, end_from_addr);

  1379     __ lea(end_to,   end_to_addr);

  1380     assert(length == count, "");        // else fix next line:

  1381     __ negptr(count);                   // negate and test the length

  1382     __ jccb(Assembler::notZero, L_load_element);

  1384     // Empty array:  Nothing to do.

  1385     __ xorptr(rax, rax);                  // return 0 on (trivial) success

  1386     __ jmp(L_done);

  1388     // ======== begin loop ========

  1389     // (Loop is rotated; its entry is L_load_element.)

  1390     // Loop control:

  1391     //   for (count = -count; count != 0; count++)

  1392     // Base pointers src, dst are biased by 8*count,to last element.

  1393     __ align(16);

  1395     __ BIND(L_store_element);

  1396     __ movptr(to_element_addr, elem);     // store the oop

  1397     __ increment(count);                // increment the count toward zero

  1398     __ jccb(Assembler::zero, L_do_card_marks);

  1400     // ======== loop entry is here ========

  1401     __ BIND(L_load_element);

  1402     __ movptr(elem, from_element_addr);   // load the oop

  1403     __ testptr(elem, elem);

  1404     __ jccb(Assembler::zero, L_store_element);

  1406     // (Could do a trick here:  Remember last successful non-null

  1407     // element stored and make a quick oop equality check on it.)

  1409     __ movptr(elem_klass, elem_klass_addr); // query the object klass

  1410     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,

  1411                         &L_store_element, NULL);

  1412       // (On fall-through, we have failed the element type check.)

  1413     // ======== end loop ========

  1415     // It was a real error; we must depend on the caller to finish the job.

  1416     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.

  1417     // Emit GC store barriers for the oops we have copied (length_arg + count),

  1418     // and report their number to the caller.

  1419     __ addl(count, length_arg);         // transfers = (length - remaining)

  1420     __ movl2ptr(rax, count);            // save the value

  1421     __ notptr(rax);                     // report (-1^K) to caller

  1422     __ movptr(to, to_arg);              // reload

  1423     assert_different_registers(to, count, rax);

  1424     gen_write_ref_array_post_barrier(to, count);

  1425     __ jmpb(L_done);

  1427     // Come here on success only.

  1428     __ BIND(L_do_card_marks);

  1429     __ movl2ptr(count, length_arg);

  1430     __ movptr(to, to_arg);                // reload

  1431     gen_write_ref_array_post_barrier(to, count);

  1432     __ xorptr(rax, rax);                  // return 0 on success

  1434     // Common exit point (success or failure).

  1435     __ BIND(L_done);

  1436     __ pop(rbx);

  1437     __ pop(rdi);

  1438     __ pop(rsi);

  1439     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);

  1440     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1441     __ ret(0);

  1443     return start;

  1444   }

  1446   //

  1447   //  Generate 'unsafe' array copy stub

  1448   //  Though just as safe as the other stubs, it takes an unscaled

  1449   //  size_t argument instead of an element count.

  1450   //

  1451   //  Input:

  1452   //    4(rsp)   - source array address

  1453   //    8(rsp)   - destination array address

  1454   //   12(rsp)   - byte count, can be zero

  1455   //

  1456   //  Output:

  1457   //    rax, ==  0  -  success

  1458   //    rax, == -1  -  need to call System.arraycopy

  1459   //

  1460   // Examines the alignment of the operands and dispatches

  1461   // to a long, int, short, or byte copy loop.

  1462   //

  1463   address generate_unsafe_copy(const char *name,

  1464                                address byte_copy_entry,

  1465                                address short_copy_entry,

  1466                                address int_copy_entry,

  1467                                address long_copy_entry) {

  1469     Label L_long_aligned, L_int_aligned, L_short_aligned;

  1471     __ align(CodeEntryAlignment);

  1472     StubCodeMark mark(this, "StubRoutines", name);

  1473     address start = __ pc();

  1475     const Register from       = rax;  // source array address

  1476     const Register to         = rdx;  // destination array address

  1477     const Register count      = rcx;  // elements count

  1479     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1480     __ push(rsi);

  1481     __ push(rdi);

  1482     Address  from_arg(rsp, 12+ 4);      // from

  1483     Address    to_arg(rsp, 12+ 8);      // to

  1484     Address count_arg(rsp, 12+12);      // byte count

  1486     // Load up:

  1487     __ movptr(from ,  from_arg);

  1488     __ movptr(to   ,    to_arg);

  1489     __ movl2ptr(count, count_arg);

  1491     // bump this on entry, not on exit:

  1492     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);

  1494     const Register bits = rsi;

  1495     __ mov(bits, from);

  1496     __ orptr(bits, to);

  1497     __ orptr(bits, count);

  1499     __ testl(bits, BytesPerLong-1);

  1500     __ jccb(Assembler::zero, L_long_aligned);

  1502     __ testl(bits, BytesPerInt-1);

  1503     __ jccb(Assembler::zero, L_int_aligned);

  1505     __ testl(bits, BytesPerShort-1);

  1506     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));

  1508     __ BIND(L_short_aligned);

  1509     __ shrptr(count, LogBytesPerShort); // size => short_count

  1510     __ movl(count_arg, count);          // update 'count'

  1511     __ jump(RuntimeAddress(short_copy_entry));

  1513     __ BIND(L_int_aligned);

  1514     __ shrptr(count, LogBytesPerInt); // size => int_count

  1515     __ movl(count_arg, count);          // update 'count'

  1516     __ jump(RuntimeAddress(int_copy_entry));

  1518     __ BIND(L_long_aligned);

  1519     __ shrptr(count, LogBytesPerLong); // size => qword_count

  1520     __ movl(count_arg, count);          // update 'count'

  1521     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.

  1522     __ pop(rsi);

  1523     __ jump(RuntimeAddress(long_copy_entry));

  1525     return start;

  1526   }

  1529   // Perform range checks on the proposed arraycopy.

  1530   // Smashes src_pos and dst_pos.  (Uses them up for temps.)

  1531   void arraycopy_range_checks(Register src,

  1532                               Register src_pos,

  1533                               Register dst,

  1534                               Register dst_pos,

  1535                               Address& length,

  1536                               Label& L_failed) {

  1537     BLOCK_COMMENT("arraycopy_range_checks:");

  1538     const Register src_end = src_pos;   // source array end position

  1539     const Register dst_end = dst_pos;   // destination array end position

  1540     __ addl(src_end, length); // src_pos + length

  1541     __ addl(dst_end, length); // dst_pos + length

  1543     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;

  1544     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));

  1545     __ jcc(Assembler::above, L_failed);

  1547     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;

  1548     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));

  1549     __ jcc(Assembler::above, L_failed);

  1551     BLOCK_COMMENT("arraycopy_range_checks done");

  1552   }

  1555   //

  1556   //  Generate generic array copy stubs

  1557   //

  1558   //  Input:

  1559   //     4(rsp)    -  src oop

  1560   //     8(rsp)    -  src_pos

  1561   //    12(rsp)    -  dst oop

  1562   //    16(rsp)    -  dst_pos

  1563   //    20(rsp)    -  element count

  1564   //

  1565   //  Output:

  1566   //    rax, ==  0  -  success

  1567   //    rax, == -1^K - failure, where K is partial transfer count

  1568   //

  1569   address generate_generic_copy(const char *name,

  1570                                 address entry_jbyte_arraycopy,

  1571                                 address entry_jshort_arraycopy,

  1572                                 address entry_jint_arraycopy,

  1573                                 address entry_oop_arraycopy,

  1574                                 address entry_jlong_arraycopy,

  1575                                 address entry_checkcast_arraycopy) {

  1576     Label L_failed, L_failed_0, L_objArray;

  1578     { int modulus = CodeEntryAlignment;

  1579       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)

  1580       int advance = target - (__ offset() % modulus);

  1581       if (advance < 0)  advance += modulus;

  1582       if (advance > 0)  __ nop(advance);

  1583     }

  1584     StubCodeMark mark(this, "StubRoutines", name);

  1586     // Short-hop target to L_failed.  Makes for denser prologue code.

  1587     __ BIND(L_failed_0);

  1588     __ jmp(L_failed);

  1589     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");

  1591     __ align(CodeEntryAlignment);

  1592     address start = __ pc();

  1594     __ enter(); // required for proper stackwalking of RuntimeStub frame

  1595     __ push(rsi);

  1596     __ push(rdi);

  1598     // bump this on entry, not on exit:

  1599     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);

  1601     // Input values

  1602     Address SRC     (rsp, 12+ 4);

  1603     Address SRC_POS (rsp, 12+ 8);

  1604     Address DST     (rsp, 12+12);

  1605     Address DST_POS (rsp, 12+16);

  1606     Address LENGTH  (rsp, 12+20);

  1608     //-----------------------------------------------------------------------

  1609     // Assembler stub will be used for this call to arraycopy

  1610     // if the following conditions are met:

  1611     //

  1612     // (1) src and dst must not be null.

  1613     // (2) src_pos must not be negative.

  1614     // (3) dst_pos must not be negative.

  1615     // (4) length  must not be negative.

  1616     // (5) src klass and dst klass should be the same and not NULL.

  1617     // (6) src and dst should be arrays.

  1618     // (7) src_pos + length must not exceed length of src.

  1619     // (8) dst_pos + length must not exceed length of dst.

  1620     //

  1622     const Register src     = rax;       // source array oop

  1623     const Register src_pos = rsi;

  1624     const Register dst     = rdx;       // destination array oop

  1625     const Register dst_pos = rdi;

  1626     const Register length  = rcx;       // transfer count

  1628     //  if (src == NULL) return -1;

  1629     __ movptr(src, SRC);      // src oop

  1630     __ testptr(src, src);

  1631     __ jccb(Assembler::zero, L_failed_0);

  1633     //  if (src_pos < 0) return -1;

  1634     __ movl2ptr(src_pos, SRC_POS);  // src_pos

  1635     __ testl(src_pos, src_pos);

  1636     __ jccb(Assembler::negative, L_failed_0);

  1638     //  if (dst == NULL) return -1;

  1639     __ movptr(dst, DST);      // dst oop

  1640     __ testptr(dst, dst);

  1641     __ jccb(Assembler::zero, L_failed_0);

  1643     //  if (dst_pos < 0) return -1;

  1644     __ movl2ptr(dst_pos, DST_POS);  // dst_pos

  1645     __ testl(dst_pos, dst_pos);

  1646     __ jccb(Assembler::negative, L_failed_0);

  1648     //  if (length < 0) return -1;

  1649     __ movl2ptr(length, LENGTH);   // length

  1650     __ testl(length, length);

  1651     __ jccb(Assembler::negative, L_failed_0);

  1653     //  if (src->klass() == NULL) return -1;

  1654     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());

  1655     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());

  1656     const Register rcx_src_klass = rcx;    // array klass

  1657     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));

  1659 #ifdef ASSERT

  1660     //  assert(src->klass() != NULL);

  1661     BLOCK_COMMENT("assert klasses not null");

  1662     { Label L1, L2;

  1663       __ testptr(rcx_src_klass, rcx_src_klass);

  1664       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL

  1665       __ bind(L1);

  1666       __ stop("broken null klass");

  1667       __ bind(L2);

  1668       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);

  1669       __ jccb(Assembler::equal, L1);      // this would be broken also

  1670       BLOCK_COMMENT("assert done");

  1671     }

  1672 #endif //ASSERT

  1674     // Load layout helper (32-bits)

  1675     //

  1676     //  |array_tag|     | header_size | element_type |     |log2_element_size|

  1677     // 32        30    24            16              8     2                 0

  1678     //

  1679     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0

  1680     //

  1682     int lh_offset = klassOopDesc::header_size() * HeapWordSize +

  1683                     Klass::layout_helper_offset_in_bytes();

  1684     Address src_klass_lh_addr(rcx_src_klass, lh_offset);

  1686     // Handle objArrays completely differently...

  1687     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);

  1688     __ cmpl(src_klass_lh_addr, objArray_lh);

  1689     __ jcc(Assembler::equal, L_objArray);

  1691     //  if (src->klass() != dst->klass()) return -1;

  1692     __ cmpptr(rcx_src_klass, dst_klass_addr);

  1693     __ jccb(Assembler::notEqual, L_failed_0);

  1695     const Register rcx_lh = rcx;  // layout helper

  1696     assert(rcx_lh == rcx_src_klass, "known alias");

  1697     __ movl(rcx_lh, src_klass_lh_addr);

  1699     //  if (!src->is_Array()) return -1;

  1700     __ cmpl(rcx_lh, Klass::_lh_neutral_value);

  1701     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp

  1703     // At this point, it is known to be a typeArray (array_tag 0x3).

  1704 #ifdef ASSERT

  1705     { Label L;

  1706       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));

  1707       __ jcc(Assembler::greaterEqual, L); // signed cmp

  1708       __ stop("must be a primitive array");

  1709       __ bind(L);

  1710     }

  1711 #endif

  1713     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);

  1714     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1716     // typeArrayKlass

  1717     //

  1718     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);

  1719     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);

  1720     //

  1721     const Register rsi_offset = rsi; // array offset

  1722     const Register src_array  = src; // src array offset

  1723     const Register dst_array  = dst; // dst array offset

  1724     const Register rdi_elsize = rdi; // log2 element size

  1726     __ mov(rsi_offset, rcx_lh);

  1727     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);

  1728     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset

  1729     __ addptr(src_array, rsi_offset);  // src array offset

  1730     __ addptr(dst_array, rsi_offset);  // dst array offset

  1731     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize

  1733     // next registers should be set before the jump to corresponding stub

  1734     const Register from       = src; // source array address

  1735     const Register to         = dst; // destination array address

  1736     const Register count      = rcx; // elements count

  1737     // some of them should be duplicated on stack

  1738 #define FROM   Address(rsp, 12+ 4)

  1739 #define TO     Address(rsp, 12+ 8)   // Not used now

  1740 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy

  1742     BLOCK_COMMENT("scale indexes to element size");

  1743     __ movl2ptr(rsi, SRC_POS);  // src_pos

  1744     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)

  1745     assert(src_array == from, "");

  1746     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize

  1747     __ movl2ptr(rdi, DST_POS);  // dst_pos

  1748     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)

  1749     assert(dst_array == to, "");

  1750     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize

  1751     __ movptr(FROM, from);      // src_addr

  1752     __ mov(rdi_elsize, rcx_lh); // log2 elsize

  1753     __ movl2ptr(count, LENGTH); // elements count

  1755     BLOCK_COMMENT("choose copy loop based on element size");

  1756     __ cmpl(rdi_elsize, 0);

  1758     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));

  1759     __ cmpl(rdi_elsize, LogBytesPerShort);

  1760     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));

  1761     __ cmpl(rdi_elsize, LogBytesPerInt);

  1762     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));

  1763 #ifdef ASSERT

  1764     __ cmpl(rdi_elsize, LogBytesPerLong);

  1765     __ jccb(Assembler::notEqual, L_failed);

  1766 #endif

  1767     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.

  1768     __ pop(rsi);

  1769     __ jump(RuntimeAddress(entry_jlong_arraycopy));

  1771   __ BIND(L_failed);

  1772     __ xorptr(rax, rax);

  1773     __ notptr(rax); // return -1

  1774     __ pop(rdi);

  1775     __ pop(rsi);

  1776     __ leave(); // required for proper stackwalking of RuntimeStub frame

  1777     __ ret(0);

  1779     // objArrayKlass

  1780   __ BIND(L_objArray);

  1781     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]

  1783     Label L_plain_copy, L_checkcast_copy;

  1784     //  test array classes for subtyping

  1785     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality

  1786     __ jccb(Assembler::notEqual, L_checkcast_copy);

  1788     // Identically typed arrays can be copied without element-wise checks.

  1789     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);

  1790     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1792   __ BIND(L_plain_copy);

  1793     __ movl2ptr(count, LENGTH); // elements count

  1794     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos

  1795     __ lea(from, Address(src, src_pos, Address::times_ptr,

  1796                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr

  1797     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos

  1798     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,

  1799                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr

  1800     __ movptr(FROM,  from);   // src_addr

  1801     __ movptr(TO,    to);     // dst_addr

  1802     __ movl(COUNT, count);  // count

  1803     __ jump(RuntimeAddress(entry_oop_arraycopy));

  1805   __ BIND(L_checkcast_copy);

  1806     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]

  1807     {

  1808       // Handy offsets:

  1809       int  ek_offset = (klassOopDesc::header_size() * HeapWordSize +

  1810                         objArrayKlass::element_klass_offset_in_bytes());

  1811       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +

  1812                         Klass::super_check_offset_offset_in_bytes());

  1814       Register rsi_dst_klass = rsi;

  1815       Register rdi_temp      = rdi;

  1816       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");

  1817       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");

  1818       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);

  1820       // Before looking at dst.length, make sure dst is also an objArray.

  1821       __ movptr(rsi_dst_klass, dst_klass_addr);

  1822       __ cmpl(dst_klass_lh_addr, objArray_lh);

  1823       __ jccb(Assembler::notEqual, L_failed);

  1825       // It is safe to examine both src.length and dst.length.

  1826       __ movl2ptr(src_pos, SRC_POS);        // reload rsi

  1827       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);

  1828       // (Now src_pos and dst_pos are killed, but not src and dst.)

  1830       // We'll need this temp (don't forget to pop it after the type check).

  1831       __ push(rbx);

  1832       Register rbx_src_klass = rbx;

  1834       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx

  1835       __ movptr(rsi_dst_klass, dst_klass_addr);

  1836       Address super_check_offset_addr(rsi_dst_klass, sco_offset);

  1837       Label L_fail_array_check;

  1838       generate_type_check(rbx_src_klass,

  1839                           super_check_offset_addr, dst_klass_addr,

  1840                           rdi_temp, NULL, &L_fail_array_check);

  1841       // (On fall-through, we have passed the array type check.)

  1842       __ pop(rbx);

  1843       __ jmp(L_plain_copy);

  1845       __ BIND(L_fail_array_check);

  1846       // Reshuffle arguments so we can call checkcast_arraycopy:

  1848       // match initial saves for checkcast_arraycopy

  1849       // push(rsi);    // already done; see above

  1850       // push(rdi);    // already done; see above

  1851       // push(rbx);    // already done; see above

  1853       // Marshal outgoing arguments now, freeing registers.

  1854       Address   from_arg(rsp, 16+ 4);   // from

  1855       Address     to_arg(rsp, 16+ 8);   // to

  1856       Address length_arg(rsp, 16+12);   // elements count

  1857       Address  ckoff_arg(rsp, 16+16);   // super_check_offset

  1858       Address  ckval_arg(rsp, 16+20);   // super_klass

  1860       Address SRC_POS_arg(rsp, 16+ 8);

  1861       Address DST_POS_arg(rsp, 16+16);

  1862       Address  LENGTH_arg(rsp, 16+20);

  1863       // push rbx, changed the incoming offsets (why not just use rbp,??)

  1864       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");

  1866       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));

  1867       __ movl2ptr(length, LENGTH_arg);    // reload elements count

  1868       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos

  1869       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos

  1871       __ movptr(ckval_arg, rbx);          // destination element type

  1872       __ movl(rbx, Address(rbx, sco_offset));

  1873       __ movl(ckoff_arg, rbx);          // corresponding class check offset

  1875       __ movl(length_arg, length);      // outgoing length argument

  1877       __ lea(from, Address(src, src_pos, Address::times_ptr,

  1878                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

  1879       __ movptr(from_arg, from);

  1881       __ lea(to, Address(dst, dst_pos, Address::times_ptr,

  1882                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));

  1883       __ movptr(to_arg, to);

  1884       __ jump(RuntimeAddress(entry_checkcast_arraycopy));

  1885     }

  1887     return start;

  1888   }

  1890   void generate_arraycopy_stubs() {

  1891     address entry;

  1892     address entry_jbyte_arraycopy;

  1893     address entry_jshort_arraycopy;

  1894     address entry_jint_arraycopy;

  1895     address entry_oop_arraycopy;

  1896     address entry_jlong_arraycopy;

  1897     address entry_checkcast_arraycopy;

  1899     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =

  1900         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,

  1901                                "arrayof_jbyte_disjoint_arraycopy");

  1902     StubRoutines::_arrayof_jbyte_arraycopy =

  1903         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,

  1904                                NULL, "arrayof_jbyte_arraycopy");

  1905     StubRoutines::_jbyte_disjoint_arraycopy =

  1906         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,

  1907                                "jbyte_disjoint_arraycopy");

  1908     StubRoutines::_jbyte_arraycopy =

  1909         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,

  1910                                &entry_jbyte_arraycopy, "jbyte_arraycopy");

  1912     StubRoutines::_arrayof_jshort_disjoint_arraycopy =

  1913         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,

  1914                                "arrayof_jshort_disjoint_arraycopy");

  1915     StubRoutines::_arrayof_jshort_arraycopy =

  1916         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,

  1917                                NULL, "arrayof_jshort_arraycopy");

  1918     StubRoutines::_jshort_disjoint_arraycopy =

  1919         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,

  1920                                "jshort_disjoint_arraycopy");

  1921     StubRoutines::_jshort_arraycopy =

  1922         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,

  1923                                &entry_jshort_arraycopy, "jshort_arraycopy");

  1925     // Next arrays are always aligned on 4 bytes at least.

  1926     StubRoutines::_jint_disjoint_arraycopy =

  1927         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,

  1928                                "jint_disjoint_arraycopy");

  1929     StubRoutines::_jint_arraycopy =

  1930         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,

  1931                                &entry_jint_arraycopy, "jint_arraycopy");

  1933     StubRoutines::_oop_disjoint_arraycopy =

  1934         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,

  1935                                "oop_disjoint_arraycopy");

  1936     StubRoutines::_oop_arraycopy =

  1937         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,

  1938                                &entry_oop_arraycopy, "oop_arraycopy");

  1940     StubRoutines::_jlong_disjoint_arraycopy =

  1941         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");

  1942     StubRoutines::_jlong_arraycopy =

  1943         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,

  1944                                     "jlong_arraycopy");

  1946     StubRoutines::_arrayof_jint_disjoint_arraycopy  =

  1947         StubRoutines::_jint_disjoint_arraycopy;

  1948     StubRoutines::_arrayof_oop_disjoint_arraycopy   =

  1949         StubRoutines::_oop_disjoint_arraycopy;

  1950     StubRoutines::_arrayof_jlong_disjoint_arraycopy =

  1951         StubRoutines::_jlong_disjoint_arraycopy;

  1953     StubRoutines::_arrayof_jint_arraycopy  = StubRoutines::_jint_arraycopy;

  1954     StubRoutines::_arrayof_oop_arraycopy   = StubRoutines::_oop_arraycopy;

  1955     StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy;

  1957     StubRoutines::_checkcast_arraycopy =

  1958         generate_checkcast_copy("checkcast_arraycopy",

  1959                                   &entry_checkcast_arraycopy);

  1961     StubRoutines::_unsafe_arraycopy =

  1962         generate_unsafe_copy("unsafe_arraycopy",

  1963                                entry_jbyte_arraycopy,

  1964                                entry_jshort_arraycopy,

  1965                                entry_jint_arraycopy,

  1966                                entry_jlong_arraycopy);

  1968     StubRoutines::_generic_arraycopy =

  1969         generate_generic_copy("generic_arraycopy",

  1970                                entry_jbyte_arraycopy,

  1971                                entry_jshort_arraycopy,

  1972                                entry_jint_arraycopy,

  1973                                entry_oop_arraycopy,

  1974                                entry_jlong_arraycopy,

  1975                                entry_checkcast_arraycopy);

  1976   }

  1978  public:

  1979   // Information about frame layout at time of blocking runtime call.

  1980   // Note that we only have to preserve callee-saved registers since

  1981   // the compilers are responsible for supplying a continuation point

  1982   // if they expect all registers to be preserved.

  1983   enum layout {

  1984     thread_off,    // last_java_sp

  1985     rbp_off,       // callee saved register

  1986     ret_pc,

  1987     framesize

  1988   };

  1990  private:

  1992 #undef  __

  1993 #define __ masm->

  1995   //------------------------------------------------------------------------------------------------------------------------

  1996   // Continuation point for throwing of implicit exceptions that are not handled in

  1997   // the current activation. Fabricates an exception oop and initiates normal

  1998   // exception dispatching in this frame.

  1999   //

  2000   // Previously the compiler (c2) allowed for callee save registers on Java calls.

  2001   // This is no longer true after adapter frames were removed but could possibly

  2002   // be brought back in the future if the interpreter code was reworked and it

  2003   // was deemed worthwhile. The comment below was left to describe what must

  2004   // happen here if callee saves were resurrected. As it stands now this stub

  2005   // could actually be a vanilla BufferBlob and have now oopMap at all.

  2006   // Since it doesn't make much difference we've chosen to leave it the

  2007   // way it was in the callee save days and keep the comment.

  2009   // If we need to preserve callee-saved values we need a callee-saved oop map and

  2010   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.

  2011   // If the compiler needs all registers to be preserved between the fault

  2012   // point and the exception handler then it must assume responsibility for that in

  2013   // AbstractCompiler::continuation_for_implicit_null_exception or

  2014   // continuation_for_implicit_division_by_zero_exception. All other implicit

  2015   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are

  2016   // either at call sites or otherwise assume that stack unwinding will be initiated,

  2017   // so caller saved registers were assumed volatile in the compiler.

  2018   address generate_throw_exception(const char* name, address runtime_entry,

  2019                                    bool restore_saved_exception_pc) {

  2021     int insts_size = 256;

  2022     int locs_size  = 32;

  2024     CodeBuffer code(name, insts_size, locs_size);

  2025     OopMapSet* oop_maps  = new OopMapSet();

  2026     MacroAssembler* masm = new MacroAssembler(&code);

  2028     address start = __ pc();

  2030     // This is an inlined and slightly modified version of call_VM

  2031     // which has the ability to fetch the return PC out of

  2032     // thread-local storage and also sets up last_Java_sp slightly

  2033     // differently than the real call_VM

  2034     Register java_thread = rbx;

  2035     __ get_thread(java_thread);

  2036     if (restore_saved_exception_pc) {

  2037       __ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));

  2038       __ push(rax);

  2039     }

  2041     __ enter(); // required for proper stackwalking of RuntimeStub frame

  2043     // pc and rbp, already pushed

  2044     __ subptr(rsp, (framesize-2) * wordSize); // prolog

  2046     // Frame is now completed as far as size and linkage.

  2048     int frame_complete = __ pc() - start;

  2050     // push java thread (becomes first argument of C function)

  2051     __ movptr(Address(rsp, thread_off * wordSize), java_thread);

  2053     // Set up last_Java_sp and last_Java_fp

  2054     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);

  2056     // Call runtime

  2057     BLOCK_COMMENT("call runtime_entry");

  2058     __ call(RuntimeAddress(runtime_entry));

  2059     // Generate oop map

  2060     OopMap* map =  new OopMap(framesize, 0);

  2061     oop_maps->add_gc_map(__ pc() - start, map);

  2063     // restore the thread (cannot use the pushed argument since arguments

  2064     // may be overwritten by C code generated by an optimizing compiler);

  2065     // however can use the register value directly if it is callee saved.

  2066     __ get_thread(java_thread);

  2068     __ reset_last_Java_frame(java_thread, true, false);

  2070     __ leave(); // required for proper stackwalking of RuntimeStub frame

  2072     // check for pending exceptions

  2073 #ifdef ASSERT

  2074     Label L;

  2075     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);

  2076     __ jcc(Assembler::notEqual, L);

  2077     __ should_not_reach_here();

  2078     __ bind(L);

  2079 #endif /* ASSERT */

  2080     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));

  2083     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);

  2084     return stub->entry_point();

  2085   }

  2088   void create_control_words() {

  2089     // Round to nearest, 53-bit mode, exceptions masked

  2090     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;

  2091     // Round to zero, 53-bit mode, exception mased

  2092     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;

  2093     // Round to nearest, 24-bit mode, exceptions masked

  2094     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;

  2095     // Round to nearest, 64-bit mode, exceptions masked

  2096     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;

  2097     // Round to nearest, 64-bit mode, exceptions masked

  2098     StubRoutines::_mxcsr_std           = 0x1F80;

  2099     // Note: the following two constants are 80-bit values

  2100     //       layout is critical for correct loading by FPU.

  2101     // Bias for strict fp multiply/divide

  2102     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000

  2103     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;

  2104     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;

  2105     // Un-Bias for strict fp multiply/divide

  2106     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000

  2107     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;

  2108     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;

  2109   }

  2111   //---------------------------------------------------------------------------

  2112   // Initialization

  2114   void generate_initial() {

  2115     // Generates all stubs and initializes the entry points

  2117     //------------------------------------------------------------------------------------------------------------------------

  2118     // entry points that exist in all platforms

  2119     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than

  2120     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.

  2121     StubRoutines::_forward_exception_entry      = generate_forward_exception();

  2123     StubRoutines::_call_stub_entry              =

  2124       generate_call_stub(StubRoutines::_call_stub_return_address);

  2125     // is referenced by megamorphic call

  2126     StubRoutines::_catch_exception_entry        = generate_catch_exception();

  2128     // These are currently used by Solaris/Intel

  2129     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();

  2131     StubRoutines::_handler_for_unsafe_access_entry =

  2132       generate_handler_for_unsafe_access();

  2134     // platform dependent

  2135     create_control_words();

  2137     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();

  2138     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();

  2139     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,

  2140                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));

  2141     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,

  2142                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));

  2143   }

  2146   void generate_all() {

  2147     // Generates all stubs and initializes the entry points

  2149     // These entry points require SharedInfo::stack0 to be set up in non-core builds

  2150     // and need to be relocatable, so they each fabricate a RuntimeStub internally.

  2151     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError),  false);

  2152     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError),  false);

  2153     StubRoutines::_throw_ArithmeticException_entry         = generate_throw_exception("ArithmeticException throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException),  true);

  2154     StubRoutines::_throw_NullPointerException_entry        = generate_throw_exception("NullPointerException throw_exception",         CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);

  2155     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);

  2156     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",           CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError),   false);

  2158     //------------------------------------------------------------------------------------------------------------------------

  2159     // entry points that are platform specific

  2161     // support for verify_oop (must happen after universe_init)

  2162     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();

  2164     // arraycopy stubs used by compilers

  2165     generate_arraycopy_stubs();

  2166   }

  2169  public:

  2170   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {

  2171     if (all) {

  2172       generate_all();

  2173     } else {

  2174       generate_initial();

  2175     }

  2176   }

  2177 }; // end class declaration

  2180 void StubGenerator_generate(CodeBuffer* code, bool all) {

  2181   StubGenerator g(code, all);

  2182 }

src/cpu/x86/vm/stubGenerator_x86_32.cpp@f8199438385b

src/cpu/x86/vm/stubGenerator_x86_32.cpp

Mercurial > jdk8-mips64-public > hotspot / file revision

src/cpu/x86/vm/stubGenerator_x86_32.cpp@f8199438385b

src/cpu/x86/vm/stubGenerator_x86_32.cpp