208 const Address r12_save(rbp, r12_off * wordSize); |
209 const Address r12_save(rbp, r12_off * wordSize); |
209 const Address rbx_save(rbp, rbx_off * wordSize); |
210 const Address rbx_save(rbp, rbx_off * wordSize); |
210 |
211 |
211 // stub code |
212 // stub code |
212 __ enter(); |
213 __ enter(); |
213 __ subq(rsp, -rsp_after_call_off * wordSize); |
214 __ subptr(rsp, -rsp_after_call_off * wordSize); |
214 |
215 |
215 // save register parameters |
216 // save register parameters |
216 #ifndef _WIN64 |
217 #ifndef _WIN64 |
217 __ movq(parameters, c_rarg5); // parameters |
218 __ movptr(parameters, c_rarg5); // parameters |
218 __ movq(entry_point, c_rarg4); // entry_point |
219 __ movptr(entry_point, c_rarg4); // entry_point |
219 #endif |
220 #endif |
220 |
221 |
221 __ movq(method, c_rarg3); // method |
222 __ movptr(method, c_rarg3); // method |
222 __ movl(result_type, c_rarg2); // result type |
223 __ movl(result_type, c_rarg2); // result type |
223 __ movq(result, c_rarg1); // result |
224 __ movptr(result, c_rarg1); // result |
224 __ movq(call_wrapper, c_rarg0); // call wrapper |
225 __ movptr(call_wrapper, c_rarg0); // call wrapper |
225 |
226 |
226 // save regs belonging to calling function |
227 // save regs belonging to calling function |
227 __ movq(rbx_save, rbx); |
228 __ movptr(rbx_save, rbx); |
228 __ movq(r12_save, r12); |
229 __ movptr(r12_save, r12); |
229 __ movq(r13_save, r13); |
230 __ movptr(r13_save, r13); |
230 __ movq(r14_save, r14); |
231 __ movptr(r14_save, r14); |
231 __ movq(r15_save, r15); |
232 __ movptr(r15_save, r15); |
232 |
233 |
233 #ifdef _WIN64 |
234 #ifdef _WIN64 |
234 const Address rdi_save(rbp, rdi_off * wordSize); |
235 const Address rdi_save(rbp, rdi_off * wordSize); |
235 const Address rsi_save(rbp, rsi_off * wordSize); |
236 const Address rsi_save(rbp, rsi_off * wordSize); |
236 |
237 |
237 __ movq(rsi_save, rsi); |
238 __ movptr(rsi_save, rsi); |
238 __ movq(rdi_save, rdi); |
239 __ movptr(rdi_save, rdi); |
239 #else |
240 #else |
240 const Address mxcsr_save(rbp, mxcsr_off * wordSize); |
241 const Address mxcsr_save(rbp, mxcsr_off * wordSize); |
241 { |
242 { |
242 Label skip_ldmx; |
243 Label skip_ldmx; |
243 __ stmxcsr(mxcsr_save); |
244 __ stmxcsr(mxcsr_save); |
244 __ movl(rax, mxcsr_save); |
245 __ movl(rax, mxcsr_save); |
245 __ andl(rax, MXCSR_MASK); // Only check control and mask bits |
246 __ andl(rax, MXCSR_MASK); // Only check control and mask bits |
246 ExternalAddress mxcsr_std(StubRoutines::amd64::mxcsr_std()); |
247 ExternalAddress mxcsr_std(StubRoutines::x86::mxcsr_std()); |
247 __ cmp32(rax, mxcsr_std); |
248 __ cmp32(rax, mxcsr_std); |
248 __ jcc(Assembler::equal, skip_ldmx); |
249 __ jcc(Assembler::equal, skip_ldmx); |
249 __ ldmxcsr(mxcsr_std); |
250 __ ldmxcsr(mxcsr_std); |
250 __ bind(skip_ldmx); |
251 __ bind(skip_ldmx); |
251 } |
252 } |
252 #endif |
253 #endif |
253 |
254 |
254 // Load up thread register |
255 // Load up thread register |
255 __ movq(r15_thread, thread); |
256 __ movptr(r15_thread, thread); |
256 __ reinit_heapbase(); |
257 __ reinit_heapbase(); |
257 |
258 |
258 #ifdef ASSERT |
259 #ifdef ASSERT |
259 // make sure we have no pending exceptions |
260 // make sure we have no pending exceptions |
260 { |
261 { |
261 Label L; |
262 Label L; |
262 __ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD); |
263 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD); |
263 __ jcc(Assembler::equal, L); |
264 __ jcc(Assembler::equal, L); |
264 __ stop("StubRoutines::call_stub: entered with pending exception"); |
265 __ stop("StubRoutines::call_stub: entered with pending exception"); |
265 __ bind(L); |
266 __ bind(L); |
266 } |
267 } |
267 #endif |
268 #endif |
272 __ movl(c_rarg3, parameter_size); |
273 __ movl(c_rarg3, parameter_size); |
273 __ testl(c_rarg3, c_rarg3); |
274 __ testl(c_rarg3, c_rarg3); |
274 __ jcc(Assembler::zero, parameters_done); |
275 __ jcc(Assembler::zero, parameters_done); |
275 |
276 |
276 Label loop; |
277 Label loop; |
277 __ movq(c_rarg2, parameters); // parameter pointer |
278 __ movptr(c_rarg2, parameters); // parameter pointer |
278 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 |
279 __ movl(c_rarg1, c_rarg3); // parameter counter is in c_rarg1 |
279 __ BIND(loop); |
280 __ BIND(loop); |
280 if (TaggedStackInterpreter) { |
281 if (TaggedStackInterpreter) { |
281 __ movq(rax, Address(c_rarg2, 0)); // get tag |
282 __ movl(rax, Address(c_rarg2, 0)); // get tag |
282 __ addq(c_rarg2, wordSize); // advance to next tag |
283 __ addptr(c_rarg2, wordSize); // advance to next tag |
283 __ pushq(rax); // pass tag |
284 __ push(rax); // pass tag |
284 } |
285 } |
285 __ movq(rax, Address(c_rarg2, 0)); // get parameter |
286 __ movptr(rax, Address(c_rarg2, 0));// get parameter |
286 __ addq(c_rarg2, wordSize); // advance to next parameter |
287 __ addptr(c_rarg2, wordSize); // advance to next parameter |
287 __ decrementl(c_rarg1); // decrement counter |
288 __ decrementl(c_rarg1); // decrement counter |
288 __ pushq(rax); // pass parameter |
289 __ push(rax); // pass parameter |
289 __ jcc(Assembler::notZero, loop); |
290 __ jcc(Assembler::notZero, loop); |
290 |
291 |
291 // call Java function |
292 // call Java function |
292 __ BIND(parameters_done); |
293 __ BIND(parameters_done); |
293 __ movq(rbx, method); // get methodOop |
294 __ movptr(rbx, method); // get methodOop |
294 __ movq(c_rarg1, entry_point); // get entry_point |
295 __ movptr(c_rarg1, entry_point); // get entry_point |
295 __ movq(r13, rsp); // set sender sp |
296 __ mov(r13, rsp); // set sender sp |
296 BLOCK_COMMENT("call Java function"); |
297 BLOCK_COMMENT("call Java function"); |
297 __ call(c_rarg1); |
298 __ call(c_rarg1); |
298 |
299 |
299 BLOCK_COMMENT("call_stub_return_address:"); |
300 BLOCK_COMMENT("call_stub_return_address:"); |
300 return_address = __ pc(); |
301 return_address = __ pc(); |
301 |
302 |
302 // store result depending on type (everything that is not |
303 // store result depending on type (everything that is not |
303 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) |
304 // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT) |
304 __ movq(c_rarg0, result); |
305 __ movptr(c_rarg0, result); |
305 Label is_long, is_float, is_double, exit; |
306 Label is_long, is_float, is_double, exit; |
306 __ movl(c_rarg1, result_type); |
307 __ movl(c_rarg1, result_type); |
307 __ cmpl(c_rarg1, T_OBJECT); |
308 __ cmpl(c_rarg1, T_OBJECT); |
308 __ jcc(Assembler::equal, is_long); |
309 __ jcc(Assembler::equal, is_long); |
309 __ cmpl(c_rarg1, T_LONG); |
310 __ cmpl(c_rarg1, T_LONG); |
317 __ movl(Address(c_rarg0, 0), rax); |
318 __ movl(Address(c_rarg0, 0), rax); |
318 |
319 |
319 __ BIND(exit); |
320 __ BIND(exit); |
320 |
321 |
321 // pop parameters |
322 // pop parameters |
322 __ leaq(rsp, rsp_after_call); |
323 __ lea(rsp, rsp_after_call); |
323 |
324 |
324 #ifdef ASSERT |
325 #ifdef ASSERT |
325 // verify that threads correspond |
326 // verify that threads correspond |
326 { |
327 { |
327 Label L, S; |
328 Label L, S; |
328 __ cmpq(r15_thread, thread); |
329 __ cmpptr(r15_thread, thread); |
329 __ jcc(Assembler::notEqual, S); |
330 __ jcc(Assembler::notEqual, S); |
330 __ get_thread(rbx); |
331 __ get_thread(rbx); |
331 __ cmpq(r15_thread, rbx); |
332 __ cmpptr(r15_thread, rbx); |
332 __ jcc(Assembler::equal, L); |
333 __ jcc(Assembler::equal, L); |
333 __ bind(S); |
334 __ bind(S); |
334 __ jcc(Assembler::equal, L); |
335 __ jcc(Assembler::equal, L); |
335 __ stop("StubRoutines::call_stub: threads must correspond"); |
336 __ stop("StubRoutines::call_stub: threads must correspond"); |
336 __ bind(L); |
337 __ bind(L); |
337 } |
338 } |
338 #endif |
339 #endif |
339 |
340 |
340 // restore regs belonging to calling function |
341 // restore regs belonging to calling function |
341 __ movq(r15, r15_save); |
342 __ movptr(r15, r15_save); |
342 __ movq(r14, r14_save); |
343 __ movptr(r14, r14_save); |
343 __ movq(r13, r13_save); |
344 __ movptr(r13, r13_save); |
344 __ movq(r12, r12_save); |
345 __ movptr(r12, r12_save); |
345 __ movq(rbx, rbx_save); |
346 __ movptr(rbx, rbx_save); |
346 |
347 |
347 #ifdef _WIN64 |
348 #ifdef _WIN64 |
348 __ movq(rdi, rdi_save); |
349 __ movptr(rdi, rdi_save); |
349 __ movq(rsi, rsi_save); |
350 __ movptr(rsi, rsi_save); |
350 #else |
351 #else |
351 __ ldmxcsr(mxcsr_save); |
352 __ ldmxcsr(mxcsr_save); |
352 #endif |
353 #endif |
353 |
354 |
354 // restore rsp |
355 // restore rsp |
355 __ addq(rsp, -rsp_after_call_off * wordSize); |
356 __ addptr(rsp, -rsp_after_call_off * wordSize); |
356 |
357 |
357 // return |
358 // return |
358 __ popq(rbp); |
359 __ pop(rbp); |
359 __ ret(0); |
360 __ ret(0); |
360 |
361 |
361 // handle return types different from T_INT |
362 // handle return types different from T_INT |
362 __ BIND(is_long); |
363 __ BIND(is_long); |
363 __ movq(Address(c_rarg0, 0), rax); |
364 __ movq(Address(c_rarg0, 0), rax); |
396 |
397 |
397 #ifdef ASSERT |
398 #ifdef ASSERT |
398 // verify that threads correspond |
399 // verify that threads correspond |
399 { |
400 { |
400 Label L, S; |
401 Label L, S; |
401 __ cmpq(r15_thread, thread); |
402 __ cmpptr(r15_thread, thread); |
402 __ jcc(Assembler::notEqual, S); |
403 __ jcc(Assembler::notEqual, S); |
403 __ get_thread(rbx); |
404 __ get_thread(rbx); |
404 __ cmpq(r15_thread, rbx); |
405 __ cmpptr(r15_thread, rbx); |
405 __ jcc(Assembler::equal, L); |
406 __ jcc(Assembler::equal, L); |
406 __ bind(S); |
407 __ bind(S); |
407 __ stop("StubRoutines::catch_exception: threads must correspond"); |
408 __ stop("StubRoutines::catch_exception: threads must correspond"); |
408 __ bind(L); |
409 __ bind(L); |
409 } |
410 } |
410 #endif |
411 #endif |
411 |
412 |
412 // set pending exception |
413 // set pending exception |
413 __ verify_oop(rax); |
414 __ verify_oop(rax); |
414 |
415 |
415 __ movq(Address(r15_thread, Thread::pending_exception_offset()), rax); |
416 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax); |
416 __ lea(rscratch1, ExternalAddress((address)__FILE__)); |
417 __ lea(rscratch1, ExternalAddress((address)__FILE__)); |
417 __ movq(Address(r15_thread, Thread::exception_file_offset()), rscratch1); |
418 __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1); |
418 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); |
419 __ movl(Address(r15_thread, Thread::exception_line_offset()), (int) __LINE__); |
419 |
420 |
420 // complete return to VM |
421 // complete return to VM |
421 assert(StubRoutines::_call_stub_return_address != NULL, |
422 assert(StubRoutines::_call_stub_return_address != NULL, |
422 "_call_stub_return_address must have been generated before"); |
423 "_call_stub_return_address must have been generated before"); |
451 |
452 |
452 #ifdef ASSERT |
453 #ifdef ASSERT |
453 // make sure this code is only executed if there is a pending exception |
454 // make sure this code is only executed if there is a pending exception |
454 { |
455 { |
455 Label L; |
456 Label L; |
456 __ cmpq(Address(r15_thread, Thread::pending_exception_offset()), (int) NULL); |
457 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL); |
457 __ jcc(Assembler::notEqual, L); |
458 __ jcc(Assembler::notEqual, L); |
458 __ stop("StubRoutines::forward exception: no pending exception (1)"); |
459 __ stop("StubRoutines::forward exception: no pending exception (1)"); |
459 __ bind(L); |
460 __ bind(L); |
460 } |
461 } |
461 #endif |
462 #endif |
462 |
463 |
463 // compute exception handler into rbx |
464 // compute exception handler into rbx |
464 __ movq(c_rarg0, Address(rsp, 0)); |
465 __ movptr(c_rarg0, Address(rsp, 0)); |
465 BLOCK_COMMENT("call exception_handler_for_return_address"); |
466 BLOCK_COMMENT("call exception_handler_for_return_address"); |
466 __ call_VM_leaf(CAST_FROM_FN_PTR(address, |
467 __ call_VM_leaf(CAST_FROM_FN_PTR(address, |
467 SharedRuntime::exception_handler_for_return_address), |
468 SharedRuntime::exception_handler_for_return_address), |
468 c_rarg0); |
469 c_rarg0); |
469 __ movq(rbx, rax); |
470 __ mov(rbx, rax); |
470 |
471 |
471 // setup rax & rdx, remove return address & clear pending exception |
472 // setup rax & rdx, remove return address & clear pending exception |
472 __ popq(rdx); |
473 __ pop(rdx); |
473 __ movq(rax, Address(r15_thread, Thread::pending_exception_offset())); |
474 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset())); |
474 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD); |
475 __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int)NULL_WORD); |
475 |
476 |
476 #ifdef ASSERT |
477 #ifdef ASSERT |
477 // make sure exception is set |
478 // make sure exception is set |
478 { |
479 { |
479 Label L; |
480 Label L; |
480 __ testq(rax, rax); |
481 __ testptr(rax, rax); |
481 __ jcc(Assembler::notEqual, L); |
482 __ jcc(Assembler::notEqual, L); |
482 __ stop("StubRoutines::forward exception: no pending exception (2)"); |
483 __ stop("StubRoutines::forward exception: no pending exception (2)"); |
483 __ bind(L); |
484 __ bind(L); |
484 } |
485 } |
485 #endif |
486 #endif |
781 |
782 |
782 address start = __ pc(); |
783 address start = __ pc(); |
783 |
784 |
784 Label L; |
785 Label L; |
785 |
786 |
786 __ pushq(rax); |
787 __ push(rax); |
787 __ pushq(c_rarg3); |
788 __ push(c_rarg3); |
788 __ pushq(c_rarg2); |
789 __ push(c_rarg2); |
789 __ pushq(c_rarg1); |
790 __ push(c_rarg1); |
790 __ pushq(c_rarg0); |
791 __ push(c_rarg0); |
791 |
792 |
792 __ movl(rax, 0x7ff00000); |
793 __ movl(rax, 0x7ff00000); |
793 __ movq(c_rarg2, inout); |
794 __ movq(c_rarg2, inout); |
794 __ movl(c_rarg3, c_rarg2); |
795 __ movl(c_rarg3, c_rarg2); |
795 __ movq(c_rarg1, c_rarg2); |
796 __ mov(c_rarg1, c_rarg2); |
796 __ movq(c_rarg0, c_rarg2); |
797 __ mov(c_rarg0, c_rarg2); |
797 __ negl(c_rarg3); |
798 __ negl(c_rarg3); |
798 __ shrq(c_rarg1, 0x20); |
799 __ shrptr(c_rarg1, 0x20); |
799 __ orl(c_rarg3, c_rarg2); |
800 __ orl(c_rarg3, c_rarg2); |
800 __ andl(c_rarg1, 0x7fffffff); |
801 __ andl(c_rarg1, 0x7fffffff); |
801 __ xorl(c_rarg2, c_rarg2); |
802 __ xorl(c_rarg2, c_rarg2); |
802 __ shrl(c_rarg3, 0x1f); |
803 __ shrl(c_rarg3, 0x1f); |
803 __ orl(c_rarg1, c_rarg3); |
804 __ orl(c_rarg1, c_rarg3); |
804 __ cmpl(rax, c_rarg1); |
805 __ cmpl(rax, c_rarg1); |
805 __ jcc(Assembler::negative, L); // NaN -> 0 |
806 __ jcc(Assembler::negative, L); // NaN -> 0 |
806 __ testq(c_rarg0, c_rarg0); // signed ? min_jint : max_jint |
807 __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint |
807 __ movl(c_rarg2, 0x80000000); |
808 __ movl(c_rarg2, 0x80000000); |
808 __ movl(rax, 0x7fffffff); |
809 __ movl(rax, 0x7fffffff); |
809 __ cmovl(Assembler::positive, c_rarg2, rax); |
810 __ cmov(Assembler::positive, c_rarg2, rax); |
810 |
811 |
811 __ bind(L); |
812 __ bind(L); |
812 __ movq(inout, c_rarg2); |
813 __ movptr(inout, c_rarg2); |
813 |
814 |
814 __ popq(c_rarg0); |
815 __ pop(c_rarg0); |
815 __ popq(c_rarg1); |
816 __ pop(c_rarg1); |
816 __ popq(c_rarg2); |
817 __ pop(c_rarg2); |
817 __ popq(c_rarg3); |
818 __ pop(c_rarg3); |
818 __ popq(rax); |
819 __ pop(rax); |
819 |
820 |
820 __ ret(0); |
821 __ ret(0); |
821 |
822 |
822 return start; |
823 return start; |
823 } |
824 } |
828 |
829 |
829 address start = __ pc(); |
830 address start = __ pc(); |
830 |
831 |
831 Label L; |
832 Label L; |
832 |
833 |
833 __ pushq(rax); |
834 __ push(rax); |
834 __ pushq(c_rarg3); |
835 __ push(c_rarg3); |
835 __ pushq(c_rarg2); |
836 __ push(c_rarg2); |
836 __ pushq(c_rarg1); |
837 __ push(c_rarg1); |
837 __ pushq(c_rarg0); |
838 __ push(c_rarg0); |
838 |
839 |
839 __ movl(rax, 0x7ff00000); |
840 __ movl(rax, 0x7ff00000); |
840 __ movq(c_rarg2, inout); |
841 __ movq(c_rarg2, inout); |
841 __ movl(c_rarg3, c_rarg2); |
842 __ movl(c_rarg3, c_rarg2); |
842 __ movq(c_rarg1, c_rarg2); |
843 __ mov(c_rarg1, c_rarg2); |
843 __ movq(c_rarg0, c_rarg2); |
844 __ mov(c_rarg0, c_rarg2); |
844 __ negl(c_rarg3); |
845 __ negl(c_rarg3); |
845 __ shrq(c_rarg1, 0x20); |
846 __ shrptr(c_rarg1, 0x20); |
846 __ orl(c_rarg3, c_rarg2); |
847 __ orl(c_rarg3, c_rarg2); |
847 __ andl(c_rarg1, 0x7fffffff); |
848 __ andl(c_rarg1, 0x7fffffff); |
848 __ xorl(c_rarg2, c_rarg2); |
849 __ xorl(c_rarg2, c_rarg2); |
849 __ shrl(c_rarg3, 0x1f); |
850 __ shrl(c_rarg3, 0x1f); |
850 __ orl(c_rarg1, c_rarg3); |
851 __ orl(c_rarg1, c_rarg3); |
887 // SIGBUS/OBJERR.) |
888 // SIGBUS/OBJERR.) |
888 address generate_handler_for_unsafe_access() { |
889 address generate_handler_for_unsafe_access() { |
889 StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access"); |
890 StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access"); |
890 address start = __ pc(); |
891 address start = __ pc(); |
891 |
892 |
892 __ pushq(0); // hole for return address-to-be |
893 __ push(0); // hole for return address-to-be |
893 __ pushaq(); // push registers |
894 __ pusha(); // push registers |
894 Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord); |
895 Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord); |
895 |
896 |
896 __ subq(rsp, frame::arg_reg_save_area_bytes); |
897 __ subptr(rsp, frame::arg_reg_save_area_bytes); |
897 BLOCK_COMMENT("call handle_unsafe_access"); |
898 BLOCK_COMMENT("call handle_unsafe_access"); |
898 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access))); |
899 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access))); |
899 __ addq(rsp, frame::arg_reg_save_area_bytes); |
900 __ addptr(rsp, frame::arg_reg_save_area_bytes); |
900 |
901 |
901 __ movq(next_pc, rax); // stuff next address |
902 __ movptr(next_pc, rax); // stuff next address |
902 __ popaq(); |
903 __ popa(); |
903 __ ret(0); // jump to next address |
904 __ ret(0); // jump to next address |
904 |
905 |
905 return start; |
906 return start; |
906 } |
907 } |
907 |
908 |
944 return_addr = 16 * wordSize, |
945 return_addr = 16 * wordSize, |
945 error_msg = 17 * wordSize |
946 error_msg = 17 * wordSize |
946 }; |
947 }; |
947 |
948 |
948 // get object |
949 // get object |
949 __ movq(rax, Address(rsp, oop_to_verify)); |
950 __ movptr(rax, Address(rsp, oop_to_verify)); |
950 |
951 |
951 // make sure object is 'reasonable' |
952 // make sure object is 'reasonable' |
952 __ testq(rax, rax); |
953 __ testptr(rax, rax); |
953 __ jcc(Assembler::zero, exit); // if obj is NULL it is OK |
954 __ jcc(Assembler::zero, exit); // if obj is NULL it is OK |
954 // Check if the oop is in the right area of memory |
955 // Check if the oop is in the right area of memory |
955 __ movq(c_rarg2, rax); |
956 __ movptr(c_rarg2, rax); |
956 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_mask()); |
957 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_mask()); |
957 __ andq(c_rarg2, c_rarg3); |
958 __ andptr(c_rarg2, c_rarg3); |
958 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_bits()); |
959 __ movptr(c_rarg3, (int64_t) Universe::verify_oop_bits()); |
959 __ cmpq(c_rarg2, c_rarg3); |
960 __ cmpptr(c_rarg2, c_rarg3); |
960 __ jcc(Assembler::notZero, error); |
961 __ jcc(Assembler::notZero, error); |
961 |
962 |
962 // set r12 to heapbase for load_klass() |
963 // set r12 to heapbase for load_klass() |
963 __ reinit_heapbase(); |
964 __ reinit_heapbase(); |
964 |
965 |
965 // make sure klass is 'reasonable' |
966 // make sure klass is 'reasonable' |
966 __ load_klass(rax, rax); // get klass |
967 __ load_klass(rax, rax); // get klass |
967 __ testq(rax, rax); |
968 __ testptr(rax, rax); |
968 __ jcc(Assembler::zero, error); // if klass is NULL it is broken |
969 __ jcc(Assembler::zero, error); // if klass is NULL it is broken |
969 // Check if the klass is in the right area of memory |
970 // Check if the klass is in the right area of memory |
970 __ movq(c_rarg2, rax); |
971 __ mov(c_rarg2, rax); |
971 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); |
972 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); |
972 __ andq(c_rarg2, c_rarg3); |
973 __ andptr(c_rarg2, c_rarg3); |
973 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); |
974 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); |
974 __ cmpq(c_rarg2, c_rarg3); |
975 __ cmpptr(c_rarg2, c_rarg3); |
975 __ jcc(Assembler::notZero, error); |
976 __ jcc(Assembler::notZero, error); |
976 |
977 |
977 // make sure klass' klass is 'reasonable' |
978 // make sure klass' klass is 'reasonable' |
978 __ load_klass(rax, rax); |
979 __ load_klass(rax, rax); |
979 __ testq(rax, rax); |
980 __ testptr(rax, rax); |
980 __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken |
981 __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken |
981 // Check if the klass' klass is in the right area of memory |
982 // Check if the klass' klass is in the right area of memory |
982 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); |
983 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_mask()); |
983 __ andq(rax, c_rarg3); |
984 __ andptr(rax, c_rarg3); |
984 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); |
985 __ movptr(c_rarg3, (int64_t) Universe::verify_klass_bits()); |
985 __ cmpq(rax, c_rarg3); |
986 __ cmpptr(rax, c_rarg3); |
986 __ jcc(Assembler::notZero, error); |
987 __ jcc(Assembler::notZero, error); |
987 |
988 |
988 // return if everything seems ok |
989 // return if everything seems ok |
989 __ bind(exit); |
990 __ bind(exit); |
990 __ movq(rax, Address(rsp, saved_rax)); // get saved rax back |
991 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back |
991 __ popq(c_rarg3); // restore c_rarg3 |
992 __ pop(c_rarg3); // restore c_rarg3 |
992 __ popq(c_rarg2); // restore c_rarg2 |
993 __ pop(c_rarg2); // restore c_rarg2 |
993 __ popq(r12); // restore r12 |
994 __ pop(r12); // restore r12 |
994 __ popfq(); // restore flags |
995 __ popf(); // restore flags |
995 __ ret(3 * wordSize); // pop caller saved stuff |
996 __ ret(3 * wordSize); // pop caller saved stuff |
996 |
997 |
997 // handle errors |
998 // handle errors |
998 __ bind(error); |
999 __ bind(error); |
999 __ movq(rax, Address(rsp, saved_rax)); // get saved rax back |
1000 __ movptr(rax, Address(rsp, saved_rax)); // get saved rax back |
1000 __ popq(c_rarg3); // get saved c_rarg3 back |
1001 __ pop(c_rarg3); // get saved c_rarg3 back |
1001 __ popq(c_rarg2); // get saved c_rarg2 back |
1002 __ pop(c_rarg2); // get saved c_rarg2 back |
1002 __ popq(r12); // get saved r12 back |
1003 __ pop(r12); // get saved r12 back |
1003 __ popfq(); // get saved flags off stack -- |
1004 __ popf(); // get saved flags off stack -- |
1004 // will be ignored |
1005 // will be ignored |
1005 |
1006 |
1006 __ pushaq(); // push registers |
1007 __ pusha(); // push registers |
1007 // (rip is already |
1008 // (rip is already |
1008 // already pushed) |
1009 // already pushed) |
1009 // debug(char* msg, int64_t pc, int64_t regs[]) |
1010 // debug(char* msg, int64_t pc, int64_t regs[]) |
1010 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and |
1011 // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and |
1011 // pushed all the registers, so now the stack looks like: |
1012 // pushed all the registers, so now the stack looks like: |
1014 // * [tos + 17] error message (char*) |
1015 // * [tos + 17] error message (char*) |
1015 // * [tos + 18] object to verify (oop) |
1016 // * [tos + 18] object to verify (oop) |
1016 // * [tos + 19] saved rax - saved by caller and bashed |
1017 // * [tos + 19] saved rax - saved by caller and bashed |
1017 // * = popped on exit |
1018 // * = popped on exit |
1018 |
1019 |
1019 __ movq(c_rarg0, Address(rsp, error_msg)); // pass address of error message |
1020 __ movptr(c_rarg0, Address(rsp, error_msg)); // pass address of error message |
1020 __ movq(c_rarg1, Address(rsp, return_addr)); // pass return address |
1021 __ movptr(c_rarg1, Address(rsp, return_addr)); // pass return address |
1021 __ movq(c_rarg2, rsp); // pass address of regs on stack |
1022 __ movq(c_rarg2, rsp); // pass address of regs on stack |
1022 __ movq(r12, rsp); // remember rsp |
1023 __ mov(r12, rsp); // remember rsp |
1023 __ subq(rsp, frame::arg_reg_save_area_bytes);// windows |
1024 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows |
1024 __ andq(rsp, -16); // align stack as required by ABI |
1025 __ andptr(rsp, -16); // align stack as required by ABI |
1025 BLOCK_COMMENT("call MacroAssembler::debug"); |
1026 BLOCK_COMMENT("call MacroAssembler::debug"); |
1026 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug))); |
1027 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64))); |
1027 __ movq(rsp, r12); // restore rsp |
1028 __ mov(rsp, r12); // restore rsp |
1028 __ popaq(); // pop registers (includes r12) |
1029 __ popa(); // pop registers (includes r12) |
1029 __ ret(3 * wordSize); // pop caller saved stuff |
1030 __ ret(3 * wordSize); // pop caller saved stuff |
1030 |
1031 |
1031 return start; |
1032 return start; |
1032 } |
1033 } |
1033 |
1034 |
1034 static address disjoint_byte_copy_entry; |
1035 static address disjoint_byte_copy_entry; |
1086 const Register from = c_rarg0; |
1087 const Register from = c_rarg0; |
1087 const Register to = c_rarg1; |
1088 const Register to = c_rarg1; |
1088 const Register count = c_rarg2; |
1089 const Register count = c_rarg2; |
1089 const Register end_from = rax; |
1090 const Register end_from = rax; |
1090 |
1091 |
1091 __ cmpq(to, from); |
1092 __ cmpptr(to, from); |
1092 __ leaq(end_from, Address(from, count, sf, 0)); |
1093 __ lea(end_from, Address(from, count, sf, 0)); |
1093 if (NOLp == NULL) { |
1094 if (NOLp == NULL) { |
1094 ExternalAddress no_overlap(no_overlap_target); |
1095 ExternalAddress no_overlap(no_overlap_target); |
1095 __ jump_cc(Assembler::belowEqual, no_overlap); |
1096 __ jump_cc(Assembler::belowEqual, no_overlap); |
1096 __ cmpq(to, end_from); |
1097 __ cmpptr(to, end_from); |
1097 __ jump_cc(Assembler::aboveEqual, no_overlap); |
1098 __ jump_cc(Assembler::aboveEqual, no_overlap); |
1098 } else { |
1099 } else { |
1099 __ jcc(Assembler::belowEqual, (*NOLp)); |
1100 __ jcc(Assembler::belowEqual, (*NOLp)); |
1100 __ cmpq(to, end_from); |
1101 __ cmpptr(to, end_from); |
1101 __ jcc(Assembler::aboveEqual, (*NOLp)); |
1102 __ jcc(Assembler::aboveEqual, (*NOLp)); |
1102 } |
1103 } |
1103 } |
1104 } |
1104 |
1105 |
1105 // Shuffle first three arg regs on Windows into Linux/Solaris locations. |
1106 // Shuffle first three arg regs on Windows into Linux/Solaris locations. |
1119 assert(nargs == 3 || nargs == 4, "else fix"); |
1120 assert(nargs == 3 || nargs == 4, "else fix"); |
1120 #ifdef _WIN64 |
1121 #ifdef _WIN64 |
1121 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, |
1122 assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9, |
1122 "unexpected argument registers"); |
1123 "unexpected argument registers"); |
1123 if (nargs >= 4) |
1124 if (nargs >= 4) |
1124 __ movq(rax, r9); // r9 is also saved_rdi |
1125 __ mov(rax, r9); // r9 is also saved_rdi |
1125 __ movq(saved_rdi, rdi); |
1126 __ movptr(saved_rdi, rdi); |
1126 __ movq(saved_rsi, rsi); |
1127 __ movptr(saved_rsi, rsi); |
1127 __ movq(rdi, rcx); // c_rarg0 |
1128 __ mov(rdi, rcx); // c_rarg0 |
1128 __ movq(rsi, rdx); // c_rarg1 |
1129 __ mov(rsi, rdx); // c_rarg1 |
1129 __ movq(rdx, r8); // c_rarg2 |
1130 __ mov(rdx, r8); // c_rarg2 |
1130 if (nargs >= 4) |
1131 if (nargs >= 4) |
1131 __ movq(rcx, rax); // c_rarg3 (via rax) |
1132 __ mov(rcx, rax); // c_rarg3 (via rax) |
1132 #else |
1133 #else |
1133 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, |
1134 assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx, |
1134 "unexpected argument registers"); |
1135 "unexpected argument registers"); |
1135 #endif |
1136 #endif |
1136 } |
1137 } |
1137 |
1138 |
1138 void restore_arg_regs() { |
1139 void restore_arg_regs() { |
1139 const Register saved_rdi = r9; |
1140 const Register saved_rdi = r9; |
1140 const Register saved_rsi = r10; |
1141 const Register saved_rsi = r10; |
1141 #ifdef _WIN64 |
1142 #ifdef _WIN64 |
1142 __ movq(rdi, saved_rdi); |
1143 __ movptr(rdi, saved_rdi); |
1143 __ movq(rsi, saved_rsi); |
1144 __ movptr(rsi, saved_rsi); |
1144 #endif |
1145 #endif |
1145 } |
1146 } |
1146 |
1147 |
1147 // Generate code for an array write pre barrier |
1148 // Generate code for an array write pre barrier |
1148 // |
1149 // |
1195 #if 0 // G1 - only |
1196 #if 0 // G1 - only |
1196 case BarrierSet::G1SATBCT: |
1197 case BarrierSet::G1SATBCT: |
1197 case BarrierSet::G1SATBCTLogging: |
1198 case BarrierSet::G1SATBCTLogging: |
1198 |
1199 |
1199 { |
1200 { |
1200 __ pushaq(); // push registers (overkill) |
1201 __ pusha(); // push registers (overkill) |
1201 // must compute element count unless barrier set interface is changed (other platforms supply count) |
1202 // must compute element count unless barrier set interface is changed (other platforms supply count) |
1202 assert_different_registers(start, end, scratch); |
1203 assert_different_registers(start, end, scratch); |
1203 __ leaq(scratch, Address(end, wordSize)); |
1204 __ lea(scratch, Address(end, wordSize)); |
1204 __ subq(scratch, start); |
1205 __ subptr(scratch, start); |
1205 __ shrq(scratch, LogBytesPerWord); |
1206 __ shrptr(scratch, LogBytesPerWord); |
1206 __ movq(c_rarg0, start); |
1207 __ mov(c_rarg0, start); |
1207 __ movq(c_rarg1, scratch); |
1208 __ mov(c_rarg1, scratch); |
1208 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)); |
1209 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post)); |
1209 __ popaq(); |
1210 __ popa(); |
1210 } |
1211 } |
1211 break; |
1212 break; |
1212 #endif // 0 G1 - only |
1213 #endif // 0 G1 - only |
1213 case BarrierSet::CardTableModRef: |
1214 case BarrierSet::CardTableModRef: |
1214 case BarrierSet::CardTableExtension: |
1215 case BarrierSet::CardTableExtension: |
1265 __ movq(to, Address(end_from, qword_count, Address::times_8, - 8)); |
1266 __ movq(to, Address(end_from, qword_count, Address::times_8, - 8)); |
1266 __ movq(Address(end_to, qword_count, Address::times_8, - 8), to); |
1267 __ movq(Address(end_to, qword_count, Address::times_8, - 8), to); |
1267 __ movq(to, Address(end_from, qword_count, Address::times_8, - 0)); |
1268 __ movq(to, Address(end_from, qword_count, Address::times_8, - 0)); |
1268 __ movq(Address(end_to, qword_count, Address::times_8, - 0), to); |
1269 __ movq(Address(end_to, qword_count, Address::times_8, - 0), to); |
1269 __ BIND(L_copy_32_bytes); |
1270 __ BIND(L_copy_32_bytes); |
1270 __ addq(qword_count, 4); |
1271 __ addptr(qword_count, 4); |
1271 __ jcc(Assembler::lessEqual, L_loop); |
1272 __ jcc(Assembler::lessEqual, L_loop); |
1272 __ subq(qword_count, 4); |
1273 __ subptr(qword_count, 4); |
1273 __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords |
1274 __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords |
1274 } |
1275 } |
1275 |
1276 |
1276 |
1277 |
1277 // Copy big chunks backward |
1278 // Copy big chunks backward |
1298 __ movq(to, Address(from, qword_count, Address::times_8, 8)); |
1299 __ movq(to, Address(from, qword_count, Address::times_8, 8)); |
1299 __ movq(Address(dest, qword_count, Address::times_8, 8), to); |
1300 __ movq(Address(dest, qword_count, Address::times_8, 8), to); |
1300 __ movq(to, Address(from, qword_count, Address::times_8, 0)); |
1301 __ movq(to, Address(from, qword_count, Address::times_8, 0)); |
1301 __ movq(Address(dest, qword_count, Address::times_8, 0), to); |
1302 __ movq(Address(dest, qword_count, Address::times_8, 0), to); |
1302 __ BIND(L_copy_32_bytes); |
1303 __ BIND(L_copy_32_bytes); |
1303 __ subq(qword_count, 4); |
1304 __ subptr(qword_count, 4); |
1304 __ jcc(Assembler::greaterEqual, L_loop); |
1305 __ jcc(Assembler::greaterEqual, L_loop); |
1305 __ addq(qword_count, 4); |
1306 __ addptr(qword_count, 4); |
1306 __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords |
1307 __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords |
1307 } |
1308 } |
1308 |
1309 |
1309 |
1310 |
1310 // Arguments: |
1311 // Arguments: |
1352 |
1353 |
1353 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1354 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1354 // r9 and r10 may be used to save non-volatile registers |
1355 // r9 and r10 may be used to save non-volatile registers |
1355 |
1356 |
1356 // 'from', 'to' and 'count' are now valid |
1357 // 'from', 'to' and 'count' are now valid |
1357 __ movq(byte_count, count); |
1358 __ movptr(byte_count, count); |
1358 __ shrq(count, 3); // count => qword_count |
1359 __ shrptr(count, 3); // count => qword_count |
1359 |
1360 |
1360 // Copy from low to high addresses. Use 'to' as scratch. |
1361 // Copy from low to high addresses. Use 'to' as scratch. |
1361 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); |
1362 __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); |
1362 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); |
1363 __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); |
1363 __ negq(qword_count); // make the count negative |
1364 __ negptr(qword_count); // make the count negative |
1364 __ jmp(L_copy_32_bytes); |
1365 __ jmp(L_copy_32_bytes); |
1365 |
1366 |
1366 // Copy trailing qwords |
1367 // Copy trailing qwords |
1367 __ BIND(L_copy_8_bytes); |
1368 __ BIND(L_copy_8_bytes); |
1368 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1369 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1369 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1370 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1370 __ incrementq(qword_count); |
1371 __ increment(qword_count); |
1371 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1372 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1372 |
1373 |
1373 // Check for and copy trailing dword |
1374 // Check for and copy trailing dword |
1374 __ BIND(L_copy_4_bytes); |
1375 __ BIND(L_copy_4_bytes); |
1375 __ testq(byte_count, 4); |
1376 __ testl(byte_count, 4); |
1376 __ jccb(Assembler::zero, L_copy_2_bytes); |
1377 __ jccb(Assembler::zero, L_copy_2_bytes); |
1377 __ movl(rax, Address(end_from, 8)); |
1378 __ movl(rax, Address(end_from, 8)); |
1378 __ movl(Address(end_to, 8), rax); |
1379 __ movl(Address(end_to, 8), rax); |
1379 |
1380 |
1380 __ addq(end_from, 4); |
1381 __ addptr(end_from, 4); |
1381 __ addq(end_to, 4); |
1382 __ addptr(end_to, 4); |
1382 |
1383 |
1383 // Check for and copy trailing word |
1384 // Check for and copy trailing word |
1384 __ BIND(L_copy_2_bytes); |
1385 __ BIND(L_copy_2_bytes); |
1385 __ testq(byte_count, 2); |
1386 __ testl(byte_count, 2); |
1386 __ jccb(Assembler::zero, L_copy_byte); |
1387 __ jccb(Assembler::zero, L_copy_byte); |
1387 __ movw(rax, Address(end_from, 8)); |
1388 __ movw(rax, Address(end_from, 8)); |
1388 __ movw(Address(end_to, 8), rax); |
1389 __ movw(Address(end_to, 8), rax); |
1389 |
1390 |
1390 __ addq(end_from, 2); |
1391 __ addptr(end_from, 2); |
1391 __ addq(end_to, 2); |
1392 __ addptr(end_to, 2); |
1392 |
1393 |
1393 // Check for and copy trailing byte |
1394 // Check for and copy trailing byte |
1394 __ BIND(L_copy_byte); |
1395 __ BIND(L_copy_byte); |
1395 __ testq(byte_count, 1); |
1396 __ testl(byte_count, 1); |
1396 __ jccb(Assembler::zero, L_exit); |
1397 __ jccb(Assembler::zero, L_exit); |
1397 __ movb(rax, Address(end_from, 8)); |
1398 __ movb(rax, Address(end_from, 8)); |
1398 __ movb(Address(end_to, 8), rax); |
1399 __ movb(Address(end_to, 8), rax); |
1399 |
1400 |
1400 __ BIND(L_exit); |
1401 __ BIND(L_exit); |
1401 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1402 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1402 restore_arg_regs(); |
1403 restore_arg_regs(); |
1403 __ xorq(rax, rax); // return 0 |
1404 __ xorptr(rax, rax); // return 0 |
1404 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1405 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1405 __ ret(0); |
1406 __ ret(0); |
1406 |
1407 |
1407 // Copy in 32-bytes chunks |
1408 // Copy in 32-bytes chunks |
1408 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1409 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1448 array_overlap_test(disjoint_byte_copy_entry, Address::times_1); |
1449 array_overlap_test(disjoint_byte_copy_entry, Address::times_1); |
1449 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1450 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1450 // r9 and r10 may be used to save non-volatile registers |
1451 // r9 and r10 may be used to save non-volatile registers |
1451 |
1452 |
1452 // 'from', 'to' and 'count' are now valid |
1453 // 'from', 'to' and 'count' are now valid |
1453 __ movq(byte_count, count); |
1454 __ movptr(byte_count, count); |
1454 __ shrq(count, 3); // count => qword_count |
1455 __ shrptr(count, 3); // count => qword_count |
1455 |
1456 |
1456 // Copy from high to low addresses. |
1457 // Copy from high to low addresses. |
1457 |
1458 |
1458 // Check for and copy trailing byte |
1459 // Check for and copy trailing byte |
1459 __ testq(byte_count, 1); |
1460 __ testl(byte_count, 1); |
1460 __ jcc(Assembler::zero, L_copy_2_bytes); |
1461 __ jcc(Assembler::zero, L_copy_2_bytes); |
1461 __ movb(rax, Address(from, byte_count, Address::times_1, -1)); |
1462 __ movb(rax, Address(from, byte_count, Address::times_1, -1)); |
1462 __ movb(Address(to, byte_count, Address::times_1, -1), rax); |
1463 __ movb(Address(to, byte_count, Address::times_1, -1), rax); |
1463 __ decrementq(byte_count); // Adjust for possible trailing word |
1464 __ decrement(byte_count); // Adjust for possible trailing word |
1464 |
1465 |
1465 // Check for and copy trailing word |
1466 // Check for and copy trailing word |
1466 __ BIND(L_copy_2_bytes); |
1467 __ BIND(L_copy_2_bytes); |
1467 __ testq(byte_count, 2); |
1468 __ testl(byte_count, 2); |
1468 __ jcc(Assembler::zero, L_copy_4_bytes); |
1469 __ jcc(Assembler::zero, L_copy_4_bytes); |
1469 __ movw(rax, Address(from, byte_count, Address::times_1, -2)); |
1470 __ movw(rax, Address(from, byte_count, Address::times_1, -2)); |
1470 __ movw(Address(to, byte_count, Address::times_1, -2), rax); |
1471 __ movw(Address(to, byte_count, Address::times_1, -2), rax); |
1471 |
1472 |
1472 // Check for and copy trailing dword |
1473 // Check for and copy trailing dword |
1473 __ BIND(L_copy_4_bytes); |
1474 __ BIND(L_copy_4_bytes); |
1474 __ testq(byte_count, 4); |
1475 __ testl(byte_count, 4); |
1475 __ jcc(Assembler::zero, L_copy_32_bytes); |
1476 __ jcc(Assembler::zero, L_copy_32_bytes); |
1476 __ movl(rax, Address(from, qword_count, Address::times_8)); |
1477 __ movl(rax, Address(from, qword_count, Address::times_8)); |
1477 __ movl(Address(to, qword_count, Address::times_8), rax); |
1478 __ movl(Address(to, qword_count, Address::times_8), rax); |
1478 __ jmp(L_copy_32_bytes); |
1479 __ jmp(L_copy_32_bytes); |
1479 |
1480 |
1480 // Copy trailing qwords |
1481 // Copy trailing qwords |
1481 __ BIND(L_copy_8_bytes); |
1482 __ BIND(L_copy_8_bytes); |
1482 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1483 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1483 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1484 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1484 __ decrementq(qword_count); |
1485 __ decrement(qword_count); |
1485 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1486 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1486 |
1487 |
1487 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1488 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1488 restore_arg_regs(); |
1489 restore_arg_regs(); |
1489 __ xorq(rax, rax); // return 0 |
1490 __ xorptr(rax, rax); // return 0 |
1490 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1491 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1491 __ ret(0); |
1492 __ ret(0); |
1492 |
1493 |
1493 // Copy in 32-bytes chunks |
1494 // Copy in 32-bytes chunks |
1494 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1495 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1495 |
1496 |
1496 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1497 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
1497 restore_arg_regs(); |
1498 restore_arg_regs(); |
1498 __ xorq(rax, rax); // return 0 |
1499 __ xorptr(rax, rax); // return 0 |
1499 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1500 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1500 __ ret(0); |
1501 __ ret(0); |
1501 |
1502 |
1502 return start; |
1503 return start; |
1503 } |
1504 } |
1546 |
1547 |
1547 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1548 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1548 // r9 and r10 may be used to save non-volatile registers |
1549 // r9 and r10 may be used to save non-volatile registers |
1549 |
1550 |
1550 // 'from', 'to' and 'count' are now valid |
1551 // 'from', 'to' and 'count' are now valid |
1551 __ movq(word_count, count); |
1552 __ movptr(word_count, count); |
1552 __ shrq(count, 2); // count => qword_count |
1553 __ shrptr(count, 2); // count => qword_count |
1553 |
1554 |
1554 // Copy from low to high addresses. Use 'to' as scratch. |
1555 // Copy from low to high addresses. Use 'to' as scratch. |
1555 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); |
1556 __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); |
1556 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); |
1557 __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); |
1557 __ negq(qword_count); |
1558 __ negptr(qword_count); |
1558 __ jmp(L_copy_32_bytes); |
1559 __ jmp(L_copy_32_bytes); |
1559 |
1560 |
1560 // Copy trailing qwords |
1561 // Copy trailing qwords |
1561 __ BIND(L_copy_8_bytes); |
1562 __ BIND(L_copy_8_bytes); |
1562 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1563 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1563 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1564 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1564 __ incrementq(qword_count); |
1565 __ increment(qword_count); |
1565 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1566 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1566 |
1567 |
1567 // Original 'dest' is trashed, so we can't use it as a |
1568 // Original 'dest' is trashed, so we can't use it as a |
1568 // base register for a possible trailing word copy |
1569 // base register for a possible trailing word copy |
1569 |
1570 |
1570 // Check for and copy trailing dword |
1571 // Check for and copy trailing dword |
1571 __ BIND(L_copy_4_bytes); |
1572 __ BIND(L_copy_4_bytes); |
1572 __ testq(word_count, 2); |
1573 __ testl(word_count, 2); |
1573 __ jccb(Assembler::zero, L_copy_2_bytes); |
1574 __ jccb(Assembler::zero, L_copy_2_bytes); |
1574 __ movl(rax, Address(end_from, 8)); |
1575 __ movl(rax, Address(end_from, 8)); |
1575 __ movl(Address(end_to, 8), rax); |
1576 __ movl(Address(end_to, 8), rax); |
1576 |
1577 |
1577 __ addq(end_from, 4); |
1578 __ addptr(end_from, 4); |
1578 __ addq(end_to, 4); |
1579 __ addptr(end_to, 4); |
1579 |
1580 |
1580 // Check for and copy trailing word |
1581 // Check for and copy trailing word |
1581 __ BIND(L_copy_2_bytes); |
1582 __ BIND(L_copy_2_bytes); |
1582 __ testq(word_count, 1); |
1583 __ testl(word_count, 1); |
1583 __ jccb(Assembler::zero, L_exit); |
1584 __ jccb(Assembler::zero, L_exit); |
1584 __ movw(rax, Address(end_from, 8)); |
1585 __ movw(rax, Address(end_from, 8)); |
1585 __ movw(Address(end_to, 8), rax); |
1586 __ movw(Address(end_to, 8), rax); |
1586 |
1587 |
1587 __ BIND(L_exit); |
1588 __ BIND(L_exit); |
1588 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1589 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1589 restore_arg_regs(); |
1590 restore_arg_regs(); |
1590 __ xorq(rax, rax); // return 0 |
1591 __ xorptr(rax, rax); // return 0 |
1591 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1592 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1592 __ ret(0); |
1593 __ ret(0); |
1593 |
1594 |
1594 // Copy in 32-bytes chunks |
1595 // Copy in 32-bytes chunks |
1595 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1596 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1635 array_overlap_test(disjoint_short_copy_entry, Address::times_2); |
1636 array_overlap_test(disjoint_short_copy_entry, Address::times_2); |
1636 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1637 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1637 // r9 and r10 may be used to save non-volatile registers |
1638 // r9 and r10 may be used to save non-volatile registers |
1638 |
1639 |
1639 // 'from', 'to' and 'count' are now valid |
1640 // 'from', 'to' and 'count' are now valid |
1640 __ movq(word_count, count); |
1641 __ movptr(word_count, count); |
1641 __ shrq(count, 2); // count => qword_count |
1642 __ shrptr(count, 2); // count => qword_count |
1642 |
1643 |
1643 // Copy from high to low addresses. Use 'to' as scratch. |
1644 // Copy from high to low addresses. Use 'to' as scratch. |
1644 |
1645 |
1645 // Check for and copy trailing word |
1646 // Check for and copy trailing word |
1646 __ testq(word_count, 1); |
1647 __ testl(word_count, 1); |
1647 __ jccb(Assembler::zero, L_copy_4_bytes); |
1648 __ jccb(Assembler::zero, L_copy_4_bytes); |
1648 __ movw(rax, Address(from, word_count, Address::times_2, -2)); |
1649 __ movw(rax, Address(from, word_count, Address::times_2, -2)); |
1649 __ movw(Address(to, word_count, Address::times_2, -2), rax); |
1650 __ movw(Address(to, word_count, Address::times_2, -2), rax); |
1650 |
1651 |
1651 // Check for and copy trailing dword |
1652 // Check for and copy trailing dword |
1652 __ BIND(L_copy_4_bytes); |
1653 __ BIND(L_copy_4_bytes); |
1653 __ testq(word_count, 2); |
1654 __ testl(word_count, 2); |
1654 __ jcc(Assembler::zero, L_copy_32_bytes); |
1655 __ jcc(Assembler::zero, L_copy_32_bytes); |
1655 __ movl(rax, Address(from, qword_count, Address::times_8)); |
1656 __ movl(rax, Address(from, qword_count, Address::times_8)); |
1656 __ movl(Address(to, qword_count, Address::times_8), rax); |
1657 __ movl(Address(to, qword_count, Address::times_8), rax); |
1657 __ jmp(L_copy_32_bytes); |
1658 __ jmp(L_copy_32_bytes); |
1658 |
1659 |
1659 // Copy trailing qwords |
1660 // Copy trailing qwords |
1660 __ BIND(L_copy_8_bytes); |
1661 __ BIND(L_copy_8_bytes); |
1661 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1662 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1662 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1663 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1663 __ decrementq(qword_count); |
1664 __ decrement(qword_count); |
1664 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1665 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1665 |
1666 |
1666 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1667 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1667 restore_arg_regs(); |
1668 restore_arg_regs(); |
1668 __ xorq(rax, rax); // return 0 |
1669 __ xorptr(rax, rax); // return 0 |
1669 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1670 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1670 __ ret(0); |
1671 __ ret(0); |
1671 |
1672 |
1672 // Copy in 32-bytes chunks |
1673 // Copy in 32-bytes chunks |
1673 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1674 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1674 |
1675 |
1675 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1676 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
1676 restore_arg_regs(); |
1677 restore_arg_regs(); |
1677 __ xorq(rax, rax); // return 0 |
1678 __ xorptr(rax, rax); // return 0 |
1678 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1679 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1679 __ ret(0); |
1680 __ ret(0); |
1680 |
1681 |
1681 return start; |
1682 return start; |
1682 } |
1683 } |
1736 if (is_oop) { |
1737 if (is_oop) { |
1737 __ movq(saved_to, to); |
1738 __ movq(saved_to, to); |
1738 } |
1739 } |
1739 |
1740 |
1740 // 'from', 'to' and 'count' are now valid |
1741 // 'from', 'to' and 'count' are now valid |
1741 __ movq(dword_count, count); |
1742 __ movptr(dword_count, count); |
1742 __ shrq(count, 1); // count => qword_count |
1743 __ shrptr(count, 1); // count => qword_count |
1743 |
1744 |
1744 // Copy from low to high addresses. Use 'to' as scratch. |
1745 // Copy from low to high addresses. Use 'to' as scratch. |
1745 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); |
1746 __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); |
1746 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); |
1747 __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); |
1747 __ negq(qword_count); |
1748 __ negptr(qword_count); |
1748 __ jmp(L_copy_32_bytes); |
1749 __ jmp(L_copy_32_bytes); |
1749 |
1750 |
1750 // Copy trailing qwords |
1751 // Copy trailing qwords |
1751 __ BIND(L_copy_8_bytes); |
1752 __ BIND(L_copy_8_bytes); |
1752 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1753 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1753 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1754 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1754 __ incrementq(qword_count); |
1755 __ increment(qword_count); |
1755 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1756 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1756 |
1757 |
1757 // Check for and copy trailing dword |
1758 // Check for and copy trailing dword |
1758 __ BIND(L_copy_4_bytes); |
1759 __ BIND(L_copy_4_bytes); |
1759 __ testq(dword_count, 1); // Only byte test since the value is 0 or 1 |
1760 __ testl(dword_count, 1); // Only byte test since the value is 0 or 1 |
1760 __ jccb(Assembler::zero, L_exit); |
1761 __ jccb(Assembler::zero, L_exit); |
1761 __ movl(rax, Address(end_from, 8)); |
1762 __ movl(rax, Address(end_from, 8)); |
1762 __ movl(Address(end_to, 8), rax); |
1763 __ movl(Address(end_to, 8), rax); |
1763 |
1764 |
1764 __ BIND(L_exit); |
1765 __ BIND(L_exit); |
1823 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1824 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
1824 // r9 and r10 may be used to save non-volatile registers |
1825 // r9 and r10 may be used to save non-volatile registers |
1825 |
1826 |
1826 assert_clean_int(count, rax); // Make sure 'count' is clean int. |
1827 assert_clean_int(count, rax); // Make sure 'count' is clean int. |
1827 // 'from', 'to' and 'count' are now valid |
1828 // 'from', 'to' and 'count' are now valid |
1828 __ movq(dword_count, count); |
1829 __ movptr(dword_count, count); |
1829 __ shrq(count, 1); // count => qword_count |
1830 __ shrptr(count, 1); // count => qword_count |
1830 |
1831 |
1831 // Copy from high to low addresses. Use 'to' as scratch. |
1832 // Copy from high to low addresses. Use 'to' as scratch. |
1832 |
1833 |
1833 // Check for and copy trailing dword |
1834 // Check for and copy trailing dword |
1834 __ testq(dword_count, 1); |
1835 __ testl(dword_count, 1); |
1835 __ jcc(Assembler::zero, L_copy_32_bytes); |
1836 __ jcc(Assembler::zero, L_copy_32_bytes); |
1836 __ movl(rax, Address(from, dword_count, Address::times_4, -4)); |
1837 __ movl(rax, Address(from, dword_count, Address::times_4, -4)); |
1837 __ movl(Address(to, dword_count, Address::times_4, -4), rax); |
1838 __ movl(Address(to, dword_count, Address::times_4, -4), rax); |
1838 __ jmp(L_copy_32_bytes); |
1839 __ jmp(L_copy_32_bytes); |
1839 |
1840 |
1840 // Copy trailing qwords |
1841 // Copy trailing qwords |
1841 __ BIND(L_copy_8_bytes); |
1842 __ BIND(L_copy_8_bytes); |
1842 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1843 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1843 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1844 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1844 __ decrementq(qword_count); |
1845 __ decrement(qword_count); |
1845 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1846 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1846 |
1847 |
1847 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); |
1848 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); |
1848 if (is_oop) { |
1849 if (is_oop) { |
1849 __ jmp(L_exit); |
1850 __ jmp(L_exit); |
1850 } |
1851 } |
1851 restore_arg_regs(); |
1852 restore_arg_regs(); |
1852 __ xorq(rax, rax); // return 0 |
1853 __ xorptr(rax, rax); // return 0 |
1853 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1854 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1854 __ ret(0); |
1855 __ ret(0); |
1855 |
1856 |
1856 // Copy in 32-bytes chunks |
1857 // Copy in 32-bytes chunks |
1857 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1858 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1919 // r9 and r10 may be used to save non-volatile registers |
1920 // r9 and r10 may be used to save non-volatile registers |
1920 |
1921 |
1921 // 'from', 'to' and 'qword_count' are now valid |
1922 // 'from', 'to' and 'qword_count' are now valid |
1922 |
1923 |
1923 // Copy from low to high addresses. Use 'to' as scratch. |
1924 // Copy from low to high addresses. Use 'to' as scratch. |
1924 __ leaq(end_from, Address(from, qword_count, Address::times_8, -8)); |
1925 __ lea(end_from, Address(from, qword_count, Address::times_8, -8)); |
1925 __ leaq(end_to, Address(to, qword_count, Address::times_8, -8)); |
1926 __ lea(end_to, Address(to, qword_count, Address::times_8, -8)); |
1926 __ negq(qword_count); |
1927 __ negptr(qword_count); |
1927 __ jmp(L_copy_32_bytes); |
1928 __ jmp(L_copy_32_bytes); |
1928 |
1929 |
1929 // Copy trailing qwords |
1930 // Copy trailing qwords |
1930 __ BIND(L_copy_8_bytes); |
1931 __ BIND(L_copy_8_bytes); |
1931 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1932 __ movq(rax, Address(end_from, qword_count, Address::times_8, 8)); |
1932 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1933 __ movq(Address(end_to, qword_count, Address::times_8, 8), rax); |
1933 __ incrementq(qword_count); |
1934 __ increment(qword_count); |
1934 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1935 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1935 |
1936 |
1936 if (is_oop) { |
1937 if (is_oop) { |
1937 __ jmp(L_exit); |
1938 __ jmp(L_exit); |
1938 } else { |
1939 } else { |
1939 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
1940 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
1940 restore_arg_regs(); |
1941 restore_arg_regs(); |
1941 __ xorq(rax, rax); // return 0 |
1942 __ xorptr(rax, rax); // return 0 |
1942 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1943 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1943 __ ret(0); |
1944 __ ret(0); |
1944 } |
1945 } |
1945 |
1946 |
1946 // Copy 64-byte chunks |
1947 // Copy 64-byte chunks |
2006 |
2007 |
2007 // 'from', 'to' and 'qword_count' are now valid |
2008 // 'from', 'to' and 'qword_count' are now valid |
2008 |
2009 |
2009 if (is_oop) { |
2010 if (is_oop) { |
2010 // Save to and count for store barrier |
2011 // Save to and count for store barrier |
2011 __ movq(saved_count, qword_count); |
2012 __ movptr(saved_count, qword_count); |
2012 // No registers are destroyed by this call |
2013 // No registers are destroyed by this call |
2013 gen_write_ref_array_pre_barrier(to, saved_count); |
2014 gen_write_ref_array_pre_barrier(to, saved_count); |
2014 } |
2015 } |
2015 |
2016 |
2016 __ jmp(L_copy_32_bytes); |
2017 __ jmp(L_copy_32_bytes); |
2017 |
2018 |
2018 // Copy trailing qwords |
2019 // Copy trailing qwords |
2019 __ BIND(L_copy_8_bytes); |
2020 __ BIND(L_copy_8_bytes); |
2020 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
2021 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
2021 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
2022 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
2022 __ decrementq(qword_count); |
2023 __ decrement(qword_count); |
2023 __ jcc(Assembler::notZero, L_copy_8_bytes); |
2024 __ jcc(Assembler::notZero, L_copy_8_bytes); |
2024 |
2025 |
2025 if (is_oop) { |
2026 if (is_oop) { |
2026 __ jmp(L_exit); |
2027 __ jmp(L_exit); |
2027 } else { |
2028 } else { |
2028 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2029 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2029 restore_arg_regs(); |
2030 restore_arg_regs(); |
2030 __ xorq(rax, rax); // return 0 |
2031 __ xorptr(rax, rax); // return 0 |
2031 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2032 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2032 __ ret(0); |
2033 __ ret(0); |
2033 } |
2034 } |
2034 |
2035 |
2035 // Copy in 32-bytes chunks |
2036 // Copy in 32-bytes chunks |
2036 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
2037 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
2037 |
2038 |
2038 if (is_oop) { |
2039 if (is_oop) { |
2039 __ BIND(L_exit); |
2040 __ BIND(L_exit); |
2040 __ leaq(rcx, Address(to, saved_count, Address::times_8, -8)); |
2041 __ lea(rcx, Address(to, saved_count, Address::times_8, -8)); |
2041 gen_write_ref_array_post_barrier(to, rcx, rax); |
2042 gen_write_ref_array_post_barrier(to, rcx, rax); |
2042 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); |
2043 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); |
2043 } else { |
2044 } else { |
2044 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2045 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2045 } |
2046 } |
2046 restore_arg_regs(); |
2047 restore_arg_regs(); |
2047 __ xorq(rax, rax); // return 0 |
2048 __ xorptr(rax, rax); // return 0 |
2048 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2049 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2049 __ ret(0); |
2050 __ ret(0); |
2050 |
2051 |
2051 return start; |
2052 return start; |
2052 } |
2053 } |
2071 Klass::secondary_super_cache_offset_in_bytes()); |
2072 Klass::secondary_super_cache_offset_in_bytes()); |
2072 Address secondary_supers_addr(sub_klass, ss_offset); |
2073 Address secondary_supers_addr(sub_klass, ss_offset); |
2073 Address super_cache_addr( sub_klass, sc_offset); |
2074 Address super_cache_addr( sub_klass, sc_offset); |
2074 |
2075 |
2075 // if the pointers are equal, we are done (e.g., String[] elements) |
2076 // if the pointers are equal, we are done (e.g., String[] elements) |
2076 __ cmpq(super_klass, sub_klass); |
2077 __ cmpptr(super_klass, sub_klass); |
2077 __ jcc(Assembler::equal, L_success); |
2078 __ jcc(Assembler::equal, L_success); |
2078 |
2079 |
2079 // check the supertype display: |
2080 // check the supertype display: |
2080 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); |
2081 Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0); |
2081 __ cmpq(super_klass, super_check_addr); // test the super type |
2082 __ cmpptr(super_klass, super_check_addr); // test the super type |
2082 __ jcc(Assembler::equal, L_success); |
2083 __ jcc(Assembler::equal, L_success); |
2083 |
2084 |
2084 // if it was a primary super, we can just fail immediately |
2085 // if it was a primary super, we can just fail immediately |
2085 __ cmpl(super_check_offset, sc_offset); |
2086 __ cmpl(super_check_offset, sc_offset); |
2086 __ jcc(Assembler::notEqual, L_miss); |
2087 __ jcc(Assembler::notEqual, L_miss); |
2089 // The repne_scan instruction uses fixed registers, which we must spill. |
2090 // The repne_scan instruction uses fixed registers, which we must spill. |
2090 // (We need a couple more temps in any case.) |
2091 // (We need a couple more temps in any case.) |
2091 // This code is rarely used, so simplicity is a virtue here. |
2092 // This code is rarely used, so simplicity is a virtue here. |
2092 inc_counter_np(SharedRuntime::_partial_subtype_ctr); |
2093 inc_counter_np(SharedRuntime::_partial_subtype_ctr); |
2093 { |
2094 { |
2094 __ pushq(rax); |
2095 __ push(rax); |
2095 __ pushq(rcx); |
2096 __ push(rcx); |
2096 __ pushq(rdi); |
2097 __ push(rdi); |
2097 assert_different_registers(sub_klass, super_klass, rax, rcx, rdi); |
2098 assert_different_registers(sub_klass, super_klass, rax, rcx, rdi); |
2098 |
2099 |
2099 __ movq(rdi, secondary_supers_addr); |
2100 __ movptr(rdi, secondary_supers_addr); |
2100 // Load the array length. |
2101 // Load the array length. |
2101 __ movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes())); |
2102 __ movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes())); |
2102 // Skip to start of data. |
2103 // Skip to start of data. |
2103 __ addq(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
2104 __ addptr(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
2104 // Scan rcx words at [rdi] for occurance of rax |
2105 // Scan rcx words at [rdi] for occurance of rax |
2105 // Set NZ/Z based on last compare |
2106 // Set NZ/Z based on last compare |
2106 __ movq(rax, super_klass); |
2107 __ movptr(rax, super_klass); |
2107 if (UseCompressedOops) { |
2108 if (UseCompressedOops) { |
2108 // Compare against compressed form. Don't need to uncompress because |
2109 // Compare against compressed form. Don't need to uncompress because |
2109 // looks like orig rax is restored in popq below. |
2110 // looks like orig rax is restored in popq below. |
2110 __ encode_heap_oop(rax); |
2111 __ encode_heap_oop(rax); |
2111 __ repne_scanl(); |
2112 __ repne_scanl(); |
2112 } else { |
2113 } else { |
2113 __ repne_scanq(); |
2114 __ repne_scan(); |
2114 } |
2115 } |
2115 |
2116 |
2116 // Unspill the temp. registers: |
2117 // Unspill the temp. registers: |
2117 __ popq(rdi); |
2118 __ pop(rdi); |
2118 __ popq(rcx); |
2119 __ pop(rcx); |
2119 __ popq(rax); |
2120 __ pop(rax); |
2120 |
2121 |
2121 __ jcc(Assembler::notEqual, L_miss); |
2122 __ jcc(Assembler::notEqual, L_miss); |
2122 } |
2123 } |
2123 |
2124 |
2124 // Success. Cache the super we found and proceed in triumph. |
2125 // Success. Cache the super we found and proceed in triumph. |
2125 __ movq(super_cache_addr, super_klass); // note: rax is dead |
2126 __ movptr(super_cache_addr, super_klass); // note: rax is dead |
2126 __ jmp(L_success); |
2127 __ jmp(L_success); |
2127 |
2128 |
2128 // Fall through on failure! |
2129 // Fall through on failure! |
2129 __ BIND(L_miss); |
2130 __ BIND(L_miss); |
2130 } |
2131 } |
2200 saved_r14_offset, |
2201 saved_r14_offset, |
2201 saved_rbp_offset, |
2202 saved_rbp_offset, |
2202 saved_rip_offset, |
2203 saved_rip_offset, |
2203 saved_rarg0_offset |
2204 saved_rarg0_offset |
2204 }; |
2205 }; |
2205 __ subq(rsp, saved_rbp_offset * wordSize); |
2206 __ subptr(rsp, saved_rbp_offset * wordSize); |
2206 __ movq(Address(rsp, saved_r13_offset * wordSize), r13); |
2207 __ movptr(Address(rsp, saved_r13_offset * wordSize), r13); |
2207 __ movq(Address(rsp, saved_r14_offset * wordSize), r14); |
2208 __ movptr(Address(rsp, saved_r14_offset * wordSize), r14); |
2208 setup_arg_regs(4); // from => rdi, to => rsi, length => rdx |
2209 setup_arg_regs(4); // from => rdi, to => rsi, length => rdx |
2209 // ckoff => rcx, ckval => r8 |
2210 // ckoff => rcx, ckval => r8 |
2210 // r9 and r10 may be used to save non-volatile registers |
2211 // r9 and r10 may be used to save non-volatile registers |
2211 #ifdef _WIN64 |
2212 #ifdef _WIN64 |
2212 // last argument (#4) is on stack on Win64 |
2213 // last argument (#4) is on stack on Win64 |
2213 const int ckval_offset = saved_rarg0_offset + 4; |
2214 const int ckval_offset = saved_rarg0_offset + 4; |
2214 __ movq(ckval, Address(rsp, ckval_offset * wordSize)); |
2215 __ movptr(ckval, Address(rsp, ckval_offset * wordSize)); |
2215 #endif |
2216 #endif |
2216 |
2217 |
2217 // check that int operands are properly extended to size_t |
2218 // check that int operands are properly extended to size_t |
2218 assert_clean_int(length, rax); |
2219 assert_clean_int(length, rax); |
2219 assert_clean_int(ckoff, rax); |
2220 assert_clean_int(ckoff, rax); |
2240 Address to_element_addr(end_to, count, TIMES_OOP, 0); |
2241 Address to_element_addr(end_to, count, TIMES_OOP, 0); |
2241 |
2242 |
2242 gen_write_ref_array_pre_barrier(to, count); |
2243 gen_write_ref_array_pre_barrier(to, count); |
2243 |
2244 |
2244 // Copy from low to high addresses, indexed from the end of each array. |
2245 // Copy from low to high addresses, indexed from the end of each array. |
2245 __ leaq(end_from, end_from_addr); |
2246 __ lea(end_from, end_from_addr); |
2246 __ leaq(end_to, end_to_addr); |
2247 __ lea(end_to, end_to_addr); |
2247 __ movq(r14_length, length); // save a copy of the length |
2248 __ movptr(r14_length, length); // save a copy of the length |
2248 assert(length == count, ""); // else fix next line: |
2249 assert(length == count, ""); // else fix next line: |
2249 __ negq(count); // negate and test the length |
2250 __ negptr(count); // negate and test the length |
2250 __ jcc(Assembler::notZero, L_load_element); |
2251 __ jcc(Assembler::notZero, L_load_element); |
2251 |
2252 |
2252 // Empty array: Nothing to do. |
2253 // Empty array: Nothing to do. |
2253 __ xorq(rax, rax); // return 0 on (trivial) success |
2254 __ xorptr(rax, rax); // return 0 on (trivial) success |
2254 __ jmp(L_done); |
2255 __ jmp(L_done); |
2255 |
2256 |
2256 // ======== begin loop ======== |
2257 // ======== begin loop ======== |
2257 // (Loop is rotated; its entry is L_load_element.) |
2258 // (Loop is rotated; its entry is L_load_element.) |
2258 // Loop control: |
2259 // Loop control: |
2260 // Base pointers src, dst are biased by 8*(count-1),to last element. |
2261 // Base pointers src, dst are biased by 8*(count-1),to last element. |
2261 __ align(16); |
2262 __ align(16); |
2262 |
2263 |
2263 __ BIND(L_store_element); |
2264 __ BIND(L_store_element); |
2264 __ store_heap_oop(to_element_addr, rax_oop); // store the oop |
2265 __ store_heap_oop(to_element_addr, rax_oop); // store the oop |
2265 __ incrementq(count); // increment the count toward zero |
2266 __ increment(count); // increment the count toward zero |
2266 __ jcc(Assembler::zero, L_do_card_marks); |
2267 __ jcc(Assembler::zero, L_do_card_marks); |
2267 |
2268 |
2268 // ======== loop entry is here ======== |
2269 // ======== loop entry is here ======== |
2269 __ BIND(L_load_element); |
2270 __ BIND(L_load_element); |
2270 __ load_heap_oop(rax_oop, from_element_addr); // load the oop |
2271 __ load_heap_oop(rax_oop, from_element_addr); // load the oop |
2271 __ testq(rax_oop, rax_oop); |
2272 __ testptr(rax_oop, rax_oop); |
2272 __ jcc(Assembler::zero, L_store_element); |
2273 __ jcc(Assembler::zero, L_store_element); |
2273 |
2274 |
2274 __ load_klass(r11_klass, rax_oop);// query the object klass |
2275 __ load_klass(r11_klass, rax_oop);// query the object klass |
2275 generate_type_check(r11_klass, ckoff, ckval, L_store_element); |
2276 generate_type_check(r11_klass, ckoff, ckval, L_store_element); |
2276 // ======== end loop ======== |
2277 // ======== end loop ======== |
2278 // It was a real error; we must depend on the caller to finish the job. |
2279 // It was a real error; we must depend on the caller to finish the job. |
2279 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. |
2280 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. |
2280 // Emit GC store barriers for the oops we have copied (r14 + rdx), |
2281 // Emit GC store barriers for the oops we have copied (r14 + rdx), |
2281 // and report their number to the caller. |
2282 // and report their number to the caller. |
2282 assert_different_registers(rax, r14_length, count, to, end_to, rcx); |
2283 assert_different_registers(rax, r14_length, count, to, end_to, rcx); |
2283 __ leaq(end_to, to_element_addr); |
2284 __ lea(end_to, to_element_addr); |
2284 gen_write_ref_array_post_barrier(to, end_to, rcx); |
2285 gen_write_ref_array_post_barrier(to, end_to, rcx); |
2285 __ movq(rax, r14_length); // original oops |
2286 __ movptr(rax, r14_length); // original oops |
2286 __ addq(rax, count); // K = (original - remaining) oops |
2287 __ addptr(rax, count); // K = (original - remaining) oops |
2287 __ notq(rax); // report (-1^K) to caller |
2288 __ notptr(rax); // report (-1^K) to caller |
2288 __ jmp(L_done); |
2289 __ jmp(L_done); |
2289 |
2290 |
2290 // Come here on success only. |
2291 // Come here on success only. |
2291 __ BIND(L_do_card_marks); |
2292 __ BIND(L_do_card_marks); |
2292 __ addq(end_to, -wordSize); // make an inclusive end pointer |
2293 __ addptr(end_to, -wordSize); // make an inclusive end pointer |
2293 gen_write_ref_array_post_barrier(to, end_to, rcx); |
2294 gen_write_ref_array_post_barrier(to, end_to, rcx); |
2294 __ xorq(rax, rax); // return 0 on success |
2295 __ xorptr(rax, rax); // return 0 on success |
2295 |
2296 |
2296 // Common exit point (success or failure). |
2297 // Common exit point (success or failure). |
2297 __ BIND(L_done); |
2298 __ BIND(L_done); |
2298 __ movq(r13, Address(rsp, saved_r13_offset * wordSize)); |
2299 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); |
2299 __ movq(r14, Address(rsp, saved_r14_offset * wordSize)); |
2300 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); |
2300 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); |
2301 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); |
2301 restore_arg_regs(); |
2302 restore_arg_regs(); |
2302 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2303 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2303 __ ret(0); |
2304 __ ret(0); |
2304 |
2305 |
2467 // (7) src_pos + length must not exceed length of src. |
2468 // (7) src_pos + length must not exceed length of src. |
2468 // (8) dst_pos + length must not exceed length of dst. |
2469 // (8) dst_pos + length must not exceed length of dst. |
2469 // |
2470 // |
2470 |
2471 |
2471 // if (src == NULL) return -1; |
2472 // if (src == NULL) return -1; |
2472 __ testq(src, src); // src oop |
2473 __ testptr(src, src); // src oop |
2473 size_t j1off = __ offset(); |
2474 size_t j1off = __ offset(); |
2474 __ jccb(Assembler::zero, L_failed_0); |
2475 __ jccb(Assembler::zero, L_failed_0); |
2475 |
2476 |
2476 // if (src_pos < 0) return -1; |
2477 // if (src_pos < 0) return -1; |
2477 __ testl(src_pos, src_pos); // src_pos (32-bits) |
2478 __ testl(src_pos, src_pos); // src_pos (32-bits) |
2478 __ jccb(Assembler::negative, L_failed_0); |
2479 __ jccb(Assembler::negative, L_failed_0); |
2479 |
2480 |
2480 // if (dst == NULL) return -1; |
2481 // if (dst == NULL) return -1; |
2481 __ testq(dst, dst); // dst oop |
2482 __ testptr(dst, dst); // dst oop |
2482 __ jccb(Assembler::zero, L_failed_0); |
2483 __ jccb(Assembler::zero, L_failed_0); |
2483 |
2484 |
2484 // if (dst_pos < 0) return -1; |
2485 // if (dst_pos < 0) return -1; |
2485 __ testl(dst_pos, dst_pos); // dst_pos (32-bits) |
2486 __ testl(dst_pos, dst_pos); // dst_pos (32-bits) |
2486 size_t j4off = __ offset(); |
2487 size_t j4off = __ offset(); |
2572 const Register r10_offset = r10; // array offset |
2573 const Register r10_offset = r10; // array offset |
2573 const Register rax_elsize = rax_lh; // element size |
2574 const Register rax_elsize = rax_lh; // element size |
2574 |
2575 |
2575 __ movl(r10_offset, rax_lh); |
2576 __ movl(r10_offset, rax_lh); |
2576 __ shrl(r10_offset, Klass::_lh_header_size_shift); |
2577 __ shrl(r10_offset, Klass::_lh_header_size_shift); |
2577 __ andq(r10_offset, Klass::_lh_header_size_mask); // array_offset |
2578 __ andptr(r10_offset, Klass::_lh_header_size_mask); // array_offset |
2578 __ addq(src, r10_offset); // src array offset |
2579 __ addptr(src, r10_offset); // src array offset |
2579 __ addq(dst, r10_offset); // dst array offset |
2580 __ addptr(dst, r10_offset); // dst array offset |
2580 BLOCK_COMMENT("choose copy loop based on element size"); |
2581 BLOCK_COMMENT("choose copy loop based on element size"); |
2581 __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize |
2582 __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize |
2582 |
2583 |
2583 // next registers should be set before the jump to corresponding stub |
2584 // next registers should be set before the jump to corresponding stub |
2584 const Register from = c_rarg0; // source array address |
2585 const Register from = c_rarg0; // source array address |
2589 // since they are the same as 'src', 'src_pos', 'dst'. |
2590 // since they are the same as 'src', 'src_pos', 'dst'. |
2590 |
2591 |
2591 __ BIND(L_copy_bytes); |
2592 __ BIND(L_copy_bytes); |
2592 __ cmpl(rax_elsize, 0); |
2593 __ cmpl(rax_elsize, 0); |
2593 __ jccb(Assembler::notEqual, L_copy_shorts); |
2594 __ jccb(Assembler::notEqual, L_copy_shorts); |
2594 __ leaq(from, Address(src, src_pos, Address::times_1, 0));// src_addr |
2595 __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr |
2595 __ leaq(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr |
2596 __ lea(to, Address(dst, dst_pos, Address::times_1, 0));// dst_addr |
2596 __ movslq(count, r11_length); // length |
2597 __ movl2ptr(count, r11_length); // length |
2597 __ jump(RuntimeAddress(byte_copy_entry)); |
2598 __ jump(RuntimeAddress(byte_copy_entry)); |
2598 |
2599 |
2599 __ BIND(L_copy_shorts); |
2600 __ BIND(L_copy_shorts); |
2600 __ cmpl(rax_elsize, LogBytesPerShort); |
2601 __ cmpl(rax_elsize, LogBytesPerShort); |
2601 __ jccb(Assembler::notEqual, L_copy_ints); |
2602 __ jccb(Assembler::notEqual, L_copy_ints); |
2602 __ leaq(from, Address(src, src_pos, Address::times_2, 0));// src_addr |
2603 __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr |
2603 __ leaq(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr |
2604 __ lea(to, Address(dst, dst_pos, Address::times_2, 0));// dst_addr |
2604 __ movslq(count, r11_length); // length |
2605 __ movl2ptr(count, r11_length); // length |
2605 __ jump(RuntimeAddress(short_copy_entry)); |
2606 __ jump(RuntimeAddress(short_copy_entry)); |
2606 |
2607 |
2607 __ BIND(L_copy_ints); |
2608 __ BIND(L_copy_ints); |
2608 __ cmpl(rax_elsize, LogBytesPerInt); |
2609 __ cmpl(rax_elsize, LogBytesPerInt); |
2609 __ jccb(Assembler::notEqual, L_copy_longs); |
2610 __ jccb(Assembler::notEqual, L_copy_longs); |
2610 __ leaq(from, Address(src, src_pos, Address::times_4, 0));// src_addr |
2611 __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr |
2611 __ leaq(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr |
2612 __ lea(to, Address(dst, dst_pos, Address::times_4, 0));// dst_addr |
2612 __ movslq(count, r11_length); // length |
2613 __ movl2ptr(count, r11_length); // length |
2613 __ jump(RuntimeAddress(int_copy_entry)); |
2614 __ jump(RuntimeAddress(int_copy_entry)); |
2614 |
2615 |
2615 __ BIND(L_copy_longs); |
2616 __ BIND(L_copy_longs); |
2616 #ifdef ASSERT |
2617 #ifdef ASSERT |
2617 { Label L; |
2618 { Label L; |
2619 __ jcc(Assembler::equal, L); |
2620 __ jcc(Assembler::equal, L); |
2620 __ stop("must be long copy, but elsize is wrong"); |
2621 __ stop("must be long copy, but elsize is wrong"); |
2621 __ bind(L); |
2622 __ bind(L); |
2622 } |
2623 } |
2623 #endif |
2624 #endif |
2624 __ leaq(from, Address(src, src_pos, Address::times_8, 0));// src_addr |
2625 __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr |
2625 __ leaq(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr |
2626 __ lea(to, Address(dst, dst_pos, Address::times_8, 0));// dst_addr |
2626 __ movslq(count, r11_length); // length |
2627 __ movl2ptr(count, r11_length); // length |
2627 __ jump(RuntimeAddress(long_copy_entry)); |
2628 __ jump(RuntimeAddress(long_copy_entry)); |
2628 |
2629 |
2629 // objArrayKlass |
2630 // objArrayKlass |
2630 __ BIND(L_objArray); |
2631 __ BIND(L_objArray); |
2631 // live at this point: r10_src_klass, src[_pos], dst[_pos] |
2632 // live at this point: r10_src_klass, src[_pos], dst[_pos] |
2638 |
2639 |
2639 // Identically typed arrays can be copied without element-wise checks. |
2640 // Identically typed arrays can be copied without element-wise checks. |
2640 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, |
2641 arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length, |
2641 r10, L_failed); |
2642 r10, L_failed); |
2642 |
2643 |
2643 __ leaq(from, Address(src, src_pos, TIMES_OOP, |
2644 __ lea(from, Address(src, src_pos, TIMES_OOP, |
2644 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr |
2645 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr |
2645 __ leaq(to, Address(dst, dst_pos, TIMES_OOP, |
2646 __ lea(to, Address(dst, dst_pos, TIMES_OOP, |
2646 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr |
2647 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr |
2647 __ movslq(count, r11_length); // length |
2648 __ movl2ptr(count, r11_length); // length |
2648 __ BIND(L_plain_copy); |
2649 __ BIND(L_plain_copy); |
2649 __ jump(RuntimeAddress(oop_copy_entry)); |
2650 __ jump(RuntimeAddress(oop_copy_entry)); |
2650 |
2651 |
2651 __ BIND(L_checkcast_copy); |
2652 __ BIND(L_checkcast_copy); |
2652 // live at this point: r10_src_klass, !r11_length |
2653 // live at this point: r10_src_klass, !r11_length |
2669 rax, L_failed); |
2670 rax, L_failed); |
2670 __ load_klass(r11_dst_klass, dst); // reload |
2671 __ load_klass(r11_dst_klass, dst); // reload |
2671 #endif |
2672 #endif |
2672 |
2673 |
2673 // Marshal the base address arguments now, freeing registers. |
2674 // Marshal the base address arguments now, freeing registers. |
2674 __ leaq(from, Address(src, src_pos, TIMES_OOP, |
2675 __ lea(from, Address(src, src_pos, TIMES_OOP, |
2675 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); |
2676 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); |
2676 __ leaq(to, Address(dst, dst_pos, TIMES_OOP, |
2677 __ lea(to, Address(dst, dst_pos, TIMES_OOP, |
2677 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); |
2678 arrayOopDesc::base_offset_in_bytes(T_OBJECT))); |
2678 __ movl(count, C_RARG4); // length (reloaded) |
2679 __ movl(count, C_RARG4); // length (reloaded) |
2679 Register sco_temp = c_rarg3; // this register is free now |
2680 Register sco_temp = c_rarg3; // this register is free now |
2680 assert_different_registers(from, to, count, sco_temp, |
2681 assert_different_registers(from, to, count, sco_temp, |
2681 r11_dst_klass, r10_src_klass); |
2682 r11_dst_klass, r10_src_klass); |
2689 generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy); |
2690 generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy); |
2690 |
2691 |
2691 // Fetch destination element klass from the objArrayKlass header. |
2692 // Fetch destination element klass from the objArrayKlass header. |
2692 int ek_offset = (klassOopDesc::header_size() * HeapWordSize + |
2693 int ek_offset = (klassOopDesc::header_size() * HeapWordSize + |
2693 objArrayKlass::element_klass_offset_in_bytes()); |
2694 objArrayKlass::element_klass_offset_in_bytes()); |
2694 __ movq(r11_dst_klass, Address(r11_dst_klass, ek_offset)); |
2695 __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset)); |
2695 __ movl(sco_temp, Address(r11_dst_klass, sco_offset)); |
2696 __ movl(sco_temp, Address(r11_dst_klass, sco_offset)); |
2696 assert_clean_int(sco_temp, rax); |
2697 assert_clean_int(sco_temp, rax); |
2697 |
2698 |
2698 // the checkcast_copy loop needs two extra arguments: |
2699 // the checkcast_copy loop needs two extra arguments: |
2699 assert(c_rarg3 == sco_temp, "#3 already in place"); |
2700 assert(c_rarg3 == sco_temp, "#3 already in place"); |
2700 __ movq(C_RARG4, r11_dst_klass); // dst.klass.element_klass |
2701 __ movptr(C_RARG4, r11_dst_klass); // dst.klass.element_klass |
2701 __ jump(RuntimeAddress(checkcast_copy_entry)); |
2702 __ jump(RuntimeAddress(checkcast_copy_entry)); |
2702 } |
2703 } |
2703 |
2704 |
2704 __ BIND(L_failed); |
2705 __ BIND(L_failed); |
2705 __ xorq(rax, rax); |
2706 __ xorptr(rax, rax); |
2706 __ notq(rax); // return -1 |
2707 __ notptr(rax); // return -1 |
2707 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2708 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2708 __ ret(0); |
2709 __ ret(0); |
2709 |
2710 |
2710 return start; |
2711 return start; |
2711 } |
2712 } |
2804 // This is an inlined and slightly modified version of call_VM |
2805 // This is an inlined and slightly modified version of call_VM |
2805 // which has the ability to fetch the return PC out of |
2806 // which has the ability to fetch the return PC out of |
2806 // thread-local storage and also sets up last_Java_sp slightly |
2807 // thread-local storage and also sets up last_Java_sp slightly |
2807 // differently than the real call_VM |
2808 // differently than the real call_VM |
2808 if (restore_saved_exception_pc) { |
2809 if (restore_saved_exception_pc) { |
2809 __ movq(rax, |
2810 __ movptr(rax, |
2810 Address(r15_thread, |
2811 Address(r15_thread, |
2811 in_bytes(JavaThread::saved_exception_pc_offset()))); |
2812 in_bytes(JavaThread::saved_exception_pc_offset()))); |
2812 __ pushq(rax); |
2813 __ push(rax); |
2813 } |
2814 } |
2814 |
2815 |
2815 __ enter(); // required for proper stackwalking of RuntimeStub frame |
2816 __ enter(); // required for proper stackwalking of RuntimeStub frame |
2816 |
2817 |
2817 assert(is_even(framesize/2), "sp not 16-byte aligned"); |
2818 assert(is_even(framesize/2), "sp not 16-byte aligned"); |
2818 |
2819 |
2819 // return address and rbp are already in place |
2820 // return address and rbp are already in place |
2820 __ subq(rsp, (framesize-4) << LogBytesPerInt); // prolog |
2821 __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog |
2821 |
2822 |
2822 int frame_complete = __ pc() - start; |
2823 int frame_complete = __ pc() - start; |
2823 |
2824 |
2824 // Set up last_Java_sp and last_Java_fp |
2825 // Set up last_Java_sp and last_Java_fp |
2825 __ set_last_Java_frame(rsp, rbp, NULL); |
2826 __ set_last_Java_frame(rsp, rbp, NULL); |
2826 |
2827 |
2827 // Call runtime |
2828 // Call runtime |
2828 __ movq(c_rarg0, r15_thread); |
2829 __ movptr(c_rarg0, r15_thread); |
2829 BLOCK_COMMENT("call runtime_entry"); |
2830 BLOCK_COMMENT("call runtime_entry"); |
2830 __ call(RuntimeAddress(runtime_entry)); |
2831 __ call(RuntimeAddress(runtime_entry)); |
2831 |
2832 |
2832 // Generate oop map |
2833 // Generate oop map |
2833 OopMap* map = new OopMap(framesize, 0); |
2834 OopMap* map = new OopMap(framesize, 0); |
2946 SharedRuntime:: |
2947 SharedRuntime:: |
2947 throw_StackOverflowError), |
2948 throw_StackOverflowError), |
2948 false); |
2949 false); |
2949 |
2950 |
2950 // entry points that are platform specific |
2951 // entry points that are platform specific |
2951 StubRoutines::amd64::_f2i_fixup = generate_f2i_fixup(); |
2952 StubRoutines::x86::_f2i_fixup = generate_f2i_fixup(); |
2952 StubRoutines::amd64::_f2l_fixup = generate_f2l_fixup(); |
2953 StubRoutines::x86::_f2l_fixup = generate_f2l_fixup(); |
2953 StubRoutines::amd64::_d2i_fixup = generate_d2i_fixup(); |
2954 StubRoutines::x86::_d2i_fixup = generate_d2i_fixup(); |
2954 StubRoutines::amd64::_d2l_fixup = generate_d2l_fixup(); |
2955 StubRoutines::x86::_d2l_fixup = generate_d2l_fixup(); |
2955 |
2956 |
2956 StubRoutines::amd64::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); |
2957 StubRoutines::x86::_float_sign_mask = generate_fp_mask("float_sign_mask", 0x7FFFFFFF7FFFFFFF); |
2957 StubRoutines::amd64::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); |
2958 StubRoutines::x86::_float_sign_flip = generate_fp_mask("float_sign_flip", 0x8000000080000000); |
2958 StubRoutines::amd64::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); |
2959 StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF); |
2959 StubRoutines::amd64::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); |
2960 StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000); |
2960 |
2961 |
2961 // support for verify_oop (must happen after universe_init) |
2962 // support for verify_oop (must happen after universe_init) |
2962 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); |
2963 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop(); |
2963 |
2964 |
2964 // arraycopy stubs used by compilers |
2965 // arraycopy stubs used by compilers |