93 |
93 |
94 #ifdef PRODUCT |
94 #ifdef PRODUCT |
95 #define inc_counter_np(counter) (0) |
95 #define inc_counter_np(counter) (0) |
96 #else |
96 #else |
97 void inc_counter_np_(int& counter) { |
97 void inc_counter_np_(int& counter) { |
|
98 // This can destroy rscratch1 if counter is far from the code cache |
98 __ incrementl(ExternalAddress((address)&counter)); |
99 __ incrementl(ExternalAddress((address)&counter)); |
99 } |
100 } |
100 #define inc_counter_np(counter) \ |
101 #define inc_counter_np(counter) \ |
101 BLOCK_COMMENT("inc_counter " #counter); \ |
102 BLOCK_COMMENT("inc_counter " #counter); \ |
102 inc_counter_np_(counter); |
103 inc_counter_np_(counter); |
1464 __ jccb(Assembler::zero, L_exit); |
1465 __ jccb(Assembler::zero, L_exit); |
1465 __ movb(rax, Address(end_from, 8)); |
1466 __ movb(rax, Address(end_from, 8)); |
1466 __ movb(Address(end_to, 8), rax); |
1467 __ movb(Address(end_to, 8), rax); |
1467 |
1468 |
1468 __ BIND(L_exit); |
1469 __ BIND(L_exit); |
1469 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
|
1470 restore_arg_regs(); |
1470 restore_arg_regs(); |
|
1471 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free |
1471 __ xorptr(rax, rax); // return 0 |
1472 __ xorptr(rax, rax); // return 0 |
1472 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1473 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1473 __ ret(0); |
1474 __ ret(0); |
1474 |
1475 |
1475 // Copy in 32-bytes chunks |
1476 // Copy in 32-bytes chunks |
1553 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1554 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1554 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1555 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1555 __ decrement(qword_count); |
1556 __ decrement(qword_count); |
1556 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1557 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1557 |
1558 |
1558 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
|
1559 restore_arg_regs(); |
1559 restore_arg_regs(); |
|
1560 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free |
1560 __ xorptr(rax, rax); // return 0 |
1561 __ xorptr(rax, rax); // return 0 |
1561 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1562 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1562 __ ret(0); |
1563 __ ret(0); |
1563 |
1564 |
1564 // Copy in 32-bytes chunks |
1565 // Copy in 32-bytes chunks |
1565 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1566 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1566 |
1567 |
1567 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); |
|
1568 restore_arg_regs(); |
1568 restore_arg_regs(); |
|
1569 inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free |
1569 __ xorptr(rax, rax); // return 0 |
1570 __ xorptr(rax, rax); // return 0 |
1570 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1571 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1571 __ ret(0); |
1572 __ ret(0); |
1572 |
1573 |
1573 return start; |
1574 return start; |
1656 __ jccb(Assembler::zero, L_exit); |
1657 __ jccb(Assembler::zero, L_exit); |
1657 __ movw(rax, Address(end_from, 8)); |
1658 __ movw(rax, Address(end_from, 8)); |
1658 __ movw(Address(end_to, 8), rax); |
1659 __ movw(Address(end_to, 8), rax); |
1659 |
1660 |
1660 __ BIND(L_exit); |
1661 __ BIND(L_exit); |
1661 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
|
1662 restore_arg_regs(); |
1662 restore_arg_regs(); |
|
1663 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free |
1663 __ xorptr(rax, rax); // return 0 |
1664 __ xorptr(rax, rax); // return 0 |
1664 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1665 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1665 __ ret(0); |
1666 __ ret(0); |
1666 |
1667 |
1667 // Copy in 32-bytes chunks |
1668 // Copy in 32-bytes chunks |
1757 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1758 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1758 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1759 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1759 __ decrement(qword_count); |
1760 __ decrement(qword_count); |
1760 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1761 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1761 |
1762 |
1762 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
|
1763 restore_arg_regs(); |
1763 restore_arg_regs(); |
|
1764 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free |
1764 __ xorptr(rax, rax); // return 0 |
1765 __ xorptr(rax, rax); // return 0 |
1765 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1766 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1766 __ ret(0); |
1767 __ ret(0); |
1767 |
1768 |
1768 // Copy in 32-bytes chunks |
1769 // Copy in 32-bytes chunks |
1769 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1770 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1770 |
1771 |
1771 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); |
|
1772 restore_arg_regs(); |
1772 restore_arg_regs(); |
|
1773 inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free |
1773 __ xorptr(rax, rax); // return 0 |
1774 __ xorptr(rax, rax); // return 0 |
1774 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1775 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1775 __ ret(0); |
1776 __ ret(0); |
1776 |
1777 |
1777 return start; |
1778 return start; |
1857 __ BIND(L_exit); |
1858 __ BIND(L_exit); |
1858 if (is_oop) { |
1859 if (is_oop) { |
1859 __ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4)); |
1860 __ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4)); |
1860 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
1861 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
1861 } |
1862 } |
1862 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); |
|
1863 restore_arg_regs(); |
1863 restore_arg_regs(); |
|
1864 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
1864 __ xorptr(rax, rax); // return 0 |
1865 __ xorptr(rax, rax); // return 0 |
1865 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1866 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1866 __ ret(0); |
1867 __ ret(0); |
1867 |
1868 |
1868 // Copy 32-bytes chunks |
1869 // Copy 32-bytes chunks |
1938 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1939 __ movq(rax, Address(from, qword_count, Address::times_8, -8)); |
1939 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1940 __ movq(Address(to, qword_count, Address::times_8, -8), rax); |
1940 __ decrement(qword_count); |
1941 __ decrement(qword_count); |
1941 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1942 __ jcc(Assembler::notZero, L_copy_8_bytes); |
1942 |
1943 |
1943 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); |
|
1944 if (is_oop) { |
1944 if (is_oop) { |
1945 __ jmp(L_exit); |
1945 __ jmp(L_exit); |
1946 } |
1946 } |
1947 restore_arg_regs(); |
1947 restore_arg_regs(); |
|
1948 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
1948 __ xorptr(rax, rax); // return 0 |
1949 __ xorptr(rax, rax); // return 0 |
1949 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1950 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1950 __ ret(0); |
1951 __ ret(0); |
1951 |
1952 |
1952 // Copy in 32-bytes chunks |
1953 // Copy in 32-bytes chunks |
1953 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1954 copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
1954 |
1955 |
1955 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); |
|
1956 __ bind(L_exit); |
1956 __ bind(L_exit); |
1957 if (is_oop) { |
1957 if (is_oop) { |
1958 Register end_to = rdx; |
1958 Register end_to = rdx; |
1959 __ leaq(end_to, Address(to, dword_count, Address::times_4, -4)); |
1959 __ leaq(end_to, Address(to, dword_count, Address::times_4, -4)); |
1960 gen_write_ref_array_post_barrier(to, end_to, rax); |
1960 gen_write_ref_array_post_barrier(to, end_to, rax); |
1961 } |
1961 } |
1962 restore_arg_regs(); |
1962 restore_arg_regs(); |
|
1963 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
1963 __ xorptr(rax, rax); // return 0 |
1964 __ xorptr(rax, rax); // return 0 |
1964 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1965 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1965 __ ret(0); |
1966 __ ret(0); |
1966 |
1967 |
1967 return start; |
1968 return start; |
2043 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
2044 copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes); |
2044 |
2045 |
2045 if (is_oop) { |
2046 if (is_oop) { |
2046 __ BIND(L_exit); |
2047 __ BIND(L_exit); |
2047 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
2048 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
2048 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); |
2049 } |
|
2050 restore_arg_regs(); |
|
2051 if (is_oop) { |
|
2052 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2049 } else { |
2053 } else { |
2050 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2054 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free |
2051 } |
2055 } |
2052 restore_arg_regs(); |
|
2053 __ xorptr(rax, rax); // return 0 |
2056 __ xorptr(rax, rax); // return 0 |
2054 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2057 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2055 __ ret(0); |
2058 __ ret(0); |
2056 |
2059 |
2057 return start; |
2060 return start; |
2125 |
2128 |
2126 if (is_oop) { |
2129 if (is_oop) { |
2127 __ BIND(L_exit); |
2130 __ BIND(L_exit); |
2128 __ lea(rcx, Address(to, saved_count, Address::times_8, -8)); |
2131 __ lea(rcx, Address(to, saved_count, Address::times_8, -8)); |
2129 gen_write_ref_array_post_barrier(to, rcx, rax); |
2132 gen_write_ref_array_post_barrier(to, rcx, rax); |
2130 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); |
2133 } |
|
2134 restore_arg_regs(); |
|
2135 if (is_oop) { |
|
2136 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2131 } else { |
2137 } else { |
2132 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); |
2138 inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free |
2133 } |
2139 } |
2134 restore_arg_regs(); |
|
2135 __ xorptr(rax, rax); // return 0 |
2140 __ xorptr(rax, rax); // return 0 |
2136 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2141 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2137 __ ret(0); |
2142 __ ret(0); |
2138 |
2143 |
2139 return start; |
2144 return start; |
2329 |
2334 |
2330 // Common exit point (success or failure). |
2335 // Common exit point (success or failure). |
2331 __ BIND(L_done); |
2336 __ BIND(L_done); |
2332 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); |
2337 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); |
2333 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); |
2338 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); |
2334 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); |
|
2335 restore_arg_regs(); |
2339 restore_arg_regs(); |
|
2340 inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free |
2336 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2341 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2337 __ ret(0); |
2342 __ ret(0); |
2338 |
2343 |
2339 return start; |
2344 return start; |
2340 } |
2345 } |