1215 // |
1215 // |
1216 // Generate code for an array write post barrier |
1216 // Generate code for an array write post barrier |
1217 // |
1217 // |
1218 // Input: |
1218 // Input: |
1219 // start - register containing starting address of destination array |
1219 // start - register containing starting address of destination array |
1220 // end - register containing ending address of destination array |
1220 // count - elements count |
1221 // scratch - scratch register |
1221 // scratch - scratch register |
1222 // |
1222 // |
1223 // The input registers are overwritten. |
1223 // The input registers are overwritten. |
1224 // The ending address is inclusive. |
1224 // |
1225 void gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) { |
1225 void gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) { |
1226 assert_different_registers(start, end, scratch); |
1226 assert_different_registers(start, count, scratch); |
1227 BarrierSet* bs = Universe::heap()->barrier_set(); |
1227 BarrierSet* bs = Universe::heap()->barrier_set(); |
1228 switch (bs->kind()) { |
1228 switch (bs->kind()) { |
1229 case BarrierSet::G1SATBCT: |
1229 case BarrierSet::G1SATBCT: |
1230 case BarrierSet::G1SATBCTLogging: |
1230 case BarrierSet::G1SATBCTLogging: |
1231 |
|
1232 { |
1231 { |
1233 __ pusha(); // push registers (overkill) |
1232 __ pusha(); // push registers (overkill) |
1234 // must compute element count unless barrier set interface is changed (other platforms supply count) |
1233 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx |
1235 assert_different_registers(start, end, scratch); |
1234 assert_different_registers(c_rarg1, start); |
1236 __ lea(scratch, Address(end, BytesPerHeapOop)); |
1235 __ mov(c_rarg1, count); |
1237 __ subptr(scratch, start); // subtract start to get #bytes |
1236 __ mov(c_rarg0, start); |
1238 __ shrptr(scratch, LogBytesPerHeapOop); // convert to element count |
1237 } else { |
1239 __ mov(c_rarg0, start); |
1238 assert_different_registers(c_rarg0, count); |
1240 __ mov(c_rarg1, scratch); |
1239 __ mov(c_rarg0, start); |
|
1240 __ mov(c_rarg1, count); |
|
1241 } |
1241 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2); |
1242 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2); |
1242 __ popa(); |
1243 __ popa(); |
1243 } |
1244 } |
1244 break; |
1245 break; |
1245 case BarrierSet::CardTableModRef: |
1246 case BarrierSet::CardTableModRef: |
1247 { |
1248 { |
1248 CardTableModRefBS* ct = (CardTableModRefBS*)bs; |
1249 CardTableModRefBS* ct = (CardTableModRefBS*)bs; |
1249 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
1250 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
1250 |
1251 |
1251 Label L_loop; |
1252 Label L_loop; |
1252 |
1253 const Register end = count; |
1253 __ shrptr(start, CardTableModRefBS::card_shift); |
1254 |
1254 __ addptr(end, BytesPerHeapOop); |
1255 __ leaq(end, Address(start, count, TIMES_OOP, 0)); // end == start+count*oop_size |
1255 __ shrptr(end, CardTableModRefBS::card_shift); |
1256 __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive |
1256 __ subptr(end, start); // number of bytes to copy |
1257 __ shrptr(start, CardTableModRefBS::card_shift); |
1257 |
1258 __ shrptr(end, CardTableModRefBS::card_shift); |
1258 intptr_t disp = (intptr_t) ct->byte_map_base; |
1259 __ subptr(end, start); // end --> cards count |
1259 if (Assembler::is_simm32(disp)) { |
1260 |
1260 Address cardtable(noreg, noreg, Address::no_scale, disp); |
1261 int64_t disp = (int64_t) ct->byte_map_base; |
1261 __ lea(scratch, cardtable); |
1262 __ mov64(scratch, disp); |
1262 } else { |
|
1263 ExternalAddress cardtable((address)disp); |
|
1264 __ lea(scratch, cardtable); |
|
1265 } |
|
1266 |
|
1267 const Register count = end; // 'end' register contains bytes count now |
|
1268 __ addptr(start, scratch); |
1263 __ addptr(start, scratch); |
1269 __ BIND(L_loop); |
1264 __ BIND(L_loop); |
1270 __ movb(Address(start, count, Address::times_1), 0); |
1265 __ movb(Address(start, count, Address::times_1), 0); |
1271 __ decrement(count); |
1266 __ decrement(count); |
1272 __ jcc(Assembler::greaterEqual, L_loop); |
1267 __ jcc(Assembler::greaterEqual, L_loop); |
1914 __ movl(rax, Address(end_from, 8)); |
1909 __ movl(rax, Address(end_from, 8)); |
1915 __ movl(Address(end_to, 8), rax); |
1910 __ movl(Address(end_to, 8), rax); |
1916 |
1911 |
1917 __ BIND(L_exit); |
1912 __ BIND(L_exit); |
1918 if (is_oop) { |
1913 if (is_oop) { |
1919 __ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4)); |
1914 gen_write_ref_array_post_barrier(saved_to, dword_count, rax); |
1920 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
|
1921 } |
1915 } |
1922 restore_arg_regs(); |
1916 restore_arg_regs(); |
1923 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
1917 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
1924 __ xorptr(rax, rax); // return 0 |
1918 __ xorptr(rax, rax); // return 0 |
1925 __ leave(); // required for proper stackwalking of RuntimeStub frame |
1919 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2010 __ ret(0); |
2004 __ ret(0); |
2011 |
2005 |
2012 // Copy in multi-bytes chunks |
2006 // Copy in multi-bytes chunks |
2013 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2007 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2014 |
2008 |
2015 __ bind(L_exit); |
2009 __ BIND(L_exit); |
2016 if (is_oop) { |
2010 if (is_oop) { |
2017 Register end_to = rdx; |
2011 gen_write_ref_array_post_barrier(to, dword_count, rax); |
2018 __ leaq(end_to, Address(to, dword_count, Address::times_4, -4)); |
2012 } |
2019 gen_write_ref_array_post_barrier(to, end_to, rax); |
|
2020 } |
|
2021 restore_arg_regs(); |
2013 restore_arg_regs(); |
2022 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
2014 inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free |
2023 __ xorptr(rax, rax); // return 0 |
2015 __ xorptr(rax, rax); // return 0 |
2024 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2016 __ leave(); // required for proper stackwalking of RuntimeStub frame |
2025 __ ret(0); |
2017 __ ret(0); |
2053 const Register to = rsi; // destination array address |
2045 const Register to = rsi; // destination array address |
2054 const Register qword_count = rdx; // elements count |
2046 const Register qword_count = rdx; // elements count |
2055 const Register end_from = from; // source array end address |
2047 const Register end_from = from; // source array end address |
2056 const Register end_to = rcx; // destination array end address |
2048 const Register end_to = rcx; // destination array end address |
2057 const Register saved_to = to; |
2049 const Register saved_to = to; |
|
2050 const Register saved_count = r11; |
2058 // End pointers are inclusive, and if count is not zero they point |
2051 // End pointers are inclusive, and if count is not zero they point |
2059 // to the last unit copied: end_to[0] := end_from[0] |
2052 // to the last unit copied: end_to[0] := end_from[0] |
2060 |
2053 |
2061 __ enter(); // required for proper stackwalking of RuntimeStub frame |
2054 __ enter(); // required for proper stackwalking of RuntimeStub frame |
2062 // Save no-overlap entry point for generate_conjoint_long_oop_copy() |
2055 // Save no-overlap entry point for generate_conjoint_long_oop_copy() |
2070 |
2063 |
2071 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
2064 setup_arg_regs(); // from => rdi, to => rsi, count => rdx |
2072 // r9 and r10 may be used to save non-volatile registers |
2065 // r9 and r10 may be used to save non-volatile registers |
2073 // 'from', 'to' and 'qword_count' are now valid |
2066 // 'from', 'to' and 'qword_count' are now valid |
2074 if (is_oop) { |
2067 if (is_oop) { |
|
2068 // Save to and count for store barrier |
|
2069 __ movptr(saved_count, qword_count); |
2075 // no registers are destroyed by this call |
2070 // no registers are destroyed by this call |
2076 gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); |
2071 gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); |
2077 } |
2072 } |
2078 |
2073 |
2079 // Copy from low to high addresses. Use 'to' as scratch. |
2074 // Copy from low to high addresses. Use 'to' as scratch. |
2102 // Copy in multi-bytes chunks |
2097 // Copy in multi-bytes chunks |
2103 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2098 copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2104 |
2099 |
2105 if (is_oop) { |
2100 if (is_oop) { |
2106 __ BIND(L_exit); |
2101 __ BIND(L_exit); |
2107 gen_write_ref_array_post_barrier(saved_to, end_to, rax); |
2102 gen_write_ref_array_post_barrier(saved_to, saved_count, rax); |
2108 } |
2103 } |
2109 restore_arg_regs(); |
2104 restore_arg_regs(); |
2110 if (is_oop) { |
2105 if (is_oop) { |
2111 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2106 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2112 } else { |
2107 } else { |
2185 // Copy in multi-bytes chunks |
2180 // Copy in multi-bytes chunks |
2186 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2181 copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes); |
2187 |
2182 |
2188 if (is_oop) { |
2183 if (is_oop) { |
2189 __ BIND(L_exit); |
2184 __ BIND(L_exit); |
2190 __ lea(rcx, Address(to, saved_count, Address::times_8, -8)); |
2185 gen_write_ref_array_post_barrier(to, saved_count, rax); |
2191 gen_write_ref_array_post_barrier(to, rcx, rax); |
|
2192 } |
2186 } |
2193 restore_arg_regs(); |
2187 restore_arg_regs(); |
2194 if (is_oop) { |
2188 if (is_oop) { |
2195 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2189 inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free |
2196 } else { |
2190 } else { |
2373 |
2367 |
2374 // It was a real error; we must depend on the caller to finish the job. |
2368 // It was a real error; we must depend on the caller to finish the job. |
2375 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. |
2369 // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops. |
2376 // Emit GC store barriers for the oops we have copied (r14 + rdx), |
2370 // Emit GC store barriers for the oops we have copied (r14 + rdx), |
2377 // and report their number to the caller. |
2371 // and report their number to the caller. |
2378 assert_different_registers(rax, r14_length, count, to, end_to, rcx); |
2372 assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1); |
2379 __ lea(end_to, to_element_addr); |
2373 Label L_post_barrier; |
2380 __ addptr(end_to, -heapOopSize); // make an inclusive end pointer |
2374 __ addptr(r14_length, count); // K = (original - remaining) oops |
2381 gen_write_ref_array_post_barrier(to, end_to, rscratch1); |
2375 __ movptr(rax, r14_length); // save the value |
2382 __ movptr(rax, r14_length); // original oops |
2376 __ notptr(rax); // report (-1^K) to caller (does not affect flags) |
2383 __ addptr(rax, count); // K = (original - remaining) oops |
2377 __ jccb(Assembler::notZero, L_post_barrier); |
2384 __ notptr(rax); // report (-1^K) to caller |
2378 __ jmp(L_done); // K == 0, nothing was copied, skip post barrier |
2385 __ jmp(L_done); |
|
2386 |
2379 |
2387 // Come here on success only. |
2380 // Come here on success only. |
2388 __ BIND(L_do_card_marks); |
2381 __ BIND(L_do_card_marks); |
2389 __ addptr(end_to, -heapOopSize); // make an inclusive end pointer |
2382 __ xorptr(rax, rax); // return 0 on success |
2390 gen_write_ref_array_post_barrier(to, end_to, rscratch1); |
2383 |
2391 __ xorptr(rax, rax); // return 0 on success |
2384 __ BIND(L_post_barrier); |
|
2385 gen_write_ref_array_post_barrier(to, r14_length, rscratch1); |
2392 |
2386 |
2393 // Common exit point (success or failure). |
2387 // Common exit point (success or failure). |
2394 __ BIND(L_done); |
2388 __ BIND(L_done); |
2395 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); |
2389 __ movptr(r13, Address(rsp, saved_r13_offset * wordSize)); |
2396 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); |
2390 __ movptr(r14, Address(rsp, saved_r14_offset * wordSize)); |