1.1 --- a/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Sat Apr 02 10:54:15 2011 -0700
1.2 +++ b/src/cpu/x86/vm/c1_LIRAssembler_x86.cpp Sun Apr 03 12:00:54 2011 +0200
1.3 @@ -3102,7 +3102,7 @@
1.4 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1.5 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1.6
1.7 - // if we don't know anything or it's an object array, just go through the generic arraycopy
1.8 + // if we don't know anything, just go through the generic arraycopy
1.9 if (default_type == NULL) {
1.10 Label done;
1.11 // save outgoing arguments on stack in case call to System.arraycopy is needed
1.12 @@ -3123,7 +3123,9 @@
1.13 store_parameter(src, 4);
1.14 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
1.15
1.16 - address entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
1.17 + address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
1.18 +
1.19 + address copyfunc_addr = StubRoutines::generic_arraycopy();
1.20
1.21 // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
1.22 #ifdef _LP64
1.23 @@ -3141,11 +3143,29 @@
1.24 // Allocate abi space for args but be sure to keep stack aligned
1.25 __ subptr(rsp, 6*wordSize);
1.26 store_parameter(j_rarg4, 4);
1.27 - __ call(RuntimeAddress(entry));
1.28 + if (copyfunc_addr == NULL) { // Use C version if stub was not generated
1.29 + __ call(RuntimeAddress(C_entry));
1.30 + } else {
1.31 +#ifndef PRODUCT
1.32 + if (PrintC1Statistics) {
1.33 + __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
1.34 + }
1.35 +#endif
1.36 + __ call(RuntimeAddress(copyfunc_addr));
1.37 + }
1.38 __ addptr(rsp, 6*wordSize);
1.39 #else
1.40 __ mov(c_rarg4, j_rarg4);
1.41 - __ call(RuntimeAddress(entry));
1.42 + if (copyfunc_addr == NULL) { // Use C version if stub was not generated
1.43 + __ call(RuntimeAddress(C_entry));
1.44 + } else {
1.45 +#ifndef PRODUCT
1.46 + if (PrintC1Statistics) {
1.47 + __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
1.48 + }
1.49 +#endif
1.50 + __ call(RuntimeAddress(copyfunc_addr));
1.51 + }
1.52 #endif // _WIN64
1.53 #else
1.54 __ push(length);
1.55 @@ -3153,13 +3173,28 @@
1.56 __ push(dst);
1.57 __ push(src_pos);
1.58 __ push(src);
1.59 - __ call_VM_leaf(entry, 5); // removes pushed parameter from the stack
1.60 +
1.61 + if (copyfunc_addr == NULL) { // Use C version if stub was not generated
1.62 + __ call_VM_leaf(C_entry, 5); // removes pushed parameter from the stack
1.63 + } else {
1.64 +#ifndef PRODUCT
1.65 + if (PrintC1Statistics) {
1.66 + __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
1.67 + }
1.68 +#endif
1.69 + __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
1.70 + }
1.71
1.72 #endif // _LP64
1.73
1.74 __ cmpl(rax, 0);
1.75 __ jcc(Assembler::equal, *stub->continuation());
1.76
1.77 + if (copyfunc_addr != NULL) {
1.78 + __ mov(tmp, rax);
1.79 + __ xorl(tmp, -1);
1.80 + }
1.81 +
1.82 // Reload values from the stack so they are where the stub
1.83 // expects them.
1.84 __ movptr (dst, Address(rsp, 0*BytesPerWord));
1.85 @@ -3167,6 +3202,12 @@
1.86 __ movptr (length, Address(rsp, 2*BytesPerWord));
1.87 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
1.88 __ movptr (src, Address(rsp, 4*BytesPerWord));
1.89 +
1.90 + if (copyfunc_addr != NULL) {
1.91 + __ subl(length, tmp);
1.92 + __ addl(src_pos, tmp);
1.93 + __ addl(dst_pos, tmp);
1.94 + }
1.95 __ jmp(*stub->entry());
1.96
1.97 __ bind(*stub->continuation());
1.98 @@ -3226,10 +3267,6 @@
1.99 __ testl(dst_pos, dst_pos);
1.100 __ jcc(Assembler::less, *stub->entry());
1.101 }
1.102 - if (flags & LIR_OpArrayCopy::length_positive_check) {
1.103 - __ testl(length, length);
1.104 - __ jcc(Assembler::less, *stub->entry());
1.105 - }
1.106
1.107 if (flags & LIR_OpArrayCopy::src_range_check) {
1.108 __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
1.109 @@ -3242,15 +3279,190 @@
1.110 __ jcc(Assembler::above, *stub->entry());
1.111 }
1.112
1.113 + if (flags & LIR_OpArrayCopy::length_positive_check) {
1.114 + __ testl(length, length);
1.115 + __ jcc(Assembler::less, *stub->entry());
1.116 + __ jcc(Assembler::zero, *stub->continuation());
1.117 + }
1.118 +
1.119 +#ifdef _LP64
1.120 + __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
1.121 + __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
1.122 +#endif
1.123 +
1.124 if (flags & LIR_OpArrayCopy::type_check) {
1.125 - if (UseCompressedOops) {
1.126 - __ movl(tmp, src_klass_addr);
1.127 - __ cmpl(tmp, dst_klass_addr);
1.128 + // We don't know the array types are compatible
1.129 + if (basic_type != T_OBJECT) {
1.130 + // Simple test for basic type arrays
1.131 + if (UseCompressedOops) {
1.132 + __ movl(tmp, src_klass_addr);
1.133 + __ cmpl(tmp, dst_klass_addr);
1.134 + } else {
1.135 + __ movptr(tmp, src_klass_addr);
1.136 + __ cmpptr(tmp, dst_klass_addr);
1.137 + }
1.138 + __ jcc(Assembler::notEqual, *stub->entry());
1.139 } else {
1.140 - __ movptr(tmp, src_klass_addr);
1.141 - __ cmpptr(tmp, dst_klass_addr);
1.142 + // For object arrays, if src is a sub class of dst then we can
1.143 + // safely do the copy.
1.144 + Label cont, slow;
1.145 +
1.146 + __ push(src);
1.147 + __ push(dst);
1.148 +
1.149 + __ load_klass(src, src);
1.150 + __ load_klass(dst, dst);
1.151 +
1.152 + __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
1.153 +
1.154 + __ push(src);
1.155 + __ push(dst);
1.156 + __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1.157 + __ pop(dst);
1.158 + __ pop(src);
1.159 +
1.160 + __ cmpl(src, 0);
1.161 + __ jcc(Assembler::notEqual, cont);
1.162 +
1.163 + __ bind(slow);
1.164 + __ pop(dst);
1.165 + __ pop(src);
1.166 +
1.167 + address copyfunc_addr = StubRoutines::checkcast_arraycopy();
1.168 + if (copyfunc_addr != NULL) { // use stub if available
1.169 + // src is not a sub class of dst so we have to do a
1.170 + // per-element check.
1.171 +
1.172 + int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
1.173 + if ((flags & mask) != mask) {
1.174 + // Check that at least both of them object arrays.
1.175 + assert(flags & mask, "one of the two should be known to be an object array");
1.176 +
1.177 + if (!(flags & LIR_OpArrayCopy::src_objarray)) {
1.178 + __ load_klass(tmp, src);
1.179 + } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
1.180 + __ load_klass(tmp, dst);
1.181 + }
1.182 + int lh_offset = klassOopDesc::header_size() * HeapWordSize +
1.183 + Klass::layout_helper_offset_in_bytes();
1.184 + Address klass_lh_addr(tmp, lh_offset);
1.185 + jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1.186 + __ cmpl(klass_lh_addr, objArray_lh);
1.187 + __ jcc(Assembler::notEqual, *stub->entry());
1.188 + }
1.189 +
1.190 +#ifndef _LP64
1.191 + // save caller save registers
1.192 + store_parameter(rax, 2);
1.193 + store_parameter(rcx, 1);
1.194 + store_parameter(rdx, 0);
1.195 +
1.196 + __ movptr(tmp, dst_klass_addr);
1.197 + __ movptr(tmp, Address(tmp, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
1.198 + __ push(tmp);
1.199 + __ movl(tmp, Address(tmp, Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc)));
1.200 + __ push(tmp);
1.201 + __ push(length);
1.202 + __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.203 + __ push(tmp);
1.204 + __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.205 + __ push(tmp);
1.206 +
1.207 + __ call_VM_leaf(copyfunc_addr, 5);
1.208 +#else
1.209 + __ movl2ptr(length, length); //higher 32bits must be null
1.210 +
1.211 + // save caller save registers: copy them to callee save registers
1.212 + __ mov(rbx, rdx);
1.213 + __ mov(r13, r8);
1.214 + __ mov(r14, r9);
1.215 +#ifndef _WIN64
1.216 + store_parameter(rsi, 1);
1.217 + store_parameter(rcx, 0);
1.218 + // on WIN64 other incoming parameters are in rdi and rsi saved
1.219 + // across the call
1.220 +#endif
1.221 +
1.222 + __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.223 + assert_different_registers(c_rarg0, dst, dst_pos, length);
1.224 + __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.225 + assert_different_registers(c_rarg1, dst, length);
1.226 +
1.227 + __ mov(c_rarg2, length);
1.228 + assert_different_registers(c_rarg2, dst);
1.229 +
1.230 +#ifdef _WIN64
1.231 + // Allocate abi space for args but be sure to keep stack aligned
1.232 + __ subptr(rsp, 6*wordSize);
1.233 + __ load_klass(c_rarg3, dst);
1.234 + __ movptr(c_rarg3, Address(c_rarg3, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
1.235 + store_parameter(c_rarg3, 4);
1.236 + __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc)));
1.237 + __ call(RuntimeAddress(copyfunc_addr));
1.238 + __ addptr(rsp, 6*wordSize);
1.239 +#else
1.240 + __ load_klass(c_rarg4, dst);
1.241 + __ movptr(c_rarg4, Address(c_rarg4, objArrayKlass::element_klass_offset_in_bytes() + sizeof(oopDesc)));
1.242 + __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset_in_bytes() + sizeof(oopDesc)));
1.243 + __ call(RuntimeAddress(copyfunc_addr));
1.244 +#endif
1.245 +
1.246 +#endif
1.247 +
1.248 +#ifndef PRODUCT
1.249 + if (PrintC1Statistics) {
1.250 + Label failed;
1.251 + __ testl(rax, rax);
1.252 + __ jcc(Assembler::notZero, failed);
1.253 + __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
1.254 + __ bind(failed);
1.255 + }
1.256 +#endif
1.257 +
1.258 + __ testl(rax, rax);
1.259 + __ jcc(Assembler::zero, *stub->continuation());
1.260 +
1.261 +#ifndef PRODUCT
1.262 + if (PrintC1Statistics) {
1.263 + __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
1.264 + }
1.265 +#endif
1.266 +
1.267 + __ mov(tmp, rax);
1.268 +
1.269 + __ xorl(tmp, -1);
1.270 +
1.271 +#ifndef _LP64
1.272 + // restore caller save registers
1.273 + assert_different_registers(tmp, rdx, rcx, rax); // result of stub will be lost
1.274 + __ movptr(rdx, Address(rsp, 0*BytesPerWord));
1.275 + __ movptr(rcx, Address(rsp, 1*BytesPerWord));
1.276 + __ movptr(rax, Address(rsp, 2*BytesPerWord));
1.277 +#else
1.278 + // restore caller save registers
1.279 + __ mov(rdx, rbx);
1.280 + __ mov(r8, r13);
1.281 + __ mov(r9, r14);
1.282 +#ifndef _WIN64
1.283 + assert_different_registers(tmp, rdx, r8, r9, rcx, rsi); // result of stub will be lost
1.284 + __ movptr(rcx, Address(rsp, 0*BytesPerWord));
1.285 + __ movptr(rsi, Address(rsp, 1*BytesPerWord));
1.286 +#else
1.287 + assert_different_registers(tmp, rdx, r8, r9); // result of stub will be lost
1.288 +#endif
1.289 +#endif
1.290 +
1.291 + __ subl(length, tmp);
1.292 + __ addl(src_pos, tmp);
1.293 + __ addl(dst_pos, tmp);
1.294 + }
1.295 +
1.296 + __ jmp(*stub->entry());
1.297 +
1.298 + __ bind(cont);
1.299 + __ pop(dst);
1.300 + __ pop(src);
1.301 }
1.302 - __ jcc(Assembler::notEqual, *stub->entry());
1.303 }
1.304
1.305 #ifdef ASSERT
1.306 @@ -3291,16 +3503,16 @@
1.307 }
1.308 #endif
1.309
1.310 - if (shift_amount > 0 && basic_type != T_OBJECT) {
1.311 - __ shlptr(length, shift_amount);
1.312 +#ifndef PRODUCT
1.313 + if (PrintC1Statistics) {
1.314 + __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
1.315 }
1.316 +#endif
1.317
1.318 #ifdef _LP64
1.319 assert_different_registers(c_rarg0, dst, dst_pos, length);
1.320 - __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
1.321 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.322 assert_different_registers(c_rarg1, length);
1.323 - __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
1.324 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
1.325 __ mov(c_rarg2, length);
1.326
1.327 @@ -3311,11 +3523,12 @@
1.328 store_parameter(tmp, 1);
1.329 store_parameter(length, 2);
1.330 #endif // _LP64
1.331 - if (basic_type == T_OBJECT) {
1.332 - __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::oop_arraycopy), 0);
1.333 - } else {
1.334 - __ call_VM_leaf(CAST_FROM_FN_PTR(address, Runtime1::primitive_arraycopy), 0);
1.335 - }
1.336 +
1.337 + bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
1.338 + bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
1.339 + const char *name;
1.340 + address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
1.341 + __ call_VM_leaf(entry, 0);
1.342
1.343 __ bind(*stub->continuation());
1.344 }
