1.1 --- a/src/share/vm/opto/library_call.cpp Tue Mar 01 10:27:15 2011 -0800
1.2 +++ b/src/share/vm/opto/library_call.cpp Tue Mar 01 14:56:48 2011 -0800
1.3 @@ -97,7 +97,7 @@
1.4 RegionNode* region);
1.5 Node* generate_current_thread(Node* &tls_output);
1.6 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
1.7 - bool disjoint_bases, const char* &name);
1.8 + bool disjoint_bases, const char* &name, bool dest_uninitialized);
1.9 Node* load_mirror_from_klass(Node* klass);
1.10 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
1.11 int nargs,
1.12 @@ -212,26 +212,26 @@
1.13 AllocateNode* alloc,
1.14 Node* src, Node* src_offset,
1.15 Node* dest, Node* dest_offset,
1.16 - Node* dest_size);
1.17 + Node* dest_size, bool dest_uninitialized);
1.18 void generate_slow_arraycopy(const TypePtr* adr_type,
1.19 Node* src, Node* src_offset,
1.20 Node* dest, Node* dest_offset,
1.21 - Node* copy_length);
1.22 + Node* copy_length, bool dest_uninitialized);
1.23 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
1.24 Node* dest_elem_klass,
1.25 Node* src, Node* src_offset,
1.26 Node* dest, Node* dest_offset,
1.27 - Node* copy_length);
1.28 + Node* copy_length, bool dest_uninitialized);
1.29 Node* generate_generic_arraycopy(const TypePtr* adr_type,
1.30 Node* src, Node* src_offset,
1.31 Node* dest, Node* dest_offset,
1.32 - Node* copy_length);
1.33 + Node* copy_length, bool dest_uninitialized);
1.34 void generate_unchecked_arraycopy(const TypePtr* adr_type,
1.35 BasicType basic_elem_type,
1.36 bool disjoint_bases,
1.37 Node* src, Node* src_offset,
1.38 Node* dest, Node* dest_offset,
1.39 - Node* copy_length);
1.40 + Node* copy_length, bool dest_uninitialized);
1.41 bool inline_unsafe_CAS(BasicType type);
1.42 bool inline_unsafe_ordered_store(BasicType type);
1.43 bool inline_fp_conversions(vmIntrinsics::ID id);
1.44 @@ -4092,7 +4092,8 @@
1.45 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
1.46 bool disjoint_bases = true;
1.47 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
1.48 - src, NULL, dest, NULL, countx);
1.49 + src, NULL, dest, NULL, countx,
1.50 + /*dest_uninitialized*/true);
1.51
1.52 // If necessary, emit some card marks afterwards. (Non-arrays only.)
1.53 if (card_mark) {
1.54 @@ -4306,7 +4307,7 @@
1.55 // Note: The condition "disjoint" applies also for overlapping copies
1.56 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
1.57 static address
1.58 -select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) {
1.59 +select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
1.60 int selector =
1.61 (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) +
1.62 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
1.63 @@ -4315,6 +4316,10 @@
1.64 name = #xxx_arraycopy; \
1.65 return StubRoutines::xxx_arraycopy(); }
1.66
1.67 +#define RETURN_STUB_PARM(xxx_arraycopy, parm) { \
1.68 + name = #xxx_arraycopy; \
1.69 + return StubRoutines::xxx_arraycopy(parm); }
1.70 +
1.71 switch (t) {
1.72 case T_BYTE:
1.73 case T_BOOLEAN:
1.74 @@ -4351,10 +4356,10 @@
1.75 case T_ARRAY:
1.76 case T_OBJECT:
1.77 switch (selector) {
1.78 - case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy);
1.79 - case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy);
1.80 - case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy);
1.81 - case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy);
1.82 + case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
1.83 + case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
1.84 + case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
1.85 + case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
1.86 }
1.87 default:
1.88 ShouldNotReachHere();
1.89 @@ -4362,6 +4367,7 @@
1.90 }
1.91
1.92 #undef RETURN_STUB
1.93 +#undef RETURN_STUB_PARM
1.94 }
1.95
1.96 //------------------------------basictype2arraycopy----------------------------
1.97 @@ -4369,7 +4375,8 @@
1.98 Node* src_offset,
1.99 Node* dest_offset,
1.100 bool disjoint_bases,
1.101 - const char* &name) {
1.102 + const char* &name,
1.103 + bool dest_uninitialized) {
1.104 const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);;
1.105 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
1.106
1.107 @@ -4395,7 +4402,7 @@
1.108 disjoint = true;
1.109 }
1.110
1.111 - return select_arraycopy_function(t, aligned, disjoint, name);
1.112 + return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
1.113 }
1.114
1.115
1.116 @@ -4451,7 +4458,8 @@
1.117 // The component types are not the same or are not recognized. Punt.
1.118 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
1.119 generate_slow_arraycopy(TypePtr::BOTTOM,
1.120 - src, src_offset, dest, dest_offset, length);
1.121 + src, src_offset, dest, dest_offset, length,
1.122 + /*dest_uninitialized*/false);
1.123 return true;
1.124 }
1.125
1.126 @@ -4564,7 +4572,7 @@
1.127
1.128 Node* original_dest = dest;
1.129 AllocateArrayNode* alloc = NULL; // used for zeroing, if needed
1.130 - bool must_clear_dest = false;
1.131 + bool dest_uninitialized = false;
1.132
1.133 // See if this is the initialization of a newly-allocated array.
1.134 // If so, we will take responsibility here for initializing it to zero.
1.135 @@ -4587,12 +4595,14 @@
1.136 adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory
1.137 // From this point on, every exit path is responsible for
1.138 // initializing any non-copied parts of the object to zero.
1.139 - must_clear_dest = true;
1.140 + // Also, if this flag is set we make sure that arraycopy interacts properly
1.141 + // with G1, eliding pre-barriers. See CR 6627983.
1.142 + dest_uninitialized = true;
1.143 } else {
1.144 // No zeroing elimination here.
1.145 alloc = NULL;
1.146 //original_dest = dest;
1.147 - //must_clear_dest = false;
1.148 + //dest_uninitialized = false;
1.149 }
1.150
1.151 // Results are placed here:
1.152 @@ -4624,10 +4634,10 @@
1.153 Node* checked_value = NULL;
1.154
1.155 if (basic_elem_type == T_CONFLICT) {
1.156 - assert(!must_clear_dest, "");
1.157 + assert(!dest_uninitialized, "");
1.158 Node* cv = generate_generic_arraycopy(adr_type,
1.159 src, src_offset, dest, dest_offset,
1.160 - copy_length);
1.161 + copy_length, dest_uninitialized);
1.162 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
1.163 checked_control = control();
1.164 checked_i_o = i_o();
1.165 @@ -4647,7 +4657,7 @@
1.166 }
1.167
1.168 // copy_length is 0.
1.169 - if (!stopped() && must_clear_dest) {
1.170 + if (!stopped() && dest_uninitialized) {
1.171 Node* dest_length = alloc->in(AllocateNode::ALength);
1.172 if (_gvn.eqv_uncast(copy_length, dest_length)
1.173 || _gvn.find_int_con(dest_length, 1) <= 0) {
1.174 @@ -4673,7 +4683,7 @@
1.175 result_memory->init_req(zero_path, memory(adr_type));
1.176 }
1.177
1.178 - if (!stopped() && must_clear_dest) {
1.179 + if (!stopped() && dest_uninitialized) {
1.180 // We have to initialize the *uncopied* part of the array to zero.
1.181 // The copy destination is the slice dest[off..off+len]. The other slices
1.182 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
1.183 @@ -4709,7 +4719,7 @@
1.184 { PreserveJVMState pjvms(this);
1.185 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
1.186 src, src_offset, dest, dest_offset,
1.187 - dest_size);
1.188 + dest_size, dest_uninitialized);
1.189 if (didit) {
1.190 // Present the results of the block-copying fast call.
1.191 result_region->init_req(bcopy_path, control());
1.192 @@ -4785,7 +4795,7 @@
1.193 Node* cv = generate_checkcast_arraycopy(adr_type,
1.194 dest_elem_klass,
1.195 src, src_offset, dest, dest_offset,
1.196 - ConvI2X(copy_length));
1.197 + ConvI2X(copy_length), dest_uninitialized);
1.198 if (cv == NULL) cv = intcon(-1); // failure (no stub available)
1.199 checked_control = control();
1.200 checked_i_o = i_o();
1.201 @@ -4808,7 +4818,7 @@
1.202 PreserveJVMState pjvms(this);
1.203 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
1.204 src, src_offset, dest, dest_offset,
1.205 - ConvI2X(copy_length));
1.206 + ConvI2X(copy_length), dest_uninitialized);
1.207
1.208 // Present the results of the fast call.
1.209 result_region->init_req(fast_path, control());
1.210 @@ -4887,7 +4897,7 @@
1.211 set_memory(slow_mem, adr_type);
1.212 set_i_o(slow_i_o);
1.213
1.214 - if (must_clear_dest) {
1.215 + if (dest_uninitialized) {
1.216 generate_clear_array(adr_type, dest, basic_elem_type,
1.217 intcon(0), NULL,
1.218 alloc->in(AllocateNode::AllocSize));
1.219 @@ -4895,7 +4905,7 @@
1.220
1.221 generate_slow_arraycopy(adr_type,
1.222 src, src_offset, dest, dest_offset,
1.223 - copy_length);
1.224 + copy_length, /*dest_uninitialized*/false);
1.225
1.226 result_region->init_req(slow_call_path, control());
1.227 result_i_o ->init_req(slow_call_path, i_o());
1.228 @@ -5139,7 +5149,7 @@
1.229 AllocateNode* alloc,
1.230 Node* src, Node* src_offset,
1.231 Node* dest, Node* dest_offset,
1.232 - Node* dest_size) {
1.233 + Node* dest_size, bool dest_uninitialized) {
1.234 // See if there is an advantage from block transfer.
1.235 int scale = exact_log2(type2aelembytes(basic_elem_type));
1.236 if (scale >= LogBytesPerLong)
1.237 @@ -5184,7 +5194,7 @@
1.238
1.239 bool disjoint_bases = true; // since alloc != NULL
1.240 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
1.241 - sptr, NULL, dptr, NULL, countx);
1.242 + sptr, NULL, dptr, NULL, countx, dest_uninitialized);
1.243
1.244 return true;
1.245 }
1.246 @@ -5197,7 +5207,8 @@
1.247 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
1.248 Node* src, Node* src_offset,
1.249 Node* dest, Node* dest_offset,
1.250 - Node* copy_length) {
1.251 + Node* copy_length, bool dest_uninitialized) {
1.252 + assert(!dest_uninitialized, "Invariant");
1.253 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
1.254 OptoRuntime::slow_arraycopy_Type(),
1.255 OptoRuntime::slow_arraycopy_Java(),
1.256 @@ -5215,10 +5226,10 @@
1.257 Node* dest_elem_klass,
1.258 Node* src, Node* src_offset,
1.259 Node* dest, Node* dest_offset,
1.260 - Node* copy_length) {
1.261 + Node* copy_length, bool dest_uninitialized) {
1.262 if (stopped()) return NULL;
1.263
1.264 - address copyfunc_addr = StubRoutines::checkcast_arraycopy();
1.265 + address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
1.266 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
1.267 return NULL;
1.268 }
1.269 @@ -5256,9 +5267,9 @@
1.270 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
1.271 Node* src, Node* src_offset,
1.272 Node* dest, Node* dest_offset,
1.273 - Node* copy_length) {
1.274 + Node* copy_length, bool dest_uninitialized) {
1.275 + assert(!dest_uninitialized, "Invariant");
1.276 if (stopped()) return NULL;
1.277 -
1.278 address copyfunc_addr = StubRoutines::generic_arraycopy();
1.279 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
1.280 return NULL;
1.281 @@ -5279,7 +5290,7 @@
1.282 bool disjoint_bases,
1.283 Node* src, Node* src_offset,
1.284 Node* dest, Node* dest_offset,
1.285 - Node* copy_length) {
1.286 + Node* copy_length, bool dest_uninitialized) {
1.287 if (stopped()) return; // nothing to do
1.288
1.289 Node* src_start = src;
1.290 @@ -5294,7 +5305,7 @@
1.291 const char* copyfunc_name = "arraycopy";
1.292 address copyfunc_addr =
1.293 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
1.294 - disjoint_bases, copyfunc_name);
1.295 + disjoint_bases, copyfunc_name, dest_uninitialized);
1.296
1.297 // Call it. Note that the count_ix value is not scaled to a byte-size.
1.298 make_runtime_call(RC_LEAF|RC_NO_FP,
