192 _igvn.replace_node(st, st->in(MemNode::Memory)); |
192 _igvn.replace_node(st, st->in(MemNode::Memory)); |
193 } |
193 } |
194 } |
194 } |
195 |
195 |
196 // Search for a memory operation for the specified memory slice. |
196 // Search for a memory operation for the specified memory slice. |
197 static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc) { |
197 static Node *scan_mem_chain(Node *mem, int alias_idx, int offset, Node *start_mem, Node *alloc, PhaseGVN *phase) { |
198 Node *orig_mem = mem; |
198 Node *orig_mem = mem; |
199 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
199 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
|
200 const TypeOopPtr *tinst = phase->C->get_adr_type(alias_idx)->isa_oopptr(); |
200 while (true) { |
201 while (true) { |
201 if (mem == alloc_mem || mem == start_mem ) { |
202 if (mem == alloc_mem || mem == start_mem ) { |
202 return mem; // hit one of our sentinals |
203 return mem; // hit one of our sentinals |
203 } else if (mem->is_MergeMem()) { |
204 } else if (mem->is_MergeMem()) { |
204 mem = mem->as_MergeMem()->memory_at(alias_idx); |
205 mem = mem->as_MergeMem()->memory_at(alias_idx); |
206 Node *in = mem->in(0); |
207 Node *in = mem->in(0); |
207 // we can safely skip over safepoints, calls, locks and membars because we |
208 // we can safely skip over safepoints, calls, locks and membars because we |
208 // already know that the object is safe to eliminate. |
209 // already know that the object is safe to eliminate. |
209 if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) { |
210 if (in->is_Initialize() && in->as_Initialize()->allocation() == alloc) { |
210 return in; |
211 return in; |
211 } else if (in->is_Call() || in->is_MemBar()) { |
212 } else if (in->is_Call()) { |
|
213 CallNode *call = in->as_Call(); |
|
214 if (!call->may_modify(tinst, phase)) { |
|
215 mem = call->in(TypeFunc::Memory); |
|
216 } |
|
217 mem = in->in(TypeFunc::Memory); |
|
218 } else if (in->is_MemBar()) { |
212 mem = in->in(TypeFunc::Memory); |
219 mem = in->in(TypeFunc::Memory); |
213 } else { |
220 } else { |
214 assert(false, "unexpected projection"); |
221 assert(false, "unexpected projection"); |
215 } |
222 } |
216 } else if (mem->is_Store()) { |
223 } else if (mem->is_Store()) { |
229 } |
236 } |
230 mem = mem->in(MemNode::Memory); |
237 mem = mem->in(MemNode::Memory); |
231 } else { |
238 } else { |
232 return mem; |
239 return mem; |
233 } |
240 } |
234 if (mem == orig_mem) |
241 assert(mem != orig_mem, "dead memory loop"); |
235 return mem; |
|
236 } |
242 } |
237 } |
243 } |
238 |
244 |
239 // |
245 // |
240 // Given a Memory Phi, compute a value Phi containing the values from stores |
246 // Given a Memory Phi, compute a value Phi containing the values from stores |
241 // on the input paths. |
247 // on the input paths. |
242 // Note: this function is recursive, its depth is limied by the "level" argument |
248 // Note: this function is recursive, its depth is limied by the "level" argument |
243 // Returns the computed Phi, or NULL if it cannot compute it. |
249 // Returns the computed Phi, or NULL if it cannot compute it. |
244 Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, Node *alloc, int level) { |
250 Node *PhaseMacroExpand::value_from_mem_phi(Node *mem, BasicType ft, const Type *phi_type, const TypeOopPtr *adr_t, Node *alloc, Node_Stack *value_phis, int level) { |
245 |
251 assert(mem->is_Phi(), "sanity"); |
246 if (level <= 0) { |
|
247 return NULL; |
|
248 } |
|
249 int alias_idx = C->get_alias_index(adr_t); |
252 int alias_idx = C->get_alias_index(adr_t); |
250 int offset = adr_t->offset(); |
253 int offset = adr_t->offset(); |
251 int instance_id = adr_t->instance_id(); |
254 int instance_id = adr_t->instance_id(); |
252 |
255 |
|
256 // Check if an appropriate value phi already exists. |
|
257 Node* region = mem->in(0); |
|
258 for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) { |
|
259 Node* phi = region->fast_out(k); |
|
260 if (phi->is_Phi() && phi != mem && |
|
261 phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) { |
|
262 return phi; |
|
263 } |
|
264 } |
|
265 // Check if an appropriate new value phi already exists. |
|
266 Node* new_phi = NULL; |
|
267 uint size = value_phis->size(); |
|
268 for (uint i=0; i < size; i++) { |
|
269 if ( mem->_idx == value_phis->index_at(i) ) { |
|
270 return value_phis->node_at(i); |
|
271 } |
|
272 } |
|
273 |
|
274 if (level <= 0) { |
|
275 return NULL; // Give up: phi tree too deep |
|
276 } |
253 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); |
277 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); |
254 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
278 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
255 |
279 |
256 uint length = mem->req(); |
280 uint length = mem->req(); |
257 GrowableArray <Node *> values(length, length, NULL); |
281 GrowableArray <Node *> values(length, length, NULL); |
|
282 |
|
283 // create a new Phi for the value |
|
284 PhiNode *phi = new (C, length) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset); |
|
285 transform_later(phi); |
|
286 value_phis->push(phi, mem->_idx); |
258 |
287 |
259 for (uint j = 1; j < length; j++) { |
288 for (uint j = 1; j < length; j++) { |
260 Node *in = mem->in(j); |
289 Node *in = mem->in(j); |
261 if (in == NULL || in->is_top()) { |
290 if (in == NULL || in->is_top()) { |
262 values.at_put(j, in); |
291 values.at_put(j, in); |
263 } else { |
292 } else { |
264 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc); |
293 Node *val = scan_mem_chain(in, alias_idx, offset, start_mem, alloc, &_igvn); |
265 if (val == start_mem || val == alloc_mem) { |
294 if (val == start_mem || val == alloc_mem) { |
266 // hit a sentinel, return appropriate 0 value |
295 // hit a sentinel, return appropriate 0 value |
267 values.at_put(j, _igvn.zerocon(ft)); |
296 values.at_put(j, _igvn.zerocon(ft)); |
268 continue; |
297 continue; |
269 } |
298 } |
278 } else if (val->is_Store()) { |
307 } else if (val->is_Store()) { |
279 values.at_put(j, val->in(MemNode::ValueIn)); |
308 values.at_put(j, val->in(MemNode::ValueIn)); |
280 } else if(val->is_Proj() && val->in(0) == alloc) { |
309 } else if(val->is_Proj() && val->in(0) == alloc) { |
281 values.at_put(j, _igvn.zerocon(ft)); |
310 values.at_put(j, _igvn.zerocon(ft)); |
282 } else if (val->is_Phi()) { |
311 } else if (val->is_Phi()) { |
283 // Check if an appropriate node already exists. |
312 val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, value_phis, level-1); |
284 Node* region = val->in(0); |
313 if (val == NULL) { |
285 Node* old_phi = NULL; |
314 return NULL; |
286 for (DUIterator_Fast kmax, k = region->fast_outs(kmax); k < kmax; k++) { |
315 } |
287 Node* phi = region->fast_out(k); |
316 values.at_put(j, val); |
288 if (phi->is_Phi() && phi != val && |
|
289 phi->as_Phi()->is_same_inst_field(phi_type, instance_id, alias_idx, offset)) { |
|
290 old_phi = phi; |
|
291 break; |
|
292 } |
|
293 } |
|
294 if (old_phi == NULL) { |
|
295 val = value_from_mem_phi(val, ft, phi_type, adr_t, alloc, level-1); |
|
296 if (val == NULL) { |
|
297 return NULL; |
|
298 } |
|
299 values.at_put(j, val); |
|
300 } else { |
|
301 values.at_put(j, old_phi); |
|
302 } |
|
303 } else { |
317 } else { |
304 return NULL; // unknown node on this path |
318 assert(false, "unknown node on this path"); |
305 } |
319 return NULL; // unknown node on this path |
306 } |
320 } |
307 } |
321 } |
308 // create a new Phi for the value |
322 } |
309 PhiNode *phi = new (C, length) PhiNode(mem->in(0), phi_type, NULL, instance_id, alias_idx, offset); |
323 // Set Phi's inputs |
310 for (uint j = 1; j < length; j++) { |
324 for (uint j = 1; j < length; j++) { |
311 if (values.at(j) == mem) { |
325 if (values.at(j) == mem) { |
312 phi->init_req(j, phi); |
326 phi->init_req(j, phi); |
313 } else { |
327 } else { |
314 phi->init_req(j, values.at(j)); |
328 phi->init_req(j, values.at(j)); |
315 } |
329 } |
316 } |
330 } |
317 transform_later(phi); |
|
318 return phi; |
331 return phi; |
319 } |
332 } |
320 |
333 |
321 // Search the last value stored into the object's field. |
334 // Search the last value stored into the object's field. |
322 Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc) { |
335 Node *PhaseMacroExpand::value_from_mem(Node *sfpt_mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, Node *alloc) { |
327 int alias_idx = C->get_alias_index(adr_t); |
340 int alias_idx = C->get_alias_index(adr_t); |
328 int offset = adr_t->offset(); |
341 int offset = adr_t->offset(); |
329 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); |
342 Node *start_mem = C->start()->proj_out(TypeFunc::Memory); |
330 Node *alloc_ctrl = alloc->in(TypeFunc::Control); |
343 Node *alloc_ctrl = alloc->in(TypeFunc::Control); |
331 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
344 Node *alloc_mem = alloc->in(TypeFunc::Memory); |
332 VectorSet visited(Thread::current()->resource_area()); |
345 Arena *a = Thread::current()->resource_area(); |
|
346 VectorSet visited(a); |
333 |
347 |
334 |
348 |
335 bool done = sfpt_mem == alloc_mem; |
349 bool done = sfpt_mem == alloc_mem; |
336 Node *mem = sfpt_mem; |
350 Node *mem = sfpt_mem; |
337 while (!done) { |
351 while (!done) { |
338 if (visited.test_set(mem->_idx)) { |
352 if (visited.test_set(mem->_idx)) { |
339 return NULL; // found a loop, give up |
353 return NULL; // found a loop, give up |
340 } |
354 } |
341 mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc); |
355 mem = scan_mem_chain(mem, alias_idx, offset, start_mem, alloc, &_igvn); |
342 if (mem == start_mem || mem == alloc_mem) { |
356 if (mem == start_mem || mem == alloc_mem) { |
343 done = true; // hit a sentinel, return appropriate 0 value |
357 done = true; // hit a sentinel, return appropriate 0 value |
344 } else if (mem->is_Initialize()) { |
358 } else if (mem->is_Initialize()) { |
345 mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); |
359 mem = mem->as_Initialize()->find_captured_store(offset, type2aelembytes(ft), &_igvn); |
346 if (mem == NULL) { |
360 if (mem == NULL) { |
360 } else if (mem->is_Phi()) { |
374 } else if (mem->is_Phi()) { |
361 // try to find a phi's unique input |
375 // try to find a phi's unique input |
362 Node *unique_input = NULL; |
376 Node *unique_input = NULL; |
363 Node *top = C->top(); |
377 Node *top = C->top(); |
364 for (uint i = 1; i < mem->req(); i++) { |
378 for (uint i = 1; i < mem->req(); i++) { |
365 Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc); |
379 Node *n = scan_mem_chain(mem->in(i), alias_idx, offset, start_mem, alloc, &_igvn); |
366 if (n == NULL || n == top || n == mem) { |
380 if (n == NULL || n == top || n == mem) { |
367 continue; |
381 continue; |
368 } else if (unique_input == NULL) { |
382 } else if (unique_input == NULL) { |
369 unique_input = n; |
383 unique_input = n; |
370 } else if (unique_input != n) { |
384 } else if (unique_input != n) { |
387 return _igvn.zerocon(ft); |
401 return _igvn.zerocon(ft); |
388 } else if (mem->is_Store()) { |
402 } else if (mem->is_Store()) { |
389 return mem->in(MemNode::ValueIn); |
403 return mem->in(MemNode::ValueIn); |
390 } else if (mem->is_Phi()) { |
404 } else if (mem->is_Phi()) { |
391 // attempt to produce a Phi reflecting the values on the input paths of the Phi |
405 // attempt to produce a Phi reflecting the values on the input paths of the Phi |
392 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, 8); |
406 Node_Stack value_phis(a, 8); |
|
407 Node * phi = value_from_mem_phi(mem, ft, ftype, adr_t, alloc, &value_phis, ValueSearchLimit); |
393 if (phi != NULL) { |
408 if (phi != NULL) { |
394 return phi; |
409 return phi; |
|
410 } else { |
|
411 // Kill all new Phis |
|
412 while(value_phis.is_nonempty()) { |
|
413 Node* n = value_phis.node(); |
|
414 _igvn.hash_delete(n); |
|
415 _igvn.subsume_node(n, C->top()); |
|
416 value_phis.pop(); |
|
417 } |
395 } |
418 } |
396 } |
419 } |
397 } |
420 } |
398 // Something go wrong. |
421 // Something go wrong. |
399 return NULL; |
422 return NULL; |
446 for (DUIterator_Fast kmax, k = use->fast_outs(kmax); |
469 for (DUIterator_Fast kmax, k = use->fast_outs(kmax); |
447 k < kmax && can_eliminate; k++) { |
470 k < kmax && can_eliminate; k++) { |
448 Node* n = use->fast_out(k); |
471 Node* n = use->fast_out(k); |
449 if (!n->is_Store() && n->Opcode() != Op_CastP2X) { |
472 if (!n->is_Store() && n->Opcode() != Op_CastP2X) { |
450 DEBUG_ONLY(disq_node = n;) |
473 DEBUG_ONLY(disq_node = n;) |
451 if (n->is_Load()) { |
474 if (n->is_Load() || n->is_LoadStore()) { |
452 NOT_PRODUCT(fail_eliminate = "Field load";) |
475 NOT_PRODUCT(fail_eliminate = "Field load";) |
453 } else { |
476 } else { |
454 NOT_PRODUCT(fail_eliminate = "Not store field referrence";) |
477 NOT_PRODUCT(fail_eliminate = "Not store field referrence";) |
455 } |
478 } |
456 can_eliminate = false; |
479 can_eliminate = false; |