214 } |
214 } |
215 // 'con' is set to true or false to kill the dominated test. |
215 // 'con' is set to true or false to kill the dominated test. |
216 Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO); |
216 Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO); |
217 set_ctrl(con, C->root()); // Constant gets a new use |
217 set_ctrl(con, C->root()); // Constant gets a new use |
218 // Hack the dominated test |
218 // Hack the dominated test |
219 _igvn.hash_delete(iff); |
219 _igvn.replace_input_of(iff, 1, con); |
220 iff->set_req(1, con); |
|
221 _igvn._worklist.push(iff); |
|
222 |
220 |
223 // If I dont have a reachable TRUE and FALSE path following the IfNode then |
221 // If I dont have a reachable TRUE and FALSE path following the IfNode then |
224 // I can assume this path reaches an infinite loop. In this case it's not |
222 // I can assume this path reaches an infinite loop. In this case it's not |
225 // important to optimize the data Nodes - either the whole compilation will |
223 // important to optimize the data Nodes - either the whole compilation will |
226 // be tossed or this path (and all data Nodes) will go dead. |
224 // be tossed or this path (and all data Nodes) will go dead. |
243 |
241 |
244 for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) { |
242 for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) { |
245 Node* cd = dp->fast_out(i); // Control-dependent node |
243 Node* cd = dp->fast_out(i); // Control-dependent node |
246 if (cd->depends_only_on_test()) { |
244 if (cd->depends_only_on_test()) { |
247 assert(cd->in(0) == dp, ""); |
245 assert(cd->in(0) == dp, ""); |
248 _igvn.hash_delete(cd); |
246 _igvn.replace_input_of(cd, 0, prevdom); |
249 cd->set_req(0, prevdom); |
|
250 set_early_ctrl(cd); |
247 set_early_ctrl(cd); |
251 _igvn._worklist.push(cd); |
|
252 IdealLoopTree *new_loop = get_loop(get_ctrl(cd)); |
248 IdealLoopTree *new_loop = get_loop(get_ctrl(cd)); |
253 if (old_loop != new_loop) { |
249 if (old_loop != new_loop) { |
254 if (!old_loop->_child) old_loop->_body.yank(cd); |
250 if (!old_loop->_child) old_loop->_body.yank(cd); |
255 if (!new_loop->_child) new_loop->_body.push(cd); |
251 if (!new_loop->_child) new_loop->_body.push(cd); |
256 } |
252 } |
950 // control, then the cloning of n is a pointless exercise, because |
946 // control, then the cloning of n is a pointless exercise, because |
951 // GVN will ensure that we end up where we started. |
947 // GVN will ensure that we end up where we started. |
952 if (!n->is_Load() || late_load_ctrl != n_ctrl) { |
948 if (!n->is_Load() || late_load_ctrl != n_ctrl) { |
953 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) { |
949 for (DUIterator_Last jmin, j = n->last_outs(jmin); j >= jmin; ) { |
954 Node *u = n->last_out(j); // Clone private computation per use |
950 Node *u = n->last_out(j); // Clone private computation per use |
955 _igvn.hash_delete(u); |
951 _igvn.rehash_node_delayed(u); |
956 _igvn._worklist.push(u); |
|
957 Node *x = n->clone(); // Clone computation |
952 Node *x = n->clone(); // Clone computation |
958 Node *x_ctrl = NULL; |
953 Node *x_ctrl = NULL; |
959 if( u->is_Phi() ) { |
954 if( u->is_Phi() ) { |
960 // Replace all uses of normal nodes. Replace Phi uses |
955 // Replace all uses of normal nodes. Replace Phi uses |
961 // individually, so the separate Nodes can sink down |
956 // individually, so the separate Nodes can sink down |
1345 assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" ); |
1336 assert( dd_r >= dom_depth(dom_lca(newuse,use)), "" ); |
1346 |
1337 |
1347 // The original user of 'use' uses 'r' instead. |
1338 // The original user of 'use' uses 'r' instead. |
1348 for (DUIterator_Last lmin, l = use->last_outs(lmin); l >= lmin;) { |
1339 for (DUIterator_Last lmin, l = use->last_outs(lmin); l >= lmin;) { |
1349 Node* useuse = use->last_out(l); |
1340 Node* useuse = use->last_out(l); |
1350 _igvn.hash_delete(useuse); |
1341 _igvn.rehash_node_delayed(useuse); |
1351 _igvn._worklist.push(useuse); |
|
1352 uint uses_found = 0; |
1342 uint uses_found = 0; |
1353 if( useuse->in(0) == use ) { |
1343 if( useuse->in(0) == use ) { |
1354 useuse->set_req(0, r); |
1344 useuse->set_req(0, r); |
1355 uses_found++; |
1345 uses_found++; |
1356 if( useuse->is_CFG() ) { |
1346 if( useuse->is_CFG() ) { |
1485 phi = hit; // Use existing phi |
1473 phi = hit; // Use existing phi |
1486 } |
1474 } |
1487 set_ctrl(phi, prev); |
1475 set_ctrl(phi, prev); |
1488 } |
1476 } |
1489 // Make 'use' use the Phi instead of the old loop body exit value |
1477 // Make 'use' use the Phi instead of the old loop body exit value |
1490 _igvn.hash_delete(use); |
1478 _igvn.replace_input_of(use, idx, phi); |
1491 _igvn._worklist.push(use); |
|
1492 use->set_req(idx, phi); |
|
1493 if( use->_idx >= new_counter ) { // If updating new phis |
1479 if( use->_idx >= new_counter ) { // If updating new phis |
1494 // Not needed for correctness, but prevents a weak assert |
1480 // Not needed for correctness, but prevents a weak assert |
1495 // in AddPNode from tripping (when we end up with different |
1481 // in AddPNode from tripping (when we end up with different |
1496 // base & derived Phis that will become the same after |
1482 // base & derived Phis that will become the same after |
1497 // IGVN does CSE). |
1483 // IGVN does CSE). |
1515 if( split_if_set ) { |
1501 if( split_if_set ) { |
1516 while( split_if_set->size() ) { |
1502 while( split_if_set->size() ) { |
1517 Node *iff = split_if_set->pop(); |
1503 Node *iff = split_if_set->pop(); |
1518 if( iff->in(1)->is_Phi() ) { |
1504 if( iff->in(1)->is_Phi() ) { |
1519 BoolNode *b = clone_iff( iff->in(1)->as_Phi(), loop ); |
1505 BoolNode *b = clone_iff( iff->in(1)->as_Phi(), loop ); |
1520 _igvn.hash_delete(iff); |
1506 _igvn.replace_input_of(iff, 1, b); |
1521 _igvn._worklist.push(iff); |
|
1522 iff->set_req(1, b); |
|
1523 } |
1507 } |
1524 } |
1508 } |
1525 } |
1509 } |
1526 if( split_bool_set ) { |
1510 if( split_bool_set ) { |
1527 while( split_bool_set->size() ) { |
1511 while( split_bool_set->size() ) { |
1528 Node *b = split_bool_set->pop(); |
1512 Node *b = split_bool_set->pop(); |
1529 Node *phi = b->in(1); |
1513 Node *phi = b->in(1); |
1530 assert( phi->is_Phi(), "" ); |
1514 assert( phi->is_Phi(), "" ); |
1531 CmpNode *cmp = clone_bool( (PhiNode*)phi, loop ); |
1515 CmpNode *cmp = clone_bool( (PhiNode*)phi, loop ); |
1532 _igvn.hash_delete(b); |
1516 _igvn.replace_input_of(b, 1, cmp); |
1533 _igvn._worklist.push(b); |
|
1534 b->set_req(1, cmp); |
|
1535 } |
1517 } |
1536 } |
1518 } |
1537 if( split_cex_set ) { |
1519 if( split_cex_set ) { |
1538 while( split_cex_set->size() ) { |
1520 while( split_cex_set->size() ) { |
1539 Node *b = split_cex_set->pop(); |
1521 Node *b = split_cex_set->pop(); |
1684 IfNode* iff = proj->in(0)->as_If(); |
1666 IfNode* iff = proj->in(0)->as_If(); |
1685 IdealLoopTree *loop = get_loop(proj); |
1667 IdealLoopTree *loop = get_loop(proj); |
1686 ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj(); |
1668 ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj(); |
1687 int ddepth = dom_depth(proj); |
1669 int ddepth = dom_depth(proj); |
1688 |
1670 |
1689 _igvn.hash_delete(iff); |
1671 _igvn.rehash_node_delayed(iff); |
1690 _igvn._worklist.push(iff); |
1672 _igvn.rehash_node_delayed(proj); |
1691 _igvn.hash_delete(proj); |
|
1692 _igvn._worklist.push(proj); |
|
1693 |
1673 |
1694 proj->set_req(0, NULL); // temporary disconnect |
1674 proj->set_req(0, NULL); // temporary disconnect |
1695 ProjNode* proj2 = proj_clone(proj, iff); |
1675 ProjNode* proj2 = proj_clone(proj, iff); |
1696 register_node(proj2, loop, iff, ddepth); |
1676 register_node(proj2, loop, iff, ddepth); |
1697 |
1677 |
1743 IfNode* iff = proj->in(0)->as_If(); |
1723 IfNode* iff = proj->in(0)->as_If(); |
1744 IdealLoopTree *loop = get_loop(proj); |
1724 IdealLoopTree *loop = get_loop(proj); |
1745 ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj(); |
1725 ProjNode *other_proj = iff->proj_out(!proj->is_IfTrue())->as_Proj(); |
1746 int ddepth = dom_depth(proj); |
1726 int ddepth = dom_depth(proj); |
1747 |
1727 |
1748 _igvn.hash_delete(iff); |
1728 _igvn.rehash_node_delayed(iff); |
1749 _igvn._worklist.push(iff); |
1729 _igvn.rehash_node_delayed(proj); |
1750 _igvn.hash_delete(proj); |
|
1751 _igvn._worklist.push(proj); |
|
1752 |
1730 |
1753 proj->set_req(0, NULL); // temporary disconnect |
1731 proj->set_req(0, NULL); // temporary disconnect |
1754 ProjNode* proj2 = proj_clone(proj, iff); |
1732 ProjNode* proj2 = proj_clone(proj, iff); |
1755 register_node(proj2, loop, iff, ddepth); |
1733 register_node(proj2, loop, iff, ddepth); |
1756 |
1734 |
1968 } |
1946 } |
1969 assert(j < use->req(), "must be there"); |
1947 assert(j < use->req(), "must be there"); |
1970 |
1948 |
1971 // clone "n" and insert it between the inputs of "n" and the use outside the loop |
1949 // clone "n" and insert it between the inputs of "n" and the use outside the loop |
1972 Node* n_clone = n->clone(); |
1950 Node* n_clone = n->clone(); |
1973 _igvn.hash_delete(use); |
1951 _igvn.replace_input_of(use, j, n_clone); |
1974 use->set_req(j, n_clone); |
|
1975 _igvn._worklist.push(use); |
|
1976 Node* use_c; |
1952 Node* use_c; |
1977 if (!use->is_Phi()) { |
1953 if (!use->is_Phi()) { |
1978 use_c = has_ctrl(use) ? get_ctrl(use) : use->in(0); |
1954 use_c = has_ctrl(use) ? get_ctrl(use) : use->in(0); |
1979 } else { |
1955 } else { |
1980 // Use in a phi is considered a use in the associated predecessor block |
1956 // Use in a phi is considered a use in the associated predecessor block |
2026 tty->print_cr("special not_peeled cloning old: %d new: %d", n->_idx, n_clone->_idx); |
2002 tty->print_cr("special not_peeled cloning old: %d new: %d", n->_idx, n_clone->_idx); |
2027 } |
2003 } |
2028 #endif |
2004 #endif |
2029 while( worklist.size() ) { |
2005 while( worklist.size() ) { |
2030 Node *use = worklist.pop(); |
2006 Node *use = worklist.pop(); |
2031 _igvn.hash_delete(use); |
2007 _igvn.rehash_node_delayed(use); |
2032 _igvn._worklist.push(use); |
|
2033 for (uint j = 1; j < use->req(); j++) { |
2008 for (uint j = 1; j < use->req(); j++) { |
2034 if (use->in(j) == n) { |
2009 if (use->in(j) == n) { |
2035 use->set_req(j, n_clone); |
2010 use->set_req(j, n_clone); |
2036 } |
2011 } |
2037 } |
2012 } |
2628 |
2601 |
2629 if ( loop->is_member(get_loop( use_c )) ) { |
2602 if ( loop->is_member(get_loop( use_c )) ) { |
2630 // use is in loop |
2603 // use is in loop |
2631 if (old_new[use->_idx] != NULL) { // null for dead code |
2604 if (old_new[use->_idx] != NULL) { // null for dead code |
2632 Node* use_clone = old_new[use->_idx]; |
2605 Node* use_clone = old_new[use->_idx]; |
2633 _igvn.hash_delete(use); |
2606 _igvn.replace_input_of(use, j, C->top()); |
2634 use->set_req(j, C->top()); |
|
2635 _igvn._worklist.push(use); |
|
2636 insert_phi_for_loop( use_clone, j, old_new[def->_idx], def, new_head_clone ); |
2607 insert_phi_for_loop( use_clone, j, old_new[def->_idx], def, new_head_clone ); |
2637 } |
2608 } |
2638 } else { |
2609 } else { |
2639 assert(is_valid_clone_loop_exit_use(loop, use, orig_exit_idx), "clone loop format"); |
2610 assert(is_valid_clone_loop_exit_use(loop, use, orig_exit_idx), "clone loop format"); |
2640 // use is not in the loop, check if the live range includes the cut |
2611 // use is not in the loop, check if the live range includes the cut |
2665 for(i = 0; i < loop->_body.size(); i++ ) { |
2636 for(i = 0; i < loop->_body.size(); i++ ) { |
2666 Node *n = loop->_body.at(i); |
2637 Node *n = loop->_body.at(i); |
2667 if (!n->is_CFG() && n->in(0) != NULL && |
2638 if (!n->is_CFG() && n->in(0) != NULL && |
2668 not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) { |
2639 not_peel.test(n->_idx) && peel.test(n->in(0)->_idx)) { |
2669 Node* n_clone = old_new[n->_idx]; |
2640 Node* n_clone = old_new[n->_idx]; |
2670 _igvn.hash_delete(n_clone); |
2641 _igvn.replace_input_of(n_clone, 0, new_head_clone); |
2671 n_clone->set_req(0, new_head_clone); |
|
2672 _igvn._worklist.push(n_clone); |
|
2673 } |
2642 } |
2674 } |
2643 } |
2675 |
2644 |
2676 // Backedge of the surviving new_head (the clone) is original last_peel |
2645 // Backedge of the surviving new_head (the clone) is original last_peel |
2677 _igvn.hash_delete(new_head_clone); |
2646 _igvn.replace_input_of(new_head_clone, LoopNode::LoopBackControl, last_peel); |
2678 new_head_clone->set_req(LoopNode::LoopBackControl, last_peel); |
|
2679 _igvn._worklist.push(new_head_clone); |
|
2680 |
2647 |
2681 // Cut first node in original not_peel set |
2648 // Cut first node in original not_peel set |
2682 _igvn.hash_delete(new_head); |
2649 _igvn.rehash_node_delayed(new_head); // Multiple edge updates: |
2683 new_head->set_req(LoopNode::EntryControl, C->top()); |
2650 new_head->set_req(LoopNode::EntryControl, C->top()); // use rehash_node_delayed / set_req instead of |
2684 new_head->set_req(LoopNode::LoopBackControl, C->top()); |
2651 new_head->set_req(LoopNode::LoopBackControl, C->top()); // multiple replace_input_of calls |
2685 _igvn._worklist.push(new_head); |
|
2686 |
2652 |
2687 // Copy head_clone back-branch info to original head |
2653 // Copy head_clone back-branch info to original head |
2688 // and remove original head's loop entry and |
2654 // and remove original head's loop entry and |
2689 // clone head's back-branch |
2655 // clone head's back-branch |
2690 _igvn.hash_delete(head); |
2656 _igvn.rehash_node_delayed(head); // Multiple edge updates |
2691 _igvn.hash_delete(head_clone); |
2657 head->set_req(LoopNode::EntryControl, head_clone->in(LoopNode::LoopBackControl)); |
2692 head->set_req(LoopNode::EntryControl, head_clone->in(LoopNode::LoopBackControl)); |
|
2693 head->set_req(LoopNode::LoopBackControl, C->top()); |
2658 head->set_req(LoopNode::LoopBackControl, C->top()); |
2694 head_clone->set_req(LoopNode::LoopBackControl, C->top()); |
2659 _igvn.replace_input_of(head_clone, LoopNode::LoopBackControl, C->top()); |
2695 _igvn._worklist.push(head); |
|
2696 _igvn._worklist.push(head_clone); |
|
2697 |
2660 |
2698 // Similarly modify the phis |
2661 // Similarly modify the phis |
2699 for (DUIterator_Fast kmax, k = head->fast_outs(kmax); k < kmax; k++) { |
2662 for (DUIterator_Fast kmax, k = head->fast_outs(kmax); k < kmax; k++) { |
2700 Node* use = head->fast_out(k); |
2663 Node* use = head->fast_out(k); |
2701 if (use->is_Phi() && use->outcnt() > 0) { |
2664 if (use->is_Phi() && use->outcnt() > 0) { |
2702 Node* use_clone = old_new[use->_idx]; |
2665 Node* use_clone = old_new[use->_idx]; |
2703 _igvn.hash_delete(use); |
2666 _igvn.rehash_node_delayed(use); // Multiple edge updates |
2704 _igvn.hash_delete(use_clone); |
2667 use->set_req(LoopNode::EntryControl, use_clone->in(LoopNode::LoopBackControl)); |
2705 use->set_req(LoopNode::EntryControl, use_clone->in(LoopNode::LoopBackControl)); |
|
2706 use->set_req(LoopNode::LoopBackControl, C->top()); |
2668 use->set_req(LoopNode::LoopBackControl, C->top()); |
2707 use_clone->set_req(LoopNode::LoopBackControl, C->top()); |
2669 _igvn.replace_input_of(use_clone, LoopNode::LoopBackControl, C->top()); |
2708 _igvn._worklist.push(use); |
|
2709 _igvn._worklist.push(use_clone); |
|
2710 } |
2670 } |
2711 } |
2671 } |
2712 |
2672 |
2713 // Step 4: update dominator tree and dominator depth |
2673 // Step 4: update dominator tree and dominator depth |
2714 |
2674 |
2790 register_new_node( opaq, u_ctrl ); |
2750 register_new_node( opaq, u_ctrl ); |
2791 Node *neg_stride = _igvn.intcon(-cle->stride_con()); |
2751 Node *neg_stride = _igvn.intcon(-cle->stride_con()); |
2792 set_ctrl(neg_stride, C->root()); |
2752 set_ctrl(neg_stride, C->root()); |
2793 Node *post = new (C, 3) AddINode( opaq, neg_stride); |
2753 Node *post = new (C, 3) AddINode( opaq, neg_stride); |
2794 register_new_node( post, u_ctrl ); |
2754 register_new_node( post, u_ctrl ); |
2795 _igvn.hash_delete(use); |
2755 _igvn.rehash_node_delayed(use); |
2796 _igvn._worklist.push(use); |
|
2797 for (uint j = 1; j < use->req(); j++) { |
2756 for (uint j = 1; j < use->req(); j++) { |
2798 if (use->in(j) == phi) |
2757 if (use->in(j) == phi) |
2799 use->set_req(j, post); |
2758 use->set_req(j, post); |
2800 } |
2759 } |
2801 // Since DU info changed, rerun loop |
2760 // Since DU info changed, rerun loop |