1.1 --- a/src/share/vm/opto/loopopts.cpp Mon Mar 21 02:30:49 2011 -0700 1.2 +++ b/src/share/vm/opto/loopopts.cpp Mon Mar 21 11:28:14 2011 -0700 1.3 @@ -42,13 +42,13 @@ 1.4 return NULL; 1.5 } 1.6 int wins = 0; 1.7 - assert( !n->is_CFG(), "" ); 1.8 - assert( region->is_Region(), "" ); 1.9 + assert(!n->is_CFG(), ""); 1.10 + assert(region->is_Region(), ""); 1.11 1.12 const Type* type = n->bottom_type(); 1.13 const TypeOopPtr *t_oop = _igvn.type(n)->isa_oopptr(); 1.14 Node *phi; 1.15 - if( t_oop != NULL && t_oop->is_known_instance_field() ) { 1.16 + if (t_oop != NULL && t_oop->is_known_instance_field()) { 1.17 int iid = t_oop->instance_id(); 1.18 int index = C->get_alias_index(t_oop); 1.19 int offset = t_oop->offset(); 1.20 @@ -57,20 +57,20 @@ 1.21 phi = PhiNode::make_blank(region, n); 1.22 } 1.23 uint old_unique = C->unique(); 1.24 - for( uint i = 1; i < region->req(); i++ ) { 1.25 + for (uint i = 1; i < region->req(); i++) { 1.26 Node *x; 1.27 Node* the_clone = NULL; 1.28 - if( region->in(i) == C->top() ) { 1.29 + if (region->in(i) == C->top()) { 1.30 x = C->top(); // Dead path? Use a dead data op 1.31 } else { 1.32 x = n->clone(); // Else clone up the data op 1.33 the_clone = x; // Remember for possible deletion. 1.34 // Alter data node to use pre-phi inputs 1.35 - if( n->in(0) == region ) 1.36 + if (n->in(0) == region) 1.37 x->set_req( 0, region->in(i) ); 1.38 - for( uint j = 1; j < n->req(); j++ ) { 1.39 + for (uint j = 1; j < n->req(); j++) { 1.40 Node *in = n->in(j); 1.41 - if( in->is_Phi() && in->in(0) == region ) 1.42 + if (in->is_Phi() && in->in(0) == region) 1.43 x->set_req( j, in->in(i) ); // Use pre-Phi input for the clone 1.44 } 1.45 } 1.46 @@ -85,7 +85,7 @@ 1.47 // happen if the singleton occurs on loop entry, as the elimination of 1.48 // the PhiNode may cause the resulting node to migrate back to a previous 1.49 // loop iteration. 1.50 - if( singleton && t == Type::TOP ) { 1.51 + if (singleton && t == Type::TOP) { 1.52 // Is_Loop() == false does not confirm the absence of a loop (e.g., an 1.53 // irreducible loop may not be indicated by an affirmative is_Loop()); 1.54 // therefore, the only top we can split thru a phi is on a backedge of 1.55 @@ -93,7 +93,7 @@ 1.56 singleton &= region->is_Loop() && (i != LoopNode::EntryControl); 1.57 } 1.58 1.59 - if( singleton ) { 1.60 + if (singleton) { 1.61 wins++; 1.62 x = ((PhaseGVN&)_igvn).makecon(t); 1.63 } else { 1.64 @@ -108,12 +108,12 @@ 1.65 // igvn->type(x) is set to x->Value() already. 1.66 x->raise_bottom_type(t); 1.67 Node *y = x->Identity(&_igvn); 1.68 - if( y != x ) { 1.69 + if (y != x) { 1.70 wins++; 1.71 x = y; 1.72 } else { 1.73 y = _igvn.hash_find(x); 1.74 - if( y ) { 1.75 + if (y) { 1.76 wins++; 1.77 x = y; 1.78 } else { 1.79 @@ -129,7 +129,7 @@ 1.80 phi->set_req( i, x ); 1.81 } 1.82 // Too few wins? 1.83 - if( wins <= policy ) { 1.84 + if (wins <= policy) { 1.85 _igvn.remove_dead_node(phi); 1.86 return NULL; 1.87 } 1.88 @@ -137,7 +137,7 @@ 1.89 // Record Phi 1.90 register_new_node( phi, region ); 1.91 1.92 - for( uint i2 = 1; i2 < phi->req(); i2++ ) { 1.93 + for (uint i2 = 1; i2 < phi->req(); i2++) { 1.94 Node *x = phi->in(i2); 1.95 // If we commoned up the cloned 'x' with another existing Node, 1.96 // the existing Node picks up a new use. We need to make the 1.97 @@ -145,24 +145,44 @@ 1.98 Node *old_ctrl; 1.99 IdealLoopTree *old_loop; 1.100 1.101 + if (x->is_Con()) { 1.102 + // Constant's control is always root. 1.103 + set_ctrl(x, C->root()); 1.104 + continue; 1.105 + } 1.106 // The occasional new node 1.107 - if( x->_idx >= old_unique ) { // Found a new, unplaced node? 1.108 - old_ctrl = x->is_Con() ? C->root() : NULL; 1.109 - old_loop = NULL; // Not in any prior loop 1.110 + if (x->_idx >= old_unique) { // Found a new, unplaced node? 1.111 + old_ctrl = NULL; 1.112 + old_loop = NULL; // Not in any prior loop 1.113 } else { 1.114 - old_ctrl = x->is_Con() ? C->root() : get_ctrl(x); 1.115 + old_ctrl = get_ctrl(x); 1.116 old_loop = get_loop(old_ctrl); // Get prior loop 1.117 } 1.118 // New late point must dominate new use 1.119 - Node *new_ctrl = dom_lca( old_ctrl, region->in(i2) ); 1.120 + Node *new_ctrl = dom_lca(old_ctrl, region->in(i2)); 1.121 + if (new_ctrl == old_ctrl) // Nothing is changed 1.122 + continue; 1.123 + 1.124 + IdealLoopTree *new_loop = get_loop(new_ctrl); 1.125 + 1.126 + // Don't move x into a loop if its uses are 1.127 + // outside of loop. Otherwise x will be cloned 1.128 + // for each use outside of this loop. 1.129 + IdealLoopTree *use_loop = get_loop(region); 1.130 + if (!new_loop->is_member(use_loop) && 1.131 + (old_loop == NULL || !new_loop->is_member(old_loop))) { 1.132 + // Take early control, later control will be recalculated 1.133 + // during next iteration of loop optimizations. 1.134 + new_ctrl = get_early_ctrl(x); 1.135 + new_loop = get_loop(new_ctrl); 1.136 + } 1.137 // Set new location 1.138 set_ctrl(x, new_ctrl); 1.139 - IdealLoopTree *new_loop = get_loop( new_ctrl ); 1.140 // If changing loop bodies, see if we need to collect into new body 1.141 - if( old_loop != new_loop ) { 1.142 - if( old_loop && !old_loop->_child ) 1.143 + if (old_loop != new_loop) { 1.144 + if (old_loop && !old_loop->_child) 1.145 old_loop->_body.yank(x); 1.146 - if( !new_loop->_child ) 1.147 + if (!new_loop->_child) 1.148 new_loop->_body.push(x); // Collect body info 1.149 } 1.150 } 1.151 @@ -174,9 +194,9 @@ 1.152 // Replace the dominated test with an obvious true or false. Place it on the 1.153 // IGVN worklist for later cleanup. Move control-dependent data Nodes on the 1.154 // live path up to the dominating control. 1.155 -void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff ) { 1.156 +void PhaseIdealLoop::dominated_by( Node *prevdom, Node *iff, bool flip ) { 1.157 #ifndef PRODUCT 1.158 - if( VerifyLoopOptimizations && PrintOpto ) tty->print_cr("dominating test"); 1.159 + if (VerifyLoopOptimizations && PrintOpto) tty->print_cr("dominating test"); 1.160 #endif 1.161 1.162 1.163 @@ -185,6 +205,12 @@ 1.164 assert( iff->Opcode() == Op_If || iff->Opcode() == Op_CountedLoopEnd, "Check this code when new subtype is added"); 1.165 int pop = prevdom->Opcode(); 1.166 assert( pop == Op_IfFalse || pop == Op_IfTrue, "" ); 1.167 + if (flip) { 1.168 + if (pop == Op_IfTrue) 1.169 + pop = Op_IfFalse; 1.170 + else 1.171 + pop = Op_IfTrue; 1.172 + } 1.173 // 'con' is set to true or false to kill the dominated test. 1.174 Node *con = _igvn.makecon(pop == Op_IfTrue ? TypeInt::ONE : TypeInt::ZERO); 1.175 set_ctrl(con, C->root()); // Constant gets a new use 1.176 @@ -197,7 +223,7 @@ 1.177 // I can assume this path reaches an infinite loop. In this case it's not 1.178 // important to optimize the data Nodes - either the whole compilation will 1.179 // be tossed or this path (and all data Nodes) will go dead. 1.180 - if( iff->outcnt() != 2 ) return; 1.181 + if (iff->outcnt() != 2) return; 1.182 1.183 // Make control-dependent data Nodes on the live path (path that will remain 1.184 // once the dominated IF is removed) become control-dependent on the 1.185 @@ -207,16 +233,16 @@ 1.186 1.187 for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) { 1.188 Node* cd = dp->fast_out(i); // Control-dependent node 1.189 - if( cd->depends_only_on_test() ) { 1.190 - assert( cd->in(0) == dp, "" ); 1.191 - _igvn.hash_delete( cd ); 1.192 + if (cd->depends_only_on_test()) { 1.193 + assert(cd->in(0) == dp, ""); 1.194 + _igvn.hash_delete(cd); 1.195 cd->set_req(0, prevdom); 1.196 - set_early_ctrl( cd ); 1.197 + set_early_ctrl(cd); 1.198 _igvn._worklist.push(cd); 1.199 IdealLoopTree *new_loop = get_loop(get_ctrl(cd)); 1.200 - if( old_loop != new_loop ) { 1.201 - if( !old_loop->_child ) old_loop->_body.yank(cd); 1.202 - if( !new_loop->_child ) new_loop->_body.push(cd); 1.203 + if (old_loop != new_loop) { 1.204 + if (!old_loop->_child) old_loop->_body.yank(cd); 1.205 + if (!new_loop->_child) new_loop->_body.push(cd); 1.206 } 1.207 --i; 1.208 --imax; 1.209 @@ -2338,6 +2364,11 @@ 1.210 } 1.211 1.212 #if !defined(PRODUCT) 1.213 + if (TraceLoopOpts) { 1.214 + tty->print("PartialPeel "); 1.215 + loop->dump_head(); 1.216 + } 1.217 + 1.218 if (TracePartialPeeling) { 1.219 tty->print_cr("before partial peel one iteration"); 1.220 Node_List wl; 1.221 @@ -2481,6 +2512,7 @@ 1.222 // Create new loop head for new phis and to hang 1.223 // the nodes being moved (sinked) from the peel region. 1.224 LoopNode* new_head = new (C, 3) LoopNode(last_peel, last_peel); 1.225 + new_head->set_unswitch_count(head->unswitch_count()); // Preserve 1.226 _igvn.register_new_node_with_optimizer(new_head); 1.227 assert(first_not_peeled->in(0) == last_peel, "last_peel <- first_not_peeled"); 1.228 first_not_peeled->set_req(0, new_head); 1.229 @@ -2651,24 +2683,23 @@ 1.230 // prevent loop-fallout uses of the pre-incremented trip counter (which are 1.231 // then alive with the post-incremented trip counter forcing an extra 1.232 // register move) 1.233 -void PhaseIdealLoop::reorg_offsets( IdealLoopTree *loop ) { 1.234 +void PhaseIdealLoop::reorg_offsets(IdealLoopTree *loop) { 1.235 + // Perform it only for canonical counted loops. 1.236 + // Loop's shape could be messed up by iteration_split_impl. 1.237 + if (!loop->_head->is_CountedLoop()) 1.238 + return; 1.239 + if (!loop->_head->as_Loop()->is_valid_counted_loop()) 1.240 + return; 1.241 1.242 CountedLoopNode *cl = loop->_head->as_CountedLoop(); 1.243 CountedLoopEndNode *cle = cl->loopexit(); 1.244 - if( !cle ) return; // The occasional dead loop 1.245 - // Find loop exit control 1.246 Node *exit = cle->proj_out(false); 1.247 - assert( exit->Opcode() == Op_IfFalse, "" ); 1.248 + Node *phi = cl->phi(); 1.249 1.250 // Check for the special case of folks using the pre-incremented 1.251 // trip-counter on the fall-out path (forces the pre-incremented 1.252 // and post-incremented trip counter to be live at the same time). 1.253 // Fix this by adjusting to use the post-increment trip counter. 1.254 - Node *phi = cl->phi(); 1.255 - if( !phi ) return; // Dead infinite loop 1.256 - 1.257 - // Shape messed up, probably by iteration_split_impl 1.258 - if (phi->in(LoopNode::LoopBackControl) != cl->incr()) return; 1.259 1.260 bool progress = true; 1.261 while (progress) { 1.262 @@ -2677,21 +2708,19 @@ 1.263 Node* use = phi->fast_out(i); // User of trip-counter 1.264 if (!has_ctrl(use)) continue; 1.265 Node *u_ctrl = get_ctrl(use); 1.266 - if( use->is_Phi() ) { 1.267 + if (use->is_Phi()) { 1.268 u_ctrl = NULL; 1.269 - for( uint j = 1; j < use->req(); j++ ) 1.270 - if( use->in(j) == phi ) 1.271 - u_ctrl = dom_lca( u_ctrl, use->in(0)->in(j) ); 1.272 + for (uint j = 1; j < use->req(); j++) 1.273 + if (use->in(j) == phi) 1.274 + u_ctrl = dom_lca(u_ctrl, use->in(0)->in(j)); 1.275 } 1.276 IdealLoopTree *u_loop = get_loop(u_ctrl); 1.277 // Look for loop-invariant use 1.278 - if( u_loop == loop ) continue; 1.279 - if( loop->is_member( u_loop ) ) continue; 1.280 + if (u_loop == loop) continue; 1.281 + if (loop->is_member(u_loop)) continue; 1.282 // Check that use is live out the bottom. Assuming the trip-counter 1.283 // update is right at the bottom, uses of of the loop middle are ok. 1.284 - if( dom_lca( exit, u_ctrl ) != exit ) continue; 1.285 - // protect against stride not being a constant 1.286 - if( !cle->stride_is_con() ) continue; 1.287 + if (dom_lca(exit, u_ctrl) != exit) continue; 1.288 // Hit! Refactor use to use the post-incremented tripcounter. 1.289 // Compute a post-increment tripcounter. 1.290 Node *opaq = new (C, 2) Opaque2Node( C, cle->incr() ); 1.291 @@ -2702,9 +2731,10 @@ 1.292 register_new_node( post, u_ctrl ); 1.293 _igvn.hash_delete(use); 1.294 _igvn._worklist.push(use); 1.295 - for( uint j = 1; j < use->req(); j++ ) 1.296 - if( use->in(j) == phi ) 1.297 + for (uint j = 1; j < use->req(); j++) { 1.298 + if (use->in(j) == phi) 1.299 use->set_req(j, post); 1.300 + } 1.301 // Since DU info changed, rerun loop 1.302 progress = true; 1.303 break;