1.1 --- a/src/share/vm/opto/loopTransform.cpp Mon Mar 21 02:30:49 2011 -0700
1.2 +++ b/src/share/vm/opto/loopTransform.cpp Mon Mar 21 11:28:14 2011 -0700
1.3 @@ -205,6 +205,8 @@
1.4 }
1.5 phase->register_new_node(addx, phase->get_ctrl(x));
1.6 phase->_igvn.replace_node(n1, addx);
1.7 + assert(phase->get_loop(phase->get_ctrl(n1)) == this, "");
1.8 + _body.yank(n1);
1.9 return addx;
1.10 }
1.11
1.12 @@ -307,15 +309,21 @@
1.13 // iterations adjusted. Therefore, we need to declare this loop as
1.14 // no longer a 'main' loop; it will need new pre and post loops before
1.15 // we can do further RCE.
1.16 +#ifndef PRODUCT
1.17 + if (TraceLoopOpts) {
1.18 + tty->print("Peel ");
1.19 + loop->dump_head();
1.20 + }
1.21 +#endif
1.22 Node *h = loop->_head;
1.23 - if( h->is_CountedLoop() ) {
1.24 + if (h->is_CountedLoop()) {
1.25 CountedLoopNode *cl = h->as_CountedLoop();
1.26 assert(cl->trip_count() > 0, "peeling a fully unrolled loop");
1.27 cl->set_trip_count(cl->trip_count() - 1);
1.28 - if( cl->is_main_loop() ) {
1.29 + if (cl->is_main_loop()) {
1.30 cl->set_normal_loop();
1.31 #ifndef PRODUCT
1.32 - if( PrintOpto && VerifyLoopOptimizations ) {
1.33 + if (PrintOpto && VerifyLoopOptimizations) {
1.34 tty->print("Peeling a 'main' loop; resetting to 'normal' ");
1.35 loop->dump_head();
1.36 }
1.37 @@ -645,6 +653,15 @@
1.38 // alignment. Useful to unroll loops that do no array accesses.
1.39 void PhaseIdealLoop::insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ) {
1.40
1.41 +#ifndef PRODUCT
1.42 + if (TraceLoopOpts) {
1.43 + if (peel_only)
1.44 + tty->print("PeelMainPost ");
1.45 + else
1.46 + tty->print("PreMainPost ");
1.47 + loop->dump_head();
1.48 + }
1.49 +#endif
1.50 C->set_major_progress();
1.51
1.52 // Find common pieces of the loop being guarded with pre & post loops
1.53 @@ -897,16 +914,19 @@
1.54 //------------------------------do_unroll--------------------------------------
1.55 // Unroll the loop body one step - make each trip do 2 iterations.
1.56 void PhaseIdealLoop::do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ) {
1.57 - assert( LoopUnrollLimit, "" );
1.58 + assert(LoopUnrollLimit, "");
1.59 + CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
1.60 + CountedLoopEndNode *loop_end = loop_head->loopexit();
1.61 + assert(loop_end, "");
1.62 #ifndef PRODUCT
1.63 - if( PrintOpto && VerifyLoopOptimizations ) {
1.64 + if (PrintOpto && VerifyLoopOptimizations) {
1.65 tty->print("Unrolling ");
1.66 loop->dump_head();
1.67 + } else if (TraceLoopOpts) {
1.68 + tty->print("Unroll %d ", loop_head->unrolled_count()*2);
1.69 + loop->dump_head();
1.70 }
1.71 #endif
1.72 - CountedLoopNode *loop_head = loop->_head->as_CountedLoop();
1.73 - CountedLoopEndNode *loop_end = loop_head->loopexit();
1.74 - assert( loop_end, "" );
1.75
1.76 // Remember loop node count before unrolling to detect
1.77 // if rounds of unroll,optimize are making progress
1.78 @@ -915,7 +935,7 @@
1.79 Node *ctrl = loop_head->in(LoopNode::EntryControl);
1.80 Node *limit = loop_head->limit();
1.81 Node *init = loop_head->init_trip();
1.82 - Node *strid = loop_head->stride();
1.83 + Node *stride = loop_head->stride();
1.84
1.85 Node *opaq = NULL;
1.86 if( adjust_min_trip ) { // If not maximally unrolling, need adjustment
1.87 @@ -955,13 +975,13 @@
1.88 // odd iteration: (trip_cnt & ~1). Then back compute a new limit.
1.89 Node *span = new (C, 3) SubINode( limit, init );
1.90 register_new_node( span, ctrl );
1.91 - Node *trip = new (C, 3) DivINode( 0, span, strid );
1.92 + Node *trip = new (C, 3) DivINode( 0, span, stride );
1.93 register_new_node( trip, ctrl );
1.94 Node *mtwo = _igvn.intcon(-2);
1.95 set_ctrl(mtwo, C->root());
1.96 Node *rond = new (C, 3) AndINode( trip, mtwo );
1.97 register_new_node( rond, ctrl );
1.98 - Node *spn2 = new (C, 3) MulINode( rond, strid );
1.99 + Node *spn2 = new (C, 3) MulINode( rond, stride );
1.100 register_new_node( spn2, ctrl );
1.101 Node *lim2 = new (C, 3) AddINode( spn2, init );
1.102 register_new_node( lim2, ctrl );
1.103 @@ -1040,17 +1060,23 @@
1.104
1.105 void PhaseIdealLoop::do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ) {
1.106 CountedLoopNode *cl = loop->_head->as_CountedLoop();
1.107 - assert( cl->trip_count() > 0, "");
1.108 + assert(cl->trip_count() > 0, "");
1.109 +#ifndef PRODUCT
1.110 + if (TraceLoopOpts) {
1.111 + tty->print("MaxUnroll %d ", cl->trip_count());
1.112 + loop->dump_head();
1.113 + }
1.114 +#endif
1.115
1.116 // If loop is tripping an odd number of times, peel odd iteration
1.117 - if( (cl->trip_count() & 1) == 1 ) {
1.118 - do_peeling( loop, old_new );
1.119 + if ((cl->trip_count() & 1) == 1) {
1.120 + do_peeling(loop, old_new);
1.121 }
1.122
1.123 // Now its tripping an even number of times remaining. Double loop body.
1.124 // Do not adjust pre-guards; they are not needed and do not exist.
1.125 - if( cl->trip_count() > 0 ) {
1.126 - do_unroll( loop, old_new, false );
1.127 + if (cl->trip_count() > 0) {
1.128 + do_unroll(loop, old_new, false);
1.129 }
1.130 }
1.131
1.132 @@ -1227,35 +1253,55 @@
1.133 // Eliminate range-checks and other trip-counter vs loop-invariant tests.
1.134 void PhaseIdealLoop::do_range_check( IdealLoopTree *loop, Node_List &old_new ) {
1.135 #ifndef PRODUCT
1.136 - if( PrintOpto && VerifyLoopOptimizations ) {
1.137 + if (PrintOpto && VerifyLoopOptimizations) {
1.138 tty->print("Range Check Elimination ");
1.139 loop->dump_head();
1.140 + } else if (TraceLoopOpts) {
1.141 + tty->print("RangeCheck ");
1.142 + loop->dump_head();
1.143 }
1.144 #endif
1.145 - assert( RangeCheckElimination, "" );
1.146 + assert(RangeCheckElimination, "");
1.147 CountedLoopNode *cl = loop->_head->as_CountedLoop();
1.148 - assert( cl->is_main_loop(), "" );
1.149 + assert(cl->is_main_loop(), "");
1.150 +
1.151 + // protect against stride not being a constant
1.152 + if (!cl->stride_is_con())
1.153 + return;
1.154
1.155 // Find the trip counter; we are iteration splitting based on it
1.156 Node *trip_counter = cl->phi();
1.157 // Find the main loop limit; we will trim it's iterations
1.158 // to not ever trip end tests
1.159 Node *main_limit = cl->limit();
1.160 +
1.161 + // Need to find the main-loop zero-trip guard
1.162 + Node *ctrl = cl->in(LoopNode::EntryControl);
1.163 + assert(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "");
1.164 + Node *iffm = ctrl->in(0);
1.165 + assert(iffm->Opcode() == Op_If, "");
1.166 + Node *bolzm = iffm->in(1);
1.167 + assert(bolzm->Opcode() == Op_Bool, "");
1.168 + Node *cmpzm = bolzm->in(1);
1.169 + assert(cmpzm->is_Cmp(), "");
1.170 + Node *opqzm = cmpzm->in(2);
1.171 + // Can not optimize a loop if pre-loop Opaque1 node is optimized
1.172 + // away and then another round of loop opts attempted.
1.173 + if (opqzm->Opcode() != Op_Opaque1)
1.174 + return;
1.175 + assert(opqzm->in(1) == main_limit, "do not understand situation");
1.176 +
1.177 // Find the pre-loop limit; we will expand it's iterations to
1.178 // not ever trip low tests.
1.179 - Node *ctrl = cl->in(LoopNode::EntryControl);
1.180 - assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" );
1.181 - Node *iffm = ctrl->in(0);
1.182 - assert( iffm->Opcode() == Op_If, "" );
1.183 Node *p_f = iffm->in(0);
1.184 - assert( p_f->Opcode() == Op_IfFalse, "" );
1.185 + assert(p_f->Opcode() == Op_IfFalse, "");
1.186 CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd();
1.187 - assert( pre_end->loopnode()->is_pre_loop(), "" );
1.188 + assert(pre_end->loopnode()->is_pre_loop(), "");
1.189 Node *pre_opaq1 = pre_end->limit();
1.190 // Occasionally it's possible for a pre-loop Opaque1 node to be
1.191 // optimized away and then another round of loop opts attempted.
1.192 // We can not optimize this particular loop in that case.
1.193 - if( pre_opaq1->Opcode() != Op_Opaque1 )
1.194 + if (pre_opaq1->Opcode() != Op_Opaque1)
1.195 return;
1.196 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
1.197 Node *pre_limit = pre_opaq->in(1);
1.198 @@ -1266,25 +1312,11 @@
1.199 // Ensure the original loop limit is available from the
1.200 // pre-loop Opaque1 node.
1.201 Node *orig_limit = pre_opaq->original_loop_limit();
1.202 - if( orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP )
1.203 + if (orig_limit == NULL || _igvn.type(orig_limit) == Type::TOP)
1.204 return;
1.205
1.206 - // Need to find the main-loop zero-trip guard
1.207 - Node *bolzm = iffm->in(1);
1.208 - assert( bolzm->Opcode() == Op_Bool, "" );
1.209 - Node *cmpzm = bolzm->in(1);
1.210 - assert( cmpzm->is_Cmp(), "" );
1.211 - Node *opqzm = cmpzm->in(2);
1.212 - if( opqzm->Opcode() != Op_Opaque1 )
1.213 - return;
1.214 - assert( opqzm->in(1) == main_limit, "do not understand situation" );
1.215 -
1.216 // Must know if its a count-up or count-down loop
1.217
1.218 - // protect against stride not being a constant
1.219 - if ( !cl->stride_is_con() ) {
1.220 - return;
1.221 - }
1.222 int stride_con = cl->stride_con();
1.223 Node *zero = _igvn.intcon(0);
1.224 Node *one = _igvn.intcon(1);
1.225 @@ -1566,16 +1598,24 @@
1.226 // have on the last iteration. This will break the loop.
1.227 bool IdealLoopTree::policy_do_remove_empty_loop( PhaseIdealLoop *phase ) {
1.228 // Minimum size must be empty loop
1.229 - if( _body.size() > 7/*number of nodes in an empty loop*/ ) return false;
1.230 + if (_body.size() > 7/*number of nodes in an empty loop*/)
1.231 + return false;
1.232
1.233 - if( !_head->is_CountedLoop() ) return false; // Dead loop
1.234 + if (!_head->is_CountedLoop())
1.235 + return false; // Dead loop
1.236 CountedLoopNode *cl = _head->as_CountedLoop();
1.237 - if( !cl->loopexit() ) return false; // Malformed loop
1.238 - if( !phase->is_member(this,phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue)) ) )
1.239 + if (!cl->loopexit())
1.240 + return false; // Malformed loop
1.241 + if (!phase->is_member(this, phase->get_ctrl(cl->loopexit()->in(CountedLoopEndNode::TestValue))))
1.242 return false; // Infinite loop
1.243 #ifndef PRODUCT
1.244 - if( PrintOpto )
1.245 - tty->print_cr("Removing empty loop");
1.246 + if (PrintOpto) {
1.247 + tty->print("Removing empty loop");
1.248 + this->dump_head();
1.249 + } else if (TraceLoopOpts) {
1.250 + tty->print("Empty ");
1.251 + this->dump_head();
1.252 + }
1.253 #endif
1.254 #ifdef ASSERT
1.255 // Ensure only one phi which is the iv.
1.256 @@ -1720,7 +1760,7 @@
1.257 //------------------------------iteration_split--------------------------------
1.258 bool IdealLoopTree::iteration_split( PhaseIdealLoop *phase, Node_List &old_new ) {
1.259 // Recursively iteration split nested loops
1.260 - if( _child && !_child->iteration_split( phase, old_new ))
1.261 + if (_child && !_child->iteration_split(phase, old_new))
1.262 return false;
1.263
1.264 // Clean out prior deadwood
1.265 @@ -1729,21 +1769,20 @@
1.266
1.267 // Look for loop-exit tests with my 50/50 guesses from the Parsing stage.
1.268 // Replace with a 1-in-10 exit guess.
1.269 - if( _parent /*not the root loop*/ &&
1.270 + if (_parent /*not the root loop*/ &&
1.271 !_irreducible &&
1.272 // Also ignore the occasional dead backedge
1.273 - !tail()->is_top() ) {
1.274 + !tail()->is_top()) {
1.275 adjust_loop_exit_prob(phase);
1.276 }
1.277
1.278 -
1.279 // Gate unrolling, RCE and peeling efforts.
1.280 - if( !_child && // If not an inner loop, do not split
1.281 + if (!_child && // If not an inner loop, do not split
1.282 !_irreducible &&
1.283 _allow_optimizations &&
1.284 - !tail()->is_top() ) { // Also ignore the occasional dead backedge
1.285 + !tail()->is_top()) { // Also ignore the occasional dead backedge
1.286 if (!_has_call) {
1.287 - if (!iteration_split_impl( phase, old_new )) {
1.288 + if (!iteration_split_impl(phase, old_new)) {
1.289 return false;
1.290 }
1.291 } else if (policy_unswitching(phase)) {
1.292 @@ -1752,16 +1791,17 @@
1.293 }
1.294
1.295 // Minor offset re-organization to remove loop-fallout uses of
1.296 - // trip counter.
1.297 - if( _head->is_CountedLoop() ) phase->reorg_offsets( this );
1.298 - if( _next && !_next->iteration_split( phase, old_new ))
1.299 + // trip counter when there was no major reshaping.
1.300 + phase->reorg_offsets(this);
1.301 +
1.302 + if (_next && !_next->iteration_split(phase, old_new))
1.303 return false;
1.304 return true;
1.305 }
1.306
1.307 //-------------------------------is_uncommon_trap_proj----------------------------
1.308 // Return true if proj is the form of "proj->[region->..]call_uct"
1.309 -bool PhaseIdealLoop::is_uncommon_trap_proj(ProjNode* proj, bool must_reason_predicate) {
1.310 +bool PhaseIdealLoop::is_uncommon_trap_proj(ProjNode* proj, Deoptimization::DeoptReason reason) {
1.311 int path_limit = 10;
1.312 assert(proj, "invalid argument");
1.313 Node* out = proj;
1.314 @@ -1772,8 +1812,8 @@
1.315 if (out->is_CallStaticJava()) {
1.316 int req = out->as_CallStaticJava()->uncommon_trap_request();
1.317 if (req != 0) {
1.318 - Deoptimization::DeoptReason reason = Deoptimization::trap_request_reason(req);
1.319 - if (!must_reason_predicate || reason == Deoptimization::Reason_predicate){
1.320 + Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req);
1.321 + if (trap_reason == reason || reason == Deoptimization::Reason_none) {
1.322 return true;
1.323 }
1.324 }
1.325 @@ -1790,15 +1830,15 @@
1.326 // other_proj->[region->..]call_uct"
1.327 //
1.328 // "must_reason_predicate" means the uct reason must be Reason_predicate
1.329 -bool PhaseIdealLoop::is_uncommon_trap_if_pattern(ProjNode *proj, bool must_reason_predicate) {
1.330 +bool PhaseIdealLoop::is_uncommon_trap_if_pattern(ProjNode *proj, Deoptimization::DeoptReason reason) {
1.331 Node *in0 = proj->in(0);
1.332 if (!in0->is_If()) return false;
1.333 // Variation of a dead If node.
1.334 if (in0->outcnt() < 2) return false;
1.335 IfNode* iff = in0->as_If();
1.336
1.337 - // we need "If(Conv2B(Opaque1(...)))" pattern for must_reason_predicate
1.338 - if (must_reason_predicate) {
1.339 + // we need "If(Conv2B(Opaque1(...)))" pattern for reason_predicate
1.340 + if (reason != Deoptimization::Reason_none) {
1.341 if (iff->in(1)->Opcode() != Op_Conv2B ||
1.342 iff->in(1)->in(1)->Opcode() != Op_Opaque1) {
1.343 return false;
1.344 @@ -1806,7 +1846,19 @@
1.345 }
1.346
1.347 ProjNode* other_proj = iff->proj_out(1-proj->_con)->as_Proj();
1.348 - return is_uncommon_trap_proj(other_proj, must_reason_predicate);
1.349 + return is_uncommon_trap_proj(other_proj, reason);
1.350 +}
1.351 +
1.352 +//-------------------------------register_control-------------------------
1.353 +void PhaseIdealLoop::register_control(Node* n, IdealLoopTree *loop, Node* pred) {
1.354 + assert(n->is_CFG(), "must be control node");
1.355 + _igvn.register_new_node_with_optimizer(n);
1.356 + loop->_body.push(n);
1.357 + set_loop(n, loop);
1.358 + // When called from beautify_loops() idom is not constructed yet.
1.359 + if (_idom != NULL) {
1.360 + set_idom(n, pred, dom_depth(pred));
1.361 + }
1.362 }
1.363
1.364 //------------------------------create_new_if_for_predicate------------------------
1.365 @@ -1843,8 +1895,10 @@
1.366 //
1.367 // We will create a region to guard the uct call if there is no one there.
1.368 // The true projecttion (if_cont) of the new_iff is returned.
1.369 -ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj) {
1.370 - assert(is_uncommon_trap_if_pattern(cont_proj, true), "must be a uct if pattern!");
1.371 +// This code is also used to clone predicates to clonned loops.
1.372 +ProjNode* PhaseIdealLoop::create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry,
1.373 + Deoptimization::DeoptReason reason) {
1.374 + assert(is_uncommon_trap_if_pattern(cont_proj, reason), "must be a uct if pattern!");
1.375 IfNode* iff = cont_proj->in(0)->as_If();
1.376
1.377 ProjNode *uncommon_proj = iff->proj_out(1 - cont_proj->_con);
1.378 @@ -1854,57 +1908,84 @@
1.379 if (!rgn->is_Region()) { // create a region to guard the call
1.380 assert(rgn->is_Call(), "must be call uct");
1.381 CallNode* call = rgn->as_Call();
1.382 + IdealLoopTree* loop = get_loop(call);
1.383 rgn = new (C, 1) RegionNode(1);
1.384 - _igvn.set_type(rgn, rgn->bottom_type());
1.385 rgn->add_req(uncommon_proj);
1.386 - set_idom(rgn, idom(uncommon_proj), dom_depth(uncommon_proj)+1);
1.387 + register_control(rgn, loop, uncommon_proj);
1.388 _igvn.hash_delete(call);
1.389 call->set_req(0, rgn);
1.390 + // When called from beautify_loops() idom is not constructed yet.
1.391 + if (_idom != NULL) {
1.392 + set_idom(call, rgn, dom_depth(rgn));
1.393 + }
1.394 }
1.395
1.396 + Node* entry = iff->in(0);
1.397 + if (new_entry != NULL) {
1.398 + // Clonning the predicate to new location.
1.399 + entry = new_entry;
1.400 + }
1.401 // Create new_iff
1.402 - uint iffdd = dom_depth(iff);
1.403 - IdealLoopTree* lp = get_loop(iff);
1.404 - IfNode *new_iff = new (C, 2) IfNode(iff->in(0), NULL, iff->_prob, iff->_fcnt);
1.405 - register_node(new_iff, lp, idom(iff), iffdd);
1.406 + IdealLoopTree* lp = get_loop(entry);
1.407 + IfNode *new_iff = new (C, 2) IfNode(entry, NULL, iff->_prob, iff->_fcnt);
1.408 + register_control(new_iff, lp, entry);
1.409 Node *if_cont = new (C, 1) IfTrueNode(new_iff);
1.410 Node *if_uct = new (C, 1) IfFalseNode(new_iff);
1.411 if (cont_proj->is_IfFalse()) {
1.412 // Swap
1.413 Node* tmp = if_uct; if_uct = if_cont; if_cont = tmp;
1.414 }
1.415 - register_node(if_cont, lp, new_iff, iffdd);
1.416 - register_node(if_uct, get_loop(rgn), new_iff, iffdd);
1.417 -
1.418 - // if_cont to iff
1.419 - _igvn.hash_delete(iff);
1.420 - iff->set_req(0, if_cont);
1.421 - set_idom(iff, if_cont, dom_depth(iff));
1.422 + register_control(if_cont, lp, new_iff);
1.423 + register_control(if_uct, get_loop(rgn), new_iff);
1.424
1.425 // if_uct to rgn
1.426 _igvn.hash_delete(rgn);
1.427 rgn->add_req(if_uct);
1.428 - Node* ridom = idom(rgn);
1.429 - Node* nrdom = dom_lca(ridom, new_iff);
1.430 - set_idom(rgn, nrdom, dom_depth(rgn));
1.431 -
1.432 + // When called from beautify_loops() idom is not constructed yet.
1.433 + if (_idom != NULL) {
1.434 + Node* ridom = idom(rgn);
1.435 + Node* nrdom = dom_lca(ridom, new_iff);
1.436 + set_idom(rgn, nrdom, dom_depth(rgn));
1.437 + }
1.438 // rgn must have no phis
1.439 assert(!rgn->as_Region()->has_phi(), "region must have no phis");
1.440
1.441 + if (new_entry == NULL) {
1.442 + // Attach if_cont to iff
1.443 + _igvn.hash_delete(iff);
1.444 + iff->set_req(0, if_cont);
1.445 + if (_idom != NULL) {
1.446 + set_idom(iff, if_cont, dom_depth(iff));
1.447 + }
1.448 + }
1.449 return if_cont->as_Proj();
1.450 }
1.451
1.452 -//------------------------------find_predicate_insertion_point--------------------------
1.453 +//--------------------------find_predicate_insertion_point-------------------
1.454 // Find a good location to insert a predicate
1.455 -ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c) {
1.456 - if (start_c == C->root() || !start_c->is_Proj())
1.457 +ProjNode* PhaseIdealLoop::find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason) {
1.458 + if (start_c == NULL || !start_c->is_Proj())
1.459 return NULL;
1.460 - if (is_uncommon_trap_if_pattern(start_c->as_Proj(), true/*Reason_Predicate*/)) {
1.461 + if (is_uncommon_trap_if_pattern(start_c->as_Proj(), reason)) {
1.462 return start_c->as_Proj();
1.463 }
1.464 return NULL;
1.465 }
1.466
1.467 +//--------------------------find_predicate------------------------------------
1.468 +// Find a predicate
1.469 +Node* PhaseIdealLoop::find_predicate(Node* entry) {
1.470 + Node* predicate = NULL;
1.471 + if (UseLoopPredicate) {
1.472 + predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
1.473 + if (predicate != NULL) { // right pattern that can be used by loop predication
1.474 + assert(entry->in(0)->in(1)->in(1)->Opcode()==Op_Opaque1, "must be");
1.475 + return entry;
1.476 + }
1.477 + }
1.478 + return NULL;
1.479 +}
1.480 +
1.481 //------------------------------Invariance-----------------------------------
1.482 // Helper class for loop_predication_impl to compute invariance on the fly and
1.483 // clone invariants.
1.484 @@ -2151,6 +2232,11 @@
1.485 return false;
1.486 }
1.487
1.488 + if (loop->_head->unique_ctrl_out()->Opcode() == Op_NeverBranch) {
1.489 + // do nothing for infinite loops
1.490 + return false;
1.491 + }
1.492 +
1.493 CountedLoopNode *cl = NULL;
1.494 if (loop->_head->is_CountedLoop()) {
1.495 cl = loop->_head->as_CountedLoop();
1.496 @@ -2158,40 +2244,22 @@
1.497 if (!cl->is_normal_loop()) return false;
1.498 }
1.499
1.500 - // Too many traps seen?
1.501 - bool tmt = C->too_many_traps(C->method(), 0, Deoptimization::Reason_predicate);
1.502 - int tc = C->trap_count(Deoptimization::Reason_predicate);
1.503 - if (tmt || tc > 0) {
1.504 - if (TraceLoopPredicate) {
1.505 - tty->print_cr("too many predicate traps: %d", tc);
1.506 - C->method()->print(); // which method has too many predicate traps
1.507 - tty->print_cr("");
1.508 - }
1.509 - return false;
1.510 - }
1.511 -
1.512 LoopNode *lpn = loop->_head->as_Loop();
1.513 Node* entry = lpn->in(LoopNode::EntryControl);
1.514
1.515 - ProjNode *predicate_proj = find_predicate_insertion_point(entry);
1.516 - if (!predicate_proj){
1.517 + ProjNode *predicate_proj = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate);
1.518 + if (!predicate_proj) {
1.519 #ifndef PRODUCT
1.520 if (TraceLoopPredicate) {
1.521 tty->print("missing predicate:");
1.522 loop->dump_head();
1.523 + lpn->dump(1);
1.524 }
1.525 #endif
1.526 return false;
1.527 }
1.528 -
1.529 ConNode* zero = _igvn.intcon(0);
1.530 set_ctrl(zero, C->root());
1.531 - Node *cond_false = new (C, 2) Conv2BNode(zero);
1.532 - register_new_node(cond_false, C->root());
1.533 - ConNode* one = _igvn.intcon(1);
1.534 - set_ctrl(one, C->root());
1.535 - Node *cond_true = new (C, 2) Conv2BNode(one);
1.536 - register_new_node(cond_true, C->root());
1.537
1.538 ResourceArea *area = Thread::current()->resource_area();
1.539 Invariance invar(area, loop);
1.540 @@ -2218,7 +2286,7 @@
1.541 ProjNode* proj = if_proj_list.pop()->as_Proj();
1.542 IfNode* iff = proj->in(0)->as_If();
1.543
1.544 - if (!is_uncommon_trap_if_pattern(proj)) {
1.545 + if (!is_uncommon_trap_if_pattern(proj, Deoptimization::Reason_none)) {
1.546 if (loop->is_loop_exit(iff)) {
1.547 // stop processing the remaining projs in the list because the execution of them
1.548 // depends on the condition of "iff" (iff->in(1)).
1.549 @@ -2242,7 +2310,8 @@
1.550 BoolNode* bol = test->as_Bool();
1.551 if (invar.is_invariant(bol)) {
1.552 // Invariant test
1.553 - new_predicate_proj = create_new_if_for_predicate(predicate_proj);
1.554 + new_predicate_proj = create_new_if_for_predicate(predicate_proj, NULL,
1.555 + Deoptimization::Reason_predicate);
1.556 Node* ctrl = new_predicate_proj->in(0)->as_If()->in(0);
1.557 BoolNode* new_predicate_bol = invar.clone(bol, ctrl)->as_Bool();
1.558
1.559 @@ -2256,8 +2325,15 @@
1.560 IfNode* new_predicate_iff = new_predicate_proj->in(0)->as_If();
1.561 _igvn.hash_delete(new_predicate_iff);
1.562 new_predicate_iff->set_req(1, new_predicate_bol);
1.563 - if (TraceLoopPredicate) tty->print_cr("invariant if%s: %d", negated ? " negated" : "", new_predicate_iff->_idx);
1.564 -
1.565 +#ifndef PRODUCT
1.566 + if (TraceLoopPredicate) {
1.567 + tty->print("Predicate invariant if%s: %d ", negated ? " negated" : "", new_predicate_iff->_idx);
1.568 + loop->dump_head();
1.569 + } else if (TraceLoopOpts) {
1.570 + tty->print("Predicate IC ");
1.571 + loop->dump_head();
1.572 + }
1.573 +#endif
1.574 } else if (cl != NULL && loop->is_range_check_if(iff, this, invar)) {
1.575 assert(proj->_con == predicate_proj->_con, "must match");
1.576
1.577 @@ -2281,8 +2357,8 @@
1.578 // lower_bound test will dominate the upper bound test and all
1.579 // cloned or created nodes will use the lower bound test as
1.580 // their declared control.
1.581 - ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj);
1.582 - ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj);
1.583 + ProjNode* lower_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
1.584 + ProjNode* upper_bound_proj = create_new_if_for_predicate(predicate_proj, NULL, Deoptimization::Reason_predicate);
1.585 assert(upper_bound_proj->in(0)->as_If()->in(0) == lower_bound_proj, "should dominate");
1.586 Node *ctrl = lower_bound_proj->in(0)->as_If()->in(0);
1.587
1.588 @@ -2311,41 +2387,24 @@
1.589 // Fall through into rest of the clean up code which will move
1.590 // any dependent nodes onto the upper bound test.
1.591 new_predicate_proj = upper_bound_proj;
1.592 +
1.593 +#ifndef PRODUCT
1.594 + if (TraceLoopOpts && !TraceLoopPredicate) {
1.595 + tty->print("Predicate RC ");
1.596 + loop->dump_head();
1.597 + }
1.598 +#endif
1.599 } else {
1.600 - // The other proj of the "iff" is a uncommon trap projection, and we can assume
1.601 - // the other proj will not be executed ("executed" means uct raised).
1.602 + // Loop variant check (for example, range check in non-counted loop)
1.603 + // with uncommon trap.
1.604 continue;
1.605 }
1.606 -
1.607 + assert(new_predicate_proj != NULL, "sanity");
1.608 // Success - attach condition (new_predicate_bol) to predicate if
1.609 invar.map_ctrl(proj, new_predicate_proj); // so that invariance test can be appropriate
1.610
1.611 - // Eliminate the old if in the loop body
1.612 - _igvn.hash_delete(iff);
1.613 - iff->set_req(1, proj->is_IfFalse() ? cond_false : cond_true);
1.614 -
1.615 - Node* ctrl = new_predicate_proj; // new control
1.616 - ProjNode* dp = proj; // old control
1.617 - assert(get_loop(dp) == loop, "guaranteed at the time of collecting proj");
1.618 - // Find nodes (depends only on the test) off the surviving projection;
1.619 - // move them outside the loop with the control of proj_clone
1.620 - for (DUIterator_Fast imax, i = dp->fast_outs(imax); i < imax; i++) {
1.621 - Node* cd = dp->fast_out(i); // Control-dependent node
1.622 - if (cd->depends_only_on_test()) {
1.623 - assert(cd->in(0) == dp, "");
1.624 - _igvn.hash_delete(cd);
1.625 - cd->set_req(0, ctrl); // ctrl, not NULL
1.626 - set_early_ctrl(cd);
1.627 - _igvn._worklist.push(cd);
1.628 - IdealLoopTree *new_loop = get_loop(get_ctrl(cd));
1.629 - if (new_loop != loop) {
1.630 - if (!loop->_child) loop->_body.yank(cd);
1.631 - if (!new_loop->_child ) new_loop->_body.push(cd);
1.632 - }
1.633 - --i;
1.634 - --imax;
1.635 - }
1.636 - }
1.637 + // Eliminate the old If in the loop body
1.638 + dominated_by( new_predicate_proj, iff, proj->_con != new_predicate_proj->_con );
1.639
1.640 hoisted = true;
1.641 C->set_major_progress();
