Thu, 20 Sep 2012 16:49:17 +0200
7023898: Intrinsify AtomicLongFieldUpdater.getAndIncrement()
Summary: use shorter instruction sequences for atomic add and atomic exchange when possible.
Reviewed-by: kvn, jrose
duke@435 | 1 | /* |
trims@2708 | 2 | * Copyright (c) 2006, 2011, Oracle and/or its affiliates. All rights reserved. |
duke@435 | 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
duke@435 | 4 | * |
duke@435 | 5 | * This code is free software; you can redistribute it and/or modify it |
duke@435 | 6 | * under the terms of the GNU General Public License version 2 only, as |
duke@435 | 7 | * published by the Free Software Foundation. |
duke@435 | 8 | * |
duke@435 | 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
duke@435 | 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
duke@435 | 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
duke@435 | 12 | * version 2 for more details (a copy is included in the LICENSE file that |
duke@435 | 13 | * accompanied this code). |
duke@435 | 14 | * |
duke@435 | 15 | * You should have received a copy of the GNU General Public License version |
duke@435 | 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
duke@435 | 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
duke@435 | 18 | * |
trims@1907 | 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
trims@1907 | 20 | * or visit www.oracle.com if you need additional information or have any |
trims@1907 | 21 | * questions. |
duke@435 | 22 | * |
duke@435 | 23 | */ |
duke@435 | 24 | |
stefank@2314 | 25 | #include "precompiled.hpp" |
stefank@2314 | 26 | #include "memory/allocation.inline.hpp" |
stefank@2314 | 27 | #include "opto/connode.hpp" |
stefank@2314 | 28 | #include "opto/loopnode.hpp" |
stefank@2314 | 29 | #include "opto/rootnode.hpp" |
duke@435 | 30 | |
duke@435 | 31 | //================= Loop Unswitching ===================== |
duke@435 | 32 | // |
duke@435 | 33 | // orig: transformed: |
duke@435 | 34 | // if (invariant-test) then |
kvn@2727 | 35 | // predicate predicate |
duke@435 | 36 | // loop loop |
duke@435 | 37 | // stmt1 stmt1 |
duke@435 | 38 | // if (invariant-test) then stmt2 |
duke@435 | 39 | // stmt2 stmt4 |
duke@435 | 40 | // else endloop |
duke@435 | 41 | // stmt3 else |
kvn@2727 | 42 | // endif predicate [clone] |
kvn@2727 | 43 | // stmt4 loop [clone] |
kvn@2727 | 44 | // endloop stmt1 [clone] |
kvn@2727 | 45 | // stmt3 |
duke@435 | 46 | // stmt4 [clone] |
duke@435 | 47 | // endloop |
duke@435 | 48 | // endif |
duke@435 | 49 | // |
duke@435 | 50 | // Note: the "else" clause may be empty |
duke@435 | 51 | |
duke@435 | 52 | //------------------------------policy_unswitching----------------------------- |
duke@435 | 53 | // Return TRUE or FALSE if the loop should be unswitched |
duke@435 | 54 | // (ie. clone loop with an invariant test that does not exit the loop) |
duke@435 | 55 | bool IdealLoopTree::policy_unswitching( PhaseIdealLoop *phase ) const { |
duke@435 | 56 | if( !LoopUnswitching ) { |
duke@435 | 57 | return false; |
duke@435 | 58 | } |
rasbold@543 | 59 | if (!_head->is_Loop()) { |
rasbold@543 | 60 | return false; |
rasbold@543 | 61 | } |
duke@435 | 62 | uint nodes_left = MaxNodeLimit - phase->C->unique(); |
duke@435 | 63 | if (2 * _body.size() > nodes_left) { |
duke@435 | 64 | return false; // Too speculative if running low on nodes. |
duke@435 | 65 | } |
duke@435 | 66 | LoopNode* head = _head->as_Loop(); |
duke@435 | 67 | if (head->unswitch_count() + 1 > head->unswitch_max()) { |
duke@435 | 68 | return false; |
duke@435 | 69 | } |
duke@435 | 70 | return phase->find_unswitching_candidate(this) != NULL; |
duke@435 | 71 | } |
duke@435 | 72 | |
duke@435 | 73 | //------------------------------find_unswitching_candidate----------------------------- |
duke@435 | 74 | // Find candidate "if" for unswitching |
duke@435 | 75 | IfNode* PhaseIdealLoop::find_unswitching_candidate(const IdealLoopTree *loop) const { |
duke@435 | 76 | |
duke@435 | 77 | // Find first invariant test that doesn't exit the loop |
duke@435 | 78 | LoopNode *head = loop->_head->as_Loop(); |
duke@435 | 79 | IfNode* unswitch_iff = NULL; |
duke@435 | 80 | Node* n = head->in(LoopNode::LoopBackControl); |
duke@435 | 81 | while (n != head) { |
duke@435 | 82 | Node* n_dom = idom(n); |
duke@435 | 83 | if (n->is_Region()) { |
duke@435 | 84 | if (n_dom->is_If()) { |
duke@435 | 85 | IfNode* iff = n_dom->as_If(); |
duke@435 | 86 | if (iff->in(1)->is_Bool()) { |
duke@435 | 87 | BoolNode* bol = iff->in(1)->as_Bool(); |
duke@435 | 88 | if (bol->in(1)->is_Cmp()) { |
duke@435 | 89 | // If condition is invariant and not a loop exit, |
duke@435 | 90 | // then found reason to unswitch. |
duke@435 | 91 | if (loop->is_invariant(bol) && !loop->is_loop_exit(iff)) { |
duke@435 | 92 | unswitch_iff = iff; |
duke@435 | 93 | } |
duke@435 | 94 | } |
duke@435 | 95 | } |
duke@435 | 96 | } |
duke@435 | 97 | } |
duke@435 | 98 | n = n_dom; |
duke@435 | 99 | } |
duke@435 | 100 | return unswitch_iff; |
duke@435 | 101 | } |
duke@435 | 102 | |
duke@435 | 103 | //------------------------------do_unswitching----------------------------- |
duke@435 | 104 | // Clone loop with an invariant test (that does not exit) and |
duke@435 | 105 | // insert a clone of the test that selects which version to |
duke@435 | 106 | // execute. |
duke@435 | 107 | void PhaseIdealLoop::do_unswitching (IdealLoopTree *loop, Node_List &old_new) { |
duke@435 | 108 | |
duke@435 | 109 | // Find first invariant test that doesn't exit the loop |
duke@435 | 110 | LoopNode *head = loop->_head->as_Loop(); |
duke@435 | 111 | |
duke@435 | 112 | IfNode* unswitch_iff = find_unswitching_candidate((const IdealLoopTree *)loop); |
duke@435 | 113 | assert(unswitch_iff != NULL, "should be at least one"); |
duke@435 | 114 | |
kvn@2665 | 115 | #ifndef PRODUCT |
kvn@2665 | 116 | if (TraceLoopOpts) { |
kvn@2665 | 117 | tty->print("Unswitch %d ", head->unswitch_count()+1); |
kvn@2665 | 118 | loop->dump_head(); |
kvn@2665 | 119 | } |
kvn@2665 | 120 | #endif |
kvn@2665 | 121 | |
duke@435 | 122 | // Need to revert back to normal loop |
duke@435 | 123 | if (head->is_CountedLoop() && !head->as_CountedLoop()->is_normal_loop()) { |
duke@435 | 124 | head->as_CountedLoop()->set_normal_loop(); |
duke@435 | 125 | } |
duke@435 | 126 | |
duke@435 | 127 | ProjNode* proj_true = create_slow_version_of_loop(loop, old_new); |
duke@435 | 128 | |
kvn@2727 | 129 | #ifdef ASSERT |
kvn@2727 | 130 | Node* uniqc = proj_true->unique_ctrl_out(); |
kvn@2727 | 131 | Node* entry = head->in(LoopNode::EntryControl); |
kvn@2727 | 132 | Node* predicate = find_predicate(entry); |
kvn@2877 | 133 | if (predicate != NULL && LoopLimitCheck && UseLoopPredicate) { |
kvn@2877 | 134 | // We may have two predicates, find first. |
kvn@2877 | 135 | entry = find_predicate(entry->in(0)->in(0)); |
kvn@2877 | 136 | if (entry != NULL) predicate = entry; |
kvn@2877 | 137 | } |
kvn@2727 | 138 | if (predicate != NULL) predicate = predicate->in(0); |
kvn@2727 | 139 | assert(proj_true->is_IfTrue() && |
kvn@2727 | 140 | (predicate == NULL && uniqc == head || |
kvn@2727 | 141 | predicate != NULL && uniqc == predicate), "by construction"); |
kvn@2727 | 142 | #endif |
duke@435 | 143 | // Increment unswitch count |
duke@435 | 144 | LoopNode* head_clone = old_new[head->_idx]->as_Loop(); |
duke@435 | 145 | int nct = head->unswitch_count() + 1; |
duke@435 | 146 | head->set_unswitch_count(nct); |
duke@435 | 147 | head_clone->set_unswitch_count(nct); |
duke@435 | 148 | |
duke@435 | 149 | // Add test to new "if" outside of loop |
duke@435 | 150 | IfNode* invar_iff = proj_true->in(0)->as_If(); |
duke@435 | 151 | Node* invar_iff_c = invar_iff->in(0); |
duke@435 | 152 | BoolNode* bol = unswitch_iff->in(1)->as_Bool(); |
duke@435 | 153 | invar_iff->set_req(1, bol); |
duke@435 | 154 | invar_iff->_prob = unswitch_iff->_prob; |
duke@435 | 155 | |
duke@435 | 156 | ProjNode* proj_false = invar_iff->proj_out(0)->as_Proj(); |
duke@435 | 157 | |
twisti@1040 | 158 | // Hoist invariant casts out of each loop to the appropriate |
duke@435 | 159 | // control projection. |
duke@435 | 160 | |
duke@435 | 161 | Node_List worklist; |
duke@435 | 162 | |
duke@435 | 163 | for (DUIterator_Fast imax, i = unswitch_iff->fast_outs(imax); i < imax; i++) { |
duke@435 | 164 | ProjNode* proj= unswitch_iff->fast_out(i)->as_Proj(); |
duke@435 | 165 | // Copy to a worklist for easier manipulation |
duke@435 | 166 | for (DUIterator_Fast jmax, j = proj->fast_outs(jmax); j < jmax; j++) { |
duke@435 | 167 | Node* use = proj->fast_out(j); |
duke@435 | 168 | if (use->Opcode() == Op_CheckCastPP && loop->is_invariant(use->in(1))) { |
duke@435 | 169 | worklist.push(use); |
duke@435 | 170 | } |
duke@435 | 171 | } |
duke@435 | 172 | ProjNode* invar_proj = invar_iff->proj_out(proj->_con)->as_Proj(); |
duke@435 | 173 | while (worklist.size() > 0) { |
duke@435 | 174 | Node* use = worklist.pop(); |
duke@435 | 175 | Node* nuse = use->clone(); |
duke@435 | 176 | nuse->set_req(0, invar_proj); |
kvn@3847 | 177 | _igvn.replace_input_of(use, 1, nuse); |
duke@435 | 178 | register_new_node(nuse, invar_proj); |
duke@435 | 179 | // Same for the clone |
duke@435 | 180 | Node* use_clone = old_new[use->_idx]; |
kvn@3847 | 181 | _igvn.replace_input_of(use_clone, 1, nuse); |
duke@435 | 182 | } |
duke@435 | 183 | } |
duke@435 | 184 | |
duke@435 | 185 | // Hardwire the control paths in the loops into if(true) and if(false) |
kvn@3847 | 186 | _igvn.rehash_node_delayed(unswitch_iff); |
duke@435 | 187 | short_circuit_if(unswitch_iff, proj_true); |
duke@435 | 188 | |
duke@435 | 189 | IfNode* unswitch_iff_clone = old_new[unswitch_iff->_idx]->as_If(); |
kvn@3847 | 190 | _igvn.rehash_node_delayed(unswitch_iff_clone); |
duke@435 | 191 | short_circuit_if(unswitch_iff_clone, proj_false); |
duke@435 | 192 | |
duke@435 | 193 | // Reoptimize loops |
duke@435 | 194 | loop->record_for_igvn(); |
duke@435 | 195 | for(int i = loop->_body.size() - 1; i >= 0 ; i--) { |
duke@435 | 196 | Node *n = loop->_body[i]; |
duke@435 | 197 | Node *n_clone = old_new[n->_idx]; |
duke@435 | 198 | _igvn._worklist.push(n_clone); |
duke@435 | 199 | } |
duke@435 | 200 | |
duke@435 | 201 | #ifndef PRODUCT |
duke@435 | 202 | if (TraceLoopUnswitching) { |
duke@435 | 203 | tty->print_cr("Loop unswitching orig: %d @ %d new: %d @ %d", |
duke@435 | 204 | head->_idx, unswitch_iff->_idx, |
duke@435 | 205 | old_new[head->_idx]->_idx, unswitch_iff_clone->_idx); |
duke@435 | 206 | } |
duke@435 | 207 | #endif |
duke@435 | 208 | |
duke@435 | 209 | C->set_major_progress(); |
duke@435 | 210 | } |
duke@435 | 211 | |
duke@435 | 212 | //-------------------------create_slow_version_of_loop------------------------ |
duke@435 | 213 | // Create a slow version of the loop by cloning the loop |
duke@435 | 214 | // and inserting an if to select fast-slow versions. |
duke@435 | 215 | // Return control projection of the entry to the fast version. |
duke@435 | 216 | ProjNode* PhaseIdealLoop::create_slow_version_of_loop(IdealLoopTree *loop, |
duke@435 | 217 | Node_List &old_new) { |
duke@435 | 218 | LoopNode* head = loop->_head->as_Loop(); |
kvn@2877 | 219 | bool counted_loop = head->is_CountedLoop(); |
duke@435 | 220 | Node* entry = head->in(LoopNode::EntryControl); |
kvn@3847 | 221 | _igvn.rehash_node_delayed(entry); |
duke@435 | 222 | IdealLoopTree* outer_loop = loop->_parent; |
duke@435 | 223 | |
duke@435 | 224 | Node *cont = _igvn.intcon(1); |
duke@435 | 225 | set_ctrl(cont, C->root()); |
kvn@651 | 226 | Node* opq = new (C, 2) Opaque1Node(C, cont); |
duke@435 | 227 | register_node(opq, outer_loop, entry, dom_depth(entry)); |
duke@435 | 228 | Node *bol = new (C, 2) Conv2BNode(opq); |
duke@435 | 229 | register_node(bol, outer_loop, entry, dom_depth(entry)); |
duke@435 | 230 | IfNode* iff = new (C, 2) IfNode(entry, bol, PROB_MAX, COUNT_UNKNOWN); |
duke@435 | 231 | register_node(iff, outer_loop, entry, dom_depth(entry)); |
duke@435 | 232 | ProjNode* iffast = new (C, 1) IfTrueNode(iff); |
duke@435 | 233 | register_node(iffast, outer_loop, iff, dom_depth(iff)); |
duke@435 | 234 | ProjNode* ifslow = new (C, 1) IfFalseNode(iff); |
duke@435 | 235 | register_node(ifslow, outer_loop, iff, dom_depth(iff)); |
duke@435 | 236 | |
duke@435 | 237 | // Clone the loop body. The clone becomes the fast loop. The |
kvn@2727 | 238 | // original pre-header will (illegally) have 3 control users |
kvn@2727 | 239 | // (old & new loops & new if). |
duke@435 | 240 | clone_loop(loop, old_new, dom_depth(head), iff); |
duke@435 | 241 | assert(old_new[head->_idx]->is_Loop(), "" ); |
duke@435 | 242 | |
duke@435 | 243 | // Fast (true) control |
kvn@2877 | 244 | Node* iffast_pred = clone_loop_predicates(entry, iffast, !counted_loop); |
kvn@3847 | 245 | _igvn.replace_input_of(head, LoopNode::EntryControl, iffast_pred); |
kvn@2727 | 246 | set_idom(head, iffast_pred, dom_depth(head)); |
duke@435 | 247 | |
duke@435 | 248 | // Slow (false) control |
kvn@3043 | 249 | Node* ifslow_pred = clone_loop_predicates(entry, ifslow, !counted_loop); |
duke@435 | 250 | LoopNode* slow_head = old_new[head->_idx]->as_Loop(); |
kvn@3847 | 251 | _igvn.replace_input_of(slow_head, LoopNode::EntryControl, ifslow_pred); |
kvn@2727 | 252 | set_idom(slow_head, ifslow_pred, dom_depth(slow_head)); |
duke@435 | 253 | |
duke@435 | 254 | recompute_dom_depth(); |
duke@435 | 255 | |
duke@435 | 256 | return iffast; |
duke@435 | 257 | } |