duke@435: /* duke@435: * Copyright 2005-2007 Sun Microsystems, Inc. All Rights Reserved. duke@435: * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. duke@435: * duke@435: * This code is free software; you can redistribute it and/or modify it duke@435: * under the terms of the GNU General Public License version 2 only, as duke@435: * published by the Free Software Foundation. duke@435: * duke@435: * This code is distributed in the hope that it will be useful, but WITHOUT duke@435: * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or duke@435: * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License duke@435: * version 2 for more details (a copy is included in the LICENSE file that duke@435: * accompanied this code). duke@435: * duke@435: * You should have received a copy of the GNU General Public License version duke@435: * 2 along with this work; if not, write to the Free Software Foundation, duke@435: * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. duke@435: * duke@435: * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, duke@435: * CA 95054 USA or visit www.sun.com if you need additional information or duke@435: * have any questions. duke@435: * duke@435: */ duke@435: duke@435: #include "incls/_precompiled.incl" duke@435: #include "incls/_macro.cpp.incl" duke@435: duke@435: duke@435: // duke@435: // Replace any references to "oldref" in inputs to "use" with "newref". duke@435: // Returns the number of replacements made. duke@435: // duke@435: int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) { duke@435: int nreplacements = 0; duke@435: uint req = use->req(); duke@435: for (uint j = 0; j < use->len(); j++) { duke@435: Node *uin = use->in(j); duke@435: if (uin == oldref) { duke@435: if (j < req) duke@435: use->set_req(j, newref); duke@435: else duke@435: use->set_prec(j, newref); duke@435: nreplacements++; duke@435: } else if (j >= req && uin == NULL) { duke@435: break; duke@435: } duke@435: } duke@435: return nreplacements; duke@435: } duke@435: duke@435: void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) { duke@435: // Copy debug information and adjust JVMState information duke@435: uint old_dbg_start = oldcall->tf()->domain()->cnt(); duke@435: uint new_dbg_start = newcall->tf()->domain()->cnt(); duke@435: int jvms_adj = new_dbg_start - old_dbg_start; duke@435: assert (new_dbg_start == newcall->req(), "argument count mismatch"); duke@435: for (uint i = old_dbg_start; i < oldcall->req(); i++) { duke@435: newcall->add_req(oldcall->in(i)); duke@435: } duke@435: newcall->set_jvms(oldcall->jvms()); duke@435: for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) { duke@435: jvms->set_map(newcall); duke@435: jvms->set_locoff(jvms->locoff()+jvms_adj); duke@435: jvms->set_stkoff(jvms->stkoff()+jvms_adj); duke@435: jvms->set_monoff(jvms->monoff()+jvms_adj); duke@435: jvms->set_endoff(jvms->endoff()+jvms_adj); duke@435: } duke@435: } duke@435: duke@435: Node* PhaseMacroExpand::opt_iff(Node* region, Node* iff) { duke@435: IfNode *opt_iff = transform_later(iff)->as_If(); duke@435: duke@435: // Fast path taken; set region slot 2 duke@435: Node *fast_taken = transform_later( new (C, 1) IfFalseNode(opt_iff) ); duke@435: region->init_req(2,fast_taken); // Capture fast-control duke@435: duke@435: // Fast path not-taken, i.e. slow path duke@435: Node *slow_taken = transform_later( new (C, 1) IfTrueNode(opt_iff) ); duke@435: return slow_taken; duke@435: } duke@435: duke@435: //--------------------copy_predefined_input_for_runtime_call-------------------- duke@435: void PhaseMacroExpand::copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call) { duke@435: // Set fixed predefined input arguments duke@435: call->init_req( TypeFunc::Control, ctrl ); duke@435: call->init_req( TypeFunc::I_O , oldcall->in( TypeFunc::I_O) ); duke@435: call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ????? duke@435: call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) ); duke@435: call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) ); duke@435: } duke@435: duke@435: //------------------------------make_slow_call--------------------------------- duke@435: CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call, const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1) { duke@435: duke@435: // Slow-path call duke@435: int size = slow_call_type->domain()->cnt(); duke@435: CallNode *call = leaf_name duke@435: ? (CallNode*)new (C, size) CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM ) duke@435: : (CallNode*)new (C, size) CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM ); duke@435: duke@435: // Slow path call has no side-effects, uses few values duke@435: copy_predefined_input_for_runtime_call(slow_path, oldcall, call ); duke@435: if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0); duke@435: if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1); duke@435: copy_call_debug_info(oldcall, call); duke@435: call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. duke@435: _igvn.hash_delete(oldcall); duke@435: _igvn.subsume_node(oldcall, call); duke@435: transform_later(call); duke@435: duke@435: return call; duke@435: } duke@435: duke@435: void PhaseMacroExpand::extract_call_projections(CallNode *call) { duke@435: _fallthroughproj = NULL; duke@435: _fallthroughcatchproj = NULL; duke@435: _ioproj_fallthrough = NULL; duke@435: _ioproj_catchall = NULL; duke@435: _catchallcatchproj = NULL; duke@435: _memproj_fallthrough = NULL; duke@435: _memproj_catchall = NULL; duke@435: _resproj = NULL; duke@435: for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) { duke@435: ProjNode *pn = call->fast_out(i)->as_Proj(); duke@435: switch (pn->_con) { duke@435: case TypeFunc::Control: duke@435: { duke@435: // For Control (fallthrough) and I_O (catch_all_index) we have CatchProj -> Catch -> Proj duke@435: _fallthroughproj = pn; duke@435: DUIterator_Fast jmax, j = pn->fast_outs(jmax); duke@435: const Node *cn = pn->fast_out(j); duke@435: if (cn->is_Catch()) { duke@435: ProjNode *cpn = NULL; duke@435: for (DUIterator_Fast kmax, k = cn->fast_outs(kmax); k < kmax; k++) { duke@435: cpn = cn->fast_out(k)->as_Proj(); duke@435: assert(cpn->is_CatchProj(), "must be a CatchProjNode"); duke@435: if (cpn->_con == CatchProjNode::fall_through_index) duke@435: _fallthroughcatchproj = cpn; duke@435: else { duke@435: assert(cpn->_con == CatchProjNode::catch_all_index, "must be correct index."); duke@435: _catchallcatchproj = cpn; duke@435: } duke@435: } duke@435: } duke@435: break; duke@435: } duke@435: case TypeFunc::I_O: duke@435: if (pn->_is_io_use) duke@435: _ioproj_catchall = pn; duke@435: else duke@435: _ioproj_fallthrough = pn; duke@435: break; duke@435: case TypeFunc::Memory: duke@435: if (pn->_is_io_use) duke@435: _memproj_catchall = pn; duke@435: else duke@435: _memproj_fallthrough = pn; duke@435: break; duke@435: case TypeFunc::Parms: duke@435: _resproj = pn; duke@435: break; duke@435: default: duke@435: assert(false, "unexpected projection from allocation node."); duke@435: } duke@435: } duke@435: duke@435: } duke@435: duke@435: duke@435: //---------------------------set_eden_pointers------------------------- duke@435: void PhaseMacroExpand::set_eden_pointers(Node* &eden_top_adr, Node* &eden_end_adr) { duke@435: if (UseTLAB) { // Private allocation: load from TLS duke@435: Node* thread = transform_later(new (C, 1) ThreadLocalNode()); duke@435: int tlab_top_offset = in_bytes(JavaThread::tlab_top_offset()); duke@435: int tlab_end_offset = in_bytes(JavaThread::tlab_end_offset()); duke@435: eden_top_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_top_offset); duke@435: eden_end_adr = basic_plus_adr(top()/*not oop*/, thread, tlab_end_offset); duke@435: } else { // Shared allocation: load from globals duke@435: CollectedHeap* ch = Universe::heap(); duke@435: address top_adr = (address)ch->top_addr(); duke@435: address end_adr = (address)ch->end_addr(); duke@435: eden_top_adr = makecon(TypeRawPtr::make(top_adr)); duke@435: eden_end_adr = basic_plus_adr(eden_top_adr, end_adr - top_adr); duke@435: } duke@435: } duke@435: duke@435: duke@435: Node* PhaseMacroExpand::make_load(Node* ctl, Node* mem, Node* base, int offset, const Type* value_type, BasicType bt) { duke@435: Node* adr = basic_plus_adr(base, offset); duke@435: const TypePtr* adr_type = TypeRawPtr::BOTTOM; duke@435: Node* value = LoadNode::make(C, ctl, mem, adr, adr_type, value_type, bt); duke@435: transform_later(value); duke@435: return value; duke@435: } duke@435: duke@435: duke@435: Node* PhaseMacroExpand::make_store(Node* ctl, Node* mem, Node* base, int offset, Node* value, BasicType bt) { duke@435: Node* adr = basic_plus_adr(base, offset); duke@435: mem = StoreNode::make(C, ctl, mem, adr, NULL, value, bt); duke@435: transform_later(mem); duke@435: return mem; duke@435: } duke@435: duke@435: //============================================================================= duke@435: // duke@435: // A L L O C A T I O N duke@435: // duke@435: // Allocation attempts to be fast in the case of frequent small objects. duke@435: // It breaks down like this: duke@435: // duke@435: // 1) Size in doublewords is computed. This is a constant for objects and duke@435: // variable for most arrays. Doubleword units are used to avoid size duke@435: // overflow of huge doubleword arrays. We need doublewords in the end for duke@435: // rounding. duke@435: // duke@435: // 2) Size is checked for being 'too large'. Too-large allocations will go duke@435: // the slow path into the VM. The slow path can throw any required duke@435: // exceptions, and does all the special checks for very large arrays. The duke@435: // size test can constant-fold away for objects. For objects with duke@435: // finalizers it constant-folds the otherway: you always go slow with duke@435: // finalizers. duke@435: // duke@435: // 3) If NOT using TLABs, this is the contended loop-back point. duke@435: // Load-Locked the heap top. If using TLABs normal-load the heap top. duke@435: // duke@435: // 4) Check that heap top + size*8 < max. If we fail go the slow ` route. duke@435: // NOTE: "top+size*8" cannot wrap the 4Gig line! Here's why: for largish duke@435: // "size*8" we always enter the VM, where "largish" is a constant picked small duke@435: // enough that there's always space between the eden max and 4Gig (old space is duke@435: // there so it's quite large) and large enough that the cost of entering the VM duke@435: // is dwarfed by the cost to initialize the space. duke@435: // duke@435: // 5) If NOT using TLABs, Store-Conditional the adjusted heap top back duke@435: // down. If contended, repeat at step 3. If using TLABs normal-store duke@435: // adjusted heap top back down; there is no contention. duke@435: // duke@435: // 6) If !ZeroTLAB then Bulk-clear the object/array. Fill in klass & mark duke@435: // fields. duke@435: // duke@435: // 7) Merge with the slow-path; cast the raw memory pointer to the correct duke@435: // oop flavor. duke@435: // duke@435: //============================================================================= duke@435: // FastAllocateSizeLimit value is in DOUBLEWORDS. duke@435: // Allocations bigger than this always go the slow route. duke@435: // This value must be small enough that allocation attempts that need to duke@435: // trigger exceptions go the slow route. Also, it must be small enough so duke@435: // that heap_top + size_in_bytes does not wrap around the 4Gig limit. duke@435: //=============================================================================j// duke@435: // %%% Here is an old comment from parseHelper.cpp; is it outdated? duke@435: // The allocator will coalesce int->oop copies away. See comment in duke@435: // coalesce.cpp about how this works. It depends critically on the exact duke@435: // code shape produced here, so if you are changing this code shape duke@435: // make sure the GC info for the heap-top is correct in and around the duke@435: // slow-path call. duke@435: // duke@435: duke@435: void PhaseMacroExpand::expand_allocate_common( duke@435: AllocateNode* alloc, // allocation node to be expanded duke@435: Node* length, // array length for an array allocation duke@435: const TypeFunc* slow_call_type, // Type of slow call duke@435: address slow_call_address // Address of slow call duke@435: ) duke@435: { duke@435: duke@435: Node* ctrl = alloc->in(TypeFunc::Control); duke@435: Node* mem = alloc->in(TypeFunc::Memory); duke@435: Node* i_o = alloc->in(TypeFunc::I_O); duke@435: Node* size_in_bytes = alloc->in(AllocateNode::AllocSize); duke@435: Node* klass_node = alloc->in(AllocateNode::KlassNode); duke@435: Node* initial_slow_test = alloc->in(AllocateNode::InitialTest); duke@435: duke@435: Node* eden_top_adr; duke@435: Node* eden_end_adr; duke@435: set_eden_pointers(eden_top_adr, eden_end_adr); duke@435: duke@435: uint raw_idx = C->get_alias_index(TypeRawPtr::BOTTOM); duke@435: assert(ctrl != NULL, "must have control"); duke@435: duke@435: // Load Eden::end. Loop invariant and hoisted. duke@435: // duke@435: // Note: We set the control input on "eden_end" and "old_eden_top" when using duke@435: // a TLAB to work around a bug where these values were being moved across duke@435: // a safepoint. These are not oops, so they cannot be include in the oop duke@435: // map, but the can be changed by a GC. The proper way to fix this would duke@435: // be to set the raw memory state when generating a SafepointNode. However duke@435: // this will require extensive changes to the loop optimization in order to duke@435: // prevent a degradation of the optimization. duke@435: // See comment in memnode.hpp, around line 227 in class LoadPNode. duke@435: Node* eden_end = make_load(ctrl, mem, eden_end_adr, 0, TypeRawPtr::BOTTOM, T_ADDRESS); duke@435: duke@435: // We need a Region and corresponding Phi's to merge the slow-path and fast-path results. duke@435: // they will not be used if "always_slow" is set duke@435: enum { slow_result_path = 1, fast_result_path = 2 }; duke@435: Node *result_region; duke@435: Node *result_phi_rawmem; duke@435: Node *result_phi_rawoop; duke@435: Node *result_phi_i_o; duke@435: duke@435: // The initial slow comparison is a size check, the comparison duke@435: // we want to do is a BoolTest::gt duke@435: bool always_slow = false; duke@435: int tv = _igvn.find_int_con(initial_slow_test, -1); duke@435: if (tv >= 0) { duke@435: always_slow = (tv == 1); duke@435: initial_slow_test = NULL; duke@435: } else { duke@435: initial_slow_test = BoolNode::make_predicate(initial_slow_test, &_igvn); duke@435: } duke@435: duke@435: if (DTraceAllocProbes) { duke@435: // Force slow-path allocation duke@435: always_slow = true; duke@435: initial_slow_test = NULL; duke@435: } duke@435: duke@435: enum { too_big_or_final_path = 1, need_gc_path = 2 }; duke@435: Node *slow_region = NULL; duke@435: Node *toobig_false = ctrl; duke@435: duke@435: assert (initial_slow_test == NULL || !always_slow, "arguments must be consistent"); duke@435: // generate the initial test if necessary duke@435: if (initial_slow_test != NULL ) { duke@435: slow_region = new (C, 3) RegionNode(3); duke@435: duke@435: // Now make the initial failure test. Usually a too-big test but duke@435: // might be a TRUE for finalizers or a fancy class check for duke@435: // newInstance0. duke@435: IfNode *toobig_iff = new (C, 2) IfNode(ctrl, initial_slow_test, PROB_MIN, COUNT_UNKNOWN); duke@435: transform_later(toobig_iff); duke@435: // Plug the failing-too-big test into the slow-path region duke@435: Node *toobig_true = new (C, 1) IfTrueNode( toobig_iff ); duke@435: transform_later(toobig_true); duke@435: slow_region ->init_req( too_big_or_final_path, toobig_true ); duke@435: toobig_false = new (C, 1) IfFalseNode( toobig_iff ); duke@435: transform_later(toobig_false); duke@435: } else { // No initial test, just fall into next case duke@435: toobig_false = ctrl; duke@435: debug_only(slow_region = NodeSentinel); duke@435: } duke@435: duke@435: Node *slow_mem = mem; // save the current memory state for slow path duke@435: // generate the fast allocation code unless we know that the initial test will always go slow duke@435: if (!always_slow) { duke@435: // allocate the Region and Phi nodes for the result duke@435: result_region = new (C, 3) RegionNode(3); duke@435: result_phi_rawmem = new (C, 3) PhiNode( result_region, Type::MEMORY, TypeRawPtr::BOTTOM ); duke@435: result_phi_rawoop = new (C, 3) PhiNode( result_region, TypeRawPtr::BOTTOM ); duke@435: result_phi_i_o = new (C, 3) PhiNode( result_region, Type::ABIO ); // I/O is used for Prefetch duke@435: duke@435: // We need a Region for the loop-back contended case. duke@435: enum { fall_in_path = 1, contended_loopback_path = 2 }; duke@435: Node *contended_region; duke@435: Node *contended_phi_rawmem; duke@435: if( UseTLAB ) { duke@435: contended_region = toobig_false; duke@435: contended_phi_rawmem = mem; duke@435: } else { duke@435: contended_region = new (C, 3) RegionNode(3); duke@435: contended_phi_rawmem = new (C, 3) PhiNode( contended_region, Type::MEMORY, TypeRawPtr::BOTTOM); duke@435: // Now handle the passing-too-big test. We fall into the contended duke@435: // loop-back merge point. duke@435: contended_region ->init_req( fall_in_path, toobig_false ); duke@435: contended_phi_rawmem->init_req( fall_in_path, mem ); duke@435: transform_later(contended_region); duke@435: transform_later(contended_phi_rawmem); duke@435: } duke@435: duke@435: // Load(-locked) the heap top. duke@435: // See note above concerning the control input when using a TLAB duke@435: Node *old_eden_top = UseTLAB duke@435: ? new (C, 3) LoadPNode ( ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM ) duke@435: : new (C, 3) LoadPLockedNode( contended_region, contended_phi_rawmem, eden_top_adr ); duke@435: duke@435: transform_later(old_eden_top); duke@435: // Add to heap top to get a new heap top duke@435: Node *new_eden_top = new (C, 4) AddPNode( top(), old_eden_top, size_in_bytes ); duke@435: transform_later(new_eden_top); duke@435: // Check for needing a GC; compare against heap end duke@435: Node *needgc_cmp = new (C, 3) CmpPNode( new_eden_top, eden_end ); duke@435: transform_later(needgc_cmp); duke@435: Node *needgc_bol = new (C, 2) BoolNode( needgc_cmp, BoolTest::ge ); duke@435: transform_later(needgc_bol); duke@435: IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN ); duke@435: transform_later(needgc_iff); duke@435: duke@435: // Plug the failing-heap-space-need-gc test into the slow-path region duke@435: Node *needgc_true = new (C, 1) IfTrueNode( needgc_iff ); duke@435: transform_later(needgc_true); duke@435: if( initial_slow_test ) { duke@435: slow_region ->init_req( need_gc_path, needgc_true ); duke@435: // This completes all paths into the slow merge point duke@435: transform_later(slow_region); duke@435: } else { // No initial slow path needed! duke@435: // Just fall from the need-GC path straight into the VM call. duke@435: slow_region = needgc_true; duke@435: } duke@435: // No need for a GC. Setup for the Store-Conditional duke@435: Node *needgc_false = new (C, 1) IfFalseNode( needgc_iff ); duke@435: transform_later(needgc_false); duke@435: duke@435: // Grab regular I/O before optional prefetch may change it. duke@435: // Slow-path does no I/O so just set it to the original I/O. duke@435: result_phi_i_o->init_req( slow_result_path, i_o ); duke@435: duke@435: i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem, duke@435: old_eden_top, new_eden_top, length); duke@435: duke@435: // Store (-conditional) the modified eden top back down. duke@435: // StorePConditional produces flags for a test PLUS a modified raw duke@435: // memory state. duke@435: Node *store_eden_top; duke@435: Node *fast_oop_ctrl; duke@435: if( UseTLAB ) { duke@435: store_eden_top = new (C, 4) StorePNode( needgc_false, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, new_eden_top ); duke@435: transform_later(store_eden_top); duke@435: fast_oop_ctrl = needgc_false; // No contention, so this is the fast path duke@435: } else { duke@435: store_eden_top = new (C, 5) StorePConditionalNode( needgc_false, contended_phi_rawmem, eden_top_adr, new_eden_top, old_eden_top ); duke@435: transform_later(store_eden_top); duke@435: Node *contention_check = new (C, 2) BoolNode( store_eden_top, BoolTest::ne ); duke@435: transform_later(contention_check); duke@435: store_eden_top = new (C, 1) SCMemProjNode(store_eden_top); duke@435: transform_later(store_eden_top); duke@435: duke@435: // If not using TLABs, check to see if there was contention. duke@435: IfNode *contention_iff = new (C, 2) IfNode ( needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN ); duke@435: transform_later(contention_iff); duke@435: Node *contention_true = new (C, 1) IfTrueNode( contention_iff ); duke@435: transform_later(contention_true); duke@435: // If contention, loopback and try again. duke@435: contended_region->init_req( contended_loopback_path, contention_true ); duke@435: contended_phi_rawmem->init_req( contended_loopback_path, store_eden_top ); duke@435: duke@435: // Fast-path succeeded with no contention! duke@435: Node *contention_false = new (C, 1) IfFalseNode( contention_iff ); duke@435: transform_later(contention_false); duke@435: fast_oop_ctrl = contention_false; duke@435: } duke@435: duke@435: // Rename successful fast-path variables to make meaning more obvious duke@435: Node* fast_oop = old_eden_top; duke@435: Node* fast_oop_rawmem = store_eden_top; duke@435: fast_oop_rawmem = initialize_object(alloc, duke@435: fast_oop_ctrl, fast_oop_rawmem, fast_oop, duke@435: klass_node, length, size_in_bytes); duke@435: duke@435: if (ExtendedDTraceProbes) { duke@435: // Slow-path call duke@435: int size = TypeFunc::Parms + 2; duke@435: CallLeafNode *call = new (C, size) CallLeafNode(OptoRuntime::dtrace_object_alloc_Type(), duke@435: CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc_base), duke@435: "dtrace_object_alloc", duke@435: TypeRawPtr::BOTTOM); duke@435: duke@435: // Get base of thread-local storage area duke@435: Node* thread = new (C, 1) ThreadLocalNode(); duke@435: transform_later(thread); duke@435: duke@435: call->init_req(TypeFunc::Parms+0, thread); duke@435: call->init_req(TypeFunc::Parms+1, fast_oop); duke@435: call->init_req( TypeFunc::Control, fast_oop_ctrl ); duke@435: call->init_req( TypeFunc::I_O , top() ) ; // does no i/o duke@435: call->init_req( TypeFunc::Memory , fast_oop_rawmem ); duke@435: call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) ); duke@435: call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) ); duke@435: transform_later(call); duke@435: fast_oop_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control); duke@435: transform_later(fast_oop_ctrl); duke@435: fast_oop_rawmem = new (C, 1) ProjNode(call,TypeFunc::Memory); duke@435: transform_later(fast_oop_rawmem); duke@435: } duke@435: duke@435: // Plug in the successful fast-path into the result merge point duke@435: result_region ->init_req( fast_result_path, fast_oop_ctrl ); duke@435: result_phi_rawoop->init_req( fast_result_path, fast_oop ); duke@435: result_phi_i_o ->init_req( fast_result_path, i_o ); duke@435: result_phi_rawmem->init_req( fast_result_path, fast_oop_rawmem ); duke@435: } else { duke@435: slow_region = ctrl; duke@435: } duke@435: duke@435: // Generate slow-path call duke@435: CallNode *call = new (C, slow_call_type->domain()->cnt()) duke@435: CallStaticJavaNode(slow_call_type, slow_call_address, duke@435: OptoRuntime::stub_name(slow_call_address), duke@435: alloc->jvms()->bci(), duke@435: TypePtr::BOTTOM); duke@435: call->init_req( TypeFunc::Control, slow_region ); duke@435: call->init_req( TypeFunc::I_O , top() ) ; // does no i/o duke@435: call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs duke@435: call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) ); duke@435: call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) ); duke@435: duke@435: call->init_req(TypeFunc::Parms+0, klass_node); duke@435: if (length != NULL) { duke@435: call->init_req(TypeFunc::Parms+1, length); duke@435: } duke@435: duke@435: // Copy debug information and adjust JVMState information, then replace duke@435: // allocate node with the call duke@435: copy_call_debug_info((CallNode *) alloc, call); duke@435: if (!always_slow) { duke@435: call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON. duke@435: } duke@435: _igvn.hash_delete(alloc); duke@435: _igvn.subsume_node(alloc, call); duke@435: transform_later(call); duke@435: duke@435: // Identify the output projections from the allocate node and duke@435: // adjust any references to them. duke@435: // The control and io projections look like: duke@435: // duke@435: // v---Proj(ctrl) <-----+ v---CatchProj(ctrl) duke@435: // Allocate Catch duke@435: // ^---Proj(io) <-------+ ^---CatchProj(io) duke@435: // duke@435: // We are interested in the CatchProj nodes. duke@435: // duke@435: extract_call_projections(call); duke@435: duke@435: // An allocate node has separate memory projections for the uses on the control and i_o paths duke@435: // Replace uses of the control memory projection with result_phi_rawmem (unless we are only generating a slow call) duke@435: if (!always_slow && _memproj_fallthrough != NULL) { duke@435: for (DUIterator_Fast imax, i = _memproj_fallthrough->fast_outs(imax); i < imax; i++) { duke@435: Node *use = _memproj_fallthrough->fast_out(i); duke@435: _igvn.hash_delete(use); duke@435: imax -= replace_input(use, _memproj_fallthrough, result_phi_rawmem); duke@435: _igvn._worklist.push(use); duke@435: // back up iterator duke@435: --i; duke@435: } duke@435: } duke@435: // Now change uses of _memproj_catchall to use _memproj_fallthrough and delete _memproj_catchall so duke@435: // we end up with a call that has only 1 memory projection duke@435: if (_memproj_catchall != NULL ) { duke@435: if (_memproj_fallthrough == NULL) { duke@435: _memproj_fallthrough = new (C, 1) ProjNode(call, TypeFunc::Memory); duke@435: transform_later(_memproj_fallthrough); duke@435: } duke@435: for (DUIterator_Fast imax, i = _memproj_catchall->fast_outs(imax); i < imax; i++) { duke@435: Node *use = _memproj_catchall->fast_out(i); duke@435: _igvn.hash_delete(use); duke@435: imax -= replace_input(use, _memproj_catchall, _memproj_fallthrough); duke@435: _igvn._worklist.push(use); duke@435: // back up iterator duke@435: --i; duke@435: } duke@435: } duke@435: duke@435: mem = result_phi_rawmem; duke@435: duke@435: // An allocate node has separate i_o projections for the uses on the control and i_o paths duke@435: // Replace uses of the control i_o projection with result_phi_i_o (unless we are only generating a slow call) duke@435: if (_ioproj_fallthrough == NULL) { duke@435: _ioproj_fallthrough = new (C, 1) ProjNode(call, TypeFunc::I_O); duke@435: transform_later(_ioproj_fallthrough); duke@435: } else if (!always_slow) { duke@435: for (DUIterator_Fast imax, i = _ioproj_fallthrough->fast_outs(imax); i < imax; i++) { duke@435: Node *use = _ioproj_fallthrough->fast_out(i); duke@435: duke@435: _igvn.hash_delete(use); duke@435: imax -= replace_input(use, _ioproj_fallthrough, result_phi_i_o); duke@435: _igvn._worklist.push(use); duke@435: // back up iterator duke@435: --i; duke@435: } duke@435: } duke@435: // Now change uses of _ioproj_catchall to use _ioproj_fallthrough and delete _ioproj_catchall so duke@435: // we end up with a call that has only 1 control projection duke@435: if (_ioproj_catchall != NULL ) { duke@435: for (DUIterator_Fast imax, i = _ioproj_catchall->fast_outs(imax); i < imax; i++) { duke@435: Node *use = _ioproj_catchall->fast_out(i); duke@435: _igvn.hash_delete(use); duke@435: imax -= replace_input(use, _ioproj_catchall, _ioproj_fallthrough); duke@435: _igvn._worklist.push(use); duke@435: // back up iterator duke@435: --i; duke@435: } duke@435: } duke@435: duke@435: // if we generated only a slow call, we are done duke@435: if (always_slow) duke@435: return; duke@435: duke@435: duke@435: if (_fallthroughcatchproj != NULL) { duke@435: ctrl = _fallthroughcatchproj->clone(); duke@435: transform_later(ctrl); duke@435: _igvn.hash_delete(_fallthroughcatchproj); duke@435: _igvn.subsume_node(_fallthroughcatchproj, result_region); duke@435: } else { duke@435: ctrl = top(); duke@435: } duke@435: Node *slow_result; duke@435: if (_resproj == NULL) { duke@435: // no uses of the allocation result duke@435: slow_result = top(); duke@435: } else { duke@435: slow_result = _resproj->clone(); duke@435: transform_later(slow_result); duke@435: _igvn.hash_delete(_resproj); duke@435: _igvn.subsume_node(_resproj, result_phi_rawoop); duke@435: } duke@435: duke@435: // Plug slow-path into result merge point duke@435: result_region ->init_req( slow_result_path, ctrl ); duke@435: result_phi_rawoop->init_req( slow_result_path, slow_result); duke@435: result_phi_rawmem->init_req( slow_result_path, _memproj_fallthrough ); duke@435: transform_later(result_region); duke@435: transform_later(result_phi_rawoop); duke@435: transform_later(result_phi_rawmem); duke@435: transform_later(result_phi_i_o); duke@435: // This completes all paths into the result merge point duke@435: } duke@435: duke@435: duke@435: // Helper for PhaseMacroExpand::expand_allocate_common. duke@435: // Initializes the newly-allocated storage. duke@435: Node* duke@435: PhaseMacroExpand::initialize_object(AllocateNode* alloc, duke@435: Node* control, Node* rawmem, Node* object, duke@435: Node* klass_node, Node* length, duke@435: Node* size_in_bytes) { duke@435: InitializeNode* init = alloc->initialization(); duke@435: // Store the klass & mark bits duke@435: Node* mark_node = NULL; duke@435: // For now only enable fast locking for non-array types duke@435: if (UseBiasedLocking && (length == NULL)) { duke@435: mark_node = make_load(NULL, rawmem, klass_node, Klass::prototype_header_offset_in_bytes() + sizeof(oopDesc), TypeRawPtr::BOTTOM, T_ADDRESS); duke@435: } else { duke@435: mark_node = makecon(TypeRawPtr::make((address)markOopDesc::prototype())); duke@435: } duke@435: rawmem = make_store(control, rawmem, object, oopDesc::mark_offset_in_bytes(), mark_node, T_ADDRESS); duke@435: rawmem = make_store(control, rawmem, object, oopDesc::klass_offset_in_bytes(), klass_node, T_OBJECT); duke@435: int header_size = alloc->minimum_header_size(); // conservatively small duke@435: duke@435: // Array length duke@435: if (length != NULL) { // Arrays need length field duke@435: rawmem = make_store(control, rawmem, object, arrayOopDesc::length_offset_in_bytes(), length, T_INT); duke@435: // conservatively small header size: duke@435: header_size = sizeof(arrayOopDesc); duke@435: ciKlass* k = _igvn.type(klass_node)->is_klassptr()->klass(); duke@435: if (k->is_array_klass()) // we know the exact header size in most cases: duke@435: header_size = Klass::layout_helper_header_size(k->layout_helper()); duke@435: } duke@435: duke@435: // Clear the object body, if necessary. duke@435: if (init == NULL) { duke@435: // The init has somehow disappeared; be cautious and clear everything. duke@435: // duke@435: // This can happen if a node is allocated but an uncommon trap occurs duke@435: // immediately. In this case, the Initialize gets associated with the duke@435: // trap, and may be placed in a different (outer) loop, if the Allocate duke@435: // is in a loop. If (this is rare) the inner loop gets unrolled, then duke@435: // there can be two Allocates to one Initialize. The answer in all these duke@435: // edge cases is safety first. It is always safe to clear immediately duke@435: // within an Allocate, and then (maybe or maybe not) clear some more later. duke@435: if (!ZeroTLAB) duke@435: rawmem = ClearArrayNode::clear_memory(control, rawmem, object, duke@435: header_size, size_in_bytes, duke@435: &_igvn); duke@435: } else { duke@435: if (!init->is_complete()) { duke@435: // Try to win by zeroing only what the init does not store. duke@435: // We can also try to do some peephole optimizations, duke@435: // such as combining some adjacent subword stores. duke@435: rawmem = init->complete_stores(control, rawmem, object, duke@435: header_size, size_in_bytes, &_igvn); duke@435: } duke@435: duke@435: // We have no more use for this link, since the AllocateNode goes away: duke@435: init->set_req(InitializeNode::RawAddress, top()); duke@435: // (If we keep the link, it just confuses the register allocator, duke@435: // who thinks he sees a real use of the address by the membar.) duke@435: } duke@435: duke@435: return rawmem; duke@435: } duke@435: duke@435: // Generate prefetch instructions for next allocations. duke@435: Node* PhaseMacroExpand::prefetch_allocation(Node* i_o, Node*& needgc_false, duke@435: Node*& contended_phi_rawmem, duke@435: Node* old_eden_top, Node* new_eden_top, duke@435: Node* length) { duke@435: if( UseTLAB && AllocatePrefetchStyle == 2 ) { duke@435: // Generate prefetch allocation with watermark check. duke@435: // As an allocation hits the watermark, we will prefetch starting duke@435: // at a "distance" away from watermark. duke@435: enum { fall_in_path = 1, pf_path = 2 }; duke@435: duke@435: Node *pf_region = new (C, 3) RegionNode(3); duke@435: Node *pf_phi_rawmem = new (C, 3) PhiNode( pf_region, Type::MEMORY, duke@435: TypeRawPtr::BOTTOM ); duke@435: // I/O is used for Prefetch duke@435: Node *pf_phi_abio = new (C, 3) PhiNode( pf_region, Type::ABIO ); duke@435: duke@435: Node *thread = new (C, 1) ThreadLocalNode(); duke@435: transform_later(thread); duke@435: duke@435: Node *eden_pf_adr = new (C, 4) AddPNode( top()/*not oop*/, thread, duke@435: _igvn.MakeConX(in_bytes(JavaThread::tlab_pf_top_offset())) ); duke@435: transform_later(eden_pf_adr); duke@435: duke@435: Node *old_pf_wm = new (C, 3) LoadPNode( needgc_false, duke@435: contended_phi_rawmem, eden_pf_adr, duke@435: TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM ); duke@435: transform_later(old_pf_wm); duke@435: duke@435: // check against new_eden_top duke@435: Node *need_pf_cmp = new (C, 3) CmpPNode( new_eden_top, old_pf_wm ); duke@435: transform_later(need_pf_cmp); duke@435: Node *need_pf_bol = new (C, 2) BoolNode( need_pf_cmp, BoolTest::ge ); duke@435: transform_later(need_pf_bol); duke@435: IfNode *need_pf_iff = new (C, 2) IfNode( needgc_false, need_pf_bol, duke@435: PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN ); duke@435: transform_later(need_pf_iff); duke@435: duke@435: // true node, add prefetchdistance duke@435: Node *need_pf_true = new (C, 1) IfTrueNode( need_pf_iff ); duke@435: transform_later(need_pf_true); duke@435: duke@435: Node *need_pf_false = new (C, 1) IfFalseNode( need_pf_iff ); duke@435: transform_later(need_pf_false); duke@435: duke@435: Node *new_pf_wmt = new (C, 4) AddPNode( top(), old_pf_wm, duke@435: _igvn.MakeConX(AllocatePrefetchDistance) ); duke@435: transform_later(new_pf_wmt ); duke@435: new_pf_wmt->set_req(0, need_pf_true); duke@435: duke@435: Node *store_new_wmt = new (C, 4) StorePNode( need_pf_true, duke@435: contended_phi_rawmem, eden_pf_adr, duke@435: TypeRawPtr::BOTTOM, new_pf_wmt ); duke@435: transform_later(store_new_wmt); duke@435: duke@435: // adding prefetches duke@435: pf_phi_abio->init_req( fall_in_path, i_o ); duke@435: duke@435: Node *prefetch_adr; duke@435: Node *prefetch; duke@435: uint lines = AllocatePrefetchDistance / AllocatePrefetchStepSize; duke@435: uint step_size = AllocatePrefetchStepSize; duke@435: uint distance = 0; duke@435: duke@435: for ( uint i = 0; i < lines; i++ ) { duke@435: prefetch_adr = new (C, 4) AddPNode( old_pf_wm, new_pf_wmt, duke@435: _igvn.MakeConX(distance) ); duke@435: transform_later(prefetch_adr); duke@435: prefetch = new (C, 3) PrefetchWriteNode( i_o, prefetch_adr ); duke@435: transform_later(prefetch); duke@435: distance += step_size; duke@435: i_o = prefetch; duke@435: } duke@435: pf_phi_abio->set_req( pf_path, i_o ); duke@435: duke@435: pf_region->init_req( fall_in_path, need_pf_false ); duke@435: pf_region->init_req( pf_path, need_pf_true ); duke@435: duke@435: pf_phi_rawmem->init_req( fall_in_path, contended_phi_rawmem ); duke@435: pf_phi_rawmem->init_req( pf_path, store_new_wmt ); duke@435: duke@435: transform_later(pf_region); duke@435: transform_later(pf_phi_rawmem); duke@435: transform_later(pf_phi_abio); duke@435: duke@435: needgc_false = pf_region; duke@435: contended_phi_rawmem = pf_phi_rawmem; duke@435: i_o = pf_phi_abio; duke@435: } else if( AllocatePrefetchStyle > 0 ) { duke@435: // Insert a prefetch for each allocation only on the fast-path duke@435: Node *prefetch_adr; duke@435: Node *prefetch; duke@435: // Generate several prefetch instructions only for arrays. duke@435: uint lines = (length != NULL) ? AllocatePrefetchLines : 1; duke@435: uint step_size = AllocatePrefetchStepSize; duke@435: uint distance = AllocatePrefetchDistance; duke@435: for ( uint i = 0; i < lines; i++ ) { duke@435: prefetch_adr = new (C, 4) AddPNode( old_eden_top, new_eden_top, duke@435: _igvn.MakeConX(distance) ); duke@435: transform_later(prefetch_adr); duke@435: prefetch = new (C, 3) PrefetchWriteNode( i_o, prefetch_adr ); duke@435: // Do not let it float too high, since if eden_top == eden_end, duke@435: // both might be null. duke@435: if( i == 0 ) { // Set control for first prefetch, next follows it duke@435: prefetch->init_req(0, needgc_false); duke@435: } duke@435: transform_later(prefetch); duke@435: distance += step_size; duke@435: i_o = prefetch; duke@435: } duke@435: } duke@435: return i_o; duke@435: } duke@435: duke@435: duke@435: void PhaseMacroExpand::expand_allocate(AllocateNode *alloc) { duke@435: expand_allocate_common(alloc, NULL, duke@435: OptoRuntime::new_instance_Type(), duke@435: OptoRuntime::new_instance_Java()); duke@435: } duke@435: duke@435: void PhaseMacroExpand::expand_allocate_array(AllocateArrayNode *alloc) { duke@435: Node* length = alloc->in(AllocateNode::ALength); duke@435: expand_allocate_common(alloc, length, duke@435: OptoRuntime::new_array_Type(), duke@435: OptoRuntime::new_array_Java()); duke@435: } duke@435: duke@435: duke@435: // we have determined that this lock/unlock can be eliminated, we simply duke@435: // eliminate the node without expanding it. duke@435: // duke@435: // Note: The membar's associated with the lock/unlock are currently not duke@435: // eliminated. This should be investigated as a future enhancement. duke@435: // duke@435: void PhaseMacroExpand::eliminate_locking_node(AbstractLockNode *alock) { duke@435: Node* mem = alock->in(TypeFunc::Memory); duke@435: duke@435: // The memory projection from a lock/unlock is RawMem duke@435: // The input to a Lock is merged memory, so extract its RawMem input duke@435: // (unless the MergeMem has been optimized away.) duke@435: if (alock->is_Lock()) { duke@435: if (mem->is_MergeMem()) duke@435: mem = mem->as_MergeMem()->in(Compile::AliasIdxRaw); duke@435: } duke@435: duke@435: extract_call_projections(alock); duke@435: // There are 2 projections from the lock. The lock node will duke@435: // be deleted when its last use is subsumed below. duke@435: assert(alock->outcnt() == 2 && _fallthroughproj != NULL && duke@435: _memproj_fallthrough != NULL, "Unexpected projections from Lock/Unlock"); duke@435: _igvn.hash_delete(_fallthroughproj); duke@435: _igvn.subsume_node(_fallthroughproj, alock->in(TypeFunc::Control)); duke@435: _igvn.hash_delete(_memproj_fallthrough); duke@435: _igvn.subsume_node(_memproj_fallthrough, mem); duke@435: return; duke@435: } duke@435: duke@435: duke@435: //------------------------------expand_lock_node---------------------- duke@435: void PhaseMacroExpand::expand_lock_node(LockNode *lock) { duke@435: duke@435: Node* ctrl = lock->in(TypeFunc::Control); duke@435: Node* mem = lock->in(TypeFunc::Memory); duke@435: Node* obj = lock->obj_node(); duke@435: Node* box = lock->box_node(); duke@435: Node *flock = lock->fastlock_node(); duke@435: duke@435: if (lock->is_eliminated()) { duke@435: eliminate_locking_node(lock); duke@435: return; duke@435: } duke@435: duke@435: // Make the merge point duke@435: Node *region = new (C, 3) RegionNode(3); duke@435: duke@435: Node *bol = transform_later(new (C, 2) BoolNode(flock,BoolTest::ne)); duke@435: Node *iff = new (C, 2) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN ); duke@435: // Optimize test; set region slot 2 duke@435: Node *slow_path = opt_iff(region,iff); duke@435: duke@435: // Make slow path call duke@435: CallNode *call = make_slow_call( (CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(), OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path, obj, box ); duke@435: duke@435: extract_call_projections(call); duke@435: duke@435: // Slow path can only throw asynchronous exceptions, which are always duke@435: // de-opted. So the compiler thinks the slow-call can never throw an duke@435: // exception. If it DOES throw an exception we would need the debug duke@435: // info removed first (since if it throws there is no monitor). duke@435: assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL && duke@435: _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock"); duke@435: duke@435: // Capture slow path duke@435: // disconnect fall-through projection from call and create a new one duke@435: // hook up users of fall-through projection to region duke@435: Node *slow_ctrl = _fallthroughproj->clone(); duke@435: transform_later(slow_ctrl); duke@435: _igvn.hash_delete(_fallthroughproj); duke@435: _fallthroughproj->disconnect_inputs(NULL); duke@435: region->init_req(1, slow_ctrl); duke@435: // region inputs are now complete duke@435: transform_later(region); duke@435: _igvn.subsume_node(_fallthroughproj, region); duke@435: duke@435: // create a Phi for the memory state duke@435: Node *mem_phi = new (C, 3) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); duke@435: Node *memproj = transform_later( new (C, 1) ProjNode(call, TypeFunc::Memory) ); duke@435: mem_phi->init_req(1, memproj ); duke@435: mem_phi->init_req(2, mem); duke@435: transform_later(mem_phi); duke@435: _igvn.hash_delete(_memproj_fallthrough); duke@435: _igvn.subsume_node(_memproj_fallthrough, mem_phi); duke@435: duke@435: duke@435: } duke@435: duke@435: //------------------------------expand_unlock_node---------------------- duke@435: void PhaseMacroExpand::expand_unlock_node(UnlockNode *unlock) { duke@435: duke@435: Node *ctrl = unlock->in(TypeFunc::Control); duke@435: Node* mem = unlock->in(TypeFunc::Memory); duke@435: Node* obj = unlock->obj_node(); duke@435: Node* box = unlock->box_node(); duke@435: duke@435: duke@435: if (unlock->is_eliminated()) { duke@435: eliminate_locking_node(unlock); duke@435: return; duke@435: } duke@435: duke@435: // No need for a null check on unlock duke@435: duke@435: // Make the merge point duke@435: RegionNode *region = new (C, 3) RegionNode(3); duke@435: duke@435: FastUnlockNode *funlock = new (C, 3) FastUnlockNode( ctrl, obj, box ); duke@435: funlock = transform_later( funlock )->as_FastUnlock(); duke@435: Node *bol = transform_later(new (C, 2) BoolNode(funlock,BoolTest::ne)); duke@435: Node *iff = new (C, 2) IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN ); duke@435: // Optimize test; set region slot 2 duke@435: Node *slow_path = opt_iff(region,iff); duke@435: duke@435: CallNode *call = make_slow_call( (CallNode *) unlock, OptoRuntime::complete_monitor_exit_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), "complete_monitor_unlocking_C", slow_path, obj, box ); duke@435: duke@435: extract_call_projections(call); duke@435: duke@435: assert ( _ioproj_fallthrough == NULL && _ioproj_catchall == NULL && duke@435: _memproj_catchall == NULL && _catchallcatchproj == NULL, "Unexpected projection from Lock"); duke@435: duke@435: // No exceptions for unlocking duke@435: // Capture slow path duke@435: // disconnect fall-through projection from call and create a new one duke@435: // hook up users of fall-through projection to region duke@435: Node *slow_ctrl = _fallthroughproj->clone(); duke@435: transform_later(slow_ctrl); duke@435: _igvn.hash_delete(_fallthroughproj); duke@435: _fallthroughproj->disconnect_inputs(NULL); duke@435: region->init_req(1, slow_ctrl); duke@435: // region inputs are now complete duke@435: transform_later(region); duke@435: _igvn.subsume_node(_fallthroughproj, region); duke@435: duke@435: // create a Phi for the memory state duke@435: Node *mem_phi = new (C, 3) PhiNode( region, Type::MEMORY, TypeRawPtr::BOTTOM); duke@435: Node *memproj = transform_later( new(C, 1) ProjNode(call, TypeFunc::Memory) ); duke@435: mem_phi->init_req(1, memproj ); duke@435: mem_phi->init_req(2, mem); duke@435: transform_later(mem_phi); duke@435: _igvn.hash_delete(_memproj_fallthrough); duke@435: _igvn.subsume_node(_memproj_fallthrough, mem_phi); duke@435: duke@435: duke@435: } duke@435: duke@435: //------------------------------expand_macro_nodes---------------------- duke@435: // Returns true if a failure occurred. duke@435: bool PhaseMacroExpand::expand_macro_nodes() { duke@435: if (C->macro_count() == 0) duke@435: return false; duke@435: // Make sure expansion will not cause node limit to be exceeded. Worst case is a duke@435: // macro node gets expanded into about 50 nodes. Allow 50% more for optimization duke@435: if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) ) duke@435: return true; duke@435: // expand "macro" nodes duke@435: // nodes are removed from the macro list as they are processed duke@435: while (C->macro_count() > 0) { duke@435: Node * n = C->macro_node(0); duke@435: assert(n->is_macro(), "only macro nodes expected here"); duke@435: if (_igvn.type(n) == Type::TOP || n->in(0)->is_top() ) { duke@435: // node is unreachable, so don't try to expand it duke@435: C->remove_macro_node(n); duke@435: continue; duke@435: } duke@435: switch (n->class_id()) { duke@435: case Node::Class_Allocate: duke@435: expand_allocate(n->as_Allocate()); duke@435: break; duke@435: case Node::Class_AllocateArray: duke@435: expand_allocate_array(n->as_AllocateArray()); duke@435: break; duke@435: case Node::Class_Lock: duke@435: expand_lock_node(n->as_Lock()); duke@435: break; duke@435: case Node::Class_Unlock: duke@435: expand_unlock_node(n->as_Unlock()); duke@435: break; duke@435: default: duke@435: assert(false, "unknown node type in macro list"); duke@435: } duke@435: if (C->failing()) return true; duke@435: } duke@435: _igvn.optimize(); duke@435: return false; duke@435: }