1.1 --- a/src/share/vm/opto/macro.cpp Fri Jan 07 03:38:19 2011 -0800
1.2 +++ b/src/share/vm/opto/macro.cpp Fri Jan 07 10:42:32 2011 -0500
1.3 @@ -1,5 +1,5 @@
1.4 /*
1.5 - * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
1.6 + * Copyright (c) 2005, 2011, Oracle and/or its affiliates. All rights reserved.
1.7 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.8 *
1.9 * This code is free software; you can redistribute it and/or modify it
1.10 @@ -1158,7 +1158,7 @@
1.11 // Note: We set the control input on "eden_end" and "old_eden_top" when using
1.12 // a TLAB to work around a bug where these values were being moved across
1.13 // a safepoint. These are not oops, so they cannot be include in the oop
1.14 - // map, but the can be changed by a GC. The proper way to fix this would
1.15 + // map, but they can be changed by a GC. The proper way to fix this would
1.16 // be to set the raw memory state when generating a SafepointNode. However
1.17 // this will require extensive changes to the loop optimization in order to
1.18 // prevent a degradation of the optimization.
1.19 @@ -1167,24 +1167,24 @@
1.20
1.21 // allocate the Region and Phi nodes for the result
1.22 result_region = new (C, 3) RegionNode(3);
1.23 - result_phi_rawmem = new (C, 3) PhiNode( result_region, Type::MEMORY, TypeRawPtr::BOTTOM );
1.24 - result_phi_rawoop = new (C, 3) PhiNode( result_region, TypeRawPtr::BOTTOM );
1.25 - result_phi_i_o = new (C, 3) PhiNode( result_region, Type::ABIO ); // I/O is used for Prefetch
1.26 + result_phi_rawmem = new (C, 3) PhiNode(result_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1.27 + result_phi_rawoop = new (C, 3) PhiNode(result_region, TypeRawPtr::BOTTOM);
1.28 + result_phi_i_o = new (C, 3) PhiNode(result_region, Type::ABIO); // I/O is used for Prefetch
1.29
1.30 // We need a Region for the loop-back contended case.
1.31 enum { fall_in_path = 1, contended_loopback_path = 2 };
1.32 Node *contended_region;
1.33 Node *contended_phi_rawmem;
1.34 - if( UseTLAB ) {
1.35 + if (UseTLAB) {
1.36 contended_region = toobig_false;
1.37 contended_phi_rawmem = mem;
1.38 } else {
1.39 contended_region = new (C, 3) RegionNode(3);
1.40 - contended_phi_rawmem = new (C, 3) PhiNode( contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1.41 + contended_phi_rawmem = new (C, 3) PhiNode(contended_region, Type::MEMORY, TypeRawPtr::BOTTOM);
1.42 // Now handle the passing-too-big test. We fall into the contended
1.43 // loop-back merge point.
1.44 - contended_region ->init_req( fall_in_path, toobig_false );
1.45 - contended_phi_rawmem->init_req( fall_in_path, mem );
1.46 + contended_region ->init_req(fall_in_path, toobig_false);
1.47 + contended_phi_rawmem->init_req(fall_in_path, mem);
1.48 transform_later(contended_region);
1.49 transform_later(contended_phi_rawmem);
1.50 }
1.51 @@ -1192,78 +1192,101 @@
1.52 // Load(-locked) the heap top.
1.53 // See note above concerning the control input when using a TLAB
1.54 Node *old_eden_top = UseTLAB
1.55 - ? new (C, 3) LoadPNode ( ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM )
1.56 - : new (C, 3) LoadPLockedNode( contended_region, contended_phi_rawmem, eden_top_adr );
1.57 + ? new (C, 3) LoadPNode (ctrl, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, TypeRawPtr::BOTTOM)
1.58 + : new (C, 3) LoadPLockedNode(contended_region, contended_phi_rawmem, eden_top_adr);
1.59
1.60 transform_later(old_eden_top);
1.61 // Add to heap top to get a new heap top
1.62 - Node *new_eden_top = new (C, 4) AddPNode( top(), old_eden_top, size_in_bytes );
1.63 + Node *new_eden_top = new (C, 4) AddPNode(top(), old_eden_top, size_in_bytes);
1.64 transform_later(new_eden_top);
1.65 // Check for needing a GC; compare against heap end
1.66 - Node *needgc_cmp = new (C, 3) CmpPNode( new_eden_top, eden_end );
1.67 + Node *needgc_cmp = new (C, 3) CmpPNode(new_eden_top, eden_end);
1.68 transform_later(needgc_cmp);
1.69 - Node *needgc_bol = new (C, 2) BoolNode( needgc_cmp, BoolTest::ge );
1.70 + Node *needgc_bol = new (C, 2) BoolNode(needgc_cmp, BoolTest::ge);
1.71 transform_later(needgc_bol);
1.72 - IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN );
1.73 + IfNode *needgc_iff = new (C, 2) IfNode(contended_region, needgc_bol, PROB_UNLIKELY_MAG(4), COUNT_UNKNOWN);
1.74 transform_later(needgc_iff);
1.75
1.76 // Plug the failing-heap-space-need-gc test into the slow-path region
1.77 - Node *needgc_true = new (C, 1) IfTrueNode( needgc_iff );
1.78 + Node *needgc_true = new (C, 1) IfTrueNode(needgc_iff);
1.79 transform_later(needgc_true);
1.80 - if( initial_slow_test ) {
1.81 - slow_region ->init_req( need_gc_path, needgc_true );
1.82 + if (initial_slow_test) {
1.83 + slow_region->init_req(need_gc_path, needgc_true);
1.84 // This completes all paths into the slow merge point
1.85 transform_later(slow_region);
1.86 } else { // No initial slow path needed!
1.87 // Just fall from the need-GC path straight into the VM call.
1.88 - slow_region = needgc_true;
1.89 + slow_region = needgc_true;
1.90 }
1.91 // No need for a GC. Setup for the Store-Conditional
1.92 - Node *needgc_false = new (C, 1) IfFalseNode( needgc_iff );
1.93 + Node *needgc_false = new (C, 1) IfFalseNode(needgc_iff);
1.94 transform_later(needgc_false);
1.95
1.96 // Grab regular I/O before optional prefetch may change it.
1.97 // Slow-path does no I/O so just set it to the original I/O.
1.98 - result_phi_i_o->init_req( slow_result_path, i_o );
1.99 + result_phi_i_o->init_req(slow_result_path, i_o);
1.100
1.101 i_o = prefetch_allocation(i_o, needgc_false, contended_phi_rawmem,
1.102 old_eden_top, new_eden_top, length);
1.103
1.104 + // Name successful fast-path variables
1.105 + Node* fast_oop = old_eden_top;
1.106 + Node* fast_oop_ctrl;
1.107 + Node* fast_oop_rawmem;
1.108 +
1.109 // Store (-conditional) the modified eden top back down.
1.110 // StorePConditional produces flags for a test PLUS a modified raw
1.111 // memory state.
1.112 - Node *store_eden_top;
1.113 - Node *fast_oop_ctrl;
1.114 - if( UseTLAB ) {
1.115 - store_eden_top = new (C, 4) StorePNode( needgc_false, contended_phi_rawmem, eden_top_adr, TypeRawPtr::BOTTOM, new_eden_top );
1.116 + if (UseTLAB) {
1.117 + Node* store_eden_top =
1.118 + new (C, 4) StorePNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1.119 + TypeRawPtr::BOTTOM, new_eden_top);
1.120 transform_later(store_eden_top);
1.121 fast_oop_ctrl = needgc_false; // No contention, so this is the fast path
1.122 + fast_oop_rawmem = store_eden_top;
1.123 } else {
1.124 - store_eden_top = new (C, 5) StorePConditionalNode( needgc_false, contended_phi_rawmem, eden_top_adr, new_eden_top, old_eden_top );
1.125 + Node* store_eden_top =
1.126 + new (C, 5) StorePConditionalNode(needgc_false, contended_phi_rawmem, eden_top_adr,
1.127 + new_eden_top, fast_oop/*old_eden_top*/);
1.128 transform_later(store_eden_top);
1.129 - Node *contention_check = new (C, 2) BoolNode( store_eden_top, BoolTest::ne );
1.130 + Node *contention_check = new (C, 2) BoolNode(store_eden_top, BoolTest::ne);
1.131 transform_later(contention_check);
1.132 store_eden_top = new (C, 1) SCMemProjNode(store_eden_top);
1.133 transform_later(store_eden_top);
1.134
1.135 // If not using TLABs, check to see if there was contention.
1.136 - IfNode *contention_iff = new (C, 2) IfNode ( needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN );
1.137 + IfNode *contention_iff = new (C, 2) IfNode (needgc_false, contention_check, PROB_MIN, COUNT_UNKNOWN);
1.138 transform_later(contention_iff);
1.139 - Node *contention_true = new (C, 1) IfTrueNode( contention_iff );
1.140 + Node *contention_true = new (C, 1) IfTrueNode(contention_iff);
1.141 transform_later(contention_true);
1.142 // If contention, loopback and try again.
1.143 - contended_region->init_req( contended_loopback_path, contention_true );
1.144 - contended_phi_rawmem->init_req( contended_loopback_path, store_eden_top );
1.145 + contended_region->init_req(contended_loopback_path, contention_true);
1.146 + contended_phi_rawmem->init_req(contended_loopback_path, store_eden_top);
1.147
1.148 // Fast-path succeeded with no contention!
1.149 - Node *contention_false = new (C, 1) IfFalseNode( contention_iff );
1.150 + Node *contention_false = new (C, 1) IfFalseNode(contention_iff);
1.151 transform_later(contention_false);
1.152 fast_oop_ctrl = contention_false;
1.153 +
1.154 + // Bump total allocated bytes for this thread
1.155 + Node* thread = new (C, 1) ThreadLocalNode();
1.156 + transform_later(thread);
1.157 + Node* alloc_bytes_adr = basic_plus_adr(top()/*not oop*/, thread,
1.158 + in_bytes(JavaThread::allocated_bytes_offset()));
1.159 + Node* alloc_bytes = make_load(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1.160 + 0, TypeLong::LONG, T_LONG);
1.161 +#ifdef _LP64
1.162 + Node* alloc_size = size_in_bytes;
1.163 +#else
1.164 + Node* alloc_size = new (C, 2) ConvI2LNode(size_in_bytes);
1.165 + transform_later(alloc_size);
1.166 +#endif
1.167 + Node* new_alloc_bytes = new (C, 3) AddLNode(alloc_bytes, alloc_size);
1.168 + transform_later(new_alloc_bytes);
1.169 + fast_oop_rawmem = make_store(fast_oop_ctrl, store_eden_top, alloc_bytes_adr,
1.170 + 0, new_alloc_bytes, T_LONG);
1.171 }
1.172
1.173 - // Rename successful fast-path variables to make meaning more obvious
1.174 - Node* fast_oop = old_eden_top;
1.175 - Node* fast_oop_rawmem = store_eden_top;
1.176 fast_oop_rawmem = initialize_object(alloc,
1.177 fast_oop_ctrl, fast_oop_rawmem, fast_oop,
1.178 klass_node, length, size_in_bytes);
1.179 @@ -1282,11 +1305,11 @@
1.180
1.181 call->init_req(TypeFunc::Parms+0, thread);
1.182 call->init_req(TypeFunc::Parms+1, fast_oop);
1.183 - call->init_req( TypeFunc::Control, fast_oop_ctrl );
1.184 - call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
1.185 - call->init_req( TypeFunc::Memory , fast_oop_rawmem );
1.186 - call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1.187 - call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1.188 + call->init_req(TypeFunc::Control, fast_oop_ctrl);
1.189 + call->init_req(TypeFunc::I_O , top()); // does no i/o
1.190 + call->init_req(TypeFunc::Memory , fast_oop_rawmem);
1.191 + call->init_req(TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr));
1.192 + call->init_req(TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr));
1.193 transform_later(call);
1.194 fast_oop_ctrl = new (C, 1) ProjNode(call,TypeFunc::Control);
1.195 transform_later(fast_oop_ctrl);
1.196 @@ -1295,10 +1318,10 @@
1.197 }
1.198
1.199 // Plug in the successful fast-path into the result merge point
1.200 - result_region ->init_req( fast_result_path, fast_oop_ctrl );
1.201 - result_phi_rawoop->init_req( fast_result_path, fast_oop );
1.202 - result_phi_i_o ->init_req( fast_result_path, i_o );
1.203 - result_phi_rawmem->init_req( fast_result_path, fast_oop_rawmem );
1.204 + result_region ->init_req(fast_result_path, fast_oop_ctrl);
1.205 + result_phi_rawoop->init_req(fast_result_path, fast_oop);
1.206 + result_phi_i_o ->init_req(fast_result_path, i_o);
1.207 + result_phi_rawmem->init_req(fast_result_path, fast_oop_rawmem);
1.208 } else {
1.209 slow_region = ctrl;
1.210 }
