1.1 --- a/src/share/vm/opto/graphKit.cpp Wed Jul 15 13:37:35 2009 -0700
1.2 +++ b/src/share/vm/opto/graphKit.cpp Thu Jul 16 14:10:42 2009 -0700
1.3 @@ -1373,11 +1373,12 @@
1.4 return st;
1.5 }
1.6
1.7 +
1.8 void GraphKit::pre_barrier(Node* ctl,
1.9 Node* obj,
1.10 Node* adr,
1.11 - uint adr_idx,
1.12 - Node *val,
1.13 + uint adr_idx,
1.14 + Node* val,
1.15 const TypeOopPtr* val_type,
1.16 BasicType bt) {
1.17 BarrierSet* bs = Universe::heap()->barrier_set();
1.18 @@ -1385,7 +1386,7 @@
1.19 switch (bs->kind()) {
1.20 case BarrierSet::G1SATBCT:
1.21 case BarrierSet::G1SATBCTLogging:
1.22 - g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt);
1.23 + g1_write_barrier_pre(obj, adr, adr_idx, val, val_type, bt);
1.24 break;
1.25
1.26 case BarrierSet::CardTableModRef:
1.27 @@ -1404,8 +1405,8 @@
1.28 Node* store,
1.29 Node* obj,
1.30 Node* adr,
1.31 - uint adr_idx,
1.32 - Node *val,
1.33 + uint adr_idx,
1.34 + Node* val,
1.35 BasicType bt,
1.36 bool use_precise) {
1.37 BarrierSet* bs = Universe::heap()->barrier_set();
1.38 @@ -1413,7 +1414,7 @@
1.39 switch (bs->kind()) {
1.40 case BarrierSet::G1SATBCT:
1.41 case BarrierSet::G1SATBCTLogging:
1.42 - g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
1.43 + g1_write_barrier_post(store, obj, adr, adr_idx, val, bt, use_precise);
1.44 break;
1.45
1.46 case BarrierSet::CardTableModRef:
1.47 @@ -1431,42 +1432,36 @@
1.48 }
1.49 }
1.50
1.51 -Node* GraphKit::store_oop_to_object(Node* ctl,
1.52 - Node* obj,
1.53 - Node* adr,
1.54 - const TypePtr* adr_type,
1.55 - Node *val,
1.56 - const TypeOopPtr* val_type,
1.57 - BasicType bt) {
1.58 +Node* GraphKit::store_oop(Node* ctl,
1.59 + Node* obj,
1.60 + Node* adr,
1.61 + const TypePtr* adr_type,
1.62 + Node* val,
1.63 + const TypeOopPtr* val_type,
1.64 + BasicType bt,
1.65 + bool use_precise) {
1.66 +
1.67 + set_control(ctl);
1.68 + if (stopped()) return top(); // Dead path ?
1.69 +
1.70 + assert(bt == T_OBJECT, "sanity");
1.71 + assert(val != NULL, "not dead path");
1.72 uint adr_idx = C->get_alias_index(adr_type);
1.73 - Node* store;
1.74 - pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
1.75 - store = store_to_memory(control(), adr, val, bt, adr_idx);
1.76 - post_barrier(control(), store, obj, adr, adr_idx, val, bt, false);
1.77 + assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
1.78 +
1.79 + pre_barrier(control(), obj, adr, adr_idx, val, val_type, bt);
1.80 + Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
1.81 + post_barrier(control(), store, obj, adr, adr_idx, val, bt, use_precise);
1.82 return store;
1.83 }
1.84
1.85 -Node* GraphKit::store_oop_to_array(Node* ctl,
1.86 - Node* obj,
1.87 - Node* adr,
1.88 - const TypePtr* adr_type,
1.89 - Node *val,
1.90 - const TypeOopPtr* val_type,
1.91 - BasicType bt) {
1.92 - uint adr_idx = C->get_alias_index(adr_type);
1.93 - Node* store;
1.94 - pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
1.95 - store = store_to_memory(control(), adr, val, bt, adr_idx);
1.96 - post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
1.97 - return store;
1.98 -}
1.99 -
1.100 +// Could be an array or object we don't know at compile time (unsafe ref.)
1.101 Node* GraphKit::store_oop_to_unknown(Node* ctl,
1.102 - Node* obj,
1.103 - Node* adr,
1.104 - const TypePtr* adr_type,
1.105 - Node *val,
1.106 - BasicType bt) {
1.107 + Node* obj, // containing obj
1.108 + Node* adr, // actual adress to store val at
1.109 + const TypePtr* adr_type,
1.110 + Node* val,
1.111 + BasicType bt) {
1.112 Compile::AliasType* at = C->alias_type(adr_type);
1.113 const TypeOopPtr* val_type = NULL;
1.114 if (adr_type->isa_instptr()) {
1.115 @@ -1485,12 +1480,7 @@
1.116 if (val_type == NULL) {
1.117 val_type = TypeInstPtr::BOTTOM;
1.118 }
1.119 -
1.120 - uint adr_idx = at->index();
1.121 - pre_barrier(ctl, obj, adr, adr_idx, val, val_type, bt);
1.122 - Node* store = store_to_memory(control(), adr, val, bt, adr_idx);
1.123 - post_barrier(control(), store, obj, adr, adr_idx, val, bt, true);
1.124 - return store;
1.125 + return store_oop(ctl, obj, adr, adr_type, val, val_type, bt, true);
1.126 }
1.127
1.128
1.129 @@ -1804,93 +1794,6 @@
1.130 }
1.131
1.132
1.133 -//------------------------------store_barrier----------------------------------
1.134 -// Insert a write-barrier store. This is to let generational GC work; we have
1.135 -// to flag all oop-stores before the next GC point.
1.136 -void GraphKit::write_barrier_post(Node* oop_store, Node* obj, Node* adr,
1.137 - Node* val, bool use_precise) {
1.138 - // No store check needed if we're storing a NULL or an old object
1.139 - // (latter case is probably a string constant). The concurrent
1.140 - // mark sweep garbage collector, however, needs to have all nonNull
1.141 - // oop updates flagged via card-marks.
1.142 - if (val != NULL && val->is_Con()) {
1.143 - // must be either an oop or NULL
1.144 - const Type* t = val->bottom_type();
1.145 - if (t == TypePtr::NULL_PTR || t == Type::TOP)
1.146 - // stores of null never (?) need barriers
1.147 - return;
1.148 - ciObject* con = t->is_oopptr()->const_oop();
1.149 - if (con != NULL
1.150 - && con->is_perm()
1.151 - && Universe::heap()->can_elide_permanent_oop_store_barriers())
1.152 - // no store barrier needed, because no old-to-new ref created
1.153 - return;
1.154 - }
1.155 -
1.156 - if (use_ReduceInitialCardMarks()
1.157 - && obj == just_allocated_object(control())) {
1.158 - // We can skip marks on a freshly-allocated object.
1.159 - // Keep this code in sync with do_eager_card_mark in runtime.cpp.
1.160 - // That routine eagerly marks the occasional object which is produced
1.161 - // by the slow path, so that we don't have to do it here.
1.162 - return;
1.163 - }
1.164 -
1.165 - if (!use_precise) {
1.166 - // All card marks for a (non-array) instance are in one place:
1.167 - adr = obj;
1.168 - }
1.169 - // (Else it's an array (or unknown), and we want more precise card marks.)
1.170 - assert(adr != NULL, "");
1.171 -
1.172 - // Get the alias_index for raw card-mark memory
1.173 - int adr_type = Compile::AliasIdxRaw;
1.174 - // Convert the pointer to an int prior to doing math on it
1.175 - Node* cast = _gvn.transform(new (C, 2) CastP2XNode(control(), adr));
1.176 - // Divide by card size
1.177 - assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
1.178 - "Only one we handle so far.");
1.179 - CardTableModRefBS* ct =
1.180 - (CardTableModRefBS*)(Universe::heap()->barrier_set());
1.181 - Node *b = _gvn.transform(new (C, 3) URShiftXNode( cast, _gvn.intcon(CardTableModRefBS::card_shift) ));
1.182 - // We store into a byte array, so do not bother to left-shift by zero
1.183 - Node *c = byte_map_base_node();
1.184 - // Combine
1.185 - Node *sb_ctl = control();
1.186 - Node *sb_adr = _gvn.transform(new (C, 4) AddPNode( top()/*no base ptr*/, c, b ));
1.187 - Node *sb_val = _gvn.intcon(0);
1.188 - // Smash zero into card
1.189 - if( !UseConcMarkSweepGC ) {
1.190 - BasicType bt = T_BYTE;
1.191 - store_to_memory(sb_ctl, sb_adr, sb_val, bt, adr_type);
1.192 - } else {
1.193 - // Specialized path for CM store barrier
1.194 - cms_card_mark( sb_ctl, sb_adr, sb_val, oop_store);
1.195 - }
1.196 -}
1.197 -
1.198 -// Specialized path for CMS store barrier
1.199 -void GraphKit::cms_card_mark(Node* ctl, Node* adr, Node* val, Node *oop_store) {
1.200 - BasicType bt = T_BYTE;
1.201 - int adr_idx = Compile::AliasIdxRaw;
1.202 - Node* mem = memory(adr_idx);
1.203 -
1.204 - // The type input is NULL in PRODUCT builds
1.205 - const TypePtr* type = NULL;
1.206 - debug_only(type = C->get_adr_type(adr_idx));
1.207 -
1.208 - // Add required edge to oop_store, optimizer does not support precedence edges.
1.209 - // Convert required edge to precedence edge before allocation.
1.210 - Node *store = _gvn.transform( new (C, 5) StoreCMNode(ctl, mem, adr, type, val, oop_store) );
1.211 - set_memory(store, adr_idx);
1.212 -
1.213 - // For CMS, back-to-back card-marks can only remove the first one
1.214 - // and this requires DU info. Push on worklist for optimizer.
1.215 - if (mem->req() > MemNode::Address && adr == mem->in(MemNode::Address))
1.216 - record_for_igvn(store);
1.217 -}
1.218 -
1.219 -
1.220 void GraphKit::round_double_arguments(ciMethod* dest_method) {
1.221 // (Note: TypeFunc::make has a cache that makes this fast.)
1.222 const TypeFunc* tf = TypeFunc::make(dest_method);
1.223 @@ -3215,6 +3118,79 @@
1.224 return NULL;
1.225 }
1.226
1.227 +//----------------------------- store barriers ----------------------------
1.228 +#define __ ideal.
1.229 +
1.230 +void GraphKit::sync_kit(IdealKit& ideal) {
1.231 + // Final sync IdealKit and graphKit.
1.232 + __ drain_delay_transform();
1.233 + set_all_memory(__ merged_memory());
1.234 + set_control(__ ctrl());
1.235 +}
1.236 +
1.237 +// vanilla/CMS post barrier
1.238 +// Insert a write-barrier store. This is to let generational GC work; we have
1.239 +// to flag all oop-stores before the next GC point.
1.240 +void GraphKit::write_barrier_post(Node* oop_store,
1.241 + Node* obj,
1.242 + Node* adr,
1.243 + Node* val,
1.244 + bool use_precise) {
1.245 + // No store check needed if we're storing a NULL or an old object
1.246 + // (latter case is probably a string constant). The concurrent
1.247 + // mark sweep garbage collector, however, needs to have all nonNull
1.248 + // oop updates flagged via card-marks.
1.249 + if (val != NULL && val->is_Con()) {
1.250 + // must be either an oop or NULL
1.251 + const Type* t = val->bottom_type();
1.252 + if (t == TypePtr::NULL_PTR || t == Type::TOP)
1.253 + // stores of null never (?) need barriers
1.254 + return;
1.255 + ciObject* con = t->is_oopptr()->const_oop();
1.256 + if (con != NULL
1.257 + && con->is_perm()
1.258 + && Universe::heap()->can_elide_permanent_oop_store_barriers())
1.259 + // no store barrier needed, because no old-to-new ref created
1.260 + return;
1.261 + }
1.262 +
1.263 + if (!use_precise) {
1.264 + // All card marks for a (non-array) instance are in one place:
1.265 + adr = obj;
1.266 + }
1.267 + // (Else it's an array (or unknown), and we want more precise card marks.)
1.268 + assert(adr != NULL, "");
1.269 +
1.270 + IdealKit ideal(gvn(), control(), merged_memory(), true);
1.271 +
1.272 + // Convert the pointer to an int prior to doing math on it
1.273 + Node* cast = __ CastPX(__ ctrl(), adr);
1.274 +
1.275 + // Divide by card size
1.276 + assert(Universe::heap()->barrier_set()->kind() == BarrierSet::CardTableModRef,
1.277 + "Only one we handle so far.");
1.278 + Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
1.279 +
1.280 + // Combine card table base and card offset
1.281 + Node* card_adr = __ AddP(__ top(), byte_map_base_node(), card_offset );
1.282 +
1.283 + // Get the alias_index for raw card-mark memory
1.284 + int adr_type = Compile::AliasIdxRaw;
1.285 + // Smash zero into card
1.286 + Node* zero = __ ConI(0);
1.287 + BasicType bt = T_BYTE;
1.288 + if( !UseConcMarkSweepGC ) {
1.289 + __ store(__ ctrl(), card_adr, zero, bt, adr_type);
1.290 + } else {
1.291 + // Specialized path for CM store barrier
1.292 + __ storeCM(__ ctrl(), card_adr, zero, oop_store, bt, adr_type);
1.293 + }
1.294 +
1.295 + // Final sync IdealKit and GraphKit.
1.296 + sync_kit(ideal);
1.297 +}
1.298 +
1.299 +// G1 pre/post barriers
1.300 void GraphKit::g1_write_barrier_pre(Node* obj,
1.301 Node* adr,
1.302 uint alias_idx,
1.303 @@ -3222,10 +3198,8 @@
1.304 const TypeOopPtr* val_type,
1.305 BasicType bt) {
1.306 IdealKit ideal(gvn(), control(), merged_memory(), true);
1.307 -#define __ ideal.
1.308 - __ declares_done();
1.309 -
1.310 - Node* thread = __ thread();
1.311 +
1.312 + Node* tls = __ thread(); // ThreadLocalStorage
1.313
1.314 Node* no_ctrl = NULL;
1.315 Node* no_base = __ top();
1.316 @@ -3248,9 +3222,9 @@
1.317
1.318 // set_control( ctl);
1.319
1.320 - Node* marking_adr = __ AddP(no_base, thread, __ ConX(marking_offset));
1.321 - Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset));
1.322 - Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset));
1.323 + Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
1.324 + Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
1.325 + Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
1.326
1.327 // Now some of the values
1.328
1.329 @@ -3278,55 +3252,52 @@
1.330 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
1.331 Node* next_indexX = next_index;
1.332 #ifdef _LP64
1.333 - // We could refine the type for what it's worth
1.334 - // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
1.335 - next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
1.336 -#endif // _LP64
1.337 + // We could refine the type for what it's worth
1.338 + // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
1.339 + next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
1.340 +#endif
1.341
1.342 // Now get the buffer location we will log the original value into and store it
1.343 -
1.344 Node *log_addr = __ AddP(no_base, buffer, next_indexX);
1.345 - // __ store(__ ctrl(), log_addr, orig, T_OBJECT, C->get_alias_index(TypeOopPtr::BOTTOM));
1.346 __ store(__ ctrl(), log_addr, orig, T_OBJECT, Compile::AliasIdxRaw);
1.347
1.348 -
1.349 // update the index
1.350 - // __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
1.351 - // This is a hack to force this store to occur before the oop store that is coming up
1.352 - __ store(__ ctrl(), index_adr, next_index, T_INT, C->get_alias_index(TypeOopPtr::BOTTOM));
1.353 + __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
1.354
1.355 } __ else_(); {
1.356
1.357 // logging buffer is full, call the runtime
1.358 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
1.359 - // __ make_leaf_call(tf, OptoRuntime::g1_wb_pre_Java(), "g1_wb_pre", orig, thread);
1.360 - __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, thread);
1.361 - } __ end_if();
1.362 - } __ end_if();
1.363 - } __ end_if();
1.364 -
1.365 - __ drain_delay_transform();
1.366 - set_control( __ ctrl());
1.367 - set_all_memory( __ merged_memory());
1.368 -
1.369 -#undef __
1.370 + __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, tls);
1.371 + } __ end_if(); // (!index)
1.372 + } __ end_if(); // (orig != NULL)
1.373 + } __ end_if(); // (!marking)
1.374 +
1.375 + // Final sync IdealKit and GraphKit.
1.376 + sync_kit(ideal);
1.377 }
1.378
1.379 //
1.380 // Update the card table and add card address to the queue
1.381 //
1.382 -void GraphKit::g1_mark_card(IdealKit* ideal, Node* card_adr, Node* store, Node* index, Node* index_adr, Node* buffer, const TypeFunc* tf) {
1.383 -#define __ ideal->
1.384 +void GraphKit::g1_mark_card(IdealKit& ideal,
1.385 + Node* card_adr,
1.386 + Node* oop_store,
1.387 + Node* index,
1.388 + Node* index_adr,
1.389 + Node* buffer,
1.390 + const TypeFunc* tf) {
1.391 +
1.392 Node* zero = __ ConI(0);
1.393 Node* no_base = __ top();
1.394 BasicType card_bt = T_BYTE;
1.395 // Smash zero into card. MUST BE ORDERED WRT TO STORE
1.396 - __ storeCM(__ ctrl(), card_adr, zero, store, card_bt, Compile::AliasIdxRaw);
1.397 + __ storeCM(__ ctrl(), card_adr, zero, oop_store, card_bt, Compile::AliasIdxRaw);
1.398
1.399 // Now do the queue work
1.400 __ if_then(index, BoolTest::ne, zero); {
1.401
1.402 - Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
1.403 + Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
1.404 Node* next_indexX = next_index;
1.405 #ifdef _LP64
1.406 // We could refine the type for what it's worth
1.407 @@ -3341,10 +3312,10 @@
1.408 } __ else_(); {
1.409 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
1.410 } __ end_if();
1.411 -#undef __
1.412 +
1.413 }
1.414
1.415 -void GraphKit::g1_write_barrier_post(Node* store,
1.416 +void GraphKit::g1_write_barrier_post(Node* oop_store,
1.417 Node* obj,
1.418 Node* adr,
1.419 uint alias_idx,
1.420 @@ -3369,10 +3340,8 @@
1.421 assert(adr != NULL, "");
1.422
1.423 IdealKit ideal(gvn(), control(), merged_memory(), true);
1.424 -#define __ ideal.
1.425 - __ declares_done();
1.426 -
1.427 - Node* thread = __ thread();
1.428 +
1.429 + Node* tls = __ thread(); // ThreadLocalStorage
1.430
1.431 Node* no_ctrl = NULL;
1.432 Node* no_base = __ top();
1.433 @@ -3394,8 +3363,8 @@
1.434
1.435 // Pointers into the thread
1.436
1.437 - Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset));
1.438 - Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset));
1.439 + Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
1.440 + Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
1.441
1.442 // Now some values
1.443
1.444 @@ -3404,18 +3373,14 @@
1.445
1.446
1.447 // Convert the store obj pointer to an int prior to doing math on it
1.448 - // Use addr not obj gets accurate card marks
1.449 -
1.450 - // Node* cast = __ CastPX(no_ctrl, adr /* obj */);
1.451 -
1.452 // Must use ctrl to prevent "integerized oop" existing across safepoint
1.453 - Node* cast = __ CastPX(__ ctrl(), ( use_precise ? adr : obj ));
1.454 + Node* cast = __ CastPX(__ ctrl(), adr);
1.455
1.456 // Divide pointer by card size
1.457 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
1.458
1.459 // Combine card table base and card offset
1.460 - Node *card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
1.461 + Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
1.462
1.463 // If we know the value being stored does it cross regions?
1.464
1.465 @@ -3439,18 +3404,17 @@
1.466 Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw);
1.467
1.468 __ if_then(card_val, BoolTest::ne, zero); {
1.469 - g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf);
1.470 + g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
1.471 } __ end_if();
1.472 } __ end_if();
1.473 } __ end_if();
1.474 } else {
1.475 - g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf);
1.476 + // Object.clone() instrinsic uses this path.
1.477 + g1_mark_card(ideal, card_adr, oop_store, index, index_adr, buffer, tf);
1.478 }
1.479
1.480 -
1.481 - __ drain_delay_transform();
1.482 - set_control( __ ctrl());
1.483 - set_all_memory( __ merged_memory());
1.484 + // Final sync IdealKit and GraphKit.
1.485 + sync_kit(ideal);
1.486 +}
1.487 #undef __
1.488
1.489 -}
