3174 return init->as_Initialize(); |
3188 return init->as_Initialize(); |
3175 } |
3189 } |
3176 } |
3190 } |
3177 return NULL; |
3191 return NULL; |
3178 } |
3192 } |
|
3193 |
|
3194 void GraphKit::g1_write_barrier_pre(Node* obj, |
|
3195 Node* adr, |
|
3196 uint alias_idx, |
|
3197 Node* val, |
|
3198 const Type* val_type, |
|
3199 BasicType bt) { |
|
3200 IdealKit ideal(gvn(), control(), merged_memory(), true); |
|
3201 #define __ ideal. |
|
3202 __ declares_done(); |
|
3203 |
|
3204 Node* thread = __ thread(); |
|
3205 |
|
3206 Node* no_ctrl = NULL; |
|
3207 Node* no_base = __ top(); |
|
3208 Node* zero = __ ConI(0); |
|
3209 |
|
3210 float likely = PROB_LIKELY(0.999); |
|
3211 float unlikely = PROB_UNLIKELY(0.999); |
|
3212 |
|
3213 BasicType active_type = in_bytes(PtrQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE; |
|
3214 assert(in_bytes(PtrQueue::byte_width_of_active()) == 4 || in_bytes(PtrQueue::byte_width_of_active()) == 1, "flag width"); |
|
3215 |
|
3216 // Offsets into the thread |
|
3217 const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 648 |
|
3218 PtrQueue::byte_offset_of_active()); |
|
3219 const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 656 |
|
3220 PtrQueue::byte_offset_of_index()); |
|
3221 const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 652 |
|
3222 PtrQueue::byte_offset_of_buf()); |
|
3223 // Now the actual pointers into the thread |
|
3224 |
|
3225 // set_control( ctl); |
|
3226 |
|
3227 Node* marking_adr = __ AddP(no_base, thread, __ ConX(marking_offset)); |
|
3228 Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset)); |
|
3229 Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset)); |
|
3230 |
|
3231 // Now some of the values |
|
3232 |
|
3233 Node* marking = __ load(no_ctrl, marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw); |
|
3234 Node* index = __ load(no_ctrl, index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw); |
|
3235 Node* buffer = __ load(no_ctrl, buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); |
|
3236 |
|
3237 // if (!marking) |
|
3238 __ if_then(marking, BoolTest::ne, zero); { |
|
3239 |
|
3240 const Type* t1 = adr->bottom_type(); |
|
3241 const Type* t2 = val->bottom_type(); |
|
3242 |
|
3243 Node* orig = __ load(no_ctrl, adr, val_type, bt, alias_idx); |
|
3244 // if (orig != NULL) |
|
3245 __ if_then(orig, BoolTest::ne, null()); { |
|
3246 |
|
3247 // load original value |
|
3248 // alias_idx correct?? |
|
3249 |
|
3250 // is the queue for this thread full? |
|
3251 __ if_then(index, BoolTest::ne, zero, likely); { |
|
3252 |
|
3253 // decrement the index |
|
3254 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t))); |
|
3255 Node* next_indexX = next_index; |
|
3256 #ifdef _LP64 |
|
3257 // We could refine the type for what it's worth |
|
3258 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue); |
|
3259 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) ); |
|
3260 #endif // _LP64 |
|
3261 |
|
3262 // Now get the buffer location we will log the original value into and store it |
|
3263 |
|
3264 Node *log_addr = __ AddP(no_base, buffer, next_indexX); |
|
3265 // __ store(__ ctrl(), log_addr, orig, T_OBJECT, C->get_alias_index(TypeOopPtr::BOTTOM)); |
|
3266 __ store(__ ctrl(), log_addr, orig, T_OBJECT, Compile::AliasIdxRaw); |
|
3267 |
|
3268 |
|
3269 // update the index |
|
3270 // __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw); |
|
3271 // This is a hack to force this store to occur before the oop store that is coming up |
|
3272 __ store(__ ctrl(), index_adr, next_index, T_INT, C->get_alias_index(TypeOopPtr::BOTTOM)); |
|
3273 |
|
3274 } __ else_(); { |
|
3275 |
|
3276 // logging buffer is full, call the runtime |
|
3277 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type(); |
|
3278 // __ make_leaf_call(tf, OptoRuntime::g1_wb_pre_Java(), "g1_wb_pre", orig, thread); |
|
3279 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", orig, thread); |
|
3280 } __ end_if(); |
|
3281 } __ end_if(); |
|
3282 } __ end_if(); |
|
3283 |
|
3284 __ drain_delay_transform(); |
|
3285 set_control( __ ctrl()); |
|
3286 set_all_memory( __ merged_memory()); |
|
3287 |
|
3288 #undef __ |
|
3289 } |
|
3290 |
|
3291 // |
|
3292 // Update the card table and add card address to the queue |
|
3293 // |
|
3294 void GraphKit::g1_mark_card(IdealKit* ideal, Node* card_adr, Node* store, Node* index, Node* index_adr, Node* buffer, const TypeFunc* tf) { |
|
3295 #define __ ideal-> |
|
3296 Node* zero = __ ConI(0); |
|
3297 Node* no_base = __ top(); |
|
3298 BasicType card_bt = T_BYTE; |
|
3299 // Smash zero into card. MUST BE ORDERED WRT TO STORE |
|
3300 __ storeCM(__ ctrl(), card_adr, zero, store, card_bt, Compile::AliasIdxRaw); |
|
3301 |
|
3302 // Now do the queue work |
|
3303 __ if_then(index, BoolTest::ne, zero); { |
|
3304 |
|
3305 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t))); |
|
3306 Node* next_indexX = next_index; |
|
3307 #ifdef _LP64 |
|
3308 // We could refine the type for what it's worth |
|
3309 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue); |
|
3310 next_indexX = _gvn.transform( new (C, 2) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) ); |
|
3311 #endif // _LP64 |
|
3312 Node* log_addr = __ AddP(no_base, buffer, next_indexX); |
|
3313 |
|
3314 __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw); |
|
3315 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw); |
|
3316 |
|
3317 } __ else_(); { |
|
3318 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread()); |
|
3319 } __ end_if(); |
|
3320 #undef __ |
|
3321 } |
|
3322 |
|
3323 void GraphKit::g1_write_barrier_post(Node* store, |
|
3324 Node* obj, |
|
3325 Node* adr, |
|
3326 uint alias_idx, |
|
3327 Node* val, |
|
3328 BasicType bt, |
|
3329 bool use_precise) { |
|
3330 // If we are writing a NULL then we need no post barrier |
|
3331 |
|
3332 if (val != NULL && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) { |
|
3333 // Must be NULL |
|
3334 const Type* t = val->bottom_type(); |
|
3335 assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be NULL"); |
|
3336 // No post barrier if writing NULLx |
|
3337 return; |
|
3338 } |
|
3339 |
|
3340 if (!use_precise) { |
|
3341 // All card marks for a (non-array) instance are in one place: |
|
3342 adr = obj; |
|
3343 } |
|
3344 // (Else it's an array (or unknown), and we want more precise card marks.) |
|
3345 assert(adr != NULL, ""); |
|
3346 |
|
3347 IdealKit ideal(gvn(), control(), merged_memory(), true); |
|
3348 #define __ ideal. |
|
3349 __ declares_done(); |
|
3350 |
|
3351 Node* thread = __ thread(); |
|
3352 |
|
3353 Node* no_ctrl = NULL; |
|
3354 Node* no_base = __ top(); |
|
3355 float likely = PROB_LIKELY(0.999); |
|
3356 float unlikely = PROB_UNLIKELY(0.999); |
|
3357 Node* zero = __ ConI(0); |
|
3358 Node* zeroX = __ ConX(0); |
|
3359 |
|
3360 // Get the alias_index for raw card-mark memory |
|
3361 const TypePtr* card_type = TypeRawPtr::BOTTOM; |
|
3362 |
|
3363 const TypeFunc *tf = OptoRuntime::g1_wb_post_Type(); |
|
3364 |
|
3365 // Get the address of the card table |
|
3366 CardTableModRefBS* ct = |
|
3367 (CardTableModRefBS*)(Universe::heap()->barrier_set()); |
|
3368 Node *card_table = __ makecon(TypeRawPtr::make((address)ct->byte_map_base)); |
|
3369 // Get base of card map |
|
3370 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); |
|
3371 |
|
3372 |
|
3373 // Offsets into the thread |
|
3374 const int index_offset = in_bytes(JavaThread::dirty_card_queue_offset() + |
|
3375 PtrQueue::byte_offset_of_index()); |
|
3376 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() + |
|
3377 PtrQueue::byte_offset_of_buf()); |
|
3378 |
|
3379 // Pointers into the thread |
|
3380 |
|
3381 Node* buffer_adr = __ AddP(no_base, thread, __ ConX(buffer_offset)); |
|
3382 Node* index_adr = __ AddP(no_base, thread, __ ConX(index_offset)); |
|
3383 |
|
3384 // Now some values |
|
3385 |
|
3386 Node* index = __ load(no_ctrl, index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw); |
|
3387 Node* buffer = __ load(no_ctrl, buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw); |
|
3388 |
|
3389 |
|
3390 // Convert the store obj pointer to an int prior to doing math on it |
|
3391 // Use addr not obj gets accurate card marks |
|
3392 |
|
3393 // Node* cast = __ CastPX(no_ctrl, adr /* obj */); |
|
3394 |
|
3395 // Must use ctrl to prevent "integerized oop" existing across safepoint |
|
3396 Node* cast = __ CastPX(__ ctrl(), ( use_precise ? adr : obj )); |
|
3397 |
|
3398 // Divide pointer by card size |
|
3399 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) ); |
|
3400 |
|
3401 // Combine card table base and card offset |
|
3402 Node *card_adr = __ AddP(no_base, card_table, card_offset ); |
|
3403 |
|
3404 // If we know the value being stored does it cross regions? |
|
3405 |
|
3406 if (val != NULL) { |
|
3407 // Does the store cause us to cross regions? |
|
3408 |
|
3409 // Should be able to do an unsigned compare of region_size instead of |
|
3410 // and extra shift. Do we have an unsigned compare?? |
|
3411 // Node* region_size = __ ConI(1 << HeapRegion::LogOfHRGrainBytes); |
|
3412 Node* xor_res = __ URShiftX ( __ XorX( cast, __ CastPX(__ ctrl(), val)), __ ConI(HeapRegion::LogOfHRGrainBytes)); |
|
3413 |
|
3414 // if (xor_res == 0) same region so skip |
|
3415 __ if_then(xor_res, BoolTest::ne, zeroX); { |
|
3416 |
|
3417 // No barrier if we are storing a NULL |
|
3418 __ if_then(val, BoolTest::ne, null(), unlikely); { |
|
3419 |
|
3420 // Ok must mark the card if not already dirty |
|
3421 |
|
3422 // load the original value of the card |
|
3423 Node* card_val = __ load(__ ctrl(), card_adr, TypeInt::INT, T_BYTE, Compile::AliasIdxRaw); |
|
3424 |
|
3425 __ if_then(card_val, BoolTest::ne, zero); { |
|
3426 g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf); |
|
3427 } __ end_if(); |
|
3428 } __ end_if(); |
|
3429 } __ end_if(); |
|
3430 } else { |
|
3431 g1_mark_card(&ideal, card_adr, store, index, index_adr, buffer, tf); |
|
3432 } |
|
3433 |
|
3434 |
|
3435 __ drain_delay_transform(); |
|
3436 set_control( __ ctrl()); |
|
3437 set_all_memory( __ merged_memory()); |
|
3438 #undef __ |
|
3439 |
|
3440 } |