4255 } |
4255 } |
4256 |
4256 |
4257 /////////////////////////////////////////////////////////////////////////////////// |
4257 /////////////////////////////////////////////////////////////////////////////////// |
4258 #ifndef SERIALGC |
4258 #ifndef SERIALGC |
4259 |
4259 |
4260 static uint num_stores = 0; |
4260 static address satb_log_enqueue_with_frame = NULL; |
4261 static uint num_null_pre_stores = 0; |
4261 static u_char* satb_log_enqueue_with_frame_end = NULL; |
4262 |
4262 |
4263 static void count_null_pre_vals(void* pre_val) { |
4263 static address satb_log_enqueue_frameless = NULL; |
4264 num_stores++; |
4264 static u_char* satb_log_enqueue_frameless_end = NULL; |
4265 if (pre_val == NULL) num_null_pre_stores++; |
|
4266 if ((num_stores % 1000000) == 0) { |
|
4267 tty->print_cr(UINT32_FORMAT " stores, " UINT32_FORMAT " (%5.2f%%) with null pre-vals.", |
|
4268 num_stores, num_null_pre_stores, |
|
4269 100.0*(float)num_null_pre_stores/(float)num_stores); |
|
4270 } |
|
4271 } |
|
4272 |
|
4273 static address satb_log_enqueue_with_frame = 0; |
|
4274 static u_char* satb_log_enqueue_with_frame_end = 0; |
|
4275 |
|
4276 static address satb_log_enqueue_frameless = 0; |
|
4277 static u_char* satb_log_enqueue_frameless_end = 0; |
|
4278 |
4265 |
4279 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions? |
4266 static int EnqueueCodeSize = 128 DEBUG_ONLY( + 256); // Instructions? |
4280 |
|
4281 // The calls to this don't work. We'd need to do a fair amount of work to |
|
4282 // make it work. |
|
4283 static void check_index(int ind) { |
|
4284 assert(0 <= ind && ind <= 64*K && ((ind % oopSize) == 0), |
|
4285 "Invariants."); |
|
4286 } |
|
4287 |
4267 |
4288 static void generate_satb_log_enqueue(bool with_frame) { |
4268 static void generate_satb_log_enqueue(bool with_frame) { |
4289 BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize); |
4269 BufferBlob* bb = BufferBlob::create("enqueue_with_frame", EnqueueCodeSize); |
4290 CodeBuffer buf(bb); |
4270 CodeBuffer buf(bb); |
4291 MacroAssembler masm(&buf); |
4271 MacroAssembler masm(&buf); |
4386 } |
4366 } |
4387 } |
4367 } |
4388 } |
4368 } |
4389 } |
4369 } |
4390 |
4370 |
4391 void MacroAssembler::g1_write_barrier_pre(Register obj, Register index, int offset, Register tmp, bool preserve_o_regs) { |
4371 void MacroAssembler::g1_write_barrier_pre(Register obj, |
4392 assert(offset == 0 || index == noreg, "choose one"); |
4372 Register index, |
4393 |
4373 int offset, |
4394 if (G1DisablePreBarrier) return; |
4374 Register pre_val, |
4395 // satb_log_barrier(tmp, obj, offset, preserve_o_regs); |
4375 Register tmp, |
|
4376 bool preserve_o_regs) { |
4396 Label filtered; |
4377 Label filtered; |
4397 // satb_log_barrier_work0(tmp, filtered); |
4378 |
|
4379 if (obj == noreg) { |
|
4380 // We are not loading the previous value so make |
|
4381 // sure that we don't trash the value in pre_val |
|
4382 // with the code below. |
|
4383 assert_different_registers(pre_val, tmp); |
|
4384 } else { |
|
4385 // We will be loading the previous value |
|
4386 // in this code so... |
|
4387 assert(offset == 0 || index == noreg, "choose one"); |
|
4388 assert(pre_val == noreg, "check this code"); |
|
4389 } |
|
4390 |
|
4391 // Is marking active? |
4398 if (in_bytes(PtrQueue::byte_width_of_active()) == 4) { |
4392 if (in_bytes(PtrQueue::byte_width_of_active()) == 4) { |
4399 ld(G2, |
4393 ld(G2, |
4400 in_bytes(JavaThread::satb_mark_queue_offset() + |
4394 in_bytes(JavaThread::satb_mark_queue_offset() + |
4401 PtrQueue::byte_offset_of_active()), |
4395 PtrQueue::byte_offset_of_active()), |
4402 tmp); |
4396 tmp); |
4411 |
4405 |
4412 // Check on whether to annul. |
4406 // Check on whether to annul. |
4413 br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered); |
4407 br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, tmp, filtered); |
4414 delayed() -> nop(); |
4408 delayed() -> nop(); |
4415 |
4409 |
4416 // satb_log_barrier_work1(tmp, offset); |
4410 // Do we need to load the previous value? |
4417 if (index == noreg) { |
4411 if (obj != noreg) { |
4418 if (Assembler::is_simm13(offset)) { |
4412 // Load the previous value... |
4419 load_heap_oop(obj, offset, tmp); |
4413 if (index == noreg) { |
|
4414 if (Assembler::is_simm13(offset)) { |
|
4415 load_heap_oop(obj, offset, tmp); |
|
4416 } else { |
|
4417 set(offset, tmp); |
|
4418 load_heap_oop(obj, tmp, tmp); |
|
4419 } |
4420 } else { |
4420 } else { |
4421 set(offset, tmp); |
4421 load_heap_oop(obj, index, tmp); |
4422 load_heap_oop(obj, tmp, tmp); |
|
4423 } |
4422 } |
4424 } else { |
4423 // Previous value has been loaded into tmp |
4425 load_heap_oop(obj, index, tmp); |
4424 pre_val = tmp; |
4426 } |
4425 } |
4427 |
4426 |
4428 // satb_log_barrier_work2(obj, tmp, offset); |
4427 assert(pre_val != noreg, "must have a real register"); |
4429 |
4428 |
4430 // satb_log_barrier_work3(tmp, filtered, preserve_o_regs); |
4429 // Is the previous value null? |
4431 |
|
4432 const Register pre_val = tmp; |
|
4433 |
|
4434 if (G1SATBBarrierPrintNullPreVals) { |
|
4435 save_frame(0); |
|
4436 mov(pre_val, O0); |
|
4437 // Save G-regs that target may use. |
|
4438 mov(G1, L1); |
|
4439 mov(G2, L2); |
|
4440 mov(G3, L3); |
|
4441 mov(G4, L4); |
|
4442 mov(G5, L5); |
|
4443 call(CAST_FROM_FN_PTR(address, &count_null_pre_vals)); |
|
4444 delayed()->nop(); |
|
4445 // Restore G-regs that target may have used. |
|
4446 mov(L1, G1); |
|
4447 mov(L2, G2); |
|
4448 mov(L3, G3); |
|
4449 mov(L4, G4); |
|
4450 mov(L5, G5); |
|
4451 restore(G0, G0, G0); |
|
4452 } |
|
4453 |
|
4454 // Check on whether to annul. |
4430 // Check on whether to annul. |
4455 br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, pre_val, filtered); |
4431 br_on_reg_cond(rc_z, /*annul*/false, Assembler::pt, pre_val, filtered); |
4456 delayed() -> nop(); |
4432 delayed() -> nop(); |
4457 |
4433 |
4458 // OK, it's not filtered, so we'll need to call enqueue. In the normal |
4434 // OK, it's not filtered, so we'll need to call enqueue. In the normal |
4459 // case, pre_val will be a scratch G-reg, but there's some cases in which |
4435 // case, pre_val will be a scratch G-reg, but there are some cases in |
4460 // it's an O-reg. In the first case, do a normal call. In the latter, |
4436 // which it's an O-reg. In the first case, do a normal call. In the |
4461 // do a save here and call the frameless version. |
4437 // latter, do a save here and call the frameless version. |
4462 |
4438 |
4463 guarantee(pre_val->is_global() || pre_val->is_out(), |
4439 guarantee(pre_val->is_global() || pre_val->is_out(), |
4464 "Or we need to think harder."); |
4440 "Or we need to think harder."); |
|
4441 |
4465 if (pre_val->is_global() && !preserve_o_regs) { |
4442 if (pre_val->is_global() && !preserve_o_regs) { |
4466 generate_satb_log_enqueue_if_necessary(true); // with frame. |
4443 generate_satb_log_enqueue_if_necessary(true); // with frame |
|
4444 |
4467 call(satb_log_enqueue_with_frame); |
4445 call(satb_log_enqueue_with_frame); |
4468 delayed()->mov(pre_val, O0); |
4446 delayed()->mov(pre_val, O0); |
4469 } else { |
4447 } else { |
4470 generate_satb_log_enqueue_if_necessary(false); // with frameless. |
4448 generate_satb_log_enqueue_if_necessary(false); // frameless |
|
4449 |
4471 save_frame(0); |
4450 save_frame(0); |
4472 call(satb_log_enqueue_frameless); |
4451 call(satb_log_enqueue_frameless); |
4473 delayed()->mov(pre_val->after_save(), O0); |
4452 delayed()->mov(pre_val->after_save(), O0); |
4474 restore(); |
4453 restore(); |
4475 } |
4454 } |
4624 #ifdef _LP64 |
4602 #ifdef _LP64 |
4625 srlx(tmp, HeapRegion::LogOfHRGrainBytes, tmp); |
4603 srlx(tmp, HeapRegion::LogOfHRGrainBytes, tmp); |
4626 #else |
4604 #else |
4627 srl(tmp, HeapRegion::LogOfHRGrainBytes, tmp); |
4605 srl(tmp, HeapRegion::LogOfHRGrainBytes, tmp); |
4628 #endif |
4606 #endif |
|
4607 |
4629 if (G1PrintCTFilterStats) { |
4608 if (G1PrintCTFilterStats) { |
4630 guarantee(tmp->is_global(), "Or stats won't work..."); |
4609 guarantee(tmp->is_global(), "Or stats won't work..."); |
4631 // This is a sleazy hack: I'm temporarily hijacking G2, which I |
4610 // This is a sleazy hack: I'm temporarily hijacking G2, which I |
4632 // promise to restore. |
4611 // promise to restore. |
4633 mov(new_val, G2); |
4612 mov(new_val, G2); |