1.1 --- a/src/cpu/x86/vm/assembler_x86_64.cpp Wed Jun 04 13:51:09 2008 -0700
1.2 +++ b/src/cpu/x86/vm/assembler_x86_64.cpp Thu Jun 05 15:57:56 2008 -0700
1.3 @@ -4405,6 +4405,129 @@
1.4 call_VM_leaf(entry_point, 3);
1.5 }
1.6
1.7 +/////////////////////////////////////////////////////////////////////////////
1.8 +#ifndef SERIALGC
1.9 +
1.10 +void MacroAssembler::g1_write_barrier_pre(Register obj, Register tmp, Register tmp2, bool tosca_live ) {
1.11 + Address in_progress(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1.12 + PtrQueue::byte_offset_of_active()));
1.13 +
1.14 + Address index(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1.15 + PtrQueue::byte_offset_of_index()));
1.16 + Address buffer(r15_thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1.17 + PtrQueue::byte_offset_of_buf()));
1.18 +
1.19 +
1.20 + Label done;
1.21 + Label runtime;
1.22 +
1.23 + // if (!marking_in_progress) goto done;
1.24 + if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
1.25 + cmpl(in_progress, 0);
1.26 + } else {
1.27 + assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");
1.28 + cmpb(in_progress, 0);
1.29 + }
1.30 + jcc(Assembler::equal, done);
1.31 +
1.32 + // if (x.f == NULL) goto done;
1.33 + cmpq(Address(obj, 0), (int)NULL_WORD);
1.34 + jcc(Assembler::equal, done);
1.35 +
1.36 + // Can we store original value in the thread's buffer?
1.37 +
1.38 + movslq(tmp, index);
1.39 + movq(tmp2, Address(obj, 0));
1.40 + cmpq(tmp, 0);
1.41 + jcc(Assembler::equal, runtime);
1.42 + subq(tmp, wordSize);
1.43 + movl(index, tmp);
1.44 + addq(tmp, buffer);
1.45 + movq(Address(tmp, 0), tmp2);
1.46 + jmp(done);
1.47 + bind(runtime);
1.48 + // save live inputs
1.49 + if (tosca_live) pushq(rax);
1.50 + pushq(obj);
1.51 + movq(c_rarg0, Address(obj, 0));
1.52 + call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, r15_thread);
1.53 + popq(obj);
1.54 + if (tosca_live) popq(rax);
1.55 + bind(done);
1.56 +}
1.57 +
1.58 +void MacroAssembler::g1_write_barrier_post(Register store_addr,
1.59 + Register new_val,
1.60 + Register tmp,
1.61 + Register tmp2) {
1.62 +
1.63 + Address index(r15_thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1.64 + PtrQueue::byte_offset_of_index()));
1.65 + Address buffer(r15_thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1.66 + PtrQueue::byte_offset_of_buf()));
1.67 + BarrierSet* bs = Universe::heap()->barrier_set();
1.68 + CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1.69 + Label done;
1.70 + Label runtime;
1.71 +
1.72 + // Does store cross heap regions?
1.73 +
1.74 + movq(tmp, store_addr);
1.75 + xorq(tmp, new_val);
1.76 + shrq(tmp, HeapRegion::LogOfHRGrainBytes);
1.77 + jcc(Assembler::equal, done);
1.78 +
1.79 + // crosses regions, storing NULL?
1.80 +
1.81 + cmpq(new_val, (int)NULL_WORD);
1.82 + jcc(Assembler::equal, done);
1.83 +
1.84 + // storing region crossing non-NULL, is card already dirty?
1.85 + const Register card_addr = tmp;
1.86 +
1.87 + movq(card_addr, store_addr);
1.88 + shrq(card_addr, CardTableModRefBS::card_shift);
1.89 +
1.90 + ExternalAddress cardtable((address) ct->byte_map_base);
1.91 + lea(tmp2, cardtable);
1.92 +
1.93 + // get the address of the card
1.94 + addq(card_addr, tmp2);
1.95 +
1.96 + assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1.97 + cmpb(Address(card_addr, 0), 0);
1.98 + jcc(Assembler::equal, done);
1.99 +
1.100 + // storing region crossing non-NULL, card is clean.
1.101 + // dirty card and log.
1.102 +
1.103 + movb(Address(card_addr, 0), 0);
1.104 +
1.105 + cmpl(index, 0);
1.106 + jcc(Assembler::equal, runtime);
1.107 + subl(index, wordSize);
1.108 + movq(tmp2, buffer);
1.109 + movslq(rscratch1, index);
1.110 + addq(tmp2, rscratch1);
1.111 + // log the card
1.112 + movq(Address(tmp2, 0), card_addr);
1.113 + jmp(done);
1.114 +
1.115 + bind(runtime);
1.116 + // save live inputs
1.117 + pushq(store_addr);
1.118 + pushq(new_val);
1.119 + call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
1.120 + popq(new_val);
1.121 + popq(store_addr);
1.122 +
1.123 + bind(done);
1.124 +
1.125 +
1.126 +}
1.127 +
1.128 +#endif // SERIALGC
1.129 +/////////////////////////////////////////////////////////////////////////////
1.130
1.131 // Calls to C land
1.132 //
1.133 @@ -4802,32 +4925,36 @@
1.134 Label& slow_case) {
1.135 assert(obj == rax, "obj must be in rax for cmpxchg");
1.136 assert_different_registers(obj, var_size_in_bytes, t1);
1.137 - Register end = t1;
1.138 - Label retry;
1.139 - bind(retry);
1.140 - ExternalAddress heap_top((address) Universe::heap()->top_addr());
1.141 - movptr(obj, heap_top);
1.142 - if (var_size_in_bytes == noreg) {
1.143 - leaq(end, Address(obj, con_size_in_bytes));
1.144 + if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
1.145 + jmp(slow_case);
1.146 } else {
1.147 - leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
1.148 + Register end = t1;
1.149 + Label retry;
1.150 + bind(retry);
1.151 + ExternalAddress heap_top((address) Universe::heap()->top_addr());
1.152 + movptr(obj, heap_top);
1.153 + if (var_size_in_bytes == noreg) {
1.154 + leaq(end, Address(obj, con_size_in_bytes));
1.155 + } else {
1.156 + leaq(end, Address(obj, var_size_in_bytes, Address::times_1));
1.157 + }
1.158 + // if end < obj then we wrapped around => object too long => slow case
1.159 + cmpq(end, obj);
1.160 + jcc(Assembler::below, slow_case);
1.161 + cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
1.162 +
1.163 + jcc(Assembler::above, slow_case);
1.164 + // Compare obj with the top addr, and if still equal, store the new
1.165 + // top addr in end at the address of the top addr pointer. Sets ZF
1.166 + // if was equal, and clears it otherwise. Use lock prefix for
1.167 + // atomicity on MPs.
1.168 + if (os::is_MP()) {
1.169 + lock();
1.170 + }
1.171 + cmpxchgptr(end, heap_top);
1.172 + // if someone beat us on the allocation, try again, otherwise continue
1.173 + jcc(Assembler::notEqual, retry);
1.174 }
1.175 - // if end < obj then we wrapped around => object too long => slow case
1.176 - cmpq(end, obj);
1.177 - jcc(Assembler::below, slow_case);
1.178 - cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
1.179 -
1.180 - jcc(Assembler::above, slow_case);
1.181 - // Compare obj with the top addr, and if still equal, store the new
1.182 - // top addr in end at the address of the top addr pointer. Sets ZF
1.183 - // if was equal, and clears it otherwise. Use lock prefix for
1.184 - // atomicity on MPs.
1.185 - if (os::is_MP()) {
1.186 - lock();
1.187 - }
1.188 - cmpxchgptr(end, heap_top);
1.189 - // if someone beat us on the allocation, try again, otherwise continue
1.190 - jcc(Assembler::notEqual, retry);
1.191 }
1.192
1.193 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
