diff -r f03d0a26bf83 -r 3bfae429e2cf src/share/vm/memory/blockOffsetTable.hpp
--- a/src/share/vm/memory/blockOffsetTable.hpp	Thu Apr 22 13:23:15 2010 -0700
+++ b/src/share/vm/memory/blockOffsetTable.hpp	Mon May 03 10:24:51 2010 -0700
@@ -140,14 +140,38 @@
            "right address out of range");
     assert(left  < right, "Heap addresses out of order");
     size_t num_cards = pointer_delta(right, left) >> LogN_words;
-    memset(&_offset_array[index_for(left)], offset, num_cards);
+
+    // Below, we may use an explicit loop instead of memset()
+    // because on certain platforms memset() can give concurrent
+    // readers "out-of-thin-air," phantom zeros; see 6948537.
+    if (UseMemSetInBOT) {
+      memset(&_offset_array[index_for(left)], offset, num_cards);
+    } else {
+      size_t i = index_for(left);
+      const size_t end = i + num_cards;
+      for (; i < end; i++) {
+        _offset_array[i] = offset;
+      }
+    }
   }
 
   void set_offset_array(size_t left, size_t right, u_char offset) {
     assert(right < _vs.committed_size(), "right address out of range");
     assert(left  <= right, "indexes out of order");
     size_t num_cards = right - left + 1;
-    memset(&_offset_array[left], offset, num_cards);
+
+    // Below, we may use an explicit loop instead of memset
+    // because on certain platforms memset() can give concurrent
+    // readers "out-of-thin-air," phantom zeros; see 6948537.
+    if (UseMemSetInBOT) {
+      memset(&_offset_array[left], offset, num_cards);
+    } else {
+      size_t i = left;
+      const size_t end = i + num_cards;
+      for (; i < end; i++) {
+        _offset_array[i] = offset;
+      }
+    }
   }
 
   void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const {