1.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
1.2 +++ b/test/gc/g1/TestStringDeduplicationTools.java Tue Mar 18 19:07:22 2014 +0100
1.3 @@ -0,0 +1,512 @@
1.4 +/*
1.5 + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
1.6 + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
1.7 + *
1.8 + * This code is free software; you can redistribute it and/or modify it
1.9 + * under the terms of the GNU General Public License version 2 only, as
1.10 + * published by the Free Software Foundation.
1.11 + *
1.12 + * This code is distributed in the hope that it will be useful, but WITHOUT
1.13 + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
1.14 + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
1.15 + * version 2 for more details (a copy is included in the LICENSE file that
1.16 + * accompanied this code).
1.17 + *
1.18 + * You should have received a copy of the GNU General Public License version
1.19 + * 2 along with this work; if not, write to the Free Software Foundation,
1.20 + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
1.21 + *
1.22 + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
1.23 + * or visit www.oracle.com if you need additional information or have any
1.24 + * questions.
1.25 + */
1.26 +
1.27 +/*
1.28 + * Common code for string deduplication tests
1.29 + */
1.30 +
1.31 +import java.lang.management.*;
1.32 +import java.lang.reflect.*;
1.33 +import java.security.*;
1.34 +import java.util.*;
1.35 +import com.oracle.java.testlibrary.*;
1.36 +import sun.misc.*;
1.37 +
1.38 +class TestStringDeduplicationTools {
1.39 + private static final String YoungGC = "YoungGC";
1.40 + private static final String FullGC = "FullGC";
1.41 +
1.42 + private static final int Xmn = 50; // MB
1.43 + private static final int Xms = 100; // MB
1.44 + private static final int Xmx = 100; // MB
1.45 + private static final int MB = 1024 * 1024;
1.46 + private static final int StringLength = 50;
1.47 +
1.48 + private static Field valueField;
1.49 + private static Unsafe unsafe;
1.50 + private static byte[] dummy;
1.51 +
1.52 + static {
1.53 + try {
1.54 + Field field = Unsafe.class.getDeclaredField("theUnsafe");
1.55 + field.setAccessible(true);
1.56 + unsafe = (Unsafe)field.get(null);
1.57 +
1.58 + valueField = String.class.getDeclaredField("value");
1.59 + valueField.setAccessible(true);
1.60 + } catch (Exception e) {
1.61 + throw new RuntimeException(e);
1.62 + }
1.63 + }
1.64 +
1.65 + private static Object getValue(String string) {
1.66 + try {
1.67 + return valueField.get(string);
1.68 + } catch (Exception e) {
1.69 + throw new RuntimeException(e);
1.70 + }
1.71 + }
1.72 +
1.73 + private static void doFullGc(int numberOfTimes) {
1.74 + for (int i = 0; i < numberOfTimes; i++) {
1.75 + System.out.println("Begin: Full GC " + (i + 1) + "/" + numberOfTimes);
1.76 + System.gc();
1.77 + System.out.println("End: Full GC " + (i + 1) + "/" + numberOfTimes);
1.78 + }
1.79 + }
1.80 +
1.81 + private static void doYoungGc(int numberOfTimes) {
1.82 + // Provoke at least numberOfTimes young GCs
1.83 + final int objectSize = 128;
1.84 + final int maxObjectInYoung = (Xmn * MB) / objectSize;
1.85 + for (int i = 0; i < numberOfTimes; i++) {
1.86 + System.out.println("Begin: Young GC " + (i + 1) + "/" + numberOfTimes);
1.87 + for (int j = 0; j < maxObjectInYoung + 1; j++) {
1.88 + dummy = new byte[objectSize];
1.89 + }
1.90 + System.out.println("End: Young GC " + (i + 1) + "/" + numberOfTimes);
1.91 + }
1.92 + }
1.93 +
1.94 + private static void forceDeduplication(int ageThreshold, String gcType) {
1.95 + // Force deduplication to happen by either causing a FullGC or a YoungGC.
1.96 + // We do several collections to also provoke a situation where the the
1.97 + // deduplication thread needs to yield while processing the queue. This
1.98 + // also tests that the references in the deduplication queue are adjusted
1.99 + // accordingly.
1.100 + if (gcType.equals(FullGC)) {
1.101 + doFullGc(3);
1.102 + } else {
1.103 + doYoungGc(ageThreshold + 3);
1.104 + }
1.105 + }
1.106 +
1.107 + private static String generateString(int id) {
1.108 + StringBuilder builder = new StringBuilder(StringLength);
1.109 +
1.110 + builder.append("DeduplicationTestString:" + id + ":");
1.111 +
1.112 + while (builder.length() < StringLength) {
1.113 + builder.append('X');
1.114 + }
1.115 +
1.116 + return builder.toString();
1.117 + }
1.118 +
1.119 + private static ArrayList<String> createStrings(int total, int unique) {
1.120 + System.out.println("Creating strings: total=" + total + ", unique=" + unique);
1.121 + if (total % unique != 0) {
1.122 + throw new RuntimeException("Total must be divisible by unique");
1.123 + }
1.124 +
1.125 + ArrayList<String> list = new ArrayList<String>(total);
1.126 + for (int j = 0; j < total / unique; j++) {
1.127 + for (int i = 0; i < unique; i++) {
1.128 + list.add(generateString(i));
1.129 + }
1.130 + }
1.131 +
1.132 + return list;
1.133 + }
1.134 +
1.135 + private static void verifyStrings(ArrayList<String> list, int uniqueExpected) {
1.136 + for (;;) {
1.137 + // Check number of deduplicated strings
1.138 + ArrayList<Object> unique = new ArrayList<Object>(uniqueExpected);
1.139 + for (String string: list) {
1.140 + Object value = getValue(string);
1.141 + boolean uniqueValue = true;
1.142 + for (Object obj: unique) {
1.143 + if (obj == value) {
1.144 + uniqueValue = false;
1.145 + break;
1.146 + }
1.147 + }
1.148 +
1.149 + if (uniqueValue) {
1.150 + unique.add(value);
1.151 + }
1.152 + }
1.153 +
1.154 + System.out.println("Verifying strings: total=" + list.size() +
1.155 + ", uniqueFound=" + unique.size() +
1.156 + ", uniqueExpected=" + uniqueExpected);
1.157 +
1.158 + if (unique.size() == uniqueExpected) {
1.159 + System.out.println("Deduplication completed");
1.160 + break;
1.161 + } else {
1.162 + System.out.println("Deduplication not completed, waiting...");
1.163 +
1.164 + // Give the deduplication thread time to complete
1.165 + try {
1.166 + Thread.sleep(1000);
1.167 + } catch (Exception e) {
1.168 + throw new RuntimeException(e);
1.169 + }
1.170 + }
1.171 + }
1.172 + }
1.173 +
1.174 + private static OutputAnalyzer runTest(String... extraArgs) throws Exception {
1.175 + String[] defaultArgs = new String[] {
1.176 + "-Xmn" + Xmn + "m",
1.177 + "-Xms" + Xms + "m",
1.178 + "-Xmx" + Xmx + "m",
1.179 + "-XX:+UseG1GC",
1.180 + "-XX:+UnlockDiagnosticVMOptions",
1.181 + "-XX:+VerifyAfterGC" // Always verify after GC
1.182 + };
1.183 +
1.184 + ArrayList<String> args = new ArrayList<String>();
1.185 + args.addAll(Arrays.asList(defaultArgs));
1.186 + args.addAll(Arrays.asList(extraArgs));
1.187 +
1.188 + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args.toArray(new String[args.size()]));
1.189 + OutputAnalyzer output = new OutputAnalyzer(pb.start());
1.190 + System.err.println(output.getStderr());
1.191 + System.out.println(output.getStdout());
1.192 + return output;
1.193 + }
1.194 +
1.195 + private static class DeduplicationTest {
1.196 + public static void main(String[] args) {
1.197 + System.out.println("Begin: DeduplicationTest");
1.198 +
1.199 + final int numberOfStrings = Integer.parseUnsignedInt(args[0]);
1.200 + final int numberOfUniqueStrings = Integer.parseUnsignedInt(args[1]);
1.201 + final int ageThreshold = Integer.parseUnsignedInt(args[2]);
1.202 + final String gcType = args[3];
1.203 +
1.204 + ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);
1.205 + forceDeduplication(ageThreshold, gcType);
1.206 + verifyStrings(list, numberOfUniqueStrings);
1.207 +
1.208 + System.out.println("End: DeduplicationTest");
1.209 + }
1.210 +
1.211 + public static OutputAnalyzer run(int numberOfStrings, int ageThreshold, String gcType, String... extraArgs) throws Exception {
1.212 + String[] defaultArgs = new String[] {
1.213 + "-XX:+UseStringDeduplication",
1.214 + "-XX:StringDeduplicationAgeThreshold=" + ageThreshold,
1.215 + DeduplicationTest.class.getName(),
1.216 + "" + numberOfStrings,
1.217 + "" + numberOfStrings / 2,
1.218 + "" + ageThreshold,
1.219 + gcType
1.220 + };
1.221 +
1.222 + ArrayList<String> args = new ArrayList<String>();
1.223 + args.addAll(Arrays.asList(extraArgs));
1.224 + args.addAll(Arrays.asList(defaultArgs));
1.225 +
1.226 + return runTest(args.toArray(new String[args.size()]));
1.227 + }
1.228 + }
1.229 +
1.230 + private static class InternedTest {
1.231 + public static void main(String[] args) {
1.232 + // This test verifies that interned strings are always
1.233 + // deduplicated when being interned, and never after
1.234 + // being interned.
1.235 +
1.236 + System.out.println("Begin: InternedTest");
1.237 +
1.238 + final int ageThreshold = Integer.parseUnsignedInt(args[0]);
1.239 + final String baseString = "DeduplicationTestString:" + InternedTest.class.getName();
1.240 +
1.241 + // Create duplicate of baseString
1.242 + StringBuilder sb1 = new StringBuilder(baseString);
1.243 + String dupString1 = sb1.toString();
1.244 + if (getValue(dupString1) == getValue(baseString)) {
1.245 + throw new RuntimeException("Values should not match");
1.246 + }
1.247 +
1.248 + // Force baseString to be inspected for deduplication
1.249 + // and be inserted into the deduplication hashtable.
1.250 + forceDeduplication(ageThreshold, FullGC);
1.251 +
1.252 + // Wait for deduplication to occur
1.253 + while (getValue(dupString1) != getValue(baseString)) {
1.254 + System.out.println("Waiting...");
1.255 + try {
1.256 + Thread.sleep(100);
1.257 + } catch (Exception e) {
1.258 + throw new RuntimeException(e);
1.259 + }
1.260 + }
1.261 +
1.262 + // Create a new duplicate of baseString
1.263 + StringBuilder sb2 = new StringBuilder(baseString);
1.264 + String dupString2 = sb2.toString();
1.265 + if (getValue(dupString2) == getValue(baseString)) {
1.266 + throw new RuntimeException("Values should not match");
1.267 + }
1.268 +
1.269 + // Intern the new duplicate
1.270 + Object beforeInternedValue = getValue(dupString2);
1.271 + String internedString = dupString2.intern();
1.272 + if (internedString != dupString2) {
1.273 + throw new RuntimeException("String should match");
1.274 + }
1.275 + if (getValue(internedString) != getValue(baseString)) {
1.276 + throw new RuntimeException("Values should match");
1.277 + }
1.278 +
1.279 + // Check original value of interned string, to make sure
1.280 + // deduplication happened on the interned string and not
1.281 + // on the base string
1.282 + if (beforeInternedValue == getValue(baseString)) {
1.283 + throw new RuntimeException("Values should not match");
1.284 + }
1.285 +
1.286 + System.out.println("End: InternedTest");
1.287 + }
1.288 +
1.289 + public static OutputAnalyzer run() throws Exception {
1.290 + return runTest("-XX:+PrintGC",
1.291 + "-XX:+PrintGCDetails",
1.292 + "-XX:+UseStringDeduplication",
1.293 + "-XX:+PrintStringDeduplicationStatistics",
1.294 + "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold,
1.295 + InternedTest.class.getName(),
1.296 + "" + DefaultAgeThreshold);
1.297 + }
1.298 + }
1.299 +
1.300 + private static class MemoryUsageTest {
1.301 + public static void main(String[] args) {
1.302 + System.out.println("Begin: MemoryUsageTest");
1.303 +
1.304 + final boolean useStringDeduplication = Boolean.parseBoolean(args[0]);
1.305 + final int numberOfStrings = LargeNumberOfStrings;
1.306 + final int numberOfUniqueStrings = 1;
1.307 +
1.308 + ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);
1.309 + forceDeduplication(DefaultAgeThreshold, FullGC);
1.310 +
1.311 + if (useStringDeduplication) {
1.312 + verifyStrings(list, numberOfUniqueStrings);
1.313 + }
1.314 +
1.315 + System.gc();
1.316 + System.out.println("Heap Memory Usage: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed());
1.317 +
1.318 + System.out.println("End: MemoryUsageTest");
1.319 + }
1.320 +
1.321 + public static OutputAnalyzer run(boolean useStringDeduplication) throws Exception {
1.322 + String[] extraArgs = new String[0];
1.323 +
1.324 + if (useStringDeduplication) {
1.325 + extraArgs = new String[] {
1.326 + "-XX:+UseStringDeduplication",
1.327 + "-XX:+PrintStringDeduplicationStatistics",
1.328 + "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold
1.329 + };
1.330 + }
1.331 +
1.332 + String[] defaultArgs = new String[] {
1.333 + "-XX:+PrintGC",
1.334 + "-XX:+PrintGCDetails",
1.335 + MemoryUsageTest.class.getName(),
1.336 + "" + useStringDeduplication
1.337 + };
1.338 +
1.339 + ArrayList<String> args = new ArrayList<String>();
1.340 + args.addAll(Arrays.asList(extraArgs));
1.341 + args.addAll(Arrays.asList(defaultArgs));
1.342 +
1.343 + return runTest(args.toArray(new String[args.size()]));
1.344 + }
1.345 + }
1.346 +
1.347 + /*
1.348 + * Tests
1.349 + */
1.350 +
1.351 + private static final int LargeNumberOfStrings = 10000;
1.352 + private static final int SmallNumberOfStrings = 10;
1.353 +
1.354 + private static final int MaxAgeThreshold = 15;
1.355 + private static final int DefaultAgeThreshold = 3;
1.356 + private static final int MinAgeThreshold = 1;
1.357 +
1.358 + private static final int TooLowAgeThreshold = MinAgeThreshold - 1;
1.359 + private static final int TooHighAgeThreshold = MaxAgeThreshold + 1;
1.360 +
1.361 + public static void testYoungGC() throws Exception {
1.362 + // Do young GC to age strings to provoke deduplication
1.363 + OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
1.364 + DefaultAgeThreshold,
1.365 + YoungGC,
1.366 + "-XX:+PrintGC",
1.367 + "-XX:+PrintStringDeduplicationStatistics");
1.368 + output.shouldNotContain("Full GC");
1.369 + output.shouldContain("GC pause (G1 Evacuation Pause) (young)");
1.370 + output.shouldContain("GC concurrent-string-deduplication");
1.371 + output.shouldContain("Deduplicated:");
1.372 + output.shouldHaveExitValue(0);
1.373 + }
1.374 +
1.375 + public static void testFullGC() throws Exception {
1.376 + // Do full GC to age strings to provoke deduplication
1.377 + OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
1.378 + DefaultAgeThreshold,
1.379 + FullGC,
1.380 + "-XX:+PrintGC",
1.381 + "-XX:+PrintStringDeduplicationStatistics");
1.382 + output.shouldNotContain("GC pause (G1 Evacuation Pause) (young)");
1.383 + output.shouldContain("Full GC");
1.384 + output.shouldContain("GC concurrent-string-deduplication");
1.385 + output.shouldContain("Deduplicated:");
1.386 + output.shouldHaveExitValue(0);
1.387 + }
1.388 +
1.389 + public static void testTableResize() throws Exception {
1.390 + // Test with StringDeduplicationResizeALot
1.391 + OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
1.392 + DefaultAgeThreshold,
1.393 + YoungGC,
1.394 + "-XX:+PrintGC",
1.395 + "-XX:+PrintStringDeduplicationStatistics",
1.396 + "-XX:+StringDeduplicationResizeALot");
1.397 + output.shouldContain("GC concurrent-string-deduplication");
1.398 + output.shouldContain("Deduplicated:");
1.399 + output.shouldNotContain("Resize Count: 0");
1.400 + output.shouldHaveExitValue(0);
1.401 + }
1.402 +
1.403 + public static void testTableRehash() throws Exception {
1.404 + // Test with StringDeduplicationRehashALot
1.405 + OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,
1.406 + DefaultAgeThreshold,
1.407 + YoungGC,
1.408 + "-XX:+PrintGC",
1.409 + "-XX:+PrintStringDeduplicationStatistics",
1.410 + "-XX:+StringDeduplicationRehashALot");
1.411 + output.shouldContain("GC concurrent-string-deduplication");
1.412 + output.shouldContain("Deduplicated:");
1.413 + output.shouldNotContain("Rehash Count: 0");
1.414 + output.shouldNotContain("Hash Seed: 0x0");
1.415 + output.shouldHaveExitValue(0);
1.416 + }
1.417 +
1.418 + public static void testAgeThreshold() throws Exception {
1.419 + OutputAnalyzer output;
1.420 +
1.421 + // Test with max age theshold
1.422 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.423 + MaxAgeThreshold,
1.424 + YoungGC,
1.425 + "-XX:+PrintGC",
1.426 + "-XX:+PrintStringDeduplicationStatistics");
1.427 + output.shouldContain("GC concurrent-string-deduplication");
1.428 + output.shouldContain("Deduplicated:");
1.429 + output.shouldHaveExitValue(0);
1.430 +
1.431 + // Test with min age theshold
1.432 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.433 + MinAgeThreshold,
1.434 + YoungGC,
1.435 + "-XX:+PrintGC",
1.436 + "-XX:+PrintStringDeduplicationStatistics");
1.437 + output.shouldContain("GC concurrent-string-deduplication");
1.438 + output.shouldContain("Deduplicated:");
1.439 + output.shouldHaveExitValue(0);
1.440 +
1.441 + // Test with too low age threshold
1.442 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.443 + TooLowAgeThreshold,
1.444 + YoungGC);
1.445 + output.shouldContain("StringDeduplicationAgeThreshold of " + TooLowAgeThreshold +
1.446 + " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);
1.447 + output.shouldHaveExitValue(1);
1.448 +
1.449 + // Test with too high age threshold
1.450 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.451 + TooHighAgeThreshold,
1.452 + YoungGC);
1.453 + output.shouldContain("StringDeduplicationAgeThreshold of " + TooHighAgeThreshold +
1.454 + " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);
1.455 + output.shouldHaveExitValue(1);
1.456 + }
1.457 +
1.458 + public static void testPrintOptions() throws Exception {
1.459 + OutputAnalyzer output;
1.460 +
1.461 + // Test without PrintGC and without PrintStringDeduplicationStatistics
1.462 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.463 + DefaultAgeThreshold,
1.464 + YoungGC);
1.465 + output.shouldNotContain("GC concurrent-string-deduplication");
1.466 + output.shouldNotContain("Deduplicated:");
1.467 + output.shouldHaveExitValue(0);
1.468 +
1.469 + // Test with PrintGC but without PrintStringDeduplicationStatistics
1.470 + output = DeduplicationTest.run(SmallNumberOfStrings,
1.471 + DefaultAgeThreshold,
1.472 + YoungGC,
1.473 + "-XX:+PrintGC");
1.474 + output.shouldContain("GC concurrent-string-deduplication");
1.475 + output.shouldNotContain("Deduplicated:");
1.476 + output.shouldHaveExitValue(0);
1.477 + }
1.478 +
1.479 + public static void testInterned() throws Exception {
1.480 + // Test that interned strings are deduplicated before being interned
1.481 + OutputAnalyzer output = InternedTest.run();
1.482 + output.shouldHaveExitValue(0);
1.483 + }
1.484 +
1.485 + public static void testMemoryUsage() throws Exception {
1.486 + // Test that memory usage is reduced after deduplication
1.487 + OutputAnalyzer output;
1.488 + final String usagePattern = "Heap Memory Usage: (\\d+)";
1.489 +
1.490 + // Run without deduplication
1.491 + output = MemoryUsageTest.run(false);
1.492 + output.shouldHaveExitValue(0);
1.493 + final long memoryUsageWithoutDedup = Long.parseLong(output.firstMatch(usagePattern, 1));
1.494 +
1.495 + // Run with deduplication
1.496 + output = MemoryUsageTest.run(true);
1.497 + output.shouldHaveExitValue(0);
1.498 + final long memoryUsageWithDedup = Long.parseLong(output.firstMatch(usagePattern, 1));
1.499 +
1.500 + // Calculate expected memory usage with deduplication enabled. This calculation does
1.501 + // not take alignment and padding into account, so it's a conservative estimate.
1.502 + final long sizeOfChar = 2; // bytes
1.503 + final long bytesSaved = (LargeNumberOfStrings - 1) * (StringLength * sizeOfChar + unsafe.ARRAY_CHAR_BASE_OFFSET);
1.504 + final long memoryUsageWithDedupExpected = memoryUsageWithoutDedup - bytesSaved;
1.505 +
1.506 + System.out.println("Memory usage summary:");
1.507 + System.out.println(" memoryUsageWithoutDedup: " + memoryUsageWithoutDedup);
1.508 + System.out.println(" memoryUsageWithDedup: " + memoryUsageWithDedup);
1.509 + System.out.println(" memoryUsageWithDedupExpected: " + memoryUsageWithDedupExpected);
1.510 +
1.511 + if (memoryUsageWithDedup > memoryUsageWithDedupExpected) {
1.512 + throw new Exception("Unexpected memory usage, memoryUsageWithDedup should less or equal to memoryUsageWithDedupExpected");
1.513 + }
1.514 + }
1.515 +}
