jdk8-mips64-public/hotspot: test/gc/g1/TestStringDeduplicationTools.java@fd8ddf2d2f6b

8038461: Test gc/g1/TestStringDeduplicationMemoryUsage.java fails with unexpected memory usage
Reviewed-by: jmasa, sjohanss

     1 /*

     2  * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    24 /*

    25  * Common code for string deduplication tests

    26  */

    28 import java.lang.management.*;

    29 import java.lang.reflect.*;

    30 import java.security.*;

    31 import java.util.*;

    32 import com.oracle.java.testlibrary.*;

    33 import sun.misc.*;

    35 class TestStringDeduplicationTools {

    36     private static final String YoungGC = "YoungGC";

    37     private static final String FullGC  = "FullGC";

    39     private static final int Xmn = 50;  // MB

    40     private static final int Xms = 100; // MB

    41     private static final int Xmx = 100; // MB

    42     private static final int MB = 1024 * 1024;

    43     private static final int StringLength = 50;

    45     private static Field valueField;

    46     private static Unsafe unsafe;

    47     private static byte[] dummy;

    49     static {

    50         try {

    51             Field field = Unsafe.class.getDeclaredField("theUnsafe");

    52             field.setAccessible(true);

    53             unsafe = (Unsafe)field.get(null);

    55             valueField = String.class.getDeclaredField("value");

    56             valueField.setAccessible(true);

    57         } catch (Exception e) {

    58             throw new RuntimeException(e);

    59         }

    60     }

    62     private static Object getValue(String string) {

    63         try {

    64             return valueField.get(string);

    65         } catch (Exception e) {

    66             throw new RuntimeException(e);

    67         }

    68     }

    70     private static void doFullGc(int numberOfTimes) {

    71         for (int i = 0; i < numberOfTimes; i++) {

    72             System.out.println("Begin: Full GC " + (i + 1) + "/" + numberOfTimes);

    73             System.gc();

    74             System.out.println("End: Full GC " + (i + 1) + "/" + numberOfTimes);

    75         }

    76     }

    78     private static void doYoungGc(int numberOfTimes) {

    79         // Provoke at least numberOfTimes young GCs

    80         final int objectSize = 128;

    81         final int maxObjectInYoung = (Xmn * MB) / objectSize;

    82         for (int i = 0; i < numberOfTimes; i++) {

    83             System.out.println("Begin: Young GC " + (i + 1) + "/" + numberOfTimes);

    84             for (int j = 0; j < maxObjectInYoung + 1; j++) {

    85                 dummy = new byte[objectSize];

    86             }

    87             System.out.println("End: Young GC " + (i + 1) + "/" + numberOfTimes);

    88         }

    89     }

    91     private static void forceDeduplication(int ageThreshold, String gcType) {

    92         // Force deduplication to happen by either causing a FullGC or a YoungGC.

    93         // We do several collections to also provoke a situation where the the

    94         // deduplication thread needs to yield while processing the queue. This

    95         // also tests that the references in the deduplication queue are adjusted

    96         // accordingly.

    97         if (gcType.equals(FullGC)) {

    98             doFullGc(3);

    99         } else {

   100             doYoungGc(ageThreshold + 3);

   101         }

   102     }

   104     private static String generateString(int id) {

   105         StringBuilder builder = new StringBuilder(StringLength);

   107         builder.append("DeduplicationTestString:" + id + ":");

   109         while (builder.length() < StringLength) {

   110             builder.append('X');

   111         }

   113         return builder.toString();

   114     }

   116     private static ArrayList<String> createStrings(int total, int unique) {

   117         System.out.println("Creating strings: total=" + total + ", unique=" + unique);

   118         if (total % unique != 0) {

   119             throw new RuntimeException("Total must be divisible by unique");

   120         }

   122         ArrayList<String> list = new ArrayList<String>(total);

   123         for (int j = 0; j < total / unique; j++) {

   124             for (int i = 0; i < unique; i++) {

   125                 list.add(generateString(i));

   126             }

   127         }

   129         return list;

   130     }

   132     private static void verifyStrings(ArrayList<String> list, int uniqueExpected) {

   133         for (;;) {

   134             // Check number of deduplicated strings

   135             ArrayList<Object> unique = new ArrayList<Object>(uniqueExpected);

   136             for (String string: list) {

   137                 Object value = getValue(string);

   138                 boolean uniqueValue = true;

   139                 for (Object obj: unique) {

   140                     if (obj == value) {

   141                         uniqueValue = false;

   142                         break;

   143                     }

   144                 }

   146                 if (uniqueValue) {

   147                     unique.add(value);

   148                 }

   149             }

   151             System.out.println("Verifying strings: total=" + list.size() +

   152                                ", uniqueFound=" + unique.size() +

   153                                ", uniqueExpected=" + uniqueExpected);

   155             if (unique.size() == uniqueExpected) {

   156                 System.out.println("Deduplication completed");

   157                 break;

   158             } else {

   159                 System.out.println("Deduplication not completed, waiting...");

   161                 // Give the deduplication thread time to complete

   162                 try {

   163                     Thread.sleep(1000);

   164                 } catch (Exception e) {

   165                     throw new RuntimeException(e);

   166                 }

   167             }

   168         }

   169     }

   171     private static OutputAnalyzer runTest(String... extraArgs) throws Exception {

   172         String[] defaultArgs = new String[] {

   173             "-Xmn" + Xmn + "m",

   174             "-Xms" + Xms + "m",

   175             "-Xmx" + Xmx + "m",

   176             "-XX:+UseG1GC",

   177             "-XX:+UnlockDiagnosticVMOptions",

   178             "-XX:+VerifyAfterGC" // Always verify after GC

   179         };

   181         ArrayList<String> args = new ArrayList<String>();

   182         args.addAll(Arrays.asList(defaultArgs));

   183         args.addAll(Arrays.asList(extraArgs));

   185         ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args.toArray(new String[args.size()]));

   186         OutputAnalyzer output = new OutputAnalyzer(pb.start());

   187         System.err.println(output.getStderr());

   188         System.out.println(output.getStdout());

   189         return output;

   190     }

   192     private static class DeduplicationTest {

   193         public static void main(String[] args) {

   194             System.out.println("Begin: DeduplicationTest");

   196             final int numberOfStrings = Integer.parseUnsignedInt(args[0]);

   197             final int numberOfUniqueStrings = Integer.parseUnsignedInt(args[1]);

   198             final int ageThreshold = Integer.parseUnsignedInt(args[2]);

   199             final String gcType = args[3];

   201             ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);

   202             forceDeduplication(ageThreshold, gcType);

   203             verifyStrings(list, numberOfUniqueStrings);

   205             System.out.println("End: DeduplicationTest");

   206         }

   208         public static OutputAnalyzer run(int numberOfStrings, int ageThreshold, String gcType, String... extraArgs) throws Exception {

   209             String[] defaultArgs = new String[] {

   210                 "-XX:+UseStringDeduplication",

   211                 "-XX:StringDeduplicationAgeThreshold=" + ageThreshold,

   212                 DeduplicationTest.class.getName(),

   213                 "" + numberOfStrings,

   214                 "" + numberOfStrings / 2,

   215                 "" + ageThreshold,

   216                 gcType

   217             };

   219             ArrayList<String> args = new ArrayList<String>();

   220             args.addAll(Arrays.asList(extraArgs));

   221             args.addAll(Arrays.asList(defaultArgs));

   223             return runTest(args.toArray(new String[args.size()]));

   224         }

   225     }

   227     private static class InternedTest {

   228         public static void main(String[] args) {

   229             // This test verifies that interned strings are always

   230             // deduplicated when being interned, and never after

   231             // being interned.

   233             System.out.println("Begin: InternedTest");

   235             final int ageThreshold = Integer.parseUnsignedInt(args[0]);

   236             final String baseString = "DeduplicationTestString:" + InternedTest.class.getName();

   238             // Create duplicate of baseString

   239             StringBuilder sb1 = new StringBuilder(baseString);

   240             String dupString1 = sb1.toString();

   241             if (getValue(dupString1) == getValue(baseString)) {

   242                 throw new RuntimeException("Values should not match");

   243             }

   245             // Force baseString to be inspected for deduplication

   246             // and be inserted into the deduplication hashtable.

   247             forceDeduplication(ageThreshold, FullGC);

   249             // Wait for deduplication to occur

   250             while (getValue(dupString1) != getValue(baseString)) {

   251                 System.out.println("Waiting...");

   252                 try {

   253                     Thread.sleep(100);

   254                 } catch (Exception e) {

   255                     throw new RuntimeException(e);

   256                 }

   257             }

   259             // Create a new duplicate of baseString

   260             StringBuilder sb2 = new StringBuilder(baseString);

   261             String dupString2 = sb2.toString();

   262             if (getValue(dupString2) == getValue(baseString)) {

   263                 throw new RuntimeException("Values should not match");

   264             }

   266             // Intern the new duplicate

   267             Object beforeInternedValue = getValue(dupString2);

   268             String internedString = dupString2.intern();

   269             if (internedString != dupString2) {

   270                 throw new RuntimeException("String should match");

   271             }

   272             if (getValue(internedString) != getValue(baseString)) {

   273                 throw new RuntimeException("Values should match");

   274             }

   276             // Check original value of interned string, to make sure

   277             // deduplication happened on the interned string and not

   278             // on the base string

   279             if (beforeInternedValue == getValue(baseString)) {

   280                 throw new RuntimeException("Values should not match");

   281             }

   283             System.out.println("End: InternedTest");

   284         }

   286         public static OutputAnalyzer run() throws Exception {

   287             return runTest("-XX:+PrintGC",

   288                            "-XX:+PrintGCDetails",

   289                            "-XX:+UseStringDeduplication",

   290                            "-XX:+PrintStringDeduplicationStatistics",

   291                            "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold,

   292                            InternedTest.class.getName(),

   293                            "" + DefaultAgeThreshold);

   294         }

   295     }

   297     private static class MemoryUsageTest {

   298         public static void main(String[] args) {

   299             System.out.println("Begin: MemoryUsageTest");

   301             final boolean useStringDeduplication = Boolean.parseBoolean(args[0]);

   302             final int numberOfStrings = LargeNumberOfStrings;

   303             final int numberOfUniqueStrings = 1;

   305             ArrayList<String> list = createStrings(numberOfStrings, numberOfUniqueStrings);

   306             forceDeduplication(DefaultAgeThreshold, FullGC);

   308             if (useStringDeduplication) {

   309                 verifyStrings(list, numberOfUniqueStrings);

   310             }

   312             System.gc();

   314             System.out.println("Heap Memory Usage: " + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed());

   315             System.out.println("Array Header Size: " + unsafe.ARRAY_CHAR_BASE_OFFSET);

   317             System.out.println("End: MemoryUsageTest");

   318         }

   320         public static OutputAnalyzer run(boolean useStringDeduplication) throws Exception {

   321             String[] extraArgs = new String[0];

   323             if (useStringDeduplication) {

   324                 extraArgs = new String[] {

   325                     "-XX:+UseStringDeduplication",

   326                     "-XX:+PrintStringDeduplicationStatistics",

   327                     "-XX:StringDeduplicationAgeThreshold=" + DefaultAgeThreshold

   328                 };

   329             }

   331             String[] defaultArgs = new String[] {

   332                 "-XX:+PrintGC",

   333                 "-XX:+PrintGCDetails",

   334                 MemoryUsageTest.class.getName(),

   335                 "" + useStringDeduplication

   336             };

   338             ArrayList<String> args = new ArrayList<String>();

   339             args.addAll(Arrays.asList(extraArgs));

   340             args.addAll(Arrays.asList(defaultArgs));

   342             return runTest(args.toArray(new String[args.size()]));

   343         }

   344     }

   346     /*

   347      * Tests

   348      */

   350     private static final int LargeNumberOfStrings = 10000;

   351     private static final int SmallNumberOfStrings = 10;

   353     private static final int MaxAgeThreshold      = 15;

   354     private static final int DefaultAgeThreshold  = 3;

   355     private static final int MinAgeThreshold      = 1;

   357     private static final int TooLowAgeThreshold   = MinAgeThreshold - 1;

   358     private static final int TooHighAgeThreshold  = MaxAgeThreshold + 1;

   360     public static void testYoungGC() throws Exception {

   361         // Do young GC to age strings to provoke deduplication

   362         OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,

   363                                                       DefaultAgeThreshold,

   364                                                       YoungGC,

   365                                                       "-XX:+PrintGC",

   366                                                       "-XX:+PrintStringDeduplicationStatistics");

   367         output.shouldNotContain("Full GC");

   368         output.shouldContain("GC pause (G1 Evacuation Pause) (young)");

   369         output.shouldContain("GC concurrent-string-deduplication");

   370         output.shouldContain("Deduplicated:");

   371         output.shouldHaveExitValue(0);

   372     }

   374     public static void testFullGC() throws Exception {

   375         // Do full GC to age strings to provoke deduplication

   376         OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,

   377                                                       DefaultAgeThreshold,

   378                                                       FullGC,

   379                                                       "-XX:+PrintGC",

   380                                                       "-XX:+PrintStringDeduplicationStatistics");

   381         output.shouldNotContain("GC pause (G1 Evacuation Pause) (young)");

   382         output.shouldContain("Full GC");

   383         output.shouldContain("GC concurrent-string-deduplication");

   384         output.shouldContain("Deduplicated:");

   385         output.shouldHaveExitValue(0);

   386     }

   388     public static void testTableResize() throws Exception {

   389         // Test with StringDeduplicationResizeALot

   390         OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,

   391                                                       DefaultAgeThreshold,

   392                                                       YoungGC,

   393                                                       "-XX:+PrintGC",

   394                                                       "-XX:+PrintStringDeduplicationStatistics",

   395                                                       "-XX:+StringDeduplicationResizeALot");

   396         output.shouldContain("GC concurrent-string-deduplication");

   397         output.shouldContain("Deduplicated:");

   398         output.shouldNotContain("Resize Count: 0");

   399         output.shouldHaveExitValue(0);

   400     }

   402     public static void testTableRehash() throws Exception {

   403         // Test with StringDeduplicationRehashALot

   404         OutputAnalyzer output = DeduplicationTest.run(LargeNumberOfStrings,

   405                                                       DefaultAgeThreshold,

   406                                                       YoungGC,

   407                                                       "-XX:+PrintGC",

   408                                                       "-XX:+PrintStringDeduplicationStatistics",

   409                                                       "-XX:+StringDeduplicationRehashALot");

   410         output.shouldContain("GC concurrent-string-deduplication");

   411         output.shouldContain("Deduplicated:");

   412         output.shouldNotContain("Rehash Count: 0");

   413         output.shouldNotContain("Hash Seed: 0x0");

   414         output.shouldHaveExitValue(0);

   415     }

   417     public static void testAgeThreshold() throws Exception {

   418         OutputAnalyzer output;

   420         // Test with max age theshold

   421         output = DeduplicationTest.run(SmallNumberOfStrings,

   422                                        MaxAgeThreshold,

   423                                        YoungGC,

   424                                        "-XX:+PrintGC",

   425                                        "-XX:+PrintStringDeduplicationStatistics");

   426         output.shouldContain("GC concurrent-string-deduplication");

   427         output.shouldContain("Deduplicated:");

   428         output.shouldHaveExitValue(0);

   430         // Test with min age theshold

   431         output = DeduplicationTest.run(SmallNumberOfStrings,

   432                                        MinAgeThreshold,

   433                                        YoungGC,

   434                                        "-XX:+PrintGC",

   435                                        "-XX:+PrintStringDeduplicationStatistics");

   436         output.shouldContain("GC concurrent-string-deduplication");

   437         output.shouldContain("Deduplicated:");

   438         output.shouldHaveExitValue(0);

   440         // Test with too low age threshold

   441         output = DeduplicationTest.run(SmallNumberOfStrings,

   442                                        TooLowAgeThreshold,

   443                                        YoungGC);

   444         output.shouldContain("StringDeduplicationAgeThreshold of " + TooLowAgeThreshold +

   445                              " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);

   446         output.shouldHaveExitValue(1);

   448         // Test with too high age threshold

   449         output = DeduplicationTest.run(SmallNumberOfStrings,

   450                                        TooHighAgeThreshold,

   451                                        YoungGC);

   452         output.shouldContain("StringDeduplicationAgeThreshold of " + TooHighAgeThreshold +

   453                              " is invalid; must be between " + MinAgeThreshold + " and " + MaxAgeThreshold);

   454         output.shouldHaveExitValue(1);

   455     }

   457     public static void testPrintOptions() throws Exception {

   458         OutputAnalyzer output;

   460         // Test without PrintGC and without PrintStringDeduplicationStatistics

   461         output = DeduplicationTest.run(SmallNumberOfStrings,

   462                                        DefaultAgeThreshold,

   463                                        YoungGC);

   464         output.shouldNotContain("GC concurrent-string-deduplication");

   465         output.shouldNotContain("Deduplicated:");

   466         output.shouldHaveExitValue(0);

   468         // Test with PrintGC but without PrintStringDeduplicationStatistics

   469         output = DeduplicationTest.run(SmallNumberOfStrings,

   470                                        DefaultAgeThreshold,

   471                                        YoungGC,

   472                                        "-XX:+PrintGC");

   473         output.shouldContain("GC concurrent-string-deduplication");

   474         output.shouldNotContain("Deduplicated:");

   475         output.shouldHaveExitValue(0);

   476     }

   478     public static void testInterned() throws Exception {

   479         // Test that interned strings are deduplicated before being interned

   480         OutputAnalyzer output = InternedTest.run();

   481         output.shouldHaveExitValue(0);

   482     }

   484     public static void testMemoryUsage() throws Exception {

   485         // Test that memory usage is reduced after deduplication

   486         OutputAnalyzer output;

   487         final String heapMemoryUsagePattern = "Heap Memory Usage: (\\d+)";

   488         final String arrayHeaderSizePattern = "Array Header Size: (\\d+)";

   490         // Run without deduplication

   491         output = MemoryUsageTest.run(false);

   492         output.shouldHaveExitValue(0);

   493         final long heapMemoryUsageWithoutDedup = Long.parseLong(output.firstMatch(heapMemoryUsagePattern, 1));

   494         final long arrayHeaderSizeWithoutDedup = Long.parseLong(output.firstMatch(arrayHeaderSizePattern, 1));

   496         // Run with deduplication

   497         output = MemoryUsageTest.run(true);

   498         output.shouldHaveExitValue(0);

   499         final long heapMemoryUsageWithDedup = Long.parseLong(output.firstMatch(heapMemoryUsagePattern, 1));

   500         final long arrayHeaderSizeWithDedup = Long.parseLong(output.firstMatch(arrayHeaderSizePattern, 1));

   502         // Sanity check to make sure one instance isn't using compressed class pointers and the other not

   503         if (arrayHeaderSizeWithoutDedup != arrayHeaderSizeWithDedup) {

   504             throw new Exception("Unexpected difference between array header sizes");

   505         }

   507         // Calculate expected memory usage with deduplication enabled. This calculation does

   508         // not take alignment and padding into account, so it's a conservative estimate.

   509         final long sizeOfChar = unsafe.ARRAY_CHAR_INDEX_SCALE;

   510         final long sizeOfCharArray = StringLength * sizeOfChar + arrayHeaderSizeWithoutDedup;

   511         final long bytesSaved = (LargeNumberOfStrings - 1) * sizeOfCharArray;

   512         final long heapMemoryUsageWithDedupExpected = heapMemoryUsageWithoutDedup - bytesSaved;

   514         System.out.println("Memory usage summary:");

   515         System.out.println("   heapMemoryUsageWithoutDedup:      " + heapMemoryUsageWithoutDedup);

   516         System.out.println("   heapMemoryUsageWithDedup:         " + heapMemoryUsageWithDedup);

   517         System.out.println("   heapMemoryUsageWithDedupExpected: " + heapMemoryUsageWithDedupExpected);

   519         if (heapMemoryUsageWithDedup > heapMemoryUsageWithDedupExpected) {

   520             throw new Exception("Unexpected memory usage, heapMemoryUsageWithDedup should be less or equal to heapMemoryUsageWithDedupExpected");

   521         }

   522     }

   523 }

Mercurial > jdk8-mips64-public > hotspot / file revision

test/gc/g1/TestStringDeduplicationTools.java@fd8ddf2d2f6b

test/gc/g1/TestStringDeduplicationTools.java