8342283: CDS cannot handle a large number of classes

src/java.base/share/classes/java/lang/ClassLoader.java

@@ -2565,22 +2565,38 @@ private void initializeJavaAssertionMaps() {
      */
     private boolean trySetObjectField(String name, Object obj) {
         Unsafe unsafe = Unsafe.getUnsafe();
-        Class<?> k = ClassLoader.class;
-        long offset;
-        offset = unsafe.objectFieldOffset(k, name);
+        long offset = unsafe.objectFieldOffset(ClassLoader.class, name);
         return unsafe.compareAndSetReference(this, offset, null, obj);
     }
 
+    private void reinitObjectField(String name, Object obj) {
+        Unsafe unsafe = Unsafe.getUnsafe();
+        long offset = unsafe.objectFieldOffset(ClassLoader.class, name);
+
+        // Extra safety: check the types
+        Object current = unsafe.getReference(this, offset);
+        if (current.getClass() != obj.getClass()) {
+            throw new IllegalStateException("Wrong field type");
+        }
+
+        unsafe.putReference(this, offset, obj);
+    }
+
     /**
-     * Called by the VM, during -Xshare:dump
+     * Called only by the VM, during -Xshare:dump.
+     *
+     * @implNote This is done while the JVM is running in single-threaded mode,
+     * and at the very end of Java bytecode execution. We know that no more classes
+     * will be loaded and none of the fields modified by this method will be used again.
      */
     private void resetArchivedStates() {
         if (parallelLockMap != null) {
-            parallelLockMap.clear();
+            reinitObjectField("parallelLockMap", new ConcurrentHashMap<>());
         }
-        packages.clear();
-        package2certs.clear();
+        reinitObjectField("packages", new ConcurrentHashMap<>());
+        reinitObjectField("package2certs", new ConcurrentHashMap<>());
         classes.clear();
+        classes.trimToSize();
         classLoaderValueMap = null;
     }
 }

test/hotspot/jtreg/TEST.groups

@@ -449,7 +449,8 @@ hotspot_appcds_dynamic = \
  -runtime/cds/appcds/LambdaWithUseImplMethodHandle.java \
  -runtime/cds/appcds/LambdaWithOldClass.java \
  -runtime/cds/appcds/LongClassListPath.java \
- -runtime/cds/appcds/LotsOfClasses.java \
+ -runtime/cds/appcds/LotsOfJRTClasses.java \
+ -runtime/cds/appcds/LotsOfSyntheticClasses.java \
  -runtime/cds/appcds/MismatchedPathTriggerMemoryRelease.java \
  -runtime/cds/appcds/NonExistClasspath.java \
  -runtime/cds/appcds/RelativePath.java \

test/hotspot/jtreg/runtime/cds/appcds/LotsOfClasses.java → test/hotspot/jtreg/runtime/cds/appcds/LotsOfJRTClasses.java

@@ -33,10 +33,10 @@
  *          this will produce an archive with over 30,000 classes.
  * @requires vm.cds
  * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
- * @run driver/timeout=500 LotsOfClasses
+ * @run driver/timeout=500 LotsOfJRTClasses
  */
 
-public class LotsOfClasses {
+public class LotsOfJRTClasses {
 
     public static void main(String[] args) throws Exception {
         ArrayList<String> list = new ArrayList<>();

test/hotspot/jtreg/runtime/cds/appcds/LotsOfSyntheticClasses.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.Files;
+
+import jdk.test.lib.cds.CDSJarUtils;
+import jdk.test.lib.cds.CDSTestUtils;
+import jdk.test.lib.process.OutputAnalyzer;
+
+/*
+ * @test
+ * @summary Try to archive lots and lots of classes.
+ * @requires vm.cds
+ * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds
+ * @run driver/timeout=500 LotsOfSyntheticClasses
+ */
+
+public class LotsOfSyntheticClasses {
+
+    // Generate 100 top-level classes, each containing 1000 nested classes.
+    // 100K total classes are more than enough to push the CDS limits.
+    private static final int NUM_CLASSES = 100;
+    private static final int NUM_NESTED_CLASSES = 1000;
+
+    private static final Path USER_DIR = Paths.get(CDSTestUtils.getOutputDir());
+    private static final Path APP_JAR = USER_DIR.resolve("test.jar");
+    private static final Path SRC_DIR = USER_DIR.resolve("src");
+
+    private static final String TOP_CLASS_NAME = "Class";
+    private static final String NESTED_CLASS_NAME = "Nested";
+    private static final String MAIN_CLASS_NAME = "Main";
+
+    public static List<String> generateClass(int idx) {
+        List<String> out = new ArrayList<>();
+        out.add("public class " + TOP_CLASS_NAME + idx + " {");
+        out.add("public " + TOP_CLASS_NAME + idx + "() {");
+        for (int c = 0; c < NUM_NESTED_CLASSES; c++) {
+            out.add("new " + NESTED_CLASS_NAME + c + "();");
+        }
+        out.add("}");
+        for (int c = 0; c < NUM_NESTED_CLASSES; c++) {
+            out.add("public static class " + NESTED_CLASS_NAME + c + " {}");
+        }
+        out.add("}");
+        return out;
+    }
+
+    public static List<String> generateMainClass() {
+        List<String> out = new ArrayList<>();
+        out.add("public class " + MAIN_CLASS_NAME + " {");
+        out.add("public static void main(String... args) {");
+        for (int c = 0; c < NUM_CLASSES; c++) {
+            out.add("new " + TOP_CLASS_NAME + c + "();");
+        }
+        out.add("System.out.println(\"Success\");");
+        out.add("}");
+        out.add("}");
+        return out;
+    }
+
+    public static String[] listAppClasses() {
+        String[] res = new String[NUM_CLASSES * NUM_NESTED_CLASSES];
+        for (int c = 0; c < NUM_CLASSES; c++) {
+            for (int sc = 0; sc < NUM_NESTED_CLASSES; sc++) {
+                res[c * NUM_NESTED_CLASSES + sc] = TOP_CLASS_NAME + c + "$" + NESTED_CLASS_NAME + sc;
+            }
+        }
+        return res;
+    }
+
+    public static void main(String[] args) throws Exception {
+        // Step 1. Generate classes and build the JAR with them.
+        {
+            SRC_DIR.toFile().mkdirs();
+
+            for (int i = 0; i < NUM_CLASSES; i++) {
+                Path file = SRC_DIR.resolve(TOP_CLASS_NAME + i + ".java");
+                Files.write(file, generateClass(i));
+            }
+
+            Path mainFile = SRC_DIR.resolve(MAIN_CLASS_NAME + ".java");
+            Files.write(mainFile, generateMainClass());
+
+            CDSJarUtils.buildFromSourceDirectory(
+                APP_JAR.toString(),
+                SRC_DIR.toString()
+            );
+        }
+
+        // Step 2. Try to dump the archive.
+        {
+            OutputAnalyzer output = TestCommon.createArchive(
+                APP_JAR.toString(),
+                listAppClasses(),
+                MAIN_CLASS_NAME
+            );
+            TestCommon.checkDump(output);
+        }
+
+        // Step 3. Try to run, touching every class.
+        {
+            TestCommon.run(
+                // Verifying dependencies for lots of classes slows down the test.
+                "-XX:+IgnoreUnrecognizedVMOptions", "-XX:-VerifyDependencies",
+                "-Xlog:cds",
+                "-cp", APP_JAR.toString(),
+                MAIN_CLASS_NAME).
+                    assertNormalExit("Success");
+        }
+
+    }
+}