[GR-67508] Improve reporting on included resources and error when the image heap gets too large.

PullRequest: graal/21453

Author: thomaswue

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/Resources.java

@@ -48,6 +48,7 @@
 
 import org.graalvm.collections.EconomicMap;
 import org.graalvm.collections.MapCursor;
+import org.graalvm.collections.UnmodifiableEconomicMap;
 import org.graalvm.nativeimage.ImageInfo;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
@@ -218,9 +219,10 @@ public String getModuleName() {
     }
 
     /**
-     * The object used to mark a resource as reachable according to the metadata. It can be obtained
-     * when accessing the {@link Resources#resources} map, and it means that even though the
-     * resource was correctly specified in the configuration, accessing it will return null.
+     * A resource marked with the NEGATIVE_QUERY_MARKER is a resource included in the image
+     * according to the resource configuration, but it does not actually exist. Trying to access it
+     * at runtime will return {@code null} and not throw a
+     * {@link com.oracle.svm.core.jdk.resources.MissingResourceRegistrationError}.
      */
     public static final ResourceStorageEntryBase NEGATIVE_QUERY_MARKER = new ResourceStorageEntryBase();
 
@@ -230,7 +232,7 @@ public String getModuleName() {
      * correctly specified in the configuration, but we do not want to throw directly (for example
      * when we try to check all the modules for a resource).
      */
-    private static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase();
+    public static final ResourceStorageEntryBase MISSING_METADATA_MARKER = new ResourceStorageEntryBase();
 
     /**
      * Embedding a resource into an image is counted as a modification. Since all resources are
@@ -275,23 +277,8 @@ public void forEachResource(BiConsumer<ModuleResourceKey, ConditionalRuntimeValu
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public ConditionalRuntimeValue<ResourceStorageEntryBase> getResource(ModuleResourceKey storageKey) {
-        return resources.get(storageKey);
-    }
-
-    @Platforms(Platform.HOSTED_ONLY.class)
-    public Iterable<ConditionalRuntimeValue<ResourceStorageEntryBase>> resources() {
-        return resources.getValues();
-    }
-
-    @Platforms(Platform.HOSTED_ONLY.class)
-    public Iterable<ModuleResourceKey> resourceKeys() {
-        return resources.getKeys();
-    }
-
-    @Platforms(Platform.HOSTED_ONLY.class)
-    public int count() {
-        return resources.size();
+    public UnmodifiableEconomicMap<ModuleResourceKey, ConditionalRuntimeValue<ResourceStorageEntryBase>> resources() {
+        return resources;
     }
 
     public static long getLastModifiedTime() {
@@ -642,8 +629,8 @@ public static InputStream createInputStream(Module module, String resourceName)
         } else if (entry == null) {
             return null;
         }
-        List<byte[]> data = entry.getData();
-        return data.isEmpty() ? null : new ByteArrayInputStream(data.get(0));
+        byte[][] data = entry.getData();
+        return data.length == 0 ? null : new ByteArrayInputStream(data[0]);
     }
 
     private static ResourceStorageEntryBase findResourceForInputStream(Module module, String resourceName) {
@@ -717,8 +704,7 @@ private static void addURLEntries(List<URL> resourcesURLs, ResourceStorageEntry
         if (entry == null) {
             return;
         }
-        int numberOfResources = entry.getData().size();
-        for (int index = 0; index < numberOfResources; index++) {
+        for (int index = 0; index < entry.getData().length; index++) {
             resourcesURLs.add(createURL(module, canonicalResourceName, index));
         }
     }
@@ -828,7 +814,7 @@ public void afterCompilation(AfterCompilationAccess access) {
          * of lazily initialized fields. Only the byte[] arrays themselves can be safely made
          * read-only.
          */
-        for (ConditionalRuntimeValue<ResourceStorageEntryBase> entry : Resources.currentLayer().resources()) {
+        for (ConditionalRuntimeValue<ResourceStorageEntryBase> entry : Resources.currentLayer().resources().getValues()) {
             var unconditionalEntry = entry.getValueUnconditionally();
             if (unconditionalEntry.hasData()) {
                 for (byte[] resource : unconditionalEntry.getData()) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/NativeImageResourceFileSystemUtil.java

@@ -40,23 +40,14 @@ public static byte[] getBytes(String resourceName, boolean readOnly) {
         if (entry == null) {
             return new byte[0];
         }
-        byte[] bytes = ((ResourceStorageEntry) entry).getData().get(0);
+        byte[] bytes = ((ResourceStorageEntry) entry).getData()[0];
         if (readOnly) {
             return bytes;
         } else {
             return Arrays.copyOf(bytes, bytes.length);
         }
     }
 
-    public static int getSize(String resourceName) {
-        Object entry = Resources.getAtRuntime(resourceName);
-        if (entry == null) {
-            return 0;
-        } else {
-            return ((ResourceStorageEntry) entry).getData().get(0).length;
-        }
-    }
-
     public static String toRegexPattern(String globPattern) {
         return Target_jdk_nio_zipfs_ZipUtils.toRegexPattern(globPattern);
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/ResourceStorageEntry.java

@@ -25,8 +25,7 @@
 
 package com.oracle.svm.core.jdk.resources;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Arrays;
 
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -36,14 +35,16 @@
 
 public final class ResourceStorageEntry extends ResourceStorageEntryBase {
 
+    private static final byte[][] EMPTY_DATA = new byte[0][];
+
     private final boolean isDirectory;
     private final boolean fromJar;
-    private List<byte[]> data;
+    private byte[][] data;
 
     public ResourceStorageEntry(boolean isDirectory, boolean fromJar) {
         this.isDirectory = isDirectory;
         this.fromJar = fromJar;
-        this.data = List.of();
+        this.data = EMPTY_DATA;
     }
 
     @Override
@@ -57,18 +58,17 @@ public boolean isFromJar() {
     }
 
     @Override
-    public List<byte[]> getData() {
+    public byte[][] getData() {
         return data;
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
     @Override
     public void addData(byte[] datum) {
-        List<byte[]> newData = new ArrayList<>(data.size() + 1);
-        newData.addAll(data);
-        newData.add(datum);
+        byte[][] newData = Arrays.copyOf(data, data.length + 1);
+        newData[data.length] = datum;
         /* Always use a compact, immutable data structure in the image heap. */
-        data = List.copyOf(newData);
+        data = newData;
     }
 
     /**
@@ -81,7 +81,7 @@ public void addData(byte[] datum) {
     public void replaceData(byte[]... replacementData) {
         VMError.guarantee(BuildPhaseProvider.isAnalysisFinished(), "Replacing data of a resource entry before analysis finished. Register standard resource instead.");
         VMError.guarantee(!BuildPhaseProvider.isCompilationFinished(), "Trying to replace data of a resource entry after compilation finished.");
-        this.data = List.of(replacementData);
+        this.data = replacementData;
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/ResourceStorageEntryBase.java

@@ -25,8 +25,6 @@
 
 package com.oracle.svm.core.jdk.resources;
 
-import java.util.List;
-
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
@@ -41,7 +39,7 @@ public boolean isFromJar() {
         throw VMError.shouldNotReachHere("This should only be called on entries with data.");
     }
 
-    public List<byte[]> getData() {
+    public byte[][] getData() {
         throw VMError.shouldNotReachHere("This should only be called on entries with data.");
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/resources/ResourceURLConnection.java

@@ -77,7 +77,7 @@ public void connect() {
         Object entry = Resources.getAtRuntime(module, resourceName, false);
         if (entry != null) {
             ResourceStorageEntry resourceStorageEntry = (ResourceStorageEntry) entry;
-            List<byte[]> bytes = resourceStorageEntry.getData();
+            byte[][] bytes = resourceStorageEntry.getData();
             isDirectory = resourceStorageEntry.isDirectory();
             String urlRef = url.getRef();
             int index = 0;
@@ -88,12 +88,12 @@ public void connect() {
                     throw new IllegalArgumentException("URL anchor '#" + urlRef + "' not allowed in " + JavaNetSubstitutions.RESOURCE_PROTOCOL + " URL");
                 }
             }
-            if (index < bytes.size()) {
-                this.data = bytes.get(index);
+            if (index < bytes.length) {
+                this.data = bytes[index];
             } else {
                 // This will happen only in case that we are creating one URL with the second URL as
                 // a context.
-                this.data = bytes.get(0);
+                this.data = bytes[0];
             }
         } else {
             this.data = null;

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/EmbeddedResourceExporter.java

@@ -120,7 +120,7 @@ public static List<ResourceReportEntry> getResourceReportEntryList(ConcurrentHas
             List<EmbeddedResourceExporter.SourceSizePair> sources = new ArrayList<>();
             for (int i = 0; i < registeredEntrySources.size(); i++) {
                 SourceAndOrigin sourceAndOrigin = registeredEntrySources.get(i);
-                int size = storageEntry.getData().get(i).length;
+                int size = storageEntry.getData()[i].length;
                 sources.add(new SourceSizePair(sourceAndOrigin.source(), sourceAndOrigin.origin(), size));
             }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/HeapBreakdownProvider.java

@@ -168,7 +168,7 @@ protected void calculate(BeforeImageWriteAccessImpl access, boolean resourcesAre
         if (!ImageLayerBuildingSupport.buildingExtensionLayer() && resourcesAreReachable) {
             /* Extract byte[] for resources. */
             int resourcesByteArrayCount = 0;
-            for (ConditionalRuntimeValue<ResourceStorageEntryBase> resourceList : Resources.currentLayer().resources()) {
+            for (ConditionalRuntimeValue<ResourceStorageEntryBase> resourceList : Resources.currentLayer().resources().getValues()) {
                 if (resourceList.getValueUnconditionally().hasData()) {
                     for (byte[] resource : resourceList.getValueUnconditionally().getData()) {
                         resourcesByteArraySize += objectLayout.getArraySize(JavaKind.Byte, resource.length, true);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java

@@ -52,6 +52,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import com.oracle.svm.core.configure.ConditionalRuntimeValue;
+import com.oracle.svm.core.jdk.resources.ResourceStorageEntryBase;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.hosted.Feature;
 import org.graalvm.nativeimage.impl.ImageSingletonsSupport;
@@ -506,10 +508,22 @@ private void printAnalysisStatistics(AnalysisUniverse universe, Collection<Strin
         String stubsFormat = "%,9d downcalls and %,d upcalls ";
         recordJsonMetric(AnalysisResults.FOREIGN_DOWNCALLS, (numForeignDowncalls >= 0 ? numForeignDowncalls : UNAVAILABLE_METRIC));
         recordJsonMetric(AnalysisResults.FOREIGN_UPCALLS, (numForeignUpcalls >= 0 ? numForeignUpcalls : UNAVAILABLE_METRIC));
-        if (numForeignDowncalls >= 0 || numForeignUpcalls >= 0) {
+        if (numForeignDowncalls > 0 || numForeignUpcalls > 0) {
             l().a(stubsFormat, numForeignDowncalls, numForeignUpcalls)
                             .doclink("registered for foreign access", "#glossary-foreign-downcall-and-upcall-registrations").println();
         }
+        int resourceCount = Resources.currentLayer().resources().size();
+        long totalResourceSize = 0;
+        for (ConditionalRuntimeValue<ResourceStorageEntryBase> value : Resources.currentLayer().resources().getValues()) {
+            if (value.getValueUnconditionally().hasData()) {
+                for (byte[] bytes : value.getValueUnconditionally().getData()) {
+                    totalResourceSize += bytes.length;
+                }
+            }
+        }
+        if (resourceCount > 0) {
+            l().a("%,9d %s registered with %s total size", resourceCount, resourceCount == 1 ? "resource access" : "resource accesses", ByteFormattingUtil.bytesToHuman(totalResourceSize)).println();
+        }
         int numLibraries = libraries.size();
         if (numLibraries > 0) {
             TreeSet<String> sortedLibraries = new TreeSet<>(libraries);
@@ -578,10 +592,15 @@ public void printCreationEnd(int imageFileSize, int heapObjectCount, long imageH
         String format = "%9s (%5.2f%%) for ";
         l().a(format, ByteFormattingUtil.bytesToHuman(codeAreaSize), Utils.toPercentage(codeAreaSize, imageFileSize))
                         .doclink("code area", "#glossary-code-area").a(":%,10d compilation units", numCompilations).println();
-        int numResources = Resources.currentLayer().count();
+        int numResources = 0;
+        for (ConditionalRuntimeValue<ResourceStorageEntryBase> entry : Resources.currentLayer().resources().getValues()) {
+            if (entry.getValueUnconditionally() != Resources.NEGATIVE_QUERY_MARKER && entry.getValueUnconditionally() != Resources.MISSING_METADATA_MARKER) {
+                numResources++;
+            }
+        }
         recordJsonMetric(ImageDetailKey.IMAGE_HEAP_RESOURCE_COUNT, numResources);
         l().a(format, ByteFormattingUtil.bytesToHuman(imageHeapSize), Utils.toPercentage(imageHeapSize, imageFileSize))
-                        .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resources", heapObjectCount, numResources).println();
+                        .doclink("image heap", "#glossary-image-heap").a(":%,9d objects and %,d resource%s", heapObjectCount, numResources, numResources == 1 ? "" : "s").println();
         long otherBytes = imageFileSize - codeAreaSize - imageHeapSize;
         if (debugInfoSize > 0) {
             recordJsonMetric(ImageDetailKey.DEBUG_INFO_SIZE, debugInfoSize); // Optional metric

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ResourcesFeature.java

@@ -200,7 +200,7 @@ public void addGlob(ConfigurationCondition condition, String module, String glob
 
         @Override
         public void addCondition(ConfigurationCondition condition, Module module, String resourcePath) {
-            var conditionalResource = Resources.currentLayer().getResource(createStorageKey(module, resourcePath));
+            var conditionalResource = Resources.currentLayer().resources().get(createStorageKey(module, resourcePath));
             if (conditionalResource != null) {
                 classInitializationSupport.addForTypeReachedTracking(condition.getType());
                 conditionalResource.getConditions().addCondition(condition);

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/NativeImage.java

@@ -50,6 +50,7 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import com.oracle.svm.hosted.ByteFormattingUtil;
 import org.graalvm.collections.Pair;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.c.CHeader;
@@ -503,6 +504,11 @@ public void build(String imageName, DebugContext debug) {
              */
             boolean padImageHeap = !SpawnIsolates.getValue() || MremapImageHeap.getValue();
             long paddedImageHeapSize = padImageHeap ? NumUtil.roundUp(imageHeapSize, alignment) : imageHeapSize;
+
+            VMError.guarantee(NumUtil.isInt(paddedImageHeapSize),
+                            "The size of the image heap is %s and therefore too large. It must be smaller than %s. This can happen when very large resource files are included in the image or a build time initialized class creates a large cache.",
+                            ByteFormattingUtil.bytesToHuman(paddedImageHeapSize),
+                            ByteFormattingUtil.bytesToHuman(Integer.MAX_VALUE));
             RelocatableBuffer heapSectionBuffer = new RelocatableBuffer(paddedImageHeapSize, objectFile.getByteOrder());
             ProgbitsSectionImpl heapSectionImpl = new BasicProgbitsSectionImpl(heapSectionBuffer.getBackingArray());
             // Note: On isolate startup the read only part of the heap will be set up as such.