Eliminate relocation entries for base layer vtables with RelativeCodePointers.

Author: peter-hofer

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/CountTrailingZerosNode.java

@@ -98,4 +98,7 @@ public ValueNode canonical(CanonicalizerTool tool, ValueNode forValue) {
     public void generate(NodeLIRBuilderTool builder, ArithmeticLIRGeneratorTool gen) {
         builder.setResult(this, gen.emitCountTrailingZeros(builder.operand(getValue())));
     }
+
+    @NodeIntrinsic
+    public static native int countLongTrailingZeros(long value);
 }

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java

@@ -33,6 +33,7 @@
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END;
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_BEGIN;
 import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_END;
+import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.CODE_START;
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_BEGIN;
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_END;
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_RELOCATABLE_BEGIN;
@@ -42,11 +43,13 @@
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_BEGIN;
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_END;
 import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.NEXT_SECTION;
+import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.VARIABLY_SIZED_DATA;
 import static com.oracle.svm.core.posix.linux.ProcFSSupport.findMapping;
 import static com.oracle.svm.core.util.PointerUtils.roundDown;
 import static com.oracle.svm.core.util.UnsignedUtils.isAMultiple;
 import static com.oracle.svm.core.util.UnsignedUtils.roundUp;
 import static jdk.graal.compiler.word.Word.signed;
+import static jdk.graal.compiler.word.Word.unsigned;
 
 import java.util.concurrent.ThreadLocalRandom;
 
@@ -84,6 +87,7 @@
 
 import jdk.graal.compiler.nodes.NamedLocationIdentity;
 import jdk.graal.compiler.nodes.PauseNode;
+import jdk.graal.compiler.replacements.nodes.CountTrailingZerosNode;
 import jdk.graal.compiler.word.Word;
 
 /**
@@ -241,28 +245,91 @@ public static void patchLayeredImageHeap() {
             return;
         }
 
-        Pointer currentSection = ImageLayerSection.getInitialLayerSection().get();
-        Word heapBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN));
+        Pointer layerSection = ImageLayerSection.getInitialLayerSection().get();
+        Pointer initialLayerImageHeap = layerSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN));
+        Pointer codeBase = layerSection.readWord(ImageLayerSection.getEntryOffset(CODE_START));
+
+        int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
+        while (layerSection.isNonNull()) {
+            Pointer data = layerSection.add(ImageLayerSection.getEntryOffset(VARIABLY_SIZED_DATA));
+            int offset = 0;
+
+            // Skip singletons table
+            long singletonTableEntryCount = data.readLong(offset);
+            UnsignedWord singletonTableAlignedSize = roundUp(unsigned(singletonTableEntryCount * referenceSize), unsigned(Long.BYTES));
+            offset += Long.BYTES + (int) singletonTableAlignedSize.rawValue();
+
+            Pointer layerHeapRelocs = layerSection.readWord(ImageLayerSection.getEntryOffset(HEAP_RELOCATABLE_BEGIN));
+            Pointer layerCode = layerSection.readWord(ImageLayerSection.getEntryOffset(CODE_START));
+            Word layerCodeOffsetToBase = (Word) layerCode.subtract(codeBase);
+            offset = applyLayerCodePointerPatches(data, offset, layerHeapRelocs, layerCodeOffsetToBase);
+
+            Word negativeCodeBase = Word.<Word> zero().subtract(codeBase);
+            offset = applyLayerCodePointerPatches(data, offset, layerHeapRelocs, negativeCodeBase);
+
+            applyLayerImageHeapRefPatches(data.add(offset), initialLayerImageHeap);
+
+            layerSection = layerSection.readWord(ImageLayerSection.getEntryOffset(NEXT_SECTION));
+        }
+
+        heapPatchStateAddr.writeWordVolatile(0, ImageHeapPatchingState.SUCCESSFUL);
+    }
 
-        Word patches = ImageLayerSection.getHeapRelativeRelocationsStart().get();
-        int endOffset = Integer.BYTES + (patches.readInt(0) * Integer.BYTES);
+    @Uninterruptible(reason = "Thread state not yet set up.")
+    private static int applyLayerCodePointerPatches(Pointer data, int startOffset, Pointer layerHeapRelocs, Word addend) {
+        int wordSize = ConfigurationValues.getTarget().wordSize;
+
+        int offset = startOffset;
+        int bitmapWordCount = (int) data.readLong(offset);
+        offset += Long.BYTES;
+        if (addend.equal(0)) {
+            offset += bitmapWordCount * Long.BYTES;
+            return offset;
+        }
 
+        for (int i = 0; i < bitmapWordCount; i++) {
+            long bits = data.readLong(offset);
+            offset += Long.BYTES;
+            int j = 0; // index of a 1-bit
+            while (bits != 0) {
+                int ntz = CountTrailingZerosNode.countLongTrailingZeros(bits);
+                j += ntz;
+
+                int at = (i * 64 + j) * wordSize;
+                Word w = layerHeapRelocs.readWord(at);
+                w = w.add(addend);
+                layerHeapRelocs.writeWord(at, w);
+
+                /*
+                 * Note that we must not shift by ntz+1 here because it can be 64, and according to
+                 * the Java Language Specification, 15.19. Shift Operators: If the promoted type of
+                 * the left-hand operand is long, then only the six lowest-order bits of the
+                 * right-hand operand are used as the shift distance. [..] The shift distance
+                 * actually used is therefore always in the range 0 to 63, inclusive.
+                 */
+                bits = (bits >>> ntz) >>> 1;
+                j++;
+            }
+        }
+        return offset;
+    }
+
+    @Uninterruptible(reason = "Thread state not yet set up.")
+    private static void applyLayerImageHeapRefPatches(Pointer patches, Pointer layerImageHeap) {
         int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
-        int offset = Integer.BYTES;
+        int count = (int) patches.readLong(0);
+        int offset = Long.BYTES;
+        int endOffset = offset + count * Integer.BYTES;
         while (offset < endOffset) {
             int heapOffset = patches.readInt(offset);
             int referenceEncoding = patches.readInt(offset + Integer.BYTES);
-
+            offset += 2 * Integer.BYTES;
             if (referenceSize == 4) {
-                heapBegin.writeInt(heapOffset, referenceEncoding);
+                layerImageHeap.writeInt(heapOffset, referenceEncoding);
             } else {
-                heapBegin.writeLong(heapOffset, referenceEncoding);
+                layerImageHeap.writeLong(heapOffset, referenceEncoding);
             }
-
-            offset += 2 * Integer.BYTES;
         }
-
-        heapPatchStateAddr.writeWordVolatile(0, ImageHeapPatchingState.SUCCESSFUL);
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -1523,6 +1523,7 @@ public static boolean printClosedArenaUponThrow() {
     }
 
     @Option(help = "Avoid linker relocations for code and instead emit address computations.", type = OptionType.Expert) //
+    @LayerVerifiedOption(severity = Severity.Error, kind = Kind.Changed, positional = false) //
     public static final HostedOptionKey<Boolean> RelativeCodePointers = new HostedOptionKey<>(false, SubstrateOptions::validateRelativeCodePointers);
 
     @Fold

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java

@@ -83,9 +83,10 @@
 import com.oracle.svm.core.heap.ReferenceHandlerThread;
 import com.oracle.svm.core.heap.ReferenceInternals;
 import com.oracle.svm.core.heap.RestrictHeapAccess;
+import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
+import com.oracle.svm.core.imagelayer.ImageLayerSection;
 import com.oracle.svm.core.jdk.PlatformNativeLibrarySupport;
 import com.oracle.svm.core.jdk.RuntimeSupport;
-import com.oracle.svm.core.layeredimagesingleton.MultiLayeredImageSingleton;
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.option.RuntimeOptionParser;
 import com.oracle.svm.core.os.CommittedMemoryProvider;
@@ -208,8 +209,16 @@ private static void setHeapBase(PointerBase heapBase) {
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE)
     @NeverInline("Heap base register is set in caller, prevent reads from floating before that.")
     private static void initCodeBase() {
-        ImageCodeInfo imageCodeInfo = MultiLayeredImageSingleton.getForLayer(ImageCodeInfo.class, MultiLayeredImageSingleton.INITIAL_LAYER_NUMBER);
-        CodePointer codeBase = imageCodeInfo.getCodeStart();
+        CodePointer codeBase;
+        if (ImageLayerBuildingSupport.buildingImageLayer()) {
+            /* Layered ImageSingleton handling is not ready yet at this point. */
+            Pointer initialLayerSection = ImageLayerSection.getInitialLayerSection().get();
+            int codeStartEntry = ImageLayerSection.getEntryOffset(ImageLayerSection.SectionEntries.CODE_START);
+            codeBase = initialLayerSection.readWord(codeStartEntry);
+        } else {
+            ImageCodeInfo imageCodeInfo = ImageSingletons.lookup(ImageCodeInfo.class);
+            codeBase = imageCodeInfo.getCodeStart();
+        }
         writeCurrentVMCodeBase(codeBase);
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/ImageLayerSection.java

@@ -32,24 +32,20 @@
 import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingleton;
 
 import jdk.graal.compiler.api.replacements.Fold;
-import jdk.graal.compiler.word.Word;
 
 public abstract class ImageLayerSection implements LayeredImageSingleton {
 
     protected final CGlobalData<Pointer> initialSectionStart;
     protected final CGlobalData<WordPointer> cachedImageFDs;
     protected final CGlobalData<WordPointer> cachedImageHeapOffsets;
     protected final CGlobalData<WordPointer> cachedImageHeapRelocations;
-    protected final CGlobalData<Word> heapRelativeRelocations;
 
     protected ImageLayerSection(CGlobalData<Pointer> initialSectionStart, CGlobalData<WordPointer> cachedImageFDs, CGlobalData<WordPointer> cachedImageHeapOffsets,
-                    CGlobalData<WordPointer> cachedImageHeapRelocations,
-                    CGlobalData<Word> heapRelativeRelocations) {
+                    CGlobalData<WordPointer> cachedImageHeapRelocations) {
         this.initialSectionStart = initialSectionStart;
         this.cachedImageFDs = cachedImageFDs;
         this.cachedImageHeapOffsets = cachedImageHeapOffsets;
         this.cachedImageHeapRelocations = cachedImageHeapRelocations;
-        this.heapRelativeRelocations = heapRelativeRelocations;
     }
 
     public enum SectionEntries {
@@ -61,7 +57,10 @@ public enum SectionEntries {
         HEAP_WRITEABLE_END,
         HEAP_WRITEABLE_PATCHED_BEGIN,
         HEAP_WRITEABLE_PATCHED_END,
-        NEXT_SECTION
+        CODE_START,
+        NEXT_SECTION,
+        VARIABLY_SIZED_DATA,
+        FIRST_SINGLETON,
     }
 
     private static ImageLayerSection singleton() {
@@ -93,11 +92,6 @@ public static CGlobalData<WordPointer> getCachedImageHeapRelocations() {
         return singleton().cachedImageHeapRelocations;
     }
 
-    @Fold
-    public static CGlobalData<Word> getHeapRelativeRelocationsStart() {
-        return singleton().heapRelativeRelocations;
-    }
-
     protected abstract int getEntryOffsetInternal(SectionEntries entry);
 
 }

substratevm/src/com.oracle.svm.hosted/resources/SharedLayerSnapshotCapnProtoSchema.capnp

@@ -168,6 +168,7 @@ struct ConstantReference {
     }
     cEntryPointLiteralCodePointer @5 :CEntryPointLiteralReference;
     cGlobalDataBasePointer @6 :Void;
+    methodOffset @7 :Void;
   }
 }
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/image/MethodPointerRelocationProvider.java

@@ -30,7 +30,9 @@
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
+import com.oracle.svm.core.meta.MethodOffset;
 import com.oracle.svm.core.meta.MethodPointer;
+import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.hosted.imagelayer.LayeredDispatchTableFeature;
 import com.oracle.svm.hosted.meta.HostedMethod;
 
@@ -52,6 +54,20 @@ public void markMethodPointerRelocation(ObjectFile.ProgbitsSectionImpl section,
         }
         section.markRelocationSite(offset, relocationKind, symbolName, addend);
     }
+
+    public void markMethodOffsetRelocation(ObjectFile.ProgbitsSectionImpl section, int offset, ObjectFile.RelocationKind relocationKind, HostedMethod target,
+                    long addend, MethodOffset methodOffset, boolean isInjectedNotCompiled) {
+        if (!imageLayer || !isInjectedNotCompiled) {
+            throw VMError.shouldNotReachHere("must be written to image heap without relocation");
+        }
+
+        /*
+         * Add a relocation entry that will be resolved to the absolute address for the symbol. At
+         * runtime, we recompute this address ourselves to become relative to the code base.
+         */
+        String symbolName = LayeredDispatchTableFeature.singleton().getSymbolName(methodOffset, target, isInjectedNotCompiled);
+        section.markRelocationSite(offset, relocationKind, symbolName, addend);
+    }
 }
 
 @AutomaticallyRegisteredFeature