Skip to content

Commit eb55583

Browse files
peter-hoferpeter-hofer
committed
[GR-65862] Base-relative code pointers in virtual dispatch tables for layered images.
PullRequest: graal/21284
2 parents aea4a00 + b12f195 commit eb55583

File tree

24 files changed

+555
-271
lines changed

24 files changed

+555
-271
lines changed

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/nodes/CountTrailingZerosNode.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,4 +98,7 @@ public ValueNode canonical(CanonicalizerTool tool, ValueNode forValue) {
9898
public void generate(NodeLIRBuilderTool builder, ArithmeticLIRGeneratorTool gen) {
9999
builder.setResult(this, gen.emitCountTrailingZeros(builder.operand(getValue())));
100100
}
101+
102+
@NodeIntrinsic
103+
public static native int countLongTrailingZeros(long value);
101104
}

‎substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunkedImageHeapLayouter.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,9 @@ public void assignObjectToPartition(ImageHeapObject info, boolean immutable, boo
118118
}
119119

120120
private ChunkedImageHeapPartition choosePartition(ImageHeapObject info, boolean immutable, boolean hasRelocatables, boolean patched) {
121+
if (patched && hasRelocatables) {
122+
throw VMError.shouldNotReachHere("Object cannot contain both relocatables and patched constants: " + info.getObject());
123+
}
121124
if (patched) {
122125
return getWritablePatched();
123126
} else if (immutable) {

‎substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/linux/LinuxImageHeapProvider.java‎

Lines changed: 141 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_END;
3434
import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_BEGIN;
3535
import static com.oracle.svm.core.Isolates.IMAGE_HEAP_WRITABLE_PATCHED_END;
36+
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.CODE_START;
3637
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_BEGIN;
3738
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_END;
3839
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_RELOCATABLE_BEGIN;
@@ -42,11 +43,13 @@
4243
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_BEGIN;
4344
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_WRITEABLE_PATCHED_END;
4445
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.NEXT_SECTION;
46+
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.VARIABLY_SIZED_DATA;
4547
import static com.oracle.svm.core.posix.linux.ProcFSSupport.findMapping;
4648
import static com.oracle.svm.core.util.PointerUtils.roundDown;
4749
import static com.oracle.svm.core.util.UnsignedUtils.isAMultiple;
4850
import static com.oracle.svm.core.util.UnsignedUtils.roundUp;
4951
import static jdk.graal.compiler.word.Word.signed;
52+
import static jdk.graal.compiler.word.Word.unsigned;
5053

5154
import java.util.concurrent.ThreadLocalRandom;
5255

@@ -71,6 +74,7 @@
7174
import com.oracle.svm.core.heap.Heap;
7275
import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
7376
import com.oracle.svm.core.imagelayer.ImageLayerSection;
77+
import com.oracle.svm.core.jdk.UninterruptibleUtils;
7478
import com.oracle.svm.core.os.AbstractImageHeapProvider;
7579
import com.oracle.svm.core.os.VirtualMemoryProvider;
7680
import com.oracle.svm.core.os.VirtualMemoryProvider.Access;
@@ -81,7 +85,10 @@
8185
import com.oracle.svm.core.util.PointerUtils;
8286
import com.oracle.svm.core.util.UnsignedUtils;
8387
import com.oracle.svm.core.util.VMError;
88+
import com.oracle.svm.hosted.imagelayer.ImageLayerSectionFeature;
89+
import com.oracle.svm.hosted.imagelayer.LayeredDispatchTableFeature;
8490

91+
import jdk.graal.compiler.nodes.NamedLocationIdentity;
8592
import jdk.graal.compiler.nodes.PauseNode;
8693
import jdk.graal.compiler.word.Word;
8794

@@ -122,6 +129,14 @@ public class LinuxImageHeapProvider extends AbstractImageHeapProvider {
122129
*/
123130
static final CGlobalData<WordPointer> CACHED_LAYERED_IMAGE_HEAP_ADDRESS_SPACE_SIZE = CGlobalDataFactory.createWord();
124131

132+
private static final class ImageHeapPatchingState {
133+
static final Word UNINITIALIZED = Word.zero();
134+
static final Word IN_PROGRESS = Word.unsigned(1);
135+
static final Word SUCCESSFUL = Word.unsigned(2);
136+
}
137+
138+
private static final CGlobalData<Word> IMAGE_HEAP_PATCHING_STATE = CGlobalDataFactory.createWord(ImageHeapPatchingState.UNINITIALIZED);
139+
125140
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
126141
private static UnsignedWord getLayeredImageHeapAddressSpaceSize() {
127142
// check if value is cached
@@ -167,6 +182,8 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve
167182
int result = -1;
168183
UnsignedWord remainingSize = initialRemainingSize;
169184

185+
patchLayeredImageHeap();
186+
170187
int layerCount = 0;
171188
Pointer currentSection = ImageLayerSection.getInitialLayerSection().get();
172189
Pointer currentHeapStart = firstHeapStart;
@@ -213,6 +230,127 @@ protected int initializeLayeredImage(Pointer firstHeapStart, Pointer selfReserve
213230
return result;
214231
}
215232

233+
/**
234+
* Apply patches to the image heap as specified by each layer. See {@link ImageLayerSection} and
235+
* {@link ImageLayerSectionFeature} for the layout of the section that contains the patches and
236+
* {@link LayeredDispatchTableFeature} where code patches are gathered.
237+
*/
238+
@Uninterruptible(reason = "Thread state not yet set up.")
239+
public static void patchLayeredImageHeap() {
240+
Word heapPatchStateAddr = IMAGE_HEAP_PATCHING_STATE.get();
241+
boolean firstIsolate = heapPatchStateAddr.logicCompareAndSwapWord(0, ImageHeapPatchingState.UNINITIALIZED, ImageHeapPatchingState.IN_PROGRESS, NamedLocationIdentity.OFF_HEAP_LOCATION);
242+
243+
if (!firstIsolate) {
244+
// spin-wait for first isolate
245+
Word state = heapPatchStateAddr.readWordVolatile(0, NamedLocationIdentity.OFF_HEAP_LOCATION);
246+
while (state.equal(ImageHeapPatchingState.IN_PROGRESS)) {
247+
PauseNode.pause();
248+
state = heapPatchStateAddr.readWordVolatile(0, NamedLocationIdentity.OFF_HEAP_LOCATION);
249+
}
250+
251+
/* Patching has already been successfully completed, nothing needs to be done. */
252+
return;
253+
}
254+
255+
Pointer layerSection = ImageLayerSection.getInitialLayerSection().get();
256+
Pointer initialLayerImageHeap = layerSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN));
257+
Pointer codeBase = layerSection.readWord(ImageLayerSection.getEntryOffset(CODE_START));
258+
259+
int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
260+
while (layerSection.isNonNull()) {
261+
Pointer data = layerSection.add(ImageLayerSection.getEntryOffset(VARIABLY_SIZED_DATA));
262+
int offset = 0;
263+
264+
offset = skipSingletonsTable(data, offset, referenceSize);
265+
266+
/* Patch code offsets to become relative to the code base. */
267+
Pointer layerHeapRelocs = layerSection.readWord(ImageLayerSection.getEntryOffset(HEAP_RELOCATABLE_BEGIN));
268+
Pointer layerCode = layerSection.readWord(ImageLayerSection.getEntryOffset(CODE_START));
269+
/*
270+
* Note that the code base can be above the layer's code section, in which case the
271+
* subtraction underflows and the additions of code address computations overflow,
272+
* giving the correct result.
273+
*/
274+
Word layerCodeOffsetToBase = (Word) layerCode.subtract(codeBase);
275+
offset = applyLayerCodePointerPatches(data, offset, layerHeapRelocs, layerCodeOffsetToBase);
276+
277+
/* Patch absolute addresses to become relative to the code base. */
278+
Word negativeCodeBase = Word.<Word> zero().subtract(codeBase);
279+
offset = applyLayerCodePointerPatches(data, offset, layerHeapRelocs, negativeCodeBase);
280+
281+
/* Patch references in the image heap. */
282+
applyLayerImageHeapRefPatches(data.add(offset), initialLayerImageHeap);
283+
284+
layerSection = layerSection.readWord(ImageLayerSection.getEntryOffset(NEXT_SECTION));
285+
}
286+
287+
heapPatchStateAddr.writeWordVolatile(0, ImageHeapPatchingState.SUCCESSFUL);
288+
}
289+
290+
@Uninterruptible(reason = "Thread state not yet set up.")
291+
private static int skipSingletonsTable(Pointer data, int offset, int referenceSize) {
292+
long singletonTableEntryCount = data.readLong(offset);
293+
UnsignedWord singletonTableAlignedSize = roundUp(unsigned(singletonTableEntryCount * referenceSize), unsigned(Long.BYTES));
294+
return offset + Long.BYTES + UnsignedUtils.safeToInt(singletonTableAlignedSize);
295+
}
296+
297+
@Uninterruptible(reason = "Thread state not yet set up.")
298+
private static int applyLayerCodePointerPatches(Pointer data, int startOffset, Pointer layerHeapRelocs, Word addend) {
299+
int wordSize = ConfigurationValues.getTarget().wordSize;
300+
301+
int offset = startOffset;
302+
long bitmapWordCountAsLong = data.readLong(offset);
303+
int bitmapWordCount = UninterruptibleUtils.NumUtil.safeToInt(bitmapWordCountAsLong);
304+
offset += Long.BYTES;
305+
if (addend.equal(0)) {
306+
/* Nothing to do. */
307+
offset += bitmapWordCount * Long.BYTES;
308+
return offset;
309+
}
310+
311+
for (int i = 0; i < bitmapWordCount; i++) {
312+
long bits = data.readLong(offset);
313+
offset += Long.BYTES;
314+
int j = 0; // index of a 1-bit
315+
while (bits != 0) {
316+
int ntz = UninterruptibleUtils.Long.countTrailingZeros(bits);
317+
j += ntz;
318+
319+
int at = (i * 64 + j) * wordSize;
320+
Word w = layerHeapRelocs.readWord(at);
321+
w = w.add(addend);
322+
layerHeapRelocs.writeWord(at, w);
323+
324+
/*
325+
* Note that we must not shift by ntz+1 here because it can be 64, which would be a
326+
* no-op according to the Java Language Specification, 15.19. Shift Operators.
327+
*/
328+
bits = (bits >>> ntz) >>> 1;
329+
j++;
330+
}
331+
}
332+
return offset;
333+
}
334+
335+
@Uninterruptible(reason = "Thread state not yet set up.")
336+
private static void applyLayerImageHeapRefPatches(Pointer patches, Pointer layerImageHeap) {
337+
int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
338+
long countAsLong = patches.readLong(0);
339+
int count = UninterruptibleUtils.NumUtil.safeToInt(countAsLong);
340+
int offset = Long.BYTES;
341+
int endOffset = offset + count * Integer.BYTES;
342+
while (offset < endOffset) {
343+
int heapOffset = patches.readInt(offset);
344+
int referenceEncoding = patches.readInt(offset + Integer.BYTES);
345+
offset += 2 * Integer.BYTES;
346+
if (referenceSize == 4) {
347+
layerImageHeap.writeInt(heapOffset, referenceEncoding);
348+
} else {
349+
layerImageHeap.writeLong(heapOffset, referenceEncoding);
350+
}
351+
}
352+
}
353+
216354
@Override
217355
@Uninterruptible(reason = "Called during isolate initialization.")
218356
public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, WordPointer basePointer, WordPointer endPointer) {
@@ -262,7 +400,9 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
262400
basePointer.write(heapBase);
263401
Pointer imageHeapStart = heapBase.add(imageHeapOffsetInAddressSpace);
264402
remainingSize = remainingSize.subtract(imageHeapOffsetInAddressSpace);
265-
if (!ImageLayerBuildingSupport.buildingImageLayer()) {
403+
if (ImageLayerBuildingSupport.buildingImageLayer()) {
404+
return initializeLayeredImage(imageHeapStart, selfReservedHeapBase, remainingSize, endPointer);
405+
} else {
266406
int result = initializeImageHeap(imageHeapStart, remainingSize, endPointer,
267407
CACHED_IMAGE_FD.get(), CACHED_IMAGE_HEAP_OFFSET.get(), CACHED_IMAGE_HEAP_RELOCATIONS.get(), MAGIC.get(),
268408
IMAGE_HEAP_BEGIN.get(), IMAGE_HEAP_END.get(),
@@ -272,8 +412,6 @@ public int initialize(Pointer reservedAddressSpace, UnsignedWord reservedSize, W
272412
freeImageHeap(selfReservedHeapBase);
273413
}
274414
return result;
275-
} else {
276-
return initializeLayeredImage(imageHeapStart, selfReservedHeapBase, remainingSize, endPointer);
277415
}
278416
}
279417

‎substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,6 @@
5555
import com.oracle.svm.core.config.ConfigurationValues;
5656
import com.oracle.svm.core.graal.RuntimeCompilation;
5757
import com.oracle.svm.core.heap.ReferenceHandler;
58-
import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
5958
import com.oracle.svm.core.jdk.VectorAPIEnabled;
6059
import com.oracle.svm.core.option.APIOption;
6160
import com.oracle.svm.core.option.APIOptionGroup;
@@ -1549,6 +1548,7 @@ public static boolean printClosedArenaUponThrow() {
15491548
}
15501549

15511550
@Option(help = "Avoid linker relocations for code and instead emit address computations.", type = OptionType.Expert) //
1551+
@LayerVerifiedOption(severity = Severity.Error, kind = Kind.Changed, positional = false) //
15521552
public static final HostedOptionKey<Boolean> RelativeCodePointers = new HostedOptionKey<>(false, SubstrateOptions::validateRelativeCodePointers);
15531553

15541554
@Fold
@@ -1562,13 +1562,6 @@ private static void validateRelativeCodePointers(HostedOptionKey<Boolean> option
15621562

15631563
UserError.guarantee(Platform.includedIn(PLATFORM_JNI.class) || Platform.includedIn(NATIVE_ONLY.class), "%s is supported only with hardware target platforms.", enabledOption);
15641564

1565-
/*
1566-
* GR-59707: Dispatch tables must potentially be patched at runtime still. Method
1567-
* offsets for dispatch need to be passed on between layer builds rather than using
1568-
* symbol names.
1569-
*/
1570-
UserError.guarantee(!ImageLayerBuildingSupport.buildingImageLayer(), "%s is currently not supported with layered images.", enabledOption);
1571-
15721565
// The concept of a code base would need to be introduced in the LLVM backend first.
15731566
UserError.guarantee(!useLLVMBackend(), "%s is currently not supported with the LLVM backend.", enabledOption);
15741567

‎substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java‎

Lines changed: 10 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import static com.oracle.svm.core.graal.nodes.WriteCurrentVMThreadNode.writeCurrentVMThread;
3030
import static com.oracle.svm.core.graal.nodes.WriteHeapBaseNode.writeCurrentVMHeapBase;
3131
import static com.oracle.svm.core.heap.RestrictHeapAccess.Access.NO_ALLOCATION;
32-
import static com.oracle.svm.core.imagelayer.ImageLayerSection.SectionEntries.HEAP_BEGIN;
3332
import static com.oracle.svm.core.util.VMError.shouldNotReachHereUnexpectedInput;
3433
import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.HAS_SIDE_EFFECT;
3534

@@ -73,7 +72,6 @@
7372
import com.oracle.svm.core.c.locale.LocaleSupport;
7473
import com.oracle.svm.core.code.CodeInfoTable;
7574
import com.oracle.svm.core.code.ImageCodeInfo;
76-
import com.oracle.svm.core.config.ConfigurationValues;
7775
import com.oracle.svm.core.container.Container;
7876
import com.oracle.svm.core.graal.meta.SubstrateForeignCallsProvider;
7977
import com.oracle.svm.core.graal.nodes.CEntryPointEnterNode;
@@ -117,7 +115,6 @@
117115
import jdk.graal.compiler.graph.Node;
118116
import jdk.graal.compiler.graph.Node.ConstantNodeParameter;
119117
import jdk.graal.compiler.graph.Node.NodeIntrinsic;
120-
import jdk.graal.compiler.nodes.NamedLocationIdentity;
121118
import jdk.graal.compiler.nodes.PauseNode;
122119
import jdk.graal.compiler.nodes.extended.ForeignCallNode;
123120
import jdk.graal.compiler.nodes.spi.LoweringTool;
@@ -213,8 +210,16 @@ private static void setHeapBase(PointerBase heapBase) {
213210
@Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE)
214211
@NeverInline("Heap base register is set in caller, prevent reads from floating before that.")
215212
private static void initCodeBase() {
216-
ImageCodeInfo imageCodeInfo = MultiLayeredImageSingleton.getForLayer(ImageCodeInfo.class, MultiLayeredImageSingleton.INITIAL_LAYER_NUMBER);
217-
CodePointer codeBase = imageCodeInfo.getCodeStart();
213+
CodePointer codeBase;
214+
if (ImageLayerBuildingSupport.buildingImageLayer()) {
215+
/* Layered ImageSingleton handling is not ready yet at this point. */
216+
Pointer initialLayerSection = ImageLayerSection.getInitialLayerSection().get();
217+
int codeStartEntry = ImageLayerSection.getEntryOffset(ImageLayerSection.SectionEntries.CODE_START);
218+
codeBase = initialLayerSection.readWord(codeStartEntry);
219+
} else {
220+
ImageCodeInfo imageCodeInfo = MultiLayeredImageSingleton.getForLayer(ImageCodeInfo.class, MultiLayeredImageSingleton.INITIAL_LAYER_NUMBER);
221+
codeBase = imageCodeInfo.getCodeStart();
222+
}
218223
writeCurrentVMCodeBase(codeBase);
219224
}
220225

@@ -260,57 +265,6 @@ public static int createIsolateSnippet(CEntryPointCreateIsolateParameters parame
260265
return runtimeCallInitializeIsolate(INITIALIZE_ISOLATE, parameters);
261266
}
262267

263-
private static final CGlobalData<Word> IMAGE_HEAP_PATCHING_STATE = CGlobalDataFactory.createWord();
264-
265-
private static final class ImageHeapPatchingState {
266-
static final Word UNINITIALIZED = Word.zero();
267-
static final Word IN_PROGRESS = Word.unsigned(1);
268-
static final Word SUCCESSFUL = Word.unsigned(2);
269-
}
270-
271-
@Uninterruptible(reason = "Thread state not yet set up.")
272-
private static void layeredPatchHeapRelativeRelocations() {
273-
Word heapPatchStateAddr = IMAGE_HEAP_PATCHING_STATE.get();
274-
boolean firstIsolate = heapPatchStateAddr.logicCompareAndSwapWord(0, ImageHeapPatchingState.UNINITIALIZED, ImageHeapPatchingState.IN_PROGRESS, NamedLocationIdentity.OFF_HEAP_LOCATION);
275-
276-
if (!firstIsolate) {
277-
// spin-wait for first isolate
278-
Word state = heapPatchStateAddr.readWordVolatile(0, LocationIdentity.ANY_LOCATION);
279-
while (state.equal(ImageHeapPatchingState.IN_PROGRESS)) {
280-
PauseNode.pause();
281-
state = heapPatchStateAddr.readWordVolatile(0, LocationIdentity.ANY_LOCATION);
282-
}
283-
284-
/*
285-
* Patching has already been successfully completed, nothing needs to be done.
286-
*/
287-
return;
288-
}
289-
290-
Pointer currentSection = ImageLayerSection.getInitialLayerSection().get();
291-
Word heapBegin = currentSection.readWord(ImageLayerSection.getEntryOffset(HEAP_BEGIN));
292-
293-
Word patches = ImageLayerSection.getHeapRelativeRelocationsStart().get();
294-
int endOffset = Integer.BYTES + (patches.readInt(0) * Integer.BYTES);
295-
296-
int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize();
297-
int offset = Integer.BYTES;
298-
while (offset < endOffset) {
299-
int heapOffset = patches.readInt(offset);
300-
int referenceEncoding = patches.readInt(offset + Integer.BYTES);
301-
302-
if (referenceSize == 4) {
303-
heapBegin.writeInt(heapOffset, referenceEncoding);
304-
} else {
305-
heapBegin.writeLong(heapOffset, referenceEncoding);
306-
}
307-
308-
offset += 2 * Integer.BYTES;
309-
}
310-
311-
heapPatchStateAddr.writeWordVolatile(0, ImageHeapPatchingState.SUCCESSFUL);
312-
}
313-
314268
/**
315269
* After parsing the isolate arguments in
316270
* {@link IsolateArgumentParser#parse(CEntryPointCreateIsolateParameters, IsolateArguments)} the
@@ -332,10 +286,6 @@ private static int createIsolate(CEntryPointCreateIsolateParameters providedPara
332286
return CEntryPointErrors.PAGE_SIZE_CHECK_FAILED;
333287
}
334288

335-
if (ImageLayerBuildingSupport.buildingImageLayer()) {
336-
layeredPatchHeapRelativeRelocations();
337-
}
338-
339289
LocaleSupport.initialize();
340290

341291
CEntryPointCreateIsolateParameters parameters = providedParameters;

0 commit comments

Comments
 (0)