From 81cbba35f5004f06af79787aa61883568fe6fd6a Mon Sep 17 00:00:00 2001 From: Ehsan Kiani Far Date: Fri, 17 Oct 2025 11:11:04 -0400 Subject: [PATCH 1/4] Z: Add support for offheap allocation in multianewarray evaluator Enhances the multianewarray evaluator to support offheap allocation on IBM Z platform. Offheap allocation for arrays with a zero-length second dimension is not supported; in such cases, a helper call is triggered to handle the allocation. signed-off-by: Ehsan Kiani Far --- .../compiler/z/codegen/J9TreeEvaluator.cpp | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index 12f33ed3381..45330c79f10 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -4885,10 +4885,6 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, TR::Register *dim2SizeReg = cg->allocateRegister(); dependencies->addPostCondition(dim2SizeReg, TR::RealRegister::AssignAny); - cursor = generateRXInstruction(cg, TR::InstOpCode::LTGF, node, dim2SizeReg, generateS390MemoryReference(dimsPtrReg, 0, cg), cursor); - iComment("Load 2st dim length."); - // The size of zero length array is already loaded in size register. Jump over the array size calculation instructions if length is 0. - TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); if (componentSize == 1) { @@ -4907,10 +4903,23 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, // allocate inline is not frequent. cursor = generateRSInstruction(cg, TR::InstOpCode::SLA, node, dim2SizeReg, trailingZeroes(componentSize), cursor); } - // Bypass dim2 size calculation if the size is zero. - cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); - // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). - cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_MASK5, node, inlineAllocFailLabel, cursor); + + TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isOffHeapAllocationEnabled = TR::Compiler->om.isOffHeapAllocationEnabled(); + if (isOffHeapAllocationEnabled) + { + // Call helper if dim2 length is negative, zero, or larger that INT32_MAX (overflow). + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BRNP, node, inlineAllocFailLabel, cursor); + } + else +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + { + // Bypass dim2 size calculation if the size is zero. + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); + // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_MASK5, node, inlineAllocFailLabel, cursor); + } int32_t headerSize= TR::Compiler->om.contiguousArrayHeaderSizeInBytes(); // size = (size + alignmentConstant - 1) & -alignmentConstant equation round up the size to a factor of alignmentConstant. @@ -5016,6 +5025,17 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, // Load the address of the first element of the first dimension array in sizeReg. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(resultReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); + +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isOffHeapAllocationEnabled) + { + // Store first element's address in data address field. + cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, sizeReg, generateS390MemoryReference(resultReg, fej9->getOffsetOfContiguousDataAddrField(), cg), cursor); + // Subtract the header size from the dim2 size, so we can move to the next leaf by adding this value to the address of the leaf's first element. + cursor = generateRILInstruction(cg, TR::InstOpCode::SLFI, node, dim2SizeReg, (int32_t)TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cursor); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + // Start setting second dim: TR::LabelSymbol *secondDimLabel = generateLabelSymbol(cg); cursor = generateS390LabelInstruction(cg, TR::InstOpCode::label, node, secondDimLabel, cursor); @@ -5040,6 +5060,18 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, generateS390MemoryReference(sizeReg, 0, cg), cursor); } +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isOffHeapAllocationEnabled) + { + // Load the first element address. + cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, + generateS390MemoryReference(dim1SizeReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); + // Store the first element address in data address field. + cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, + (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + // Load the next leaf address in dim1SizeReg. cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); // Load the next element address of the first dim array in sizeReg. @@ -5107,9 +5139,7 @@ J9::Z::TreeEvaluator::multianewArrayEvaluator(TR::Node * node, TR::CodeGenerator if ((nDims == 2) && (componentSize > 0) && comp->target().cpu.isAtLeast(OMR_PROCESSOR_S390_Z196) - && !comp->suppressAllocationInlining() - // Temporarily disable inline allocation for offHeap. - && !TR::Compiler->om.isOffHeapAllocationEnabled()) + && !comp->suppressAllocationInlining()) { return generateMultianewArrayWithInlineAllocators(node, cg, componentSize); } From 9229e518983a59f27f48e054745eebe187a001b6 Mon Sep 17 00:00:00 2001 From: Ehsan Kiani Far Date: Mon, 20 Oct 2025 12:45:30 -0400 Subject: [PATCH 2/4] Z: Add support for inline zero-length offheap 2D array allocation Enhances the multianewarray evaluator to support zero-length offheap allocation on IBM Z platform. signed-off-by: Ehsan Kiani Far --- .../compiler/z/codegen/J9TreeEvaluator.cpp | 72 +++++++++++-------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index 45330c79f10..ef65a56eac2 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -4905,21 +4905,10 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, } TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); -#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) - bool isOffHeapAllocationEnabled = TR::Compiler->om.isOffHeapAllocationEnabled(); - if (isOffHeapAllocationEnabled) - { - // Call helper if dim2 length is negative, zero, or larger that INT32_MAX (overflow). - cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BRNP, node, inlineAllocFailLabel, cursor); - } - else -#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ - { - // Bypass dim2 size calculation if the size is zero. - cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); - // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). - cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_MASK5, node, inlineAllocFailLabel, cursor); - } + // Bypass dim2 size calculation if the size is zero. + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); + // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_MASK5, node, inlineAllocFailLabel, cursor); int32_t headerSize= TR::Compiler->om.contiguousArrayHeaderSizeInBytes(); // size = (size + alignmentConstant - 1) & -alignmentConstant equation round up the size to a factor of alignmentConstant. @@ -5027,12 +5016,17 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, generateS390MemoryReference(resultReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isOffHeapAllocationEnabled = TR::Compiler->om.isOffHeapAllocationEnabled(); if (isOffHeapAllocationEnabled) { // Store first element's address in data address field. cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, sizeReg, generateS390MemoryReference(resultReg, fej9->getOffsetOfContiguousDataAddrField(), cg), cursor); // Subtract the header size from the dim2 size, so we can move to the next leaf by adding this value to the address of the leaf's first element. cursor = generateRILInstruction(cg, TR::InstOpCode::SLFI, node, dim2SizeReg, (int32_t)TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cursor); + // The length of the second dimension is stored in the upper 32 bits of the scratch register. + // Comparing the full 64-bit scratch register with its lower 32 bits sets the condition code to COND_BNE if the second dimension is non-zero. + cursor = generateRREInstruction(cg, TR::InstOpCode::CGFR, node, scratchReg, scratchReg, cursor); + // The condition code has already been set and must remain unchanged during the second dimension allocation loop in this code path. } #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ @@ -5046,16 +5040,10 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, scratchReg, generateS390MemoryReference(dim1SizeReg, static_cast(TR::Compiler->om.offsetOfContiguousArraySizeField()), cg), cursor); int32_t shiftAmount = TR::Compiler->om.compressedReferenceShift(); - if (shiftAmount > 0) - { - // Calculate the compressed reference of leaf array in the higher 32bits of dim2SizeReg. - cursor = generateRIEInstruction(cg, TR::InstOpCode::RISBG, node, dim2SizeReg, dim1SizeReg, 0, 31, 32-shiftAmount, cursor); - // Store the compressed 32 bit leaf address in the relevant element of the first dim array. - cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, dim2SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); - } - else + + if (shiftAmount == 0) { - // Store the 32 bit leaf address in the relevant element of the first dim array. + // Store the leaf address in the relevant element of the first dim array. cursor = generateRXInstruction(cg, (TR::Compiler->om.compressObjectReferences() ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, dim1SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); } @@ -5063,17 +5051,43 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) if (isOffHeapAllocationEnabled) { + if (shiftAmount > 0) + { + // Compress the leaf array reference and store it in the first element of the dimension array. + // Avoid using any instructions that could affect the branch condition in this code path. + cursor = generateRSInstruction(cg, TR::InstOpCode::SRLG, node, dim1SizeReg, dim1SizeReg, shiftAmount, cursor); + cursor = generateRXInstruction(cg, TR::InstOpCode::ST, node, dim1SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); + // Decompress the leaf array reference. + cursor = generateRSInstruction(cg, TR::InstOpCode::SLLG, node, dim1SizeReg, dim1SizeReg, shiftAmount, cursor); + } // Load the first element address. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); - // Store the first element address in data address field. - cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, - (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); + // Store the address of the first element in the data address field only if the second dimension length is non-zero. + cursor = generateRSInstruction(cg, TR::InstOpCode::STOCG, node, dim1SizeReg, getMaskForBranchCondition(TR::InstOpCode::COND_BNE), + generateS390MemoryReference(dim1SizeReg, (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); + // Load the next leaf address in dim1SizeReg. + cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, dim2SizeReg, 0, cg), cursor); } + else #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + { + if (shiftAmount > 0) + { + // Calculate the compressed reference of leaf array in the higher 32bits of dim2SizeReg. + cursor = generateRIEInstruction(cg, TR::InstOpCode::RISBG, node, dim2SizeReg, dim1SizeReg, 0, 31, 32-shiftAmount, cursor); + // Store the compressed 32 bit leaf address in the relevant element of the first dim array. + cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, dim2SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); + // Load the next leaf address in dim1SizeReg. + cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); + } + else + { + // Load the next leaf address in dim1SizeReg. + cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, dim2SizeReg, 0, cg), cursor); + } + } - // Load the next leaf address in dim1SizeReg. - cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); // Load the next element address of the first dim array in sizeReg. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(sizeReg, elementSize, cg), cursor); cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRCT, node, scratchReg, secondDimLabel, cursor); From 03e3d37bb32fa30913b7025baeec7d9c934b14a9 Mon Sep 17 00:00:00 2001 From: Ehsan Kiani Far Date: Mon, 27 Oct 2025 11:32:53 -0400 Subject: [PATCH 3/4] Z: Enable 2D array inlining with non-zero constant second dimension Previously, inlining of constant 2D arrays with a non-zero second dimension was disabled on z due to limitations in handling such cases. However, recent improvements have resolved those issues, making it safe and beneficial to inline these arrays. Remove the constraint that prevented inlining of non-zero constant second dimension 2D arrays. signed-off-by: Ehsan Kiani Far --- runtime/compiler/env/VMJ9.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runtime/compiler/env/VMJ9.cpp b/runtime/compiler/env/VMJ9.cpp index 83fdf0911e2..190c1a76245 100644 --- a/runtime/compiler/env/VMJ9.cpp +++ b/runtime/compiler/env/VMJ9.cpp @@ -3513,11 +3513,11 @@ TR_J9VMBase::lowerMultiANewArray(TR::Compilation * comp, TR::Node * root, TR::Tr root->setNumChildren(3); static bool recreateRoot = feGetEnv("TR_LowerMultiANewArrayRecreateRoot") ? true : false; -#if defined(TR_HOST_POWER) +#if defined(TR_HOST_POWER) || defined(TR_HOST_S390) if (!comp->target().is64Bit() || recreateRoot || dims > 2) #else if (!comp->target().is64Bit() || recreateRoot || dims > 2 || secondDimConstNonZero) -#endif /* defined(TR_HOST_POWER) */ +#endif /* defined(TR_HOST_POWER) || defined(TR_HOST_S390) */ TR::Node::recreate(root, TR::acall); return treeTop; From e9a7d49b957d108c0127e799490ea2ba025972b8 Mon Sep 17 00:00:00 2001 From: ehsankianifar Date: Tue, 2 Dec 2025 12:02:40 -0500 Subject: [PATCH 4/4] Z: Move off-heap zero length allocation OOL Refactored off-heap 2D array allocation logic to improve clarity and maintainability on IBM Z platform. Previously, STOCG was used to store the data address only when the length was non-zero, relying on condition codes to control flow. This approach introduced hidden dependencies on allocator condition codes, making the code harder to read and prone to future bugs. Updated the implementation to allocate non-zero length off-heap arrays inline, while handling zero-length arrays out of line. This removes reliance on condition codes for allocation decisions, simplifies control flow, and ensures consistent behavior across allocation paths. signed-off-by: Ehsan Kiani Far --- .../compiler/z/codegen/J9TreeEvaluator.cpp | 93 +++++++++++-------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index ef65a56eac2..15afc2289cb 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -4905,6 +4905,7 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, } TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); + // Bypass dim2 size calculation if the size is zero. cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). @@ -5017,16 +5018,19 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) bool isOffHeapAllocationEnabled = TR::Compiler->om.isOffHeapAllocationEnabled(); + TR::LabelSymbol *zeroLengthSecondDimLabel = NULL; if (isOffHeapAllocationEnabled) { // Store first element's address in data address field. cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, sizeReg, generateS390MemoryReference(resultReg, fej9->getOffsetOfContiguousDataAddrField(), cg), cursor); - // Subtract the header size from the dim2 size, so we can move to the next leaf by adding this value to the address of the leaf's first element. - cursor = generateRILInstruction(cg, TR::InstOpCode::SLFI, node, dim2SizeReg, (int32_t)TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cursor); + // If the second dimension length is zero, allocate the leaf array OOL. + zeroLengthSecondDimLabel = generateLabelSymbol(cg); // The length of the second dimension is stored in the upper 32 bits of the scratch register. - // Comparing the full 64-bit scratch register with its lower 32 bits sets the condition code to COND_BNE if the second dimension is non-zero. + // Comparing the full 64-bit scratch register with its lower 32 bits sets the condition code to COND_BE if the second dimension is zero. cursor = generateRREInstruction(cg, TR::InstOpCode::CGFR, node, scratchReg, scratchReg, cursor); - // The condition code has already been set and must remain unchanged during the second dimension allocation loop in this code path. + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BE, node, zeroLengthSecondDimLabel, cursor); + // Subtract the header size from the dim2 size, so we can move to the next leaf by adding this value to the address of the leaf's first element. + cursor = generateRILInstruction(cg, TR::InstOpCode::SLFI, node, dim2SizeReg, (int32_t)TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cursor); } #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ @@ -5040,8 +5044,14 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, scratchReg, generateS390MemoryReference(dim1SizeReg, static_cast(TR::Compiler->om.offsetOfContiguousArraySizeField()), cg), cursor); int32_t shiftAmount = TR::Compiler->om.compressedReferenceShift(); - - if (shiftAmount == 0) + if (shiftAmount > 0) + { + // Calculate the compressed reference of leaf array in the higher 32bits of dim2SizeReg. + cursor = generateRIEInstruction(cg, TR::InstOpCode::RISBG, node, dim2SizeReg, dim1SizeReg, 0, 31, 32-shiftAmount, cursor); + // Store the compressed 32 bit leaf address in the relevant element of the first dim array. + cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, dim2SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); + } + else { // Store the leaf address in the relevant element of the first dim array. cursor = generateRXInstruction(cg, (TR::Compiler->om.compressObjectReferences() ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, dim1SizeReg, @@ -5051,48 +5061,55 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) if (isOffHeapAllocationEnabled) { - if (shiftAmount > 0) - { - // Compress the leaf array reference and store it in the first element of the dimension array. - // Avoid using any instructions that could affect the branch condition in this code path. - cursor = generateRSInstruction(cg, TR::InstOpCode::SRLG, node, dim1SizeReg, dim1SizeReg, shiftAmount, cursor); - cursor = generateRXInstruction(cg, TR::InstOpCode::ST, node, dim1SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); - // Decompress the leaf array reference. - cursor = generateRSInstruction(cg, TR::InstOpCode::SLLG, node, dim1SizeReg, dim1SizeReg, shiftAmount, cursor); - } // Load the first element address. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); - // Store the address of the first element in the data address field only if the second dimension length is non-zero. - cursor = generateRSInstruction(cg, TR::InstOpCode::STOCG, node, dim1SizeReg, getMaskForBranchCondition(TR::InstOpCode::COND_BNE), - generateS390MemoryReference(dim1SizeReg, (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); - // Load the next leaf address in dim1SizeReg. - cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, dim2SizeReg, 0, cg), cursor); + // Store the first element address in data address field. + cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, + (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); } - else #endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ - { - if (shiftAmount > 0) - { - // Calculate the compressed reference of leaf array in the higher 32bits of dim2SizeReg. - cursor = generateRIEInstruction(cg, TR::InstOpCode::RISBG, node, dim2SizeReg, dim1SizeReg, 0, 31, 32-shiftAmount, cursor); - // Store the compressed 32 bit leaf address in the relevant element of the first dim array. - cursor = generateRXInstruction(cg, TR::InstOpCode::STFH, node, dim2SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); - // Load the next leaf address in dim1SizeReg. - cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); - } - else - { - // Load the next leaf address in dim1SizeReg. - cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, dim2SizeReg, 0, cg), cursor); - } - } + // Load the next leaf address in dim1SizeReg. + cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); // Load the next element address of the first dim array in sizeReg. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(sizeReg, elementSize, cg), cursor); cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRCT, node, scratchReg, secondDimLabel, cursor); cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_B, node, controlFlowEndLabel, cursor); - iComment("Allocation done!"); + + /********************************************* OOL zero length offheap leaf allocator *********************************************/ +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isOffHeapAllocationEnabled) + { + TR_S390OutOfLineCodeSection *zeroLengthSecondDimAlloc = new (cg->trHeapMemory()) TR_S390OutOfLineCodeSection(zeroLengthSecondDimLabel, controlFlowEndLabel, cg); + cg->getS390OutOfLineCodeSectionList().push_front(zeroLengthSecondDimAlloc); + zeroLengthSecondDimAlloc->swapInstructionListsWithCompilation(); + generateS390LabelInstruction(cg, TR::InstOpCode::label, node, zeroLengthSecondDimLabel); + + // Store the class field. + generateRXInstruction(cg, (compressedObjectHeaders ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, classReg, + generateS390MemoryReference(dim1SizeReg, static_cast(TR::Compiler->om.offsetOfObjectVftField()), cg)); + TR::Register *leafArrayRefReg = dim1SizeReg; + if (shiftAmount > 0) + { + leafArrayRefReg = dim2SizeReg; + // Calculate the compressed reference of leaf array in leafArrayRefReg. + generateRSInstruction(cg, TR::InstOpCode::SRLG, node, leafArrayRefReg, dim1SizeReg, shiftAmount); + } + // Store the leaf address in the relevant element of the first dim array. + generateRXInstruction(cg, (TR::Compiler->om.compressObjectReferences() ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, leafArrayRefReg, + generateS390MemoryReference(sizeReg, 0, cg)); + // Load the next leaf address in dim1SizeReg. + size_t leafArraySize = OMR::align(TR::Compiler->om.discontiguousArrayHeaderSizeInBytes(), alignmentConstant); + generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, leafArraySize, cg)); + // Load the next element address of the first dim array in sizeReg. + generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(sizeReg, elementSize, cg)); + generateS390BranchInstruction(cg, TR::InstOpCode::BRCT, node, scratchReg, zeroLengthSecondDimLabel); + + generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BRC, node, controlFlowEndLabel); + zeroLengthSecondDimAlloc->swapInstructionListsWithCompilation(); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ TR::LabelSymbol *slowPathLabel = generateLabelSymbol(cg); cursor = generateS390LabelInstruction(cg, TR::InstOpCode::label, node, inlineAllocFailLabel, cursor);