diff --git a/runtime/compiler/env/VMJ9.cpp b/runtime/compiler/env/VMJ9.cpp index 83fdf0911e2..190c1a76245 100644 --- a/runtime/compiler/env/VMJ9.cpp +++ b/runtime/compiler/env/VMJ9.cpp @@ -3513,11 +3513,11 @@ TR_J9VMBase::lowerMultiANewArray(TR::Compilation * comp, TR::Node * root, TR::Tr root->setNumChildren(3); static bool recreateRoot = feGetEnv("TR_LowerMultiANewArrayRecreateRoot") ? true : false; -#if defined(TR_HOST_POWER) +#if defined(TR_HOST_POWER) || defined(TR_HOST_S390) if (!comp->target().is64Bit() || recreateRoot || dims > 2) #else if (!comp->target().is64Bit() || recreateRoot || dims > 2 || secondDimConstNonZero) -#endif /* defined(TR_HOST_POWER) */ +#endif /* defined(TR_HOST_POWER) || defined(TR_HOST_S390) */ TR::Node::recreate(root, TR::acall); return treeTop; diff --git a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp index 12f33ed3381..15afc2289cb 100644 --- a/runtime/compiler/z/codegen/J9TreeEvaluator.cpp +++ b/runtime/compiler/z/codegen/J9TreeEvaluator.cpp @@ -4885,10 +4885,6 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, TR::Register *dim2SizeReg = cg->allocateRegister(); dependencies->addPostCondition(dim2SizeReg, TR::RealRegister::AssignAny); - cursor = generateRXInstruction(cg, TR::InstOpCode::LTGF, node, dim2SizeReg, generateS390MemoryReference(dimsPtrReg, 0, cg), cursor); - iComment("Load 2st dim length."); - // The size of zero length array is already loaded in size register. Jump over the array size calculation instructions if length is 0. - TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); if (componentSize == 1) { @@ -4907,6 +4903,9 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, // allocate inline is not frequent. cursor = generateRSInstruction(cg, TR::InstOpCode::SLA, node, dim2SizeReg, trailingZeroes(componentSize), cursor); } + + TR::LabelSymbol *zeroSecondDimLabel = generateLabelSymbol(cg); + // Bypass dim2 size calculation if the size is zero. cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BZ, node, zeroSecondDimLabel, cursor); // Call helper if the length is negative or length * componentSize is larger that INT32_MAX (overflow). @@ -5016,6 +5015,25 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, // Load the address of the first element of the first dimension array in sizeReg. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(resultReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); + +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + bool isOffHeapAllocationEnabled = TR::Compiler->om.isOffHeapAllocationEnabled(); + TR::LabelSymbol *zeroLengthSecondDimLabel = NULL; + if (isOffHeapAllocationEnabled) + { + // Store first element's address in data address field. + cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, sizeReg, generateS390MemoryReference(resultReg, fej9->getOffsetOfContiguousDataAddrField(), cg), cursor); + // If the second dimension length is zero, allocate the leaf array OOL. + zeroLengthSecondDimLabel = generateLabelSymbol(cg); + // The length of the second dimension is stored in the upper 32 bits of the scratch register. + // Comparing the full 64-bit scratch register with its lower 32 bits sets the condition code to COND_BE if the second dimension is zero. + cursor = generateRREInstruction(cg, TR::InstOpCode::CGFR, node, scratchReg, scratchReg, cursor); + cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BE, node, zeroLengthSecondDimLabel, cursor); + // Subtract the header size from the dim2 size, so we can move to the next leaf by adding this value to the address of the leaf's first element. + cursor = generateRILInstruction(cg, TR::InstOpCode::SLFI, node, dim2SizeReg, (int32_t)TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cursor); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + // Start setting second dim: TR::LabelSymbol *secondDimLabel = generateLabelSymbol(cg); cursor = generateS390LabelInstruction(cg, TR::InstOpCode::label, node, secondDimLabel, cursor); @@ -5035,18 +5053,63 @@ static TR::Register * generateMultianewArrayWithInlineAllocators(TR::Node *node, } else { - // Store the 32 bit leaf address in the relevant element of the first dim array. + // Store the leaf address in the relevant element of the first dim array. cursor = generateRXInstruction(cg, (TR::Compiler->om.compressObjectReferences() ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, dim1SizeReg, generateS390MemoryReference(sizeReg, 0, cg), cursor); } +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isOffHeapAllocationEnabled) + { + // Load the first element address. + cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, + generateS390MemoryReference(dim1SizeReg, TR::Compiler->om.contiguousArrayHeaderSizeInBytes(), cg), cursor); + // Store the first element address in data address field. + cursor = generateRXInstruction(cg, TR::InstOpCode::STG, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, + (fej9->getOffsetOfContiguousDataAddrField() - TR::Compiler->om.contiguousArrayHeaderSizeInBytes()), cg), cursor); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ + // Load the next leaf address in dim1SizeReg. cursor = generateRREInstruction(cg, TR::InstOpCode::ALGFR, node, dim1SizeReg, dim2SizeReg, cursor); // Load the next element address of the first dim array in sizeReg. cursor = generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(sizeReg, elementSize, cg), cursor); cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRCT, node, scratchReg, secondDimLabel, cursor); cursor = generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_B, node, controlFlowEndLabel, cursor); - iComment("Allocation done!"); + + /********************************************* OOL zero length offheap leaf allocator *********************************************/ +#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) + if (isOffHeapAllocationEnabled) + { + TR_S390OutOfLineCodeSection *zeroLengthSecondDimAlloc = new (cg->trHeapMemory()) TR_S390OutOfLineCodeSection(zeroLengthSecondDimLabel, controlFlowEndLabel, cg); + cg->getS390OutOfLineCodeSectionList().push_front(zeroLengthSecondDimAlloc); + zeroLengthSecondDimAlloc->swapInstructionListsWithCompilation(); + generateS390LabelInstruction(cg, TR::InstOpCode::label, node, zeroLengthSecondDimLabel); + + // Store the class field. + generateRXInstruction(cg, (compressedObjectHeaders ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, classReg, + generateS390MemoryReference(dim1SizeReg, static_cast(TR::Compiler->om.offsetOfObjectVftField()), cg)); + TR::Register *leafArrayRefReg = dim1SizeReg; + if (shiftAmount > 0) + { + leafArrayRefReg = dim2SizeReg; + // Calculate the compressed reference of leaf array in leafArrayRefReg. + generateRSInstruction(cg, TR::InstOpCode::SRLG, node, leafArrayRefReg, dim1SizeReg, shiftAmount); + } + // Store the leaf address in the relevant element of the first dim array. + generateRXInstruction(cg, (TR::Compiler->om.compressObjectReferences() ? TR::InstOpCode::ST : TR::InstOpCode::STG), node, leafArrayRefReg, + generateS390MemoryReference(sizeReg, 0, cg)); + // Load the next leaf address in dim1SizeReg. + size_t leafArraySize = OMR::align(TR::Compiler->om.discontiguousArrayHeaderSizeInBytes(), alignmentConstant); + generateRXInstruction(cg, TR::InstOpCode::LA, node, dim1SizeReg, generateS390MemoryReference(dim1SizeReg, leafArraySize, cg)); + // Load the next element address of the first dim array in sizeReg. + generateRXInstruction(cg, TR::InstOpCode::LA, node, sizeReg, generateS390MemoryReference(sizeReg, elementSize, cg)); + generateS390BranchInstruction(cg, TR::InstOpCode::BRCT, node, scratchReg, zeroLengthSecondDimLabel); + + generateS390BranchInstruction(cg, TR::InstOpCode::BRC, TR::InstOpCode::COND_BRC, node, controlFlowEndLabel); + zeroLengthSecondDimAlloc->swapInstructionListsWithCompilation(); + } +#endif /* J9VM_GC_SPARSE_HEAP_ALLOCATION */ TR::LabelSymbol *slowPathLabel = generateLabelSymbol(cg); cursor = generateS390LabelInstruction(cg, TR::InstOpCode::label, node, inlineAllocFailLabel, cursor); @@ -5107,9 +5170,7 @@ J9::Z::TreeEvaluator::multianewArrayEvaluator(TR::Node * node, TR::CodeGenerator if ((nDims == 2) && (componentSize > 0) && comp->target().cpu.isAtLeast(OMR_PROCESSOR_S390_Z196) - && !comp->suppressAllocationInlining() - // Temporarily disable inline allocation for offHeap. - && !TR::Compiler->om.isOffHeapAllocationEnabled()) + && !comp->suppressAllocationInlining()) { return generateMultianewArrayWithInlineAllocators(node, cg, componentSize); }