Index: clang/lib/CodeGen/CGStmtOpenMP.cpp =================================================================== --- clang/lib/CodeGen/CGStmtOpenMP.cpp +++ clang/lib/CodeGen/CGStmtOpenMP.cpp @@ -2662,7 +2662,8 @@ } // Add simd metadata to the collapsed loop. Do not generate // another loop for if clause. Support for if clause is done earlier. - OMPBuilder.applySimd(CLI, /*IfCond*/ nullptr, Simdlen, Safelen); + OMPBuilder.applySimd(CLI, /*AlignedVars*/ {}, /*Alignment*/ nullptr, + /*IfCond*/ nullptr, Simdlen, Safelen); return; } }; Index: llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h =================================================================== --- llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -617,11 +617,16 @@ /// to the cloned loop. The cloned loop is executed when ifCond is evaluated /// to false. /// - /// \param Loop The loop to simd-ize. - /// \param IfCond The value which corresponds to the if clause condition. + /// \param Loop The loop to simd-ize. + /// \param AlignedVars The variables which need to aligned. + /// \param Alignment The value of alignment. + /// \param IfCond The value which corresponds to the if clause + /// condition. /// \param Simdlen The Simdlen length to apply to the simd loop. /// \param Safelen The Safelen length to apply to the simd loop. - void applySimd(CanonicalLoopInfo *Loop, Value *IfCond, ConstantInt *Simdlen, + void applySimd(CanonicalLoopInfo *Loop, + llvm::ArrayRef AlignedVars, + llvm::Value *Alignment, Value *IfCond, ConstantInt *Simdlen, ConstantInt *Safelen); /// Generator for '#omp flush' Index: llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp =================================================================== --- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -2965,12 +2965,16 @@ Builder.CreateBr(NewBlocks.front()); } -void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, Value *IfCond, - ConstantInt *Simdlen, ConstantInt *Safelen) { +void OpenMPIRBuilder::applySimd(CanonicalLoopInfo *CanonicalLoop, + ArrayRef AlignedVars, Value *Alignment, + Value *IfCond, ConstantInt *Simdlen, + ConstantInt *Safelen) { LLVMContext &Ctx = Builder.getContext(); Function *F = CanonicalLoop->getFunction(); + const int DefaultAlignment = 16; + // TODO: We should not rely on pass manager. Currently we use pass manager // only for getting llvm::Loop which corresponds to given CanonicalLoopInfo // object. We should have a method which returns all blocks between @@ -2985,6 +2989,30 @@ Loop *L = LI.getLoopFor(CanonicalLoop->getHeader()); + for (auto *AlignedItem : AlignedVars) { + assert(isa(AlignedItem) && + "Value which needs to be aligned must represented by alloca " + "instruction"); + Value *AlignPtrInstruction = nullptr; + AllocaInst *AllocaInstruction = dyn_cast(AlignedItem); + Builder.SetInsertPoint(CanonicalLoop->getPreheader()->getTerminator()); + Type *AllocatedVarType = AllocaInstruction->getAllocatedType(); + if (isa(AllocatedVarType)) + AlignPtrInstruction = Builder.CreateInBoundsGEP( + AllocaInstruction->getAllocatedType(), AllocaInstruction, + SmallVector{Builder.getInt64(0), Builder.getInt64(0)}, + "arraydecay"); + else if (isa(AllocatedVarType)) + AlignPtrInstruction = Builder.CreateLoad( + AllocaInstruction->getAllocatedType(), AllocaInstruction); + assert(AlignPtrInstruction && + "Aligned variables must be either pointer or array type"); + + Builder.CreateAlignmentAssumption( + F->getParent()->getDataLayout(), AlignPtrInstruction, + Alignment ? Alignment : Builder.getInt64(DefaultAlignment)); + } + if (IfCond) { ValueToValueMapTy VMap; createIfVersion(CanonicalLoop, IfCond, VMap, "simd"); Index: llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp =================================================================== --- llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp +++ llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp @@ -1771,7 +1771,8 @@ CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); // Simd-ize the loop. - OMPBuilder.applySimd(CLI, /* IfCond */ nullptr, /* Simdlen */ nullptr, + OMPBuilder.applySimd(CLI, /* AlignedVars */ {}, /* Alignment */ nullptr, + /* IfCond */ nullptr, /* Simdlen */ nullptr, /* Safelen */ nullptr); OMPBuilder.finalize(); @@ -1797,13 +1798,137 @@ })); } +TEST_F(OpenMPIRBuilderTest, ApplySimdiDefaultAligned) { + OpenMPIRBuilder OMPBuilder(*M); + IRBuilder<> Builder(BB); + AllocaInst *Alloc1 = + Builder.CreateAlloca(Builder.getInt8PtrTy(), Builder.getInt64(1)); + AllocaInst *Alloc2 = Builder.CreateAlloca( + ArrayType::get(Builder.getInt32Ty(), 10), Builder.getInt64(1)); + SmallVector AlignedVars; + auto Int8Ty = Builder.getInt8Ty(); + Instruction *MallocInstr = CallInst::CreateMalloc( + Alloc2, Builder.getInt64Ty(), Int8Ty, ConstantExpr::getSizeOf(Int8Ty), + Builder.getInt64(400), nullptr, ""); + Builder.CreateStore(MallocInstr, Alloc1); + + AlignedVars.push_back(Alloc1); + AlignedVars.push_back(Alloc2); + + CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); + + // Simd-ize the loop. + OMPBuilder.applySimd(CLI, AlignedVars, /* Alignment */ nullptr, + /* IfCond */ nullptr, /* Simdlen */ nullptr, + /* Safelen */ nullptr); + + OMPBuilder.finalize(); + EXPECT_FALSE(verifyModule(*M, &errs())); + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + LoopInfo &LI = FAM.getResult(*F); + + const std::vector &TopLvl = LI.getTopLevelLoops(); + EXPECT_EQ(TopLvl.size(), 1u); + + Loop *L = TopLvl.front(); + EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses")); + EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable")); + + // Check for llvm.access.group metadata attached to the printf + // function in the loop body. + BasicBlock *LoopBody = CLI->getBody(); + EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) { + return I.getMetadata("llvm.access.group") != nullptr; + })); + + // Check if number of assumption instructions is equal to number of aligned + // variables + BasicBlock *LoopPreheader = CLI->getPreheader(); + size_t NumAssummptionCallsInPreheader = count_if( + *LoopPreheader, [](Instruction &I) { return isa(I); }); + EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size()); +} + +TEST_F(OpenMPIRBuilderTest, ApplySimdiCustomAligned) { + OpenMPIRBuilder OMPBuilder(*M); + IRBuilder<> Builder(BB); + const int AlignmentValue = 32; + AllocaInst *Alloc1 = + Builder.CreateAlloca(Builder.getInt8PtrTy(), Builder.getInt64(1)); + AllocaInst *Alloc2 = Builder.CreateAlloca( + ArrayType::get(Builder.getInt32Ty(), 10), Builder.getInt64(1)); + SmallVector AlignedVars; + auto Int8Ty = Builder.getInt8Ty(); + Instruction *MallocInstr = CallInst::CreateMalloc( + Alloc2, Builder.getInt64Ty(), Int8Ty, ConstantExpr::getSizeOf(Int8Ty), + Builder.getInt64(400), nullptr, ""); + Builder.CreateStore(MallocInstr, Alloc1); + + AlignedVars.push_back(Alloc1); + AlignedVars.push_back(Alloc2); + + CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); + + // Simd-ize the loop. + OMPBuilder.applySimd(CLI, AlignedVars, Builder.getInt64(AlignmentValue), + /* IfCond */ nullptr, /* Simdlen */ nullptr, + /* Safelen */ nullptr); + + OMPBuilder.finalize(); + EXPECT_FALSE(verifyModule(*M, &errs())); + + PassBuilder PB; + FunctionAnalysisManager FAM; + PB.registerFunctionAnalyses(FAM); + LoopInfo &LI = FAM.getResult(*F); + + const std::vector &TopLvl = LI.getTopLevelLoops(); + EXPECT_EQ(TopLvl.size(), 1u); + + Loop *L = TopLvl.front(); + EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses")); + EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable")); + + // Check for llvm.access.group metadata attached to the printf + // function in the loop body. + BasicBlock *LoopBody = CLI->getBody(); + EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) { + return I.getMetadata("llvm.access.group") != nullptr; + })); + + // Check if number of assumption instructions is equal to number of aligned + // variables + BasicBlock *LoopPreheader = CLI->getPreheader(); + size_t NumAssummptionCallsInPreheader = count_if( + *LoopPreheader, [](Instruction &I) { return isa(I); }); + EXPECT_EQ(NumAssummptionCallsInPreheader, AlignedVars.size()); + + // Check if variables are correctly aligned + for (auto &Instr : LoopPreheader->getInstList()) { + if (isa(Instr)) { + auto AssumeInstruction = dyn_cast(&Instr); + if (AssumeInstruction->getNumTotalBundleOperands()) { + auto Bundle = AssumeInstruction->getOperandBundleAt(0); + if (Bundle.getTagName() == "align") { + EXPECT_TRUE(isa(Bundle.Inputs[1])); + auto ConstIntVal = dyn_cast(Bundle.Inputs[1]); + EXPECT_EQ(ConstIntVal->getSExtValue(), AlignmentValue); + } + } + } + } +} TEST_F(OpenMPIRBuilderTest, ApplySimdlen) { OpenMPIRBuilder OMPBuilder(*M); CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); // Simd-ize the loop. - OMPBuilder.applySimd(CLI, /* IfCond */ nullptr, + OMPBuilder.applySimd(CLI, /* AlignedVars */ {}, /* Alignment */ nullptr, + /* IfCond */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3), /* Safelen */ nullptr); @@ -1837,9 +1962,9 @@ CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); // Simd-ize the loop. - OMPBuilder.applySimd(CLI, /* IfCond */ nullptr, - /* Simdlen */ nullptr, - ConstantInt::get(Type::getInt32Ty(Ctx), 3)); + OMPBuilder.applySimd( + CLI, /* AlignedVars */ {}, /* Alignment */ nullptr, /* IfCond */ nullptr, + /* Simdlen */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 3)); OMPBuilder.finalize(); EXPECT_FALSE(verifyModule(*M, &errs())); @@ -1871,7 +1996,8 @@ CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); // Simd-ize the loop. - OMPBuilder.applySimd(CLI, /* IfCond */ nullptr, + OMPBuilder.applySimd(CLI, /* AlignedVars */ {}, /* Alignment */ nullptr, + /* IfCond */ nullptr, ConstantInt::get(Type::getInt32Ty(Ctx), 2), ConstantInt::get(Type::getInt32Ty(Ctx), 3)); @@ -1916,7 +2042,8 @@ CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32); // Simd-ize the loop with if condition - OMPBuilder.applySimd(CLI, IfCmp, ConstantInt::get(Type::getInt32Ty(Ctx), 3), + OMPBuilder.applySimd(CLI, /* AlignedVars */ {}, /* Alignment */ nullptr, + IfCmp, ConstantInt::get(Type::getInt32Ty(Ctx), 3), /* Safelen */ nullptr); OMPBuilder.finalize(); Index: mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp =================================================================== --- mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -996,7 +996,7 @@ safelen = builder.getInt64(safelenVar.value()); ompBuilder->applySimd( - loopInfo, + loopInfo, {}, nullptr, loop.if_expr() ? moduleTranslation.lookupValue(loop.if_expr()) : nullptr, simdlen, safelen);