Index: flang/include/flang/Optimizer/Builder/PPCIntrinsicCall.h =================================================================== --- flang/include/flang/Optimizer/Builder/PPCIntrinsicCall.h +++ flang/include/flang/Optimizer/Builder/PPCIntrinsicCall.h @@ -45,6 +45,21 @@ Xor }; +/// Enums used to templatize and share lowering of PowerPC MMA intrinsics. +enum class MMAOp { + AssembleAcc, + AssemblePair, + DisassembleAcc, + DisassemblePair, +}; + +enum class MMAHandlerOp { + NoOp, + SubToFunc, + SubToFuncReverseArgOnLE, + FirstArgIsResult, +}; + // Wrapper struct to encapsulate information for a vector type. Preserves // sign of eleTy if eleTy is signed/unsigned integer. Helps with vector type // conversions. @@ -120,13 +135,16 @@ PPCIntrinsicLibrary() = delete; PPCIntrinsicLibrary(const PPCIntrinsicLibrary &) = delete; + // PPC MMA intrinsic generic handler + template + void genMmaIntr(llvm::ArrayRef); + // PPC intrinsic handlers. template void genMtfsf(llvm::ArrayRef); fir::ExtendedValue genVecAbs(mlir::Type resultType, llvm::ArrayRef args); - template fir::ExtendedValue genVecAddAndMulSubXor(mlir::Type resultType, Index: flang/lib/Optimizer/Builder/IntrinsicCall.cpp =================================================================== --- flang/lib/Optimizer/Builder/IntrinsicCall.cpp +++ flang/lib/Optimizer/Builder/IntrinsicCall.cpp @@ -5746,6 +5746,9 @@ if (const IntrinsicHandler *handler = findIntrinsicHandler(name)) if (!handler->argLoweringRules.hasDefaultRules()) return &handler->argLoweringRules; + if (const IntrinsicHandler *ppcHandler = findPPCIntrinsicHandler(name)) + if (!ppcHandler->argLoweringRules.hasDefaultRules()) + return &ppcHandler->argLoweringRules; return nullptr; } Index: flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp =================================================================== --- flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp +++ flang/lib/Optimizer/Builder/PPCIntrinsicCall.cpp @@ -14,8 +14,8 @@ //===----------------------------------------------------------------------===// #include "flang/Optimizer/Builder/PPCIntrinsicCall.h" +#include "flang/Evaluate/common.h" #include "flang/Optimizer/Builder/FIRBuilder.h" -#include "flang/Optimizer/Builder/IntrinsicCall.h" #include "flang/Optimizer/Builder/MutableBox.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" @@ -25,6 +25,40 @@ // PPC specific intrinsic handlers. static constexpr IntrinsicHandler ppcHandlers[]{ + {"__ppc_mma_assemble_acc", + static_cast( + &PI::genMmaIntr), + {{{"acc", asAddr}, + {"arg1", asValue}, + {"arg2", asValue}, + {"arg3", asValue}, + {"arg4", asValue}}}, + /*isElemental=*/true}, + {"__ppc_mma_assemble_pair", + static_cast( + &PI::genMmaIntr), + {{{"pair", asAddr}, {"arg1", asValue}, {"arg2", asValue}}}, + /*isElemental=*/true}, + {"__ppc_mma_build_acc", + static_cast( + &PI::genMmaIntr), + {{{"acc", asAddr}, + {"arg1", asValue}, + {"arg2", asValue}, + {"arg3", asValue}, + {"arg4", asValue}}}, + /*isElemental=*/true}, + {"__ppc_mma_disassemble_acc", + static_cast( + &PI::genMmaIntr), + {{{"data", asAddr}, {"acc", asValue}}}, + /*isElemental=*/true}, + {"__ppc_mma_disassemble_pair", + static_cast( + &PI::genMmaIntr), + {{{"data", asAddr}, {"pair", asValue}}}, + /*isElemental=*/true}, {"__ppc_mtfsf", static_cast(&PI::genMtfsf), {{{"mask", asValue}, {"r", asValue}}}, @@ -326,6 +360,103 @@ return ppcMathOps.equal_range(name); } +static mlir::FunctionType genMmaVpFuncType(mlir::MLIRContext *context, + int quadCnt, int pairCnt, int vecCnt, + int intCnt = 0, + int vecElemBitSize = 8, + int intBitSize = 32) { + // Constructs a function type with the following signature: + // Result type: __vector_pair + // Arguments: + // quadCnt: number of arguments that has __vector_quad type, followed by + // pairCnt: number of arguments that has __vector_pair type, followed by + // vecCnt: number of arguments that has vector(integer) type, followed by + // intCnt: number of arguments that has integer type + // vecElemBitSize: specifies the size of vector elements in bits + // intBitSize: specifies the size of integer arguments in bits + auto vType{mlir::VectorType::get( + 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))}; + auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; + auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; + auto iType{mlir::IntegerType::get(context, intBitSize)}; + llvm::SmallVector argTypes; + for (int i = 0; i < quadCnt; ++i) { + argTypes.push_back(vqType); + } + for (int i = 0; i < pairCnt; ++i) { + argTypes.push_back(vpType); + } + for (int i = 0; i < vecCnt; ++i) { + argTypes.push_back(vType); + } + for (int i = 0; i < intCnt; ++i) { + argTypes.push_back(iType); + } + + return mlir::FunctionType::get(context, argTypes, {vpType}); +} + +static mlir::FunctionType genMmaVqFuncType(mlir::MLIRContext *context, + int quadCnt, int pairCnt, int vecCnt, + int intCnt = 0, + int vecElemBitSize = 8, + int intBitSize = 32) { + // Constructs a function type with the following signature: + // Result type: __vector_quad + // Arguments: + // quadCnt: number of arguments that has __vector_quad type, followed by + // pairCnt: number of arguments that has __vector_pair type, followed by + // vecCnt: number of arguments that has vector(integer) type, followed by + // intCnt: number of arguments that has integer type + // vecElemBitSize: specifies the size of vector elements in bits + // intBitSize: specifies the size of integer arguments in bits + auto vType{mlir::VectorType::get( + 128 / vecElemBitSize, mlir::IntegerType::get(context, vecElemBitSize))}; + auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; + auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; + auto iType{mlir::IntegerType::get(context, intBitSize)}; + llvm::SmallVector argTypes; + for (int i = 0; i < quadCnt; ++i) { + argTypes.push_back(vqType); + } + for (int i = 0; i < pairCnt; ++i) { + argTypes.push_back(vpType); + } + for (int i = 0; i < vecCnt; ++i) { + argTypes.push_back(vType); + } + for (int i = 0; i < intCnt; ++i) { + argTypes.push_back(iType); + } + + return mlir::FunctionType::get(context, argTypes, {vqType}); +} + +mlir::FunctionType genMmaDisassembleFuncType(mlir::MLIRContext *context, + MMAOp mmaOp) { + auto vType{mlir::VectorType::get(16, mlir::IntegerType::get(context, 8))}; + llvm::SmallVector members; + + if (mmaOp == MMAOp::DisassembleAcc) { + auto vqType{fir::VectorType::get(512, mlir::IntegerType::get(context, 1))}; + members.push_back(vType); + members.push_back(vType); + members.push_back(vType); + members.push_back(vType); + auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)}; + return mlir::FunctionType::get(context, {vqType}, {resType}); + } else if (mmaOp == MMAOp::DisassemblePair) { + auto vpType{fir::VectorType::get(256, mlir::IntegerType::get(context, 1))}; + members.push_back(vType); + members.push_back(vType); + auto resType{mlir::LLVM::LLVMStructType::getLiteral(context, members)}; + return mlir::FunctionType::get(context, {vpType}, {resType}); + } else { + llvm_unreachable( + "Unsupported intrinsic code for function signature generator"); + } +} + //===----------------------------------------------------------------------===// // PowerPC specific intrinsic handlers. //===----------------------------------------------------------------------===// @@ -1130,4 +1261,114 @@ return shftRes; } +const char *getMmaIrIntrName(MMAOp mmaOp) { + switch (mmaOp) { + case MMAOp::AssembleAcc: + return "llvm.ppc.mma.assemble.acc"; + case MMAOp::AssemblePair: + return "llvm.ppc.vsx.assemble.pair"; + case MMAOp::DisassembleAcc: + return "llvm.ppc.mma.disassemble.acc"; + case MMAOp::DisassemblePair: + return "llvm.ppc.vsx.disassemble.pair"; + } +} + +mlir::FunctionType getMmaIrFuncType(mlir::MLIRContext *context, MMAOp mmaOp) { + switch (mmaOp) { + case MMAOp::AssembleAcc: + return genMmaVqFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 4); + case MMAOp::AssemblePair: + return genMmaVpFuncType(context, /*Quad*/ 0, /*Pair*/ 0, /*Vector*/ 2); + case MMAOp::DisassembleAcc: + return genMmaDisassembleFuncType(context, mmaOp); + case MMAOp::DisassemblePair: + return genMmaDisassembleFuncType(context, mmaOp); + } +} + +template +void PPCIntrinsicLibrary::genMmaIntr(llvm::ArrayRef args) { + auto context{builder.getContext()}; + mlir::FunctionType intrFuncType{getMmaIrFuncType(context, IntrId)}; + mlir::func::FuncOp funcOp{ + builder.addNamedFunction(loc, getMmaIrIntrName(IntrId), intrFuncType)}; + llvm::SmallVector intrArgs; + + // Depending on SubToFunc, change the subroutine call to a function call. + // First argument represents the result. Rest of the arguments + // are shifted one position to form the actual argument list. + size_t argStart{0}; + size_t argStep{1}; + size_t e{args.size()}; + if (HandlerOp == MMAHandlerOp::SubToFunc) { + // The first argument becomes function result. Start from the second + // argument. + argStart = 1; + } else if (HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE) { + // Reverse argument order on little-endian target only. + // The reversal does not depend on the setting of non-native-order option. + if (Fortran::evaluate::isHostLittleEndian) { + // Load the arguments in reverse order. + argStart = args.size() - 1; + // The first argument becomes function result. Stop at the second + // argument. + e = 0; + argStep = -1; + } else { + // Load the arguments in natural order. + // The first argument becomes function result. Start from the second + // argument. + argStart = 1; + } + } + + for (size_t i = argStart, j = 0; i != e; i += argStep, ++j) { + auto v{fir::getBase(args[i])}; + if (i == 0 && HandlerOp == MMAHandlerOp::FirstArgIsResult) { + // First argument is passed in as an address. We need to load + // the content to match the LLVM interface. + v = builder.create(loc, v); + } + auto vType{v.getType()}; + mlir::Type targetType{intrFuncType.getInput(j)}; + if (vType != targetType) { + if (targetType.isa()) { + // Perform vector type conversion for arguments passed by value. + auto eleTy{vType.dyn_cast().getEleTy()}; + auto len{vType.dyn_cast().getLen()}; + mlir::VectorType mlirType = mlir::VectorType::get(len, eleTy); + auto v0{builder.createConvert(loc, mlirType, v)}; + auto v1{builder.create(loc, targetType, v0)}; + intrArgs.push_back(v1); + } else if (targetType.isa() && + vType.isa()) { + auto v0{builder.createConvert(loc, targetType, v)}; + intrArgs.push_back(v0); + } else { + llvm::errs() << "\nUnexpected type conversion requested: " + << " from " << vType << " to " << targetType << "\n"; + llvm_unreachable("Unsupported type conversion for argument to PowerPC " + "MMA intrinsic"); + } + } else { + intrArgs.push_back(v); + } + } + auto callSt{builder.create(loc, funcOp, intrArgs)}; + if (HandlerOp == MMAHandlerOp::SubToFunc || + HandlerOp == MMAHandlerOp::SubToFuncReverseArgOnLE || + HandlerOp == MMAHandlerOp::FirstArgIsResult) { + // Convert pointer type if needed. + mlir::Value callResult{callSt.getResult(0)}; + mlir::Value destPtr{fir::getBase(args[0])}; + mlir::Type callResultPtrType{builder.getRefType(callResult.getType())}; + if (destPtr.getType() != callResultPtrType) { + destPtr = builder.create(loc, callResultPtrType, destPtr); + } + // Copy the result. + builder.create(loc, callResult, destPtr); + } +} + } // namespace fir Index: flang/lib/Optimizer/Dialect/FIROps.cpp =================================================================== --- flang/lib/Optimizer/Dialect/FIROps.cpp +++ flang/lib/Optimizer/Dialect/FIROps.cpp @@ -947,20 +947,21 @@ } static std::optional getVectorElementType(mlir::Type ty) { - if (mlir::isa(ty)) { - auto elemTy = mlir::dyn_cast(ty).getEleTy(); - - // fir.vector<4:ui32> is converted to mlir.vector<4xi32> - if (elemTy.isUnsignedInteger()) { - elemTy = mlir::IntegerType::get( - ty.getContext(), - mlir::dyn_cast(elemTy).getWidth()); - } - return elemTy; - } else if (mlir::isa(ty)) - return mlir::dyn_cast(ty).getElementType(); + mlir::Type elemTy; + if (mlir::isa(ty)) + elemTy = mlir::dyn_cast(ty).getEleTy(); + else if (mlir::isa(ty)) + elemTy = mlir::dyn_cast(ty).getElementType(); + else + return std::nullopt; - return std::nullopt; + // e.g. fir.vector<4:ui32> => mlir.vector<4xi32> + // e.g. mlir.vector<4xui32> => mlir.vector<4xi32> + if (elemTy.isUnsignedInteger()) { + elemTy = mlir::IntegerType::get( + ty.getContext(), mlir::dyn_cast(elemTy).getWidth()); + } + return elemTy; } static std::optional getVectorLen(mlir::Type ty) { Index: flang/lib/Semantics/check-call.cpp =================================================================== --- flang/lib/Semantics/check-call.cpp +++ flang/lib/Semantics/check-call.cpp @@ -298,7 +298,8 @@ actualFirstSymbol && actualFirstSymbol->attrs().test(Attr::ASYNCHRONOUS)}; bool actualIsVolatile{ actualFirstSymbol && actualFirstSymbol->attrs().test(Attr::VOLATILE)}; - if (const auto *derived{evaluate::GetDerivedTypeSpec(actualType.type())}) { + const auto *derived{evaluate::GetDerivedTypeSpec(actualType.type())}; + if (derived && !derived->IsVectorType()) { if (dummy.type.type().IsAssumedType()) { if (!derived->parameters().empty()) { // 15.5.2.4(2) messages.Say( Index: flang/lib/Semantics/semantics.cpp =================================================================== --- flang/lib/Semantics/semantics.cpp +++ flang/lib/Semantics/semantics.cpp @@ -518,8 +518,11 @@ .statement.v.source == "__ppc_types")) { // Don't try to read the builtins module when we're actually building it. } else if (frontModule && - std::get>(frontModule->value().t) - .statement.v.source == "__ppc_intrinsics") { + (std::get>(frontModule->value().t) + .statement.v.source == "__ppc_intrinsics" || + std::get>( + frontModule->value().t) + .statement.v.source == "mma")) { // The derived type definition for the vectors is needed. context_.UsePPCBuiltinTypesModule(); } else { Index: flang/tools/f18/CMakeLists.txt =================================================================== --- flang/tools/f18/CMakeLists.txt +++ flang/tools/f18/CMakeLists.txt @@ -10,6 +10,7 @@ "__fortran_type_info" "__ppc_types" "__ppc_intrinsics" + "mma" "__cuda_builtins" "ieee_arithmetic" "ieee_exceptions" @@ -32,7 +33,8 @@ set(depends "") elseif(${filename} STREQUAL "__ppc_types") set(depends "") - elseif(${filename} STREQUAL "__ppc_intrinsics") + elseif(${filename} STREQUAL "__ppc_intrinsics" OR + ${filename} STREQUAL "mma") set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__ppc_types.mod) else() set(depends ${FLANG_INTRINSIC_MODULES_DIR}/__fortran_builtins.mod) @@ -47,7 +49,8 @@ # The module contains PPC vector types that needs the PPC target. set(opts "") - if(${filename} STREQUAL "__ppc_intrinsics") + if(${filename} STREQUAL "__ppc_intrinsics" OR + ${filename} STREQUAL "mma") set(opts "--target=ppc64le") endif()