diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -33,42 +33,64 @@ // Atomic Operations //===----------------------------------------------------------------------===// -def AtomicBinOpXchg : I64EnumAttrCase<"xchg", 0>; -def AtomicBinOpAdd : I64EnumAttrCase<"add", 1>; -def AtomicBinOpSub : I64EnumAttrCase<"sub", 2>; -def AtomicBinOpAnd : I64EnumAttrCase<"_and", 3>; -def AtomicBinOpNand : I64EnumAttrCase<"nand", 4>; -def AtomicBinOpOr : I64EnumAttrCase<"_or", 5>; -def AtomicBinOpXor : I64EnumAttrCase<"_xor", 6>; -def AtomicBinOpMax : I64EnumAttrCase<"max", 7>; -def AtomicBinOpMin : I64EnumAttrCase<"min", 8>; -def AtomicBinOpUMax : I64EnumAttrCase<"umax", 9>; -def AtomicBinOpUMin : I64EnumAttrCase<"umin", 10>; -def AtomicBinOpFAdd : I64EnumAttrCase<"fadd", 11>; -def AtomicBinOpFSub : I64EnumAttrCase<"fsub", 12>; -def AtomicBinOp : I64EnumAttr< +def AtomicBinOpXchg : LLVM_EnumAttrCase<"xchg", "xchg", "Xchg", 0>; +def AtomicBinOpAdd : LLVM_EnumAttrCase<"add", "add", "Add", 1>; +def AtomicBinOpSub : LLVM_EnumAttrCase<"sub", "sub", "Sub", 2>; +def AtomicBinOpAnd : LLVM_EnumAttrCase<"_and", "_and", "And", 3>; +def AtomicBinOpNand : LLVM_EnumAttrCase<"nand", "nand", "Nand", 4>; +def AtomicBinOpOr : LLVM_EnumAttrCase<"_or", "_or", "Or", 5>; +def AtomicBinOpXor : LLVM_EnumAttrCase<"_xor", "_xor", "Xor", 6>; +def AtomicBinOpMax : LLVM_EnumAttrCase<"max", "max", "Max", 7>; +def AtomicBinOpMin : LLVM_EnumAttrCase<"min", "min", "Min", 8>; +def AtomicBinOpUMax : LLVM_EnumAttrCase<"umax", "umax", "UMax", 9>; +def AtomicBinOpUMin : LLVM_EnumAttrCase<"umin", "umin", "UMin", 10>; +def AtomicBinOpFAdd : LLVM_EnumAttrCase<"fadd", "fadd", "FAdd", 11>; +def AtomicBinOpFSub : LLVM_EnumAttrCase<"fsub", "fsub", "FSub", 12>; +def AtomicBinOpFMax : LLVM_EnumAttrCase<"fmax", "fmax", "FMax", 13>; +def AtomicBinOpFMin : LLVM_EnumAttrCase<"fmin", "fmin", "FMin", 14>; +def AtomicBinOpUIncWrap : LLVM_EnumAttrCase<"uinc_wrap", + "uinc_wrap", "UIncWrap", 15>; +def AtomicBinOpUDecWrap : LLVM_EnumAttrCase<"udec_wrap", + "udec_wrap", "UDecWrap", 16>; + +// Sentinel case that terminates the LLVM enum. +def AtomicBadBinOp : LLVM_EnumAttrCase<"", "", "BAD_BINOP", 0>; + +def AtomicBinOp : LLVM_EnumAttr< "AtomicBinOp", + "::llvm::AtomicRMWInst::BinOp", "llvm.atomicrmw binary operations", [AtomicBinOpXchg, AtomicBinOpAdd, AtomicBinOpSub, AtomicBinOpAnd, AtomicBinOpNand, AtomicBinOpOr, AtomicBinOpXor, AtomicBinOpMax, AtomicBinOpMin, AtomicBinOpUMax, AtomicBinOpUMin, AtomicBinOpFAdd, - AtomicBinOpFSub]> { + AtomicBinOpFSub, AtomicBinOpFMax, AtomicBinOpFMin, AtomicBinOpUIncWrap, + AtomicBinOpUDecWrap], + [AtomicBadBinOp]> { let cppNamespace = "::mlir::LLVM"; } -def AtomicOrderingNotAtomic : I64EnumAttrCase<"not_atomic", 0>; -def AtomicOrderingUnordered : I64EnumAttrCase<"unordered", 1>; -def AtomicOrderingMonotonic : I64EnumAttrCase<"monotonic", 2>; -def AtomicOrderingAcquire : I64EnumAttrCase<"acquire", 4>; -def AtomicOrderingRelease : I64EnumAttrCase<"release", 5>; -def AtomicOrderingAcquireRelease : I64EnumAttrCase<"acq_rel", 6>; -def AtomicOrderingSequentiallyConsistent : I64EnumAttrCase<"seq_cst", 7>; -def AtomicOrdering : I64EnumAttr< +def AtomicOrderingNotAtomic : LLVM_EnumAttrCase<"not_atomic", + "not_atomic", "NotAtomic", 0>; +def AtomicOrderingUnordered : LLVM_EnumAttrCase<"unordered", + "unordered", "Unordered", 1>; +def AtomicOrderingMonotonic : LLVM_EnumAttrCase<"monotonic", + "monotonic", "Monotonic", 2>; +def AtomicOrderingAcquire : LLVM_EnumAttrCase<"acquire", + "acquire", "Acquire", 4>; +def AtomicOrderingRelease : LLVM_EnumAttrCase<"release", + "release", "Release", 5>; +def AtomicOrderingAcquireRelease : + LLVM_EnumAttrCase<"acq_rel", "acq_rel", "AcquireRelease", 6>; +def AtomicOrderingSequentiallyConsistent : + LLVM_EnumAttrCase<"seq_cst", "seq_cst", "SequentiallyConsistent", 7>; +def AtomicOrdering : LLVM_EnumAttr< "AtomicOrdering", + "::llvm::AtomicOrdering", "Atomic ordering for LLVM's memory model", [AtomicOrderingNotAtomic, AtomicOrderingUnordered, AtomicOrderingMonotonic, AtomicOrderingAcquire, AtomicOrderingRelease, AtomicOrderingAcquireRelease, - AtomicOrderingSequentiallyConsistent]> { + AtomicOrderingSequentiallyConsistent + ]> { let cppNamespace = "::mlir::LLVM"; } @@ -380,59 +402,75 @@ } //===----------------------------------------------------------------------===// -// FCmp Predicates +// FCmp and ICmp Predicates //===----------------------------------------------------------------------===// -// Predicate for float comparisons -def FCmpPredicateFALSE : I64EnumAttrCase<"_false", 0>; -def FCmpPredicateOEQ : I64EnumAttrCase<"oeq", 1>; -def FCmpPredicateOGT : I64EnumAttrCase<"ogt", 2>; -def FCmpPredicateOGE : I64EnumAttrCase<"oge", 3>; -def FCmpPredicateOLT : I64EnumAttrCase<"olt", 4>; -def FCmpPredicateOLE : I64EnumAttrCase<"ole", 5>; -def FCmpPredicateONE : I64EnumAttrCase<"one", 6>; -def FCmpPredicateORD : I64EnumAttrCase<"ord", 7>; -def FCmpPredicateUEQ : I64EnumAttrCase<"ueq", 8>; -def FCmpPredicateUGT : I64EnumAttrCase<"ugt", 9>; -def FCmpPredicateUGE : I64EnumAttrCase<"uge", 10>; -def FCmpPredicateULT : I64EnumAttrCase<"ult", 11>; -def FCmpPredicateULE : I64EnumAttrCase<"ule", 12>; -def FCmpPredicateUNE : I64EnumAttrCase<"une", 13>; -def FCmpPredicateUNO : I64EnumAttrCase<"uno", 14>; -def FCmpPredicateTRUE : I64EnumAttrCase<"_true", 15>; - -def FCmpPredicate : I64EnumAttr< +// Predicates for float comparisons +def FCmpPredicateFALSE : LLVM_EnumAttrCase<"_false", "_false", "FCMP_FALSE", 0>; +def FCmpPredicateOEQ : LLVM_EnumAttrCase<"oeq", "oeq", "FCMP_OEQ", 1>; +def FCmpPredicateOGT : LLVM_EnumAttrCase<"ogt", "ogt", "FCMP_OGT", 2>; +def FCmpPredicateOGE : LLVM_EnumAttrCase<"oge", "oge", "FCMP_OGE", 3>; +def FCmpPredicateOLT : LLVM_EnumAttrCase<"olt", "olt", "FCMP_OLT", 4>; +def FCmpPredicateOLE : LLVM_EnumAttrCase<"ole", "ole", "FCMP_OLE", 5>; +def FCmpPredicateONE : LLVM_EnumAttrCase<"one", "one", "FCMP_ONE", 6>; +def FCmpPredicateORD : LLVM_EnumAttrCase<"ord", "ord", "FCMP_ORD", 7>; +def FCmpPredicateUEQ : LLVM_EnumAttrCase<"ueq", "ueq", "FCMP_UEQ", 8>; +def FCmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "FCMP_UGT", 9>; +def FCmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "FCMP_UGE", 10>; +def FCmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "FCMP_ULT", 11>; +def FCmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "FCMP_ULE", 12>; +def FCmpPredicateUNE : LLVM_EnumAttrCase<"une", "une", "FCMP_UNE", 13>; +def FCmpPredicateUNO : LLVM_EnumAttrCase<"uno", "uno", "FCMP_UNO", 14>; +def FCmpPredicateTRUE : LLVM_EnumAttrCase<"_true", "_true", "FCMP_TRUE", 15>; + +// Predicates for integer comparisons. +def ICmpPredicateEQ : LLVM_EnumAttrCase<"eq", "eq", "ICMP_EQ", 0>; +def ICmpPredicateNE : LLVM_EnumAttrCase<"ne", "ne", "ICMP_NE", 1>; +def ICmpPredicateSLT : LLVM_EnumAttrCase<"slt", "slt", "ICMP_SLT", 2>; +def ICmpPredicateSLE : LLVM_EnumAttrCase<"sle", "sle", "ICMP_SLE", 3>; +def ICmpPredicateSGT : LLVM_EnumAttrCase<"sgt", "sgt", "ICMP_SGT", 4>; +def ICmpPredicateSGE : LLVM_EnumAttrCase<"sge", "sge", "ICMP_SGE", 5>; +def ICmpPredicateULT : LLVM_EnumAttrCase<"ult", "ult", "ICMP_ULT", 6>; +def ICmpPredicateULE : LLVM_EnumAttrCase<"ule", "ule", "ICMP_ULE", 7>; +def ICmpPredicateUGT : LLVM_EnumAttrCase<"ugt", "ugt", "ICMP_UGT", 8>; +def ICmpPredicateUGE : LLVM_EnumAttrCase<"uge", "uge", "ICMP_UGE", 9>; + +// Sentinel cases that terminate the float and integer LLVM enum subranges. +def ICmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_ICMP_PREDICATE", 0>; +def FCmpPredicateBad : LLVM_EnumAttrCase<"", "", "BAD_FCMP_PREDICATE", 0>; + +// LLVM's predicate enum contains the floating-point and integer comparison +// cases, while the LLVM dialect uses two separate enums. The floating-point +// predicate enum thus uses the integer comparison cases as sentinels and +// vice versa. +def FCmpPredicate : LLVM_EnumAttr< "FCmpPredicate", + "::llvm::CmpInst::Predicate", "llvm.fcmp comparison predicate", [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, - FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE], + [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, + ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, + ICmpPredicateUGT, ICmpPredicateUGE, FCmpPredicateBad, ICmpPredicateBad ]> { let cppNamespace = "::mlir::LLVM"; } -//===----------------------------------------------------------------------===// -// ICmp Predicates -//===----------------------------------------------------------------------===// - -// Predicate for integer comparisons. -def ICmpPredicateEQ : I64EnumAttrCase<"eq", 0>; -def ICmpPredicateNE : I64EnumAttrCase<"ne", 1>; -def ICmpPredicateSLT : I64EnumAttrCase<"slt", 2>; -def ICmpPredicateSLE : I64EnumAttrCase<"sle", 3>; -def ICmpPredicateSGT : I64EnumAttrCase<"sgt", 4>; -def ICmpPredicateSGE : I64EnumAttrCase<"sge", 5>; -def ICmpPredicateULT : I64EnumAttrCase<"ult", 6>; -def ICmpPredicateULE : I64EnumAttrCase<"ule", 7>; -def ICmpPredicateUGT : I64EnumAttrCase<"ugt", 8>; -def ICmpPredicateUGE : I64EnumAttrCase<"uge", 9>; -def ICmpPredicate : I64EnumAttr< +def ICmpPredicate : LLVM_EnumAttr< "ICmpPredicate", - "llvm.icmp comparison predicate", + "::llvm::CmpInst::Predicate", + "lvm.icmp comparison predicate", [ICmpPredicateEQ, ICmpPredicateNE, ICmpPredicateSLT, ICmpPredicateSLE, ICmpPredicateSGT, ICmpPredicateSGE, ICmpPredicateULT, ICmpPredicateULE, - ICmpPredicateUGT, ICmpPredicateUGE]> { + ICmpPredicateUGT, ICmpPredicateUGE], + [FCmpPredicateFALSE, FCmpPredicateOEQ, FCmpPredicateOGT, FCmpPredicateOGE, + FCmpPredicateOLT, FCmpPredicateOLE, FCmpPredicateONE, FCmpPredicateORD, + FCmpPredicateUEQ, FCmpPredicateUGT, FCmpPredicateUGE, FCmpPredicateULT, + FCmpPredicateULE, FCmpPredicateUNE, FCmpPredicateUNO, FCmpPredicateTRUE, + FCmpPredicateBad, ICmpPredicateBad + ]> { let cppNamespace = "::mlir::LLVM"; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td @@ -312,14 +312,24 @@ // - `name`: name of the C++ enum class in MLIR API; // - `llvmName`: name of the C++ enum in LLVM API; // - `description`: textual description for documentation purposes; -// - `cases`: list of enum cases. +// - `cases`: list of enum cases; +// - `sentinels`: optional list of enum cases used as sentinels. // For example, `LLVM_EnumAttr cases> : + list cases, + list sentinels = []> : I64EnumAttr { + // List of all cases used as sentinels in LLVM. + list llvmSentinels = sentinels; + // The equivalent enum class name in LLVM. string llvmClassName = llvmName; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -122,12 +122,13 @@ let hasCustomAssemblyFormat = 1; string llvmInstName = "ICmp"; string llvmBuilder = [{ - $res = builder.CreateICmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateICmp( + convertICmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *iCmpInst = cast(inst); - $res = $_builder.create<$_qualCppClassName>( - $_location, getICmpPredicate(iCmpInst->getPredicate()), $lhs, $rhs); + $res = $_builder.create<$_qualCppClassName>($_location, + convertICmpPredicateFromLLVM(iCmpInst->getPredicate()), $lhs, $rhs); }]; // Set the $predicate index to -1 to indicate there is no matching operand // and decrement the following indices. @@ -145,12 +146,12 @@ let hasCustomAssemblyFormat = 1; string llvmInstName = "FCmp"; string llvmBuilder = [{ - $res = builder.CreateFCmp(getLLVMCmpPredicate($predicate), $lhs, $rhs); + $res = builder.CreateFCmp(convertFCmpPredicateToLLVM($predicate), $lhs, $rhs); }]; string mlirBuilder = [{ auto *fCmpInst = cast(inst); auto op = $_builder.create<$_qualCppClassName>( - $_location, getFCmpPredicate(fCmpInst->getPredicate()), $lhs, $rhs); + $_location, convertFCmpPredicateFromLLVM(fCmpInst->getPredicate()), $lhs, $rhs); moduleImport.setFastmathFlagsAttr(inst, op); $res = op; }]; @@ -1670,15 +1671,15 @@ }]; string llvmInstName = "AtomicRMW"; string llvmBuilder = [{ - $res = builder.CreateAtomicRMW(getLLVMAtomicBinOp($bin_op), $ptr, $val, - llvm::MaybeAlign(), - getLLVMAtomicOrdering($ordering)); + $res = builder.CreateAtomicRMW( + convertAtomicBinOpToLLVM($bin_op), $ptr, $val, llvm::MaybeAlign(), + convertAtomicOrderingToLLVM($ordering)); }]; string mlirBuilder = [{ auto *atomicInst = cast(inst); $res = $_builder.create($_location, $_resultType, - getLLVMAtomicBinOp(atomicInst->getOperation()), $ptr, $val, - getLLVMAtomicOrdering(atomicInst->getOrdering())); + convertAtomicBinOpFromLLVM(atomicInst->getOperation()), $ptr, $val, + convertAtomicOrderingFromLLVM(atomicInst->getOrdering())); }]; // Only $ptr and $val are llvm instruction operands. list llvmArgIndices = [-1, 0, 1, -1]; @@ -1706,15 +1707,15 @@ string llvmInstName = "AtomicCmpXchg"; string llvmBuilder = [{ $res = builder.CreateAtomicCmpXchg($ptr, $cmp, $val, llvm::MaybeAlign(), - getLLVMAtomicOrdering($success_ordering), - getLLVMAtomicOrdering($failure_ordering)); + convertAtomicOrderingToLLVM($success_ordering), + convertAtomicOrderingToLLVM($failure_ordering)); }]; string mlirBuilder = [{ auto *cmpXchgInst = cast(inst); $res = $_builder.create( $_location, $_resultType, $ptr, $cmp, $val, - getLLVMAtomicOrdering(cmpXchgInst->getSuccessOrdering()), - getLLVMAtomicOrdering(cmpXchgInst->getFailureOrdering())); + convertAtomicOrderingFromLLVM(cmpXchgInst->getSuccessOrdering()), + convertAtomicOrderingFromLLVM(cmpXchgInst->getFailureOrdering())); }]; } @@ -1724,14 +1725,14 @@ string llvmInstName = "Fence"; let llvmBuilder = [{ llvm::LLVMContext &llvmContext = builder.getContext(); - builder.CreateFence(getLLVMAtomicOrdering($ordering), + builder.CreateFence(convertAtomicOrderingToLLVM($ordering), llvmContext.getOrInsertSyncScopeID($syncscope)); }]; string mlirBuilder = [{ llvm::FenceInst *fenceInst = cast(inst); $_op = $_builder.create( $_location, - getLLVMAtomicOrdering(fenceInst->getOrdering()), + convertAtomicOrderingFromLLVM(fenceInst->getOrdering()), getLLVMSyncScope(fenceInst)); }]; let hasCustomAssemblyFormat = 1; diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -2267,7 +2267,8 @@ if (!ptrType.isOpaque() && valType != ptrType.getElementType()) return emitOpError("expected LLVM IR element type for operand #0 to " "match type for operand #1"); - if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub) { + if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub || + getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) { if (!mlir::LLVM::isCompatibleFloatingPointType(valType)) return emitOpError("expected LLVM IR floating point type"); } else if (getBinOp() == AtomicBinOp::xchg) { diff --git a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp --- a/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.cpp @@ -28,123 +28,6 @@ #include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc" -/// Convert MLIR integer comparison predicate to LLVM IR comparison predicate. -static llvm::CmpInst::Predicate getLLVMCmpPredicate(ICmpPredicate p) { - switch (p) { - case LLVM::ICmpPredicate::eq: - return llvm::CmpInst::Predicate::ICMP_EQ; - case LLVM::ICmpPredicate::ne: - return llvm::CmpInst::Predicate::ICMP_NE; - case LLVM::ICmpPredicate::slt: - return llvm::CmpInst::Predicate::ICMP_SLT; - case LLVM::ICmpPredicate::sle: - return llvm::CmpInst::Predicate::ICMP_SLE; - case LLVM::ICmpPredicate::sgt: - return llvm::CmpInst::Predicate::ICMP_SGT; - case LLVM::ICmpPredicate::sge: - return llvm::CmpInst::Predicate::ICMP_SGE; - case LLVM::ICmpPredicate::ult: - return llvm::CmpInst::Predicate::ICMP_ULT; - case LLVM::ICmpPredicate::ule: - return llvm::CmpInst::Predicate::ICMP_ULE; - case LLVM::ICmpPredicate::ugt: - return llvm::CmpInst::Predicate::ICMP_UGT; - case LLVM::ICmpPredicate::uge: - return llvm::CmpInst::Predicate::ICMP_UGE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::CmpInst::Predicate getLLVMCmpPredicate(FCmpPredicate p) { - switch (p) { - case LLVM::FCmpPredicate::_false: - return llvm::CmpInst::Predicate::FCMP_FALSE; - case LLVM::FCmpPredicate::oeq: - return llvm::CmpInst::Predicate::FCMP_OEQ; - case LLVM::FCmpPredicate::ogt: - return llvm::CmpInst::Predicate::FCMP_OGT; - case LLVM::FCmpPredicate::oge: - return llvm::CmpInst::Predicate::FCMP_OGE; - case LLVM::FCmpPredicate::olt: - return llvm::CmpInst::Predicate::FCMP_OLT; - case LLVM::FCmpPredicate::ole: - return llvm::CmpInst::Predicate::FCMP_OLE; - case LLVM::FCmpPredicate::one: - return llvm::CmpInst::Predicate::FCMP_ONE; - case LLVM::FCmpPredicate::ord: - return llvm::CmpInst::Predicate::FCMP_ORD; - case LLVM::FCmpPredicate::ueq: - return llvm::CmpInst::Predicate::FCMP_UEQ; - case LLVM::FCmpPredicate::ugt: - return llvm::CmpInst::Predicate::FCMP_UGT; - case LLVM::FCmpPredicate::uge: - return llvm::CmpInst::Predicate::FCMP_UGE; - case LLVM::FCmpPredicate::ult: - return llvm::CmpInst::Predicate::FCMP_ULT; - case LLVM::FCmpPredicate::ule: - return llvm::CmpInst::Predicate::FCMP_ULE; - case LLVM::FCmpPredicate::une: - return llvm::CmpInst::Predicate::FCMP_UNE; - case LLVM::FCmpPredicate::uno: - return llvm::CmpInst::Predicate::FCMP_UNO; - case LLVM::FCmpPredicate::_true: - return llvm::CmpInst::Predicate::FCMP_TRUE; - } - llvm_unreachable("incorrect comparison predicate"); -} - -static llvm::AtomicRMWInst::BinOp getLLVMAtomicBinOp(AtomicBinOp op) { - switch (op) { - case LLVM::AtomicBinOp::xchg: - return llvm::AtomicRMWInst::BinOp::Xchg; - case LLVM::AtomicBinOp::add: - return llvm::AtomicRMWInst::BinOp::Add; - case LLVM::AtomicBinOp::sub: - return llvm::AtomicRMWInst::BinOp::Sub; - case LLVM::AtomicBinOp::_and: - return llvm::AtomicRMWInst::BinOp::And; - case LLVM::AtomicBinOp::nand: - return llvm::AtomicRMWInst::BinOp::Nand; - case LLVM::AtomicBinOp::_or: - return llvm::AtomicRMWInst::BinOp::Or; - case LLVM::AtomicBinOp::_xor: - return llvm::AtomicRMWInst::BinOp::Xor; - case LLVM::AtomicBinOp::max: - return llvm::AtomicRMWInst::BinOp::Max; - case LLVM::AtomicBinOp::min: - return llvm::AtomicRMWInst::BinOp::Min; - case LLVM::AtomicBinOp::umax: - return llvm::AtomicRMWInst::BinOp::UMax; - case LLVM::AtomicBinOp::umin: - return llvm::AtomicRMWInst::BinOp::UMin; - case LLVM::AtomicBinOp::fadd: - return llvm::AtomicRMWInst::BinOp::FAdd; - case LLVM::AtomicBinOp::fsub: - return llvm::AtomicRMWInst::BinOp::FSub; - } - llvm_unreachable("incorrect atomic binary operator"); -} - -static llvm::AtomicOrdering getLLVMAtomicOrdering(AtomicOrdering ordering) { - switch (ordering) { - case LLVM::AtomicOrdering::not_atomic: - return llvm::AtomicOrdering::NotAtomic; - case LLVM::AtomicOrdering::unordered: - return llvm::AtomicOrdering::Unordered; - case LLVM::AtomicOrdering::monotonic: - return llvm::AtomicOrdering::Monotonic; - case LLVM::AtomicOrdering::acquire: - return llvm::AtomicOrdering::Acquire; - case LLVM::AtomicOrdering::release: - return llvm::AtomicOrdering::Release; - case LLVM::AtomicOrdering::acq_rel: - return llvm::AtomicOrdering::AcquireRelease; - case LLVM::AtomicOrdering::seq_cst: - return llvm::AtomicOrdering::SequentiallyConsistent; - } - llvm_unreachable("incorrect atomic ordering"); -} - static llvm::FastMathFlags getFastmathFlags(FastmathFlagsInterface &op) { using llvmFMF = llvm::FastMathFlags; using FuncT = void (llvmFMF::*)(bool); diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -97,127 +97,6 @@ } } -static ICmpPredicate getICmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::ICMP_EQ: - return LLVM::ICmpPredicate::eq; - case llvm::CmpInst::Predicate::ICMP_NE: - return LLVM::ICmpPredicate::ne; - case llvm::CmpInst::Predicate::ICMP_SLT: - return LLVM::ICmpPredicate::slt; - case llvm::CmpInst::Predicate::ICMP_SLE: - return LLVM::ICmpPredicate::sle; - case llvm::CmpInst::Predicate::ICMP_SGT: - return LLVM::ICmpPredicate::sgt; - case llvm::CmpInst::Predicate::ICMP_SGE: - return LLVM::ICmpPredicate::sge; - case llvm::CmpInst::Predicate::ICMP_ULT: - return LLVM::ICmpPredicate::ult; - case llvm::CmpInst::Predicate::ICMP_ULE: - return LLVM::ICmpPredicate::ule; - case llvm::CmpInst::Predicate::ICMP_UGT: - return LLVM::ICmpPredicate::ugt; - case llvm::CmpInst::Predicate::ICMP_UGE: - return LLVM::ICmpPredicate::uge; - } - llvm_unreachable("incorrect integer comparison predicate"); -} - -static FCmpPredicate getFCmpPredicate(llvm::CmpInst::Predicate pred) { - switch (pred) { - default: - llvm_unreachable("incorrect comparison predicate"); - case llvm::CmpInst::Predicate::FCMP_FALSE: - return LLVM::FCmpPredicate::_false; - case llvm::CmpInst::Predicate::FCMP_TRUE: - return LLVM::FCmpPredicate::_true; - case llvm::CmpInst::Predicate::FCMP_OEQ: - return LLVM::FCmpPredicate::oeq; - case llvm::CmpInst::Predicate::FCMP_ONE: - return LLVM::FCmpPredicate::one; - case llvm::CmpInst::Predicate::FCMP_OLT: - return LLVM::FCmpPredicate::olt; - case llvm::CmpInst::Predicate::FCMP_OLE: - return LLVM::FCmpPredicate::ole; - case llvm::CmpInst::Predicate::FCMP_OGT: - return LLVM::FCmpPredicate::ogt; - case llvm::CmpInst::Predicate::FCMP_OGE: - return LLVM::FCmpPredicate::oge; - case llvm::CmpInst::Predicate::FCMP_ORD: - return LLVM::FCmpPredicate::ord; - case llvm::CmpInst::Predicate::FCMP_ULT: - return LLVM::FCmpPredicate::ult; - case llvm::CmpInst::Predicate::FCMP_ULE: - return LLVM::FCmpPredicate::ule; - case llvm::CmpInst::Predicate::FCMP_UGT: - return LLVM::FCmpPredicate::ugt; - case llvm::CmpInst::Predicate::FCMP_UGE: - return LLVM::FCmpPredicate::uge; - case llvm::CmpInst::Predicate::FCMP_UNO: - return LLVM::FCmpPredicate::uno; - case llvm::CmpInst::Predicate::FCMP_UEQ: - return LLVM::FCmpPredicate::ueq; - case llvm::CmpInst::Predicate::FCMP_UNE: - return LLVM::FCmpPredicate::une; - } - llvm_unreachable("incorrect floating point comparison predicate"); -} - -static AtomicOrdering getLLVMAtomicOrdering(llvm::AtomicOrdering ordering) { - switch (ordering) { - case llvm::AtomicOrdering::NotAtomic: - return LLVM::AtomicOrdering::not_atomic; - case llvm::AtomicOrdering::Unordered: - return LLVM::AtomicOrdering::unordered; - case llvm::AtomicOrdering::Monotonic: - return LLVM::AtomicOrdering::monotonic; - case llvm::AtomicOrdering::Acquire: - return LLVM::AtomicOrdering::acquire; - case llvm::AtomicOrdering::Release: - return LLVM::AtomicOrdering::release; - case llvm::AtomicOrdering::AcquireRelease: - return LLVM::AtomicOrdering::acq_rel; - case llvm::AtomicOrdering::SequentiallyConsistent: - return LLVM::AtomicOrdering::seq_cst; - } - llvm_unreachable("incorrect atomic ordering"); -} - -static AtomicBinOp getLLVMAtomicBinOp(llvm::AtomicRMWInst::BinOp binOp) { - switch (binOp) { - case llvm::AtomicRMWInst::Xchg: - return LLVM::AtomicBinOp::xchg; - case llvm::AtomicRMWInst::Add: - return LLVM::AtomicBinOp::add; - case llvm::AtomicRMWInst::Sub: - return LLVM::AtomicBinOp::sub; - case llvm::AtomicRMWInst::And: - return LLVM::AtomicBinOp::_and; - case llvm::AtomicRMWInst::Nand: - return LLVM::AtomicBinOp::nand; - case llvm::AtomicRMWInst::Or: - return LLVM::AtomicBinOp::_or; - case llvm::AtomicRMWInst::Xor: - return LLVM::AtomicBinOp::_xor; - case llvm::AtomicRMWInst::Max: - return LLVM::AtomicBinOp::max; - case llvm::AtomicRMWInst::Min: - return LLVM::AtomicBinOp::min; - case llvm::AtomicRMWInst::UMax: - return LLVM::AtomicBinOp::umax; - case llvm::AtomicRMWInst::UMin: - return LLVM::AtomicBinOp::umin; - case llvm::AtomicRMWInst::FAdd: - return LLVM::AtomicBinOp::fadd; - case llvm::AtomicRMWInst::FSub: - return LLVM::AtomicBinOp::fsub; - default: - llvm_unreachable("unsupported atomic binary operation"); - } -} - /// Converts the sync scope identifier of `fenceInst` to the string /// representation necessary to build the LLVM dialect fence operation. static StringRef getLLVMSyncScope(llvm::FenceInst *fenceInst) { diff --git a/mlir/test/Target/LLVMIR/Import/instructions.ll b/mlir/test/Target/LLVMIR/Import/instructions.ll --- a/mlir/test/Target/LLVMIR/Import/instructions.ll +++ b/mlir/test/Target/LLVMIR/Import/instructions.ll @@ -393,6 +393,14 @@ %12 = atomicrmw fadd ptr %ptr2, float %val2 acquire ; CHECK: llvm.atomicrmw fsub %[[PTR2]], %[[VAL2]] acquire %13 = atomicrmw fsub ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmax %[[PTR2]], %[[VAL2]] acquire + %14 = atomicrmw fmax ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw fmin %[[PTR2]], %[[VAL2]] acquire + %15 = atomicrmw fmin ptr %ptr2, float %val2 acquire + ; CHECK: llvm.atomicrmw uinc_wrap %[[PTR1]], %[[VAL1]] acquire + %16 = atomicrmw uinc_wrap ptr %ptr1, i32 %val1 acquire + ; CHECK: llvm.atomicrmw udec_wrap %[[PTR1]], %[[VAL1]] acquire + %17 = atomicrmw udec_wrap ptr %ptr1, i32 %val1 acquire ret void } diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir --- a/mlir/test/Target/LLVMIR/llvmir.mlir +++ b/mlir/test/Target/LLVMIR/llvmir.mlir @@ -1406,28 +1406,36 @@ %0 = llvm.atomicrmw fadd %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw fsub ptr %{{.*}}, float %{{.*}} monotonic %1 = llvm.atomicrmw fsub %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmax ptr %{{.*}}, float %{{.*}} monotonic + %2 = llvm.atomicrmw fmax %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + // CHECK: atomicrmw fmin ptr %{{.*}}, float %{{.*}} monotonic + %3 = llvm.atomicrmw fmin %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw xchg ptr %{{.*}}, float %{{.*}} monotonic - %2 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 + %4 = llvm.atomicrmw xchg %f32_ptr, %f32 monotonic : !llvm.ptr, f32 // CHECK: atomicrmw add ptr %{{.*}}, i32 %{{.*}} acquire - %3 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 + %5 = llvm.atomicrmw add %i32_ptr, %i32 acquire : !llvm.ptr, i32 // CHECK: atomicrmw sub ptr %{{.*}}, i32 %{{.*}} release - %4 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 + %6 = llvm.atomicrmw sub %i32_ptr, %i32 release : !llvm.ptr, i32 // CHECK: atomicrmw and ptr %{{.*}}, i32 %{{.*}} acq_rel - %5 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 + %7 = llvm.atomicrmw _and %i32_ptr, %i32 acq_rel : !llvm.ptr, i32 // CHECK: atomicrmw nand ptr %{{.*}}, i32 %{{.*}} seq_cst - %6 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 + %8 = llvm.atomicrmw nand %i32_ptr, %i32 seq_cst : !llvm.ptr, i32 // CHECK: atomicrmw or ptr %{{.*}}, i32 %{{.*}} monotonic - %7 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %9 = llvm.atomicrmw _or %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw xor ptr %{{.*}}, i32 %{{.*}} monotonic - %8 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %10 = llvm.atomicrmw _xor %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw max ptr %{{.*}}, i32 %{{.*}} monotonic - %9 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %11 = llvm.atomicrmw max %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw min ptr %{{.*}}, i32 %{{.*}} monotonic - %10 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %12 = llvm.atomicrmw min %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umax ptr %{{.*}}, i32 %{{.*}} monotonic - %11 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %13 = llvm.atomicrmw umax %i32_ptr, %i32 monotonic : !llvm.ptr, i32 // CHECK: atomicrmw umin ptr %{{.*}}, i32 %{{.*}} monotonic - %12 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + %14 = llvm.atomicrmw umin %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw uinc_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %15 = llvm.atomicrmw uinc_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 + // CHECK: atomicrmw udec_wrap ptr %{{.*}}, i32 %{{.*}} monotonic + %16 = llvm.atomicrmw udec_wrap %i32_ptr, %i32 monotonic : !llvm.ptr, i32 llvm.return } diff --git a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp --- a/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp +++ b/mlir/tools/mlir-tblgen/LLVMIRConversionGen.cpp @@ -365,6 +365,18 @@ return cases; } + + std::vector getAllSentinels() const { + const auto *inits = def->getValueAsListInit("llvmSentinels"); + + std::vector cases; + cases.reserve(inits->size()); + + for (const llvm::Init *init : *inits) + cases.emplace_back(cast(init)); + + return cases; + } }; // Wraper class around a Tablegen definition of a C-style LLVM enum attribute. @@ -473,6 +485,12 @@ os << formatv(" return {0}::{1}::{2};\n", cppNamespace, cppClassName, cppEnumerant); } + for (const auto &enumerant : enumAttr.getAllSentinels()) { + StringRef llvmEnumerant = enumerant.getLLVMEnumerant(); + os << formatv(" case {0}::{1}:\n", llvmClass, llvmEnumerant); + os << formatv(" llvm_unreachable(\"unhandled sentinel {0}::{1}\");\n", + enumAttr.getLLVMClassName(), llvmEnumerant); + } os << " }\n"; os << formatv(" llvm_unreachable(\"unknown {0} type\");",