diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h --- a/clang/include/clang/Basic/IdentifierTable.h +++ b/clang/include/clang/Basic/IdentifierTable.h @@ -56,7 +56,7 @@ /// of a pointer to one of these classes. enum { IdentifierInfoAlignment = 8 }; -static constexpr int ObjCOrBuiltinIDBits = 16; +static constexpr int ObjCOrBuiltinIDBits = 17; /// One of these records is kept for each identifier that /// is lexed. This contains information about whether the token was \#define'd, diff --git a/clang/include/clang/Basic/riscv_vector.td b/clang/include/clang/Basic/riscv_vector.td --- a/clang/include/clang/Basic/riscv_vector.td +++ b/clang/include/clang/Basic/riscv_vector.td @@ -599,9 +599,13 @@ IntrinsicTypes = {ResultType, Ops[1]->getType()}; Ops[0] = Builder.CreateBitCast(Ops[0], ResultType->getPointerTo()); }], - ManualCodegenMask= [{ + ManualCodegenMask = [{ // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); IntrinsicTypes = {ResultType, Ops[3]->getType()}; Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); }] in { @@ -644,7 +648,11 @@ ManualCodegenMask = [{ { // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 5) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 6) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); IntrinsicTypes = {ResultType, Ops[4]->getType()}; Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); Value *NewVL = Ops[2]; @@ -681,7 +689,11 @@ }], ManualCodegenMask= [{ // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 5) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 6) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); IntrinsicTypes = {ResultType, Ops[4]->getType()}; Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); }] in { @@ -701,7 +713,11 @@ }], ManualCodegenMask = [{ // Move mask to right before vl. - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 5) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 6) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); IntrinsicTypes = {ResultType, Ops[2]->getType(), Ops[4]->getType()}; Ops[1] = Builder.CreateBitCast(Ops[1], ResultType->getPointerTo()); }] in { @@ -885,7 +901,10 @@ Operands.push_back(Ops[2 * NF + 1]); Operands.push_back(Ops[NF]); Operands.push_back(Ops[2 * NF + 2]); - Operands.push_back(Ops[2 * NF + 3]); + if (Ops.size() == 2 * NF + 4) + Operands.push_back(Ops[2 * NF + 3]); + else + Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1)); assert(Operands.size() == NF + 4); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); @@ -958,7 +977,10 @@ Operands.push_back(Ops[2 * NF + 1]); Operands.push_back(Ops[NF]); Operands.push_back(Ops[2 * NF + 3]); - Operands.push_back(Ops[2 * NF + 4]); + if (Ops.size() == 2 * NF + 5) + Operands.push_back(Ops[2 * NF + 4]); + else + Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1)); Value *NewVL = Ops[2 * NF + 2]; assert(Operands.size() == NF + 4); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); @@ -1032,7 +1054,10 @@ Operands.push_back(Ops[2 * NF + 2]); Operands.push_back(Ops[NF]); Operands.push_back(Ops[2 * NF + 3]); - Operands.push_back(Ops[2 * NF + 4]); + if (Ops.size() == 2 * NF + 5) + Operands.push_back(Ops[2 * NF + 4]); + else + Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1)); assert(Operands.size() == NF + 5); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); @@ -1099,7 +1124,10 @@ Operands.push_back(Ops[2 * NF + 2]); Operands.push_back(Ops[NF]); Operands.push_back(Ops[2 * NF + 3]); - Operands.push_back(Ops[2 * NF + 4]); + if (Ops.size() == 2 * NF + 5) + Operands.push_back(Ops[2 * NF + 4]); + else + Operands.push_back(ConstantInt::get(Ops.back()->getType(), 1)); assert(Operands.size() == NF + 5); llvm::Function *F = CGM.getIntrinsic(ID, IntrinsicTypes); llvm::Value *LoadValue = Builder.CreateCall(F, Operands, ""); @@ -1318,7 +1346,11 @@ }], ManualCodegenMask = [{ { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); // maskedoff, op1, mask, vl IntrinsicTypes = {ResultType, cast(ResultType)->getElementType(), @@ -1348,7 +1380,11 @@ }], ManualCodegenMask = [{ { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); // maskedoff, op1, mask, vl IntrinsicTypes = {ResultType, cast(ResultType)->getElementType(), @@ -1395,7 +1431,11 @@ }], ManualCodegenMask = [{ { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); // maskedoff, op1, mask, vl IntrinsicTypes = {ResultType, Ops[1]->getType(), @@ -1427,7 +1467,11 @@ }], ManualCodegenMask = [{ { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); // maskedoff, op1, mask, vl IntrinsicTypes = {ResultType, Ops[1]->getType(), @@ -1462,7 +1506,11 @@ }], ManualCodegenMask = [{ { - std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); + if (Ops.size() == 4) { + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1); + Ops.push_back(ConstantInt::get(Ops.back()->getType(), 1)); + } else if (Ops.size() == 5) + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2); // maskedoff, op1, mask, vl IntrinsicTypes = {ResultType, Ops[1]->getType(), diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c b/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c --- a/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics/vadd-policy.c @@ -42,3 +42,30 @@ vint8m1_t test_vadd_ta(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) { return vadd_vv_i8m1_mt(mask, maskedoff, op1, op2, vl, VE_TAIL_AGNOSTIC); } + +// CHECK-RV64-LABEL: @test_generic_vadd( +// CHECK-RV64-NEXT: entry: +// CHECK-RV64-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vadd.nxv8i8.nxv8i8.i64( [[OP1:%.*]], [[OP2:%.*]], i64 [[VL:%.*]]) +// CHECK-RV64-NEXT: ret [[TMP0]] +// +vint8m1_t test_generic_vadd(vint8m1_t op1, vint8m1_t op2, size_t vl) { + return vadd(op1, op2, vl); +} + +// CHECK-RV64-LABEL: @test_generic_vadd_m( +// CHECK-RV64-NEXT: entry: +// CHECK-RV64-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vadd.mask.nxv8i8.nxv8i8.i64( [[MASKEDOFF:%.*]], [[OP1:%.*]], [[OP2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 1) +// CHECK-RV64-NEXT: ret [[TMP0]] +// +vint8m1_t test_generic_vadd_m(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) { + return vadd(mask, maskedoff, op1, op2, vl); +} + +// CHECK-RV64-LABEL: @test_generic_vadd_mt( +// CHECK-RV64-NEXT: entry: +// CHECK-RV64-NEXT: [[TMP0:%.*]] = call @llvm.riscv.vadd.mask.nxv8i8.nxv8i8.i64( [[MASKEDOFF:%.*]], [[OP1:%.*]], [[OP2:%.*]], [[MASK:%.*]], i64 [[VL:%.*]], i64 0) +// CHECK-RV64-NEXT: ret [[TMP0]] +// +vint8m1_t test_generic_vadd_mt(vbool8_t mask, vint8m1_t maskedoff, vint8m1_t op1, vint8m1_t op2, size_t vl) { + return vadd(mask, maskedoff, op1, op2, vl, VE_TAIL_UNDISTURBED); +} diff --git a/clang/utils/TableGen/RISCVVEmitter.cpp b/clang/utils/TableGen/RISCVVEmitter.cpp --- a/clang/utils/TableGen/RISCVVEmitter.cpp +++ b/clang/utils/TableGen/RISCVVEmitter.cpp @@ -158,6 +158,7 @@ bool HasMaskedOffOperand; bool HasVL; bool HasPolicy; + bool IsPolicyIntrinsic; bool HasNoMaskedOverloaded; bool HasAutoDef; // There is automiatic definition in header std::string ManualCodegen; @@ -173,7 +174,8 @@ RVVIntrinsic(StringRef Name, StringRef Suffix, StringRef MangledName, StringRef MangledSuffix, StringRef IRName, bool HasSideEffects, bool IsMask, bool HasMaskedOffOperand, bool HasVL, - bool HasPolicy, bool HasNoMaskedOverloaded, bool HasAutoDef, + bool HasPolicy, bool IsPolicyIntrinsic, + bool HasNoMaskedOverloaded, bool HasAutoDef, StringRef ManualCodegen, const RVVTypes &Types, const std::vector &IntrinsicTypes, StringRef RequiredExtension, unsigned NF); @@ -185,6 +187,7 @@ bool hasMaskedOffOperand() const { return HasMaskedOffOperand; } bool hasVL() const { return HasVL; } bool hasPolicy() const { return HasPolicy; } + bool isPolicyIntrinsic() const { return IsPolicyIntrinsic; } bool hasNoMaskedOverloaded() const { return HasNoMaskedOverloaded; } bool hasManualCodegen() const { return !ManualCodegen.empty(); } bool hasAutoDef() const { return HasAutoDef; } @@ -201,9 +204,6 @@ // init the RVVIntrinsic ID and IntrinsicTypes. void emitCodeGenSwitchBody(raw_ostream &o) const; - // Emit the define macors for mask intrinsics using _mt intrinsics. - void emitIntrinsicMaskMacro(raw_ostream &o) const; - // Emit the macros for mapping C/C++ intrinsic function to builtin functions. void emitIntrinsicMacro(raw_ostream &o) const; @@ -764,14 +764,16 @@ StringRef NewMangledName, StringRef MangledSuffix, StringRef IRName, bool HasSideEffects, bool IsMask, bool HasMaskedOffOperand, bool HasVL, bool HasPolicy, - bool HasNoMaskedOverloaded, bool HasAutoDef, - StringRef ManualCodegen, const RVVTypes &OutInTypes, + bool IsPolicyIntrinsic, bool HasNoMaskedOverloaded, + bool HasAutoDef, StringRef ManualCodegen, + const RVVTypes &OutInTypes, const std::vector &NewIntrinsicTypes, StringRef RequiredExtension, unsigned NF) : IRName(IRName), HasSideEffects(HasSideEffects), IsMask(IsMask), HasMaskedOffOperand(HasMaskedOffOperand), HasVL(HasVL), - HasPolicy(HasPolicy), HasNoMaskedOverloaded(HasNoMaskedOverloaded), - HasAutoDef(HasAutoDef), ManualCodegen(ManualCodegen.str()), NF(NF) { + HasPolicy(HasPolicy), IsPolicyIntrinsic(IsPolicyIntrinsic), + HasNoMaskedOverloaded(HasNoMaskedOverloaded), HasAutoDef(HasAutoDef), + ManualCodegen(ManualCodegen.str()), NF(NF) { // Init Name and MangledName Name = NewName.str(); @@ -785,7 +787,7 @@ MangledName += "_" + MangledSuffix.str(); if (IsMask) { Name += "_m"; - if (HasPolicy) + if (IsPolicyIntrinsic) Name += "t"; } // Init RISC-V extensions @@ -839,9 +841,15 @@ if (isMask()) { if (hasVL()) { - if (hasPolicy()) - OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);\n"; - else + if (hasPolicy()) { + if (isPolicyIntrinsic()) + OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 2);\n"; + else { + OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);\n"; + OS << " Ops.push_back(ConstantInt::get(Ops.back()->getType(), " + "1));\n"; + } + } else OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end() - 1);\n"; } else { OS << " std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end());\n"; @@ -882,24 +890,6 @@ OS << ")\n"; } -void RVVIntrinsic::emitIntrinsicMaskMacro(raw_ostream &OS) const { - OS << "#define " << getName().drop_back() << "("; - if (!InputTypes.empty()) { - ListSeparator LS; - for (unsigned i = 0, e = InputTypes.size() - 1; i != e; ++i) - OS << LS << "op" << i; - } - OS << ") \\\n"; - OS << "__builtin_rvv_" << getName() << "("; - ListSeparator LS; - if (!InputTypes.empty()) { - for (unsigned i = 0, e = InputTypes.size() - 1; i != e; ++i) - OS << LS << "(" << InputTypes[i]->getTypeStr() << ")(op" << i << ")"; - } - OS << LS << "(size_t)VE_TAIL_AGNOSTIC"; - OS << ")\n"; -} - void RVVIntrinsic::emitMangledFuncDef(raw_ostream &OS) const { OS << "__attribute__((clang_builtin_alias("; OS << "__builtin_rvv_" << getName() << ")))\n"; @@ -1014,12 +1004,6 @@ Inst.emitIntrinsicMacro(OS); }); - // Use _mt to implement _m intrinsics. - emitArchMacroAndBody(Defs, OS, [](raw_ostream &OS, const RVVIntrinsic &Inst) { - if (Inst.isMask() && Inst.hasPolicy()) - Inst.emitIntrinsicMaskMacro(OS); - }); - OS << "#define __riscv_v_intrinsic_overloading 1\n"; // Print Overloaded APIs @@ -1065,7 +1049,9 @@ std::stable_sort(Defs.begin(), Defs.end(), [](const std::unique_ptr &A, const std::unique_ptr &B) { - return A->getIRName() < B->getIRName(); + return A->getIRName() < B->getIRName() || + (A->getIRName() == B->getIRName() && + A->isPolicyIntrinsic() == false); }); // Print switch body when the ir name or ManualCodegen changes from previous // iteration. @@ -1073,7 +1059,8 @@ for (auto &Def : Defs) { StringRef CurIRName = Def->getIRName(); if (CurIRName != PrevDef->getIRName() || - (Def->getManualCodegen() != PrevDef->getManualCodegen())) { + (Def->getManualCodegen() != PrevDef->getManualCodegen()) || + (Def->isPolicyIntrinsic() != PrevDef->isPolicyIntrinsic())) { PrevDef->emitCodeGenSwitchBody(OS); } PrevDef = Def.get(); @@ -1183,8 +1170,9 @@ ProtoMaskSeq.push_back("z"); } + SmallVector ProtoMaskPolicySeq = ProtoMaskSeq; if (HasPolicy) { - ProtoMaskSeq.push_back("z"); + ProtoMaskPolicySeq.push_back("z"); } // Create Intrinsics for each type and LMUL. @@ -1201,8 +1189,9 @@ Out.push_back(std::make_unique( Name, SuffixStr, MangledName, MangledSuffixStr, IRName, HasSideEffects, /*IsMask=*/false, /*HasMaskedOffOperand=*/false, - HasVL, HasPolicy, HasNoMaskedOverloaded, HasAutoDef, ManualCodegen, - Types.getValue(), IntrinsicTypes, RequiredExtension, NF)); + HasVL, HasPolicy, /*IsPolicyIntrinsic*/ false, + HasNoMaskedOverloaded, HasAutoDef, ManualCodegen, Types.getValue(), + IntrinsicTypes, RequiredExtension, NF)); if (HasMask) { // Create a mask intrinsic Optional MaskTypes = @@ -1210,8 +1199,18 @@ Out.push_back(std::make_unique( Name, SuffixStr, MangledName, MangledSuffixStr, IRNameMask, HasSideEffects, /*IsMask=*/true, HasMaskedOffOperand, HasVL, - HasPolicy, HasNoMaskedOverloaded, HasAutoDef, ManualCodegenMask, - MaskTypes.getValue(), IntrinsicTypes, RequiredExtension, NF)); + HasPolicy, /*IsPolicyIntrinsic=*/false, HasNoMaskedOverloaded, + HasAutoDef, ManualCodegenMask, MaskTypes.getValue(), + IntrinsicTypes, RequiredExtension, NF)); + if (HasPolicy) { + MaskTypes = computeTypes(I, Log2LMUL, NF, ProtoMaskPolicySeq); + Out.push_back(std::make_unique( + Name, SuffixStr, MangledName, MangledSuffixStr, IRNameMask, + HasSideEffects, /*IsMask=*/true, HasMaskedOffOperand, HasVL, + HasPolicy, /*IsPolicyIntrinsic=*/true, HasNoMaskedOverloaded, + HasAutoDef, ManualCodegenMask, MaskTypes.getValue(), + IntrinsicTypes, RequiredExtension, NF)); + } } } // end for Log2LMULList } // end for TypeRange