Index: clang/include/clang/Basic/TargetBuiltins.h =================================================================== --- clang/include/clang/Basic/TargetBuiltins.h +++ clang/include/clang/Basic/TargetBuiltins.h @@ -240,6 +240,7 @@ bool isOverloadCvt() const { return Flags & IsOverloadCvt; } bool isAppendSVALL() const { return Flags & IsAppendSVALL; } bool isInsertOp1SVALL() const { return Flags & IsInsertOp1SVALL; } + bool isPrefetch() const { return Flags & IsPrefetch; } uint64_t getBits() const { return Flags; } bool isFlagSet(uint64_t Flag) const { return Flags & Flag; } Index: clang/include/clang/Basic/arm_sve.td =================================================================== --- clang/include/clang/Basic/arm_sve.td +++ clang/include/clang/Basic/arm_sve.td @@ -58,6 +58,7 @@ // ------------------- // prototype: return (arg, arg, ...) // +// v: void // x: vector of signed integers // u: vector of unsigned integers // d: default @@ -83,6 +84,7 @@ // M: svfloat32_t // N: svfloat64_t +// J: Prefetch type (sv_prfop) // A: pointer to int8_t // B: pointer to int16_t // C: pointer to int32_t @@ -179,6 +181,7 @@ def IsByteIndexed : FlagType<0x01000000>; def IsAppendSVALL : FlagType<0x02000000>; // Appends SV_ALL as the last operand. def IsInsertOp1SVALL : FlagType<0x04000000>; // Inserts SV_ALL as the second operand. +def IsPrefetch : FlagType<0x08000000>; // Contiguous prefetches. // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h class ImmCheckType { @@ -196,6 +199,7 @@ def ImmCheckLaneIndexDot : ImmCheckType<9>; // 0..(128/(4*sizeinbits(elt)) - 1) def ImmCheckComplexRot90_270 : ImmCheckType<10>; // [90,270] def ImmCheckComplexRotAll90 : ImmCheckType<11>; // [0, 90, 180,270] +def ImmCheck0_13 : ImmCheckType<12>; // 0..13 class ImmCheck { int Arg = arg; @@ -475,6 +479,21 @@ // Store one vector, with no truncation, non-temporal (scalar base, VL displacement) def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore], MemEltTyDefault, "aarch64_sve_stnt1">; +//////////////////////////////////////////////////////////////////////////////// +// Prefetches + +// Prefetch (Scalar base) +def SVPRFB : MInst<"svprfb", "vPcJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH : MInst<"svprfh", "vPcJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW : MInst<"svprfw", "vPcJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD : MInst<"svprfd", "vPcJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; + +// Prefetch (Scalar base, VL displacement) +def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPclJ", "c", [IsPrefetch], MemEltTyInt8, "aarch64_sve_prf">; +def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPclJ", "s", [IsPrefetch], MemEltTyInt16, "aarch64_sve_prf">; +def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPclJ", "i", [IsPrefetch], MemEltTyInt32, "aarch64_sve_prf">; +def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPclJ", "l", [IsPrefetch], MemEltTyInt64, "aarch64_sve_prf">; + //////////////////////////////////////////////////////////////////////////////// // Integer arithmetic Index: clang/lib/CodeGen/CGBuiltin.cpp =================================================================== --- clang/lib/CodeGen/CGBuiltin.cpp +++ clang/lib/CodeGen/CGBuiltin.cpp @@ -7568,6 +7568,13 @@ } } +constexpr unsigned SVEBitsPerBlock = 128; + +static llvm::VectorType* getSVEVectorForElementType(llvm::Type *EltTy) { + unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits(); + return llvm::VectorType::get(EltTy, { NumElts, true }); +} + // Reinterpret the input predicate so that it can be used to correctly isolate // the elements of the specified datatype. Value *CodeGenFunction::EmitSVEPredicateCast(Value *Pred, @@ -7707,6 +7714,26 @@ return Builder.CreateCall(F, Ops); } +Value *CodeGenFunction::EmitSVEPrefetchLoad(SVETypeFlags TypeFlags, + SmallVectorImpl &Ops, + unsigned BuiltinID) { + auto *MemEltTy = SVEBuiltinMemEltTy(TypeFlags); + auto *VectorTy = getSVEVectorForElementType(MemEltTy); + auto *MemoryTy = llvm::VectorType::get(MemEltTy, VectorTy->getElementCount()); + + Value *Predicate = EmitSVEPredicateCast(Ops[0], MemoryTy); + Value *BasePtr = Builder.CreateBitCast(Ops[1], MemoryTy->getPointerTo()); + Value *Offset = Ops.size() > 3 ? Ops[2] : Builder.getInt32(0); + BasePtr = Builder.CreateGEP(MemoryTy, BasePtr, Offset); + + // Prefetch intriniscs always expect an i8* + BasePtr = Builder.CreateBitCast(BasePtr, llvm::PointerType::getUnqual(Int8Ty)); + Value *PrfOp = Ops.back(); + + Function *F = CGM.getIntrinsic(BuiltinID, Predicate->getType()); + return Builder.CreateCall(F, {Predicate, BasePtr, PrfOp}); +} + Value *CodeGenFunction::EmitSVEMaskedLoad(const CallExpr *E, llvm::Type *ReturnTy, SmallVectorImpl &Ops, @@ -7759,13 +7786,6 @@ return Builder.CreateCall(F, {Val, Predicate, BasePtr}); } -constexpr unsigned SVEBitsPerBlock = 128; - -static llvm::VectorType* getSVEVectorForElementType(llvm::Type *EltTy) { - unsigned NumElts = SVEBitsPerBlock / EltTy->getScalarSizeInBits(); - return llvm::VectorType::get(EltTy, { NumElts, true }); -} - // Limit the usage of scalable llvm IR generated by the ACLE by using the // sve dup.x intrinsic instead of IRBuilder::CreateVectorSplat. Value *CodeGenFunction::EmitSVEDupX(Value* Scalar) { @@ -7847,6 +7867,8 @@ return EmitSVEGatherLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); else if (TypeFlags.isScatterStore()) return EmitSVEScatterStore(TypeFlags, Ops, Builtin->LLVMIntrinsic); + else if (TypeFlags.isPrefetch()) + return EmitSVEPrefetchLoad(TypeFlags, Ops, Builtin->LLVMIntrinsic); else if (Builtin->LLVMIntrinsic != 0) { if (TypeFlags.getMergeType() == SVETypeFlags::MergeZeroExp) InsertExplicitZeroOperand(Builder, Ty, Ops); Index: clang/lib/CodeGen/CodeGenFunction.h =================================================================== --- clang/lib/CodeGen/CodeGenFunction.h +++ clang/lib/CodeGen/CodeGenFunction.h @@ -3927,6 +3927,9 @@ llvm::Value *EmitSVEMaskedStore(const CallExpr *, SmallVectorImpl &Ops, unsigned BuiltinID); + llvm::Value *EmitSVEPrefetchLoad(SVETypeFlags TypeFlags, + SmallVectorImpl &Ops, + unsigned BuiltinID); llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, Index: clang/lib/Sema/SemaChecking.cpp =================================================================== --- clang/lib/Sema/SemaChecking.cpp +++ clang/lib/Sema/SemaChecking.cpp @@ -2042,6 +2042,10 @@ if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 31)) HasError = true; break; + case SVETypeFlags::ImmCheck0_13: + if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 0, 13)) + HasError = true; + break; case SVETypeFlags::ImmCheck1_16: if (SemaBuiltinConstantArgRange(TheCall, ArgNum, 1, 16)) HasError = true; Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfb.c @@ -0,0 +1,104 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +void test_svprfb(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 0) + return svprfb(pg, base, SV_PLDL1KEEP); +} + +void test_svprfb_1(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_1 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 1) + return svprfb(pg, base, SV_PLDL1STRM); +} + +void test_svprfb_2(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_2 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 2) + return svprfb(pg, base, SV_PLDL2KEEP); +} + +void test_svprfb_3(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_3 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 3) + return svprfb(pg, base, SV_PLDL2STRM); +} + +void test_svprfb_4(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_4 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 4) + return svprfb(pg, base, SV_PLDL3KEEP); +} + +void test_svprfb_5(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_5 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 5) + return svprfb(pg, base, SV_PLDL3STRM); +} + +void test_svprfb_6(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_6 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 8) + return svprfb(pg, base, SV_PSTL1KEEP); +} + +void test_svprfb_7(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_7 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 9) + return svprfb(pg, base, SV_PSTL1STRM); +} + +void test_svprfb_8(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_8 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 10) + return svprfb(pg, base, SV_PSTL2KEEP); +} + +void test_svprfb_9(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_9 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 11) + return svprfb(pg, base, SV_PSTL2STRM); +} + +void test_svprfb_10(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_10 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 12) + return svprfb(pg, base, SV_PSTL3KEEP); +} + +void test_svprfb_11(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfb_11 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %base, i32 13) + return svprfb(pg, base, SV_PSTL3STRM); +} + +void test_svprfb_vnum(svbool_t pg, const void *base, int64_t vnum) +{ + // CHECK-LABEL: test_svprfb_vnum + // CHECK: %[[BASE:.*]] = bitcast i8* %base to * + // CHECK: %[[GEP:.*]] = getelementptr , * %[[BASE]], i64 %vnum, i64 0 + // CHECK: @llvm.aarch64.sve.prf.nxv16i1( %pg, i8* %[[GEP]], i32 0) + return svprfb_vnum(pg, base, vnum, SV_PLDL1KEEP); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfd.c @@ -0,0 +1,118 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +void test_svprfd(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 0) + return svprfd(pg, base, SV_PLDL1KEEP); +} + +void test_svprfd_1(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_1 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 1) + return svprfd(pg, base, SV_PLDL1STRM); +} + +void test_svprfd_2(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_2 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 2) + return svprfd(pg, base, SV_PLDL2KEEP); +} + +void test_svprfd_3(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_3 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 3) + return svprfd(pg, base, SV_PLDL2STRM); +} + +void test_svprfd_4(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_4 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 4) + return svprfd(pg, base, SV_PLDL3KEEP); +} + +void test_svprfd_5(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_5 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 5) + return svprfd(pg, base, SV_PLDL3STRM); +} + +void test_svprfd_6(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_6 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 8) + return svprfd(pg, base, SV_PSTL1KEEP); +} + +void test_svprfd_7(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_7 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 9) + return svprfd(pg, base, SV_PSTL1STRM); +} + +void test_svprfd_8(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_8 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 10) + return svprfd(pg, base, SV_PSTL2KEEP); +} + +void test_svprfd_9(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_9 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 11) + return svprfd(pg, base, SV_PSTL2STRM); +} + +void test_svprfd_10(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_10 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 12) + return svprfd(pg, base, SV_PSTL3KEEP); +} + +void test_svprfd_11(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfd_11 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %base, i32 13) + return svprfd(pg, base, SV_PSTL3STRM); +} + +void test_svprfd_vnum(svbool_t pg, const void *base, int64_t vnum) +{ + // CHECK-LABEL: test_svprfd_vnum + // CHECK-DAG: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv2i1( %pg) + // CHECK-DAG: %[[BASE:.*]] = bitcast i8* %base to * + // CHECK-DAG: %[[GEP:.*]] = getelementptr , * %[[BASE]], i64 %vnum + // CHECK-DAG: %[[I8_BASE:.*]] = bitcast * %[[GEP]] to i8* + // CHECK: @llvm.aarch64.sve.prf.nxv2i1( %[[PG]], i8* %[[I8_BASE]], i32 0) + return svprfd_vnum(pg, base, vnum, SV_PLDL1KEEP); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfh.c @@ -0,0 +1,118 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +void test_svprfh(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 0) + return svprfh(pg, base, SV_PLDL1KEEP); +} + +void test_svprfh_1(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_1 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 1) + return svprfh(pg, base, SV_PLDL1STRM); +} + +void test_svprfh_2(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_2 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 2) + return svprfh(pg, base, SV_PLDL2KEEP); +} + +void test_svprfh_3(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_3 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 3) + return svprfh(pg, base, SV_PLDL2STRM); +} + +void test_svprfh_4(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_4 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 4) + return svprfh(pg, base, SV_PLDL3KEEP); +} + +void test_svprfh_5(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_5 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 5) + return svprfh(pg, base, SV_PLDL3STRM); +} + +void test_svprfh_6(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_6 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 8) + return svprfh(pg, base, SV_PSTL1KEEP); +} + +void test_svprfh_7(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_7 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 9) + return svprfh(pg, base, SV_PSTL1STRM); +} + +void test_svprfh_8(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_8 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 10) + return svprfh(pg, base, SV_PSTL2KEEP); +} + +void test_svprfh_9(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_9 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 11) + return svprfh(pg, base, SV_PSTL2STRM); +} + +void test_svprfh_10(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_10 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 12) + return svprfh(pg, base, SV_PSTL3KEEP); +} + +void test_svprfh_11(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfh_11 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %base, i32 13) + return svprfh(pg, base, SV_PSTL3STRM); +} + +void test_svprfh_vnum(svbool_t pg, const void *base, int64_t vnum) +{ + // CHECK-LABEL: test_svprfh_vnum + // CHECK-DAG: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv8i1( %pg) + // CHECK-DAG: %[[BASE:.*]] = bitcast i8* %base to * + // CHECK-DAG: %[[GEP:.*]] = getelementptr , * %[[BASE]], i64 %vnum + // CHECK-DAG: %[[I8_BASE:.*]] = bitcast * %[[GEP]] to i8* + // CHECK: @llvm.aarch64.sve.prf.nxv8i1( %[[PG]], i8* %[[I8_BASE]], i32 0) + return svprfh_vnum(pg, base, vnum, SV_PLDL1KEEP); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/acle_sve_prfw.c @@ -0,0 +1,118 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +void test_svprfw(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 0) + return svprfw(pg, base, SV_PLDL1KEEP); +} + +void test_svprfw_1(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_1 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 1) + return svprfw(pg, base, SV_PLDL1STRM); +} + +void test_svprfw_2(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_2 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 2) + return svprfw(pg, base, SV_PLDL2KEEP); +} + +void test_svprfw_3(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_3 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 3) + return svprfw(pg, base, SV_PLDL2STRM); +} + +void test_svprfw_4(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_4 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 4) + return svprfw(pg, base, SV_PLDL3KEEP); +} + +void test_svprfw_5(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_5 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 5) + return svprfw(pg, base, SV_PLDL3STRM); +} + +void test_svprfw_6(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_6 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 8) + return svprfw(pg, base, SV_PSTL1KEEP); +} + +void test_svprfw_7(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_7 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 9) + return svprfw(pg, base, SV_PSTL1STRM); +} + +void test_svprfw_8(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_8 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 10) + return svprfw(pg, base, SV_PSTL2KEEP); +} + +void test_svprfw_9(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_9 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 11) + return svprfw(pg, base, SV_PSTL2STRM); +} + +void test_svprfw_10(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_10 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 12) + return svprfw(pg, base, SV_PSTL3KEEP); +} + +void test_svprfw_11(svbool_t pg, const void *base) +{ + // CHECK-LABEL: test_svprfw_11 + // CHECK: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %base, i32 13) + return svprfw(pg, base, SV_PSTL3STRM); +} + +void test_svprfw_vnum(svbool_t pg, const void *base, int64_t vnum) +{ + // CHECK-LABEL: test_svprfw_vnum + // CHECK-DAG: %[[PG:.*]] = call @llvm.aarch64.sve.convert.from.svbool.nxv4i1( %pg) + // CHECK-DAG: %[[BASE:.*]] = bitcast i8* %base to * + // CHECK-DAG: %[[GEP:.*]] = getelementptr , * %[[BASE]], i64 %vnum + // CHECK-DAG: %[[I8_BASE:.*]] = bitcast * %[[GEP]] to i8* + // CHECK: @llvm.aarch64.sve.prf.nxv4i1( %[[PG]], i8* %[[I8_BASE]], i32 0) + return svprfw_vnum(pg, base, vnum, SV_PLDL1KEEP); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfb.c @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +#include + +void test_svprfb(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value 14 is outside the valid range [0, 13]}} + return svprfb(pg, base, 14); +} + +void test_svprfb_1(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value -1 is outside the valid range [0, 13]}} + return svprfb(pg, base, -1); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfd.c @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +#include + +void test_svprfd(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value 14 is outside the valid range [0, 13]}} + return svprfd(pg, base, 14); +} + +void test_svprfd_1(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value -1 is outside the valid range [0, 13]}} + return svprfd(pg, base, -1); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfh.c @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +#include + +void test_svprfh(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value 14 is outside the valid range [0, 13]}} + return svprfh(pg, base, 14); +} + +void test_svprfh_1(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value -1 is outside the valid range [0, 13]}} + return svprfh(pg, base, -1); +} Index: clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c =================================================================== --- /dev/null +++ clang/test/CodeGen/aarch64-sve-intrinsics/negative/acle_sve_prfw.c @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s +// RUN: %clang_cc1 -D__ARM_FEATURE_SVE -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -fallow-half-arguments-and-returns -fsyntax-only -verify %s + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +#include + +void test_svprfw(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value 14 is outside the valid range [0, 13]}} + return svprfw(pg, base, 14); +} + +void test_svprfw_1(svbool_t pg, const void *base) +{ + // expected-error@+1 {{argument value -1 is outside the valid range [0, 13]}} + return svprfw(pg, base, -1); +} Index: clang/utils/TableGen/SveEmitter.cpp =================================================================== --- clang/utils/TableGen/SveEmitter.cpp +++ clang/utils/TableGen/SveEmitter.cpp @@ -537,6 +537,15 @@ Immediate = true; PredicatePattern = true; break; + case 'J': + Predicate = false; + Float = false; + ElementBitwidth = Bitwidth = 32; + NumVectors = 0; + Signed = true; + Immediate = true; + PrefetchOp = true; + break; case 'k': Predicate = false; Signed = true; @@ -704,6 +713,9 @@ if (T.isPredicatePattern()) ImmChecks.emplace_back( I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_31")); + else if (T.isPrefetchOp()) + ImmChecks.emplace_back( + I - 1, Emitter.getEnumValueForImmCheck("ImmCheck0_13")); } } @@ -1006,6 +1018,22 @@ OS << " SV_ALL = 31\n"; OS << "} sv_pattern;\n\n"; + OS << "typedef enum\n"; + OS << "{\n"; + OS << " SV_PLDL1KEEP = 0,\n"; + OS << " SV_PLDL1STRM = 1,\n"; + OS << " SV_PLDL2KEEP = 2,\n"; + OS << " SV_PLDL2STRM = 3,\n"; + OS << " SV_PLDL3KEEP = 4,\n"; + OS << " SV_PLDL3STRM = 5,\n"; + OS << " SV_PSTL1KEEP = 8,\n"; + OS << " SV_PSTL1STRM = 9,\n"; + OS << " SV_PSTL2KEEP = 10,\n"; + OS << " SV_PSTL2STRM = 11,\n"; + OS << " SV_PSTL3KEEP = 12,\n"; + OS << " SV_PSTL3STRM = 13\n"; + OS << "} sv_prfop;\n\n"; + OS << "/* Function attributes */\n"; OS << "#define __aio static inline __attribute__((__always_inline__, " "__nodebug__, __overloadable__))\n\n";