diff --git a/clang/include/clang/Basic/AArch64SVETypeFlags.h b/clang/include/clang/Basic/AArch64SVETypeFlags.h new file mode 100644 --- /dev/null +++ b/clang/include/clang/Basic/AArch64SVETypeFlags.h @@ -0,0 +1,67 @@ +//===- AArch64SVETypeFlags.h - Flags used to generate ACLE builtins- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_BASIC_AARCH64SVETYPEFLAGS_H +#define LLVM_CLANG_BASIC_AARCH64SVETYPEFLAGS_H + +#include + +namespace clang { + +/// Flags to identify the types for overloaded SVE builtins. +class SVETypeFlags { + uint64_t Flags; + +public: + /// These must be kept in sync with the flags in + /// include/clang/Basic/arm_sve.td. + static const uint64_t MemEltTypeOffset = 4; // Bit offset of MemEltTypeMask + static const uint64_t EltTypeMask = 0x00000000000f; + static const uint64_t MemEltTypeMask = 0x000000000070; + static const uint64_t IsLoad = 0x000000000080; + + enum EltType { + Invalid, + Int8, + Int16, + Int32, + Int64, + Float16, + Float32, + Float64, + Bool8, + Bool16, + Bool32, + Bool64 + }; + + enum MemEltTy { + MemEltTyDefault, + MemEltTyInt8, + MemEltTyInt16, + MemEltTyInt32, + MemEltTyInt64 + }; + + SVETypeFlags(uint64_t F) : Flags(F) { } + SVETypeFlags(EltType ET, bool IsUnsigned) : Flags(ET) { } + + EltType getEltType() const { return (EltType)(Flags & EltTypeMask); } + MemEltTy getMemEltType() const { + return (MemEltTy)((Flags & MemEltTypeMask) >> MemEltTypeOffset); + } + + bool isLoad() const { return Flags & IsLoad; } + + uint64_t getBits() const { return Flags; } + bool isFlagSet(uint64_t Flag) const { return Flags & Flag; } +}; + +} // end namespace clang + +#endif diff --git a/clang/include/clang/Basic/BuiltinsAArch64.def b/clang/include/clang/Basic/BuiltinsAArch64.def --- a/clang/include/clang/Basic/BuiltinsAArch64.def +++ b/clang/include/clang/Basic/BuiltinsAArch64.def @@ -99,19 +99,6 @@ BUILTIN(__builtin_arm_tcancel, "vWUIi", "n") BUILTIN(__builtin_arm_ttest, "WUi", "nc") -// SVE -BUILTIN(__builtin_sve_svld1_s16, "q8sq16bSsC*", "n") -BUILTIN(__builtin_sve_svld1_s32, "q4iq16bSiC*", "n") -BUILTIN(__builtin_sve_svld1_s64, "q2Wiq16bSWiC*", "n") -BUILTIN(__builtin_sve_svld1_s8, "q16Scq16bScC*", "n") -BUILTIN(__builtin_sve_svld1_u16, "q8Usq16bUsC*", "n") -BUILTIN(__builtin_sve_svld1_u32, "q4Uiq16bUiC*", "n") -BUILTIN(__builtin_sve_svld1_u64, "q2UWiq16bUWiC*", "n") -BUILTIN(__builtin_sve_svld1_u8, "q16Ucq16bUcC*", "n") -BUILTIN(__builtin_sve_svld1_f64, "q2dq16bdC*", "n") -BUILTIN(__builtin_sve_svld1_f32, "q4fq16bfC*", "n") -BUILTIN(__builtin_sve_svld1_f16, "q8hq16bhC*", "n") - TARGET_HEADER_BUILTIN(_BitScanForward, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_BitScanReverse, "UcUNi*UNi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") TARGET_HEADER_BUILTIN(_BitScanForward64, "UcUNi*ULLi", "nh", "intrin.h", ALL_MS_LANGUAGES, "") diff --git a/clang/include/clang/Basic/BuiltinsSVE.def b/clang/include/clang/Basic/BuiltinsSVE.def new file mode 100644 --- /dev/null +++ b/clang/include/clang/Basic/BuiltinsSVE.def @@ -0,0 +1,20 @@ +//===--- BuiltinsSVE.def - SVE Builtin function database --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the SVE-specific builtin function database. Users of +// this file must define the BUILTIN macro to make use of this information. +// +//===----------------------------------------------------------------------===// + +// The format of this database matches clang/Basic/Builtins.def. + +#define GET_SVE_BUILTINS +#include "clang/Basic/arm_sve_builtins.inc" +#undef GET_SVE_BUILTINS + +#undef BUILTIN diff --git a/clang/include/clang/Basic/CMakeLists.txt b/clang/include/clang/Basic/CMakeLists.txt --- a/clang/include/clang/Basic/CMakeLists.txt +++ b/clang/include/clang/Basic/CMakeLists.txt @@ -60,7 +60,12 @@ clang_tablegen(arm_mve_builtin_aliases.inc -gen-arm-mve-builtin-aliases SOURCE arm_mve.td TARGET ClangARMMveBuiltinAliases) - +clang_tablegen(arm_sve_builtins.inc -gen-arm-sve-builtins + SOURCE arm_sve.td + TARGET ClangARMSveBuiltins) +clang_tablegen(arm_sve_codegenmap.inc -gen-arm-sve-codegenmap + SOURCE arm_sve.td + TARGET ClangARMSveCodeGenMap) clang_tablegen(arm_cde_builtins.inc -gen-arm-cde-builtin-def SOURCE arm_cde.td TARGET ClangARMCdeBuiltinsDef) diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h --- a/clang/include/clang/Basic/TargetBuiltins.h +++ b/clang/include/clang/Basic/TargetBuiltins.h @@ -41,11 +41,22 @@ }; } + namespace SVE { + enum { + LastNEONBuiltin = NEON::FirstTSBuiltin - 1, +#define BUILTIN(ID, TYPE, ATTRS) BI##ID, +#include "clang/Basic/BuiltinsSVE.def" + FirstTSBuiltin, + }; + } + /// AArch64 builtins namespace AArch64 { enum { LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1, LastNEONBuiltin = NEON::FirstTSBuiltin - 1, + FirstSVEBuiltin = NEON::FirstTSBuiltin, + LastSVEBuiltin = SVE::FirstTSBuiltin - 1, #define BUILTIN(ID, TYPE, ATTRS) BI##ID, #include "clang/Basic/BuiltinsAArch64.def" LastTSBuiltin diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -12,3 +12,110 @@ // https://developer.arm.com/architectures/system-architectures/software-standards/acle // //===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Instruction definitions +//===----------------------------------------------------------------------===// +// Every intrinsic subclasses "Inst". An intrinsic has a name, a prototype and +// a sequence of typespecs. +// +// The name is the base name of the intrinsic, for example "svld1". This is +// then mangled by the tblgen backend to add type information ("svld1_s16"). +// +// A typespec is a sequence of uppercase characters (modifiers) followed by one +// lowercase character. A typespec encodes a particular "base type" of the +// intrinsic. +// +// An example typespec is "Us" - unsigned short - svuint16_t. The available +// typespec codes are given below. +// +// The string given to an Inst class is a sequence of typespecs. The intrinsic +// is instantiated for every typespec in the sequence. For example "sdUsUd". +// +// The prototype is a string that defines the return type of the intrinsic +// and the type of each argument. The return type and every argument gets a +// "modifier" that can change in some way the "base type" of the intrinsic. +// +// The modifier 'd' means "default" and does not modify the base type in any +// way. The available modifiers are given below. +// +// Typespecs +// --------- +// c: char +// s: short +// i: int +// l: long +// f: float +// h: half-float +// d: double + +// Typespec modifiers +// ------------------ +// P: boolean +// U: unsigned + +// Prototype modifiers +// ------------------- +// prototype: return (arg, arg, ...) +// +// d: default +// c: const pointer type +// P: predicate type + +class MergeType { + int Value = val; +} +def MergeNone : MergeType<0>; +def MergeAny : MergeType<1>; +def MergeOp1 : MergeType<2>; +def MergeZero : MergeType<3>; +def MergeAnyExp : MergeType<4>; // Use merged builtin with explicit +def MergeZeroExp : MergeType<5>; // generation of its inactive argument. + +class MemEltTy { + int Value = val; +} +def MemEltTyDefault : MemEltTy<0>; +def MemEltTyInt8 : MemEltTy<1>; +def MemEltTyInt16 : MemEltTy<2>; +def MemEltTyInt32 : MemEltTy<3>; +def MemEltTyInt64 : MemEltTy<4>; + +class FlagType { + int Value = val; +} + +// These must be kept in sync with the flags in utils/TableGen/SveEmitter.h +// and include/clang/Basic/TargetBuiltins.h +def NoFlags : FlagType<0x00000000>; +// 0x00000001 => EltType +// ... +// 0x0000000f => EltType +// 0x00000010 => MemEltType +// ... +// 0x00000070 => MemEltType +def IsLoad : FlagType<0x00000080>; + +// Every intrinsic subclasses Inst. +class Inst ft, MemEltTy met> { + string Name = n; + string Prototype = p; + string Types = t; + string ArchGuard = ""; + int Merge = mt.Value; + string LLVMIntrinsic = i; + list Flags = ft; + int MemEltType = met.Value; +} + +// MInst: Instructions which access memory +class MInst f, + MemEltTy met=MemEltTyDefault, string i=""> + : Inst {} + +//////////////////////////////////////////////////////////////////////////////// +// Loads + +// Load one vector (scalar base) +def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad]>; diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp --- a/clang/lib/Basic/Targets/AArch64.cpp +++ b/clang/lib/Basic/Targets/AArch64.cpp @@ -28,6 +28,10 @@ #define BUILTIN(ID, TYPE, ATTRS) \ {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr}, +#include "clang/Basic/BuiltinsSVE.def" + +#define BUILTIN(ID, TYPE, ATTRS) \ + {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr}, #define LANGBUILTIN(ID, TYPE, ATTRS, LANG) \ {#ID, TYPE, ATTRS, nullptr, LANG, nullptr}, #define TARGET_HEADER_BUILTIN(ID, TYPE, ATTRS, HEADER, LANGS, FEATURE) \ diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -23,6 +23,7 @@ #include "clang/AST/Attr.h" #include "clang/AST/Decl.h" #include "clang/AST/OSLog.h" +#include "clang/Basic/AArch64SVETypeFlags.h" #include "clang/Basic/TargetBuiltins.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/CGFunctionInfo.h" @@ -4576,7 +4577,7 @@ }; namespace { -struct NeonIntrinsicInfo { +struct ARMVectorIntrinsicInfo { const char *NameHint; unsigned BuiltinID; unsigned LLVMIntrinsic; @@ -4586,7 +4587,7 @@ bool operator<(unsigned RHSBuiltinID) const { return BuiltinID < RHSBuiltinID; } - bool operator<(const NeonIntrinsicInfo &TE) const { + bool operator<(const ARMVectorIntrinsicInfo &TE) const { return BuiltinID < TE.BuiltinID; } }; @@ -4604,7 +4605,7 @@ Intrinsic::LLVMIntrinsic, Intrinsic::AltLLVMIntrinsic, \ TypeModifier } -static const NeonIntrinsicInfo ARMSIMDIntrinsicMap [] = { +static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = { NEONMAP2(vabd_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), NEONMAP2(vabdq_v, arm_neon_vabdu, arm_neon_vabds, Add1ArgType | UnsignedAlts), NEONMAP1(vabs_v, arm_neon_vabs, 0), @@ -4885,7 +4886,7 @@ NEONMAP0(vzipq_v) }; -static const NeonIntrinsicInfo AArch64SIMDIntrinsicMap[] = { +static const ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap[] = { NEONMAP1(vabs_v, aarch64_neon_abs, 0), NEONMAP1(vabsq_v, aarch64_neon_abs, 0), NEONMAP0(vaddhn_v), @@ -5054,7 +5055,7 @@ NEONMAP0(vtstq_v), }; -static const NeonIntrinsicInfo AArch64SISDIntrinsicMap[] = { +static const ARMVectorIntrinsicInfo AArch64SISDIntrinsicMap[] = { NEONMAP1(vabdd_f64, aarch64_sisd_fabd, Add1ArgType), NEONMAP1(vabds_f32, aarch64_sisd_fabd, Add1ArgType), NEONMAP1(vabsd_s64, aarch64_neon_abs, Add1ArgType), @@ -5284,15 +5285,32 @@ #undef NEONMAP1 #undef NEONMAP2 +#define SVEMAP1(NameBase, LLVMIntrinsic, TypeModifier) \ + { \ + #NameBase, SVE::BI__builtin_sve_##NameBase, Intrinsic::LLVMIntrinsic, 0, \ + TypeModifier \ + } + +#define SVEMAP2(NameBase, TypeModifier) \ + { #NameBase, SVE::BI__builtin_sve_##NameBase, 0, 0, TypeModifier } +static const ARMVectorIntrinsicInfo AArch64SVEIntrinsicMap[] = { +#define GET_SVE_LLVM_INTRINSIC_MAP +#include "clang/Basic/arm_sve_codegenmap.inc" +#undef GET_SVE_LLVM_INTRINSIC_MAP +}; + +#undef SVEMAP1 +#undef SVEMAP2 + static bool NEONSIMDIntrinsicsProvenSorted = false; static bool AArch64SIMDIntrinsicsProvenSorted = false; static bool AArch64SISDIntrinsicsProvenSorted = false; +static bool AArch64SVEIntrinsicsProvenSorted = false; - -static const NeonIntrinsicInfo * -findNeonIntrinsicInMap(ArrayRef IntrinsicMap, - unsigned BuiltinID, bool &MapProvenSorted) { +static const ARMVectorIntrinsicInfo * +findARMVectorIntrinsicInMap(ArrayRef IntrinsicMap, + unsigned BuiltinID, bool &MapProvenSorted) { #ifndef NDEBUG if (!MapProvenSorted) { @@ -5301,7 +5319,8 @@ } #endif - const NeonIntrinsicInfo *Builtin = llvm::lower_bound(IntrinsicMap, BuiltinID); + const ARMVectorIntrinsicInfo *Builtin = + llvm::lower_bound(IntrinsicMap, BuiltinID); if (Builtin != IntrinsicMap.end() && Builtin->BuiltinID == BuiltinID) return Builtin; @@ -5348,10 +5367,9 @@ return CGM.getIntrinsic(IntrinsicID, Tys); } -static Value *EmitCommonNeonSISDBuiltinExpr(CodeGenFunction &CGF, - const NeonIntrinsicInfo &SISDInfo, - SmallVectorImpl &Ops, - const CallExpr *E) { +static Value *EmitCommonNeonSISDBuiltinExpr( + CodeGenFunction &CGF, const ARMVectorIntrinsicInfo &SISDInfo, + SmallVectorImpl &Ops, const CallExpr *E) { unsigned BuiltinID = SISDInfo.BuiltinID; unsigned int Int = SISDInfo.LLVMIntrinsic; unsigned Modifier = SISDInfo.TypeModifier; @@ -6864,7 +6882,7 @@ // Many NEON builtins have identical semantics and uses in ARM and // AArch64. Emit these in a single function. auto IntrinsicMap = makeArrayRef(ARMSIMDIntrinsicMap); - const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( + const ARMVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap( IntrinsicMap, BuiltinID, NEONSIMDIntrinsicsProvenSorted); if (Builtin) return EmitCommonNeonBuiltinExpr( @@ -7436,9 +7454,40 @@ return Builder.CreateMaskedLoad(BasePtr, Align(1), Predicate, Splat0); } +Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + // Find out if any arguments are required to be integer constant expressions. + unsigned ICEArguments = 0; + ASTContext::GetBuiltinTypeError Error; + getContext().GetBuiltinType(BuiltinID, Error, &ICEArguments); + assert(Error == ASTContext::GE_None && "Should not codegen an error"); + + llvm::SmallVector Ops; + for (unsigned i = 0, e = E->getNumArgs(); i != e; i++) { + if ((ICEArguments & (1 << i)) == 0) + Ops.push_back(EmitScalarExpr(E->getArg(i))); + else + llvm_unreachable("Not yet implemented"); + } + + auto *Builtin = findARMVectorIntrinsicInMap(AArch64SVEIntrinsicMap, BuiltinID, + AArch64SVEIntrinsicsProvenSorted); + SVETypeFlags TypeFlags(Builtin->TypeModifier); + llvm::Type *Ty = ConvertType(E->getType()); + if (TypeFlags.isLoad()) + return EmitSVEMaskedLoad(Ty, Ops); + + /// Should not happen + return nullptr; +} + Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch) { + if (BuiltinID >= AArch64::FirstSVEBuiltin && + BuiltinID <= AArch64::LastSVEBuiltin) + return EmitAArch64SVEBuiltinExpr(BuiltinID, E); + unsigned HintID = static_cast(-1); switch (BuiltinID) { default: break; @@ -7472,27 +7521,6 @@ return Builder.CreateCall(F, llvm::ConstantInt::get(Int32Ty, HintID)); } - switch (BuiltinID) { - case AArch64::BI__builtin_sve_svld1_u8: - case AArch64::BI__builtin_sve_svld1_u16: - case AArch64::BI__builtin_sve_svld1_u32: - case AArch64::BI__builtin_sve_svld1_u64: - case AArch64::BI__builtin_sve_svld1_s8: - case AArch64::BI__builtin_sve_svld1_s16: - case AArch64::BI__builtin_sve_svld1_s32: - case AArch64::BI__builtin_sve_svld1_s64: - case AArch64::BI__builtin_sve_svld1_f16: - case AArch64::BI__builtin_sve_svld1_f32: - case AArch64::BI__builtin_sve_svld1_f64: { - llvm::SmallVector Ops = {EmitScalarExpr(E->getArg(0)), - EmitScalarExpr(E->getArg(1))}; - llvm::Type *Ty = ConvertType(E->getType()); - return EmitSVEMaskedLoad(Ty, Ops); - } - default: - break; - } - if (BuiltinID == AArch64::BI__builtin_arm_prefetch) { Value *Address = EmitScalarExpr(E->getArg(0)); Value *RW = EmitScalarExpr(E->getArg(1)); @@ -7891,7 +7919,7 @@ } auto SISDMap = makeArrayRef(AArch64SISDIntrinsicMap); - const NeonIntrinsicInfo *Builtin = findNeonIntrinsicInMap( + const ARMVectorIntrinsicInfo *Builtin = findARMVectorIntrinsicInMap( SISDMap, BuiltinID, AArch64SISDIntrinsicsProvenSorted); if (Builtin) { @@ -8731,8 +8759,8 @@ // Not all intrinsics handled by the common case work for AArch64 yet, so only // defer to common code if it's been added to our special map. - Builtin = findNeonIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, - AArch64SIMDIntrinsicsProvenSorted); + Builtin = findARMVectorIntrinsicInMap(AArch64SIMDIntrinsicMap, BuiltinID, + AArch64SIMDIntrinsicsProvenSorted); if (Builtin) return EmitCommonNeonBuiltinExpr( diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -3904,6 +3904,7 @@ llvm::Value *EmitSVEPredicateCast(llvm::Value *Pred, llvm::VectorType *VTy); llvm::Value *EmitSVEMaskedLoad(llvm::Type *ReturnTy, SmallVectorImpl &Ops); + llvm::Value *EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch); diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp --- a/clang/utils/TableGen/SveEmitter.cpp +++ b/clang/utils/TableGen/SveEmitter.cpp @@ -29,6 +29,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/TableGen/Record.h" #include "llvm/TableGen/Error.h" +#include "clang/Basic/AArch64SVETypeFlags.h" #include #include #include @@ -36,26 +37,535 @@ using namespace llvm; -//===----------------------------------------------------------------------===// -// SVEEmitter -//===----------------------------------------------------------------------===// +enum ClassKind { + ClassNone, + ClassS, // signed/unsigned, e.g., "_s8", "_u8" suffix + ClassG, // Overloaded name without type suffix +}; + +using TypeSpec = std::string; +using SVETypeFlags = clang::SVETypeFlags; namespace { +class SVEType { + TypeSpec TS; + bool Float, Signed, Immediate, Void, Constant, Pointer; + bool DefaultType, IsScalable, Predicate, PredicatePattern, PrefetchOp; + unsigned Bitwidth, ElementBitwidth, NumVectors; + +public: + SVEType() : SVEType(TypeSpec(), 'v') {} + + SVEType(TypeSpec TS, char CharMod) + : TS(TS), Float(false), Signed(true), Immediate(false), Void(false), + Constant(false), Pointer(false), DefaultType(false), IsScalable(true), + Predicate(false), PredicatePattern(false), PrefetchOp(false), + Bitwidth(128), ElementBitwidth(~0U), NumVectors(1) { + if (!TS.empty()) + applyTypespec(); + applyModifier(CharMod); + } + + /// Return the value in SVETypeFlags for this type. + unsigned getTypeFlags() const; + + bool isPointer() const { return Pointer; } + bool isVoidPointer() const { return Pointer && Void; } + bool isSigned() const { return Signed; } + bool isImmediate() const { return Immediate; } + bool isScalar() const { return NumVectors == 0; } + bool isVector() const { return NumVectors > 0; } + bool isScalableVector() const { return isVector() && IsScalable; } + bool isChar() const { return ElementBitwidth == 8; } + bool isVoid() const { return Void & !Pointer; } + bool isDefault() const { return DefaultType; } + bool isFloat() const { return Float; } + bool isInteger() const { return !Float && !Predicate; } + bool isScalarPredicate() const { return !Float && ElementBitwidth == 1; } + bool isPredicateVector() const { return Predicate; } + bool isPredicatePattern() const { return PredicatePattern; } + bool isPrefetchOp() const { return PrefetchOp; } + bool isConstant() const { return Constant; } + unsigned getElementSizeInBits() const { return ElementBitwidth; } + unsigned getNumVectors() const { return NumVectors; } + + unsigned getNumElements() const { + assert(ElementBitwidth != ~0U); + return Bitwidth / ElementBitwidth; + } + unsigned getSizeInBits() const { + return Bitwidth; + } + + /// Return the string representation of a type, which is an encoded + /// string for passing to the BUILTIN() macro in Builtins.def. + std::string builtin_str() const; + +private: + /// Creates the type based on the typespec string in TS. + void applyTypespec(); + + /// Applies a prototype modifier to the type. + void applyModifier(char Mod); +}; + + +class SVEEmitter; + +/// The main grunt class. This represents an instantiation of an intrinsic with +/// a particular typespec and prototype. +class Intrinsic { + /// The unmangled name. + std::string Name; + + /// The name of the corresponding LLVM IR intrinsic. + std::string LLVMName; + + /// Intrinsic prototype. + std::string Proto; + + /// The base type spec for this intrinsic. + TypeSpec BaseTypeSpec; + + /// The base class kind. Most intrinsics use ClassS, which has full type + /// info for integers (_s32/_u32), or ClassG which is used for overloaded + /// intrinsics. + ClassKind Class; + + /// The architectural #ifdef guard. + std::string Guard; + + /// The types of return value [0] and parameters [1..]. + std::vector Types; + + /// The "base type", which is VarType('d', BaseTypeSpec). + SVEType BaseType; + + /// The type of the memory element + enum MemEltType { + MemEltTypeDefault, + MemEltTypeInt8, + MemEltTypeInt16, + MemEltTypeInt32, + MemEltTypeInt64, + MemEltTypeInvalid + } MemEltTy; + + SVETypeFlags Flags; + +public: + /// The type of predication. + enum MergeType { + MergeNone, + MergeAny, + MergeOp1, + MergeZero, + MergeAnyExp, + MergeZeroExp, + MergeInvalid + } Merge; + + Intrinsic(StringRef Name, StringRef Proto, int64_t MT, int64_t MET, + StringRef LLVMName, SVETypeFlags Flags, TypeSpec BT, ClassKind Class, + SVEEmitter &Emitter, StringRef Guard) + : Name(Name.str()), LLVMName(LLVMName), Proto(Proto.str()), + BaseTypeSpec(BT), Class(Class), Guard(Guard.str()), BaseType(BT, 'd'), + MemEltTy(MemEltType(MET)), Flags(Flags), Merge(MergeType(MT)) { + // Types[0] is the return value. + for (unsigned I = 0; I < Proto.size(); ++I) + Types.emplace_back(BaseTypeSpec, Proto[I]); + } + + ~Intrinsic()=default; + + std::string getName() const { return Name; } + std::string getLLVMName() const { return LLVMName; } + std::string getProto() const { return Proto; } + TypeSpec getBaseTypeSpec() const { return BaseTypeSpec; } + SVEType getBaseType() const { return BaseType; } + + StringRef getGuard() const { return Guard; } + ClassKind getClassKind() const { return Class; } + MergeType getMergeType() const { return Merge; } + + SVEType getReturnType() const { return Types[0]; } + ArrayRef getTypes() const { return Types; } + SVEType getParamType(unsigned I) const { return Types[I + 1]; } + unsigned getNumParams() const { return Proto.size() - 1; } + + SVETypeFlags getFlags() const { return Flags; } + bool isFlagSet(uint64_t Flag) const { return Flags.isFlagSet(Flag);} + + int64_t getMemEltTypeEnum() const { + int64_t METEnum = (MemEltTy << SVETypeFlags::MemEltTypeOffset); + assert((METEnum &~ SVETypeFlags::MemEltTypeMask) == 0 && "Bad MemEltTy"); + return METEnum; + } + + /// Return the type string for a BUILTIN() macro in Builtins.def. + std::string getBuiltinTypeStr(); + + /// Return the name, mangled with type information. The name is mangled for + /// ClassS, so will add type suffixes such as _u32/_s32. + std::string getMangledName() const { return mangleName(ClassS); } + + /// Returns true if the intrinsic is overloaded, in that it should also generate + /// a short form without the type-specifiers, e.g. 'svld1(..)' instead of + /// 'svld1_u32(..)'. + static bool isOverloadedIntrinsic(StringRef Name) { + auto BrOpen = Name.find("["); + auto BrClose = Name.find(']'); + return BrOpen != std::string::npos && BrClose != std::string::npos; + } + + /// Emits the intrinsic declaration to the ostream. + void emitIntrinsic(raw_ostream &OS) const; + +private: + std::string getMergeSuffix() const; + std::string mangleName(ClassKind LocalCK) const; + std::string replaceTemplatedArgs(std::string Name, TypeSpec TS, + std::string Proto) const; +}; + class SVEEmitter { +private: + RecordKeeper &Records; + public: - // run - Emit arm_sve.h - void run(raw_ostream &o); + SVEEmitter(RecordKeeper &R) : Records(R) {} + + /// Emit arm_sve.h. + void createHeader(raw_ostream &o); + + /// Emit all the __builtin prototypes and code needed by Sema. + void createBuiltins(raw_ostream &o); + + /// Emit all the information needed to map builtin -> LLVM IR intrinsic. + void createCodeGenMap(raw_ostream &o); + + /// Create intrinsic and add it to \p Out + void createIntrinsic(Record *R, SmallVectorImpl> &Out); }; } // end anonymous namespace //===----------------------------------------------------------------------===// +// Type implementation +//===----------------------------------------------------------------------===// + +unsigned SVEType::getTypeFlags() const { + if (isFloat()) { + switch (ElementBitwidth) { + case 16: return SVETypeFlags::Float16; + case 32: return SVETypeFlags::Float32; + case 64: return SVETypeFlags::Float64; + default: llvm_unreachable("Unhandled float element bitwidth!"); + } + } + + if (isPredicateVector()) { + switch (ElementBitwidth) { + case 8: return SVETypeFlags::Bool8; + case 16: return SVETypeFlags::Bool16; + case 32: return SVETypeFlags::Bool32; + case 64: return SVETypeFlags::Bool64; + default: llvm_unreachable("Unhandled predicate element bitwidth!"); + } + } + + switch (ElementBitwidth) { + case 8: return SVETypeFlags::Int8; + case 16: return SVETypeFlags::Int16; + case 32: return SVETypeFlags::Int32; + case 64: return SVETypeFlags::Int64; + default: llvm_unreachable("Unhandled integer element bitwidth!"); + } +} + +std::string SVEType::builtin_str() const { + std::string S; + if (isVoid()) + return "v"; + + if (isVoidPointer()) + S += "v"; + else if (!Float) + switch (ElementBitwidth) { + case 1: S += "b"; break; + case 8: S += "c"; break; + case 16: S += "s"; break; + case 32: S += "i"; break; + case 64: S += "Wi"; break; + case 128: S += "LLLi"; break; + default: llvm_unreachable("Unhandled case!"); + } + else + switch (ElementBitwidth) { + case 16: S += "h"; break; + case 32: S += "f"; break; + case 64: S += "d"; break; + default: llvm_unreachable("Unhandled case!"); + } + + if (!isFloat()) { + if ((isChar() || isPointer()) && !isVoidPointer()) { + // Make chars and typed pointers explicitly signed. + if (Signed) + S = "S" + S; + else if (!Signed) + S = "U" + S; + } else if (!isVoidPointer() && !Signed) { + S = "U" + S; + } + } + + // Constant indices are "int", but have the "constant expression" modifier. + if (isImmediate()) { + assert(!isFloat() && "fp immediates are not supported"); + S = "I" + S; + } + + if (isScalar()) { + if (Constant) S += "C"; + if (Pointer) S += "*"; + return S; + } + + assert(isScalableVector() && "Unsupported type"); + return "q" + utostr(getNumElements() * NumVectors) + S; +} + +void SVEType::applyTypespec() { + for (char I : TS) { + switch (I) { + case 'P': + Predicate = true; + ElementBitwidth = 1; + break; + case 'U': + Signed = false; + break; + case 'c': + ElementBitwidth = 8; + break; + case 's': + ElementBitwidth = 16; + break; + case 'i': + ElementBitwidth = 32; + break; + case 'l': + ElementBitwidth = 64; + break; + case 'h': + Float = true; + ElementBitwidth = 16; + break; + case 'f': + Float = true; + ElementBitwidth = 32; + break; + case 'd': + Float = true; + ElementBitwidth = 64; + break; + default: + llvm_unreachable("Unhandled type code!"); + } + } + assert(ElementBitwidth != ~0U && "Bad element bitwidth!"); +} + +void SVEType::applyModifier(char Mod) { + switch (Mod) { + case 'v': + Void = true; + break; + case 'd': + DefaultType = true; + break; + case 'c': + Constant = true; + LLVM_FALLTHROUGH; + case 'p': + Pointer = true; + Bitwidth = ElementBitwidth; + NumVectors = 0; + break; + case 'P': + Signed = true; + Float = false; + Predicate = true; + Bitwidth = 16; + ElementBitwidth = 1; + break; + default: + llvm_unreachable("Unhandled character!"); + } +} + + +//===----------------------------------------------------------------------===// +// Intrinsic implementation +//===----------------------------------------------------------------------===// + +std::string Intrinsic::getBuiltinTypeStr() { + std::string S; + + SVEType RetT = getReturnType(); + // Since the return value must be one type, return a vector type of the + // appropriate width which we will bitcast. An exception is made for + // returning structs of 2, 3, or 4 vectors which are returned in a sret-like + // fashion, storing them to a pointer arg. + if (RetT.getNumVectors() > 1) { + S += "vv*"; // void result with void* first argument + } else + S += RetT.builtin_str(); + + for (unsigned I = 0; I < getNumParams(); ++I) + S += getParamType(I).builtin_str(); + + return S; +} + +std::string Intrinsic::replaceTemplatedArgs(std::string Name, TypeSpec TS, + std::string Proto) const { + std::string Ret = Name; + while (Ret.find('{') != std::string::npos) { + size_t Pos = Ret.find('{'); + size_t End = Ret.find('}'); + unsigned NumChars = End - Pos + 1; + assert(NumChars == 3 && "Unexpected template argument"); + + SVEType T; + char C = Ret[Pos+1]; + switch(C) { + default: + llvm_unreachable("Unknown predication specifier"); + case 'd': + T = SVEType(TS, 'd'); + break; + case '0': + case '1': + case '2': + case '3': + T = SVEType(TS, Proto[C - '0']); + break; + } + + // Replace templated arg with the right suffix (e.g. u32) + std::string TypeCode; + if (T.isInteger()) + TypeCode = T.isSigned() ? 's' : 'u'; + else if (T.isPredicateVector()) + TypeCode = 'b'; + else + TypeCode = 'f'; + Ret.replace(Pos, NumChars, TypeCode + utostr(T.getElementSizeInBits())); + } + + return Ret; +} + +// ACLE function names have a merge style postfix. +std::string Intrinsic::getMergeSuffix() const { + switch (getMergeType()) { + default: + llvm_unreachable("Unknown predication specifier"); + case MergeNone: return ""; + case MergeAny: + case MergeAnyExp: return "_x"; + case MergeOp1: return "_m"; + case MergeZero: + case MergeZeroExp: return "_z"; + } +} + +std::string Intrinsic::mangleName(ClassKind LocalCK) const { + std::string S = getName(); + + if (LocalCK == ClassG) { + // Remove the square brackets and everything in between. + while (S.find("[") != std::string::npos) { + auto Start = S.find("["); + auto End = S.find(']'); + S.erase(Start, (End-Start)+1); + } + } else { + // Remove the square brackets. + while (S.find("[") != std::string::npos) { + auto BrPos = S.find('['); + if (BrPos != std::string::npos) + S.erase(BrPos, 1); + BrPos = S.find(']'); + if (BrPos != std::string::npos) + S.erase(BrPos, 1); + } + } + + // Replace all {d} like expressions with e.g. 'u32' + return replaceTemplatedArgs(S, getBaseTypeSpec(), getProto()) + + getMergeSuffix(); +} + +void Intrinsic::emitIntrinsic(raw_ostream &OS) const { + // Use the preprocessor to enable the non-overloaded builtins. + if (getClassKind() != ClassG || getProto().size() <= 1) { + OS << "#define " << mangleName(getClassKind()) + << "(...) __builtin_sve_" << mangleName(ClassS) + << "(__VA_ARGS__)\n"; + } else { + llvm_unreachable("Not yet implemented. Overloaded intrinsics will follow " + "in a future patch"); + } +} + +//===----------------------------------------------------------------------===// // SVEEmitter implementation //===----------------------------------------------------------------------===// +void SVEEmitter::createIntrinsic( + Record *R, SmallVectorImpl> &Out) { + StringRef Name = R->getValueAsString("Name"); + StringRef Proto = R->getValueAsString("Prototype"); + StringRef Types = R->getValueAsString("Types"); + StringRef Guard = R->getValueAsString("ArchGuard"); + StringRef LLVMName = R->getValueAsString("LLVMIntrinsic"); + int64_t Merge = R->getValueAsInt("Merge"); + std::vector FlagsList = R->getValueAsListOfDefs("Flags"); + int64_t MemEltType = R->getValueAsInt("MemEltType"); + + int64_t Flags = 0; + for (auto FlagRec : FlagsList) + Flags |= FlagRec->getValueAsInt("Value"); + + // Extract type specs from string + SmallVector TypeSpecs; + TypeSpec Acc; + for (char I : Types) { + Acc.push_back(I); + if (islower(I)) { + TypeSpecs.push_back(TypeSpec(Acc)); + Acc.clear(); + } + } + + // Remove duplicate type specs. + std::sort(TypeSpecs.begin(), TypeSpecs.end()); + TypeSpecs.erase(std::unique(TypeSpecs.begin(), TypeSpecs.end()), + TypeSpecs.end()); -void SVEEmitter::run(raw_ostream &OS) { + // Create an Intrinsic for each type spec. + for (auto TS : TypeSpecs) { + Out.push_back(std::make_unique(Name, Proto, Merge, MemEltType, + LLVMName, Flags, TS, ClassS, + *this, Guard)); + } +} + +void SVEEmitter::createHeader(raw_ostream &OS) { OS << "/*===---- arm_sve.h - ARM SVE intrinsics " "-----------------------------------===\n" " *\n" @@ -77,7 +587,9 @@ OS << "#else\n\n"; OS << "#include \n\n"; - OS << "#ifndef __cplusplus\n"; + OS << "#ifdef __cplusplus\n"; + OS << "extern \"C\" {\n"; + OS << "#else\n"; OS << "#include \n"; OS << "#endif\n\n"; @@ -99,25 +611,120 @@ OS << "typedef __SVFloat64_t svfloat64_t;\n"; OS << "typedef __SVBool_t svbool_t;\n\n"; - OS << "#define svld1_u8(...) __builtin_sve_svld1_u8(__VA_ARGS__)\n"; - OS << "#define svld1_u16(...) __builtin_sve_svld1_u16(__VA_ARGS__)\n"; - OS << "#define svld1_u32(...) __builtin_sve_svld1_u32(__VA_ARGS__)\n"; - OS << "#define svld1_u64(...) __builtin_sve_svld1_u64(__VA_ARGS__)\n"; - OS << "#define svld1_s8(...) __builtin_sve_svld1_s8(__VA_ARGS__)\n"; - OS << "#define svld1_s16(...) __builtin_sve_svld1_s16(__VA_ARGS__)\n"; - OS << "#define svld1_s32(...) __builtin_sve_svld1_s32(__VA_ARGS__)\n"; - OS << "#define svld1_s64(...) __builtin_sve_svld1_s64(__VA_ARGS__)\n"; - OS << "#define svld1_f16(...) __builtin_sve_svld1_f16(__VA_ARGS__)\n"; - OS << "#define svld1_f32(...) __builtin_sve_svld1_f32(__VA_ARGS__)\n"; - OS << "#define svld1_f64(...) __builtin_sve_svld1_f64(__VA_ARGS__)\n"; - - OS << "#endif /*__ARM_FEATURE_SVE */\n"; + SmallVector, 128> Defs; + std::vector RV = Records.getAllDerivedDefinitions("Inst"); + for (auto *R : RV) + createIntrinsic(R, Defs); + + // Sort intrinsics in header file by following order/priority: + // - Architectural guard (i.e. does it require SVE2 or SVE2_AES) + // - Class (is intrinsic overloaded or not) + // - Intrinsic name + std::stable_sort( + Defs.begin(), Defs.end(), [](const std::unique_ptr &A, + const std::unique_ptr &B) { + return A->getGuard() < B->getGuard() || + (unsigned)A->getClassKind() < (unsigned)B->getClassKind() || + A->getName() < B->getName(); + }); + + StringRef InGuard = ""; + for (auto &I : Defs) { + // Emit #endif/#if pair if needed. + if (I->getGuard() != InGuard) { + if (!InGuard.empty()) + OS << "#endif //" << InGuard << "\n"; + InGuard = I->getGuard(); + if (!InGuard.empty()) + OS << "\n#if " << InGuard << "\n"; + } + + // Actually emit the intrinsic declaration. + I->emitIntrinsic(OS); + } + + if (!InGuard.empty()) + OS << "#endif //" << InGuard << "\n"; + + OS << "#ifdef __cplusplus\n"; + OS << "} // extern \"C\"\n"; + OS << "#endif\n\n"; + OS << "#endif /*__ARM_FEATURE_SVE */\n\n"; OS << "#endif /* __ARM_SVE_H */\n"; } +void SVEEmitter::createBuiltins(raw_ostream &OS) { + std::vector RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + // The mappings must be sorted based on BuiltinID. + llvm::sort(Defs, [](const std::unique_ptr &A, + const std::unique_ptr &B) { + return A->getMangledName() < B->getMangledName(); + }); + + OS << "#ifdef GET_SVE_BUILTINS\n"; + for (auto &Def : Defs) { + // Only create BUILTINs for non-overloaded intrinsics, as overloaded + // declarations only live in the header file. + if (Def->getClassKind() != ClassG) + OS << "BUILTIN(__builtin_sve_" << Def->getMangledName() << ", \"" + << Def->getBuiltinTypeStr() << "\", \"n\")\n"; + } + OS << "#endif\n\n"; +} + +void SVEEmitter::createCodeGenMap(raw_ostream &OS) { + std::vector RV = Records.getAllDerivedDefinitions("Inst"); + SmallVector, 128> Defs; + for (auto *R : RV) + createIntrinsic(R, Defs); + + // The mappings must be sorted based on BuiltinID. + llvm::sort(Defs, [](const std::unique_ptr &A, + const std::unique_ptr &B) { + return A->getMangledName() < B->getMangledName(); + }); + + OS << "#ifdef GET_SVE_LLVM_INTRINSIC_MAP\n"; + for (auto &Def : Defs) { + // Builtins only exist for non-overloaded intrinsics, overloaded + // declarations only live in the header file. + if (Def->getClassKind() == ClassG) + continue; + + assert(!Def->isFlagSet(SVETypeFlags::EltTypeMask) && + !Def->isFlagSet(SVETypeFlags::MemEltTypeMask) && + "Unexpected mask value"); + uint64_t Flags = Def->getFlags().getBits() | + Def->getBaseType().getTypeFlags() | + Def->getMemEltTypeEnum(); + auto FlagString = std::to_string(Flags); + + std::string LLVMName = Def->getLLVMName(); + std::string Builtin = Def->getMangledName(); + if (!LLVMName.empty()) + OS << "SVEMAP1(" << Builtin << ", " << LLVMName << ", " << FlagString + << "),\n"; + else + OS << "SVEMAP2(" << Builtin << ", " << FlagString << "),\n"; + } + OS << "#endif\n\n"; +} + namespace clang { void EmitSveHeader(RecordKeeper &Records, raw_ostream &OS) { - SVEEmitter().run(OS); + SVEEmitter(Records).createHeader(OS); +} + +void EmitSveBuiltins(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createBuiltins(OS); +} + +void EmitSveCodeGenMap(RecordKeeper &Records, raw_ostream &OS) { + SVEEmitter(Records).createCodeGenMap(OS); } } // End namespace clang diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp --- a/clang/utils/TableGen/TableGen.cpp +++ b/clang/utils/TableGen/TableGen.cpp @@ -71,6 +71,8 @@ GenArmMveBuiltinCG, GenArmMveBuiltinAliases, GenArmSveHeader, + GenArmSveBuiltins, + GenArmSveCodeGenMap, GenArmCdeHeader, GenArmCdeBuiltinDef, GenArmCdeBuiltinSema, @@ -188,6 +190,10 @@ "Generate ARM NEON tests for clang"), clEnumValN(GenArmSveHeader, "gen-arm-sve-header", "Generate arm_sve.h for clang"), + clEnumValN(GenArmSveBuiltins, "gen-arm-sve-builtins", + "Generate arm_sve_builtins.inc for clang"), + clEnumValN(GenArmSveCodeGenMap, "gen-arm-sve-codegenmap", + "Generate arm_sve_codegenmap.inc for clang"), clEnumValN(GenArmMveHeader, "gen-arm-mve-header", "Generate arm_mve.h for clang"), clEnumValN(GenArmMveBuiltinDef, "gen-arm-mve-builtin-def", @@ -372,6 +378,12 @@ case GenArmSveHeader: EmitSveHeader(Records, OS); break; + case GenArmSveBuiltins: + EmitSveBuiltins(Records, OS); + break; + case GenArmSveCodeGenMap: + EmitSveCodeGenMap(Records, OS); + break; case GenArmCdeHeader: EmitCdeHeader(Records, OS); break; diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h --- a/clang/utils/TableGen/TableGenBackends.h +++ b/clang/utils/TableGen/TableGenBackends.h @@ -92,6 +92,8 @@ void EmitNeonTest2(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitSveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSveBuiltins(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); +void EmitSveCodeGenMap(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS); void EmitMveBuiltinDef(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);