diff --git a/clang/include/clang/Basic/BuiltinsBPF.def b/clang/include/clang/Basic/BuiltinsBPF.def new file mode 100644 --- /dev/null +++ b/clang/include/clang/Basic/BuiltinsBPF.def @@ -0,0 +1,24 @@ +//===--- BuiltinsBPF.def - BPF Builtin function database --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the BPF-specific builtin function database. Users of +// this file must define the BUILTIN macro to make use of this information. +// +//===----------------------------------------------------------------------===// + +// The format of this database matches clang/Basic/Builtins.def. + +#if defined(BUILTIN) && !defined(TARGET_BUILTIN) +# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS) +#endif + +// Get record field information. +TARGET_BUILTIN(__builtin_get_field_info, "Ui.", "t", "") + +#undef BUILTIN +#undef TARGET_BUILTIN diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9947,6 +9947,11 @@ "%select{non-pointer|function pointer|void pointer}0 argument to " "'__builtin_launder' is not allowed">; +def err_get_field_info_not_field : Error< + "__builtin_get_field_info argument %0 not a field access">; +def err_get_field_info_not_const: Error< + "__builtin_get_field_info argument %0 not a constant">; + def err_bit_cast_non_trivially_copyable : Error< "__builtin_bit_cast %select{source|destination}0 type must be trivially copyable">; def err_bit_cast_type_size_mismatch : Error< diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h --- a/clang/include/clang/Basic/TargetBuiltins.h +++ b/clang/include/clang/Basic/TargetBuiltins.h @@ -52,6 +52,16 @@ }; } + /// BPF builtins + namespace BPF { + enum { + LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1, + #define BUILTIN(ID, TYPE, ATTRS) BI##ID, + #include "clang/Basic/BuiltinsBPF.def" + LastTSBuiltin + }; + } + /// PPC builtins namespace PPC { enum { diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -11047,6 +11047,7 @@ bool CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckAArch64BuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckBPFBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap --- a/clang/include/clang/module.modulemap +++ b/clang/include/clang/module.modulemap @@ -35,6 +35,7 @@ textual header "Basic/BuiltinsAArch64.def" textual header "Basic/BuiltinsAMDGPU.def" textual header "Basic/BuiltinsARM.def" + textual header "Basic/BuiltinsBPF.def" textual header "Basic/Builtins.def" textual header "Basic/BuiltinsHexagon.def" textual header "Basic/BuiltinsLe64.def" diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h --- a/clang/lib/Basic/Targets/BPF.h +++ b/clang/lib/Basic/Targets/BPF.h @@ -22,6 +22,8 @@ namespace targets { class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo { + static const Builtin::Info BuiltinInfo[]; + public: BPFTargetInfo(const llvm::Triple &Triple, const TargetOptions &) : TargetInfo(Triple) { @@ -54,7 +56,7 @@ Features[Name] = Enabled; } - ArrayRef getTargetBuiltins() const override { return None; } + ArrayRef getTargetBuiltins() const override; const char *getClobbers() const override { return ""; } diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp --- a/clang/lib/Basic/Targets/BPF.cpp +++ b/clang/lib/Basic/Targets/BPF.cpp @@ -13,11 +13,20 @@ #include "BPF.h" #include "Targets.h" #include "clang/Basic/MacroBuilder.h" +#include "clang/Basic/TargetBuiltins.h" #include "llvm/ADT/StringRef.h" using namespace clang; using namespace clang::targets; +const Builtin::Info BPFTargetInfo::BuiltinInfo[] = { +#define BUILTIN(ID, TYPE, ATTRS) \ + {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr}, +#define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) \ + {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, FEATURE}, +#include "clang/Basic/BuiltinsBPF.def" +}; + void BPFTargetInfo::getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const { Builder.defineMacro("__bpf__"); @@ -34,3 +43,8 @@ void BPFTargetInfo::fillValidCPUList(SmallVectorImpl &Values) const { Values.append(std::begin(ValidCPUNames), std::end(ValidCPUNames)); } + +ArrayRef BPFTargetInfo::getTargetBuiltins() const { + return llvm::makeArrayRef(BuiltinInfo, clang::BPF::LastTSBuiltin - + Builtin::FirstTSBuiltin); +} diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -4242,6 +4242,9 @@ case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch); + case llvm::Triple::bpfeb: + case llvm::Triple::bpfel: + return CGF->EmitBPFBuiltinExpr(BuiltinID, E); case llvm::Triple::x86: case llvm::Triple::x86_64: return CGF->EmitX86BuiltinExpr(BuiltinID, E); @@ -9300,6 +9303,41 @@ } } +Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + assert(BuiltinID == BPF::BI__builtin_get_field_info && + "unexpected ARM builtin"); + + const Expr *Arg = E->getArg(0); + bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField; + bool IsError = false; + + if (!getDebugInfo()) { + CGM.Error(E->getExprLoc(), "using builtin_get_field_info() without -g"); + IsError = true; + } + + if (IsError) + return IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(); + + // Enable underlying preserve_*_access_index() generation. + bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; + IsInPreservedAIRegion = true; + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(); + IsInPreservedAIRegion = OldIsInPreservedAIRegion; + + ConstantInt *C = cast(EmitScalarExpr(E->getArg(1))); + Value *InfoKind = ConstantInt::get(Int32Ty, C->getZExtValue()); + + // Built the IR for the get_field_info intrinsic. + llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::bpf_get_field_info, + {FieldAddr->getType()}); + return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind}); +} + llvm::Value *CodeGenFunction:: BuildVector(ArrayRef Ops) { assert((Ops.size() & (Ops.size() - 1)) == 0 && diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3990,9 +3990,19 @@ const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); Address Addr = base.getAddress(); unsigned Idx = RL.getLLVMFieldNo(field); - if (Idx != 0) - // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + if (!IsInPreservedAIRegion) { + if (Idx != 0) + // For structs, we GEP to the field that the record layout suggests. + Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + } else { + const RecordDecl *rec = field->getParent(); + llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( + getContext().getRecordType(rec), rec->getLocation()); + Addr = Builder.CreatePreserveStructAccessIndex(Addr, Idx, + getDebugInfoFIndex(rec, field->getFieldIndex()), + DbgInfo); + } + // Get the access type. llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize); diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -3760,6 +3760,7 @@ llvm::Value *vectorWrapScalar16(llvm::Value *Op); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch); + llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *BuildVector(ArrayRef Ops); llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -1540,6 +1540,11 @@ if (CheckAArch64BuiltinFunctionCall(BuiltinID, TheCall)) return ExprError(); break; + case llvm::Triple::bpfeb: + case llvm::Triple::bpfel: + if (CheckBPFBuiltinFunctionCall(BuiltinID, TheCall)) + return ExprError(); + break; case llvm::Triple::hexagon: if (CheckHexagonBuiltinFunctionCall(BuiltinID, TheCall)) return ExprError(); @@ -1940,6 +1945,36 @@ return SemaBuiltinConstantArgRange(TheCall, i, l, u + l); } +bool Sema::CheckBPFBuiltinFunctionCall(unsigned BuiltinID, + CallExpr *TheCall) { + assert(BuiltinID == BPF::BI__builtin_get_field_info && + "unexpected ARM builtin"); + + if (checkArgCount(*this, TheCall, 2)) + return true; + + // The first argument needs to be a record field access + Expr *Arg = TheCall->getArg(0); + if (Arg->getType()->getAsPlaceholderType() || + (Arg->IgnoreParens()->getObjectKind() != OK_BitField && + !dyn_cast(Arg->IgnoreParens()))) { + Diag(Arg->getBeginLoc(), diag::err_get_field_info_not_field) + << 1 << Arg->getSourceRange(); + return true; + } + + // The second argument needs to be a constant int + llvm::APSInt Value; + if (!TheCall->getArg(1)->isIntegerConstantExpr(Value, Context)) { + Diag(Arg->getBeginLoc(), diag::err_get_field_info_not_const) + << 2 << Arg->getSourceRange(); + return true; + } + + TheCall->setType(Context.UnsignedIntTy); + return false; +} + bool Sema::CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall) { struct BuiltinAndString { unsigned BuiltinID; diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -20,4 +20,7 @@ Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; def int_bpf_pseudo : GCCBuiltin<"__builtin_bpf_pseudo">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>; + def int_bpf_get_field_info : GCCBuiltin<"__builtin_bpf_get_field_info">, + Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; } diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -50,6 +50,28 @@ // addr = preserve_struct_access_index(base, gep_index, di_index) // !llvm.preserve.access.index // +// Bitfield member access needs special attention. User cannot take the +// address of a bitfield acceess. To facilitate kernel verifier +// for easy bitfield code optimization, a new clang intrinsic is introduced: +// uint32_t __builtin_get_field_info(member_access, info_kind) +// In IR, a chain with two (or more) intrinsic calls will be generated: +// ... +// addr = preserve_struct_access_index(base, 1, 1) !struct s +// uint32_t result = bpf_get_field_info(addr, info_kind) +// +// Suppose the info_kind is BTF_FIELD_SIGNNESS, +// The above two IR intrinsics will be replaced with +// a relocatable insn: +// signness = /* signness of member_access */ +// and signness can be changed by bpf loader based on the +// types on the host. +// +// User can also test whether a field exists or not with +// uint32_t result = bpf_get_field_info(member_access, BTF_FIELD_EXISTENCE) +// The field will be always available (result = 1) during initial +// compilation, but bpf loader can patch with the correct value +// on the target host where the member_access may or may not be available. +// //===----------------------------------------------------------------------===// #include "BPF.h" @@ -95,14 +117,16 @@ BPFPreserveArrayAI = 1, BPFPreserveUnionAI = 2, BPFPreserveStructAI = 3, + BPFGetFieldInfoAI = 4, }; std::map GEPGlobals; // A map to link preserve_*_access_index instrinsic calls. std::map> AIChain; - // A map to hold all the base preserve_*_access_index instrinsic calls. - // The base call is not an input of any other preserve_*_access_index - // intrinsics. + // A map to hold all the base preserve_*_access_index + // and get_field_info instrinsic calls. + // The base call is an input of something else other than + // these intrinsics. std::map BaseAICalls; bool doTransformation(Module &M); @@ -122,10 +146,12 @@ bool removePreserveAccessIndexIntrinsic(Module &M); void replaceWithGEP(std::vector &CallList, uint32_t NumOfZerosIndex, uint32_t DIIndex); + uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, + uint32_t AccessIndex); Value *computeBaseAndAccessKey(CallInst *Call, std::string &AccessKey, uint32_t Kind, MDNode *&BaseMeta); - bool getAccessIndex(const Value *IndexValue, uint64_t &AccessIndex); + uint64_t getConstant(const Value *IndexValue); bool transformGEPChain(Module &M, CallInst *Call, uint32_t Kind); }; } // End anonymous namespace @@ -206,8 +232,7 @@ TypeMeta = Call->getMetadata(LLVMContext::MD_preserve_access_index); if (!TypeMeta) report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic"); - AccessIndex = cast(Call->getArgOperand(2)) - ->getZExtValue(); + AccessIndex = getConstant(Call->getArgOperand(2)); return true; } if (GV->getName().startswith("llvm.preserve.union.access.index")) { @@ -215,8 +240,7 @@ TypeMeta = Call->getMetadata(LLVMContext::MD_preserve_access_index); if (!TypeMeta) report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic"); - AccessIndex = cast(Call->getArgOperand(1)) - ->getZExtValue(); + AccessIndex = getConstant(Call->getArgOperand(1)); return true; } if (GV->getName().startswith("llvm.preserve.struct.access.index")) { @@ -224,8 +248,13 @@ TypeMeta = Call->getMetadata(LLVMContext::MD_preserve_access_index); if (!TypeMeta) report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic"); - AccessIndex = cast(Call->getArgOperand(2)) - ->getZExtValue(); + AccessIndex = getConstant(Call->getArgOperand(2)); + return true; + } + if (GV->getName().startswith("llvm.bpf.get.field.info")) { + Kind = BPFGetFieldInfoAI; + TypeMeta = nullptr; + AccessIndex = getConstant(Call->getArgOperand(1)); return true; } @@ -307,6 +336,9 @@ uint32_t ParentAI, const MDNode *ChildType) { const DIType *PType = stripQualifiers(cast(ParentType)); + if (!ChildType) + return true; // get_field_info, no type comparison needed. + const DIType *CType = stripQualifiers(cast(ChildType)); // Child is a derived/pointer type, which is due to type casting. @@ -465,14 +497,106 @@ } /// Get access index from the preserve_*_access_index intrinsic calls. -bool BPFAbstractMemberAccess::getAccessIndex(const Value *IndexValue, - uint64_t &AccessIndex) { +uint64_t BPFAbstractMemberAccess::getConstant(const Value *IndexValue) { const ConstantInt *CV = dyn_cast(IndexValue); - if (!CV) - return false; + assert(CV); - AccessIndex = CV->getValue().getZExtValue(); - return true; + return CV->getValue().getZExtValue(); +} + +uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, + uint32_t AccessIndex) { + if (InfoKind == BPFCoreSharedInfo::BTF_FIELD_EXISTENCE) + return 1; + + uint32_t Tag = CTy->getTag(); + if (InfoKind == BPFCoreSharedInfo::BTF_FIELD_SIGNNESS) { + assert(Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type); + auto *MemberTy = cast(CTy->getElements()[AccessIndex]); + const DIType *BaseTy = stripQualifiers(MemberTy->getBaseType()); + const auto *BTy = dyn_cast(BaseTy); + while (!BTy) { + const auto *CompTy = dyn_cast(BaseTy); + assert(CompTy); + assert(CompTy->getTag() == dwarf::DW_TAG_enumeration_type); + BaseTy = stripQualifiers(CompTy->getBaseType()); + BTy = dyn_cast(BaseTy); + } + uint32_t Encoding = BTy->getEncoding(); + return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char); + } + + // BTF_FIELD_BF_* + uint32_t PatchImm = 0; + assert(Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type); + auto *MemberTy = cast(CTy->getElements()[AccessIndex]); + assert(MemberTy->isBitField()); + uint64_t OffsetInBits = MemberTy->getOffsetInBits(); + uint64_t SizeInBits = MemberTy->getSizeInBits(); + + // FIXME: we did not handle the case like + // int a; + // int f1:4; + // int f2:7 + // ... + // where a byte storage is not enough, we should + // use 2-bytes. + assert(SizeInBits <= 63); + switch (InfoKind) { + default: + assert("Invalid InfoKind"); + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_BYTE_OFFSET_64BIT_ALIGN: + PatchImm = (OffsetInBits & ~0x3f) >> 3; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_LSHIFT_64BIT_ALIGN: + PatchImm = OffsetInBits & 0x3f; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_RSHIFT_64BIT_ALIGN: + PatchImm = 64 - SizeInBits; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_BYTE_SIZE: + if (SizeInBits <= 8) + PatchImm = 1; + else if (SizeInBits <= 16) + PatchImm = 2; + else if (SizeInBits <= 32) + PatchImm = 4; + else + PatchImm = 8; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_BYTE_OFFSET_SIZE_ALIGN: + if (SizeInBits <= 8) + PatchImm = OffsetInBits >> 3; + else if (SizeInBits <= 16) + PatchImm = (OffsetInBits >> 3) & ~0x1; + else if (SizeInBits <= 32) + PatchImm = (OffsetInBits >> 3) & ~0x3; + else + PatchImm = (OffsetInBits >> 3) & ~0x7; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_LSHIFT_SIZE_ALIGN: + if (SizeInBits <= 8) + PatchImm = OffsetInBits & 0x7; + else if (SizeInBits <= 16) + PatchImm = OffsetInBits & 0xf; + else if (SizeInBits <= 32) + PatchImm = OffsetInBits & 0x1f; + else + PatchImm = OffsetInBits & 0x3f; + break; + case BPFCoreSharedInfo::BTF_FIELD_BF_RSHIFT_SIZE_ALIGN: + if (SizeInBits <= 8) + PatchImm = 8 - SizeInBits; + else if (SizeInBits <= 16) + PatchImm = 16 - SizeInBits; + else if (SizeInBits <= 32) + PatchImm = 32 - SizeInBits; + else + PatchImm = 64 - SizeInBits; + break; + } + return PatchImm; } /// Compute the base of the whole preserve_*_access_index chains, i.e., the base @@ -504,7 +628,8 @@ // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ... // we will skip them. uint32_t FirstIndex = 0; - uint32_t AccessOffset = 0; + uint32_t PatchImm = 0; // AccessOffset or the BitField Info Encoding + uint32_t InfoKind = BPFCoreSharedInfo::BTF_FIELD_ACCESS_OFFSET; while (CallStack.size()) { auto StackElem = CallStack.top(); Call = StackElem.first; @@ -519,17 +644,19 @@ // struct or union type TypeName = Ty->getName(); TypeMeta = Ty; - AccessOffset += FirstIndex * Ty->getSizeInBits() >> 3; + + PatchImm += FirstIndex * Ty->getSizeInBits() >> 3; break; } + assert(Kind == BPFGetFieldInfoAI && + "Reaching get_field_info and cannot find top type name"); + // Array entries will always be consumed for accumulative initial index. CallStack.pop(); // BPFPreserveArrayAI - uint64_t AccessIndex; - if (!getAccessIndex(Call->getArgOperand(2), AccessIndex)) - return nullptr; + uint64_t AccessIndex = getConstant(Call->getArgOperand(2)); DIType *BaseTy = nullptr; bool CheckElemType = false; @@ -569,7 +696,7 @@ else TypeName = CTy->getName(); TypeMeta = CTy; - AccessOffset += FirstIndex * CTy->getSizeInBits() >> 3; + PatchImm += FirstIndex * CTy->getSizeInBits() >> 3; break; } } @@ -584,30 +711,46 @@ Kind = StackElem.second; CallStack.pop(); + if (Kind == BPFGetFieldInfoAI) + break; + + // Find next element if it exists + if (CallStack.size()) { + auto StackElem2 = CallStack.top(); + uint32_t Kind2 = StackElem2.second; + if (Kind2 == BPFGetFieldInfoAI) { + CallInst *Call2 = StackElem2.first; + InfoKind = getConstant(Call2->getArgOperand(1)); + } + } + // Access Index - uint64_t AccessIndex; uint32_t ArgIndex = (Kind == BPFPreserveUnionAI) ? 1 : 2; - if (!getAccessIndex(Call->getArgOperand(ArgIndex), AccessIndex)) - return nullptr; + uint64_t AccessIndex = getConstant(Call->getArgOperand(ArgIndex)); AccessKey += ":" + std::to_string(AccessIndex); MDNode *MDN = Call->getMetadata(LLVMContext::MD_preserve_access_index); // At this stage, it cannot be pointer type. auto *CTy = cast(stripQualifiers(cast(MDN))); uint32_t Tag = CTy->getTag(); - if (Tag == dwarf::DW_TAG_structure_type) { - auto *MemberTy = cast(CTy->getElements()[AccessIndex]); - AccessOffset += MemberTy->getOffsetInBits() >> 3; - } else if (Tag == dwarf::DW_TAG_array_type) { - auto *EltTy = stripQualifiers(CTy->getBaseType()); - AccessOffset += AccessIndex * calcArraySize(CTy, 1) * - EltTy->getSizeInBits() >> 3; + if (InfoKind == BPFCoreSharedInfo::BTF_FIELD_ACCESS_OFFSET) { + if (Tag == dwarf::DW_TAG_structure_type) { + auto *MemberTy = cast(CTy->getElements()[AccessIndex]); + PatchImm += MemberTy->getOffsetInBits() >> 3; + } else if (Tag == dwarf::DW_TAG_array_type) { + auto *EltTy = stripQualifiers(CTy->getBaseType()); + PatchImm += AccessIndex * calcArraySize(CTy, 1) * + EltTy->getSizeInBits() >> 3; + } + } else { + PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex); } } - // Access key is the type name + access string, uniquely identifying + // Access key is the type name + patch immediate string, uniquely identifying // one kernel memory access. - AccessKey = TypeName + ":" + std::to_string(AccessOffset) + "$" + AccessKey; + AccessKey = TypeName + ":" + std::to_string(InfoKind) + ":" + + std::to_string(PatchImm) + "$" + AccessKey; return Base; } @@ -646,25 +789,36 @@ GV = GEPGlobals[AccessKey]; } - // Load the global variable. - auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); - BB->getInstList().insert(Call->getIterator(), LDInst); + if (Kind == BPFGetFieldInfoAI) { + // Load the global variable. + auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); + BB->getInstList().insert(Call->getIterator(), LDInst); + auto *TcInst = new TruncInst(LDInst, Call->getType()); + BB->getInstList().insert(Call->getIterator(), TcInst); - // Generate a BitCast - auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext())); - BB->getInstList().insert(Call->getIterator(), BCInst); + Call->replaceAllUsesWith(TcInst); + Call->eraseFromParent(); + } else { + // Load the global variable. + auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); + BB->getInstList().insert(Call->getIterator(), LDInst); - // Generate a GetElementPtr - auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), - BCInst, LDInst); - BB->getInstList().insert(Call->getIterator(), GEP); + // Generate a BitCast + auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext())); + BB->getInstList().insert(Call->getIterator(), BCInst); - // Generate a BitCast - auto *BCInst2 = new BitCastInst(GEP, Call->getType()); - BB->getInstList().insert(Call->getIterator(), BCInst2); + // Generate a GetElementPtr + auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), + BCInst, LDInst); + BB->getInstList().insert(Call->getIterator(), GEP); - Call->replaceAllUsesWith(BCInst2); - Call->eraseFromParent(); + // Generate a BitCast + auto *BCInst2 = new BitCastInst(GEP, Call->getType()); + BB->getInstList().insert(Call->getIterator(), BCInst2); + + Call->replaceAllUsesWith(BCInst2); + Call->eraseFromParent(); + } return true; } diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -13,6 +13,18 @@ class BPFCoreSharedInfo { public: + enum OffsetRelocKind : uint32_t { + BTF_FIELD_ACCESS_OFFSET = 0, + BTF_FIELD_EXISTENCE, + BTF_FIELD_SIGNNESS, + BTF_FIELD_BF_BYTE_OFFSET_64BIT_ALIGN, + BTF_FIELD_BF_LSHIFT_64BIT_ALIGN, + BTF_FIELD_BF_RSHIFT_64BIT_ALIGN, + BTF_FIELD_BF_BYTE_SIZE, + BTF_FIELD_BF_BYTE_OFFSET_SIZE_ALIGN, + BTF_FIELD_BF_LSHIFT_SIZE_ALIGN, + BTF_FIELD_BF_RSHIFT_SIZE_ALIGN, + }; /// The attribute attached to globals representing a member offset static const std::string AmaAttr; /// The section name to identify a patchable external global diff --git a/llvm/lib/Target/BPF/BTF.h b/llvm/lib/Target/BPF/BTF.h --- a/llvm/lib/Target/BPF/BTF.h +++ b/llvm/lib/Target/BPF/BTF.h @@ -76,7 +76,7 @@ SecExternRelocSize = 8, BPFFuncInfoSize = 8, BPFLineInfoSize = 16, - BPFOffsetRelocSize = 12, + BPFOffsetRelocSize = 16, BPFExternRelocSize = 8, }; @@ -251,6 +251,7 @@ uint32_t InsnOffset; ///< Byte offset in this section uint32_t TypeID; ///< TypeID for the relocation uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t RelocKind; ///< What to patch the instruction }; /// Specifying offset relocation's in one section. diff --git a/llvm/lib/Target/BPF/BTFDebug.h b/llvm/lib/Target/BPF/BTFDebug.h --- a/llvm/lib/Target/BPF/BTFDebug.h +++ b/llvm/lib/Target/BPF/BTFDebug.h @@ -228,6 +228,7 @@ const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc uint32_t TypeID; ///< Type ID uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t RelocKind; ///< What to patch the instruction }; /// Represent one extern relocation. @@ -254,7 +255,7 @@ StringMap> FileContent; std::map> DataSecEntries; std::vector StructTypes; - std::map AccessOffsets; + std::map PatchImms; std::map>> FixupDerivedTypes; diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp --- a/llvm/lib/Target/BPF/BTFDebug.cpp +++ b/llvm/lib/Target/BPF/BTFDebug.cpp @@ -844,6 +844,7 @@ Asm->EmitLabelReference(OffsetRelocInfo.Label, 4); OS.EmitIntValue(OffsetRelocInfo.TypeID, 4); OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4); + OS.EmitIntValue(OffsetRelocInfo.RelocKind, 4); } } } @@ -965,15 +966,19 @@ unsigned RootId = populateStructType(RootTy); size_t FirstDollar = AccessPattern.find_first_of('$'); size_t FirstColon = AccessPattern.find_first_of(':'); + size_t SecondColon = AccessPattern.find_first_of(':', FirstColon + 1); StringRef IndexPattern = AccessPattern.substr(FirstDollar + 1); - StringRef OffsetStr = AccessPattern.substr(FirstColon + 1, - FirstDollar - FirstColon); + StringRef RelocKindStr = AccessPattern.substr(FirstColon + 1, + SecondColon - FirstColon); + StringRef PatchImmStr = AccessPattern.substr(SecondColon + 1, + FirstDollar - SecondColon); BTFOffsetReloc OffsetReloc; OffsetReloc.Label = ORSym; OffsetReloc.OffsetNameOff = addString(IndexPattern); OffsetReloc.TypeID = RootId; - AccessOffsets[AccessPattern.str()] = std::stoi(OffsetStr); + OffsetReloc.RelocKind = std::stoull(RelocKindStr); + PatchImms[AccessPattern.str()] = std::stoull(PatchImmStr); OffsetRelocTable[SecNameOff].push_back(OffsetReloc); } @@ -1154,8 +1159,8 @@ const GlobalValue *GVal = MO.getGlobal(); auto *GVar = dyn_cast(GVal); if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { - // Emit "mov ri, " for abstract member accesses. - int64_t Imm = AccessOffsets[GVar->getName().str()]; + // Emit "mov ri, " for patched immediate. + int64_t Imm = PatchImms[GVar->getName().str()]; OutMI.setOpcode(BPF::MOV_ri); OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); OutMI.addOperand(MCOperand::createImm(Imm));