diff --git a/clang/include/clang/Basic/BuiltinsBPF.def b/clang/include/clang/Basic/BuiltinsBPF.def new file mode 100644 --- /dev/null +++ b/clang/include/clang/Basic/BuiltinsBPF.def @@ -0,0 +1,24 @@ +//===--- BuiltinsBPF.def - BPF Builtin function database --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the BPF-specific builtin function database. Users of +// this file must define the BUILTIN macro to make use of this information. +// +//===----------------------------------------------------------------------===// + +// The format of this database matches clang/Basic/Builtins.def. + +#if defined(BUILTIN) && !defined(TARGET_BUILTIN) +# define TARGET_BUILTIN(ID, TYPE, ATTRS, FEATURE) BUILTIN(ID, TYPE, ATTRS) +#endif + +// Get record field information. +TARGET_BUILTIN(__builtin_preserve_field_info, "Ui.", "t", "") + +#undef BUILTIN +#undef TARGET_BUILTIN diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9950,6 +9950,11 @@ "%select{non-pointer|function pointer|void pointer}0 argument to " "'__builtin_launder' is not allowed">; +def err_preserve_field_info_not_field : Error< + "__builtin_preserve_field_info argument %0 not a field access">; +def err_preserve_field_info_not_const: Error< + "__builtin_preserve_field_info argument %0 not a constant">; + def err_bit_cast_non_trivially_copyable : Error< "__builtin_bit_cast %select{source|destination}0 type must be trivially copyable">; def err_bit_cast_type_size_mismatch : Error< diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h --- a/clang/include/clang/Basic/TargetBuiltins.h +++ b/clang/include/clang/Basic/TargetBuiltins.h @@ -52,6 +52,16 @@ }; } + /// BPF builtins + namespace BPF { + enum { + LastTIBuiltin = clang::Builtin::FirstTSBuiltin - 1, + #define BUILTIN(ID, TYPE, ATTRS) BI##ID, + #include "clang/Basic/BuiltinsBPF.def" + LastTSBuiltin + }; + } + /// PPC builtins namespace PPC { enum { diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -11050,6 +11050,7 @@ bool CheckARMBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckAArch64BuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); + bool CheckBPFBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall); bool CheckHexagonBuiltinArgument(unsigned BuiltinID, CallExpr *TheCall); diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap --- a/clang/include/clang/module.modulemap +++ b/clang/include/clang/module.modulemap @@ -34,6 +34,7 @@ textual header "Basic/BuiltinsAArch64.def" textual header "Basic/BuiltinsAMDGPU.def" textual header "Basic/BuiltinsARM.def" + textual header "Basic/BuiltinsBPF.def" textual header "Basic/Builtins.def" textual header "Basic/BuiltinsHexagon.def" textual header "Basic/BuiltinsLe64.def" diff --git a/clang/lib/Basic/Targets/BPF.h b/clang/lib/Basic/Targets/BPF.h --- a/clang/lib/Basic/Targets/BPF.h +++ b/clang/lib/Basic/Targets/BPF.h @@ -22,6 +22,8 @@ namespace targets { class LLVM_LIBRARY_VISIBILITY BPFTargetInfo : public TargetInfo { + static const Builtin::Info BuiltinInfo[]; + public: BPFTargetInfo(const llvm::Triple &Triple, const TargetOptions &) : TargetInfo(Triple) { @@ -54,7 +56,7 @@ Features[Name] = Enabled; } - ArrayRef getTargetBuiltins() const override { return None; } + ArrayRef getTargetBuiltins() const override; const char *getClobbers() const override { return ""; } diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp --- a/clang/lib/Basic/Targets/BPF.cpp +++ b/clang/lib/Basic/Targets/BPF.cpp @@ -13,11 +13,18 @@ #include "BPF.h" #include "Targets.h" #include "clang/Basic/MacroBuilder.h" +#include "clang/Basic/TargetBuiltins.h" #include "llvm/ADT/StringRef.h" using namespace clang; using namespace clang::targets; +const Builtin::Info BPFTargetInfo::BuiltinInfo[] = { +#define BUILTIN(ID, TYPE, ATTRS) \ + {#ID, TYPE, ATTRS, nullptr, ALL_LANGUAGES, nullptr}, +#include "clang/Basic/BuiltinsBPF.def" +}; + void BPFTargetInfo::getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const { Builder.defineMacro("__bpf__"); @@ -34,3 +41,8 @@ void BPFTargetInfo::fillValidCPUList(SmallVectorImpl &Values) const { Values.append(std::begin(ValidCPUNames), std::end(ValidCPUNames)); } + +ArrayRef BPFTargetInfo::getTargetBuiltins() const { + return llvm::makeArrayRef(BuiltinInfo, clang::BPF::LastTSBuiltin - + Builtin::FirstTSBuiltin); +} diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -4242,6 +4242,9 @@ case llvm::Triple::aarch64: case llvm::Triple::aarch64_be: return CGF->EmitAArch64BuiltinExpr(BuiltinID, E, Arch); + case llvm::Triple::bpfeb: + case llvm::Triple::bpfel: + return CGF->EmitBPFBuiltinExpr(BuiltinID, E); case llvm::Triple::x86: case llvm::Triple::x86_64: return CGF->EmitX86BuiltinExpr(BuiltinID, E); @@ -9300,6 +9303,37 @@ } } +Value *CodeGenFunction::EmitBPFBuiltinExpr(unsigned BuiltinID, + const CallExpr *E) { + assert(BuiltinID == BPF::BI__builtin_preserve_field_info && + "unexpected ARM builtin"); + + const Expr *Arg = E->getArg(0); + bool IsBitField = Arg->IgnoreParens()->getObjectKind() == OK_BitField; + + if (!getDebugInfo()) { + CGM.Error(E->getExprLoc(), "using builtin_preserve_field_info() without -g"); + return IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(); + } + + // Enable underlying preserve_*_access_index() generation. + bool OldIsInPreservedAIRegion = IsInPreservedAIRegion; + IsInPreservedAIRegion = true; + Value *FieldAddr = IsBitField ? EmitLValue(Arg).getBitFieldPointer() + : EmitLValue(Arg).getPointer(); + IsInPreservedAIRegion = OldIsInPreservedAIRegion; + + ConstantInt *C = cast(EmitScalarExpr(E->getArg(1))); + Value *InfoKind = ConstantInt::get(Int64Ty, C->getSExtValue()); + + // Built the IR for the preserve_field_info intrinsic. + llvm::Function *FnGetFieldInfo = llvm::Intrinsic::getDeclaration( + &CGM.getModule(), llvm::Intrinsic::bpf_preserve_field_info, + {FieldAddr->getType()}); + return Builder.CreateCall(FnGetFieldInfo, {FieldAddr, InfoKind}); +} + llvm::Value *CodeGenFunction:: BuildVector(ArrayRef Ops) { assert((Ops.size() & (Ops.size() - 1)) == 0 && diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3990,9 +3990,19 @@ const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); Address Addr = base.getAddress(); unsigned Idx = RL.getLLVMFieldNo(field); - if (Idx != 0) - // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + if (!IsInPreservedAIRegion) { + if (Idx != 0) + // For structs, we GEP to the field that the record layout suggests. + Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + } else { + const RecordDecl *rec = field->getParent(); + llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( + getContext().getRecordType(rec), rec->getLocation()); + Addr = Builder.CreatePreserveStructAccessIndex(Addr, Idx, + getDebugInfoFIndex(rec, field->getFieldIndex()), + DbgInfo); + } + // Get the access type. llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize); diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -3760,6 +3760,7 @@ llvm::Value *vectorWrapScalar16(llvm::Value *Op); llvm::Value *EmitAArch64BuiltinExpr(unsigned BuiltinID, const CallExpr *E, llvm::Triple::ArchType Arch); + llvm::Value *EmitBPFBuiltinExpr(unsigned BuiltinID, const CallExpr *E); llvm::Value *BuildVector(ArrayRef Ops); llvm::Value *EmitX86BuiltinExpr(unsigned BuiltinID, const CallExpr *E); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -1540,6 +1540,11 @@ if (CheckAArch64BuiltinFunctionCall(BuiltinID, TheCall)) return ExprError(); break; + case llvm::Triple::bpfeb: + case llvm::Triple::bpfel: + if (CheckBPFBuiltinFunctionCall(BuiltinID, TheCall)) + return ExprError(); + break; case llvm::Triple::hexagon: if (CheckHexagonBuiltinFunctionCall(BuiltinID, TheCall)) return ExprError(); @@ -1940,6 +1945,36 @@ return SemaBuiltinConstantArgRange(TheCall, i, l, u + l); } +bool Sema::CheckBPFBuiltinFunctionCall(unsigned BuiltinID, + CallExpr *TheCall) { + assert(BuiltinID == BPF::BI__builtin_preserve_field_info && + "unexpected ARM builtin"); + + if (checkArgCount(*this, TheCall, 2)) + return true; + + // The first argument needs to be a record field access + Expr *Arg = TheCall->getArg(0); + if (Arg->getType()->getAsPlaceholderType() || + (Arg->IgnoreParens()->getObjectKind() != OK_BitField && + !dyn_cast(Arg->IgnoreParens()))) { + Diag(Arg->getBeginLoc(), diag::err_preserve_field_info_not_field) + << 1 << Arg->getSourceRange(); + return true; + } + + // The second argument needs to be a constant int + llvm::APSInt Value; + if (!TheCall->getArg(1)->isIntegerConstantExpr(Value, Context)) { + Diag(Arg->getBeginLoc(), diag::err_preserve_field_info_not_const) + << 2 << Arg->getSourceRange(); + return true; + } + + TheCall->setType(Context.UnsignedIntTy); + return false; +} + bool Sema::CheckHexagonBuiltinCpu(unsigned BuiltinID, CallExpr *TheCall) { struct BuiltinAndString { unsigned BuiltinID; diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -20,4 +20,7 @@ Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], [IntrReadMem]>; def int_bpf_pseudo : GCCBuiltin<"__builtin_bpf_pseudo">, Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty]>; + def int_bpf_preserve_field_info : GCCBuiltin<"__builtin_bpf_preserve_field_info">, + Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<1>]>; } diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -15,7 +15,7 @@ namespace llvm { class BPFTargetMachine; -ModulePass *createBPFAbstractMemberAccess(); +ModulePass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); FunctionPass *createBPFISelDag(BPFTargetMachine &TM); FunctionPass *createBPFMISimplifyPatchablePass(); diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -50,6 +50,28 @@ // addr = preserve_struct_access_index(base, gep_index, di_index) // !llvm.preserve.access.index // +// Bitfield member access needs special attention. User cannot take the +// address of a bitfield acceess. To facilitate kernel verifier +// for easy bitfield code optimization, a new clang intrinsic is introduced: +// uint32_t __builtin_preserve_field_info(member_access, info_kind) +// In IR, a chain with two (or more) intrinsic calls will be generated: +// ... +// addr = preserve_struct_access_index(base, 1, 1) !struct s +// uint32_t result = bpf_preserve_field_info(addr, info_kind) +// +// Suppose the info_kind is FIELD_SIGNEDNESS, +// The above two IR intrinsics will be replaced with +// a relocatable insn: +// signness = /* signness of member_access */ +// and signness can be changed by bpf loader based on the +// types on the host. +// +// User can also test whether a field exists or not with +// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE) +// The field will be always available (result = 1) during initial +// compilation, but bpf loader can patch with the correct value +// on the target host where the member_access may or may not be available +// //===----------------------------------------------------------------------===// #include "BPF.h" @@ -88,7 +110,11 @@ public: static char ID; - BPFAbstractMemberAccess() : ModulePass(ID) {} + TargetMachine *TM; + // Add optional BPFTargetMachine parameter so that BPF backend can add the phase + // with target machine to find out the endianness. The default constructor (without + // parameters) is used by the pass manager for managing purposes. + BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) : ModulePass(ID), TM(TM) {} struct CallInfo { uint32_t Kind; @@ -102,13 +128,14 @@ BPFPreserveArrayAI = 1, BPFPreserveUnionAI = 2, BPFPreserveStructAI = 3, + BPFPreserveFieldInfoAI = 4, }; std::map GEPGlobals; // A map to link preserve_*_access_index instrinsic calls. std::map> AIChain; // A map to hold all the base preserve_*_access_index instrinsic calls. - // The base call is not an input of any other preserve_*_access_index + // The base call is not an input of any other preserve_* // intrinsics. std::map BaseAICalls; @@ -127,6 +154,8 @@ bool removePreserveAccessIndexIntrinsic(Module &M); void replaceWithGEP(std::vector &CallList, uint32_t NumOfZerosIndex, uint32_t DIIndex); + uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, + uint32_t AccessIndex); Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo, std::string &AccessKey, MDNode *&BaseMeta); @@ -139,8 +168,8 @@ INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE, "abstracting struct/union member accessees", false, false) -ModulePass *llvm::createBPFAbstractMemberAccess() { - return new BPFAbstractMemberAccess(); +ModulePass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { + return new BPFAbstractMemberAccess(TM); } bool BPFAbstractMemberAccess::runOnModule(Module &M) { @@ -231,6 +260,17 @@ CInfo.Base = Call->getArgOperand(0); return true; } + if (GV->getName().startswith("llvm.bpf.preserve.field.info")) { + CInfo.Kind = BPFPreserveFieldInfoAI; + CInfo.Metadata = nullptr; + // Check validity of info_kind as clang did not check this. + uint64_t InfoKind = getConstant(Call->getArgOperand(1)); + if (InfoKind == BPFCoreSharedInfo::FIELD_ACCESS_OFFSET || + InfoKind >= BPFCoreSharedInfo::MAX_FIELD_RELOC_KIND) + report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic"); + CInfo.AccessIndex = InfoKind; + return true; + } return false; } @@ -306,6 +346,9 @@ bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType, uint32_t ParentAI, const MDNode *ChildType) { + if (!ChildType) + return true; // preserve_field_info, no type comparison needed. + const DIType *PType = stripQualifiers(cast(ParentType)); const DIType *CType = stripQualifiers(cast(ChildType)); @@ -463,7 +506,97 @@ return CV->getValue().getZExtValue(); } -/// Compute the base of the whole preserve_*_access_index chains, i.e., the base +uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind, + DICompositeType *CTy, + uint32_t AccessIndex) { + if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE) + return 1; + + uint32_t Tag = CTy->getTag(); + assert(Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type); + auto *MemberTy = cast(CTy->getElements()[AccessIndex]); + + if (InfoKind == BPFCoreSharedInfo::FIELD_SIGNEDNESS) { + const DIType *BaseTy = stripQualifiers(MemberTy->getBaseType()); + // Only basic types and enum types have signedness. + const auto *BTy = dyn_cast(BaseTy); + while (!BTy) { + const auto *CompTy = dyn_cast(BaseTy); + // Report an error if the member expression does not have signedness. + if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type) + report_fatal_error("Invalid member expression for llvm.bpf.preserve.field.info"); + BaseTy = stripQualifiers(CompTy->getBaseType()); + BTy = dyn_cast(BaseTy); + } + uint32_t Encoding = BTy->getEncoding(); + return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char); + } + + if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_SIZE) { + uint32_t SizeInBits = MemberTy->getSizeInBits(); + if (!MemberTy->isBitField()) + return SizeInBits >> 3; + + uint32_t OffsetInBits = MemberTy->getOffsetInBits(); + uint32_t Start = OffsetInBits >> 3; + uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3; + return End - Start + 1; + } + + if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_OFFSET) + return MemberTy->getOffsetInBits() >> 3; + + if (InfoKind == BPFCoreSharedInfo::FIELD_LSHIFT_64BIT_BUF) { + uint32_t SizeInBits = MemberTy->getSizeInBits(); + const Triple &Triple = TM->getTargetTriple(); + if (!MemberTy->isBitField()) { + if (SizeInBits > 64) + report_fatal_error("too big member size for llvm.bpf.preserve.field.info"); + + /* endianness */ + if (Triple.getArch() == Triple::bpfeb) + return 0; + else + return 64 - SizeInBits; + } + + uint32_t OffsetInBits = MemberTy->getOffsetInBits(); + uint32_t Start = OffsetInBits >> 3; + uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3; + uint32_t NumOfBytes = End - Start + 1; + if (NumOfBytes > 8) + report_fatal_error("too big member size for llvm.bpf.preserve.field.info"); + + /* endianness */ + if (Triple.getArch() == Triple::bpfeb) + return OffsetInBits - (Start << 3); + else + return ((Start + 8) << 3) - OffsetInBits - SizeInBits; + } + + if (InfoKind == BPFCoreSharedInfo::FIELD_RSHIFT_AFTER_LSHIFT_64BIT_BUF) { + uint32_t SizeInBits = MemberTy->getSizeInBits(); + if (!MemberTy->isBitField()) { + if (SizeInBits > 64) + report_fatal_error("too big member size for llvm.bpf.preserve.field.info"); + + return 64 - SizeInBits; + } + + uint32_t OffsetInBits = MemberTy->getOffsetInBits(); + uint32_t Start = OffsetInBits >> 3; + uint32_t End = (OffsetInBits + SizeInBits - 1) >> 3; + uint32_t NumOfBytes = End - Start + 1; + if (NumOfBytes > 8) + report_fatal_error("too big member size for llvm.bpf.preserve.field.info"); + + return 64 - SizeInBits; + } + + llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind"); +} + +/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base /// pointer of the first preserve_*_access_index call, and construct the access /// string, which will be the name of a global variable. Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call, @@ -492,7 +625,8 @@ // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ... // we will skip them. uint32_t FirstIndex = 0; - uint32_t AccessOffset = 0; + uint32_t PatchImm = 0; // AccessOffset or the requested field info + uint32_t InfoKind = BPFCoreSharedInfo::FIELD_ACCESS_OFFSET; while (CallStack.size()) { auto StackElem = CallStack.top(); Call = StackElem.first; @@ -507,10 +641,13 @@ // struct or union type TypeName = Ty->getName(); TypeMeta = Ty; - AccessOffset += FirstIndex * Ty->getSizeInBits() >> 3; + PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3); break; } + assert(CInfo.Kind == BPFPreserveFieldInfoAI && + "Reaching preserve_field_info and cannot find top type name"); + // Array entries will always be consumed for accumulative initial index. CallStack.pop(); @@ -555,7 +692,7 @@ else TypeName = CTy->getName(); TypeMeta = CTy; - AccessOffset += FirstIndex * CTy->getSizeInBits() >> 3; + PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3); break; } } @@ -569,6 +706,20 @@ CInfo = StackElem.second; CallStack.pop(); + if (CInfo.Kind == BPFPreserveFieldInfoAI) + break; + + // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI, + // the action will be extracting field info. + if (CallStack.size()) { + auto StackElem2 = CallStack.top(); + CallInfo CInfo2 = StackElem2.second; + if (CInfo2.Kind == BPFPreserveFieldInfoAI) { + InfoKind = CInfo2.AccessIndex; + assert(CallStack.size() == 1); + } + } + // Access Index uint64_t AccessIndex = CInfo.AccessIndex; AccessKey += ":" + std::to_string(AccessIndex); @@ -577,19 +728,24 @@ // At this stage, it cannot be pointer type. auto *CTy = cast(stripQualifiers(cast(MDN))); uint32_t Tag = CTy->getTag(); - if (Tag == dwarf::DW_TAG_structure_type) { - auto *MemberTy = cast(CTy->getElements()[AccessIndex]); - AccessOffset += MemberTy->getOffsetInBits() >> 3; - } else if (Tag == dwarf::DW_TAG_array_type) { - auto *EltTy = stripQualifiers(CTy->getBaseType()); - AccessOffset += AccessIndex * calcArraySize(CTy, 1) * - EltTy->getSizeInBits() >> 3; + if (InfoKind == BPFCoreSharedInfo::FIELD_ACCESS_OFFSET) { + if (Tag == dwarf::DW_TAG_structure_type) { + auto *MemberTy = cast(CTy->getElements()[AccessIndex]); + PatchImm += MemberTy->getOffsetInBits() >> 3; + } else if (Tag == dwarf::DW_TAG_array_type) { + auto *EltTy = stripQualifiers(CTy->getBaseType()); + PatchImm += AccessIndex * calcArraySize(CTy, 1) * + (EltTy->getSizeInBits() >> 3); + } + } else { + PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex); } } - // Access key is the type name + access string, uniquely identifying - // one kernel memory access. - AccessKey = TypeName + ":" + std::to_string(AccessOffset) + "$" + AccessKey; + // Access key is the type name + reloc type + patched imm + access string, + // uniquely identifying one relocation. + AccessKey = TypeName + ":" + std::to_string(InfoKind) + ":" + + std::to_string(PatchImm) + "$" + AccessKey; return Base; } @@ -605,21 +761,11 @@ if (!Base) return false; - // Do the transformation - // For any original GEP Call and Base %2 like - // %4 = bitcast %struct.net_device** %dev1 to i64* - // it is transformed to: - // %6 = load sk_buff:50:$0:0:0:2:0 - // %7 = bitcast %struct.sk_buff* %2 to i8* - // %8 = getelementptr i8, i8* %7, %6 - // %9 = bitcast i8* %8 to i64* - // using %9 instead of %4 - // The original Call inst is removed. BasicBlock *BB = Call->getParent(); GlobalVariable *GV; if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) { - GV = new GlobalVariable(M, Type::getInt64Ty(BB->getContext()), false, + GV = new GlobalVariable(M, Type::getInt32Ty(BB->getContext()), false, GlobalVariable::ExternalLinkage, NULL, AccessKey); GV->addAttribute(BPFCoreSharedInfo::AmaAttr); GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); @@ -628,8 +774,27 @@ GV = GEPGlobals[AccessKey]; } + if (CInfo.Kind == BPFPreserveFieldInfoAI) { + // Load the global variable which represents the returned field info. + auto *LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV); + BB->getInstList().insert(Call->getIterator(), LDInst); + Call->replaceAllUsesWith(LDInst); + Call->eraseFromParent(); + return true; + } + + // For any original GEP Call and Base %2 like + // %4 = bitcast %struct.net_device** %dev1 to i64* + // it is transformed to: + // %6 = load sk_buff:50:$0:0:0:2:0 + // %7 = bitcast %struct.sk_buff* %2 to i8* + // %8 = getelementptr i8, i8* %7, %6 + // %9 = bitcast i8* %8 to i64* + // using %9 instead of %4 + // The original Call inst is removed. + // Load the global variable. - auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); + auto *LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV); BB->getInstList().insert(Call->getIterator(), LDInst); // Generate a BitCast diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -13,6 +13,17 @@ class BPFCoreSharedInfo { public: + enum OffsetRelocKind : uint32_t { + FIELD_ACCESS_OFFSET = 0, + FIELD_EXISTENCE, + FIELD_SIGNEDNESS, + FIELD_BYTE_SIZE, + FIELD_BYTE_OFFSET, + FIELD_LSHIFT_64BIT_BUF, + FIELD_RSHIFT_AFTER_LSHIFT_64BIT_BUF, + + MAX_FIELD_RELOC_KIND, + }; /// The attribute attached to globals representing a member offset static const std::string AmaAttr; /// The section name to identify a patchable external global diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -94,7 +94,7 @@ void BPFPassConfig::addIRPasses() { - addPass(createBPFAbstractMemberAccess()); + addPass(createBPFAbstractMemberAccess(&getBPFTargetMachine())); TargetPassConfig::addIRPasses(); } diff --git a/llvm/lib/Target/BPF/BTF.h b/llvm/lib/Target/BPF/BTF.h --- a/llvm/lib/Target/BPF/BTF.h +++ b/llvm/lib/Target/BPF/BTF.h @@ -17,7 +17,7 @@ /// /// The binary layout for .BTF.ext section: /// struct ExtHeader -/// FuncInfo, LineInfo, OffsetReloc and ExternReloc subsections +/// FuncInfo, LineInfo, FieldReloc and ExternReloc subsections /// The FuncInfo subsection is defined as below: /// BTFFuncInfo Size /// struct SecFuncInfo for ELF section #1 @@ -32,12 +32,12 @@ /// struct SecLineInfo for ELF section #2 /// A number of struct BPFLineInfo for ELF section #2 /// ... -/// The OffsetReloc subsection is defined as below: -/// BPFOffsetReloc Size -/// struct SecOffsetReloc for ELF section #1 -/// A number of struct BPFOffsetReloc for ELF section #1 -/// struct SecOffsetReloc for ELF section #2 -/// A number of struct BPFOffsetReloc for ELF section #2 +/// The FieldReloc subsection is defined as below: +/// BPFFieldReloc Size +/// struct SecFieldReloc for ELF section #1 +/// A number of struct BPFFieldReloc for ELF section #1 +/// struct SecFieldReloc for ELF section #2 +/// A number of struct BPFFieldReloc for ELF section #2 /// ... /// The ExternReloc subsection is defined as below: /// BPFExternReloc Size @@ -72,11 +72,11 @@ BTFDataSecVarSize = 12, SecFuncInfoSize = 8, SecLineInfoSize = 8, - SecOffsetRelocSize = 8, + SecFieldRelocSize = 8, SecExternRelocSize = 8, BPFFuncInfoSize = 8, BPFLineInfoSize = 16, - BPFOffsetRelocSize = 12, + BPFFieldRelocSize = 16, BPFExternRelocSize = 8, }; @@ -213,8 +213,8 @@ uint32_t FuncInfoLen; ///< Length of func info section uint32_t LineInfoOff; ///< Offset of line info section uint32_t LineInfoLen; ///< Length of line info section - uint32_t OffsetRelocOff; ///< Offset of offset reloc section - uint32_t OffsetRelocLen; ///< Length of offset reloc section + uint32_t FieldRelocOff; ///< Offset of offset reloc section + uint32_t FieldRelocLen; ///< Length of offset reloc section uint32_t ExternRelocOff; ///< Offset of extern reloc section uint32_t ExternRelocLen; ///< Length of extern reloc section }; @@ -247,16 +247,17 @@ }; /// Specifying one offset relocation. -struct BPFOffsetReloc { +struct BPFFieldReloc { uint32_t InsnOffset; ///< Byte offset in this section uint32_t TypeID; ///< TypeID for the relocation uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t RelocKind; ///< What to patch the instruction }; /// Specifying offset relocation's in one section. -struct SecOffsetReloc { +struct SecFieldReloc { uint32_t SecNameOff; ///< Section name index in the .BTF string table - uint32_t NumOffsetReloc; ///< Number of offset reloc's in this section + uint32_t NumFieldReloc; ///< Number of offset reloc's in this section }; /// Specifying one offset relocation. diff --git a/llvm/lib/Target/BPF/BTFDebug.h b/llvm/lib/Target/BPF/BTFDebug.h --- a/llvm/lib/Target/BPF/BTFDebug.h +++ b/llvm/lib/Target/BPF/BTFDebug.h @@ -224,10 +224,11 @@ }; /// Represent one offset relocation. -struct BTFOffsetReloc { +struct BTFFieldReloc { const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc uint32_t TypeID; ///< Type ID uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t RelocKind; ///< What to patch the instruction }; /// Represent one extern relocation. @@ -249,12 +250,12 @@ std::unordered_map DIToIdMap; std::map> FuncInfoTable; std::map> LineInfoTable; - std::map> OffsetRelocTable; + std::map> FieldRelocTable; std::map> ExternRelocTable; StringMap> FileContent; std::map> DataSecEntries; std::vector StructTypes; - std::map AccessOffsets; + std::map PatchImms; std::map>> FixupDerivedTypes; @@ -300,7 +301,7 @@ void processGlobals(bool ProcessingMapDef); /// Generate one offset relocation record. - void generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym, + void generateFieldReloc(const MachineInstr *MI, const MCSymbol *ORSym, DIType *RootTy, StringRef AccessPattern); /// Populating unprocessed struct type. diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp --- a/llvm/lib/Target/BPF/BTFDebug.cpp +++ b/llvm/lib/Target/BPF/BTFDebug.cpp @@ -754,7 +754,7 @@ void BTFDebug::emitBTFExtSection() { // Do not emit section if empty FuncInfoTable and LineInfoTable. if (!FuncInfoTable.size() && !LineInfoTable.size() && - !OffsetRelocTable.size() && !ExternRelocTable.size()) + !FieldRelocTable.size() && !ExternRelocTable.size()) return; MCContext &Ctx = OS.getContext(); @@ -766,8 +766,8 @@ // Account for FuncInfo/LineInfo record size as well. uint32_t FuncLen = 4, LineLen = 4; - // Do not account for optional OffsetReloc/ExternReloc. - uint32_t OffsetRelocLen = 0, ExternRelocLen = 0; + // Do not account for optional FieldReloc/ExternReloc. + uint32_t FieldRelocLen = 0, ExternRelocLen = 0; for (const auto &FuncSec : FuncInfoTable) { FuncLen += BTF::SecFuncInfoSize; FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize; @@ -776,17 +776,17 @@ LineLen += BTF::SecLineInfoSize; LineLen += LineSec.second.size() * BTF::BPFLineInfoSize; } - for (const auto &OffsetRelocSec : OffsetRelocTable) { - OffsetRelocLen += BTF::SecOffsetRelocSize; - OffsetRelocLen += OffsetRelocSec.second.size() * BTF::BPFOffsetRelocSize; + for (const auto &FieldRelocSec : FieldRelocTable) { + FieldRelocLen += BTF::SecFieldRelocSize; + FieldRelocLen += FieldRelocSec.second.size() * BTF::BPFFieldRelocSize; } for (const auto &ExternRelocSec : ExternRelocTable) { ExternRelocLen += BTF::SecExternRelocSize; ExternRelocLen += ExternRelocSec.second.size() * BTF::BPFExternRelocSize; } - if (OffsetRelocLen) - OffsetRelocLen += 4; + if (FieldRelocLen) + FieldRelocLen += 4; if (ExternRelocLen) ExternRelocLen += 4; @@ -795,8 +795,8 @@ OS.EmitIntValue(FuncLen, 4); OS.EmitIntValue(LineLen, 4); OS.EmitIntValue(FuncLen + LineLen, 4); - OS.EmitIntValue(OffsetRelocLen, 4); - OS.EmitIntValue(FuncLen + LineLen + OffsetRelocLen, 4); + OS.EmitIntValue(FieldRelocLen, 4); + OS.EmitIntValue(FuncLen + LineLen + FieldRelocLen, 4); OS.EmitIntValue(ExternRelocLen, 4); // Emit func_info table. @@ -832,18 +832,19 @@ } // Emit offset reloc table. - if (OffsetRelocLen) { - OS.AddComment("OffsetReloc"); - OS.EmitIntValue(BTF::BPFOffsetRelocSize, 4); - for (const auto &OffsetRelocSec : OffsetRelocTable) { + if (FieldRelocLen) { + OS.AddComment("FieldReloc"); + OS.EmitIntValue(BTF::BPFFieldRelocSize, 4); + for (const auto &FieldRelocSec : FieldRelocTable) { OS.AddComment("Offset reloc section string offset=" + - std::to_string(OffsetRelocSec.first)); - OS.EmitIntValue(OffsetRelocSec.first, 4); - OS.EmitIntValue(OffsetRelocSec.second.size(), 4); - for (const auto &OffsetRelocInfo : OffsetRelocSec.second) { - Asm->EmitLabelReference(OffsetRelocInfo.Label, 4); - OS.EmitIntValue(OffsetRelocInfo.TypeID, 4); - OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4); + std::to_string(FieldRelocSec.first)); + OS.EmitIntValue(FieldRelocSec.first, 4); + OS.EmitIntValue(FieldRelocSec.second.size(), 4); + for (const auto &FieldRelocInfo : FieldRelocSec.second) { + Asm->EmitLabelReference(FieldRelocInfo.Label, 4); + OS.EmitIntValue(FieldRelocInfo.TypeID, 4); + OS.EmitIntValue(FieldRelocInfo.OffsetNameOff, 4); + OS.EmitIntValue(FieldRelocInfo.RelocKind, 4); } } } @@ -959,22 +960,26 @@ } /// Generate a struct member offset relocation. -void BTFDebug::generateOffsetReloc(const MachineInstr *MI, +void BTFDebug::generateFieldReloc(const MachineInstr *MI, const MCSymbol *ORSym, DIType *RootTy, StringRef AccessPattern) { unsigned RootId = populateStructType(RootTy); size_t FirstDollar = AccessPattern.find_first_of('$'); size_t FirstColon = AccessPattern.find_first_of(':'); + size_t SecondColon = AccessPattern.find_first_of(':', FirstColon + 1); StringRef IndexPattern = AccessPattern.substr(FirstDollar + 1); - StringRef OffsetStr = AccessPattern.substr(FirstColon + 1, - FirstDollar - FirstColon); - - BTFOffsetReloc OffsetReloc; - OffsetReloc.Label = ORSym; - OffsetReloc.OffsetNameOff = addString(IndexPattern); - OffsetReloc.TypeID = RootId; - AccessOffsets[AccessPattern.str()] = std::stoi(OffsetStr); - OffsetRelocTable[SecNameOff].push_back(OffsetReloc); + StringRef RelocKindStr = AccessPattern.substr(FirstColon + 1, + SecondColon - FirstColon); + StringRef PatchImmStr = AccessPattern.substr(SecondColon + 1, + FirstDollar - SecondColon); + + BTFFieldReloc FieldReloc; + FieldReloc.Label = ORSym; + FieldReloc.OffsetNameOff = addString(IndexPattern); + FieldReloc.TypeID = RootId; + FieldReloc.RelocKind = std::stoull(RelocKindStr); + PatchImms[AccessPattern.str()] = std::stoul(PatchImmStr); + FieldRelocTable[SecNameOff].push_back(FieldReloc); } void BTFDebug::processLDimm64(const MachineInstr *MI) { @@ -982,7 +987,7 @@ // will generate an .BTF.ext record. // // If the insn is "r2 = LD_imm64 @__BTF_...", - // add this insn into the .BTF.ext OffsetReloc subsection. + // add this insn into the .BTF.ext FieldReloc subsection. // Relocation looks like: // . SecName: // . InstOffset @@ -1013,7 +1018,7 @@ MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index); DIType *Ty = dyn_cast(MDN); - generateOffsetReloc(MI, ORSym, Ty, GVar->getName()); + generateFieldReloc(MI, ORSym, Ty, GVar->getName()); } else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() && GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) { MCSymbol *ORSym = OS.getContext().createTempSymbol(); @@ -1154,8 +1159,8 @@ const GlobalValue *GVal = MO.getGlobal(); auto *GVar = dyn_cast(GVal); if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) { - // Emit "mov ri, " for abstract member accesses. - int64_t Imm = AccessOffsets[GVar->getName().str()]; + // Emit "mov ri, " for patched immediate. + uint32_t Imm = PatchImms[GVar->getName().str()]; OutMI.setOpcode(BPF::MOV_ri); OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg())); OutMI.addOperand(MCOperand::createImm(Imm));