diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -2378,6 +2378,47 @@ int *pb =__builtin_preserve_access_index(&v->c[3].b); __builtin_preserve_access_index(v->j); +``__builtin_preserve_bitfield_info`` +------------------------------------ + +``__builtin_preserve_bitfield_info`` permits the structure bitfield access +to be relocatable/verifiable under bpf compile-once run-everywhere framework. +Debuginfo (typically with ``-g``) is needed, otherwise, the compiler will +exit with an error. The intrinsic returns the memory address of the first +bitfield in the group of contiguous run of bitfields including the one +expressed in the first argument. The second argument returns the following +information which can be used by application for a generic way to extract +the bitfield values. The semantics of these fields are similar to the +definition in ``clang::CodeGen::CGFieldInfo``. + * the signness of the bitfield + * the size of the bitfield + * the offset within the contiguous run of bitfields + +**Syntax**: + +.. code-block:: c + + void * __builtin_preserve_bitfield_info(expr, unsigned *buf) + +**Example of Use**: + +.. code-block:: c + + struct t { + int a; + int f1:8; + int f2:1; + int f3:2; + int b; + }; + struct t *v = ...; + unsigned info; + void *p =__builtin_preserve_bitfield_info(v->f3, &info); + // p = (void *)v + 4, pointing to field f1 + // signness = info >> 31 + // size = (info >> 16) & 0x7fff + // offset = info & 0xffff + Multiprecision Arithmetic Builtins ---------------------------------- diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -1469,6 +1469,7 @@ BUILTIN(__builtin_char_memchr, "c*cC*iz", "n") BUILTIN(__builtin_dump_struct, "ivC*v*", "tn") BUILTIN(__builtin_preserve_access_index, "v.", "t") +BUILTIN(__builtin_preserve_bitfield_info, "v*.", "t") // Safestack builtins BUILTIN(__builtin___get_unsafe_stack_start, "v*", "Fn") diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9916,4 +9916,15 @@ "__builtin_bit_cast %select{source|destination}0 type must be trivially copyable">; def err_bit_cast_type_size_mismatch : Error< "__builtin_bit_cast source size does not equal destination size (%0 vs %1)">; + +def err_preserve_bitfield_info_not_bitfield : Error< + "__builtin_preserve_bitfield_info arg %0 not a bitfield access">; +def err_preserve_bitfield_info_not_ptr: Error< + "__builtin_preserve_bitfield_info arg %0 not a pointer">; +def err_preserve_bitfield_info_ptr_invalid_align: Error< + "__builtin_preserve_bitfield_info arg %0 must be a pointer" + " with memory alignment of %1-bit">; +def err_preserve_bitfield_info_ptr_invalid_size: Error< + "__builtin_preserve_bitfield_info arg %0 must be a pointer" + " with pointee size of %1-bit instead of %2-bit">; } // end of sema component. diff --git a/clang/lib/CodeGen/CGBuilder.h b/clang/lib/CodeGen/CGBuilder.h --- a/clang/lib/CodeGen/CGBuilder.h +++ b/clang/lib/CodeGen/CGBuilder.h @@ -303,6 +303,7 @@ Address CreatePreserveStructAccessIndex(Address Addr, unsigned Index, unsigned FieldIndex, + unsigned PreservedIndex, llvm::MDNode *DbgInfo) { llvm::StructType *ElTy = cast(Addr.getElementType()); const llvm::DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); @@ -310,7 +311,9 @@ auto Offset = CharUnits::fromQuantity(Layout->getElementOffset(Index)); return Address(CreatePreserveStructAccessIndex(Addr.getPointer(), - Index, FieldIndex, DbgInfo), + Index, FieldIndex, + PreservedIndex, + DbgInfo), Addr.getAlignment().alignmentAtOffset(Offset)); } }; diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -1871,9 +1871,9 @@ return RValue::get(EmitScalarExpr(E->getArg(0))); } - // Nested builtin_preserve_access_index() not supported + // Nested builtin_preserve_{access_index, bitfield_info}() not supported if (IsInPreservedAIRegion) { - CGM.Error(E->getExprLoc(), "nested builtin_preserve_access_index() not supported"); + CGM.Error(E->getExprLoc(), "nested builtin_preserve_*() not supported"); return RValue::get(EmitScalarExpr(E->getArg(0))); } @@ -1883,6 +1883,41 @@ return RValue::get(Res); } + case Builtin::BI__builtin_preserve_bitfield_info: { + if (!getDebugInfo()) { + CGM.Error(E->getExprLoc(), "using builtin_preserve_bitfield_info() without -g"); + return RValue::get(EmitLValue(E->getArg(0)).getBitFieldPointer()); + } + + if (IsInPreservedAIRegion) { + CGM.Error(E->getExprLoc(), "nested builtin_preserve_*() not supported"); + return RValue::get(EmitLValue(E->getArg(0)).getBitFieldPointer()); + } + + IsInPreservedAIRegion = true; + + // Emit relocation for the first member of bitfield group. + LValue LV = EmitLValue(E->getArg(0)); + Value *BitFieldPtr = LV.getBitFieldPointer(); + unsigned AS = cast(BitFieldPtr->getType())->getAddressSpace(); + Value *Res = Builder.CreateBitCast(BitFieldPtr, Int8Ty->getPointerTo(AS)); + + // Store the bitfield info in user memory. + const CGBitFieldInfo &Info = LV.getBitFieldInfo(); + uint32_t Val = Info.IsSigned << 31 | + Info.Size << 16 | + Info.Offset; + Value *InfoVal = ConstantInt::get(Int32Ty, Val); + Address InfoPtr = EmitPointerWithAlignment(E->getArg(1)); + llvm::PointerType *Int32PtrTy = + Int32Ty->getPointerTo(InfoPtr.getAddressSpace()); + Builder.CreateStore(InfoVal, + Builder.CreateBitCast(InfoPtr, Int32PtrTy)); + + IsInPreservedAIRegion = false; + return RValue::get(Res); + } + case Builtin::BI__builtin_cimag: case Builtin::BI__builtin_cimagf: case Builtin::BI__builtin_cimagl: diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3903,19 +3903,24 @@ } /// Get the field index in the debug info. The debug info structure/union -/// will ignore the unnamed bitfields. +/// will ignore the unnamed bitfields. The \p PreservedIndex is the +/// field index recorded as relocation for structures. unsigned CodeGenFunction::getDebugInfoFIndex(const RecordDecl *Rec, - unsigned FieldIndex) { - unsigned I = 0, Skipped = 0; + unsigned FieldIndex, + unsigned &PreservedIndex) { + unsigned I = 0, Skipped = 0, BitFieldsInGroup = 0; for (auto F : Rec->getDefinition()->fields()) { if (I == FieldIndex) break; if (F->isUnnamedBitfield()) Skipped++; + else + BitFieldsInGroup = F->isBitField() ? BitFieldsInGroup + 1 : 0; I++; } + PreservedIndex = FieldIndex - Skipped - BitFieldsInGroup; return FieldIndex - Skipped; } @@ -3957,8 +3962,11 @@ unsigned idx = CGF.CGM.getTypes().getCGRecordLayout(rec).getLLVMFieldNo(field); - return CGF.Builder.CreatePreserveStructAccessIndex( - base, idx, CGF.getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo); + unsigned FieldIndex = field->getFieldIndex(), PreservedIndex; + unsigned DIFieldIndex = CGF.getDebugInfoFIndex(rec, FieldIndex, + PreservedIndex); + return CGF.Builder.CreatePreserveStructAccessIndex(base, idx, + DIFieldIndex, PreservedIndex, DbgInfo); } static bool hasAnyVptr(const QualType Type, const ASTContext &Context) { @@ -3990,9 +3998,25 @@ const CGBitFieldInfo &Info = RL.getBitFieldInfo(field); Address Addr = base.getAddress(); unsigned Idx = RL.getLLVMFieldNo(field); - if (Idx != 0) - // For structs, we GEP to the field that the record layout suggests. - Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + if (!IsInPreservedAIRegion) { + if (Idx != 0) + // For structs, we GEP to the field that the record layout suggests. + Addr = Builder.CreateStructGEP(Addr, Idx, field->getName()); + } else { + const RecordDecl *rec = field->getParent(); + llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( + getContext().getRecordType(rec), rec->getLocation()); + + // Preserve access index for the first bitfield member + // in the contiguous run of bitfields with this field. + unsigned FieldIndex = field->getFieldIndex(); + unsigned PreservedIndex; + unsigned DIFieldIndex = getDebugInfoFIndex(rec, FieldIndex, + PreservedIndex); + Addr = Builder.CreatePreserveStructAccessIndex(Addr, Idx, + DIFieldIndex, PreservedIndex, DbgInfo); + } + // Get the access type. llvm::Type *FieldIntTy = llvm::Type::getIntNTy(getLLVMContext(), Info.StorageSize); @@ -4072,9 +4096,12 @@ // Remember the original union field index llvm::DIType *DbgInfo = getDebugInfo()->getOrCreateRecordType( getContext().getRecordType(rec), rec->getLocation()); + unsigned PreservedIndex; addr = Address( Builder.CreatePreserveUnionAccessIndex( - addr.getPointer(), getDebugInfoFIndex(rec, field->getFieldIndex()), DbgInfo), + addr.getPointer(), + getDebugInfoFIndex(rec, field->getFieldIndex(), PreservedIndex), + DbgInfo), addr.getAlignment()); } diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h --- a/clang/lib/CodeGen/CodeGenFunction.h +++ b/clang/lib/CodeGen/CodeGenFunction.h @@ -2653,9 +2653,10 @@ /// Converts Location to a DebugLoc, if debug information is enabled. llvm::DebugLoc SourceLocToDebugLoc(SourceLocation Location); - /// Get the record field index as represented in debug info. - unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex); - + /// Get the record field index as represented in debug info and + /// the record field index used for relocation. + unsigned getDebugInfoFIndex(const RecordDecl *Rec, unsigned FieldIndex, + unsigned &PreservedIndex); //===--------------------------------------------------------------------===// // Declaration Emission diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -201,6 +201,61 @@ return false; } +static bool SemaBuiltinPreserveBI(Sema &S, CallExpr *TheCall) { + if (checkArgCount(S, TheCall, 2)) + return true; + + // The first argument must be a bitfield access + Expr *Arg = TheCall->getArg(0); + if (Arg->getType()->getAsPlaceholderType() || + Arg->IgnoreParens()->getObjectKind() != OK_BitField) { + S.Diag(Arg->getBeginLoc(), diag::err_preserve_bitfield_info_not_bitfield) + << 1 /* first argument */ << Arg->getSourceRange(); + return true; + } + + // The second argument must be a pointer and aligned at 32-bit + // boundary as later a 32-bit store insn will be generated. + // Note that higher alignment (e.g. 64-bit) is okay. + // The size of pointee must be 32-bit. + Arg = TheCall->getArg(1); + QualType ArgType = Arg->getType(); + if (ArgType->isPointerType()) { + bool IsAligned = false; + QualType Pointee = ArgType->getPointeeType(); + unsigned IntAlign = S.Context.getTargetInfo().getIntAlign(); + if (!Pointee->isIncompleteType()) { + unsigned PointeeAlign = S.Context.getTypeAlign(Pointee); + if (!(PointeeAlign & (IntAlign - 1))) + IsAligned = true; + } + if (!IsAligned) { + S.Diag(Arg->getBeginLoc(), diag::err_preserve_bitfield_info_ptr_invalid_align) + << 2 /* second argument */ + << IntAlign + << Arg->getSourceRange(); + return true; + } + + unsigned IntSize = S.Context.getTargetInfo().getIntWidth(); + unsigned PointeeSize = S.Context.getTypeSize(Pointee); + if (IntSize != PointeeSize) { + S.Diag(Arg->getBeginLoc(), diag::err_preserve_bitfield_info_ptr_invalid_size) + << 2 /* second argument */ + << IntSize << PointeeSize + << Arg->getSourceRange(); + return true; + } + } else { + S.Diag(Arg->getBeginLoc(), diag::err_preserve_bitfield_info_not_ptr) + << 2 /* second argument */ << Arg->getSourceRange(); + return true; + } + + TheCall->setType(S.Context.VoidPtrTy); + return false; +} + static bool SemaBuiltinOverflow(Sema &S, CallExpr *TheCall) { if (checkArgCount(S, TheCall, 3)) return true; @@ -1429,6 +1484,10 @@ if (SemaBuiltinPreserveAI(*this, TheCall)) return ExprError(); break; + case Builtin::BI__builtin_preserve_bitfield_info: + if (SemaBuiltinPreserveBI(*this, TheCall)) + return ExprError(); + break; case Builtin::BI__builtin_call_with_static_chain: if (SemaBuiltinCallWithStaticChain(*this, TheCall)) return ExprError(); diff --git a/clang/test/CodeGen/builtin-preserve-bitfield-info-2.c b/clang/test/CodeGen/builtin-preserve-bitfield-info-2.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/builtin-preserve-bitfield-info-2.c @@ -0,0 +1,31 @@ +// RUN: %clang -target x86_64 -emit-llvm -O2 -S -g %s -o - | FileCheck %s + +struct s { + int a; + int b1:8; + int b2:3; + int b3:4; +}; + +#define _(x, y) __builtin_preserve_bitfield_info((x), (y)) +void process(void *); +unsigned test(struct s *arg) { + unsigned info1, info2; + void *p1, *p2; + + p1 = _(arg->b2, &info1); + p2 = _(arg->b3, &info2); + process(p1); + process(p2); + // the last 16 bit is the offset, so + // return value should be 8 + 11 = 19 + // At -O2, compiler should do this optimization. + return (info1 & 0xffff) + (info2 & 0xffff); +} + +// CHECK: define dso_local i32 @test +// CHECK: tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %{{[0-9a-z]+}}, i32 1, i32 2, i32 1), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[STRUCT_S:[0-9]+]] +// CHECK: tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %{{[0-9a-z]+}}, i32 1, i32 3, i32 1), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[STRUCT_S:[0-9]+]] +// CHECK: ret i32 19 +// +// CHECK: ![[STRUCT_S]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s" diff --git a/clang/test/CodeGen/builtin-preserve-bitfield-info.c b/clang/test/CodeGen/builtin-preserve-bitfield-info.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/builtin-preserve-bitfield-info.c @@ -0,0 +1,54 @@ +// RUN: %clang -target x86_64 -emit-llvm -S -g %s -o - | FileCheck %s + +struct s { + int a; + int b1:8; + int b2:3; + int b3:4; + int c; +}; + +extern int bpf_probe_read(void *, unsigned, void *); +int test(struct s *arg, int generic) { + unsigned info, sign, size, offset; + union { + unsigned char uc; + char sc; + } v; + void *p, *s; + int ret; + char c; + + p = __builtin_preserve_bitfield_info(arg->b3, &info); + if (generic) { + // try to be generic + // Assume all bitfields can be read within a byte. + sign = info >> 31; + size = (info >> 16) & 0x7fff; + offset = info & 0xffff; + s = p + (offset >> 3); + + bpf_probe_read(&v, sizeof(v), s); + if (sign) { + v.sc = v.sc << (offset & 0x7); + ret = v.sc >> (8 - size); + } else { + v.uc = v.uc << (offset & 0x7); + ret = v.uc >> (8 - size); + } + } else { + // use specific knowledge for this field + bpf_probe_read(&c, sizeof(c), p + 1); + c = c << 3; + ret = c >> 4; + } + + return ret; +} + +// CHECK: define dso_local i32 @test +// CHECK: call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %{{[0-9a-z]+}}, i32 1, i32 3, i32 1), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[STRUCT_S:[0-9]+]] +// CHECK: store i32 -2147221493, i32* %{{[0-9a-z]+}} +// +// CHECK: ![[STRUCT_S]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s" + diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -17679,7 +17679,8 @@ declare @llvm.preserve.struct.access.index.p0i8.p0s_struct.anon.0s( base, i32 gep_index, - i32 di_index) + i32 di_index, + i32 preserved_index) Overview: """"""""" @@ -17696,6 +17697,10 @@ The ``base`` is the structure base address. The ``gep_index`` is the struct member index based on IR structures. The ``di_index`` is the struct member index based on debuginfo. +The ``preserved_index`` is the debuginfo struct member index which corresponds to +``gep_index``. The ``preserved_index`` is the same as ``di_index`` for non bitfield members. +For bitfield members, the ``preserved_index`` corresponds to the first bitfield member +in the contiguous run of bitfields include ``di_index``. Semantics: """""""""" diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h --- a/llvm/include/llvm/IR/IRBuilder.h +++ b/llvm/include/llvm/IR/IRBuilder.h @@ -2546,7 +2546,9 @@ } Value *CreatePreserveStructAccessIndex(Value *Base, unsigned Index, - unsigned FieldIndex, MDNode *DbgInfo) { + unsigned FieldIndex, + unsigned PreservedIndex, + MDNode *DbgInfo) { assert(isa(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index."); auto *BaseType = Base->getType(); @@ -2561,8 +2563,9 @@ M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); Value *DIIndex = getInt32(FieldIndex); + Value *PreservedDIIndex = getInt32(PreservedIndex); CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, - {Base, GEPIndex, DIIndex}); + {Base, GEPIndex, DIIndex, PreservedDIIndex}); if (DbgInfo) Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -1255,9 +1255,9 @@ [IntrNoMem, ImmArg<1>]>; def int_preserve_struct_access_index : Intrinsic<[llvm_anyptr_ty], [llvm_anyptr_ty, llvm_i32_ty, - llvm_i32_ty], + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>, - ImmArg<2>]>; + ImmArg<2>, ImmArg<3>]>; //===----------------------------------------------------------------------===// // Target-specific intrinsics diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -47,7 +47,8 @@ // addr = preserve_array_access_index(base, dimension, index) // addr = preserve_union_access_index(base, di_index) // !llvm.preserve.access.index -// addr = preserve_struct_access_index(base, gep_index, di_index) +// addr = preserve_struct_access_index(base, gep_index, di_index, +// preserved_index) // !llvm.preserve.access.index // //===----------------------------------------------------------------------===// @@ -65,12 +66,14 @@ #include "llvm/IR/Value.h" #include "llvm/Pass.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include #include #define DEBUG_TYPE "bpf-abstract-member-access" namespace llvm { const std::string BPFCoreSharedInfo::AmaAttr = "btf_ama"; +const std::string BPFCoreSharedInfo::BitFieldAttr = "btf_bitfield_di_findex"; const std::string BPFCoreSharedInfo::PatchableExtSecName = ".BPF.patchable_externs"; } // namespace llvm @@ -104,6 +107,7 @@ // The base call is not an input of any other preserve_*_access_index // intrinsics. std::map BaseAICalls; + std::map> BitfieldAccesses; bool doTransformation(Module &M); @@ -124,6 +128,7 @@ uint32_t NumOfZerosIndex, uint32_t DIIndex); Value *computeBaseAndAccessKey(CallInst *Call, std::string &AccessKey, + uint32_t &DIFieldIndex, uint32_t Kind, MDNode *&BaseMeta); bool getAccessIndex(const Value *IndexValue, uint64_t &AccessIndex); bool transformGEPChain(Module &M, CallInst *Call, uint32_t Kind); @@ -287,7 +292,8 @@ // . addr = preserve_union_access_index(base, di_index) // is transformed to // addr = base, i.e., all usages of "addr" are replaced by "base". - // . addr = preserve_struct_access_index(base, gep_index, di_index) + // . addr = preserve_struct_access_index(base, gep_index, di_index, + // preserved_index) // is transformed to // addr = GEP(base, 0, gep_index) replaceWithGEP(PreserveArrayIndexCalls, 1, 2); @@ -480,6 +486,7 @@ /// string, which will be the name of a global variable. Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call, std::string &AccessKey, + uint32_t &DIFieldIndex, uint32_t Kind, MDNode *&TypeMeta) { Value *Base = nullptr; @@ -586,7 +593,14 @@ // Access Index uint64_t AccessIndex; - uint32_t ArgIndex = (Kind == BPFPreserveUnionAI) ? 1 : 2; + uint32_t ArgIndex; + if (Kind == BPFPreserveUnionAI) + ArgIndex = 1; + else if (Kind == BPFPreserveStructAI) + ArgIndex = 3; + else + ArgIndex = 2; + if (!getAccessIndex(Call->getArgOperand(ArgIndex), AccessIndex)) return nullptr; AccessKey += ":" + std::to_string(AccessIndex); @@ -603,6 +617,18 @@ AccessOffset += AccessIndex * calcArraySize(CTy, 1) * EltTy->getSizeInBits() >> 3; } + + if (Kind == BPFPreserveStructAI) { + ArgIndex = 2; + uint64_t DbgFieldIndex; + if (!getAccessIndex(Call->getArgOperand(ArgIndex), DbgFieldIndex)) + return nullptr; + if (DbgFieldIndex != AccessIndex) { + // A bitfield access which is not the leading bitfield in + // the bitfield group. + DIFieldIndex = DbgFieldIndex; + } + } } // Access key is the type name + access string, uniquely identifying @@ -618,8 +644,9 @@ uint32_t Kind) { std::string AccessKey; MDNode *TypeMeta; + uint32_t DIFieldIndex = 0; Value *Base = - computeBaseAndAccessKey(Call, AccessKey, Kind, TypeMeta); + computeBaseAndAccessKey(Call, AccessKey, DIFieldIndex, Kind, TypeMeta); if (!Base) return false; @@ -645,6 +672,8 @@ } else { GV = GEPGlobals[AccessKey]; } + if (DIFieldIndex) + BitfieldAccesses[GV].insert(DIFieldIndex); // Load the global variable. auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV); @@ -682,5 +711,15 @@ Transformed = transformGEPChain(M, C.first, C.second) || Transformed; } + // Set bitfield attributes for globals. + for (auto &B : BitfieldAccesses) { + GlobalVariable *GV = B.first; + std::string AttrStr; + for (auto &DIFieldIndex : B.second) + AttrStr += std::to_string(DIFieldIndex) + "$"; + AttrStr.pop_back(); + GV->addAttribute(BPFCoreSharedInfo::BitFieldAttr, AttrStr); + } + return removePreserveAccessIndexIntrinsic(M) || Transformed; } diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -15,6 +15,8 @@ public: /// The attribute attached to globals representing a member offset static const std::string AmaAttr; + /// The attribute attached to globals representing dbg bitfield indices + static const std::string BitFieldAttr; /// The section name to identify a patchable external global static const std::string PatchableExtSecName; }; diff --git a/llvm/lib/Target/BPF/BTF.h b/llvm/lib/Target/BPF/BTF.h --- a/llvm/lib/Target/BPF/BTF.h +++ b/llvm/lib/Target/BPF/BTF.h @@ -76,7 +76,7 @@ SecExternRelocSize = 8, BPFFuncInfoSize = 8, BPFLineInfoSize = 16, - BPFOffsetRelocSize = 12, + BPFOffsetRelocSize = 16, BPFExternRelocSize = 8, }; @@ -248,9 +248,10 @@ /// Specifying one offset relocation. struct BPFOffsetReloc { - uint32_t InsnOffset; ///< Byte offset in this section - uint32_t TypeID; ///< TypeID for the relocation - uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t InsnOffset; ///< Byte offset in this section + uint32_t TypeID; ///< TypeID for the relocation + uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t BitfieldMembers; ///< The string to encode all accessed bitfield members }; /// Specifying offset relocation's in one section. diff --git a/llvm/lib/Target/BPF/BTFDebug.h b/llvm/lib/Target/BPF/BTFDebug.h --- a/llvm/lib/Target/BPF/BTFDebug.h +++ b/llvm/lib/Target/BPF/BTFDebug.h @@ -225,9 +225,10 @@ /// Represent one offset relocation. struct BTFOffsetReloc { - const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc - uint32_t TypeID; ///< Type ID - uint32_t OffsetNameOff; ///< The string to traverse types + const MCSymbol *Label; ///< MCSymbol identifying insn for the reloc + uint32_t TypeID; ///< Type ID + uint32_t OffsetNameOff; ///< The string to traverse types + uint32_t BitfieldMembers; ///< The string to encode all accessed bitfield members }; /// Represent one extern relocation. @@ -301,7 +302,8 @@ /// Generate one offset relocation record. void generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym, - DIType *RootTy, StringRef AccessPattern); + DIType *RootTy, StringRef AccessPattern, + StringRef BitFieldAccesses); /// Populating unprocessed struct type. unsigned populateStructType(const DIType *Ty); diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp --- a/llvm/lib/Target/BPF/BTFDebug.cpp +++ b/llvm/lib/Target/BPF/BTFDebug.cpp @@ -850,6 +850,7 @@ Asm->EmitLabelReference(OffsetRelocInfo.Label, 4); OS.EmitIntValue(OffsetRelocInfo.TypeID, 4); OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4); + OS.EmitIntValue(OffsetRelocInfo.BitfieldMembers, 4); } } } @@ -967,7 +968,8 @@ /// Generate a struct member offset relocation. void BTFDebug::generateOffsetReloc(const MachineInstr *MI, const MCSymbol *ORSym, DIType *RootTy, - StringRef AccessPattern) { + StringRef AccessPattern, + StringRef BitFieldAccesses) { unsigned RootId = populateStructType(RootTy); size_t FirstDollar = AccessPattern.find_first_of('$'); size_t FirstColon = AccessPattern.find_first_of(':'); @@ -979,6 +981,7 @@ OffsetReloc.Label = ORSym; OffsetReloc.OffsetNameOff = addString(IndexPattern); OffsetReloc.TypeID = RootId; + OffsetReloc.BitfieldMembers = addString(BitFieldAccesses); AccessOffsets[AccessPattern.str()] = std::stoi(OffsetStr); OffsetRelocTable[SecNameOff].push_back(OffsetReloc); } @@ -1017,9 +1020,13 @@ MCSymbol *ORSym = OS.getContext().createTempSymbol(); OS.EmitLabel(ORSym); + StringRef BitFieldAccesses; + BitFieldAccesses = + GVar->getAttribute(BPFCoreSharedInfo::BitFieldAttr).getValueAsString(); + MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index); DIType *Ty = dyn_cast(MDN); - generateOffsetReloc(MI, ORSym, Ty, GVar->getName()); + generateOffsetReloc(MI, ORSym, Ty, GVar->getName(), BitFieldAccesses); } else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() && GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) { MCSymbol *ORSym = OS.getContext().createTempSymbol(); diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-bitfield.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-bitfield.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-bitfield.ll @@ -0,0 +1,89 @@ +; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s +; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s +; Source code: +; struct s { +; int a; +; int b1:8; +; int b2:3; +; int b3:4; +; }; +; #define _(x) __builtin_preserve_access_index(x) +; int test(struct s *arg) { return _(arg->b2 + arg->b3); } +; Compilation flag: +; clang -target bpf -O2 -g -S -emit-llvm test.c + +%struct.s = type { i32, i16 } + +; Function Attrs: nounwind readonly +define dso_local i32 @test(%struct.s* readonly %arg) local_unnamed_addr #0 !dbg !7 { +entry: + call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !19, metadata !DIExpression()), !dbg !20 + %0 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 2, i32 1), !dbg !21, !llvm.preserve.access.index !12 + %bf.load = load i16, i16* %0, align 4, !dbg !21 + %bf.shl = shl i16 %bf.load, 5, !dbg !21 + %bf.ashr = ashr i16 %bf.shl, 13, !dbg !21 + %1 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 3, i32 1), !dbg !21, !llvm.preserve.access.index !12 + %bf.load1 = load i16, i16* %1, align 4, !dbg !21 + %bf.shl2 = shl i16 %bf.load1, 1, !dbg !21 + %bf.ashr3 = ashr i16 %bf.shl2, 12, !dbg !21 + %narrow = add nsw i16 %bf.ashr3, %bf.ashr, !dbg !21 + %add = sext i16 %narrow to i32, !dbg !21 + ret i32 %add, !dbg !22 +} + +; CHECK-LABEL: test +; CHECK: r2 = 4 +; CHECK: r1 += r2 +; +; CHECK: .long 1 # BTF_KIND_STRUCT(id = 2) +; +; CHECK: .byte 115 # string offset=1 +; CHECK: .ascii ".text" # string offset=27 +; CHECK: .ascii "0:1" # string offset=33 +; CHECK: .ascii "2$3" # string offset=37 + +; CHECK: .long 16 # OffsetReloc +; CHECK-NEXT: .long 27 # Offset reloc section string offset=27 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long .Ltmp{{[0-9]+}} +; CHECK-NEXT: .long 2 +; CHECK-NEXT: .long 33 +; CHECK-NEXT: .long 37 + +; Function Attrs: nounwind readnone +declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s*, i32, i32, i32) #1 + +; Function Attrs: nounwind readnone speculatable willreturn +declare void @llvm.dbg.value(metadata, metadata, metadata) #2 + +attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind readnone speculatable willreturn } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git d07e49be1b25894d97cc5e6f73068ac80a55a977)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git d07e49be1b25894d97cc5e6f73068ac80a55a977)"} +!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !18) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !11} +!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64) +!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 64, elements: !13) +!13 = !{!14, !15, !16, !17} +!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !12, file: !1, line: 2, baseType: !10, size: 32) +!15 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !12, file: !1, line: 3, baseType: !10, size: 8, offset: 32, flags: DIFlagBitField, extraData: i64 32) +!16 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !12, file: !1, line: 4, baseType: !10, size: 3, offset: 40, flags: DIFlagBitField, extraData: i64 32) +!17 = !DIDerivedType(tag: DW_TAG_member, name: "b3", scope: !12, file: !1, line: 5, baseType: !10, size: 4, offset: 43, flags: DIFlagBitField, extraData: i64 32) +!18 = !{!19} +!19 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 8, type: !11) +!20 = !DILocation(line: 0, scope: !7) +!21 = !DILocation(line: 8, column: 34, scope: !7) +!22 = !DILocation(line: 8, column: 27, scope: !7) diff --git a/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll b/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll --- a/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll +++ b/llvm/test/CodeGen/BPF/CORE/intrinsic-struct.ll @@ -14,7 +14,7 @@ define dso_local i32 @test(%struct.s* %arg) local_unnamed_addr #0 !dbg !7 { entry: call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !17, metadata !DIExpression()), !dbg !18 - %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1), !dbg !19, !llvm.preserve.access.index !12 + %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1, i32 1), !dbg !19, !llvm.preserve.access.index !12 %1 = bitcast i32* %0 to i8*, !dbg !19 %call = tail call i32 @get_value(i8* %1) #4, !dbg !20 ret i32 %call, !dbg !21 @@ -28,17 +28,18 @@ ; CHECK: exit ; ; CHECK: .section .BTF.ext,"",@progbits -; CHECK: .long 12 # OffsetReloc +; CHECK: .long 16 # OffsetReloc ; CHECK-NEXT: .long 20 # Offset reloc section string offset=20 ; CHECK-NEXT: .long 1 ; CHECK-NEXT: .long [[RELOC]] ; CHECK-NEXT: .long 2 ; CHECK-NEXT: .long 26 +; CHECK-NEXT: .long 0 declare dso_local i32 @get_value(i8*) local_unnamed_addr #1 ; Function Attrs: nounwind readnone -declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32 immarg, i32 immarg) #2 +declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32 immarg, i32 immarg, i32 immarg) #2 ; Function Attrs: nounwind readnone speculatable declare void @llvm.dbg.value(metadata, metadata, metadata) #3 diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-bitfield.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-bitfield.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-bitfield.ll @@ -0,0 +1,112 @@ +; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s +; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s +; Source code: +; struct s { +; int a; +; int b1:8; +; int b2:3; +; int b3:4; +; }; +; +; #define _(x, y) __builtin_preserve_bitfield_info((x), (y)) +; void process(void *); +; unsigned test(struct s *arg) { +; unsigned info1, info2; +; void *p1, *p2; +; +; p1 = _(arg->b2, &info1); +; p2 = _(arg->b3, &info2); +; process(p1); +; process(p2); +; // the last 16 bit is the offset, so +; // return value should be 8 + 11 = 19 +; // At -O2, compiler should do this optimization. +; return (info1 & 0xffff) + (info2 & 0xffff); +; } + +%struct.s = type { i32, i16 } + +; Function Attrs: nounwind +define dso_local i32 @test(%struct.s* %arg) local_unnamed_addr #0 !dbg !7 { +entry: + call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !20, metadata !DIExpression()), !dbg !26 + %0 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 2, i32 1), !dbg !27, !llvm.preserve.access.index !12 + %1 = bitcast i16* %0 to i8*, !dbg !27 + call void @llvm.dbg.value(metadata i32 -2147287032, metadata !21, metadata !DIExpression()), !dbg !26 + call void @llvm.dbg.value(metadata i8* %1, metadata !23, metadata !DIExpression()), !dbg !26 + %2 = tail call i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s* %arg, i32 1, i32 3, i32 1), !dbg !28, !llvm.preserve.access.index !12 + %3 = bitcast i16* %2 to i8*, !dbg !28 + call void @llvm.dbg.value(metadata i32 -2147221493, metadata !22, metadata !DIExpression()), !dbg !26 + call void @llvm.dbg.value(metadata i8* %3, metadata !25, metadata !DIExpression()), !dbg !26 + tail call void @process(i8* %1) #4, !dbg !29 + tail call void @process(i8* %3) #4, !dbg !30 + ret i32 19, !dbg !31 +} + +; CHECK-LABEL: test +; CHECK: r1 = 4 +; CHECK: r{{[0-9]+}} += r1 +; CHECK: r0 = 19 +; CHECK: exit +; +; CHECK: .ascii ".text" # string offset=40 +; CHECK: .ascii "0:1" # string offset=83 +; CHECK: .ascii "2$3" # string offset=87 +; CHECK: .long 16 # OffsetReloc +; CHECK-NEXT: .long 40 # Offset reloc section string offset=40 +; CHECK-NEXT: .long 1 +; CHECK-NEXT: .long .Ltmp{{[0-9]+}} +; CHECK-NEXT: .long 2 +; CHECK-NEXT: .long 83 +; CHECK-NEXT: .long 87 + +; Function Attrs: nounwind readnone +declare i16* @llvm.preserve.struct.access.index.p0i16.p0s_struct.ss(%struct.s*, i32, i32, i32) #1 + +declare dso_local void @process(i8*) local_unnamed_addr #2 + +; Function Attrs: nounwind readnone speculatable willreturn +declare void @llvm.dbg.value(metadata, metadata, metadata) #3 + +attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { nounwind readnone } +attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind readnone speculatable willreturn } +attributes #4 = { nounwind } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!3, !4, !5} +!llvm.ident = !{!6} + +!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git d07e49be1b25894d97cc5e6f73068ac80a55a977)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core") +!2 = !{} +!3 = !{i32 2, !"Dwarf Version", i32 4} +!4 = !{i32 2, !"Debug Info Version", i32 3} +!5 = !{i32 1, !"wchar_size", i32 4} +!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git d07e49be1b25894d97cc5e6f73068ac80a55a977)"} +!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 10, type: !8, scopeLine: 10, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !19) +!8 = !DISubroutineType(types: !9) +!9 = !{!10, !11} +!10 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned) +!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64) +!12 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 64, elements: !13) +!13 = !{!14, !16, !17, !18} +!14 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !12, file: !1, line: 2, baseType: !15, size: 32) +!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!16 = !DIDerivedType(tag: DW_TAG_member, name: "b1", scope: !12, file: !1, line: 3, baseType: !15, size: 8, offset: 32, flags: DIFlagBitField, extraData: i64 32) +!17 = !DIDerivedType(tag: DW_TAG_member, name: "b2", scope: !12, file: !1, line: 4, baseType: !15, size: 3, offset: 40, flags: DIFlagBitField, extraData: i64 32) +!18 = !DIDerivedType(tag: DW_TAG_member, name: "b3", scope: !12, file: !1, line: 5, baseType: !15, size: 4, offset: 43, flags: DIFlagBitField, extraData: i64 32) +!19 = !{!20, !21, !22, !23, !25} +!20 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 10, type: !11) +!21 = !DILocalVariable(name: "info1", scope: !7, file: !1, line: 11, type: !10) +!22 = !DILocalVariable(name: "info2", scope: !7, file: !1, line: 11, type: !10) +!23 = !DILocalVariable(name: "p1", scope: !7, file: !1, line: 12, type: !24) +!24 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64) +!25 = !DILocalVariable(name: "p2", scope: !7, file: !1, line: 12, type: !24) +!26 = !DILocation(line: 0, scope: !7) +!27 = !DILocation(line: 14, column: 8, scope: !7) +!28 = !DILocation(line: 15, column: 8, scope: !7) +!29 = !DILocation(line: 16, column: 3, scope: !7) +!30 = !DILocation(line: 17, column: 3, scope: !7) +!31 = !DILocation(line: 21, column: 3, scope: !7)