diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -1956,6 +1956,14 @@ let LangOpts = [COnly]; } +def BPFPreserveStaticOffset : InheritableAttr, + TargetSpecificAttr { + let Spellings = [Clang<"preserve_static_offset">]; + let Subjects = SubjectList<[Record], ErrorDiag>; + let Documentation = [BPFPreserveStaticOffsetDocs]; + let LangOpts = [COnly]; +} + def BTFDeclTag : InheritableAttr { let Spellings = [Clang<"btf_decl_tag">]; let Args = [StringArgument<"BTFDeclTag">]; diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -2195,6 +2195,43 @@ struct or union, similar to clang ``__builtin_preserve_access_index()``. }]; } + +def BPFPreserveStaticOffsetDocs : Documentation { + let Category = DocCatFunction; + let Content = [{ +Clang supports the ``__attribute__((preserve_static_offset))`` +attribute for the BPF target. This attribute may be attached to a +struct or union declaration. Reading or writing fields of types having +such annotation is guaranteed to generate LDX/ST/STX instruction with +offset corresponding to the field. + +For example: + +.. code-block:: c + + struct foo { + int a; + int b; + }; + + struct bar { + int a; + struct foo b; + } __attribute__((preserve_static_offset)); + + void buz(struct bar *g) { + g->a.b = 42; + } + +The assignment to ``g``'s field would produce an ST instruction with +offset 8: ``*(u32)(r1 + 8) = 42;``. + +Without this attribute generated instructions might be different, +depending on optimizations behavior. E.g. the example above could be +rewritten as ``r1 += 8; *(u32)(r1 + 0) = 42;``. + }]; +} + def BTFDeclTagDocs : Documentation { let Category = DocCatFunction; let Content = [{ diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp --- a/clang/lib/CodeGen/CGExpr.cpp +++ b/clang/lib/CodeGen/CGExpr.cpp @@ -3690,6 +3690,30 @@ return eltType; } +static bool hasBPFPreserveStaticOffset(const RecordDecl *D) { + return D->getAttr(); +} + +static bool hasBPFPreserveStaticOffset(const Expr *E) { + if (auto *PtrType = dyn_cast(E->getType())) + if (auto *BaseDecl = PtrType->getPointeeType()->getAsRecordDecl()) + return hasBPFPreserveStaticOffset(BaseDecl); + + return false; +} + +// Wraps Addr with a call to llvm.preserve.static.offset intrinsic. +static Address wrapWithBPFPreserveStaticOffset(CodeGenFunction &CGF, + Address &Addr) { + if (!CGF.getTarget().getTriple().isBPF()) + return Addr; + + llvm::Function *Fn = + CGF.CGM.getIntrinsic(llvm::Intrinsic::preserve_static_offset); + llvm::CallInst *Call = CGF.Builder.CreateCall(Fn, {Addr.getPointer()}); + return Address(Call, Addr.getElementType(), Addr.getAlignment()); +} + /// Given an array base, check whether its member access belongs to a record /// with preserve_access_index attribute or not. static bool IsPreserveAIArrayBase(CodeGenFunction &CGF, const Expr *ArrayBase) { @@ -3751,6 +3775,9 @@ CharUnits eltAlign = getArrayElementAlign(addr.getAlignment(), indices.back(), eltSize); + if (Base && hasBPFPreserveStaticOffset(Base)) + addr = wrapWithBPFPreserveStaticOffset(CGF, addr); + llvm::Value *eltPtr; auto LastIndex = dyn_cast(indices.back()); if (!LastIndex || @@ -4358,6 +4385,8 @@ Address Addr = base.getAddress(*this); unsigned Idx = RL.getLLVMFieldNo(field); const RecordDecl *rec = field->getParent(); + if (hasBPFPreserveStaticOffset(rec)) + Addr = wrapWithBPFPreserveStaticOffset(*this, Addr); if (!UseVolatile) { if (!IsInPreservedAIRegion && (!getDebugInfo() || !rec->hasAttr())) { @@ -4430,6 +4459,8 @@ } Address addr = base.getAddress(*this); + if (hasBPFPreserveStaticOffset(rec)) + addr = wrapWithBPFPreserveStaticOffset(*this, addr); if (auto *ClassDef = dyn_cast(rec)) { if (CGM.getCodeGenOpts().StrictVTablePointers && ClassDef->isDynamicClass()) { diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -7595,6 +7595,12 @@ Rec->addAttr(::new (S.Context) BPFPreserveAccessIndexAttr(S.Context, AL)); } +static void handleBPFPreserveStaticOffsetAttr(Sema &S, Decl *D, + const ParsedAttr &AL) { + auto *Rec = cast(D); + Rec->addAttr(::new (S.Context) BPFPreserveStaticOffsetAttr(S.Context, AL)); +} + static bool hasBTFDeclTagAttr(Decl *D, StringRef Tag) { for (const auto *I : D->specific_attrs()) { if (I->getBTFDeclTag() == Tag) @@ -8861,6 +8867,9 @@ case ParsedAttr::AT_BPFPreserveAccessIndex: handleBPFPreserveAccessIndexAttr(S, D, AL); break; + case ParsedAttr::AT_BPFPreserveStaticOffset: + handleBPFPreserveStaticOffsetAttr(S, D, AL); + break; case ParsedAttr::AT_BTFDeclTag: handleBTFDeclTagAttr(S, D, AL); break; diff --git a/clang/test/CodeGen/bpf-preserve-static-offset-arr.c b/clang/test/CodeGen/bpf-preserve-static-offset-arr.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/bpf-preserve-static-offset-arr.c @@ -0,0 +1,33 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2 +// REQUIRES: bpf-registered-target +// RUN: %clang -cc1 -triple bpf -disable-llvm-passes -S -emit-llvm -o - %s \ +// RUN: | FileCheck %s + +// Check that call to preserve.static.offset is generated when array +// member of a struct marked with __attribute__((preserve_static_offset)) +// is accessed. + +#define __ctx __attribute__((preserve_static_offset)) + +struct foo { + struct { + int a; + } b[7]; +} __ctx; + +// CHECK-LABEL: define dso_local i32 @arr_access +// CHECK-SAME: (ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call ptr @llvm.preserve.static.offset(ptr [[TMP0]]) +// CHECK-NEXT: [[B:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr [[TMP1]], i32 0, i32 0 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds [7 x %struct.anon], ptr [[B]], i64 0, i64 2 +// CHECK-NEXT: [[A:%.*]] = getelementptr inbounds [[STRUCT_ANON:%.*]], ptr [[ARRAYIDX]], i32 0, i32 0 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[A]], align 4 +// CHECK-NEXT: ret i32 [[TMP2]] +// +int arr_access(struct foo *p) { + return p->b[2].a; +} diff --git a/clang/test/CodeGen/bpf-preserve-static-offset-bitfield.c b/clang/test/CodeGen/bpf-preserve-static-offset-bitfield.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/bpf-preserve-static-offset-bitfield.c @@ -0,0 +1,31 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2 +// REQUIRES: bpf-registered-target +// RUN: %clang -cc1 -triple bpf -disable-llvm-passes -S -emit-llvm -o - %s \ +// RUN: | FileCheck %s + +// Check that call to preserve.static.offset is generated when bitfield +// from a struct marked with __attribute__((preserve_static_offset)) is +// accessed. + +#define __ctx __attribute__((preserve_static_offset)) + +struct foo { + unsigned a:1; +} __ctx; + +// CHECK-LABEL: define dso_local void @lvalue_bitfield +// CHECK-SAME: (ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call ptr @llvm.preserve.static.offset(ptr [[TMP0]]) +// CHECK-NEXT: [[BF_LOAD:%.*]] = load i8, ptr [[TMP1]], align 4 +// CHECK-NEXT: [[BF_CLEAR:%.*]] = and i8 [[BF_LOAD]], -2 +// CHECK-NEXT: [[BF_SET:%.*]] = or i8 [[BF_CLEAR]], 1 +// CHECK-NEXT: store i8 [[BF_SET]], ptr [[TMP1]], align 4 +// CHECK-NEXT: ret void +// +void lvalue_bitfield(struct foo *p) { + p->a = 1; +} diff --git a/clang/test/CodeGen/bpf-preserve-static-offset-lvalue.c b/clang/test/CodeGen/bpf-preserve-static-offset-lvalue.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/bpf-preserve-static-offset-lvalue.c @@ -0,0 +1,28 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2 +// REQUIRES: bpf-registered-target +// RUN: %clang -cc1 -triple bpf -disable-llvm-passes -S -emit-llvm -o - %s \ +// RUN: | FileCheck %s + +// Check that call to preserve.static.offset is generated when field of +// a struct marked with __attribute__((preserve_static_offset)) is accessed. + +#define __ctx __attribute__((preserve_static_offset)) + +struct foo { + int a; +} __ctx; + +// CHECK-LABEL: define dso_local void @lvalue +// CHECK-SAME: (ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call ptr @llvm.preserve.static.offset(ptr [[TMP0]]) +// CHECK-NEXT: [[A:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr [[TMP1]], i32 0, i32 0 +// CHECK-NEXT: store i32 42, ptr [[A]], align 4 +// CHECK-NEXT: ret void +// +void lvalue(struct foo *p) { + p->a = 42; +} diff --git a/clang/test/CodeGen/bpf-preserve-static-offset-non-bpf.c b/clang/test/CodeGen/bpf-preserve-static-offset-non-bpf.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/bpf-preserve-static-offset-non-bpf.c @@ -0,0 +1,18 @@ +// REQUIRES: x86-registered-target +// RUN: %clang -cc1 -triple x86_64 -disable-llvm-passes -S -emit-llvm -o - %s \ +// RUN: | FileCheck %s + +// Verify that __attribute__((preserve_static_offset)) +// has no effect for non-BPF target. + +#define __ctx __attribute__((preserve_static_offset)) + +struct foo { + int a; +} __ctx; + +// CHECK-NOT: @llvm_preserve_static_offset + +int bar(struct foo *p) { + return p->a; +} diff --git a/clang/test/CodeGen/bpf-preserve-static-offset-pai.c b/clang/test/CodeGen/bpf-preserve-static-offset-pai.c new file mode 100644 --- /dev/null +++ b/clang/test/CodeGen/bpf-preserve-static-offset-pai.c @@ -0,0 +1,29 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 2 +// REQUIRES: bpf-registered-target +// RUN: %clang -cc1 -triple bpf -disable-llvm-passes -S -emit-llvm -o - %s \ +// RUN: | FileCheck %s + +// Verify that preserve_static_offset does not interfere with +// preserve_access_index at IR generation stage. + +#define __ctx __attribute__((preserve_static_offset)) +#define __pai __attribute__((preserve_access_index)) + +struct foo { + int a; +} __ctx __pai; + +// CHECK-LABEL: define dso_local i32 @bar +// CHECK-SAME: (ptr noundef [[P:%.*]]) #[[ATTR0:[0-9]+]] { +// CHECK-NEXT: entry: +// CHECK-NEXT: [[P_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: store ptr [[P]], ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[P_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = call ptr @llvm.preserve.static.offset(ptr [[TMP0]]) +// CHECK-NEXT: [[A:%.*]] = getelementptr inbounds [[STRUCT_FOO:%.*]], ptr [[TMP1]], i32 0, i32 0 +// CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[A]], align 4 +// CHECK-NEXT: ret i32 [[TMP2]] +// +int bar(struct foo *p) { + return p->a; +} diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -23,6 +23,7 @@ // CHECK-NEXT: Availability ((SubjectMatchRule_record, SubjectMatchRule_enum, SubjectMatchRule_enum_constant, SubjectMatchRule_field, SubjectMatchRule_function, SubjectMatchRule_namespace, SubjectMatchRule_objc_category, SubjectMatchRule_objc_implementation, SubjectMatchRule_objc_interface, SubjectMatchRule_objc_method, SubjectMatchRule_objc_property, SubjectMatchRule_objc_protocol, SubjectMatchRule_record, SubjectMatchRule_type_alias, SubjectMatchRule_variable)) // CHECK-NEXT: AvailableOnlyInDefaultEvalMethod (SubjectMatchRule_type_alias) // CHECK-NEXT: BPFPreserveAccessIndex (SubjectMatchRule_record) +// CHECK-NEXT: BPFPreserveStaticOffset (SubjectMatchRule_record) // CHECK-NEXT: BTFDeclTag (SubjectMatchRule_variable, SubjectMatchRule_function, SubjectMatchRule_record, SubjectMatchRule_field, SubjectMatchRule_type_alias) // CHECK-NEXT: BuiltinAlias (SubjectMatchRule_function) // CHECK-NEXT: CFAuditedTransfer (SubjectMatchRule_function) diff --git a/clang/test/Sema/bpf-attr-preserve-static-offset.c b/clang/test/Sema/bpf-attr-preserve-static-offset.c new file mode 100644 --- /dev/null +++ b/clang/test/Sema/bpf-attr-preserve-static-offset.c @@ -0,0 +1,27 @@ +// RUN: %clang_cc1 -fsyntax-only -ast-dump -triple bpf-pc-linux-gnu %s | FileCheck %s + +// The 'preserve_static_offset' attribute should be propagated to +// inline declarations (foo's 'b', 'bb', 'c' but not 'd'). +// +// CHECK: RecordDecl {{.*}} struct foo definition +// CHECK-NEXT: BPFPreserveStaticOffsetAttr +// CHECK-NEXT: FieldDecl {{.*}} a +// CHECK-NEXT: RecordDecl {{.*}} struct definition +// CHECK-NEXT: FieldDecl {{.*}} aa +// CHECK-NEXT: FieldDecl {{.*}} b +// CHECK-NEXT: RecordDecl {{.*}} union bar definition +// CHECK-NEXT: BPFPreserveStaticOffsetAttr +// CHECK-NEXT: FieldDecl {{.*}} a +// CHECK-NEXT: FieldDecl {{.*}} b + +struct foo { + int a; + struct { + int aa; + } b; +} __attribute__((preserve_static_offset)); + +union bar { + int a; + long b; +} __attribute__((preserve_static_offset)); diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2441,6 +2441,10 @@ [IntrNoMem, ImmArg>, ImmArg>]>; +def int_preserve_static_offset : DefaultAttrsIntrinsic<[llvm_ptr_ty], + [llvm_ptr_ty], + [IntrNoMem, IntrSpeculatable, + ReadNone >]>; //===------------ Intrinsics to perform common vector shuffles ------------===// diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -37,4 +37,43 @@ def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">, Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty], [IntrNoMem]>; + def int_bpf_getelementptr_and_load : ClangBuiltin<"__builtin_bpf_getelementptr_and_load">, + Intrinsic<[llvm_any_ty], + [llvm_ptr_ty, // base ptr for getelementptr + llvm_i1_ty, // volatile + llvm_i8_ty, // atomic order + llvm_i8_ty, // synscope id + llvm_i8_ty, // alignment + llvm_i1_ty, // inbounds + llvm_vararg_ty], // indices for getelementptr insn + [IntrNoCallback, + IntrNoFree, + IntrWillReturn, + NoCapture >, + ImmArg >, // volatile + ImmArg >, // atomic order + ImmArg >, // synscope id + ImmArg >, // alignment + ImmArg >, // inbounds + ]>; + def int_bpf_getelementptr_and_store : ClangBuiltin<"__builtin_bpf_getelementptr_and_store">, + Intrinsic<[], + [llvm_any_ty, // value to store + llvm_ptr_ty, // base ptr for getelementptr + llvm_i1_ty, // volatile + llvm_i8_ty, // atomic order + llvm_i8_ty, // syncscope id + llvm_i8_ty, // alignment + llvm_i1_ty, // inbounds + llvm_vararg_ty], // indexes for getelementptr insn + [IntrNoCallback, + IntrNoFree, + IntrWillReturn, + NoCapture >, + ImmArg >, // volatile + ImmArg >, // atomic order + ImmArg >, // syncscope id + ImmArg >, // alignment + ImmArg >, // inbounds + ]>; } diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_BPF_BPF_H #include "MCTargetDesc/BPFMCTargetDesc.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" #include "llvm/Target/TargetMachine.h" @@ -62,6 +63,24 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; + +class BPFPreserveStaticOffsetPass + : public PassInfoMixin { + bool AllowPartial; + +public: + BPFPreserveStaticOffsetPass(bool AllowPartial) : AllowPartial(AllowPartial) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } + + static std::pair + reconstructLoad(CallInst *Call); + + static std::pair + reconstructStore(CallInst *Call); +}; + } // namespace llvm #endif diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -170,8 +170,6 @@ bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, const MDNode *ChildMeta); bool removePreserveAccessIndexIntrinsic(Function &F); - void replaceWithGEP(std::vector &CallList, - uint32_t NumOfZerosIndex, uint32_t DIIndex); bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment, uint32_t &StartBitOffset, uint32_t &EndBitOffset); @@ -183,7 +181,6 @@ std::string &AccessKey, MDNode *&BaseMeta); MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, std::string &AccessKey, bool &IsInt32Ret); - uint64_t getConstant(const Value *IndexValue); bool transformGEPChain(CallInst *Call, CallInfo &CInfo); }; @@ -324,6 +321,12 @@ return Call->getParamElementType(0); } +static uint64_t getConstant(const Value *IndexValue) { + const ConstantInt *CV = dyn_cast(IndexValue); + assert(CV); + return CV->getValue().getZExtValue(); +} + /// Check whether a call is a preserve_*_access_index intrinsic call or not. bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &CInfo) { @@ -408,26 +411,36 @@ return false; } -void BPFAbstractMemberAccess::replaceWithGEP(std::vector &CallList, - uint32_t DimensionIndex, - uint32_t GEPIndex) { - for (auto *Call : CallList) { - uint32_t Dimension = 1; - if (DimensionIndex > 0) - Dimension = getConstant(Call->getArgOperand(DimensionIndex)); - - Constant *Zero = - ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); - SmallVector IdxList; - for (unsigned I = 0; I < Dimension; ++I) - IdxList.push_back(Zero); - IdxList.push_back(Call->getArgOperand(GEPIndex)); - - auto *GEP = GetElementPtrInst::CreateInBounds( - getBaseElementType(Call), Call->getArgOperand(0), IdxList, "", Call); - Call->replaceAllUsesWith(GEP); - Call->eraseFromParent(); - } +static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex, + uint32_t GEPIndex) { + uint32_t Dimension = 1; + if (DimensionIndex > 0) + Dimension = getConstant(Call->getArgOperand(DimensionIndex)); + + Constant *Zero = + ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); + SmallVector IdxList; + for (unsigned I = 0; I < Dimension; ++I) + IdxList.push_back(Zero); + IdxList.push_back(Call->getArgOperand(GEPIndex)); + + auto *GEP = GetElementPtrInst::CreateInBounds( + getBaseElementType(Call), Call->getArgOperand(0), IdxList, "", Call); + Call->replaceAllUsesWith(GEP); + Call->eraseFromParent(); +} + +void BPFCoreSharedInfo::removeArrayAccessCall(CallInst *Call) { + replaceWithGEP(Call, 1, 2); +} + +void BPFCoreSharedInfo::removeStructAccessCall(CallInst *Call) { + replaceWithGEP(Call, 0, 1); +} + +void BPFCoreSharedInfo::removeUnionAccessCall(CallInst *Call) { + Call->replaceAllUsesWith(Call->getArgOperand(0)); + Call->eraseFromParent(); } bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { @@ -462,12 +475,12 @@ // . addr = preserve_struct_access_index(base, gep_index, di_index) // is transformed to // addr = GEP(base, 0, gep_index) - replaceWithGEP(PreserveArrayIndexCalls, 1, 2); - replaceWithGEP(PreserveStructIndexCalls, 0, 1); - for (auto *Call : PreserveUnionIndexCalls) { - Call->replaceAllUsesWith(Call->getArgOperand(0)); - Call->eraseFromParent(); - } + for (CallInst *Call : PreserveArrayIndexCalls) + BPFCoreSharedInfo::removeArrayAccessCall(Call); + for (CallInst *Call : PreserveStructIndexCalls) + BPFCoreSharedInfo::removeStructAccessCall(Call); + for (CallInst *Call : PreserveUnionIndexCalls) + BPFCoreSharedInfo::removeUnionAccessCall(Call); return Found; } @@ -632,12 +645,6 @@ } } -uint64_t BPFAbstractMemberAccess::getConstant(const Value *IndexValue) { - const ConstantInt *CV = dyn_cast(IndexValue); - assert(CV); - return CV->getValue().getZExtValue(); -} - /// Get the start and the end of storage offset for \p MemberTy. void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment, diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_BPF_BPFCORE_H #include "llvm/ADT/StringRef.h" +#include "llvm/IR/Instructions.h" namespace llvm { @@ -71,6 +72,9 @@ static Instruction *insertPassThrough(Module *M, BasicBlock *BB, Instruction *Input, Instruction *Before); + static void removeArrayAccessCall(CallInst *Call); + static void removeStructAccessCall(CallInst *Call); + static void removeUnionAccessCall(CallInst *Call); }; } // namespace llvm diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp --- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -12,6 +12,8 @@ // The following are done for IR adjustment: // - remove __builtin_bpf_passthrough builtins. Target independent IR // optimizations are done and those builtins can be removed. +// - remove llvm.bpf.getelementptr.and.load builtins. +// - remove llvm.bpf.getelementptr.and.store builtins. // //===----------------------------------------------------------------------===// @@ -24,6 +26,7 @@ #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicsBPF.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -51,6 +54,7 @@ bool removePassThroughBuiltin(Module &M); bool removeCompareBuiltin(Module &M); bool sinkMinMax(Module &M); + bool removeGEPBuiltins(Module &M); }; } // End anonymous namespace @@ -361,10 +365,62 @@ AU.addRequired(); } +static void unrollGEPLoad(CallInst *Call) { + auto [GEP, Load] = BPFPreserveStaticOffsetPass::reconstructLoad(Call); + GEP->insertBefore(Call); + Load->insertBefore(Call); + Call->replaceAllUsesWith(Load); + Call->eraseFromParent(); +} + +static void unrollGEPStore(CallInst *Call) { + auto [GEP, Store] = BPFPreserveStaticOffsetPass::reconstructStore(Call); + GEP->insertBefore(Call); + Store->insertBefore(Call); + Call->eraseFromParent(); +} + +static bool removeGEPBuiltinsInFunc(Function &F) { + SmallVector GEPLoads; + SmallVector GEPStores; + for (auto &BB : F) + for (auto &Insn : BB) + if (auto *Call = dyn_cast(&Insn)) + if (auto *Called = Call->getCalledFunction()) + switch (Called->getIntrinsicID()) { + case Intrinsic::bpf_getelementptr_and_load: + GEPLoads.push_back(Call); + break; + case Intrinsic::bpf_getelementptr_and_store: + GEPStores.push_back(Call); + break; + } + + if (GEPLoads.empty() && GEPStores.empty()) + return false; + + for_each(GEPLoads, unrollGEPLoad); + for_each(GEPStores, unrollGEPStore); + + return true; +} + +// Rewrites the following builtins: +// - llvm.bpf.getelementptr.and.load +// - llvm.bpf.getelementptr.and.store +// As (load (getelementptr ...)) or (store (getelementptr ...)). +bool BPFCheckAndAdjustIR::removeGEPBuiltins(Module &M) { + bool Changed = false; + for (auto &F : M) + Changed = removeGEPBuiltinsInFunc(F) || Changed; + return Changed; +} + bool BPFCheckAndAdjustIR::adjustIR(Module &M) { bool Changed = removePassThroughBuiltin(M); Changed = removeCompareBuiltin(M) || Changed; Changed = sinkMinMax(M) || Changed; + Changed = removeGEPBuiltins(M) || Changed; return Changed; } diff --git a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp @@ -0,0 +1,680 @@ +//===------ BPFPreserveStaticOffset.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TLDR: replaces llvm.preserve.static.offset + GEP + load / store +// with llvm.bpf.getelementptr.and.load / store +// +// This file implements BPFPreserveStaticOffsetPass transformation. +// This transformation address two BPF verifier specific issues: +// +// (a) Access to the fields of some structural types is allowed only +// using load and store instructions with static immediate offsets. +// +// Examples of such types are `struct __sk_buff` and `struct +// bpf_sock_ops`. This is so because offsets of the fields of +// these structures do not match real offsets in the running +// kernel. During BPF program load LDX and STX instructions +// referring to the fields of these types are rewritten so that +// offsets match real offsets. For this rewrite to happen field +// offsets have to be encoded as immediate operands of the +// instructions. +// +// See kernel/bpf/verifier.c:convert_ctx_access function in the +// Linux kernel source tree for details. +// +// (b) Pointers to context parameters of BPF programs must not be +// modified before access. +// +// During BPF program verification a tag PTR_TO_CTX is tracked for +// register values. In case if register with such tag is modified +// BPF program is not allowed to read or write memory using this +// register. See kernel/bpf/verifier.c:check_mem_access function +// in the Linux kernel source tree for details. +// +// The following sequence of the IR instructions: +// +// %x = getelementptr %ptr, %constant_offset +// %y = load %x +// +// Is translated as a single machine instruction: +// +// LDW %ptr, %constant_offset +// +// In order for cases (a) and (b) to work the sequence %x-%y above has +// to be preserved by the IR passes. +// +// However, several optimization passes might sink `load` instruction +// or hoist `getelementptr` instruction so that the instructions are +// no longer in sequence. Examples of such passes are: +// SimplifyCFGPass, InstCombinePass, GVNPass. +// After such modification the verifier would reject the BPF program. +// +// To avoid this issue the patterns like (load/store (getelementptr ...)) +// are replaced by calls to BPF specific intrinsic functions: +// - llvm.bpf.getelementptr.and.load +// - llvm.bpf.getelementptr.and.store +// +// These calls are lowered back to (load/store (getelementptr ...)) +// by BPFCheckAndAdjustIR pass right before the translation from IR to +// machine instructions. +// +// The transformation is split into the following steps: +// - When IR is generated from AST the calls to intrinsic function +// llvm.preserve.static.offset are inserted. +// - BPFPreserveStaticOffsetPass is executed as early as possible +// with AllowPatial set to true, this handles marked GEP chains +// with constant offsets. +// - BPFPreserveStaticOffsetPass is executed at ScalarOptimizerLateEPCallback +// with AllowPatial set to false, this handles marked GEP chains +// with offsets that became constant after loop unrolling, e.g. +// to handle the following code: +// +// struct context { int x[4]; } __attribute__((preserve_static_offset)); +// +// struct context *ctx = ...; +// #pragma clang loop unroll(full) +// for (int i = 0; i < 4; ++i) +// foo(ctx->x[i]); +// +// The early BPFPreserveStaticOffsetPass run is necessary to allow +// additional GVN / CSE opportunities after functions inlining. +// The relative order of optimization applied to function: +// - early stage (1) +// - ... +// - function inlining (2) +// - ... +// - loop unrolling +// - ... +// - ScalarOptimizerLateEPCallback (3) +// +// When function A is inlined into function B all optimizations for A +// are already done, while some passes remain for B. In case if +// BPFPreserveStaticOffsetPass is done at (3) but not done at (1) +// the code after (2) would contain a mix of +// (load (gep %p)) and (get.and.load %p) usages: +// - the (load (gep %p)) would come from the calling function; +// - the (get.and.load %p) would come from the callee function. +// Thus clobbering CSE / GVN passes done after inlining. + +#include "BPF.h" +#include "BPFCORE.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/Argument.h" +#include "llvm/IR/Attributes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/InstIterator.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" + +#define DEBUG_TYPE "bpf-preserve-static-offset" + +using namespace llvm; + +static const unsigned GepAndLoadFirstIdxArg = 6; +static const unsigned GepAndStoreFirstIdxArg = 7; + +static bool isIntrinsicCall(Value *I, Intrinsic::ID Id) { + if (auto *Call = dyn_cast(I)) + if (Function *Func = Call->getCalledFunction()) + return Func->getIntrinsicID() == Id; + return false; +} + +static bool isPreserveStaticOffsetCall(Value *I) { + return isIntrinsicCall(I, Intrinsic::preserve_static_offset); +} + +static CallInst *isGEPAndLoad(Value *I) { + if (isIntrinsicCall(I, Intrinsic::bpf_getelementptr_and_load)) + return cast(I); + return nullptr; +} + +static CallInst *isGEPAndStore(Value *I) { + if (isIntrinsicCall(I, Intrinsic::bpf_getelementptr_and_store)) + return cast(I); + return nullptr; +} + +template +static DILocation *mergeDILocations(SmallVector &Insns) { + DILocation *Merged = (*Insns.begin())->getDebugLoc(); + for (T *I : Insns) + Merged = DILocation::getMergedLocation(Merged, I->getDebugLoc()); + return Merged; +} + +static CallInst *makeIntrinsicCall(Module *M, + Intrinsic::BPFIntrinsics Intrinsic, + ArrayRef Types, + ArrayRef Args) { + + Function *Fn = Intrinsic::getDeclaration(M, Intrinsic, Types); + return CallInst::Create(Fn, Args); +} + +static void setParamElementType(CallInst *Call, unsigned ArgNo, Type *Type) { + LLVMContext &C = Call->getContext(); + Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ElementType, Type)); +} + +static void setParamReadNone(CallInst *Call, unsigned ArgNo) { + LLVMContext &C = Call->getContext(); + Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ReadNone)); +} + +static void setParamReadOnly(CallInst *Call, unsigned ArgNo) { + LLVMContext &C = Call->getContext(); + Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::ReadOnly)); +} + +static void setParamWriteOnly(CallInst *Call, unsigned ArgNo) { + LLVMContext &C = Call->getContext(); + Call->addParamAttr(ArgNo, Attribute::get(C, Attribute::WriteOnly)); +} + +namespace { +struct GEPChainInfo { + bool InBounds; + Type *SourceElementType; + SmallVector Indices; + SmallVector Members; + + GEPChainInfo() { reset(); } + + void reset() { + InBounds = true; + SourceElementType = nullptr; + Indices.clear(); + Members.clear(); + } +}; +} // Anonymous namespace + +template > +static void fillCommonArgs(LLVMContext &C, SmallVector &Args, + GEPChainInfo &GEP, T *Insn) { + Type *Int8Ty = Type::getInt8Ty(C); + Type *Int1Ty = Type::getInt1Ty(C); + // Implementation of Align guarantees that ShiftValue < 64 + unsigned AlignShiftValue = Log2_64(Insn->getAlign().value()); + Args.push_back(GEP.Members[0]->getPointerOperand()); + Args.push_back(ConstantInt::get(Int1Ty, Insn->isVolatile())); + Args.push_back(ConstantInt::get(Int8Ty, (unsigned)Insn->getOrdering())); + Args.push_back(ConstantInt::get(Int8Ty, (unsigned)Insn->getSyncScopeID())); + Args.push_back(ConstantInt::get(Int8Ty, AlignShiftValue)); + Args.push_back(ConstantInt::get(Int1Ty, GEP.InBounds)); + Args.append(GEP.Indices.begin(), GEP.Indices.end()); +} + +static Instruction *makeGEPAndLoad(Module *M, GEPChainInfo &GEP, + LoadInst *Load) { + SmallVector Args; + fillCommonArgs(M->getContext(), Args, GEP, Load); + CallInst *Call = makeIntrinsicCall(M, Intrinsic::bpf_getelementptr_and_load, + {Load->getType()}, Args); + setParamElementType(Call, 0, GEP.SourceElementType); + Call->applyMergedLocation(mergeDILocations(GEP.Members), Load->getDebugLoc()); + Call->setName((*GEP.Members.rbegin())->getName()); + if (Load->isUnordered()) { + Call->setOnlyReadsMemory(); + Call->setOnlyAccessesArgMemory(); + setParamReadOnly(Call, 0); + } + for (unsigned I = GepAndLoadFirstIdxArg; I < Args.size(); ++I) + Call->addParamAttr(I, Attribute::ImmArg); + Call->setAAMetadata(Load->getAAMetadata()); + return Call; +} + +static Instruction *makeGEPAndStore(Module *M, GEPChainInfo &GEP, + StoreInst *Store) { + SmallVector Args; + Args.push_back(Store->getValueOperand()); + fillCommonArgs(M->getContext(), Args, GEP, Store); + CallInst *Call = + makeIntrinsicCall(M, Intrinsic::bpf_getelementptr_and_store, + {Store->getValueOperand()->getType()}, Args); + setParamElementType(Call, 1, GEP.SourceElementType); + if (Store->getValueOperand()->getType()->isPointerTy()) + setParamReadNone(Call, 0); + Call->applyMergedLocation(mergeDILocations(GEP.Members), + Store->getDebugLoc()); + if (Store->isUnordered()) { + Call->setOnlyWritesMemory(); + Call->setOnlyAccessesArgMemory(); + setParamWriteOnly(Call, 1); + } + for (unsigned I = GepAndStoreFirstIdxArg; I < Args.size(); ++I) + Call->addParamAttr(I, Attribute::ImmArg); + Call->setAAMetadata(Store->getAAMetadata()); + return Call; +} + +static unsigned getOperandAsUnsigned(CallInst *Call, unsigned ArgNo) { + if (auto *Int = dyn_cast(Call->getOperand(ArgNo))) + return Int->getValue().getZExtValue(); + std::string Report; + raw_string_ostream ReportS(Report); + ReportS << "Expecting ConstantInt as argument #" << ArgNo << " of " << *Call + << "\n"; + report_fatal_error(StringRef(Report)); +} + +static GetElementPtrInst *reconstructGEP(CallInst *Call, int Delta) { + SmallVector Indices; + Indices.append(Call->data_operands_begin() + 6 + Delta, + Call->data_operands_end()); + Type *GEPPointeeType = Call->getParamElementType(Delta); + auto *GEP = + GetElementPtrInst::Create(GEPPointeeType, Call->getOperand(Delta), + ArrayRef(Indices), Call->getName()); + GEP->setIsInBounds(getOperandAsUnsigned(Call, 5 + Delta)); + return GEP; +} + +template > +static void reconstructCommon(CallInst *Call, GetElementPtrInst *GEP, T *Insn, + int Delta) { + Insn->setVolatile(getOperandAsUnsigned(Call, 1 + Delta)); + Insn->setOrdering((AtomicOrdering)getOperandAsUnsigned(Call, 2 + Delta)); + Insn->setSyncScopeID(getOperandAsUnsigned(Call, 3 + Delta)); + unsigned AlignShiftValue = getOperandAsUnsigned(Call, 4 + Delta); + Insn->setAlignment(Align(1ULL << AlignShiftValue)); + GEP->setDebugLoc(Call->getDebugLoc()); + Insn->setDebugLoc(Call->getDebugLoc()); + Insn->setAAMetadata(Call->getAAMetadata()); +} + +std::pair +BPFPreserveStaticOffsetPass::reconstructLoad(CallInst *Call) { + GetElementPtrInst *GEP = reconstructGEP(Call, 0); + Type *ReturnType = Call->getFunctionType()->getReturnType(); + auto *Load = new LoadInst(ReturnType, GEP, "", + /* These would be set in reconstructCommon */ + false, Align(1)); + reconstructCommon(Call, GEP, Load, 0); + return std::pair{GEP, Load}; +} + +std::pair +BPFPreserveStaticOffsetPass::reconstructStore(CallInst *Call) { + GetElementPtrInst *GEP = reconstructGEP(Call, 1); + auto *Store = new StoreInst(Call->getOperand(0), GEP, + /* These would be set in reconstructCommon */ + false, Align(1)); + reconstructCommon(Call, GEP, Store, 1); + return std::pair{GEP, Store}; +} + +static bool isZero(Value *V) { + auto *CI = dyn_cast(V); + return CI && CI->isZero(); +} + +// Given a chain of GEP instructions collect information necessary to +// merge this chain as a single GEP instruction of form: +// getelementptr %, ptr %p, i32 0, , , ... +static bool foldGEPChainAsStructAccess(SmallVector &GEPs, + GEPChainInfo &Info) { + if (GEPs.empty()) + return false; + + if (!all_of(GEPs, [=](GetElementPtrInst *GEP) { + return GEP->hasAllConstantIndices(); + })) + return false; + + GetElementPtrInst *First = GEPs[0]; + Info.InBounds = First->isInBounds(); + Info.SourceElementType = First->getSourceElementType(); + Type *ResultElementType = First->getResultElementType(); + Info.Indices.append(First->idx_begin(), First->idx_end()); + Info.Members.push_back(First); + + for (auto *Iter = GEPs.begin() + 1; Iter != GEPs.end(); ++Iter) { + GetElementPtrInst *GEP = *Iter; + if (!isZero(*GEP->idx_begin())) { + Info.reset(); + return false; + } + if (!GEP->getSourceElementType() || + GEP->getSourceElementType() != ResultElementType) { + Info.reset(); + return false; + } + Info.InBounds &= GEP->isInBounds(); + Info.Indices.append(GEP->idx_begin() + 1, GEP->idx_end()); + Info.Members.push_back(GEP); + ResultElementType = GEP->getResultElementType(); + } + + return true; +} + +// Given a chain of GEP instructions collect information necessary to +// merge this chain as a single GEP instruction of form: +// getelementptr i8, ptr %p, i64 %offset +static bool foldGEPChainAsU8Access(SmallVector &GEPs, + GEPChainInfo &Info) { + if (GEPs.empty()) + return false; + + GetElementPtrInst *First = GEPs[0]; + const DataLayout &DL = First->getModule()->getDataLayout(); + LLVMContext &C = First->getContext(); + Type *PtrTy = First->getType()->getScalarType(); + APInt Offset(DL.getIndexTypeSizeInBits(PtrTy), 0); + for (GetElementPtrInst *GEP : GEPs) { + if (!GEP->accumulateConstantOffset(DL, Offset)) { + Info.reset(); + return false; + } + Info.InBounds &= GEP->isInBounds(); + Info.Members.push_back(GEP); + } + Info.SourceElementType = Type::getInt8Ty(C); + Info.Indices.push_back(ConstantInt::get(C, Offset)); + + return true; +} + +static void reportNonStaticGEPChain(Instruction *Insn) { + auto Msg = DiagnosticInfoUnsupported( + *Insn->getFunction(), + Twine("Non-constant offset in access to a field of a type marked " + "with preserve_static_offset might be rejected by BPF verifier") + .concat(Insn->getDebugLoc() + ? "" + : " (pass -g option to get exact location)"), + Insn->getDebugLoc(), DS_Warning); + Insn->getContext().diagnose(Msg); +} + +static bool allZeroIndices(SmallVector &GEPs) { + return GEPs.empty() || all_of(GEPs, [=](GetElementPtrInst *GEP) { + return GEP->hasAllZeroIndices(); + }); +} + +static bool tryToReplaceWithGEPBuiltin(Instruction *LoadOrStoreTemplate, + SmallVector &GEPs, + Instruction *InsnToReplace) { + GEPChainInfo GEPChain; + if (!foldGEPChainAsStructAccess(GEPs, GEPChain) && + !foldGEPChainAsU8Access(GEPs, GEPChain)) { + return false; + } + Module *M = InsnToReplace->getModule(); + if (auto *Load = dyn_cast(LoadOrStoreTemplate)) { + Instruction *Replacement = makeGEPAndLoad(M, GEPChain, Load); + Replacement->insertBefore(InsnToReplace); + InsnToReplace->replaceAllUsesWith(Replacement); + } + if (auto *Store = dyn_cast(LoadOrStoreTemplate)) { + Instruction *Replacement = makeGEPAndStore(M, GEPChain, Store); + Replacement->insertBefore(InsnToReplace); + } + return true; +} + +// Check if U->getPointerOperand() == I +static bool isPointerOperand(Value *I, User *U) { + if (auto *L = dyn_cast(U)) + return L->getPointerOperand() == I; + if (auto *S = dyn_cast(U)) + return S->getPointerOperand() == I; + if (auto *GEP = dyn_cast(U)) + return GEP->getPointerOperand() == I; + if (auto *Call = isGEPAndLoad(U)) + return Call->getArgOperand(0) == I; + if (auto *Call = isGEPAndStore(U)) + return Call->getArgOperand(1) == I; + return false; +} + +static bool isInlineableCall(User *U) { + if (auto *Call = dyn_cast(U)) + return Call->hasFnAttr(Attribute::InlineHint); + return false; +} + +static void rewriteAccessChain(Instruction *Insn, + SmallVector &GEPs, + SmallVector &Visited, + bool AllowPatial, bool &StillUsed); + +static void rewriteUses(Instruction *Insn, + SmallVector &GEPs, + SmallVector &Visited, bool AllowPatial, + bool &StillUsed) { + for (User *U : Insn->users()) { + auto *UI = dyn_cast(U); + if (UI && (isPointerOperand(Insn, UI) || isPreserveStaticOffsetCall(UI) || + isInlineableCall(UI))) + rewriteAccessChain(UI, GEPs, Visited, AllowPatial, StillUsed); + else + LLVM_DEBUG({ + llvm::dbgs() << "unsupported usage in BPFPreserveStaticOffsetPass:\n"; + llvm::dbgs() << " Insn: " << *Insn << "\n"; + llvm::dbgs() << " User: " << *U << "\n"; + }); + } +} + +// A DFS traversal of GEP chain trees starting from Root. +// +// Recursion descends through GEP instructions and +// llvm.preserve.static.offset calls. Recursion stops at any other +// instruction. If load or store instruction is reached it is replaced +// by a call to `llvm.bpf.getelementptr.and.load` or +// `llvm.bpf.getelementptr.and.store` intrinsic. +// If `llvm.bpf.getelementptr.and.load/store` is reached the accumulated +// GEPs are merged into the intrinsic call. +// If nested calls to `llvm.preserve.static.offset` are encountered these +// calls are marked for deletion. +// +// Parameters description: +// - Insn - current position in the tree +// - GEPs - GEP instructions for the current branch +// - Visited - a list of visited instructions in DFS order, +// order is important for unused instruction deletion. +// - AllowPartial - when true GEP chains that can't be folded are +// not reported, otherwise diagnostic message is show for such chains. +// - StillUsed - set to true if one of the GEP chains could not be +// folded, makes sense when AllowPartial is false, means that root +// preserve.static.offset call is still in use and should remain +// until the next run of this pass. +static void rewriteAccessChain(Instruction *Insn, + SmallVector &GEPs, + SmallVector &Visited, + bool AllowPatial, bool &StillUsed) { + auto MarkAndTraverseUses = [&]() { + Visited.push_back(Insn); + rewriteUses(Insn, GEPs, Visited, AllowPatial, StillUsed); + }; + auto TryToReplace = [&](Instruction *LoadOrStore) { + // Do nothing for (preserve.static.offset (load/store ..)) or for + // GEPs with zero indices. Such constructs lead to zero offset and + // are simplified by other passes. + if (allZeroIndices(GEPs)) + return; + if (tryToReplaceWithGEPBuiltin(LoadOrStore, GEPs, Insn)) { + Visited.push_back(Insn); + return; + } + if (!AllowPatial) + reportNonStaticGEPChain(Insn); + StillUsed = true; + }; + if (isa(Insn) || isa(Insn)) { + TryToReplace(Insn); + } else if (isGEPAndLoad(Insn)) { + auto [GEP, Load] = + BPFPreserveStaticOffsetPass::reconstructLoad(cast(Insn)); + GEPs.push_back(GEP); + TryToReplace(Load); + GEPs.pop_back(); + delete Load; + delete GEP; + } else if (isGEPAndStore(Insn)) { + // This case can't be merged with the above because + // `delete Load` / `delete Store` wants a concrete type, + // destructor of Instruction is protected. + auto [GEP, Store] = + BPFPreserveStaticOffsetPass::reconstructStore(cast(Insn)); + GEPs.push_back(GEP); + TryToReplace(Store); + GEPs.pop_back(); + delete Store; + delete GEP; + } else if (auto *GEP = dyn_cast(Insn)) { + GEPs.push_back(GEP); + MarkAndTraverseUses(); + GEPs.pop_back(); + } else if (isPreserveStaticOffsetCall(Insn)) { + MarkAndTraverseUses(); + } else if (isInlineableCall(Insn)) { + // Preserve preserve.static.offset call for parameters of + // functions that might be inlined. These would be removed on a + // second pass after inlining. + // Might happen when a pointer to a preserve_static_offset + // structure is passed as parameter of a function that would be + // inlined inside a loop that would be unrolled. + if (AllowPatial) + StillUsed = true; + } else { + SmallString<128> Buf; + raw_svector_ostream BufStream(Buf); + BufStream << *Insn; + report_fatal_error( + Twine("Unexpected rewriteAccessChain Insn = ").concat(Buf)); + } +} + +static void removeMarkerCall(Instruction *Marker) { + Marker->replaceAllUsesWith(Marker->getOperand(0)); + Marker->eraseFromParent(); +} + +static bool rewriteAccessChain(Instruction *Marker, bool AllowPatial, + SmallPtrSetImpl &RemovedMarkers) { + SmallVector GEPs; + SmallVector Visited; + bool StillUsed = false; + rewriteUses(Marker, GEPs, Visited, AllowPatial, StillUsed); + // Check if Visited instructions could be removed, iterate in + // reverse to unblock instructions higher in the chain. + for (auto V = Visited.rbegin(); V != Visited.rend(); ++V) { + if (isPreserveStaticOffsetCall(*V)) { + removeMarkerCall(*V); + RemovedMarkers.insert(*V); + } else if ((*V)->use_empty()) { + (*V)->eraseFromParent(); + } + } + return StillUsed; +} + +static std::vector +collectPreserveStaticOffsetCalls(Function &F) { + std::vector Calls; + for (Instruction &Insn : instructions(F)) + if (isPreserveStaticOffsetCall(&Insn)) + Calls.push_back(&Insn); + return Calls; +} + +bool isPreserveArrayIndex(Value *V) { + return isIntrinsicCall(V, Intrinsic::preserve_array_access_index); +} + +bool isPreserveStructIndex(Value *V) { + return isIntrinsicCall(V, Intrinsic::preserve_struct_access_index); +} + +bool isPreserveUnionIndex(Value *V) { + return isIntrinsicCall(V, Intrinsic::preserve_union_access_index); +} + +static void removePAICalls(Instruction *Marker) { + auto IsPointerOperand = [](Value *Op, User *U) { + if (auto *GEP = dyn_cast(U)) + return GEP->getPointerOperand() == Op; + if (isPreserveStaticOffsetCall(U) || isPreserveArrayIndex(U) || + isPreserveStructIndex(U) || isPreserveUnionIndex(U)) + return cast(U)->getArgOperand(0) == Op; + return false; + }; + + SmallVector WorkList; + WorkList.push_back(Marker); + do { + Value *V = WorkList.pop_back_val(); + for (User *U : V->users()) + if (IsPointerOperand(V, U)) + WorkList.push_back(U); + auto *Call = dyn_cast(V); + if (!Call) + continue; + if (isPreserveArrayIndex(V)) + BPFCoreSharedInfo::removeArrayAccessCall(Call); + if (isPreserveStructIndex(V)) + BPFCoreSharedInfo::removeStructAccessCall(Call); + if (isPreserveUnionIndex(V)) + BPFCoreSharedInfo::removeUnionAccessCall(Call); + } while (!WorkList.empty()); +} + +// Look for sequences: +// - llvm.preserve.static.offset -> getelementptr... -> load +// - llvm.preserve.static.offset -> getelementptr... -> store +// And replace those with calls to intrinsics: +// - llvm.bpf.getelementptr.and.load +// - llvm.bpf.getelementptr.and.store +static bool rewriteFunction(Function &F, bool AllowPartial) { + LLVM_DEBUG(dbgs() << "********** BPFPreserveStaticOffsetPass (AllowPartial=" + << AllowPartial << ") ************\n"); + + auto MarkerCalls = collectPreserveStaticOffsetCalls(F); + SmallPtrSet RemovedMarkers; + + LLVM_DEBUG(dbgs() << "There are " << MarkerCalls.size() + << " preserve.static.offset calls\n"); + + if (MarkerCalls.empty()) + return false; + + for (auto *Call : MarkerCalls) + removePAICalls(Call); + + for (auto *Call : MarkerCalls) { + if (RemovedMarkers.contains(Call)) + continue; + bool StillUsed = rewriteAccessChain(Call, AllowPartial, RemovedMarkers); + if (!StillUsed || !AllowPartial) + removeMarkerCall(Call); + } + + return true; +} + +PreservedAnalyses +llvm::BPFPreserveStaticOffsetPass::run(Function &F, + FunctionAnalysisManager &AM) { + return rewriteFunction(F, AllowPartial) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -105,11 +105,16 @@ FPM.addPass(BPFIRPeepholePass()); return true; } + if (PassName == "bpf-preserve-static-offset") { + FPM.addPass(BPFPreserveStaticOffsetPass(false)); + return true; + } return false; }); PB.registerPipelineStartEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { FunctionPassManager FPM; + FPM.addPass(BPFPreserveStaticOffsetPass(true)); FPM.addPass(BPFAbstractMemberAccessPass(this)); FPM.addPass(BPFPreserveDITypePass()); FPM.addPass(BPFIRPeepholePass()); @@ -119,6 +124,12 @@ OptimizationLevel Level) { FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))); }); + PB.registerScalarOptimizerLateEPCallback( + [=](FunctionPassManager &FPM, OptimizationLevel Level) { + // Run this after loop unrolling but before + // SimplifyCFGPass(... .sinkCommonInsts(true)) + FPM.addPass(BPFPreserveStaticOffsetPass(false)); + }); PB.registerPipelineEarlySimplificationEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { MPM.addPass(BPFAdjustOptPass()); diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -26,6 +26,7 @@ BPFISelLowering.cpp BPFMCInstLower.cpp BPFPreserveDIType.cpp + BPFPreserveStaticOffset.cpp BPFRegisterInfo.cpp BPFSelectionDAGInfo.cpp BPFSubtarget.cpp diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-align.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-align.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-align.ll @@ -0,0 +1,66 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a load instruction for a field with non-standard +; alignment by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; typedef int aligned_int __attribute__((aligned(128))); +; +; struct foo { +; int _; +; aligned_int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->a); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, [124 x i8], i32, [124 x i8] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 2 + %1 = load i32, ptr %a, align 128, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: %[[a1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 7, i1 true, i32 immarg 0, i32 immarg 2) +; ^^^^ +; alignment 2**7 +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK-NEXT: call void @consume(i32 noundef %[[a1]]) +; CHECK: attributes #[[v2]] = { memory(argmem: read) } + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 128} +!3 = !{!"foo", !4, i64 0, !4, i64 128} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-arr-pai.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-arr-pai.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-arr-pai.ll @@ -0,0 +1,93 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; #define __pai __attribute__((preserve_access_index)) +; +; struct bar { +; int a[7]; +; } __pai __ctx; +; +; int buz(struct bar *p) { +; return p->a[5]; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -debug-info-kind=limited -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { [7 x i32] } + +; Function Attrs: nounwind +define dso_local i32 @buz(ptr noundef %p) #0 !dbg !10 { +entry: + call void @llvm.dbg.value(metadata ptr %p, metadata !18, metadata !DIExpression()), !dbg !19 + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !20 + %1 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bar) %0, i32 0, i32 0), !dbg !20, !llvm.preserve.access.index !14 + %2 = call ptr @llvm.preserve.array.access.index.p0.p0(ptr elementtype([7 x i32]) %1, i32 1, i32 5), !dbg !21, !llvm.preserve.access.index !3 + %3 = load i32, ptr %2, align 4, !dbg !21, !tbaa !22 + ret i32 %3, !dbg !26 +} + +; CHECK: define dso_local i32 @buz(ptr noundef %[[p:.*]]) {{.*}} { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.dbg.value +; CHECK-NEXT: %[[v5:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.bar) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 0, i32 immarg 5) +; CHECK-SAME: #[[v6:.*]], !tbaa +; CHECK-NEXT: ret i32 %[[v5]] +; CHECK-NEXT: } +; CHECK: attributes #[[v6]] = { memory(argmem: read) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.array.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!7, !8} +!llvm.ident = !{!9} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{!3} +!3 = !DICompositeType(tag: DW_TAG_array_type, baseType: !4, size: 224, elements: !5) +!4 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!5 = !{!6} +!6 = !DISubrange(count: 7) +!7 = !{i32 2, !"Debug Info Version", i32 3} +!8 = !{i32 1, !"wchar_size", i32 4} +!9 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!10 = distinct !DISubprogram(name: "buz", scope: !1, file: !1, line: 8, type: !11, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17) +!11 = !DISubroutineType(types: !12) +!12 = !{!4, !13} +!13 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !14, size: 64) +!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bar", file: !1, line: 4, size: 224, elements: !15) +!15 = !{!16} +!16 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !14, file: !1, line: 5, baseType: !3, size: 224) +!17 = !{!18} +!18 = !DILocalVariable(name: "p", arg: 1, scope: !10, file: !1, line: 8, type: !13) +!19 = !DILocation(line: 0, scope: !10) +!20 = !DILocation(line: 9, column: 13, scope: !10) +!21 = !DILocation(line: 9, column: 10, scope: !10) +!22 = !{!23, !23, i64 0} +!23 = !{!"int", !24, i64 0} +!24 = !{!"omnipotent char", !25, i64 0} +!25 = !{!"Simple C/C++ TBAA"} +!26 = !DILocation(line: 9, column: 3, scope: !10) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-atomic.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-atomic.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-atomic.ll @@ -0,0 +1,66 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of atomic load instruction by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int _; +; int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; int r; +; __atomic_load(&p->a, &r, 2); +; consume(r); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %1 = load atomic i32, ptr %a acquire, align 4 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: %[[a1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr elementtype(%struct.foo) %[[p:.*]], +; i1 false, i8 4, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; ^^^^ +; atomic order +; CHECK-NOT: #{{[0-9]+}} +; CHECK-NEXT: call void @consume(i32 noundef %[[a1]]) + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +declare void @consume(i32 noundef) #3 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-2.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-2.ll @@ -0,0 +1,82 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds GEP chains that end by +; getelementptr.and.load. +; +; Source (modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->b.bb); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; And modified to fold last getelementptr/load as a single +; getelementptr.and.load. + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %bb1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr readonly elementtype(%struct.bar) %b, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) + #4, !tbaa !2 + call void @consume(i32 noundef %bb1) + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[bb1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK-NEXT: call void @consume(i32 noundef %[[bb1]]) +; CHECK: attributes #[[v2]] = { memory(argmem: read) } + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-oob.ll @@ -0,0 +1,73 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset keeps track of 'inbounds' flags while +; folding chain of GEP instructions. +; +; Source (IR modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a[2]; +; }; +; +; struct bar { +; int a; +; struct foo b; +; } __ctx; +; +; extern void consume(int); +; +; void buz(struct bar *p) { +; consume(p->b.a[1]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; Modified to remove one of the 'inbounds' from one of the GEP instructions. + +%struct.bar = type { i32, %struct.foo } +%struct.foo = type { [2 x i32] } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %a = getelementptr %struct.foo, ptr %b, i32 0, i32 0 + %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 1 + %1 = load i32, ptr %arrayidx, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: %[[v1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.bar) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 false, +; ^^^^^^^^ +; not inbounds +; CHECK-SAME: i32 immarg 0, i32 immarg 1, i32 immarg 0, i64 immarg 1) +; ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +; folded gep chain +; CHECK-NEXT: call void @consume(i32 noundef %[[v1]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-oob.ll @@ -0,0 +1,74 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; The GEP chain in this example has unexpected shape and thus is +; folded as i8 access. +; +; Source (IR modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char a[2]; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; extern void consume(char); +; +; void buz(struct bar *p) { +; consume((&p->b)[1].a[1]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; Modified to remove 'inbounds' from one of the GEP instructions. + +%struct.bar = type { i8, %struct.foo } +%struct.foo = type { [2 x i8] } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %arrayidx = getelementptr inbounds %struct.foo, ptr %b, i64 1 +; ^^^^^ +; folded as i8 access because of this index + %a = getelementptr %struct.foo, ptr %arrayidx, i32 0, i32 0 + %arrayidx1 = getelementptr inbounds [2 x i8], ptr %a, i64 0, i64 1 + %1 = load i8, ptr %arrayidx1, align 1, !tbaa !2 + call void @consume(i8 noundef signext %1) + ret void +} + +; CHECK: %[[v1:.*]] = call i8 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i8 +; CHECK-SAME: (ptr readonly elementtype(i8) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 0, i1 false, i64 immarg 4) +; ^^^^^^^^ ^^^^^^^^^^^^ +; not inbounds ---' | +; offset from 'struct bar' start -------------' +; CHECK-NEXT: call void @consume(i8 noundef signext %[[v1]]) + +declare void @consume(i8 noundef signext) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"omnipotent char", !4, i64 0} +!4 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-type-mismatch.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-type-mismatch.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8-type-mismatch.ll @@ -0,0 +1,73 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; The GEP chain in this example has unexpected shape and thus is +; folded as i8 access. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char aa; +; char bb; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; extern void consume(char); +; +; void buz(struct bar *p) { +; consume(((struct foo *)(((char*)&p->b) + 1))->bb); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i8, %struct.foo } +%struct.foo = type { i8, i8 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %add.ptr = getelementptr inbounds i8, ptr %b, i64 1 +; ~~ +; these types do not match, thus GEP chain is folded as an offset +; ~~~~~~~~~~~ + %bb = getelementptr inbounds %struct.foo, ptr %add.ptr, i32 0, i32 1 + %1 = load i8, ptr %bb, align 1, !tbaa !2 + call void @consume(i8 noundef signext %1) + ret void +} + +; CHECK: %[[bb1:.*]] = call i8 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i8 +; CHECK-SAME: (ptr readonly elementtype(i8) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 0, i1 true, i64 immarg 3) +; ^^^^^^^^^^^^ +; offset from 'struct bar' start +; CHECK-NEXT: call void @consume(i8 noundef signext %[[bb1]]) + +declare void @consume(i8 noundef signext) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 1} +!3 = !{!"foo", !4, i64 0, !4, i64 1} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain-u8.ll @@ -0,0 +1,71 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; The GEP chain in this example has unexpected shape and thus is +; folded as i8 access. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char a[2]; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; extern void consume(char); +; +; void buz(struct bar *p) { +; consume((&p->b)[1].a[1]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i8, %struct.foo } +%struct.foo = type { [2 x i8] } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %arrayidx = getelementptr inbounds %struct.foo, ptr %b, i64 1 +; ^^^^^ +; folded as i8 access because of this index + %a = getelementptr inbounds %struct.foo, ptr %arrayidx, i32 0, i32 0 + %arrayidx1 = getelementptr inbounds [2 x i8], ptr %a, i64 0, i64 1 + %1 = load i8, ptr %arrayidx1, align 1, !tbaa !2 + call void @consume(i8 noundef signext %1) + ret void +} + +; CHECK: %[[v1:.*]] = call i8 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i8 +; CHECK-SAME: (ptr readonly elementtype(i8) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 0, i1 true, i64 immarg 4) +; ^^^^^^^^^^^^ +; offset from 'struct bar' start +; CHECK-NEXT: call void @consume(i8 noundef signext %[[v1]]) + +declare void @consume(i8 noundef signext) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"omnipotent char", !4, i64 0} +!4 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-chain.ll @@ -0,0 +1,68 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a[2]; +; }; +; +; struct bar { +; int a; +; struct foo b; +; } __ctx; +; +; extern void consume(int); +; +; void buz(struct bar *p) { +; consume(p->b.a[1]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i32, %struct.foo } +%struct.foo = type { [2 x i32] } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %a = getelementptr inbounds %struct.foo, ptr %b, i32 0, i32 0 + %arrayidx = getelementptr inbounds [2 x i32], ptr %a, i64 0, i64 1 + %1 = load i32, ptr %arrayidx, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: %[[v1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.bar) %{{[^,]+}}, +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, +; CHECK-SAME: i32 immarg 0, i32 immarg 1, i32 immarg 0, i64 immarg 1) +; ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +; folded gep chain +; CHECK-NEXT: call void @consume(i32 noundef %[[v1]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-inline.ll @@ -0,0 +1,85 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check position of bpf-preserve-static-offset pass in the pipeline: +; - preserve.static.offset call is preserved if address is passed as +; a parameter to an inline-able function; +; - second bpf-preserve-static-offset pass (after inlining) should introduce +; getelementptr.and.load call using the preserved marker. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; extern void consume(int); +; +; static inline void bar(struct bar *p){ +; consume(p->bb); +; } +; +; void quux(struct foo *p) { +; bar(&p->b); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @quux(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + call void @bar(ptr noundef %b) + ret void +} + +; Function Attrs: inlinehint nounwind +define internal void @bar(ptr noundef %p) #1 { +entry: + %bb = getelementptr inbounds %struct.bar, ptr %p, i32 0, i32 1 + %0 = load i32, ptr %bb, align 4, !tbaa !2 + call void @consume(i32 noundef %0) + ret void +} + +; CHECK: define dso_local void @quux(ptr nocapture noundef readonly %[[p:.*]]) +; CHECK: %[[bb_i1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 0, i32 immarg 1, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK-NEXT: tail call void @consume(i32 noundef %[[bb_i1]]) +; CHECK: attributes #[[v2]] = { memory(argmem: read) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +declare void @consume(i32 noundef) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { inlinehint nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"bar", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-non-const.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-non-const.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-non-const.ll @@ -0,0 +1,75 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s 2>&1 | FileCheck %s +; +; If load offset is not a constant bpf-preserve-static-offset should report a +; warning and remove preserve.static.offset call. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a[7]; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p, unsigned long i) { +; consume(p->a[i]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -debug-info-kind=line-tables-only -triple bpf \ +; -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +; CHECK: warning: some-file.c:10:11: in function bar void (ptr, i64): +; CHECK-SAME: Non-constant offset in access to a field of a type marked with +; CHECK-SAME: preserve_static_offset might be rejected by BPF verifier + +%struct.foo = type { [7 x i32] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p, i64 noundef %i) #0 !dbg !5 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !8 + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 0, !dbg !8 + %arrayidx = getelementptr inbounds [7 x i32], ptr %a, i64 0, i64 %i, !dbg !9 + %1 = load i32, ptr %arrayidx, align 4, !dbg !9, !tbaa !10 + call void @consume(i32 noundef %1), !dbg !14 + ret void, !dbg !15 +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]], i64 noundef %[[i:.*]]) +; CHECK: %[[a:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 0, !dbg +; CHECK-NEXT: %[[arrayidx:.*]] = getelementptr inbounds [7 x i32], ptr %[[a]], i64 0, i64 %[[i]], !dbg +; CHECK-NEXT: %[[v5:.*]] = load i32, ptr %[[arrayidx]], align 4, !dbg {{.*}}, !tbaa +; CHECK-NEXT: call void @consume(i32 noundef %[[v5]]), !dbg + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!5 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 9, type: !6, scopeLine: 9, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!6 = !DISubroutineType(types: !7) +!7 = !{} +!8 = !DILocation(line: 10, column: 14, scope: !5) +!9 = !DILocation(line: 10, column: 11, scope: !5) +!10 = !{!11, !11, i64 0} +!11 = !{!"int", !12, i64 0} +!12 = !{!"omnipotent char", !13, i64 0} +!13 = !{!"Simple C/C++ TBAA"} +!14 = !DILocation(line: 10, column: 3, scope: !5) +!15 = !DILocation(line: 11, column: 1, scope: !5) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-ptr-pai.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-ptr-pai.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-ptr-pai.ll @@ -0,0 +1,114 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; #define __pai __attribute__((preserve_access_index)) +; +; struct bar { +; int a; +; int b; +; } __pai; +; +; struct buz { +; int _1; +; struct bar *b; +; } __pai __ctx; +; +; void foo(struct buz *p) { +; p->b->b = 42; +; } +; +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes \ +; -debug-info-kind=limited -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.buz = type { i32, ptr } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @foo(ptr noundef %p) #0 !dbg !5 { +entry: + call void @llvm.dbg.value(metadata ptr %p, metadata !20, metadata !DIExpression()), !dbg !21 + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !22 + %1 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.buz) %0, i32 1, i32 1), !dbg !22, !llvm.preserve.access.index !9 + %2 = load ptr, ptr %1, align 8, !dbg !22, !tbaa !23 + %3 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bar) %2, i32 1, i32 1), !dbg !29, !llvm.preserve.access.index !15 + store i32 42, ptr %3, align 4, !dbg !30, !tbaa !31 + ret void, !dbg !33 +} + +; CHECK: define dso_local void @foo(ptr noundef %[[p:.*]]) {{.*}} { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.dbg.value +; CHECK-NEXT: %[[v5:.*]] = call ptr (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.p0 +; CHECK-SAME: (ptr readonly elementtype(%struct.buz) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 3, i1 true, i32 immarg 0, i32 immarg 1) +; CHECK-SAME: #[[v6:.*]], !tbaa +; CHECK-NEXT: %[[v8:.*]] = +; CHECK-SAME: call ptr @llvm.preserve.struct.access.index.p0.p0 +; CHECK-SAME: (ptr elementtype(%struct.bar) %[[v5]], i32 1, i32 1), +; CHECK-SAME: !dbg ![[#]], !llvm.preserve.access.index ![[#]] +; CHECK-NEXT: store i32 42, ptr %[[v8]], align 4, !dbg ![[#]], !tbaa +; CHECK-NEXT: ret void, !dbg +; CHECK-NEXT: } + +; CHECK : attributes #[[v6]] = { memory(argmem: read) } + + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!5 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 14, type: !6, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !19) +!6 = !DISubroutineType(types: !7) +!7 = !{null, !8} +!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64) +!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "buz", file: !1, line: 9, size: 128, elements: !10) +!10 = !{!11, !13} +!11 = !DIDerivedType(tag: DW_TAG_member, name: "_1", scope: !9, file: !1, line: 10, baseType: !12, size: 32) +!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!13 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !9, file: !1, line: 11, baseType: !14, size: 64, offset: 64) +!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64) +!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bar", file: !1, line: 4, size: 64, elements: !16) +!16 = !{!17, !18} +!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !15, file: !1, line: 5, baseType: !12, size: 32) +!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !15, file: !1, line: 6, baseType: !12, size: 32, offset: 32) +!19 = !{!20} +!20 = !DILocalVariable(name: "p", arg: 1, scope: !5, file: !1, line: 14, type: !8) +!21 = !DILocation(line: 0, scope: !5) +!22 = !DILocation(line: 15, column: 6, scope: !5) +!23 = !{!24, !28, i64 8} +!24 = !{!"buz", !25, i64 0, !28, i64 8} +!25 = !{!"int", !26, i64 0} +!26 = !{!"omnipotent char", !27, i64 0} +!27 = !{!"Simple C/C++ TBAA"} +!28 = !{!"any pointer", !26, i64 0} +!29 = !DILocation(line: 15, column: 9, scope: !5) +!30 = !DILocation(line: 15, column: 11, scope: !5) +!31 = !{!32, !25, i64 4} +!32 = !{!"bar", !25, i64 0, !25, i64 4} +!33 = !DILocation(line: 16, column: 1, scope: !5) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-simple.ll @@ -0,0 +1,71 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a simple load instruction by bpf-preserve-static-offset. +; Verify: +; - presence of gep.and.load intrinsic call +; - correct attributes for intrinsic call +; - presence of tbaa annotations +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int _; +; int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->a); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %1 = load i32, ptr %a, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[a1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; CHECK-SAME: #[[v1:.*]], !tbaa +; CHECK-NEXT: call void @consume(i32 noundef %[[a1]]) + +; CHECK: declare i32 +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, {{.*}}) #[[v2:.*]] + +; CHECK: attributes #[[v2]] = { nocallback nofree nounwind willreturn } +; CHECK: attributes #[[v1]] = { memory(argmem: read) } + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-struct-pai.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-struct-pai.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-struct-pai.ll @@ -0,0 +1,105 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; #define __pai __attribute__((preserve_access_index)) +; +; struct foo { +; int a; +; int b; +; }; +; +; struct bar { +; int _1; +; int _2; +; struct foo c; +; } __pai __ctx; +; +; int buz(struct bar *p) { +; return p->c.b; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes \ +; -debug-info-kind=limited -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i32, i32, %struct.foo } +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local i32 @buz(ptr noundef %p) #0 !dbg !5 { +entry: + call void @llvm.dbg.value(metadata ptr %p, metadata !20, metadata !DIExpression()), !dbg !21 + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !22 + %1 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bar) %0, i32 2, i32 2), !dbg !22, !llvm.preserve.access.index !10 + %b = getelementptr inbounds %struct.foo, ptr %1, i32 0, i32 1, !dbg !23 + %2 = load i32, ptr %b, align 4, !dbg !23, !tbaa !24 + ret i32 %2, !dbg !30 +} + +; CHECK: define dso_local i32 @buz(ptr noundef %[[p:.*]]) {{.*}} { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.dbg.value +; CHECK-NEXT: %[[b1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.bar) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 2, i32 immarg 1) +; CHECK-SAME: #[[v5:.*]], !tbaa +; CHECK-NEXT: ret i32 %[[b1]] +; CHECK-NEXT: } + +; CHECK: attributes #[[v5]] = { memory(argmem: read) } + + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!5 = distinct !DISubprogram(name: "buz", scope: !1, file: !1, line: 15, type: !6, scopeLine: 15, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !19) +!6 = !DISubroutineType(types: !7) +!7 = !{!8, !9} +!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64) +!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bar", file: !1, line: 9, size: 128, elements: !11) +!11 = !{!12, !13, !14} +!12 = !DIDerivedType(tag: DW_TAG_member, name: "_1", scope: !10, file: !1, line: 10, baseType: !8, size: 32) +!13 = !DIDerivedType(tag: DW_TAG_member, name: "_2", scope: !10, file: !1, line: 11, baseType: !8, size: 32, offset: 32) +!14 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !10, file: !1, line: 12, baseType: !15, size: 64, offset: 64) +!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !1, line: 4, size: 64, elements: !16) +!16 = !{!17, !18} +!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !15, file: !1, line: 5, baseType: !8, size: 32) +!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !15, file: !1, line: 6, baseType: !8, size: 32, offset: 32) +!19 = !{!20} +!20 = !DILocalVariable(name: "p", arg: 1, scope: !5, file: !1, line: 15, type: !9) +!21 = !DILocation(line: 0, scope: !5) +!22 = !DILocation(line: 16, column: 13, scope: !5) +!23 = !DILocation(line: 16, column: 15, scope: !5) +!24 = !{!25, !26, i64 12} +!25 = !{!"bar", !26, i64 0, !26, i64 4, !29, i64 8} +!26 = !{!"int", !27, i64 0} +!27 = !{!"omnipotent char", !28, i64 0} +!28 = !{!"Simple C/C++ TBAA"} +!29 = !{!"foo", !26, i64 0, !26, i64 4} +!30 = !DILocation(line: 16, column: 3, scope: !5) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-align.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-align.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-align.ll @@ -0,0 +1,67 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that getelementptr.and.load unroll restores alignment spec. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; typedef int aligned_int __attribute__((aligned(128))); +; +; struct foo { +; int _; +; aligned_int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->a); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, [124 x i8], i32, [124 x i8] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %a1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr readonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 7, i1 true, i32 immarg 0, i32 immarg 2) + #4, !tbaa !2 + call void @consume(i32 noundef %a1) + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[a11:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 2 +; CHECK: %[[v2:.*]] = load i32, ptr %[[a11]], align 128 +; CHECK: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 128} +!3 = !{!"foo", !4, i64 0, !4, i64 128} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-oob.ll @@ -0,0 +1,74 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that getelementptr.and.load unroll can skip 'inbounds' flag. +; +; Source (IR modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; extern void consume(int); +; +; void buz(struct foo *p) { +; consume(p->b.bb); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - +; +; Modified to set 'inbounds' flag to false. + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %bb1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr readonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 false, i32 immarg 0, i32 immarg 1, i32 immarg 1) + #4, !tbaa !2 + call void @consume(i32 noundef %bb1) + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[bb11:.*]] = getelementptr %struct.foo, ptr %[[p]], i32 0, i32 1, i32 1 +; CHECK: %[[v2:.*]] = load i32, ptr %[[bb11]], align 4 +; CHECK: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-u8.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-u8.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain-u8.ll @@ -0,0 +1,68 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.load when direct memory offset is +; used instead of field indexes. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char aa; +; char bb; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; extern void consume(char); +; +; void buz(struct bar *p) { +; consume(((struct foo *)(((char*)&p->b) + 1))->bb); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %bb1 = call i8 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i8 + (ptr readonly elementtype(i8) %p, + i1 false, i8 0, i8 1, i8 0, i1 true, i64 immarg 3) + #4, !tbaa !2 + call void @consume(i8 noundef signext %bb1) + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[bb11:.*]] = getelementptr inbounds i8, ptr %[[p]], i64 3 +; CHECK: %[[v2:.*]] = load i8, ptr %[[bb11]], align 1 +; CHECK: call void @consume(i8 noundef signext %[[v2]]) + +declare void @consume(i8 noundef signext) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i8 @llvm.bpf.getelementptr.and.load.i8(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 1} +!3 = !{!"foo", !4, i64 0, !4, i64 1} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-chain.ll @@ -0,0 +1,73 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.load when several field indexes +; are specified in a chain. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; extern void consume(int); +; +; void buz(struct foo *p) { +; consume(p->b.bb); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %bb1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr readonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1, i32 immarg 1) + #4, !tbaa !2 + call void @consume(i32 noundef %bb1) + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[bb11:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1, i32 1 +; CHECK: %[[v2:.*]] = load i32, ptr %[[bb11]], align 4 +; CHECK: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-simple.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-simple.ll @@ -0,0 +1,65 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.load. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; int b; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p){ +; consume(p->b); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %b1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr readonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) + #4, !tbaa !2 + call void @consume(i32 noundef %b1) + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) #[[v1:.*]] { +; CHECK: %[[b11:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1 +; CHECK-NEXT: %[[v2:.*]] = load i32, ptr %[[b11]], align 4 +; CHECK-NEXT: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } +attributes #4 = { memory(argmem: read) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-volatile.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-volatile.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-undo-volatile.ll @@ -0,0 +1,64 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that unroll of getelementptr.and.load restores volatile. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; volatile int b; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p){ +; consume(p->b); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %b1 = call i32 (ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.load.i32 + (ptr elementtype(%struct.foo) %p, + i1 true, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1), + !tbaa !2 + call void @consume(i32 noundef %b1) + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[b11:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1 +; CHECK: %[[v2:.*]] = load volatile i32, ptr %[[b11]], align 4 +; CHECK: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nounwind willreturn +declare i32 @llvm.bpf.getelementptr.and.load.i32(ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #3 = { nocallback nofree nounwind willreturn } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-union-pai.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-union-pai.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-union-pai.ll @@ -0,0 +1,110 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; #define __pai __attribute__((preserve_access_index)) +; +; struct foo { +; char a[10]; +; } __pai; +; +; struct bar { +; int a; +; int b; +; } __pai; +; +; union buz { +; struct foo a; +; struct bar b; +; } __pai __ctx; +; +; int quux(union buz *p) { +; return p->b.b; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -debug-info-kind=limited -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local i32 @quux(ptr noundef %p) #0 !dbg !5 { +entry: + call void @llvm.dbg.value(metadata ptr %p, metadata !26, metadata !DIExpression()), !dbg !27 + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !28 + %1 = call ptr @llvm.preserve.union.access.index.p0.p0(ptr %0, i32 1), !dbg !28, !llvm.preserve.access.index !10 + %2 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bar) %1, i32 1, i32 1), !dbg !29, !llvm.preserve.access.index !21 + %3 = load i32, ptr %2, align 4, !dbg !29, !tbaa !30 + ret i32 %3, !dbg !33 +} + +; CHECK: define dso_local i32 @quux(ptr noundef %[[p:.*]]) {{.*}} { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.dbg.value +; CHECK-NEXT: %[[v5:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.bar) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; CHECK-SAME: #[[v6:.*]], !tbaa +; CHECK-NEXT: ret i32 %[[v5]] +; CHECK-NEXT: } +; CHECK: attributes #[[v6]] = { memory(argmem: read) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.union.access.index.p0.p0(ptr, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!5 = distinct !DISubprogram(name: "quux", scope: !1, file: !1, line: 18, type: !6, scopeLine: 18, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !25) +!6 = !DISubroutineType(types: !7) +!7 = !{!8, !9} +!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64) +!10 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "buz", file: !1, line: 13, size: 96, elements: !11) +!11 = !{!12, !20} +!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !10, file: !1, line: 14, baseType: !13, size: 80) +!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !1, line: 4, size: 80, elements: !14) +!14 = !{!15} +!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !13, file: !1, line: 5, baseType: !16, size: 80) +!16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 80, elements: !18) +!17 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) +!18 = !{!19} +!19 = !DISubrange(count: 10) +!20 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !10, file: !1, line: 15, baseType: !21, size: 64) +!21 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bar", file: !1, line: 8, size: 64, elements: !22) +!22 = !{!23, !24} +!23 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !21, file: !1, line: 9, baseType: !8, size: 32) +!24 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !21, file: !1, line: 10, baseType: !8, size: 32, offset: 32) +!25 = !{!26} +!26 = !DILocalVariable(name: "p", arg: 1, scope: !5, file: !1, line: 18, type: !9) +!27 = !DILocation(line: 0, scope: !5) +!28 = !DILocation(line: 19, column: 13, scope: !5) +!29 = !DILocation(line: 19, column: 15, scope: !5) +!30 = !{!31, !31, i64 0} +!31 = !{!"omnipotent char", !32, i64 0} +!32 = !{!"Simple C/C++ TBAA"} +!33 = !DILocation(line: 19, column: 3, scope: !5) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll-inline.ll @@ -0,0 +1,108 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check position of bpf-preserve-static-offset pass in the pipeline: +; - preserve.static.offset call is preserved if address is passed as +; a parameter to an inline-able function; +; - second bpf-preserve-static-offset pass (after inlining) should introduce +; getelementptr.and.load call using the preserved marker after loops +; unrolling; +; - readonly and tbaa attributes should allow replacement of +; getelementptr.and.load calls by CSE transformation. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; int b[4]; +; } __ctx; +; +; extern void consume(int); +; +; static inline void bar(int * restrict p) { +; consume(p[1]); +; } +; +; void quux(struct foo *p){ +; unsigned long i = 0; +; #pragma clang loop unroll(full) +; while (i < 2) { +; bar(p->b); +; ++i; +; } +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, [4 x i32] } + +; Function Attrs: nounwind +define dso_local void @quux(ptr noundef %p) #0 { +entry: + br label %while.cond + +while.cond: ; preds = %while.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %while.body ] + %cmp = icmp ult i64 %i.0, 2 + br i1 %cmp, label %while.body, label %while.end + +while.body: ; preds = %while.cond + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %arraydecay = getelementptr inbounds [4 x i32], ptr %b, i64 0, i64 0 + call void @bar(ptr noundef %arraydecay) + %inc = add i64 %i.0, 1 + br label %while.cond, !llvm.loop !2 + +while.end: ; preds = %while.cond + ret void +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: inlinehint nounwind +define internal void @bar(ptr noalias noundef %p) #2 { +entry: + %arrayidx = getelementptr inbounds i32, ptr %p, i64 1 + %0 = load i32, ptr %arrayidx, align 4, !tbaa !5 + call void @consume(i32 noundef %0) + ret void +} + +; CHECK: define dso_local void @quux(ptr nocapture noundef readonly %[[p:.*]]) +; CHECK: %[[v1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 0, i32 immarg 1, i64 immarg 1) +; CHECK: tail call void @consume(i32 noundef %[[v1]]) +; CHECK: tail call void @consume(i32 noundef %[[v1]]) + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #3 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +declare void @consume(i32 noundef) #4 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { inlinehint nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #4 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = distinct !{!2, !3, !4} +!3 = !{!"llvm.loop.mustprogress"} +!4 = !{!"llvm.loop.unroll.full"} +!5 = !{!6, !6, i64 0} +!6 = !{!"int", !7, i64 0} +!7 = !{!"omnipotent char", !8, i64 0} +!8 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-unroll.ll @@ -0,0 +1,95 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check position of bpf-preserve-static-offset pass in the pipeline: +; preserve.static.offset call should be preserved long enough to allow +; introduction of getelementptr.and.load after loops unrolling. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; int b[4]; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p){ +; unsigned long i = 0; +; #pragma clang loop unroll(full) +; while (i < 2) +; consume(p->b[i++]); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, [4 x i32] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + br label %while.cond + +while.cond: ; preds = %while.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %while.body ] + %cmp = icmp ult i64 %i.0, 2 + br i1 %cmp, label %while.body, label %while.end + +while.body: ; preds = %while.cond + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %inc = add i64 %i.0, 1 + %arrayidx = getelementptr inbounds [4 x i32], ptr %b, i64 0, i64 %i.0 + %1 = load i32, ptr %arrayidx, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + br label %while.cond, !llvm.loop !6 + +while.end: ; preds = %while.cond + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef readonly %[[p:.*]]) +; CHECK: %[[v1:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 0, i32 immarg 1, i64 immarg 0) +; CHECK-SAME: #[[attrs:.*]], !tbaa +; CHECK-NEXT: tail call void @consume(i32 noundef %[[v1]]) +; CHECK-NEXT: %[[v2:.*]] = tail call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr readonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 0, i32 immarg 1, i64 immarg 1) +; CHECK-SAME: #[[attrs]], !tbaa +; CHECK-NEXT: tail call void @consume(i32 noundef %[[v2]]) +; CHECK: attributes #[[attrs]] = { memory(argmem: read) } + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +declare void @consume(i32 noundef) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #3 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} +!6 = distinct !{!6, !7, !8} +!7 = !{!"llvm.loop.mustprogress"} +!8 = !{!"llvm.loop.unroll.full"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-volatile.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-volatile.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-volatile.ll @@ -0,0 +1,62 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a volatile load instruction by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int _; +; volatile int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->a); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %1 = load volatile i32, ptr %a, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: %[[a1:.*]] = call i32 (ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.load.i32 +; CHECK-SAME: (ptr elementtype(%struct.foo) %{{[^,]+}}, +; CHECK-SAME: i1 true, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; ^^^^^^^^ +; volatile +; CHECK-NOT: #{{[0-9]+}} +; CHECK-NEXT: call void @consume(i32 noundef %[[a1]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/load-zero.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/load-zero.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/load-zero.ll @@ -0,0 +1,57 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that loads from zero offset are not modified by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p) { +; consume(p->a); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 0 + %1 = load i32, ptr %a, align 4, !tbaa !2 + call void @consume(i32 noundef %1) + ret void +} + +; CHECK: entry: +; CHECK-NEXT: %[[a:.*]] = getelementptr inbounds %struct.foo, ptr %[[p:.*]], i32 0, i32 0 +; CHECK-NEXT: %[[v2:.*]] = load i32, ptr %[[a]], align 4, !tbaa +; CHECK-NEXT: call void @consume(i32 noundef %[[v2]]) + +declare void @consume(i32 noundef) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 0} +!3 = !{!"foo", !4, i64 0} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-align.ll @@ -0,0 +1,59 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a store instruction for a field with non-standard +; alignment by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; typedef int aligned_int __attribute__((aligned(128))); +; +; struct foo { +; int _; +; aligned_int a; +; } __ctx; +; +; void bar(struct foo *p) { +; p->a = 7; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, [124 x i8], i32, [124 x i8] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 2 + store i32 7, ptr %a, align 128, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 7, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 7, i1 true, i32 immarg 0, i32 immarg 2) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 128} +!3 = !{!"foo", !4, i64 0, !4, i64 128} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-atomic.ll @@ -0,0 +1,60 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of atomic store instruction by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int _; +; int a; +; } __ctx; +; +; void bar(struct foo *p) { +; int r; +; r = 7; +; __atomic_store(&p->a, &r, 3); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + store atomic i32 7, ptr %a release, align 4 + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 7, +; CHECK-SAME: ptr elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 5, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; CHECK-NOT: #{{[0-9]+}} +; CHECK-NEXT: ret void + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #2 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-2.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-2.ll @@ -0,0 +1,77 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds GEP chains that end by +; getelementptr.and.store. +; +; Source (modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; void buz(struct foo *p) { +; p->b.bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; And modified to fold last getelementptr/store as a single +; getelementptr.and.store. + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr writeonly elementtype(%struct.bar) %b, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 42, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-oob.ll @@ -0,0 +1,67 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset keeps track of 'inbounds' flags while +; folding chain of GEP instructions. +; +; Source (IR modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; void buz(struct foo *p) { +; p->b.bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; Modified to remove one of the 'inbounds' from one of the GEP instructions. + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %bb = getelementptr %struct.bar, ptr %b, i32 0, i32 1 + store i32 42, ptr %bb, align 4, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 42, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 false, i32 immarg 0, i32 immarg 1, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8-oob.ll @@ -0,0 +1,67 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; The GEP chain in this example has type mismatch and thus is +; folded as i8 access. +; +; Source (modified by hand): +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char aa; +; char bb; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; void buz(struct bar *p) { +; ((struct foo *)(((char*)&p->b) + 1))->bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - +; +; Modified to remove one of the 'inbounds' from one of the getelementptr. + +%struct.bar = type { i8, %struct.foo } +%struct.foo = type { i8, i8 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %add.ptr = getelementptr i8, ptr %b, i64 1 + %bb = getelementptr inbounds %struct.foo, ptr %add.ptr, i32 0, i32 1 + store i8 42, ptr %bb, align 1, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i8, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i8 +; CHECK-SAME: (i8 42, +; CHECK-SAME: ptr writeonly elementtype(i8) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 0, i1 false, i64 immarg 3) +; CHECK-SAME: #[[v2:.*]], !tbaa ![[v3:.*]] +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 1} +!3 = !{!"foo", !4, i64 0, !4, i64 1} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain-u8.ll @@ -0,0 +1,68 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; The GEP chain in this example has type mismatch and thus is +; folded as i8 access. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char aa; +; char bb; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; void buz(struct bar *p) { +; ((struct foo *)(((char*)&p->b) + 1))->bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bar = type { i8, %struct.foo } +%struct.foo = type { i8, i8 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.bar, ptr %0, i32 0, i32 1 + %add.ptr = getelementptr inbounds i8, ptr %b, i64 1 +; ~~ +; these types do not match, thus GEP chain is folded as an offset +; ~~~~~~~~~~~ + %bb = getelementptr inbounds %struct.foo, ptr %add.ptr, i32 0, i32 1 + store i8 42, ptr %bb, align 1, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i8, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i8 +; CHECK-SAME: (i8 42, +; CHECK-SAME: ptr writeonly elementtype(i8) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 0, i1 true, i64 immarg 3) +; CHECK-SAME: #[[v2:.*]], !tbaa ![[v3:.*]] +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 1} +!3 = !{!"foo", !4, i64 0, !4, i64 1} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-chain.ll @@ -0,0 +1,64 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that bpf-preserve-static-offset folds chain of GEP instructions. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; void buz(struct foo *p) { +; p->b.bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %bb = getelementptr inbounds %struct.bar, ptr %b, i32 0, i32 1 + store i32 42, ptr %bb, align 4, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 42, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-pai.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-pai.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-pai.ll @@ -0,0 +1,136 @@ +; RUN: opt -passes=bpf-preserve-static-offset -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; #define __pai __attribute__((preserve_access_index)) +; +; struct foo { +; char a[10]; +; } __pai; +; +; struct bar { +; int a; +; int b; +; } __pai; +; +; struct buz { +; int _1; +; int _2; +; int _3; +; union { +; struct foo a; +; struct bar b[7]; +; }; +; } __pai __ctx; +; +; void quux(struct buz *p) { +; p->b[5].b = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes \ +; -debug-info-kind=limited -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.buz = type { i32, i32, i32, %union.anon } +%union.anon = type { [7 x %struct.bar] } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @quux(ptr noundef %p) #0 !dbg !31 { +entry: + call void @llvm.dbg.value(metadata ptr %p, metadata !36, metadata !DIExpression()), !dbg !37 + %0 = call ptr @llvm.preserve.static.offset(ptr %p), !dbg !38 + %1 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.buz) %0, i32 3, i32 3), !dbg !38, !llvm.preserve.access.index !4 + %2 = call ptr @llvm.preserve.union.access.index.p0.p0(ptr %1, i32 1), !dbg !38, !llvm.preserve.access.index !3 + %3 = call ptr @llvm.preserve.array.access.index.p0.p0(ptr elementtype([7 x %struct.bar]) %2, i32 1, i32 5), !dbg !39, !llvm.preserve.access.index !21 + %4 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bar) %3, i32 1, i32 1), !dbg !40, !llvm.preserve.access.index !22 + store i32 42, ptr %4, align 4, !dbg !41, !tbaa !42 + ret void, !dbg !45 +} + +; CHECK: define dso_local void @quux(ptr noundef %[[p:.*]]) {{.*}} { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.dbg.value +; CHECK-NEXT: call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 42, +; CHECK-SAME: ptr writeonly elementtype(i8) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 56) +; CHECK-SAME: #[[v5:.*]], !tbaa +; CHECK-NEXT: ret void, !dbg +; CHECK-NEXT: } +; CHECK: attributes #[[v5]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.union.access.index.p0.p0(ptr, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.array.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.value(metadata, metadata, metadata) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!28, !29} +!llvm.ident = !{!30} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{!3, !21} +!3 = distinct !DICompositeType(tag: DW_TAG_union_type, scope: !4, file: !1, line: 17, size: 448, elements: !11) +!4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "buz", file: !1, line: 13, size: 544, elements: !5) +!5 = !{!6, !8, !9, !10} +!6 = !DIDerivedType(tag: DW_TAG_member, name: "_1", scope: !4, file: !1, line: 14, baseType: !7, size: 32) +!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!8 = !DIDerivedType(tag: DW_TAG_member, name: "_2", scope: !4, file: !1, line: 15, baseType: !7, size: 32, offset: 32) +!9 = !DIDerivedType(tag: DW_TAG_member, name: "_3", scope: !4, file: !1, line: 16, baseType: !7, size: 32, offset: 64) +!10 = !DIDerivedType(tag: DW_TAG_member, scope: !4, file: !1, line: 17, baseType: !3, size: 448, offset: 96) +!11 = !{!12, !20} +!12 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 18, baseType: !13, size: 80) +!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo", file: !1, line: 4, size: 80, elements: !14) +!14 = !{!15} +!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !13, file: !1, line: 5, baseType: !16, size: 80) +!16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 80, elements: !18) +!17 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) +!18 = !{!19} +!19 = !DISubrange(count: 10) +!20 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !3, file: !1, line: 19, baseType: !21, size: 448) +!21 = !DICompositeType(tag: DW_TAG_array_type, baseType: !22, size: 448, elements: !26) +!22 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bar", file: !1, line: 8, size: 64, elements: !23) +!23 = !{!24, !25} +!24 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !22, file: !1, line: 9, baseType: !7, size: 32) +!25 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !22, file: !1, line: 10, baseType: !7, size: 32, offset: 32) +!26 = !{!27} +!27 = !DISubrange(count: 7) +!28 = !{i32 2, !"Debug Info Version", i32 3} +!29 = !{i32 1, !"wchar_size", i32 4} +!30 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!31 = distinct !DISubprogram(name: "quux", scope: !1, file: !1, line: 23, type: !32, scopeLine: 23, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !35) +!32 = !DISubroutineType(types: !33) +!33 = !{null, !34} +!34 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !4, size: 64) +!35 = !{!36} +!36 = !DILocalVariable(name: "p", arg: 1, scope: !31, file: !1, line: 23, type: !34) +!37 = !DILocation(line: 0, scope: !31) +!38 = !DILocation(line: 24, column: 6, scope: !31) +!39 = !DILocation(line: 24, column: 3, scope: !31) +!40 = !DILocation(line: 24, column: 11, scope: !31) +!41 = !DILocation(line: 24, column: 13, scope: !31) +!42 = !{!43, !43, i64 0} +!43 = !{!"omnipotent char", !44, i64 0} +!44 = !{!"Simple C/C++ TBAA"} +!45 = !DILocation(line: 25, column: 1, scope: !31) diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-simple.ll @@ -0,0 +1,60 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a simple store instruction by bpf-preserve-static-offset. +; Verify: +; - presence of gep.and.store intrinsic call +; - correct attributes for intrinsic call +; - presence of tbaa annotations +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int _; +; int a; +; } __ctx; +; +; void bar(struct foo *p) { +; p->a = 7; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + store i32 7, ptr %a, align 4, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 7, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) +; CHECK-SAME: #[[v2:.*]], !tbaa +; CHECK: attributes #[[v2]] = { memory(argmem: write) } + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-align.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-align.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-align.ll @@ -0,0 +1,62 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that getelementptr.and.store unroll restores alignment spec. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; typedef int aligned_int __attribute__((aligned(128))); +; +; struct foo { +; int _; +; aligned_int a; +; } __ctx; +; +; void bar(struct foo *p) { +; p->a = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, [124 x i8], i32, [124 x i8] } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr writeonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 7, i1 true, i32 immarg 0, i32 immarg 2) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[v2:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 2 +; CHECK: store i32 42, ptr %[[v2]], align 128 +; CHECK: ret void + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 128} +!3 = !{!"foo", !4, i64 0, !4, i64 128} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-oob.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-oob.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-oob.ll @@ -0,0 +1,67 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that getelementptr.and.load unroll can skip 'inbounds' flag. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; void buz(struct foo *p) { +; p->b.bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr writeonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 false, i32 immarg 0, i32 immarg 1, i32 immarg 1) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[v2:.*]] = getelementptr %struct.foo, ptr %[[p]], i32 0, i32 1, i32 1 +; CHECK: store i32 42, ptr %[[v2]], align 4 +; CHECK: ret void + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-u8.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-u8.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain-u8.ll @@ -0,0 +1,62 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.store when direct memory offset is +; used instead of field indexes. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; char aa; +; char bb; +; }; +; +; struct bar { +; char a; +; struct foo b; +; } __ctx; +; +; void buz(struct bar *p) { +; ((struct foo *)(((char*)&p->b) + 1))->bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + call void (i8, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i8 + (i8 42, + ptr writeonly elementtype(i8) %p, + i1 false, i8 0, i8 1, i8 0, i1 true, i64 immarg 3) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[v2:.*]] = getelementptr inbounds i8, ptr %[[p]], i64 3 +; CHECK: store i8 42, ptr %[[v2]], align 1 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i8(i8, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 1} +!3 = !{!"foo", !4, i64 0, !4, i64 1} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-chain.ll @@ -0,0 +1,68 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.store when several field indexes +; are specified in a chain. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct bar { +; int aa; +; int bb; +; }; +; +; struct foo { +; int a; +; struct bar b; +; } __ctx; +; +; void buz(struct foo *p) { +; p->b.bb = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, %struct.bar } +%struct.bar = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @buz(ptr noundef %p) #0 { +entry: + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr writeonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1, i32 immarg 1) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @buz(ptr noundef %[[p:.*]]) +; CHECK: %[[v2:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1, i32 1 +; CHECK: store i32 42, ptr %[[v2]], align 4 +; CHECK: ret void + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 8} +!3 = !{!"foo", !4, i64 0, !7, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!"bar", !4, i64 0, !4, i64 4} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-simple.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-simple.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-simple.ll @@ -0,0 +1,61 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check unroll of getelementptr.and.store. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; int b; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p){ +; p->b = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr writeonly elementtype(%struct.foo) %p, + i1 false, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1) + #3, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: %[[v2:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1 +; CHECK: store i32 42, ptr %[[v2]], align 4 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } +attributes #3 = { memory(argmem: write) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-volatile.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-volatile.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-undo-volatile.ll @@ -0,0 +1,61 @@ +; RUN: opt --bpf-check-and-opt-ir -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that unroll of getelementptr.and.store restores volatile. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; volatile int b; +; } __ctx; +; +; extern void consume(int); +; +; void bar(struct foo *p){ +; p->b = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=sroa,bpf-preserve-static-offset -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + call void (i32, ptr, i1, i8, i8, i8, i1, ...) + @llvm.bpf.getelementptr.and.store.i32 + (i32 42, + ptr elementtype(%struct.foo) %p, + i1 true, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1), + !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr noundef %[[p:.*]]) +; CHECK: entry: +; CHECK: %[[v2:.*]] = getelementptr inbounds %struct.foo, ptr %[[p]], i32 0, i32 1 +; CHECK: store volatile i32 42, ptr %[[v2]], align 4 + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +; Function Attrs: nocallback nofree nounwind willreturn +declare void @llvm.bpf.getelementptr.and.store.i32(i32, ptr nocapture, i1 immarg, i8 immarg, i8 immarg, i8 immarg, i1 immarg, ...) #2 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nounwind willreturn } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-unroll-inline.ll @@ -0,0 +1,104 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check position of bpf-preserve-static-offset pass in the pipeline: +; - preserve.static.offset call is preserved if address is passed as +; a parameter to an inline-able function; +; - second bpf-preserve-static-offset pass (after inlining) should introduce +; getelementptr.and.store call using the preserved marker after loops +; unrolling; +; - memory(argmem: readwrite) and tbaa attributes should allow +; removing one getelementptr.and.store call. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; int b[4]; +; } __ctx; +; +; static inline void bar(int * restrict p, unsigned long i) { +; p[0] = i; +; } +; +; void quux(struct foo *p){ +; unsigned long i = 0; +; #pragma clang loop unroll(full) +; while (i < 2) { +; bar(p->b, i); +; ++i; +; } +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, [4 x i32] } + +; Function Attrs: nounwind +define dso_local void @quux(ptr noundef %p) #0 { +entry: + br label %while.cond + +while.cond: ; preds = %while.body, %entry + %i.0 = phi i64 [ 0, %entry ], [ %inc, %while.body ] + %cmp = icmp ult i64 %i.0, 2 + br i1 %cmp, label %while.body, label %while.end + +while.body: ; preds = %while.cond + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + %arraydecay = getelementptr inbounds [4 x i32], ptr %b, i64 0, i64 0 + call void @bar(ptr noundef %arraydecay, i64 noundef %i.0) + %inc = add i64 %i.0, 1 + br label %while.cond, !llvm.loop !2 + +while.end: ; preds = %while.cond + ret void +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture) #1 + +; Function Attrs: inlinehint nounwind +define internal void @bar(ptr noalias noundef %p, i64 noundef %i) #2 { +entry: + %conv = trunc i64 %i to i32 + %arrayidx = getelementptr inbounds i32, ptr %p, i64 0 + store i32 %conv, ptr %arrayidx, align 4, !tbaa !5 + ret void +} + +; CHECK: define dso_local void @quux(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK-NEXT: entry: +; CHECK-NEXT: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 1, +; CHECK-SAME: ptr writeonly elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 false, i8 0, i8 1, i8 2, i1 true, i64 immarg 0, i32 immarg 1) +; CHECK-NEXT: ret void + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #3 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) +declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) } +attributes #2 = { inlinehint nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #3 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = distinct !{!2, !3, !4} +!3 = !{!"llvm.loop.mustprogress"} +!4 = !{!"llvm.loop.unroll.full"} +!5 = !{!6, !6, i64 0} +!6 = !{!"int", !7, i64 0} +!7 = !{!"omnipotent char", !8, i64 0} +!8 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-volatile.ll @@ -0,0 +1,56 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check handling of a volatile store instruction by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; volatile int b; +; } __ctx; +; +; void bar(struct foo *p) { +; p->b = 42; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32, i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %b = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 1 + store volatile i32 42, ptr %b, align 4, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef %[[p:.*]]) +; CHECK: tail call void (i32, ptr, i1, i8, i8, i8, i1, ...) +; CHECK-SAME: @llvm.bpf.getelementptr.and.store.i32 +; CHECK-SAME: (i32 42, +; CHECK-SAME: ptr elementtype(%struct.foo) %[[p]], +; CHECK-SAME: i1 true, i8 0, i8 1, i8 2, i1 true, i32 immarg 0, i32 immarg 1), +; CHECK-NOT: #{{[0-9]+}} +; CHECK-SAME: !tbaa + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"foo", !4, i64 0, !4, i64 4} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll b/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/preserve-static-offset/store-zero.ll @@ -0,0 +1,51 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -S -o - %s | FileCheck %s +; +; Check that stores from zero offset are not modified by bpf-preserve-static-offset. +; +; Source: +; #define __ctx __attribute__((preserve_static_offset)) +; +; struct foo { +; int a; +; } __ctx; +; +; void bar(struct foo *p) { +; p->a = 0; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.foo = type { i32 } + +; Function Attrs: nounwind +define dso_local void @bar(ptr noundef %p) #0 { +entry: + %0 = call ptr @llvm.preserve.static.offset(ptr %p) + %a = getelementptr inbounds %struct.foo, ptr %0, i32 0, i32 0 + store i32 0, ptr %a, align 4, !tbaa !2 + ret void +} + +; CHECK: define dso_local void @bar(ptr nocapture noundef writeonly %[[p:.*]]) +; CHECK-NEXT: entry: +; CHECK-NEXT: store i32 0, ptr %[[p]], align 4, !tbaa +; CHECK-NEXT: ret void + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare ptr @llvm.preserve.static.offset(ptr readnone) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang c899a1ca75d0f1b559204eff79a2578d2cafc7ab)"} +!2 = !{!3, !4, i64 0} +!3 = !{!"foo", !4, i64 0} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"}