diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -62,6 +62,15 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; + +class BPFHoistArgumentAccessPass + : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + } // namespace llvm #endif diff --git a/llvm/lib/Target/BPF/BPFHoistArgumentAccess.cpp b/llvm/lib/Target/BPF/BPFHoistArgumentAccess.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFHoistArgumentAccess.cpp @@ -0,0 +1,361 @@ +// ===------ BPFHoistArgumentAccess.cpp ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// ===----------------------------------------------------------------------===// +// +// BPF verifier limits access patterns allowed for certain data +// types. E.g. struct __sk_buff and struct bpf_sock_ops. For these +// types only BASE + static-offset memory loads are allowed. +// +// This is so because offsets of the fields of these structures do not +// match real offsets in the running kernel. During BPF program +// load/verification loads and stores to the fields of these types are +// rewritten so that offsets match real offsets. For this rewrite to +// happen static offsets have to be encoded in the instructions. +// +// See kernel/bpf/verifier.c:convert_ctx_access function in the Linux +// kernel source tree for details. +// +// During instruction selection phase the following sequence of +// instructions: +// +// %x = getelementptr %ptr, %imm +// %y = load %x +// +// Is translated as a single load instruction with embedded offset, +// e.g. 'LDW %ptr, %imm', which matches access pattern necessary for +// the restricted set of types described above. +// +// However, several optimization passes might sink load/store instruction +// or hoist getelementptr instruction so that load/store pointer operand +// becomes a PHI. +// +// The BPFHoistArgumentAccess undoes sinking of the load / store +// instructions by hoisting load/store instructions to predecessor basic +// blocks: +// - transformation looks for loads or stores where base pointer is +// defined via phi and some of the incoming values are getelementptr +// such that: +// - pointer operand is a function argument +// - all indices are constant; +// (alternatively, offset via preserve access index constructs is +// also recognized) +// - if some predecessor basic block has multiple successors an +// intermediate basic block is inserted between predecessor and phi's +// basic block (thus guaranteeing that all predecessors have single +// successor); +// - if some incoming values for predecessor basic block do not match the +// getelementptr pattern a single intermediate basic block is is shared +// by all these predecessors and phi's basic block; +// - load/store instruction and all instructions preceding it are hoisted +// to predecessor basic blocks; +// +// For example: +// +// bb.then.1: bb.then.1: +// %a = getelementptr br label %if.then.2 +// br label %if.then.2 +// bb.then.2: +// bb.then.2: %a1 = getelementptr +// br label %if.end call void @foo +// %l1 = load %a1 +// bb.else: --> br label %if.end +// %b = call ptr @bar +// br label %if.end bb.else: +// %b = call ptr @bar +// if.end: call void @foo +// %p = phi [%a, label %bb.then.2], %l2 = load %b +// [%b, label %if.end] br label %if.end +// call void @foo +// %l = load %p if.end: +// %l = phi [%l1, label %bb.then.2], +// [%l2, label %if.end] + +#include "BPF.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/ValueMapper.h" + +#define DEBUG_TYPE "bpf-hoist-argument-access" + +using namespace llvm; + +[[noreturn]] static void reportUnexpected(Value *Value) { + SmallString<64> Str; + raw_svector_ostream Stream(Str); + Stream << *Value; + report_fatal_error(Twine("Unexpected value ").concat(Str)); +} + +using GEPChain = SmallVector; + +GEPChain getHoistGEPChain(Value *Root) { + GEPChain Chain; + if (auto *GEP = dyn_cast(Root)) { + Chain.push_back(GEP); + } else if (auto *Call = dyn_cast(Root)) { + Chain.push_back(cast(Call->getArgOperand(1))); + Chain.push_back(Call); + } else { + reportUnexpected(Root); + } + return Chain; +} + +static bool isStaticArgumentAccess(Value *Insn) { + if (auto *GEP = dyn_cast(Insn)) { + // Access to function argument via GEP with static offset. + // Containing BB should have single successor. + if (!GEP->hasAllConstantIndices()) + return false; + if (!isa(GEP->getPointerOperand())) + return false; + } else if (auto *Call = dyn_cast(Insn)) { + // Access to function argument via preserve access index: + // + // @"llvm.foo:0:20$0:3" = external global i64, !llvm.preserve.access.index !0 + // ... + // %field.offset = load i64, ptr @"llvm.foo:0:20$0:3" + // %ptr.offset = getelementptr i8, ptr %argument, i64 %field.offset + // %ptr.result = call ptr @llvm.bpf.passthrough.p0.p0(i32 0, ptr %ptr.offset) + // + // Call BB should have a single successor, call and GEP should be + // in the same BB. + if (Call->getIntrinsicID() != Intrinsic::bpf_passthrough) + return false; + auto *GEP = dyn_cast(Call->getArgOperand(1)); + if (!GEP) + return false; + if (GEP->getNumOperands() != 2) + return false; + if (!isa(GEP->getPointerOperand())) + return false; + auto *OffsetLoad = dyn_cast(GEP->getOperand(1)); + if (!OffsetLoad) + return false; + auto *GV = dyn_cast(OffsetLoad->getPointerOperand()); + if (!GV) + return false; + if (!GV->getMetadata(LLVMContext::MD_preserve_access_index)) + return false; + } else + return false; + return true; +} + +static Value *getPointerOperand(Instruction *Insn) { + if (auto *Load = dyn_cast(Insn)) + return Load->getPointerOperand(); + if (auto *Store = dyn_cast(Insn)) + return Store->getPointerOperand(); + return nullptr; +} + +static Instruction *copyToBB(GEPChain &Chain, BasicBlock *BB) { + Instruction *Clone = nullptr; + IRBuilder<> Builder(BB->getTerminator()); + ValueToValueMapTy VMap; + for (Instruction *I : Chain) { + Clone = I->clone(); + VMap[I] = Clone; + RemapInstruction(Clone, VMap, RF_IgnoreMissingLocals | RF_NoModuleLevelChanges); + Builder.Insert(Clone, I->getName()); + } + assert(Clone); + return Clone; +} + +static void eraseUnused(GEPChain &Chain) { + for (Instruction *I : reverse(Chain)) + if (I->use_empty()) + I->eraseFromParent(); +} + +static void ensureSingleSuccessor(PHINode *Phi) { + SmallVector CatchAll; + SmallVector NeedSplit; + for (unsigned I = 0; I < Phi->getNumIncomingValues(); ++I) { + Value *Incoming = Phi->getIncomingValue(I); + BasicBlock *BB = Phi->getIncomingBlock(I); + if (isStaticArgumentAccess(Incoming)) { + if (BB->getSingleSuccessor()) + continue; + NeedSplit.push_back(BB); + continue; + } + CatchAll.push_back(BB); + } + for (BasicBlock *BB : NeedSplit) + SplitBlockPredecessors(Phi->getParent(), {BB}, ".sa.lh"); + if (CatchAll.size() > 1 || + (CatchAll.size() == 1 && !CatchAll[0]->getSinglePredecessor())) + SplitBlockPredecessors(Phi->getParent(), CatchAll, ".lh"); + for (unsigned I = 0; I < Phi->getNumIncomingValues(); ++I) { + Value *Incoming = Phi->getIncomingValue(I); + BasicBlock *BB = Phi->getIncomingBlock(I); + if (!isStaticArgumentAccess(Incoming)) + continue; + GEPChain Chain = getHoistGEPChain(Incoming); + if (all_of(Chain, [&](Instruction *I) { return I->getParent() == BB; })) + continue; + Phi->setIncomingValue(I, copyToBB(Chain, BB)); + eraseUnused(Chain); + } +} + +static bool hasUsesAfter(Instruction *Insn, Instruction *After) { + for (User *U: Insn->users()) { + Instruction *UI = dyn_cast(U); + if (!UI) + continue; + if (UI->getParent() != After->getParent()) + return true; + if (After->comesBefore(UI)) + return true; + } + return false; +} + +static void hoistHeadToPredecessors(PHINode *Phi, Instruction *UpTo) { + ensureSingleSuccessor(Phi); + BasicBlock *ThisBB = UpTo->getParent(); + unsigned NumPreds = Phi->getNumIncomingValues(); + // Prepare replacement PHI nodes for instructions that would be + // hoisted to predecessor blocks: + // - HoistMap[&Insn] == nullptr if &Insn is used only + // in the part of ThisBB that is being hoisted; + // - HoistMap[&Insn] == PHINode otherwise. + SmallDenseMap HoistMap; + SmallVector Hoisted; + for (Instruction &Insn : *ThisBB) { + if (isa(&Insn)) { + if (!hasUsesAfter(&Insn, UpTo)) { + HoistMap[&Insn] = nullptr; + Hoisted.push_back(&Insn); + } + } else { + HoistMap[&Insn] = hasUsesAfter(&Insn, UpTo) + ? PHINode::Create(Insn.getType(), NumPreds, Insn.getName()) + : nullptr; + Hoisted.push_back(&Insn); + } + if (&Insn == UpTo) + break; + } + // For each predecessor BB, copy instructions [ThisBB->begin()..UpTo] + // from ThisBB to predecessor BB. + for (unsigned I = 0; I < NumPreds; ++I) { + BasicBlock *PredBB = Phi->getIncomingBlock(I); + ValueToValueMapTy PredVMap; + IRBuilder<> Builder(PredBB->getTerminator()); + for (Instruction &Insn : *ThisBB) { + if (auto *SrcPhi = dyn_cast(&Insn)) { + PredVMap[SrcPhi] = SrcPhi->getIncomingValue(I); + continue; + } + Instruction *Clone = Insn.clone(); + PredVMap[&Insn] = Clone; + RemapInstruction(Clone, PredVMap, RF_IgnoreMissingLocals | RF_NoModuleLevelChanges); + Builder.Insert(Clone, Clone->getName()); + // If &Insn is used in the remaining part of ThisBB + // add incoming value for corresponding PHI. + if (auto *NewPhi = HoistMap[&Insn]) + NewPhi->addIncoming(Clone, PredBB); + if (&Insn == UpTo) + break; + } + } + // Replace cloned instructions with PHI nodes. + IRBuilder<> Builder(ThisBB->getFirstNonPHI()); + for (Instruction *Insn: reverse(Hoisted)) { + PHINode *ReplacingPHI = HoistMap[Insn]; + if (ReplacingPHI) { + Insn->replaceAllUsesWith(ReplacingPHI); + Builder.Insert(ReplacingPHI, Insn->getName()); + } + Insn->eraseFromParent(); + } +} + +static bool expandSelects(Function &F) { + SmallVector Worklist; + for (BasicBlock &BB : F) + for (Instruction &Insn : BB) { + Value *Pointer = getPointerOperand(&Insn); + if (!Pointer) + continue; + SelectInst *Select = dyn_cast(Pointer); + if (!Select) + continue; + if (isStaticArgumentAccess(Select->getTrueValue()) || + isStaticArgumentAccess(Select->getFalseValue())) + Worklist.push_back(Select); + } + + if (Worklist.empty()) + return false; + + for (SelectInst *Select : Worklist) { + LLVM_DEBUG(dbgs() << "expanding " << *Select << "\n"); + StringRef BaseName = Select->getParent()->getName(); + BasicBlock *TrueBB = nullptr; + BasicBlock *FalseBB = nullptr; + SplitBlockAndInsertIfThenElse(Select->getCondition(), Select, &TrueBB, &FalseBB); + PHINode *Phi = PHINode::Create(Select->getType(), 2, Select->getName(), Select); + Phi->addIncoming(Select->getTrueValue(), TrueBB); + Phi->addIncoming(Select->getFalseValue(), FalseBB); + TrueBB->setName(Twine(BaseName).concat(".select.true")); + FalseBB->setName(Twine(BaseName).concat(".select.false")); + Phi->getParent()->setName(Twine(BaseName).concat(".select.tail")); + Select->replaceAllUsesWith(Phi); + Select->eraseFromParent(); + } + + return true; +} + +static bool shouldHoist(Instruction *Insn) { + Value *Pointer = getPointerOperand(Insn); + if (!Pointer) + return false; + auto *Phi = dyn_cast(Pointer); + if (!Phi || Phi->getParent() != Insn->getParent()) + return false; + for (Use &Incoming : Phi->incoming_values()) + if (isStaticArgumentAccess(Incoming)) + return true; + return false; +} + +static bool hoistGetElementPtrLoads(Function &F) { + SmallVector Worklist; + for (BasicBlock &BB : F) + for (Instruction &Insn : BB) + if (shouldHoist(&Insn)) + Worklist.push_back(&Insn); + LLVM_DEBUG(dbgs() << "Going to hoist " << Worklist.size() + << " loads/stores\n"); + if (Worklist.empty()) + return false; + for (Instruction *Insn: Worklist) { + LLVM_DEBUG(dbgs() << "Hoisting " << *Insn << "\n"); + Value *Pointer = getPointerOperand(Insn); + if (!Pointer || !isa(Pointer)) + reportUnexpected(Insn); + hoistHeadToPredecessors(cast(Pointer), Insn); + } + return true; +} + +PreservedAnalyses BPFHoistArgumentAccessPass::run(Function &F, + FunctionAnalysisManager &AM) { + bool Changed = false; + Changed |= expandSelects(F); + Changed |= hoistGetElementPtrLoads(F); + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -105,6 +105,10 @@ FPM.addPass(BPFIRPeepholePass()); return true; } + if (PassName == "bpf-hoist-argument-access") { + FPM.addPass(BPFHoistArgumentAccessPass()); + return true; + } return false; }); PB.registerPipelineStartEPCallback( @@ -123,6 +127,12 @@ [=](ModulePassManager &MPM, OptimizationLevel) { MPM.addPass(BPFAdjustOptPass()); }); + PB.registerOptimizerLastEPCallback( + [=](ModulePassManager &MPM, OptimizationLevel) { + FunctionPassManager FPM; + FPM.addPass(BPFHoistArgumentAccessPass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + }); } void BPFPassConfig::addIRPasses() { diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -20,6 +20,7 @@ BPFAsmPrinter.cpp BPFCheckAndAdjustIR.cpp BPFFrameLowering.cpp + BPFHoistArgumentAccess.cpp BPFInstrInfo.cpp BPFIRPeephole.cpp BPFISelDAGToDAG.cpp diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-1.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-1.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-1.ll @@ -0,0 +1,489 @@ +; RUN: opt -S -passes=bpf-hoist-argument-access -mtriple=bpf-pc-linux < %s | FileCheck %s + +%struct.foo = type { i32, i32, i32 } +declare void @consume(i32 noundef) +declare i32 @produce() +@gptr = external global ptr +@gptr2 = external global ptr + +define void @load.1(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @load.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[T1:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[V2:%.*]] = phi i32 [ [[T0]], %if.then ], [ [[T1]], %if.else ] +; CHECK-NEXT: call void @consume(i32 [[V2]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @load.one_hand_nonconst.1(ptr %ctx, i1 %cond, i32 %idx) { +; CHECK-LABEL: define void @load.one_hand_nonconst.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]], i32 [[IDX:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 [[IDX]], i32 1 +; CHECK-NEXT: [[T1:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[V2:%.*]] = phi i32 [ [[T0]], %if.then ], [ [[T1]], %if.else ] +; CHECK-NEXT: call void @consume(i32 [[V2]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 %idx, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @load.one_hand_nonarg.1(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @load.one_hand_nonarg.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 0 +; CHECK-NEXT: [[T1:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[V2:%.*]] = phi i32 [ [[T1]], %if.then ], [ [[T0]], %if.else ] +; CHECK-NEXT: call void @consume(i32 [[V2]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @load.one_hand_many_succ.1(ptr %ctx, i1 %cond, i1 %cond2) { +; CHECK: define void @load.one_hand_many_succ.1(ptr %[[ctx:.*]], i1 %[[cond:.*]], i1 %[[cond2:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 %[[cond]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: br i1 %[[cond]], label %if.end.sa.lh, label %if.else +; CHECK: if.else: +; CHECK-NEXT: %[[p2:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i32 0, i32 1 +; CHECK-NEXT: %[[T0:.*]] = load i32, ptr %[[p2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end.sa.lh: +; CHECK-NEXT: %[[p11:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i32 0, i32 0 +; CHECK-NEXT: %[[T1:.*]] = load i32, ptr %[[p11]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: %[[v3:.*]] = phi i32 [ %[[T0]], %if.else ], +; CHECK-SAME: [ %[[T1]], %if.end.sa.lh ] +; CHECK-NEXT: call void @consume(i32 %[[v3]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br i1 %cond, label %if.end, label %if.else +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @noop.1(i1 %cond) { +; CHECK-LABEL: define void @noop.1 +; CHECK-SAME: (i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 0 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 1 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P1]], %if.then ], [ [[P2]], %if.else ] +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void @consume(i32 [[V]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @noop.2(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @noop.2 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[P1]], %if.then ], [ [[P2]], %if.else ] +; CHECK-NEXT: br label %if.end.2 +; CHECK: if.end.2: +; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P]], align 4 +; CHECK-NEXT: call void @consume(i32 [[V]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + br label %if.end.2 +if.end.2: + %v = load i32, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @store.1(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @store.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: store i32 42, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: store i32 42, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + store i32 42, ptr %p + ret void +} + +define void @two_times.no_uses_after.1(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @two_times.no_uses_after.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: store i32 [[T0]], ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[T1:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: store i32 [[T1]], ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + store i32 %v, ptr %p + ret void +} + +define void @two_times.with_use_after.2(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @two_times.with_use_after.2 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[T0:%.*]] = load i32, ptr [[P1]], align 4 +; CHECK-NEXT: store i32 [[T0]], ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[T1:%.*]] = load i32, ptr [[P2]], align 4 +; CHECK-NEXT: store i32 [[T1]], ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[V2:%.*]] = phi i32 [ [[T0]], %if.then ], [ [[T1]], %if.else ] +; CHECK-NEXT: call void @consume(i32 [[V2]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %v = load i32, ptr %p + store i32 %v, ptr %p + call void @consume(i32 %v) + ret void +} + +define void @phi.1(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @phi.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[A:%.*]] = load i32, ptr @gptr, align 4 +; CHECK-NEXT: store i32 [[A]], ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[B:%.*]] = load i32, ptr @gptr2, align 4 +; CHECK-NEXT: store i32 [[B]], ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + %a = load i32, ptr @gptr + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + %b = load i32, ptr @gptr2 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %ab = phi i32 [%a, %if.then], [%b, %if.else] + store i32 %ab, ptr %p + ret void +} + +define void @phi.2(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @phi.2 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[A:%.*]] = load i32, ptr @gptr, align 4 +; CHECK-NEXT: store i32 42, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[B:%.*]] = load i32, ptr @gptr2, align 4 +; CHECK-NEXT: store i32 42, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: [[AB:%.*]] = phi i32 [ [[A]], %if.then ], [ [[B]], %if.else ] +; CHECK-NEXT: call void @consume(i32 [[AB]]) +; CHECK-NEXT: ret void +; +entry: + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + %a = load i32, ptr @gptr + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + %b = load i32, ptr @gptr2 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %ab = phi i32 [%a, %if.then], [%b, %if.else] + store i32 42, ptr %p + call void @consume(i32 %ab) + ret void +} + +define void @phi.3(ptr %ctx, i1 %cond) { +; CHECK-LABEL: define void @phi.3 +; CHECK-SAME: (ptr [[CTX:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[K:%.*]] = call i32 @produce() +; CHECK-NEXT: br i1 [[COND]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: [[A:%.*]] = load i32, ptr @gptr, align 4 +; CHECK-NEXT: call void @consume(i32 [[K]]) +; CHECK-NEXT: store i32 42, ptr [[P1]], align 4 +; CHECK-NEXT: call void @consume(i32 [[A]]) +; CHECK-NEXT: store i32 7, ptr [[P1]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: [[B:%.*]] = load i32, ptr @gptr2, align 4 +; CHECK-NEXT: call void @consume(i32 [[K]]) +; CHECK-NEXT: store i32 42, ptr [[P2]], align 4 +; CHECK-NEXT: call void @consume(i32 [[B]]) +; CHECK-NEXT: store i32 7, ptr [[P2]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: ret void +; +entry: + %k = call i32 @produce() + br i1 %cond, label %if.then, label %if.else +if.then: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + %a = load i32, ptr @gptr + br label %if.end +if.else: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + %b = load i32, ptr @gptr2 + br label %if.end +if.end: + %p = phi ptr [%p1, %if.then], [%p2, %if.else] + %ab = phi i32 [%a, %if.then], [%b, %if.else] + ;; value 'k' does not depend on phi or GEP chains that would be + ;; lifted, check that implementation handles such cases. + call void @consume(i32 %k) + store i32 42, ptr %p + call void @consume(i32 %ab) + store i32 7, ptr %p + ret void +} + +define void @switch.1(ptr %ctx, i32 %smth) { +; CHECK-LABEL: define void @switch.1 +; CHECK-SAME: (ptr [[CTX:%.*]], i32 [[SMTH:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: switch i32 [[SMTH]], label [[EXIT:%.*]] [ +; CHECK-NEXT: i32 1, label [[BB1:%.*]] +; CHECK-NEXT: i32 2, label [[BB2:%.*]] +; CHECK-NEXT: i32 3, label [[BB3:%.*]] +; CHECK-NEXT: i32 4, label [[BB4:%.*]] +; CHECK-NEXT: ] +; CHECK: bb1: +; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 0 +; CHECK-NEXT: store i32 42, ptr [[P1]], align 4 +; CHECK-NEXT: br label %end +; CHECK: bb2: +; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds %struct.foo, ptr [[CTX]], i32 0, i32 1 +; CHECK-NEXT: store i32 42, ptr [[P2]], align 4 +; CHECK-NEXT: br label %end +; CHECK: bb3: +; CHECK-NEXT: [[P3:%.*]] = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 1 +; CHECK-NEXT: br label %end.lh +; CHECK: bb4: +; CHECK-NEXT: [[P4:%.*]] = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 2 +; CHECK-NEXT: br label %end.lh +; CHECK: end.lh: +; CHECK-NEXT: [[P_PH:%.*]] = phi ptr [ [[P4]], [[BB4]] ], [ [[P3]], [[BB3]] ] +; CHECK-NEXT: store i32 42, ptr [[P_PH]], align 4 +; CHECK-NEXT: br label %end +; CHECK: end: +; CHECK-NEXT: br label %exit +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + switch i32 %smth, label %exit [ + i32 1, label %bb1 + i32 2, label %bb2 + i32 3, label %bb3 + i32 4, label %bb4 + ] +bb1: + %p1 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 0 + br label %end +bb2: + %p2 = getelementptr inbounds %struct.foo, ptr %ctx, i32 0, i32 1 + br label %end +bb3: + %p3 = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 1 + br label %end +bb4: + %p4 = getelementptr inbounds %struct.foo, ptr @gptr, i32 0, i32 2 + br label %end +end: + %p = phi ptr [%p1, %bb1], [%p2, %bb2], [%p3, %bb3], [%p4, %bb4] + store i32 42, ptr %p + br label %exit +exit: + ret void +} diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-2.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-2.ll @@ -0,0 +1,117 @@ +; RUN: opt -S -passes=bpf-hoist-argument-access -mtriple=bpf-pc-linux < %s | FileCheck %s + +; Input generated from the following C code: +; +; #define __pai __attribute__((preserve_access_index)) +; +; struct bpf_sock { +; int bound_dev_if; +; int family; +; } __ctx; +; +; struct bpf_sockopt { +; int _; +; struct bpf_sock *sk; +; int level; +; int optlen; +; } __pai; +; +; extern void magic2(int); +; +; void known_load_sink_example_1(struct bpf_sockopt *ctx) +; { +; unsigned g = 0; +; if (ctx->level == 42) +; magic2(ctx->sk->family); +; else +; magic2(ctx->optlen); +; } +; +; Using command: +; +; clang --target=bpf -emit-llvm -g -O2 -S -o - +; +; Plus some manual cleanup of unused metadata. + +%struct.bpf_sock = type { i32, i32 } + +@"llvm.bpf_sockopt:0:16$0:2" = external global i64, !llvm.preserve.access.index !0 #0 +@"llvm.bpf_sockopt:0:20$0:3" = external global i64, !llvm.preserve.access.index !0 #0 +@"llvm.bpf_sockopt:0:8$0:1" = external global i64, !llvm.preserve.access.index !0 #0 + +define void @known_load_sink_example_1(ptr noundef readonly %ctx) { +; CHECK: define void @known_load_sink_example_1(ptr noundef readonly %[[ctx:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %[[T0:.*]] = load i64, ptr @"llvm.bpf_sockopt:0:16$0:2", align 8 +; CHECK-NEXT: %[[T1:.*]] = getelementptr i8, ptr %[[ctx]], i64 %[[T0]] +; CHECK-NEXT: %[[T2:.*]] = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 0, ptr %[[T1]]) +; CHECK-NEXT: %[[T3:.*]] = load i32, ptr %[[T2]], align 8 +; CHECK-NEXT: %[[cmp:.*]] = icmp eq i32 %[[T3]], 42 +; CHECK-NEXT: br i1 %[[cmp]], label %if.then, label %if.else +; CHECK: if.then: +; CHECK-NEXT: %[[T4:.*]] = load i64, ptr @"llvm.bpf_sockopt:0:8$0:1", align 8 +; CHECK-NEXT: %[[T5:.*]] = getelementptr i8, ptr %[[ctx]], i64 %[[T4]] +; CHECK-NEXT: %[[T6:.*]] = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 2, ptr %[[T5]]) +; CHECK-NEXT: %[[T7:.*]] = load ptr, ptr %[[T6]], align 8 +; CHECK-NEXT: %[[family:.*]] = getelementptr inbounds %struct.bpf_sock, ptr %[[T7]], i64 0, i32 1 +; CHECK-NEXT: %[[T8:.*]] = load i32, ptr %[[family]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.else: +; CHECK-NEXT: %[[T9:.*]] = load i64, ptr @"llvm.bpf_sockopt:0:20$0:3", align 8 +; CHECK-NEXT: %[[T10:.*]] = getelementptr i8, ptr %[[ctx]], i64 %[[T9]] +; CHECK-NEXT: %[[T11:.*]] = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 1, ptr %[[T10]]) +; CHECK-NEXT: %[[T12:.*]] = load i32, ptr %[[T11]], align 4 +; CHECK-NEXT: br label %if.end +; CHECK: if.end: +; CHECK-NEXT: %[[T13:.*]] = phi i32 [ %[[T12]], %if.else ], +; CHECK-SAME: [ %[[T8]], %if.then ] +; CHECK-NEXT: tail call void @magic2(i32 noundef %[[T13]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %0 = load i64, ptr @"llvm.bpf_sockopt:0:16$0:2", align 8 + %1 = getelementptr i8, ptr %ctx, i64 %0 + %2 = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 0, ptr %1) + %3 = load i32, ptr %2, align 8 + %cmp = icmp eq i32 %3, 42 + br i1 %cmp, label %if.then, label %if.else + +if.then: ; preds = %entry + %4 = load i64, ptr @"llvm.bpf_sockopt:0:8$0:1", align 8 + %5 = getelementptr i8, ptr %ctx, i64 %4 + %6 = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 2, ptr %5) + %7 = load ptr, ptr %6, align 8 + %family = getelementptr inbounds %struct.bpf_sock, ptr %7, i64 0, i32 1 + br label %if.end + +if.else: ; preds = %entry + %8 = load i64, ptr @"llvm.bpf_sockopt:0:20$0:3", align 8 + %9 = getelementptr i8, ptr %ctx, i64 %8 + %10 = tail call ptr @llvm.bpf.passthrough.p0.p0(i32 1, ptr %9) + br label %if.end + +if.end: ; preds = %if.else, %if.then + %.sink3 = phi ptr [ %10, %if.else ], [ %family, %if.then ] + %11 = load i32, ptr %.sink3, align 4 + tail call void @magic2(i32 noundef %11) + ret void +} + +declare void @magic2(i32 noundef) + +; Function Attrs: nofree nosync nounwind memory(none) +declare ptr @llvm.bpf.passthrough.p0.p0(i32, ptr) + +!0 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bpf_sockopt", file: !1, line: 9, size: 192, elements: !2) +!1 = !DIFile(filename: "some.file", directory: "/some/dir", checksumkind: CSK_MD5, checksum: "00000000000000000000000000000000") +!2 = !{!3, !5, !13, !14} +!3 = !DIDerivedType(tag: DW_TAG_member, name: "_", scope: !0, file: !1, line: 10, baseType: !4, size: 32) +!4 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!5 = !DIDerivedType(tag: DW_TAG_member, name: "sk", scope: !0, file: !1, line: 11, baseType: !6, size: 64, offset: 64) +!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64) +!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bpf_sock", file: !1, line: 4, size: 64, elements: !8) +!8 = !{!9, !10} +!9 = !DIDerivedType(tag: DW_TAG_member, name: "bound_dev_if", scope: !7, file: !1, line: 5, baseType: !4, size: 32) +!10 = !DIDerivedType(tag: DW_TAG_member, name: "family", scope: !7, file: !1, line: 6, baseType: !4, size: 32, offset: 32) +!13 = !DIDerivedType(tag: DW_TAG_member, name: "level", scope: !0, file: !1, line: 12, baseType: !4, size: 32, offset: 128) +!14 = !DIDerivedType(tag: DW_TAG_member, name: "optlen", scope: !0, file: !1, line: 13, baseType: !4, size: 32, offset: 160) diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-3.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-3.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-3.ll @@ -0,0 +1,150 @@ +; RUN: opt -S -passes=bpf-hoist-argument-access -mtriple=bpf-pc-linux < %s | FileCheck %s + +%struct.foo = type { i32, i32 } +declare void @consume(i32) +declare void @consume_ptr(ptr) +declare ptr @llvm.bpf.passthrough.p0.p0(i32, ptr) + +@"llvm.bpf_sockopt:0:20$0:3" = external global i64, !llvm.preserve.access.index !0 + +define void @select.1(ptr %ctx, i1 %cond) { +; CHECK: define void @select.1(ptr %[[ctx:.*]], i1 %[[cond:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 %[[cond]], label %entry.select.true, label %entry.select.false +; CHECK: entry.select.true: +; CHECK-NEXT: %[[a2:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 0 +; CHECK-NEXT: %[[T0:.*]] = load i32, ptr %[[a2]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.false: +; CHECK-NEXT: %[[b3:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 1 +; CHECK-NEXT: %[[T1:.*]] = load i32, ptr %[[b3]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.tail: +; CHECK-NEXT: %[[v5:.*]] = phi i32 [ %[[T0]], %entry.select.true ], +; CHECK-SAME: [ %[[T1]], %entry.select.false ] +; CHECK-NEXT: call void @consume(i32 %[[v5]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %a = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 0 + %b = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 1 + %p = select i1 %cond, ptr %a, ptr %b + %v = load i32, ptr %p, align 4 + call void @consume(i32 %v) + ret void +} + +define void @select.one-hand.1(ptr %ctx, i1 %cond, i64 %idx) { +; CHECK: define void @select.one-hand.1(ptr %[[ctx:.*]], i1 %[[cond:.*]], i64 %[[idx:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %[[b:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 %[[idx]], i32 1 +; CHECK-NEXT: br i1 %[[cond]], label %entry.select.true, label %entry.select.false +; CHECK: entry.select.true: +; CHECK-NEXT: %[[a2:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 0 +; CHECK-NEXT: %[[T0:.*]] = load i32, ptr %[[a2]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.false: +; CHECK-NEXT: %[[T1:.*]] = load i32, ptr %[[b]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.tail: +; CHECK-NEXT: %[[v4:.*]] = phi i32 [ %[[T0]], %entry.select.true ], +; CHECK-SAME: [ %[[T1]], %entry.select.false ] +; CHECK-NEXT: call void @consume(i32 %[[v4]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %a = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 0 + %b = getelementptr inbounds %struct.foo, ptr %ctx, i64 %idx, i32 1 + %p = select i1 %cond, ptr %a, ptr %b + %v = load i32, ptr %p, align 4 + call void @consume(i32 %v) + ret void +} + +define void @select.use_after.1(ptr %ctx, i1 %cond) { +; CHECK: define void @select.use_after.1(ptr %[[ctx:.*]], i1 %[[cond:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %[[a:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 0 +; CHECK-NEXT: %[[b:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 1 +; CHECK-NEXT: br i1 %[[cond]], label %entry.select.true, label %entry.select.false +; CHECK: entry.select.true: +; CHECK-NEXT: %[[a2:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 0 +; CHECK-NEXT: %[[T0:.*]] = load i32, ptr %[[a2]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.false: +; CHECK-NEXT: %[[b3:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 1 +; CHECK-NEXT: %[[T1:.*]] = load i32, ptr %[[b3]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.tail: +; CHECK-NEXT: %[[v5:.*]] = phi i32 [ %[[T0]], %entry.select.true ], +; CHECK-SAME: [ %[[T1]], %entry.select.false ] +; CHECK-NEXT: call void @consume(i32 %[[v5]]) +; CHECK-NEXT: call void @consume_ptr(ptr %[[a]]) +; CHECK-NEXT: call void @consume_ptr(ptr %[[b]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %a = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 0 + %b = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 1 + %p = select i1 %cond, ptr %a, ptr %b + %v = load i32, ptr %p, align 4 + call void @consume(i32 %v) + call void @consume_ptr(ptr %a) + call void @consume_ptr(ptr %b) + ret void +} + +define void @select.noop.1(ptr %ctx, i1 %cond, i64 %idx) { +; CHECK: define void @select.noop.1(ptr %[[ctx:.*]], i1 %[[cond:.*]], i64 %[[idx:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %[[a:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 %[[idx]], i32 0 +; CHECK-NEXT: %[[b:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 %[[idx]], i32 1 +; CHECK-NEXT: %[[p:.*]] = select i1 %[[cond]], ptr %[[a]], ptr %[[b]] +; CHECK-NEXT: %[[v:.*]] = load i32, ptr %[[p]], align 4 +; CHECK-NEXT: call void @consume(i32 %[[v]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %a = getelementptr inbounds %struct.foo, ptr %ctx, i64 %idx, i32 0 + %b = getelementptr inbounds %struct.foo, ptr %ctx, i64 %idx, i32 1 + %p = select i1 %cond, ptr %a, ptr %b + %v = load i32, ptr %p, align 4 + call void @consume(i32 %v) + ret void +} + +define void @select.pai.1(ptr %ctx, i1 %cond) { +; CHECK: define void @select.pai.1(ptr %[[ctx:.*]], i1 %[[cond:.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: %[[pai_offset:.*]] = load i64, ptr @"llvm.bpf_sockopt:0:20$0:3", align 8 +; CHECK-NEXT: br i1 %[[cond]], label %entry.select.true, label %entry.select.false +; CHECK: entry.select.true: +; CHECK-NEXT: %[[ctx_offset2:.*]] = getelementptr i8, ptr %[[ctx]], i64 %[[pai_offset]] +; CHECK-NEXT: %[[a3:.*]] = call ptr @llvm.bpf.passthrough.p0.p0(i32 1, ptr %[[ctx_offset2]]) +; CHECK-NEXT: %[[T0:.*]] = load i32, ptr %[[a3]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.false: +; CHECK-NEXT: %[[b4:.*]] = getelementptr inbounds %struct.foo, ptr %[[ctx]], i64 0, i32 1 +; CHECK-NEXT: %[[T1:.*]] = load i32, ptr %[[b4]], align 4 +; CHECK-NEXT: br label %entry.select.tail +; CHECK: entry.select.tail: +; CHECK-NEXT: %[[v6:.*]] = phi i32 [ %[[T0]], %entry.select.true ], +; CHECK-SAME: [ %[[T1]], %entry.select.false ] +; CHECK-NEXT: call void @consume(i32 %[[v6]]) +; CHECK-NEXT: ret void +; CHECK-NEXT: } +entry: + %pai.offset = load i64, ptr @"llvm.bpf_sockopt:0:20$0:3", align 8 + %ctx.offset = getelementptr i8, ptr %ctx, i64 %pai.offset + %a = call ptr @llvm.bpf.passthrough.p0.p0(i32 1, ptr %ctx.offset) + %b = getelementptr inbounds %struct.foo, ptr %ctx, i64 0, i32 1 + %p = select i1 %cond, ptr %a, ptr %b + %v = load i32, ptr %p, align 4 + call void @consume(i32 %v) + ret void +} + +!0 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bpf_sockopt", file: !1, line: 9, size: 192, elements: !2) +!1 = !DIFile(filename: "some.file", directory: "/some/dir", checksumkind: CSK_MD5, checksum: "00000000000000000000000000000000") +!2 = !{!3, !3, !3, !3} +!3 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-canary-1.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-1.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-1.ll @@ -0,0 +1,71 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -mcpu=v4 -o - %s \ +; RUN: | llc -mcpu=v4 - | FileCheck %s +; +; Source: +; struct __sk_buff { +; int _; +; int priority; +; int mark; +; int tc_index; +; }; +; +; int store_sink_example(struct __sk_buff *ctx) { +; switch (ctx->priority) { +; case 10: +; ctx->mark = 3; +; break; +; case 20: +; ctx->priority = 4; +; break; +; } +; return 0; +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.__sk_buff = type { i32, i32, i32, i32 } + +@__ctx = global %struct.__sk_buff zeroinitializer, align 4 + +; Function Attrs: nounwind +define dso_local i32 @store_sink_example(ptr noundef %ctx) #0 { +; CHECK: {{.*}} = *(u32 *)(r1 + 4) +; CHECK: *(u32 *)(r1 + 8) = 3 +; CHECK: *(u32 *)(r1 + 4) = 4 +entry: + %priority = getelementptr inbounds %struct.__sk_buff, ptr %ctx, i32 0, i32 1 + %0 = load i32, ptr %priority, align 4, !tbaa !2 + switch i32 %0, label %sw.epilog [ + i32 10, label %sw.bb + i32 20, label %sw.bb1 + ] + +sw.bb: ; preds = %entry + %mark = getelementptr inbounds %struct.__sk_buff, ptr %ctx, i32 0, i32 2 + store i32 3, ptr %mark, align 4, !tbaa !7 + br label %sw.epilog + +sw.bb1: ; preds = %entry + %priority2 = getelementptr inbounds %struct.__sk_buff, ptr %ctx, i32 0, i32 1 + store i32 4, ptr %priority2, align 4, !tbaa !2 + br label %sw.epilog + +sw.epilog: ; preds = %sw.bb1, %sw.bb, %entry + ret i32 0 +} + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang 53c495f835926142c10c80d7d0505f59b1e46e49)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"__sk_buff", !4, i64 0, !4, i64 4, !4, i64 8, !4, i64 12} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!3, !4, i64 8} diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-canary-2.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-2.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-2.ll @@ -0,0 +1,70 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -mcpu=v4 -o - %s \ +; RUN: | llc -mcpu=v4 - | FileCheck %s +; +; Source: +; struct bpf_sockopt { +; int family; +; int level; +; int optlen; +; }; +; +; extern void consume(int); +; +; void load_sink_example(struct bpf_sockopt *ctx) +; { +; if (ctx->level == 42) +; consume(ctx->family); +; else +; consume(ctx->optlen); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -o - \ +; | opt -passes=function(sroa) -S -o - + +%struct.bpf_sockopt = type { i32, i32, i32 } + +; Function Attrs: nounwind +define dso_local void @load_sink_example(ptr noundef %ctx) #0 { +; CHECK: {{.*}} = *(u32 *)(r1 + 4) +; CHECK: w1 = *(u32 *)(r1 + 0) +; CHECK: w1 = *(u32 *)(r1 + 8) +entry: + %level = getelementptr inbounds %struct.bpf_sockopt, ptr %ctx, i32 0, i32 1 + %0 = load i32, ptr %level, align 4, !tbaa !2 + %cmp = icmp eq i32 %0, 42 + br i1 %cmp, label %if.then, label %if.else + +if.then: ; preds = %entry + %family = getelementptr inbounds %struct.bpf_sockopt, ptr %ctx, i32 0, i32 0 + %1 = load i32, ptr %family, align 4, !tbaa !7 + call void @consume(i32 noundef %1) + br label %if.end + +if.else: ; preds = %entry + %optlen = getelementptr inbounds %struct.bpf_sockopt, ptr %ctx, i32 0, i32 2 + %2 = load i32, ptr %optlen, align 4, !tbaa !8 + call void @consume(i32 noundef %2) + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void +} + +declare void @consume(i32 noundef) #1 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang 53c495f835926142c10c80d7d0505f59b1e46e49)"} +!2 = !{!3, !4, i64 4} +!3 = !{!"bpf_sockopt", !4, i64 0, !4, i64 4, !4, i64 8} +!4 = !{!"int", !5, i64 0} +!5 = !{!"omnipotent char", !6, i64 0} +!6 = !{!"Simple C/C++ TBAA"} +!7 = !{!3, !4, i64 0} +!8 = !{!3, !4, i64 8} diff --git a/llvm/test/CodeGen/BPF/hoist-arg-access-canary-3.ll b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-3.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/BPF/hoist-arg-access-canary-3.ll @@ -0,0 +1,116 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux -mcpu=v4 -o - %s \ +; RUN: | llc -mcpu=v4 - | FileCheck %s +; +; Source: +; struct bpf_sockopt { +; int family; +; int level; +; int optlen; +; } __attribute__((preserve_access_index)); +; +; extern void consume(int); +; +; void load_sink_example(struct bpf_sockopt *ctx) +; { +; if (ctx->level == 42) +; consume(ctx->family); +; else +; consume(ctx->optlen); +; } +; +; Compilation flag: +; clang -cc1 -O2 -triple bpf -S -emit-llvm -disable-llvm-passes -debug-info-kind=limited + +%struct.bpf_sockopt = type { i32, i32, i32 } + +; Function Attrs: nounwind +define dso_local void @load_sink_example(ptr noundef %ctx) #0 !dbg !5 { +; CHECK: {{.*}} = *(u32 *)(r1 + 4) +; CHECK: w1 = *(u32 *)(r1 + 0) +; CHECK: w1 = *(u32 *)(r1 + 8) +entry: + %ctx.addr = alloca ptr, align 8 + store ptr %ctx, ptr %ctx.addr, align 8, !tbaa !18 + call void @llvm.dbg.declare(metadata ptr %ctx.addr, metadata !17, metadata !DIExpression()), !dbg !22 + %0 = load ptr, ptr %ctx.addr, align 8, !dbg !23, !tbaa !18 + %1 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bpf_sockopt) %0, i32 1, i32 1), !dbg !25, !llvm.preserve.access.index !10 + %2 = load i32, ptr %1, align 4, !dbg !25, !tbaa !26 + %cmp = icmp eq i32 %2, 42, !dbg !29 + br i1 %cmp, label %if.then, label %if.else, !dbg !30 + +if.then: ; preds = %entry + %3 = load ptr, ptr %ctx.addr, align 8, !dbg !31, !tbaa !18 + %4 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bpf_sockopt) %3, i32 0, i32 0), !dbg !32, !llvm.preserve.access.index !10 + %5 = load i32, ptr %4, align 4, !dbg !32, !tbaa !33 + call void @consume(i32 noundef %5), !dbg !34 + br label %if.end, !dbg !34 + +if.else: ; preds = %entry + %6 = load ptr, ptr %ctx.addr, align 8, !dbg !35, !tbaa !18 + %7 = call ptr @llvm.preserve.struct.access.index.p0.p0(ptr elementtype(%struct.bpf_sockopt) %6, i32 2, i32 2), !dbg !36, !llvm.preserve.access.index !10 + %8 = load i32, ptr %7, align 4, !dbg !36, !tbaa !37 + call void @consume(i32 noundef %8), !dbg !38 + br label %if.end + +if.end: ; preds = %if.else, %if.then + ret void, !dbg !39 +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare void @llvm.dbg.declare(metadata, metadata, metadata) #1 + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none) +declare ptr @llvm.preserve.struct.access.index.p0.p0(ptr, i32 immarg, i32 immarg) #2 + +declare void @consume(i32 noundef) #3 + +attributes #0 = { nounwind "no-trapping-math"="true" "stack-protector-buffer-size"="8" } +attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } +attributes #2 = { nocallback nofree nosync nounwind willreturn memory(none) } +attributes #3 = { "no-trapping-math"="true" "stack-protector-buffer-size"="8" } + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3} +!llvm.ident = !{!4} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0 (/home/eddy/work/llvm-project/clang 53c495f835926142c10c80d7d0505f59b1e46e49)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!2 = !{i32 2, !"Debug Info Version", i32 3} +!3 = !{i32 1, !"wchar_size", i32 4} +!4 = !{!"clang version 18.0.0 (/home/eddy/work/llvm-project/clang 53c495f835926142c10c80d7d0505f59b1e46e49)"} +!5 = distinct !DISubprogram(name: "load_sink_example", scope: !6, file: !6, line: 9, type: !7, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16) +!6 = !DIFile(filename: "some-file.c", directory: "/some/dir/") +!7 = !DISubroutineType(types: !8) +!8 = !{null, !9} +!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64) +!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "bpf_sockopt", file: !6, line: 1, size: 96, elements: !11) +!11 = !{!12, !14, !15} +!12 = !DIDerivedType(tag: DW_TAG_member, name: "family", scope: !10, file: !6, line: 2, baseType: !13, size: 32) +!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!14 = !DIDerivedType(tag: DW_TAG_member, name: "level", scope: !10, file: !6, line: 3, baseType: !13, size: 32, offset: 32) +!15 = !DIDerivedType(tag: DW_TAG_member, name: "optlen", scope: !10, file: !6, line: 4, baseType: !13, size: 32, offset: 64) +!16 = !{!17} +!17 = !DILocalVariable(name: "ctx", arg: 1, scope: !5, file: !6, line: 9, type: !9) +!18 = !{!19, !19, i64 0} +!19 = !{!"any pointer", !20, i64 0} +!20 = !{!"omnipotent char", !21, i64 0} +!21 = !{!"Simple C/C++ TBAA"} +!22 = !DILocation(line: 9, column: 44, scope: !5) +!23 = !DILocation(line: 11, column: 7, scope: !24) +!24 = distinct !DILexicalBlock(scope: !5, file: !6, line: 11, column: 7) +!25 = !DILocation(line: 11, column: 12, scope: !24) +!26 = !{!27, !28, i64 4} +!27 = !{!"bpf_sockopt", !28, i64 0, !28, i64 4, !28, i64 8} +!28 = !{!"int", !20, i64 0} +!29 = !DILocation(line: 11, column: 18, scope: !24) +!30 = !DILocation(line: 11, column: 7, scope: !5) +!31 = !DILocation(line: 12, column: 13, scope: !24) +!32 = !DILocation(line: 12, column: 18, scope: !24) +!33 = !{!27, !28, i64 0} +!34 = !DILocation(line: 12, column: 5, scope: !24) +!35 = !DILocation(line: 14, column: 13, scope: !24) +!36 = !DILocation(line: 14, column: 18, scope: !24) +!37 = !{!27, !28, i64 8} +!38 = !DILocation(line: 14, column: 5, scope: !24) +!39 = !DILocation(line: 15, column: 1, scope: !5) +