diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h
--- a/llvm/lib/Target/BPF/BPF.h
+++ b/llvm/lib/Target/BPF/BPF.h
@@ -15,6 +15,7 @@
namespace llvm {
class BPFTargetMachine;
+ModulePass *createBPFAdjustOpt();
ModulePass *createBPFCheckAndAdjustIR();
FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM);
@@ -26,6 +27,7 @@
FunctionPass *createBPFMIPreEmitPeepholePass();
FunctionPass *createBPFMIPreEmitCheckingPass();
+void initializeBPFAdjustOptPass(PassRegistry&);
void initializeBPFCheckAndAdjustIRPass(PassRegistry&);
void initializeBPFAbstractMemberAccessPass(PassRegistry&);
diff --git a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
new file mode 100644
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp
@@ -0,0 +1,310 @@
+//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Adjust optimization to make the code more kernel verifier friendly.
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPF.h"
+#include "BPFCORE.h"
+#include "BPFTargetMachine.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/User.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+#define DEBUG_TYPE "bpf-adjust-opt"
+
+using namespace llvm;
+
+static cl::opt<bool>
+ DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden,
+ cl::desc("BPF: Disable Serializing ICMP insns."),
+ cl::init(false));
+
+static cl::opt<bool> DisableBPFavoidSpeculation(
+ "bpf-disable-avoid-speculation", cl::Hidden,
+ cl::desc("BPF: Disable Avoiding Speculative Code Motion."),
+ cl::init(false));
+
+namespace {
+
+class BPFAdjustOpt final : public ModulePass {
+ struct PassThroughInfo {
+ Instruction *Input;
+ Instruction *UsedInst;
+ uint32_t OpIdx;
+ PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx)
+ : Input(I), UsedInst(U), OpIdx(Idx) {}
+ };
+
+public:
+ static char ID;
+ Module *Mod;
+
+ BPFAdjustOpt() : ModulePass(ID) {}
+ bool runOnModule(Module &M) override;
+
+private:
+ SmallVector<PassThroughInfo, 16> PassThroughs;
+
+ void adjustBasicBlock(BasicBlock &BB);
+ bool serializeICMPCrossBB(BasicBlock &BB);
+ void adjustInst(Instruction &I);
+ bool serializeICMPInBB(Instruction &I);
+ bool avoidSpeculation(Instruction &I);
+ bool insertPassThrough();
+};
+
+} // End anonymous namespace
+
+char BPFAdjustOpt::ID = 0;
+INITIALIZE_PASS(BPFAdjustOpt, "bpf-adjust-opt", "BPF Adjust Optimization",
+ false, false)
+
+ModulePass *llvm::createBPFAdjustOpt() { return new BPFAdjustOpt(); }
+
+bool BPFAdjustOpt::runOnModule(Module &M) {
+ Mod = &M;
+ for (Function &F : M)
+ for (auto &BB : F) {
+ adjustBasicBlock(BB);
+ for (auto &I : BB)
+ adjustInst(I);
+ }
+
+ return insertPassThrough();
+}
+
+bool BPFAdjustOpt::insertPassThrough() {
+ for (auto &Info : PassThroughs) {
+ auto *CI = BPFCoreSharedInfo::insertPassThrough(
+ Mod, Info.UsedInst->getParent(), Info.Input, Info.UsedInst);
+ Info.UsedInst->setOperand(Info.OpIdx, CI);
+ }
+
+ return !PassThroughs.empty();
+}
+
+// To avoid combining conditionals in the same basic block by
+// instrcombine optimization.
+bool BPFAdjustOpt::serializeICMPInBB(Instruction &I) {
+ // For:
+ // comp1 = icmp <opcode> ...;
+ // comp2 = icmp <opcode> ...;
+ // ... or comp1 comp2 ...
+ // changed to:
+ // comp1 = icmp <opcode> ...;
+ // comp2 = icmp <opcode> ...;
+ // new_comp1 = __builtin_bpf_passthrough(seq_num, comp1)
+ // ... or new_comp1 comp2 ...
+ if (I.getOpcode() != Instruction::Or)
+ return false;
+ auto *Icmp1 = dyn_cast<ICmpInst>(I.getOperand(0));
+ if (!Icmp1)
+ return false;
+ auto *Icmp2 = dyn_cast<ICmpInst>(I.getOperand(1));
+ if (!Icmp2)
+ return false;
+
+ Value *Icmp1Op0 = Icmp1->getOperand(0);
+ Value *Icmp2Op0 = Icmp2->getOperand(0);
+ if (Icmp1Op0 != Icmp2Op0)
+ return false;
+
+ // Now we got two icmp instructions which feed into
+ // an "or" instruction.
+ PassThroughInfo Info(Icmp1, &I, 0);
+ PassThroughs.push_back(Info);
+ return true;
+}
+
+// To avoid combining conditionals in the same basic block by
+// instrcombine optimization.
+bool BPFAdjustOpt::serializeICMPCrossBB(BasicBlock &BB) {
+ // For:
+ // B1:
+ // comp1 = icmp <opcode> ...;
+ // if (comp1) goto B2 else B3;
+ // B2:
+ // comp2 = icmp <opcode> ...;
+ // if (comp2) goto B4 else B5;
+ // B4:
+ // ...
+ // changed to:
+ // B1:
+ // comp1 = icmp <opcode> ...;
+ // comp1 = __builtin_bpf_passthrough(seq_num, comp1);
+ // if (comp1) goto B2 else B3;
+ // B2:
+ // comp2 = icmp <opcode> ...;
+ // if (comp2) goto B4 else B5;
+ // B4:
+ // ...
+
+ // Check basic predecessors, if two of them (say B1, B2) are using
+ // icmp instructions to generate conditions and one is the predesessor
+ // of another (e.g., B1 is the predecessor of B2). Add a passthrough
+ // barrier after icmp inst of block B1.
+ BasicBlock *B2 = BB.getSinglePredecessor();
+ if (!B2)
+ return false;
+
+ BasicBlock *B1 = B2->getSinglePredecessor();
+ if (!B1)
+ return false;
+
+ Instruction *TI = B2->getTerminator();
+ auto *BI = dyn_cast<BranchInst>(TI);
+ if (!BI || !BI->isConditional())
+ return false;
+ auto *Cond = dyn_cast<ICmpInst>(BI->getCondition());
+ if (!Cond || B2->getFirstNonPHI() != Cond)
+ return false;
+ Value *B2Op0 = Cond->getOperand(0);
+ auto Cond2Op = Cond->getPredicate();
+
+ TI = B1->getTerminator();
+ BI = dyn_cast<BranchInst>(TI);
+ if (!BI || !BI->isConditional())
+ return false;
+ Cond = dyn_cast<ICmpInst>(BI->getCondition());
+ if (!Cond)
+ return false;
+ Value *B1Op0 = Cond->getOperand(0);
+ auto Cond1Op = Cond->getPredicate();
+
+ if (B1Op0 != B2Op0)
+ return false;
+
+ if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) {
+ if (Cond2Op != ICmpInst::ICMP_SLT && Cond1Op != ICmpInst::ICMP_SLE)
+ return false;
+ } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) {
+ if (Cond2Op != ICmpInst::ICMP_SGT && Cond1Op != ICmpInst::ICMP_SGE)
+ return false;
+ } else {
+ return false;
+ }
+
+ PassThroughInfo Info(Cond, BI, 0);
+ PassThroughs.push_back(Info);
+
+ return true;
+}
+
+// To avoid speculative hoisting certain computations out of
+// a basic block.
+bool BPFAdjustOpt::avoidSpeculation(Instruction &I) {
+ if (auto *LdInst = dyn_cast<LoadInst>(&I)) {
+ if (auto *GV = dyn_cast<GlobalVariable>(LdInst->getOperand(0))) {
+ if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) ||
+ GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr))
+ return false;
+ }
+ }
+
+ if (!dyn_cast<LoadInst>(&I) && !dyn_cast<CallInst>(&I))
+ return false;
+
+ // For:
+ // B1:
+ // var = ...
+ // ...
+ // /* icmp may not be in the same block as var = ... */
+ // comp1 = icmp <opcode> var, <const>;
+ // if (comp1) goto B2 else B3;
+ // B2:
+ // ... var ...
+ // change to:
+ // B1:
+ // var = ...
+ // ...
+ // /* icmp may not be in the same block as var = ... */
+ // comp1 = icmp <opcode> var, <const>;
+ // if (comp1) goto B2 else B3;
+ // B2:
+ // var = __builtin_bpf_passthrough(seq_num, var);
+ // ... var ...
+ bool isCandidate = false;
+ SmallVector<PassThroughInfo, 4> Candidates;
+ for (User *U : I.users()) {
+ Instruction *Inst = dyn_cast<Instruction>(U);
+ if (!Inst)
+ continue;
+
+ // May cover a little bit more than the
+ // above pattern.
+ if (auto *Icmp1 = dyn_cast<ICmpInst>(Inst)) {
+ Value *Icmp1Op1 = Icmp1->getOperand(1);
+ if (!isa<Constant>(Icmp1Op1))
+ return false;
+ isCandidate = true;
+ continue;
+ }
+
+ // Ignore the use in the same basic block as the definition.
+ if (Inst->getParent() == I.getParent())
+ continue;
+
+ // use in a different basic block, If there is a call or
+ // load/store insn before this instruction in this basic
+ // block. Most likely it cannot be hoisted out. Skip it.
+ for (auto &I2 : *Inst->getParent()) {
+ if (dyn_cast<CallInst>(&I2))
+ return false;
+ if (dyn_cast<LoadInst>(&I2) || dyn_cast<StoreInst>(&I2))
+ return false;
+ if (&I2 == Inst)
+ break;
+ }
+
+ // It should be used in a GEP or a simple arithmetic like
+ // ZEXT/SEXT which is used for GEP.
+ if (Inst->getOpcode() == Instruction::ZExt ||
+ Inst->getOpcode() == Instruction::SExt) {
+ PassThroughInfo Info(&I, Inst, 0);
+ Candidates.push_back(Info);
+ } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
+ // traverse GEP inst to find Use operand index
+ unsigned i, e;
+ for (i = 1, e = GI->getNumOperands(); i != e; ++i) {
+ Value *V = GI->getOperand(i);
+ if (V == &I)
+ break;
+ }
+ if (i == e)
+ continue;
+
+ PassThroughInfo Info(&I, GI, i);
+ Candidates.push_back(Info);
+ }
+ }
+
+ if (!isCandidate || Candidates.empty())
+ return false;
+
+ PassThroughs.insert(PassThroughs.end(), Candidates.begin(), Candidates.end());
+ return true;
+}
+
+void BPFAdjustOpt::adjustBasicBlock(BasicBlock &BB) {
+ if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB))
+ return;
+}
+
+void BPFAdjustOpt::adjustInst(Instruction &I) {
+ if (!DisableBPFserializeICMP && serializeICMPInBB(I))
+ return;
+ if (!DisableBPFavoidSpeculation && avoidSpeculation(I))
+ return;
+}
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -39,6 +39,7 @@
PassRegistry &PR = *PassRegistry::getPassRegistry();
initializeBPFAbstractMemberAccessPass(PR);
initializeBPFPreserveDITypePass(PR);
+ initializeBPFAdjustOptPass(PR);
initializeBPFCheckAndAdjustIRPass(PR);
initializeBPFMIPeepholePass(PR);
initializeBPFMIPeepholeTruncElimPass(PR);
@@ -112,6 +113,11 @@
PM.add(createCFGSimplificationPass(
SimplifyCFGOptions().hoistCommonInsts(true)));
});
+ Builder.addExtension(
+ PassManagerBuilder::EP_ModuleOptimizerEarly,
+ [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
+ PM.add(createBPFAdjustOpt());
+ });
}
void BPFPassConfig::addIRPasses() {
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -14,6 +14,7 @@
add_llvm_target(BPFCodeGen
BPFAbstractMemberAccess.cpp
+ BPFAdjustOpt.cpp
BPFAsmPrinter.cpp
BPFCheckAndAdjustIR.cpp
BPFFrameLowering.cpp
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll
@@ -0,0 +1,90 @@
+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+;
+; Source:
+; int foo();
+; int bar(int);
+; int test() {
+; int ret = foo();
+; if (ret <= 0 || ret > 7)
+; return 0;
+; return bar(ret);
+; }
+; Compilation flag:
+; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() #0 {
+entry:
+ %retval = alloca i32, align 4
+ %ret = alloca i32, align 4
+ %cleanup.dest.slot = alloca i32, align 4
+ %0 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+ %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+ store i32 %call, i32* %ret, align 4, !tbaa !2
+ %1 = load i32, i32* %ret, align 4, !tbaa !2
+ %cmp = icmp sle i32 %1, 0
+ br i1 %cmp, label %if.then, label %lor.lhs.false
+
+; CHECK: [[REG1:r[0-9]+]] <<= 32
+; CHECK: [[REG1]] s>>= 32
+; CHECK: [[REG2:r[0-9]+]] = 1
+; CHECK: if [[REG2]] s> [[REG1]] goto
+; CHECK: if [[REG1]] s> 7 goto
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: if [[REG1]] > 6 goto
+
+lor.lhs.false: ; preds = %entry
+ %2 = load i32, i32* %ret, align 4, !tbaa !2
+ %cmp1 = icmp sgt i32 %2, 7
+ br i1 %cmp1, label %if.then, label %if.end
+
+if.then: ; preds = %lor.lhs.false, %entry
+ store i32 0, i32* %retval, align 4
+ store i32 1, i32* %cleanup.dest.slot, align 4
+ br label %cleanup
+
+if.end: ; preds = %lor.lhs.false
+ %3 = load i32, i32* %ret, align 4, !tbaa !2
+ %call2 = call i32 @bar(i32 %3)
+ store i32 %call2, i32* %retval, align 4
+ store i32 1, i32* %cleanup.dest.slot, align 4
+ br label %cleanup
+
+cleanup: ; preds = %if.end, %if.then
+ %4 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3
+ %5 = load i32, i32* %retval, align 4
+ ret i32 %5
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+declare dso_local i32 @bar(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll
@@ -0,0 +1,97 @@
+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+;
+; Source:
+; int foo();
+; int bar(int);
+; int test() {
+; int ret = foo();
+; if (ret <= 0)
+; return 0;
+; if (ret > 7)
+; return 0;
+; return bar(ret);
+; }
+; Compilation flag:
+; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() #0 {
+entry:
+ %retval = alloca i32, align 4
+ %ret = alloca i32, align 4
+ %cleanup.dest.slot = alloca i32, align 4
+ %0 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+ %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+ store i32 %call, i32* %ret, align 4, !tbaa !2
+ %1 = load i32, i32* %ret, align 4, !tbaa !2
+ %cmp = icmp sle i32 %1, 0
+ br i1 %cmp, label %if.then, label %if.end
+
+; CHECK: [[REG1:r[0-9]+]] <<= 32
+; CHECK: [[REG1]] s>>= 32
+; CHECK: [[REG2:r[0-9]+]] = 1
+; CHECK: if [[REG2]] s> [[REG1]] goto
+; CHECK: if [[REG1]] s> 7 goto
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: if [[REG1]] > 6 goto
+
+if.then: ; preds = %entry
+ store i32 0, i32* %retval, align 4
+ store i32 1, i32* %cleanup.dest.slot, align 4
+ br label %cleanup
+
+if.end: ; preds = %entry
+ %2 = load i32, i32* %ret, align 4, !tbaa !2
+ %cmp1 = icmp sgt i32 %2, 7
+ br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2: ; preds = %if.end
+ store i32 0, i32* %retval, align 4
+ store i32 1, i32* %cleanup.dest.slot, align 4
+ br label %cleanup
+
+if.end3: ; preds = %if.end
+ %3 = load i32, i32* %ret, align 4, !tbaa !2
+ %call4 = call i32 @bar(i32 %3)
+ store i32 %call4, i32* %retval, align 4
+ store i32 1, i32* %cleanup.dest.slot, align 4
+ br label %cleanup
+
+cleanup: ; preds = %if.end3, %if.then2, %if.then
+ %4 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3
+ %5 = load i32, i32* %retval, align 4
+ ret i32 %5
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+declare dso_local i32 @bar(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll
@@ -0,0 +1,84 @@
+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+;
+; Source:
+; unsigned long foo();
+; void *test(void *p) {
+; unsigned long ret = foo();
+; if (ret <= 7)
+; p += ret;
+; return p;
+; }
+; Compilation flag:
+; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i8* @test(i8* %p) #0 {
+entry:
+ %p.addr = alloca i8*, align 8
+ %ret = alloca i64, align 8
+ store i8* %p, i8** %p.addr, align 8, !tbaa !2
+ %0 = bitcast i64* %ret to i8*
+ call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #3
+ %call = call i64 bitcast (i64 (...)* @foo to i64 ()*)()
+ store i64 %call, i64* %ret, align 8, !tbaa !6
+ %1 = load i64, i64* %ret, align 8, !tbaa !6
+ %cmp = icmp ule i64 %1, 7
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %2 = load i64, i64* %ret, align 8, !tbaa !6
+ %3 = load i8*, i8** %p.addr, align 8, !tbaa !2
+ %add.ptr = getelementptr i8, i8* %3, i64 %2
+ store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %4 = load i8*, i8** %p.addr, align 8, !tbaa !2
+ %5 = bitcast i64* %ret to i8*
+ call void @llvm.lifetime.end.p0i8(i64 8, i8* %5) #3
+ ret i8* %4
+}
+; CHECK-COMMON: [[REG6:r[0-9]+]] = r1
+; CHECK-COMMON: call foo
+
+; CHECK: if r0 > 7 goto [[LABEL:.*]]
+; CHECK: [[REG6]] += r0
+; CHECK: [[LABEL]]:
+; CHECK: r0 = [[REG6]]
+
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: r0 += [[REG1:r[0-9]+]]
+; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8
+; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]]
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: [[LABEL]]:
+
+; CHECK-COMMON: exit
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i64 @foo(...) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"long", !4, i64 0}
diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll
new file mode 100644
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll
@@ -0,0 +1,91 @@
+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+;
+; Source:
+; unsigned foo();
+; void *test(void *p) {
+; unsigned ret = foo();
+; if (ret <= 7)
+; p += ret;
+; return p;
+; }
+; Compilation flag:
+; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i8* @test(i8* %p) #0 {
+entry:
+ %p.addr = alloca i8*, align 8
+ %ret = alloca i32, align 4
+ store i8* %p, i8** %p.addr, align 8, !tbaa !2
+ %0 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+ %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+ store i32 %call, i32* %ret, align 4, !tbaa !6
+ %1 = load i32, i32* %ret, align 4, !tbaa !6
+ %cmp = icmp ule i32 %1, 7
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %2 = load i32, i32* %ret, align 4, !tbaa !6
+ %3 = load i8*, i8** %p.addr, align 8, !tbaa !2
+ %idx.ext = zext i32 %2 to i64
+ %add.ptr = getelementptr i8, i8* %3, i64 %idx.ext
+ store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %4 = load i8*, i8** %p.addr, align 8, !tbaa !2
+ %5 = bitcast i32* %ret to i8*
+ call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #3
+ ret i8* %4
+}
+
+; CHECK-COMMON: [[REG6:r[0-9]+]] = r1
+; CHECK-COMMON: call foo
+
+; CHECK: r0 <<= 32
+; CHECK: r0 >>= 32
+; CHECK: if r0 > 7 goto [[LABEL:.*]]
+; CHECK: [[REG6]] += r0
+; CHECK: [[LABEL]]:
+; CHECK: r0 = [[REG6]]
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] = r0
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: r0 += [[REG1]]
+; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8
+; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]]
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: [[LABEL]]:
+
+; CHECK-COMMON: exit
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"int", !4, i64 0}