BPF: add AdjustOpt IR pass to generate verifier friendly codes

[yonghong-song]

Summary

Add an IR phase right before main module optimization.
This is to modify IR to restrict certain downward optimizations
in order to generate verifier friendly code.

> prevent certain instcombine optimizations, handling both
  in-block/cross-block instcombines.
> avoid speculative code motion if the variable used in
  condition is also used in the later blocks.

Internally, a bpf IR builtin

result = __builtin_bpf_passthrough(seq_num, result)

is used to enforce ordering. This builtin is only used
during target independent IR optimizations and it will
be removed at the beginning of target dependent IR
optimizations.

For example, removing the following workaround,

• • a/tools/testing/selftests/bpf/progs/test_sysctl_loop1.c +++ b/tools/testing/selftests/bpf/progs/test_sysctl_loop1.c @@ -47,7 +47,7 @@ int sysctl_tcp_mem(struct bpf_sysctl *ctx) /* a workaround to prevent compiler from generating
    • codes verifier cannot handle yet. */
• volatile int ret; + int ret;

this patch is able to generate code which passed the verifier.

To disable optimization, users need to use "opt" command like below:

clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
// disable icmp serialization
opt -O2 -bpf-disable-serialize-icmp test.ll | llvm-dis > t.ll
// disable avoid-speculation
opt -O2 -bpf-disable-avoid-speculation test.ll | llvm-dis > t.ll
llc t.ll

[yonghong-song]

now all bpf selftests can pass.

[yonghong-song]

add another adjustopt phase right after clang codegen to prevent certain early CSE optimization for __builtin_preserve_* intrinsics.

[yonghong-song]

A few changes:

• AdjustOpt IR pass only called at EP_ModuleOptimizerEarly now
• remove the pass which deals with a condition is used in more than one branch insns. This will be added to BPFCheckAndAdjustIR later.

[ast]

is this based on D87153 ?
the passthrough will be removed later, right?

[yonghong-song]

In D85570#2266620, @ast wrote:

is this based on D87153 ?
the passthrough will be removed later, right?

Yes, this is on top of D87153 and all these passthrough builtin will be removed.

[yonghong-song]

minor change: add some more conditions to further restrict transformation.

[yonghong-song]

add seq_num arg to passthrough builtin

[yonghong-song]

• remove RFC tag as proper tests are added
• added "opt" options so the added optimizations can be disabled.

llvm/lib/Target/BPF/BPF.h

 
 #include "MCTargetDesc/BPFMCTargetDesc.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Target/TargetMachine.h"
 
 namespace llvm {
 class BPFTargetMachine;
 
+ModulePass *createBPFAdjustOpt();
 ModulePass *createBPFCheckAndAdjustIR();
 
 FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM);
 FunctionPass *createBPFPreserveDIType();
 FunctionPass *createBPFISelDag(BPFTargetMachine &TM);
 FunctionPass *createBPFMISimplifyPatchablePass();
 FunctionPass *createBPFMIPeepholePass();
 FunctionPass *createBPFMIPeepholeTruncElimPass();
 FunctionPass *createBPFMIPreEmitPeepholePass();
 FunctionPass *createBPFMIPreEmitCheckingPass();
 
+void initializeBPFAdjustOptPass(PassRegistry&);
 void initializeBPFCheckAndAdjustIRPass(PassRegistry&);
 
 void initializeBPFAbstractMemberAccessLegacyPassPass(PassRegistry &);
 void initializeBPFPreserveDITypePass(PassRegistry&);
 void initializeBPFMISimplifyPatchablePass(PassRegistry&);
 void initializeBPFMIPeepholePass(PassRegistry&);
 void initializeBPFMIPeepholeTruncElimPass(PassRegistry&);
 void initializeBPFMIPreEmitPeepholePass(PassRegistry&);

llvm/lib/Target/BPF/BPFAdjustOpt.cpp

+//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Adjust optimization to make the code more kernel verifier friendly.
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPF.h"
+#include "BPFCORE.h"
+#include "BPFTargetMachine.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/User.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+
+#define DEBUG_TYPE "bpf-adjust-opt"
+
+using namespace llvm;
+
+static cl::opt<bool>
+    DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden,
+                            cl::desc("BPF: Disable Serializing ICMP insns."),
+                            cl::init(false));
+
+static cl::opt<bool> DisableBPFavoidSpeculation(
+    "bpf-disable-avoid-speculation", cl::Hidden,
+    cl::desc("BPF: Disable Avoiding Speculative Code Motion."),
+    cl::init(false));
+
+namespace {
+
+class BPFAdjustOpt final : public ModulePass {
+  struct PassThroughInfo {
+    Instruction *Input;
+    Instruction *UsedInst;
+    uint32_t OpIdx;
+    PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx)
+        : Input(I), UsedInst(U), OpIdx(Idx) {}
+  };
+
+public:
+  static char ID;
+  Module *Mod;
+
+  BPFAdjustOpt() : ModulePass(ID) {}
+  bool runOnModule(Module &M) override;
+
+private:
+  SmallVector<PassThroughInfo, 16> PassThroughs;
+
+  void adjustBasicBlock(BasicBlock &BB);
+  bool serializeICMPCrossBB(BasicBlock &BB);
+  void adjustInst(Instruction &I);
+  bool serializeICMPInBB(Instruction &I);
+  bool avoidSpeculation(Instruction &I);
+  bool insertPassThrough();
+};
+
+} // End anonymous namespace
+
+char BPFAdjustOpt::ID = 0;
+INITIALIZE_PASS(BPFAdjustOpt, "bpf-adjust-opt", "BPF Adjust Optimization",
+                false, false)
+
+ModulePass *llvm::createBPFAdjustOpt() { return new BPFAdjustOpt(); }
+
+bool BPFAdjustOpt::runOnModule(Module &M) {
+  Mod = &M;
+  for (Function &F : M)
+    for (auto &BB : F) {
+      adjustBasicBlock(BB);
+      for (auto &I : BB)
+        adjustInst(I);
+    }
+
+  return insertPassThrough();
+}
+
+bool BPFAdjustOpt::insertPassThrough() {
+  for (auto &Info : PassThroughs) {
+    auto *CI = BPFCoreSharedInfo::insertPassThrough(
+        Mod, Info.UsedInst->getParent(), Info.Input, Info.UsedInst);
+    Info.UsedInst->setOperand(Info.OpIdx, CI);
+  }
+
+  return !PassThroughs.empty();
+}
+
+// To avoid combining conditionals in the same basic block by
+// instrcombine optimization.
+bool BPFAdjustOpt::serializeICMPInBB(Instruction &I) {
+  // For:
+  //   comp1 = icmp <opcode> ...;
+  //   comp2 = icmp <opcode> ...;
+  //   ... or comp1 comp2 ...
+  // changed to:
+  //   comp1 = icmp <opcode> ...;
+  //   comp2 = icmp <opcode> ...;
+  //   new_comp1 = __builtin_bpf_passthrough(seq_num, comp1)
+  //   ... or new_comp1 comp2 ...
+  if (I.getOpcode() != Instruction::Or)
+    return false;
+  auto *Icmp1 = dyn_cast<ICmpInst>(I.getOperand(0));
+  if (!Icmp1)
+    return false;
+  auto *Icmp2 = dyn_cast<ICmpInst>(I.getOperand(1));
+  if (!Icmp2)
+    return false;
+
+  Value *Icmp1Op0 = Icmp1->getOperand(0);
+  Value *Icmp2Op0 = Icmp2->getOperand(0);
+  if (Icmp1Op0 != Icmp2Op0)
+    return false;
+
+  // Now we got two icmp instructions which feed into
+  // an "or" instruction.
+  PassThroughInfo Info(Icmp1, &I, 0);
+  PassThroughs.push_back(Info);
+  return true;
+}
+
+// To avoid combining conditionals in the same basic block by
+// instrcombine optimization.
+bool BPFAdjustOpt::serializeICMPCrossBB(BasicBlock &BB) {
+  // For:
+  //   B1:
+  //     comp1 = icmp <opcode> ...;
+  //     if (comp1) goto B2 else B3;
+  //   B2:
+  //     comp2 = icmp <opcode> ...;
+  //     if (comp2) goto B4 else B5;
+  //   B4:
+  //     ...
+  // changed to:
+  //   B1:
+  //     comp1 = icmp <opcode> ...;
+  //     comp1 = __builtin_bpf_passthrough(seq_num, comp1);
+  //     if (comp1) goto B2 else B3;
+  //   B2:
+  //     comp2 = icmp <opcode> ...;
+  //     if (comp2) goto B4 else B5;
+  //   B4:
+  //     ...
+
+  // Check basic predecessors, if two of them (say B1, B2) are using
+  // icmp instructions to generate conditions and one is the predesessor
+  // of another (e.g., B1 is the predecessor of B2). Add a passthrough
+  // barrier after icmp inst of block B1.
+  BasicBlock *B2 = BB.getSinglePredecessor();
+  if (!B2)
+    return false;
+
+  BasicBlock *B1 = B2->getSinglePredecessor();
+  if (!B1)
+    return false;
+
+  Instruction *TI = B2->getTerminator();
+  auto *BI = dyn_cast<BranchInst>(TI);
+  if (!BI || !BI->isConditional())
+    return false;
+  auto *Cond = dyn_cast<ICmpInst>(BI->getCondition());
+  if (!Cond || B2->getFirstNonPHI() != Cond)
+    return false;
+  Value *B2Op0 = Cond->getOperand(0);
+  auto Cond2Op = Cond->getPredicate();
+
+  TI = B1->getTerminator();
+  BI = dyn_cast<BranchInst>(TI);
+  if (!BI || !BI->isConditional())
+    return false;
+  Cond = dyn_cast<ICmpInst>(BI->getCondition());
+  if (!Cond)
+    return false;
+  Value *B1Op0 = Cond->getOperand(0);
+  auto Cond1Op = Cond->getPredicate();
+
+  if (B1Op0 != B2Op0)
+    return false;
+
+  if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) {
+    if (Cond2Op != ICmpInst::ICMP_SLT && Cond1Op != ICmpInst::ICMP_SLE)
+      return false;
+  } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) {
+    if (Cond2Op != ICmpInst::ICMP_SGT && Cond1Op != ICmpInst::ICMP_SGE)
+      return false;
+  } else {
+    return false;
+  }
+
+  PassThroughInfo Info(Cond, BI, 0);
+  PassThroughs.push_back(Info);
+
+  return true;
+}
+
+// To avoid speculative hoisting certain computations out of
+// a basic block.
+bool BPFAdjustOpt::avoidSpeculation(Instruction &I) {
+  if (auto *LdInst = dyn_cast<LoadInst>(&I)) {
+    if (auto *GV = dyn_cast<GlobalVariable>(LdInst->getOperand(0))) {
+      if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) ||
+          GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr))
+        return false;
+    }
+  }
+
+  if (!dyn_cast<LoadInst>(&I) && !dyn_cast<CallInst>(&I))
+    return false;
+
+  // For:
+  //   B1:
+  //     var = ...
+  //     ...
+  //     /* icmp may not be in the same block as var = ... */
+  //     comp1 = icmp <opcode> var, <const>;
+  //     if (comp1) goto B2 else B3;
+  //   B2:
+  //     ... var ...
+  // change to:
+  //   B1:
+  //     var = ...
+  //     ...
+  //     /* icmp may not be in the same block as var = ... */
+  //     comp1 = icmp <opcode> var, <const>;
+  //     if (comp1) goto B2 else B3;
+  //   B2:
+  //     var = __builtin_bpf_passthrough(seq_num, var);
+  //     ... var ...
+  bool isCandidate = false;
+  SmallVector<PassThroughInfo, 4> Candidates;
+  for (User *U : I.users()) {
+    Instruction *Inst = dyn_cast<Instruction>(U);
+    if (!Inst)
+      continue;
+
+    // May cover a little bit more than the
+    // above pattern.
+    if (auto *Icmp1 = dyn_cast<ICmpInst>(Inst)) {
+      Value *Icmp1Op1 = Icmp1->getOperand(1);
+      if (!isa<Constant>(Icmp1Op1))
+        return false;
+      isCandidate = true;
+      continue;
+    }
+
+    // Ignore the use in the same basic block as the definition.
+    if (Inst->getParent() == I.getParent())
+      continue;
+
+    // use in a different basic block, If there is a call or
+    // load/store insn before this instruction in this basic
+    // block. Most likely it cannot be hoisted out. Skip it.
+    for (auto &I2 : *Inst->getParent()) {
+      if (dyn_cast<CallInst>(&I2))
+        return false;
+      if (dyn_cast<LoadInst>(&I2) || dyn_cast<StoreInst>(&I2))
+        return false;
+      if (&I2 == Inst)
+        break;
+    }
+
+    // It should be used in a GEP or a simple arithmetic like
+    // ZEXT/SEXT which is used for GEP.
+    if (Inst->getOpcode() == Instruction::ZExt ||
+        Inst->getOpcode() == Instruction::SExt) {
+      PassThroughInfo Info(&I, Inst, 0);
+      Candidates.push_back(Info);
+    } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
+      // traverse GEP inst to find Use operand index
+      unsigned i, e;
+      for (i = 1, e = GI->getNumOperands(); i != e; ++i) {
+        Value *V = GI->getOperand(i);
+        if (V == &I)
+          break;
+      }
+      if (i == e)
+        continue;
+
+      PassThroughInfo Info(&I, GI, i);
+      Candidates.push_back(Info);
+    }
+  }
+
+  if (!isCandidate || Candidates.empty())
+    return false;
+
+  PassThroughs.insert(PassThroughs.end(), Candidates.begin(), Candidates.end());
+  return true;
+}
+
+void BPFAdjustOpt::adjustBasicBlock(BasicBlock &BB) {
+  if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB))
+    return;
+}
+
+void BPFAdjustOpt::adjustInst(Instruction &I) {
+  if (!DisableBPFserializeICMP && serializeICMPInBB(I))
+    return;
+  if (!DisableBPFavoidSpeculation && avoidSpeculation(I))
+    return;
+}

llvm/lib/Target/BPF/BPFTargetMachine.cpp

extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() {
   // Register the target.
   RegisterTargetMachine<BPFTargetMachine> X(getTheBPFleTarget());
   RegisterTargetMachine<BPFTargetMachine> Y(getTheBPFbeTarget());
   RegisterTargetMachine<BPFTargetMachine> Z(getTheBPFTarget());
 
   PassRegistry &PR = *PassRegistry::getPassRegistry();
   initializeBPFAbstractMemberAccessLegacyPassPass(PR);
   initializeBPFPreserveDITypePass(PR);
+  initializeBPFAdjustOptPass(PR);
   initializeBPFCheckAndAdjustIRPass(PR);
   initializeBPFMIPeepholePass(PR);
   initializeBPFMIPeepholeTruncElimPass(PR);
 }
 
 // DataLayout: little or big endian
 static std::string computeDataLayout(const Triple &TT) {
   if (TT.getArch() == Triple::bpfeb)
Builder.addExtension(
       });
 
   Builder.addExtension(
       PassManagerBuilder::EP_Peephole,
       [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
         PM.add(createCFGSimplificationPass(
             SimplifyCFGOptions().hoistCommonInsts(true)));
       });
+  Builder.addExtension(
+      PassManagerBuilder::EP_ModuleOptimizerEarly,
+      [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
+        PM.add(createBPFAdjustOpt());
+      });
 }
 
 void BPFTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB,
                                                     bool DebugPassManager) {
   PB.registerPipelineStartEPCallback([=](ModulePassManager &MPM) {
     FunctionPassManager FPM(DebugPassManager);
     FPM.addPass(BPFAbstractMemberAccessPass(this));
     FPM.addPass(BPFPreserveDITypePass());

llvm/lib/Target/BPF/CMakeLists.txt

 set(LLVM_TARGET_DEFINITIONS BPF.td)
 
 tablegen(LLVM BPFGenAsmMatcher.inc -gen-asm-matcher)
 tablegen(LLVM BPFGenAsmWriter.inc -gen-asm-writer)
 tablegen(LLVM BPFGenCallingConv.inc -gen-callingconv)
 tablegen(LLVM BPFGenDAGISel.inc -gen-dag-isel)
 tablegen(LLVM BPFGenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM BPFGenInstrInfo.inc -gen-instr-info)
 tablegen(LLVM BPFGenMCCodeEmitter.inc -gen-emitter)
 tablegen(LLVM BPFGenRegisterInfo.inc -gen-register-info)
 tablegen(LLVM BPFGenSubtargetInfo.inc -gen-subtarget)
 
 add_public_tablegen_target(BPFCommonTableGen)
 
 add_llvm_target(BPFCodeGen
   BPFAbstractMemberAccess.cpp
+  BPFAdjustOpt.cpp
   BPFAsmPrinter.cpp
   BPFCheckAndAdjustIR.cpp
   BPFFrameLowering.cpp
   BPFInstrInfo.cpp
   BPFISelDAGToDAG.cpp
   BPFISelLowering.cpp
   BPFMCInstLower.cpp
   BPFPreserveDIType.cpp

llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll

+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+;
+; Source:
+;   int foo();
+;   int bar(int);
+;   int test() {
+;     int ret = foo();
+;     if (ret <= 0 || ret > 7)
+;       return 0;
+;     return bar(ret);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %0 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+  %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+  store i32 %call, i32* %ret, align 4, !tbaa !2
+  %1 = load i32, i32* %ret, align 4, !tbaa !2
+  %cmp = icmp sle i32 %1, 0
+  br i1 %cmp, label %if.then, label %lor.lhs.false
+
+; CHECK:         [[REG1:r[0-9]+]] <<= 32
+; CHECK:         [[REG1]] s>>= 32
+; CHECK:         [[REG2:r[0-9]+]] = 1
+; CHECK:         if [[REG2]] s> [[REG1]] goto
+; CHECK:         if [[REG1]] s> 7 goto
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: if [[REG1]] > 6 goto
+
+lor.lhs.false:                                    ; preds = %entry
+  %2 = load i32, i32* %ret, align 4, !tbaa !2
+  %cmp1 = icmp sgt i32 %2, 7
+  br i1 %cmp1, label %if.then, label %if.end
+
+if.then:                                          ; preds = %lor.lhs.false, %entry
+  store i32 0, i32* %retval, align 4
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup
+
+if.end:                                           ; preds = %lor.lhs.false
+  %3 = load i32, i32* %ret, align 4, !tbaa !2
+  %call2 = call i32 @bar(i32 %3)
+  store i32 %call2, i32* %retval, align 4
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end, %if.then
+  %4 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3
+  %5 = load i32, i32* %retval, align 4
+  ret i32 %5
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+declare dso_local i32 @bar(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}

llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll

+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
+;
+; Source:
+;   int foo();
+;   int bar(int);
+;   int test() {
+;     int ret = foo();
+;     if (ret <= 0)
+;       return 0;
+;     if (ret > 7)
+;       return 0;
+;     return bar(ret);
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i32 @test() #0 {
+entry:
+  %retval = alloca i32, align 4
+  %ret = alloca i32, align 4
+  %cleanup.dest.slot = alloca i32, align 4
+  %0 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+  %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+  store i32 %call, i32* %ret, align 4, !tbaa !2
+  %1 = load i32, i32* %ret, align 4, !tbaa !2
+  %cmp = icmp sle i32 %1, 0
+  br i1 %cmp, label %if.then, label %if.end
+
+; CHECK:         [[REG1:r[0-9]+]] <<= 32
+; CHECK:         [[REG1]] s>>= 32
+; CHECK:         [[REG2:r[0-9]+]] = 1
+; CHECK:         if [[REG2]] s> [[REG1]] goto
+; CHECK:         if [[REG1]] s> 7 goto
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: if [[REG1]] > 6 goto
+
+if.then:                                          ; preds = %entry
+  store i32 0, i32* %retval, align 4
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup
+
+if.end:                                           ; preds = %entry
+  %2 = load i32, i32* %ret, align 4, !tbaa !2
+  %cmp1 = icmp sgt i32 %2, 7
+  br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2:                                         ; preds = %if.end
+  store i32 0, i32* %retval, align 4
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup
+
+if.end3:                                          ; preds = %if.end
+  %3 = load i32, i32* %ret, align 4, !tbaa !2
+  %call4 = call i32 @bar(i32 %3)
+  store i32 %call4, i32* %retval, align 4
+  store i32 1, i32* %cleanup.dest.slot, align 4
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.end3, %if.then2, %if.then
+  %4 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3
+  %5 = load i32, i32* %retval, align 4
+  ret i32 %5
+}
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+declare dso_local i32 @bar(i32) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"int", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}

llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll

+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+;
+; Source:
+;   unsigned long foo();
+;   void *test(void *p) {
+;     unsigned long ret = foo();
+;     if (ret <= 7)
+;       p += ret;
+;     return p;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i8* @test(i8* %p) #0 {
+entry:
+  %p.addr = alloca i8*, align 8
+  %ret = alloca i64, align 8
+  store i8* %p, i8** %p.addr, align 8, !tbaa !2
+  %0 = bitcast i64* %ret to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #3
+  %call = call i64 bitcast (i64 (...)* @foo to i64 ()*)()
+  store i64 %call, i64* %ret, align 8, !tbaa !6
+  %1 = load i64, i64* %ret, align 8, !tbaa !6
+  %cmp = icmp ule i64 %1, 7
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %2 = load i64, i64* %ret, align 8, !tbaa !6
+  %3 = load i8*, i8** %p.addr, align 8, !tbaa !2
+  %add.ptr = getelementptr i8, i8* %3, i64 %2
+  store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %4 = load i8*, i8** %p.addr, align 8, !tbaa !2
+  %5 = bitcast i64* %ret to i8*
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* %5) #3
+  ret i8* %4
+}
+; CHECK-COMMON:  [[REG6:r[0-9]+]] = r1
+; CHECK-COMMON:  call foo
+
+; CHECK:         if r0 > 7 goto [[LABEL:.*]]
+; CHECK:         [[REG6]] += r0
+; CHECK:         [[LABEL]]:
+; CHECK:         r0 = [[REG6]]
+
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: r0 += [[REG1:r[0-9]+]]
+; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8
+; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]]
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: [[LABEL]]:
+
+; CHECK-COMMON:  exit
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i64 @foo(...) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"long", !4, i64 0}

llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll

+; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
+; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
+; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
+;
+; Source:
+;   unsigned foo();
+;   void *test(void *p) {
+;     unsigned ret = foo();
+;     if (ret <= 7)
+;       p += ret;
+;     return p;
+;   }
+; Compilation flag:
+;   clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c
+
+; Function Attrs: nounwind
+define dso_local i8* @test(i8* %p) #0 {
+entry:
+  %p.addr = alloca i8*, align 8
+  %ret = alloca i32, align 4
+  store i8* %p, i8** %p.addr, align 8, !tbaa !2
+  %0 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3
+  %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)()
+  store i32 %call, i32* %ret, align 4, !tbaa !6
+  %1 = load i32, i32* %ret, align 4, !tbaa !6
+  %cmp = icmp ule i32 %1, 7
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %ret, align 4, !tbaa !6
+  %3 = load i8*, i8** %p.addr, align 8, !tbaa !2
+  %idx.ext = zext i32 %2 to i64
+  %add.ptr = getelementptr i8, i8* %3, i64 %idx.ext
+  store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  %4 = load i8*, i8** %p.addr, align 8, !tbaa !2
+  %5 = bitcast i32* %ret to i8*
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #3
+  ret i8* %4
+}
+
+; CHECK-COMMON:  [[REG6:r[0-9]+]] = r1
+; CHECK-COMMON:  call foo
+
+; CHECK:         r0 <<= 32
+; CHECK:         r0 >>= 32
+; CHECK:         if r0 > 7 goto [[LABEL:.*]]
+; CHECK:         [[REG6]] += r0
+; CHECK:         [[LABEL]]:
+; CHECK:         r0 = [[REG6]]
+
+; CHECK-DISABLE: [[REG1:r[0-9]+]] = r0
+; CHECK-DISABLE: [[REG1]] <<= 32
+; CHECK-DISABLE: [[REG1]] >>= 32
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: r0 += [[REG1]]
+; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8
+; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]]
+; CHECK-DISABLE: r0 = [[REG6]]
+; CHECK-DISABLE: [[LABEL]]:
+
+; CHECK-COMMON:  exit
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare dso_local i32 @foo(...) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1
+
+attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind willreturn }
+attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { nounwind }
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"}
+!2 = !{!3, !3, i64 0}
+!3 = !{!"any pointer", !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{!7, !7, i64 0}
+!7 = !{!"int", !4, i64 0}