[TTI][BPF]: Undo specific transform-preventing passes and add one TTI hook

[yonghong-song]

Summary

LLVM optimization may generate certain codes which cannot be
handled by kernel verifier, e.g., some optimizations in
InstCombine and SimplifyCFG as bpf verifier is implemented
in kernel and it cannot perform complicated code analysis like
llvm compiler. Memory, verification speed and verifier complexity
are all part of considerations when adding new analysis
in verifier.

To avoid such harmful transformation, BPF backend has implemented
some passes, esp. through 'barrier' builtin function to prevent
certain InstCombine and SimplifyCFG transformations.
In these BPF backend passes pattern matching are used
to capture some specific patterns to prevent some
llvm transformations. But such pattern matching may not be precise
and may prevent some useful transformations. It would be great
if we can directly disable llvm transformations and this will
also avoid bpf specific transformation-preventing passes.
The following is 'git show --stat' to show that we can remove
lots of bpf hacking codes by adding one TTI hook.

llvm/include/llvm/Analysis/TargetTransformInfo.h        |   9 +++
llvm/include/llvm/Analysis/TargetTransformInfoImpl.h    |   2 +
llvm/include/llvm/IR/IntrinsicsBPF.td                   |   3 -
llvm/include/llvm/Transforms/InstCombine/InstCombiner.h |   1 +
llvm/include/llvm/Transforms/Utils/LoopUtils.h          |   8 +-
llvm/lib/Analysis/TargetTransformInfo.cpp               |   4 +
llvm/lib/Target/BPF/BPF.h                               |   7 --
llvm/lib/Target/BPF/BPFAdjustOpt.cpp                    | 393 --------------------------------------------------------------------------------------------
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp             |  45 +----------
llvm/lib/Target/BPF/BPFTargetMachine.cpp                |   5 --
llvm/lib/Target/BPF/BPFTargetTransformInfo.h            |   3 +
llvm/lib/Target/BPF/CMakeLists.txt                      |   1 -
llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp |   4 +
llvm/lib/Transforms/Scalar/LICM.cpp                     |  27 ++++---
llvm/lib/Transforms/Utils/SimplifyCFG.cpp               |   3 +
llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll               |  14 +---
llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll               |  10 +--
llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll               |  12 +--
llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll               |  12 +--
llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll        |  17 +---
llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll        |  22 +-----
21 files changed, 69 insertions(+), 533 deletions(-)

Below are detailed explanations for three transformations
which hurts bpf verification.

FoldAndOrOfICmpsUsingRanges

The following is an example to show how FoldAndOrOfICmpsUsingRanges
transformation may generate codes which hurts bpf verifier.
For bpf prog in linux/tools/testing/selftests/bpf/progs/map_kptr_fail.c:

...
id = ctx->protocol;
if (id < 4 || id > 12)
  return 0;
*(u64 *)((void *)v + id) = 0;
...

With FoldAndOrOfICmpsUsingRanges, the find pseudo code looks like:

...
id = ctx->protocol;
tmp = id;
tmp += -13;
if (tmp < 0xfffffff7) goto next;
v += id;
*v = 0;
next:

In the above code, the verifier considers 'id' in 'v += id' as a arbitrary
unsigned integer so later '*v = 0' is considered as possible out-of-bound
memory access. This is because the verifier, as a post analysis tool,
does not know the relationship of tmp/id at 'v += id' point. Although it
is possible to improve verifier to track tmp/id relationship, this would
increase bpf verifier complexity a lot. llvm FoldAndOrOfICmpsUsingRanges
does the transformation based on pattern matching and it certainly aware
tmp/id relationship.

The actual verification failure looks like below:

; id = ctx->protocol;
9: (61) r1 = *(u32 *)(r6 +16)         ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6_w=ctx(off=0,imm=0)
; if (id < 4 || id > 12)
10: (bc) w2 = w1                      ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
11: (04) w2 += -13                    ; R2=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
12: (a6) if w2 < 0xfffffff7 goto pc+3         ; R2=scalar(umin=4294967287,umax=4294967295,var_off=(0xfffffff0; 0xf),s32_min=-9,s32_max=-1)
; *(u64 *)((void *)v + id) = 0;
13: (0f) r0 += r1                     ; R0_w=map_value(off=0,ks=4,vs=32,umax=4294967295,var_off=(0x0; 0xffffffff)) R1=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
14: (b7) r1 = 0                       ; R1_w=0
; *(u64 *)((void *)v + id) = 0;
15: (7b) *(u64 *)(r0 +0) = r1
R0 unbounded memory access, make sure to bounds check any such access

FoldTwoEntryPHINode

The following is an example to show FoldTwoEntryPHINode transformation
may generate codes which hurts bpf verifier.
For bpf prog in linux/tools/testing/selftests/bpf/progs/test_tc_dtime.c:

static void inc_errs(__u32 idx)
{
      if (test < 9)
              errs[test][idx]++;
      else
              errs[UKN_TEST][idx]++;
}
...
if (skb->tstamp == 0xb9fbeef)
  inc_errs(2);
...

With FoldTwoEntryPHINode, the final generated code looks like

...
r1 = test;
r2 = skb->tstamp;
if (r2 != 0xb9fbeef) goto next;
w2 = w1; // w1/w2 are lower 32 bit values of r1/r2.
if (w1 >= 9)
  w2 = 9;
tmp = r2 * 28;
r3 = errs + tmp;
... *r3 ...
...

next:

In the above code, for the case where 'w1 >= 9' is false, verifier
concludes that 'r2' at 'tmp = r2 * 28' as an arbitrary scalar which
caused verificaiton failure for later dereference of r3.

The actual verification failure looks like below:

8: (18) r1 = 0xffffc900001ca230       ; R1_w=map_value(off=560,ks=4,vs=564,imm=0)
10: (61) r1 = *(u32 *)(r1 +0)         ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
; if (skb->tstamp == EGRESS_ENDHOST_MAGIC)
11: (79) r2 = *(u64 *)(r6 +152)       ; R2_w=scalar() R6=ctx(off=0,imm=0)
; if (skb->tstamp == EGRESS_ENDHOST_MAGIC)
12: (55) if r2 != 0xb9fbeef goto pc+10        ; R2_w=195018479
13: (bc) w2 = w1                      ; R1_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff))
; if (test < __NR_TESTS)
14: (a6) if w1 < 0x9 goto pc+1 16: R0=2 R1_w=scalar(umax=8,var_off=(0x0; 0xf)) R2_w=scalar(umax=4294967295,var_off=(0x0; 0xffffffff)) R6=ctx(off=0,imm=0) R10=fp0
;
16: (27) r2 *= 28                     ; R2_w=scalar(umax=120259084260,var_off=(0x0; 0x1ffffffffc),s32_max=2147483644,u32_max=-4)
17: (18) r3 = 0xffffc900001ca118      ; R3_w=map_value(off=280,ks=4,vs=564,imm=0)
19: (0f) r3 += r2                     ; R2_w=scalar(umax=120259084260,var_off=(0x0; 0x1ffffffffc),s32_max=2147483644,u32_max=-4) R3_w=map_value(off=280,ks=4,vs=564,umax=120259084260,var_off=(0x0; 0x1ffffffffc),s32_max=2147483644,u32_max=-4)
20: (61) r2 = *(u32 *)(r3 +0)
R3 unbounded memory access, make sure to bounds check any such access

The unit test adjust-opt-speculative2.ll also shows how FoldTwoEntryPHINode
might hurt verifier.

The original code:

unsigned foo();
void *test(void *p) {
  unsigned ret = foo();
  if (ret <= 7)
    p += ret;
  return p;
}

Compiled with clang -target bpf -O2 -S t.c, with FoldTwoEntryPHINode enabled,
the following code is generated:

1:    r6 = r1
2:    call foo
3:    r1 = r0
4:    r1 <<= 32
5:    r1 >>= 32
6:    r2 = 8
7:    if r2 > r1 goto LBB0_2
8:    r0 = 0
   LBB0_2:
9:    r0 <<= 32
10:   r0 >>= 32
11:   r6 += r0
12:   r0 = r6
13:   exit

In the above example, insn 3 establishes r1 and r0 equivalence. Insns 4-7
establishes r1 < 8 if branch is taken. However, with branch taken, later
verifier is not able to ensure r0 < 8 and 'r6 += r0' may have a verificaiton
error.

With FoldTwoEntryPHINode disabled, the following code is generated:

1:    r6 = r1
2:    call foo
3:    r0 <<= 32
4:    r0 >>= 32
5:    if r0 > 7 goto LBB0_2
6:    r6 += r0
   LBB0_2:
7:    r0 = r6
8:    exit

The 'r6 += r0' is safe as the verifier can deduce 'r0 <= 7' based on the branch.

MinMaxHoisting

Furthermore, recently we hit another issue related LICM
MinMaxHoisting transformation (https://reviews.llvm.org/D147078)
which also hurts verifier.
For bpf prog in linux/tools/testing/selftests/bpf/progs/loop6.c:

The original code:

for (i = 0; (i < VIRTIO_MAX_SGS) && (i < out_sgs); i++) {
        for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX);
             sgp = __sg_next(sgp)) {
                bpf_probe_read_kernel(&len, sizeof(len), &sgp->length);
                length1 += len;
                n++;
        }
}

After MinMaxHoisting,

upper = MIN(VIRTIO_MAX_SGS, out_sgs);
for (i = 0; i < upper; i++) {
        for (n = 0, sgp = get_sgp(sgs, i); sgp && (n < SG_MAX);
             sgp = __sg_next(sgp)) {
                bpf_probe_read_kernel(&len, sizeof(len), &sgp->length);
                length1 += len;
                n++;
        }
}

The verifier is not able to verify properly since it assumes the loop upper
is a arbitrary scalar (up to 32bit integer). The actual verification failure
looks like:

...
119: (15) if r1 == 0x0 goto pc+1
The sequence of 8193 jumps is too complex.

We have a draft to show bpf backend
implementation to undo the transformation (https://reviews.llvm.org/D147990).

Proposal

We feel adding proper TTI hooks is a better solution.
TTI provides a mechanism so backend can influence the
transformation. This also helps remove the associated bpf backend
pattern matching transformations.

This patch undos previous InstCombine/SimplifyCFG
transformation-preventing passes and adds one TTI hook
TTI->needsPreserveRangeInfoInVerification() such that
the above mentioned transformations can be disabled
by the target. The hook name needsPreserveRangeInfoInVerification()
implies that the transformation is disabled due to
later downstrean code verification.

Another possible solution is to legalize IR for verificaiton requirement.
This may require to add the verification requirement to IR, or
establish certain illegal code patterns, etc. This approach
requires more thought as downstream verification capability and
new code pattern verification failure is also a moving target.

[yonghong-song]

The previous patch caused several bpf selftest failures. Remove two flags and changed a few tests so bpf selftests can pass.

[yonghong-song]

• Using TTI hooks instead of flags

[yonghong-song]

• do not add TTI to InstCombinerImpl, use the existing one in InstCombiner
• add a code example to illustrate the problem in Summary

[yonghong-song]

• add TTI hook for LICM/HoistMinMax transformations.
• add detailed code analysis in commit message.

[yonghong-song]

@mkazantsev @nikic @chandlerc @lebedev.ri @spatel Ping. Could you help take a look at this patch? Thanks!

[yonghong-song]

Ping again. @mkazantsev @nikic @chandlerc @lebedev.ri @spatel Could you help review the patch and share your opinion? Thanks!

[jemarch]

In my opinion this new approach of using target hooks is way better than trying to amend the effect of certain passes by adding "counter passes", which is not only a moving target, but also fragile and IMO a waste of effort. We do have the same problem with the GCC BPF backend, and will be using a similar strategy to what Yonghong is proposing here.

[RKSimon]

llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll

41

You've left a lot of orphan CHECK-DISABLE in various files where you've removed the bpf-disable-serialize-icmp RUN

[nikic]

This has already been discussed in D147078. All the middle-end maintainers who chimed in on that one were opposed to the proposal, and I don't think their responses will be different this time. I don't really want to repeat that discussion one more time here, because it felt a bit like talking at a wall. If you want to pursue this further, I would suggest starting an RFC on Discourse, because it seems pretty clear that we don't have a consensus between BPF and middle-end maintainers here.

PS: I would recommend shifting your thinking about these undo transforms from "precisely undo what this pass did" to "legalize IR for BPF". The draft patch at D147990 is needlessly complex because it tries to precisely undo what LICM does (down to loop invariance checks!) instead of treating it as a generic legalization problem.

[davidxl]

Can we have a middle ground here. Instead of having too many new TTI interfaces here, a single one can be used instead: TTI:needsPreserveRangeInfoInVerification(). If this returns true, some passes (or subpasses in instcombine) can be turned off for the target. Similar things are done in vectorizer.

Transformations that are guarded by this check also need to be checked case by case. If the verifier can be enhanced without too much compile time overhead, it should probably be done there -- it adds more benefit of allowing more flexibility in source patterns.

Legalization can be a longer term thing to to think about. Like undoing transformations (excluding pure canonicalization ones), it does add unnecessary compile time overhead.

[yonghong-song]

@davidxl Thanks for your suggestions. Adding a single TTI->needsPreserveRangeInfoInVerification() sounds a reasonable idea. Legalization of IR for BPF verification is a great idea but it requires more thought, e.g., how IR will be enhanced to encode verification requirement and how middle end optimization reacts to such requirement as typically it is not clear whether a particular optimization will hurt bpf verification or not unless running though bpf verifier or having a deep knowledge of bpf verifier. We can continue to think and discuss 'legalization of IR for BPF verification' idea, but it could be great we can have current TTI->needsPreserveRangeInfoInVerification() approach which gcc community intends to do the same.

Also as you mentioned, bpf verifier is a moving target as well and we are constantly improving verifier as well. Yes, once we fixed verifier, after some times, old kernels are considered not important, we might undo some hooks in llvm.

I will post another version of the patch soon.

[yonghong-song]

• use one hook TTI->needsPreserveRangeInfoInVerification() instead of three hooks
• remove unused checks in the tests

[davidxl]

Considering the overall longer term maintenance cost to middle-end maintainers and the cost the BPF backend developers, this approach seems like the least intrusive method to me. Middle end maintainers (nikic@) need to chime in if there are more concerns on the approach to unblock the progress. The assumption is that more longer term solution will be explored further (e.g. legalization, or maintaining range info etc).

[yonghong-song]

@nikic Could you comment on @davidxl suggestion? I would be great if we can find a path forward for this particular issue. Thanks!

[wenlei]

My two cents:

I think that in general honoring canonicalization is good, but it's not always a clear cut. Legality and profitability are sometimes relative, and in the case of BPF, I do think there's case to be made for mid-end to look at TTI more then what is traditionally allowed. The failure mode here is different from profitability, where you may just end up with inefficient code if you don't undo later; for BPF though, it can generate program that will be rejected by verifier (not run slower). Technically BPF backend can define what is legal for that target, i.e. one could argue that BPF target requires preserving predicates in certain form (needs to be well defined though), which would then restrict certain optimization from the mid-end.

Perhaps BPF is an uncharted territory in the sense that it's so restrictive that its requirements are often at odds with more canonicalizations, and that requiring backend to undo everything is bordering impractical. A special case grant for untraditional use of TTI may be reasonable here - we can make the reasons clear so it won't set bad precedence for others.

[nikic]

In D147968#4416082, @wenlei wrote:

I think that in general honoring canonicalization is good, but it's not always a clear cut. Legality and profitability are sometimes relative, and in the case of BPF, I do think there's case to be made for mid-end to look at TTI more then what is traditionally allowed. The failure mode here is different from profitability, where you may just end up with inefficient code if you don't undo later; for BPF though, it can generate program that will be rejected by verifier (not run slower). Technically BPF backend can define what is legal for that target, i.e. one could argue that BPF target requires preserving predicates in certain form (needs to be well defined though), which would then restrict certain optimization from the mid-end.

I am, generally speaking, not opposed to doing TTI-based legality checks in special circumstances. I have approved exceptions for doing so in the past. A recent one was using TTI in InstCombine to check whether a certain address space cast is legal for the target. Without TTI, we assume that all address space casts are illegal. This exception made sense to me, because addrspacecast semantics are fundamentally target-dependent, and the legality check is compatible with the LangRef semantics.

The case here is very different, because the legality conditions for BPF are not well-defined and not compatible with IR semantics. BPF considers transforms "illegal" that are clearly legal under our operational semantics, and, to the best of my knowledge, there is no principled way a maintainer could determine whether or not a given transform would be "legal" for BPF or not.

We have other targets with somewhat "unusual" legality requirements, where the target does not accept arbitrary inputs. These include things like wasm (which has verifier requirements) and gpu targets like amdgpu and nvptx (which also have verifier requirements, as well as convergence restrictions). These additional legality requirements are handled in one of two ways: Either the backend takes the responsibility of converting IR into a legal form (e.g. CFG structurization or removal of irreducible cycles) or the additional legality requirements are encoded into the IR, e.g. through the use special intrinsics, operand bundles and/or token values. This makes the legality constraint part of the normal IR semantics, and we can use our normal reasoning and tools to determine whether transforms are legal or not.

I maintain my position that the BPF target should be using either of those approaches (or a combination of them). Otherwise we will end up with checks for the BPF target littered over random places in the code base, with a new check being added every time a commit breaks the BPF verifier.

[yonghong-song]

@nikic Thanks for your comment and pointer! I will study Webassembly/AMDGPU/NVPTX to see how they resolve their respective 'legality' issues and will report back once I got some understanding about their strategy and commonality/difference w.r.t. BPF.

llvm/include/llvm/IR/IntrinsicsBPF.td

let TargetPrefix = "bpf" in {  // All intrinsics start with "llvm.bpf."
   def int_bpf_preserve_type_info : ClangBuiltin<"__builtin_bpf_preserve_type_info">,
               Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
               [IntrNoMem]>;
   def int_bpf_preserve_enum_value : ClangBuiltin<"__builtin_bpf_preserve_enum_value">,
               Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_ptr_ty, llvm_i64_ty],
               [IntrNoMem]>;
   def int_bpf_passthrough : ClangBuiltin<"__builtin_bpf_passthrough">,
               Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]>;
-  def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">,
-              Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty],
-              [IntrNoMem]>;
 }

llvm/include/llvm/Transforms/InstCombine/InstCombiner.h

 namespace llvm {
 
 class AAResults;
 class AssumptionCache;
 class ProfileSummaryInfo;
 class TargetLibraryInfo;
 class TargetTransformInfo;
 
+extern cl::opt<bool> EnableFoldAndOrOfICmpsUsingRanges;
+
 /// The core instruction combiner logic.
 ///
 /// This class provides both the logic to recursively visit instructions and
 /// combine them.
 class LLVM_LIBRARY_VISIBILITY InstCombiner {
   /// Only used to call target specific intrinsic combining.
   /// It must **NOT** be used for any other purpose, as InstCombine is a
   /// target-independent canonicalization transform.

llvm/include/llvm/Transforms/Utils/Local.h

 
 /// This function is used to do simplification of a CFG.  For example, it
 /// adjusts branches to branches to eliminate the extra hop, it eliminates
 /// unreachable basic blocks, and does other peephole optimization of the CFG.
 /// It returns true if a modification was made, possibly deleting the basic
 /// block that was pointed to. LoopHeaders is an optional input parameter
 /// providing the set of loop headers that SimplifyCFG should not eliminate.
 extern cl::opt<bool> RequireAndPreserveDomTree;
+extern cl::opt<bool> AllowFoldTwoEntryPHINode;
 bool simplifyCFG(BasicBlock *BB, const TargetTransformInfo &TTI,
                  DomTreeUpdater *DTU = nullptr,
                  const SimplifyCFGOptions &Options = {},
                  ArrayRef<WeakVH> LoopHeaders = {});
 
 /// This function is used to flatten a CFG. For example, it uses parallel-and
 /// and parallel-or mode to collapse if-conditions and merge if-regions with
 /// identical statements.

llvm/lib/Target/BPF/BPF.h

 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include "llvm/Target/TargetMachine.h"
 
 namespace llvm {
 class BPFTargetMachine;
 class PassRegistry;
 
-ModulePass *createBPFAdjustOpt();
 ModulePass *createBPFCheckAndAdjustIR();
 
 FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM);
 FunctionPass *createBPFPreserveDIType();
 FunctionPass *createBPFIRPeephole();
 FunctionPass *createBPFISelDag(BPFTargetMachine &TM);
 FunctionPass *createBPFMISimplifyPatchablePass();
 FunctionPass *createBPFMIPeepholePass();
 FunctionPass *createBPFMIPeepholeTruncElimPass();
 FunctionPass *createBPFMIPreEmitPeepholePass();
 FunctionPass *createBPFMIPreEmitCheckingPass();
 
 void initializeBPFAbstractMemberAccessLegacyPassPass(PassRegistry &);
-void initializeBPFAdjustOptPass(PassRegistry&);
 void initializeBPFCheckAndAdjustIRPass(PassRegistry&);
 void initializeBPFDAGToDAGISelPass(PassRegistry &);
 void initializeBPFIRPeepholePass(PassRegistry &);
 void initializeBPFMIPeepholePass(PassRegistry&);
 void initializeBPFMIPeepholeTruncElimPass(PassRegistry &);
 void initializeBPFMIPreEmitCheckingPass(PassRegistry&);
 void initializeBPFMIPreEmitPeepholePass(PassRegistry &);
 void initializeBPFMISimplifyPatchablePass(PassRegistry &);

 };
 
 class BPFIRPeepholePass : public PassInfoMixin<BPFIRPeepholePass> {
 public:
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   static bool isRequired() { return true; }
 };
-
-class BPFAdjustOptPass : public PassInfoMixin<BPFAdjustOptPass> {
-public:
-  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
-};
 } // namespace llvm
 
 #endif

llvm/lib/Target/BPF/BPFAdjustOpt.cpp

-//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Adjust optimization to make the code more kernel verifier friendly.
-//
-//===----------------------------------------------------------------------===//
-
-#include "BPF.h"
-#include "BPFCORE.h"
-#include "BPFTargetMachine.h"
-#include "llvm/IR/Instruction.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicsBPF.h"
-#include "llvm/IR/Module.h"
-#include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/Type.h"
-#include "llvm/IR/User.h"
-#include "llvm/IR/Value.h"
-#include "llvm/Pass.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-
-#define DEBUG_TYPE "bpf-adjust-opt"
-
-using namespace llvm;
-using namespace llvm::PatternMatch;
-
-static cl::opt<bool>
-    DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden,
-                            cl::desc("BPF: Disable Serializing ICMP insns."),
-                            cl::init(false));
-
-static cl::opt<bool> DisableBPFavoidSpeculation(
-    "bpf-disable-avoid-speculation", cl::Hidden,
-    cl::desc("BPF: Disable Avoiding Speculative Code Motion."),
-    cl::init(false));
-
-namespace {
-
-class BPFAdjustOpt final : public ModulePass {
-public:
-  static char ID;
-
-  BPFAdjustOpt() : ModulePass(ID) {}
-  bool runOnModule(Module &M) override;
-};
-
-class BPFAdjustOptImpl {
-  struct PassThroughInfo {
-    Instruction *Input;
-    Instruction *UsedInst;
-    uint32_t OpIdx;
-    PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx)
-        : Input(I), UsedInst(U), OpIdx(Idx) {}
-  };
-
-public:
-  BPFAdjustOptImpl(Module *M) : M(M) {}
-
-  bool run();
-
-private:
-  Module *M;
-  SmallVector<PassThroughInfo, 16> PassThroughs;
-
-  bool adjustICmpToBuiltin();
-  void adjustBasicBlock(BasicBlock &BB);
-  bool serializeICMPCrossBB(BasicBlock &BB);
-  void adjustInst(Instruction &I);
-  bool serializeICMPInBB(Instruction &I);
-  bool avoidSpeculation(Instruction &I);
-  bool insertPassThrough();
-};
-
-} // End anonymous namespace
-
-char BPFAdjustOpt::ID = 0;
-INITIALIZE_PASS(BPFAdjustOpt, "bpf-adjust-opt", "BPF Adjust Optimization",
-                false, false)
-
-ModulePass *llvm::createBPFAdjustOpt() { return new BPFAdjustOpt(); }
-
-bool BPFAdjustOpt::runOnModule(Module &M) { return BPFAdjustOptImpl(&M).run(); }
-
-bool BPFAdjustOptImpl::run() {
-  bool Changed = adjustICmpToBuiltin();
-
-  for (Function &F : *M)
-    for (auto &BB : F) {
-      adjustBasicBlock(BB);
-      for (auto &I : BB)
-        adjustInst(I);
-    }
-  return insertPassThrough() || Changed;
-}
-
-// Commit acabad9ff6bf ("[InstCombine] try to canonicalize icmp with
-// trunc op into mask and cmp") added a transformation to
-// convert "(conv)a < power_2_const" to "a & <const>" in certain
-// cases and bpf kernel verifier has to handle the resulted code
-// conservatively and this may reject otherwise legitimate program.
-// Here, we change related icmp code to a builtin which will
-// be restored to original icmp code later to prevent that
-// InstCombine transformatin.
-bool BPFAdjustOptImpl::adjustICmpToBuiltin() {
-  bool Changed = false;
-  ICmpInst *ToBeDeleted = nullptr;
-  for (Function &F : *M)
-    for (auto &BB : F)
-      for (auto &I : BB) {
-        if (ToBeDeleted) {
-          ToBeDeleted->eraseFromParent();
-          ToBeDeleted = nullptr;
-        }
-
-        auto *Icmp = dyn_cast<ICmpInst>(&I);
-        if (!Icmp)
-          continue;
-
-        Value *Op0 = Icmp->getOperand(0);
-        if (!isa<TruncInst>(Op0))
-          continue;
-
-        auto ConstOp1 = dyn_cast<ConstantInt>(Icmp->getOperand(1));
-        if (!ConstOp1)
-          continue;
-
-        auto ConstOp1Val = ConstOp1->getValue().getZExtValue();
-        auto Op = Icmp->getPredicate();
-        if (Op == ICmpInst::ICMP_ULT || Op == ICmpInst::ICMP_UGE) {
-          if ((ConstOp1Val - 1) & ConstOp1Val)
-            continue;
-        } else if (Op == ICmpInst::ICMP_ULE || Op == ICmpInst::ICMP_UGT) {
-          if (ConstOp1Val & (ConstOp1Val + 1))
-            continue;
-        } else {
-          continue;
-        }
-
-        Constant *Opcode =
-            ConstantInt::get(Type::getInt32Ty(BB.getContext()), Op);
-        Function *Fn = Intrinsic::getDeclaration(
-            M, Intrinsic::bpf_compare, {Op0->getType(), ConstOp1->getType()});
-        auto *NewInst = CallInst::Create(Fn, {Opcode, Op0, ConstOp1});
-        NewInst->insertBefore(&I);
-        Icmp->replaceAllUsesWith(NewInst);
-        Changed = true;
-        ToBeDeleted = Icmp;
-      }
-
-  return Changed;
-}
-
-bool BPFAdjustOptImpl::insertPassThrough() {
-  for (auto &Info : PassThroughs) {
-    auto *CI = BPFCoreSharedInfo::insertPassThrough(
-        M, Info.UsedInst->getParent(), Info.Input, Info.UsedInst);
-    Info.UsedInst->setOperand(Info.OpIdx, CI);
-  }
-
-  return !PassThroughs.empty();
-}
-
-// To avoid combining conditionals in the same basic block by
-// instrcombine optimization.
-bool BPFAdjustOptImpl::serializeICMPInBB(Instruction &I) {
-  // For:
-  //   comp1 = icmp <opcode> ...;
-  //   comp2 = icmp <opcode> ...;
-  //   ... or comp1 comp2 ...
-  // changed to:
-  //   comp1 = icmp <opcode> ...;
-  //   comp2 = icmp <opcode> ...;
-  //   new_comp1 = __builtin_bpf_passthrough(seq_num, comp1)
-  //   ... or new_comp1 comp2 ...
-  Value *Op0, *Op1;
-  // Use LogicalOr (accept `or i1` as well as `select i1 Op0, true, Op1`)
-  if (!match(&I, m_LogicalOr(m_Value(Op0), m_Value(Op1))))
-    return false;
-  auto *Icmp1 = dyn_cast<ICmpInst>(Op0);
-  if (!Icmp1)
-    return false;
-  auto *Icmp2 = dyn_cast<ICmpInst>(Op1);
-  if (!Icmp2)
-    return false;
-
-  Value *Icmp1Op0 = Icmp1->getOperand(0);
-  Value *Icmp2Op0 = Icmp2->getOperand(0);
-  if (Icmp1Op0 != Icmp2Op0)
-    return false;
-
-  // Now we got two icmp instructions which feed into
-  // an "or" instruction.
-  PassThroughInfo Info(Icmp1, &I, 0);
-  PassThroughs.push_back(Info);
-  return true;
-}
-
-// To avoid combining conditionals in the same basic block by
-// instrcombine optimization.
-bool BPFAdjustOptImpl::serializeICMPCrossBB(BasicBlock &BB) {
-  // For:
-  //   B1:
-  //     comp1 = icmp <opcode> ...;
-  //     if (comp1) goto B2 else B3;
-  //   B2:
-  //     comp2 = icmp <opcode> ...;
-  //     if (comp2) goto B4 else B5;
-  //   B4:
-  //     ...
-  // changed to:
-  //   B1:
-  //     comp1 = icmp <opcode> ...;
-  //     comp1 = __builtin_bpf_passthrough(seq_num, comp1);
-  //     if (comp1) goto B2 else B3;
-  //   B2:
-  //     comp2 = icmp <opcode> ...;
-  //     if (comp2) goto B4 else B5;
-  //   B4:
-  //     ...
-
-  // Check basic predecessors, if two of them (say B1, B2) are using
-  // icmp instructions to generate conditions and one is the predesessor
-  // of another (e.g., B1 is the predecessor of B2). Add a passthrough
-  // barrier after icmp inst of block B1.
-  BasicBlock *B2 = BB.getSinglePredecessor();
-  if (!B2)
-    return false;
-
-  BasicBlock *B1 = B2->getSinglePredecessor();
-  if (!B1)
-    return false;
-
-  Instruction *TI = B2->getTerminator();
-  auto *BI = dyn_cast<BranchInst>(TI);
-  if (!BI || !BI->isConditional())
-    return false;
-  auto *Cond = dyn_cast<ICmpInst>(BI->getCondition());
-  if (!Cond || B2->getFirstNonPHI() != Cond)
-    return false;
-  Value *B2Op0 = Cond->getOperand(0);
-  auto Cond2Op = Cond->getPredicate();
-
-  TI = B1->getTerminator();
-  BI = dyn_cast<BranchInst>(TI);
-  if (!BI || !BI->isConditional())
-    return false;
-  Cond = dyn_cast<ICmpInst>(BI->getCondition());
-  if (!Cond)
-    return false;
-  Value *B1Op0 = Cond->getOperand(0);
-  auto Cond1Op = Cond->getPredicate();
-
-  if (B1Op0 != B2Op0)
-    return false;
-
-  if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) {
-    if (Cond2Op != ICmpInst::ICMP_SLT && Cond2Op != ICmpInst::ICMP_SLE)
-      return false;
-  } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) {
-    if (Cond2Op != ICmpInst::ICMP_SGT && Cond2Op != ICmpInst::ICMP_SGE)
-      return false;
-  } else if (Cond1Op == ICmpInst::ICMP_ULT || Cond1Op == ICmpInst::ICMP_ULE) {
-    if (Cond2Op != ICmpInst::ICMP_UGT && Cond2Op != ICmpInst::ICMP_UGE)
-      return false;
-  } else if (Cond1Op == ICmpInst::ICMP_UGT || Cond1Op == ICmpInst::ICMP_UGE) {
-    if (Cond2Op != ICmpInst::ICMP_ULT && Cond2Op != ICmpInst::ICMP_ULE)
-      return false;
-  } else {
-    return false;
-  }
-
-  PassThroughInfo Info(Cond, BI, 0);
-  PassThroughs.push_back(Info);
-
-  return true;
-}
-
-// To avoid speculative hoisting certain computations out of
-// a basic block.
-bool BPFAdjustOptImpl::avoidSpeculation(Instruction &I) {
-  if (auto *LdInst = dyn_cast<LoadInst>(&I)) {
-    if (auto *GV = dyn_cast<GlobalVariable>(LdInst->getOperand(0))) {
-      if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) ||
-          GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr))
-        return false;
-    }
-  }
-
-  if (!isa<LoadInst>(&I) && !isa<CallInst>(&I))
-    return false;
-
-  // For:
-  //   B1:
-  //     var = ...
-  //     ...
-  //     /* icmp may not be in the same block as var = ... */
-  //     comp1 = icmp <opcode> var, <const>;
-  //     if (comp1) goto B2 else B3;
-  //   B2:
-  //     ... var ...
-  // change to:
-  //   B1:
-  //     var = ...
-  //     ...
-  //     /* icmp may not be in the same block as var = ... */
-  //     comp1 = icmp <opcode> var, <const>;
-  //     if (comp1) goto B2 else B3;
-  //   B2:
-  //     var = __builtin_bpf_passthrough(seq_num, var);
-  //     ... var ...
-  bool isCandidate = false;
-  SmallVector<PassThroughInfo, 4> Candidates;
-  for (User *U : I.users()) {
-    Instruction *Inst = dyn_cast<Instruction>(U);
-    if (!Inst)
-      continue;
-
-    // May cover a little bit more than the
-    // above pattern.
-    if (auto *Icmp1 = dyn_cast<ICmpInst>(Inst)) {
-      Value *Icmp1Op1 = Icmp1->getOperand(1);
-      if (!isa<Constant>(Icmp1Op1))
-        return false;
-      isCandidate = true;
-      continue;
-    }
-
-    // Ignore the use in the same basic block as the definition.
-    if (Inst->getParent() == I.getParent())
-      continue;
-
-    // use in a different basic block, If there is a call or
-    // load/store insn before this instruction in this basic
-    // block. Most likely it cannot be hoisted out. Skip it.
-    for (auto &I2 : *Inst->getParent()) {
-      if (isa<CallInst>(&I2))
-        return false;
-      if (isa<LoadInst>(&I2) || isa<StoreInst>(&I2))
-        return false;
-      if (&I2 == Inst)
-        break;
-    }
-
-    // It should be used in a GEP or a simple arithmetic like
-    // ZEXT/SEXT which is used for GEP.
-    if (Inst->getOpcode() == Instruction::ZExt ||
-        Inst->getOpcode() == Instruction::SExt) {
-      PassThroughInfo Info(&I, Inst, 0);
-      Candidates.push_back(Info);
-    } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
-      // traverse GEP inst to find Use operand index
-      unsigned i, e;
-      for (i = 1, e = GI->getNumOperands(); i != e; ++i) {
-        Value *V = GI->getOperand(i);
-        if (V == &I)
-          break;
-      }
-      if (i == e)
-        continue;
-
-      PassThroughInfo Info(&I, GI, i);
-      Candidates.push_back(Info);
-    }
-  }
-
-  if (!isCandidate || Candidates.empty())
-    return false;
-
-  llvm::append_range(PassThroughs, Candidates);
-  return true;
-}
-
-void BPFAdjustOptImpl::adjustBasicBlock(BasicBlock &BB) {
-  if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB))
-    return;
-}
-
-void BPFAdjustOptImpl::adjustInst(Instruction &I) {
-  if (!DisableBPFserializeICMP && serializeICMPInBB(I))
-    return;
-  if (!DisableBPFavoidSpeculation && avoidSpeculation(I))
-    return;
-}
-
-PreservedAnalyses BPFAdjustOptPass::run(Module &M, ModuleAnalysisManager &AM) {
-  return BPFAdjustOptImpl(&M).run() ? PreservedAnalyses::none()
-                                    : PreservedAnalyses::all();
-}

llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp

 public:
   static char ID;
   BPFCheckAndAdjustIR() : ModulePass(ID) {}
 
 private:
   void checkIR(Module &M);
   bool adjustIR(Module &M);
   bool removePassThroughBuiltin(Module &M);
-  bool removeCompareBuiltin(Module &M);
 };
 } // End anonymous namespace
 
 char BPFCheckAndAdjustIR::ID = 0;
 INITIALIZE_PASS(BPFCheckAndAdjustIR, DEBUG_TYPE, "BPF Check And Adjust IR",
                 false, false)
 
 ModulePass *llvm::createBPFCheckAndAdjustIR() {
for (auto &BB : F)
         Changed = true;
         Value *Arg = Call->getArgOperand(1);
         Call->replaceAllUsesWith(Arg);
         ToBeDeleted = Call;
       }
   return Changed;
 }
 
-bool BPFCheckAndAdjustIR::removeCompareBuiltin(Module &M) {
-  // Remove __builtin_bpf_compare()'s which are used to prevent
-  // certain IR optimizations. Now major IR optimizations are done,
-  // remove them.
-  bool Changed = false;
-  CallInst *ToBeDeleted = nullptr;
-  for (Function &F : M)
-    for (auto &BB : F)
-      for (auto &I : BB) {
-        if (ToBeDeleted) {
-          ToBeDeleted->eraseFromParent();
-          ToBeDeleted = nullptr;
-        }
-
-        auto *Call = dyn_cast<CallInst>(&I);
-        if (!Call)
-          continue;
-        auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand());
-        if (!GV)
-          continue;
-        if (!GV->getName().startswith("llvm.bpf.compare"))
-          continue;
-
-        Changed = true;
-        Value *Arg0 = Call->getArgOperand(0);
-        Value *Arg1 = Call->getArgOperand(1);
-        Value *Arg2 = Call->getArgOperand(2);
-
-        auto OpVal = cast<ConstantInt>(Arg0)->getValue().getZExtValue();
-        CmpInst::Predicate Opcode = (CmpInst::Predicate)OpVal;
-
-        auto *ICmp = new ICmpInst(Opcode, Arg1, Arg2);
-        ICmp->insertBefore(Call);
-
-        Call->replaceAllUsesWith(ICmp);
-        ToBeDeleted = Call;
-      }
-  return Changed;
-}
-
 bool BPFCheckAndAdjustIR::adjustIR(Module &M) {
-  bool Changed = removePassThroughBuiltin(M);
-  Changed = removeCompareBuiltin(M) || Changed;
-  return Changed;
+  return removePassThroughBuiltin(M);
 }
 
 bool BPFCheckAndAdjustIR::runOnModule(Module &M) {
   checkIR(M);
   return adjustIR(M);
 }

llvm/lib/Target/BPF/BPFTargetMachine.cpp

extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() {
   RegisterTargetMachine<BPFTargetMachine> X(getTheBPFleTarget());
   RegisterTargetMachine<BPFTargetMachine> Y(getTheBPFbeTarget());
   RegisterTargetMachine<BPFTargetMachine> Z(getTheBPFTarget());
 
   PassRegistry &PR = *PassRegistry::getPassRegistry();
   initializeBPFAbstractMemberAccessLegacyPassPass(PR);
   initializeBPFPreserveDITypePass(PR);
   initializeBPFIRPeepholePass(PR);
-  initializeBPFAdjustOptPass(PR);
   initializeBPFCheckAndAdjustIRPass(PR);
   initializeBPFMIPeepholePass(PR);
   initializeBPFMIPeepholeTruncElimPass(PR);
   initializeBPFDAGToDAGISelPass(PR);
 }
 
 // DataLayout: little or big endian
 static std::string computeDataLayout(const Triple &TT) {
PB.registerPipelineStartEPCallback(
         FPM.addPass(BPFPreserveDITypePass());
         FPM.addPass(BPFIRPeepholePass());
         MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
       });
   PB.registerPeepholeEPCallback([=](FunctionPassManager &FPM,
                                     OptimizationLevel Level) {
     FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
   });
-  PB.registerPipelineEarlySimplificationEPCallback(
-      [=](ModulePassManager &MPM, OptimizationLevel) {
-        MPM.addPass(BPFAdjustOptPass());
-      });
 }
 
 void BPFPassConfig::addIRPasses() {
   addPass(createBPFCheckAndAdjustIR());
   TargetPassConfig::addIRPasses();
 }
 
 TargetTransformInfo

llvm/lib/Target/BPF/BPFTargetTransformInfo.h

 
 #ifndef LLVM_LIB_TARGET_BPF_BPFTARGETTRANSFORMINFO_H
 #define LLVM_LIB_TARGET_BPF_BPFTARGETTRANSFORMINFO_H
 
 #include "BPFTargetMachine.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/CodeGen/BasicTTIImpl.h"
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+#include "llvm/Transforms/InstCombine/InstCombiner.h"
+#include "llvm/Transforms/Utils/Local.h"
 
 namespace llvm {
 class BPFTTIImpl : public BasicTTIImplBase<BPFTTIImpl> {
   typedef BasicTTIImplBase<BPFTTIImpl> BaseT;
   typedef TargetTransformInfo TTI;
   friend BaseT;
 
   const BPFSubtarget *ST;
   const BPFTargetLowering *TLI;
 
   const BPFSubtarget *getST() const { return ST; }
   const BPFTargetLowering *getTLI() const { return TLI; }
 
 public:
   explicit BPFTTIImpl(const BPFTargetMachine *TM, const Function &F)
       : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)),
-        TLI(ST->getTargetLowering()) {}
+        TLI(ST->getTargetLowering()) {
+    // Disable certain optimizations which may cause verification failure.
+    AllowFoldTwoEntryPHINode = false;
+    EnableFoldAndOrOfICmpsUsingRanges = false;
+  }
 
   int getIntImmCost(const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind) {
     if (Imm.getBitWidth() <= 64 && isInt<32>(Imm.getSExtValue()))
       return TTI::TCC_Free;
 
     return TTI::TCC_Basic;
   }
 

llvm/lib/Target/BPF/CMakeLists.txt

 tablegen(LLVM BPFGenMCCodeEmitter.inc -gen-emitter)
 tablegen(LLVM BPFGenRegisterInfo.inc -gen-register-info)
 tablegen(LLVM BPFGenSubtargetInfo.inc -gen-subtarget)
 
 add_public_tablegen_target(BPFCommonTableGen)
 
 add_llvm_target(BPFCodeGen
   BPFAbstractMemberAccess.cpp
-  BPFAdjustOpt.cpp
   BPFAsmPrinter.cpp
   BPFCheckAndAdjustIR.cpp
   BPFFrameLowering.cpp
   BPFInstrInfo.cpp
   BPFIRPeephole.cpp
   BPFISelDAGToDAG.cpp
   BPFISelLowering.cpp
   BPFMCInstLower.cpp

llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp

 #include "llvm/Transforms/InstCombine/InstCombiner.h"
 #include "llvm/Transforms/Utils/Local.h"
 
 using namespace llvm;
 using namespace PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
+/// FoldAndOrOfICmpsUsingRanges optimization is enabled by default.
+cl::opt<bool> llvm::EnableFoldAndOrOfICmpsUsingRanges(
+    "enable-instcombine-foldAndOrOfICmpsUsingRanges", cl::Hidden, cl::init(true),
+    cl::desc("Enable FoldAndOrOfICmpsUsingRanges transformation"));
+
 /// This is the complement of getICmpCode, which turns an opcode and two
 /// operands into either a constant true or false, or a brand new ICmp
 /// instruction. The sign is passed in to determine which kind of predicate to
 /// use in the new icmp instruction.
 static Value *getNewICmpValue(unsigned Code, bool Sign, Value *LHS, Value *RHS,
                               InstCombiner::BuilderTy &Builder) {
   ICmpInst::Predicate NewPred;
   if (Constant *TorF = getPredForICmpCode(Code, Sign, LHS->getType(), NewPred))

 
 /// Fold (icmp Pred1 V1, C1) & (icmp Pred2 V2, C2)
 /// or   (icmp Pred1 V1, C1) | (icmp Pred2 V2, C2)
 /// into a single comparison using range-based reasoning.
 /// NOTE: This is also used for logical and/or, must be poison-safe!
 Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1,
                                                      ICmpInst *ICmp2,
                                                      bool IsAnd) {
+  if (!EnableFoldAndOrOfICmpsUsingRanges)
+    return nullptr;
+
   ICmpInst::Predicate Pred1, Pred2;
   Value *V1, *V2;
   const APInt *C1, *C2;
   if (!match(ICmp1, m_ICmp(Pred1, m_Value(V1), m_APInt(C1))) ||
       !match(ICmp2, m_ICmp(Pred2, m_Value(V2), m_APInt(C2))))
     return nullptr;
 
   // Look through add of a constant offset on V1, V2, or both operands. This

llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp

 using namespace llvm;
 using namespace PatternMatch;
 
 #define DEBUG_TYPE "instcombine"
 
 // How many times is a select replaced by one of its operands?
 STATISTIC(NumSel, "Number of select opts");
 
-
 /// Compute Result = In1+In2, returning true if the result overflowed for this
 /// type.
 static bool addWithOverflow(APInt &Result, const APInt &In1,
                             const APInt &In2, bool IsSigned = false) {
   bool Overflow;
   if (IsSigned)
     Result = In1.sadd_ov(In2, Overflow);
   else

llvm/lib/Transforms/Utils/SimplifyCFG.cpp

 #define DEBUG_TYPE "simplifycfg"
 
 cl::opt<bool> llvm::RequireAndPreserveDomTree(
     "simplifycfg-require-and-preserve-domtree", cl::Hidden,
 
     cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
              "into preserving DomTree,"));
 
+cl::opt<bool> llvm::AllowFoldTwoEntryPHINode(
+    "allow-fold-twoentry-phi-node", cl::Hidden, cl::init(true),
+    cl::desc("Allow FoldTwoEntryPHINode transformation"));
+
 // Chosen as 2 so as to be cheap, but still to have enough power to fold
 // a select, so the "clamp" idiom (of a min followed by a max) will be caught.
 // To catch this, we need to fold a compare and a select, hence '2' being the
 // minimum reasonable default.
 static cl::opt<unsigned> PHINodeFoldingThreshold(
     "phi-node-folding-threshold", cl::Hidden, cl::init(2),
     cl::desc(
         "Control the amount of phi node folding to perform (default = 2)"));
static bool FoldCondBranchOnValueKnownInPredecessor(BranchInst *BI,
   } while (Result == std::nullopt);
   return EverChanged;
 }
 
 /// Given a BB that starts with the specified two-entry PHI node,
 /// see if we can eliminate it.
 static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
                                 DomTreeUpdater *DTU, const DataLayout &DL) {
+  if (!AllowFoldTwoEntryPHINode)
+    return false;
+
   // Ok, this is a two entry PHI node.  Check to see if this is a simple "if
   // statement", which has a very simple dominance structure.  Basically, we
   // are trying to find the condition that is being branched on, which
   // subsequently causes this merge to happen.  We really want control
   // dependence information for this check, but simplifycfg can't keep it up
   // to date, and this catches most of the cases we care about anyway.
   BasicBlock *BB = PN->getParent();
 

llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll

 ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
 ; RUN: opt -passes='default<O2>' -mtriple=bpf-pc-linux %s | llvm-dis > %t1
 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
-; RUN: opt -passes='default<O2>' -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
 ;
 ; Source:
 ;   int foo();
 ;   int bar(int);
 ;   int test() {
 ;     int ret = foo();
 ;     if (ret <= 0 || ret > 7)
 ;       return 0;

 ; CHECK:         [[REG2:r[0-9]+]] = 1
 ; CHECK:         if [[REG2]] s> [[REG1]] goto
 ; CHECK:         if [[REG1]] s> 7 goto
 
 ; CHECK-DISABLE: [[REG1:r[0-9]+]] += -8
 ; CHECK-DISABLE: [[REG1]] <<= 32
 ; CHECK-DISABLE: [[REG1]] >>= 32
 ; CHECK-DISABLE: [[REG2:r[0-9]+]] = 4294967289
 ; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto
 
 lor.lhs.false:                                    ; preds = %entry
   %1 = load i32, ptr %ret, align 4, !tbaa !2
   %cmp1 = icmp sgt i32 %1, 7
   br i1 %cmp1, label %if.then, label %if.end
 
 if.then:                                          ; preds = %lor.lhs.false, %entry
   store i32 0, ptr %retval, align 4

llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll

 ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s
 ;
 ; Source:
 ;   int foo();
 ;   int bar(int);
 ;   int test() {
 ;     int ret = foo();
 ;     if (ret <= 0)
 ;       return 0;

llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll

 ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s
+; RUN: llc %t1 -o - | FileCheck %s
 ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s
+; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s
 ;
 ; Source:
 ;   int test1(unsigned long a) {
 ;     if ((unsigned)a <= 3) return 2;
 ;     return 3;
 ;   }
 ;   int test2(unsigned long a) {
 ;     if ((unsigned)a < 4) return 2;
if.end:                                           ; preds = %entry
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
   %1 = load i32, ptr %retval, align 4
   ret i32 %1
 }
 
 ; CHECK-LABEL: test1
-; CHECK-V1:    if r[[#]] > r[[#]] goto
-; CHECK-V3:    if w[[#]] < 4 goto
+; CHECK:       r[[#]] &= r[[#]]
+; CHECK:       if r[[#]] == 0 goto
 
 ; Function Attrs: nounwind
 define dso_local i32 @test2(i64 %a) #0 {
 entry:
   %retval = alloca i32, align 4
   %a.addr = alloca i64, align 8
   store i64 %a, ptr %a.addr, align 8, !tbaa !3
   %0 = load i64, ptr %a.addr, align 8, !tbaa !3
if.end:                                           ; preds = %entry
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
   %1 = load i32, ptr %retval, align 4
   ret i32 %1
 }
 
 ; CHECK-LABEL: test2
-; CHECK-V1:    if r[[#]] > r[[#]] goto
-; CHECK-V3:    if w[[#]] < 4 goto
+; CHECK:    r[[#]] &= r[[#]]
+; CHECK:    if r[[#]] == 0 goto
 
 attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 7, !"frame-pointer", i32 2}
 !2 = !{!"clang version 14.0.0 (https://github.com/llvm/llvm-project.git b7892f95881c891032742e0cd81861b845512653)"}
 !3 = !{!4, !4, i64 0}
 !4 = !{!"long", !5, i64 0}
 !5 = !{!"omnipotent char", !6, i64 0}
 !6 = !{!"Simple C/C++ TBAA"}

llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll

 ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s
+; RUN: llc %t1 -o - | FileCheck %s
 ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1
-; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s
+; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s
 ;
 ; Source:
 ;   int test1(unsigned long a) {
 ;     if ((unsigned)a > 3) return 2;
 ;     return 3;
 ;   }
 ;   int test2(unsigned long a) {
 ;     if ((unsigned)a >= 4) return 2;
if.end:                                           ; preds = %entry
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
   %1 = load i32, ptr %retval, align 4
   ret i32 %1
 }
 
 ; CHECK-LABEL: test1
-; CHECK-V1:    if r[[#]] > 3 goto
-; CHECK-V3:    if w[[#]] > 3 goto
+; CHECK:       r[[#]] &= r[[#]]
+; CHECK:       if r[[#]] == 0 goto
 
 ; Function Attrs: nounwind
 define dso_local i32 @test2(i64 %a) #0 {
 entry:
   %retval = alloca i32, align 4
   %a.addr = alloca i64, align 8
   store i64 %a, ptr %a.addr, align 8, !tbaa !3
   %0 = load i64, ptr %a.addr, align 8, !tbaa !3
if.end:                                           ; preds = %entry
   br label %return
 
 return:                                           ; preds = %if.end, %if.then
   %1 = load i32, ptr %retval, align 4
   ret i32 %1
 }
 
 ; CHECK-LABEL: test2
-; CHECK-V1:    if r[[#]] > 3 goto
-; CHECK-V3:    if w[[#]] > 3 goto
+; CHECK:       r[[#]] &= r[[#]]
+; CHECK:       if r[[#]] == 0 goto
 
 attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" }
 
 !llvm.module.flags = !{!0, !1}
 !llvm.ident = !{!2}
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 7, !"frame-pointer", i32 2}
 !2 = !{!"clang version 14.0.0 (https://github.com/llvm/llvm-project.git 930ccf0191b4a33332d924522e5676fff583f083)"}
 !3 = !{!4, !4, i64 0}
 !4 = !{!"long", !5, i64 0}
 !5 = !{!"omnipotent char", !6, i64 0}
 !6 = !{!"Simple C/C++ TBAA"}

llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll

 ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
 ;
 ; Source:
 ;   unsigned long foo();
 ;   ptr test(ptr p) {
 ;     unsigned long ret = foo();
 ;     if (ret <= 7)
 ;       p += ret;
 ;     return p;

llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll

 ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1
 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s
-; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1
-; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s
 ;
 ; Source:
 ;   unsigned foo();
 ;   ptr test(ptr p) {
 ;     unsigned ret = foo();
 ;     if (ret <= 7)
 ;       p += ret;
 ;     return p;