Specify branch probabilities for callbr dests

[void]

Summary

callbr's indirect branches aren't expected to be taken, so reduce their
probabilities to 0 while increasing the default destination to 1. This
allows some code improvements through block placement.

[nickdesaulniers]

Cool, IIRC when upstream kernel devs took a look (via some objtool bug reports) they mentioned that our block placement was backwards. The callbr-asm-branch-folding.ll change makes the most sense to me; the indirect branch is moved out of the way. (Though I would have thought branch predictors assume forward branches are taken). Did this result in any measurable performance difference?

llvm/test/CodeGen/X86/callbr-asm.ll

81

The top of this file mentions ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py. Did update_llc_test_checks.py really remove this .p2align?

[void]

Rebase patch.

[void]

llvm/test/CodeGen/X86/callbr-asm.ll

81

I don't know how this was removed. I've never had to deal with update_llc_test_checks.py.

[void]

Run update_llc_test_checks.py on the testcase.

[void]

PTAL.

llvm/test/CodeGen/X86/callbr-asm.ll

81

Okay. I figured out how to run it. Done.

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
   visitInlineAsm(&I);
   CopyToExportRegsIfNeeded(&I);
 
   // Retrieve successors.
   MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
   Return->setInlineAsmBrDefaultTarget();
 
   // Update successor info.
-  addSuccessorWithProb(CallBrMBB, Return);
+  addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
   for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
     MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
-    addSuccessorWithProb(CallBrMBB, Target);
+    addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
     CallBrMBB->addInlineAsmBrIndirectTarget(Target);
   }
   CallBrMBB->normalizeSuccProbs();
 
   // Drop into default successor.
   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
                           MVT::Other, getControlRoot(),
                           DAG.getBasicBlock(Return)));

llvm/test/CodeGen/AArch64/callbr-asm-label.ll

 ; RUN: llc < %s -mtriple=aarch64-linux-gnu | FileCheck %s
 
 @X = common local_unnamed_addr global i32 0, align 4
 
 define i32 @test1() {
 ; CHECK-LABEL: test1:
 ; CHECK:         .word b
 ; CHECK-NEXT:    .word .Ltmp0
+; CHECK-LABEL: .LBB0_1: // %cleanup
 ; CHECK-LABEL: .Ltmp0:
-; CHECK-LABEL: .LBB0_1: // %l_yes
-; CHECK-LABEL: .LBB0_2: // %cleanup
+; CHECK-LABEL: .LBB0_2: // %indirect
 entry:
-  callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "X"(i8* blockaddress(@test1, %l_yes))
-          to label %cleanup [label %l_yes]
+  callbr void asm sideeffect "1:\0A\09.word b, ${0:l}\0A\09", "X"(i8* blockaddress(@test1, %indirect))
+          to label %cleanup [label %indirect]
 
-l_yes:
+indirect:
   br label %cleanup
 
 cleanup:
-  %retval.0 = phi i32 [ 1, %l_yes ], [ 0, %entry ]
+  %retval.0 = phi i32 [ 1, %indirect ], [ 0, %entry ]
   ret i32 %retval.0
 }
 
 define void @test2() {
 ; CHECK-LABEL: test2:
 entry:
   %0 = load i32, i32* @X, align 4
   %and = and i32 %0, 1

llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll

7:                                                ; preds = %3, %4, %5
   ret i32 %8
 }
 
 declare dso_local i32 @g(...) local_unnamed_addr
 
 declare dso_local i32 @i(...) local_unnamed_addr
 
 ; CHECK-LABEL: <test2>:
-; CHECK:         bl #0 <test2+0x10>
+; CHECK:         bl #0 <test2+0x18>
 ; CHECK-LABEL: <$d.5>:
 ; CHECK-LABEL: <$x.6>:
-; CHECK-NEXT:    b #16 <$x.8+0x4>
+; CHECK-NEXT:    b #-20 <test2+0x18>
 define hidden i32 @test2() local_unnamed_addr {
   %1 = load i32, i32* @l, align 4
   %2 = icmp eq i32 %1, 0
   br i1 %2, label %10, label %3
 
 3:                                                ; preds = %0
   %4 = tail call i32 bitcast (i32 (...)* @g to i32 ()*)()
   %5 = icmp eq i32 %4, 0

llvm/test/CodeGen/X86/callbr-asm-branch-folding.ll

 ; CHECK-NEXT:    movl %edx, %ebx
 ; CHECK-NEXT:    movl %esi, %r12d
 ; CHECK-NEXT:    movq %rdi, %r15
 ; CHECK-NEXT:    callq c
 ; CHECK-NEXT:    movl %eax, %r13d
 ; CHECK-NEXT:    movq %r15, %rdi
 ; CHECK-NEXT:    callq l
 ; CHECK-NEXT:    testl %eax, %eax
-; CHECK-NEXT:    je .LBB0_1
-; CHECK-NEXT:  .LBB0_10: # %cleanup
-; CHECK-NEXT:    addq $8, %rsp
-; CHECK-NEXT:    popq %rbx
-; CHECK-NEXT:    popq %r12
-; CHECK-NEXT:    popq %r13
-; CHECK-NEXT:    popq %r14
-; CHECK-NEXT:    popq %r15
-; CHECK-NEXT:    popq %rbp
-; CHECK-NEXT:    retq
-; CHECK-NEXT:  .LBB0_1: # %if.end
+; CHECK-NEXT:    jne .LBB0_10
+; CHECK-NEXT:  # %bb.1: # %if.end
 ; CHECK-NEXT:    movl %ebx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
 ; CHECK-NEXT:    cmpl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    # implicit-def: $ebx
 ; CHECK-NEXT:    # implicit-def: $r14d
 ; CHECK-NEXT:    je .LBB0_4
 ; CHECK-NEXT:  # %bb.2: # %if.then4
 ; CHECK-NEXT:    movslq %r12d, %rdi
 ; CHECK-NEXT:    callq m

 ; CHECK-NEXT:    popq %r12
 ; CHECK-NEXT:    popq %r13
 ; CHECK-NEXT:    popq %r14
 ; CHECK-NEXT:    popq %r15
 ; CHECK-NEXT:    popq %rbp
 ; CHECK-NEXT:    jmp k # TAILCALL
 ; CHECK-NEXT:  .LBB0_9: # %if.else
 ; CHECK-NEXT:    incq 0
-; CHECK-NEXT:    jmp .LBB0_10
+; CHECK-NEXT:  .LBB0_10: # %cleanup
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
 entry:
   %call = tail call i32 @c()
   %call1 = tail call i32 @l(i32* %o)
   %tobool = icmp eq i32 %call1, 0
   br i1 %tobool, label %if.end, label %cleanup
 
 if.end:                                           ; preds = %entry
   %0 = load i32, i32* @e

llvm/test/CodeGen/X86/callbr-asm.ll

 
 fail:
   ret i32 1
 }
 
 ; Test 3 - asm-goto implements a loop. The loop gets recognized, but many loop
 ; transforms fail due to canonicalization having callbr exceptions. Trivial
 ; blocks at labels 1 and 3 also don't get simplified due to callbr.
-define dso_local i32 @test3(i32 %a) {
+define i32 @test3(i32 %a) {
 ; CHECK-LABEL: test3:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:  .Ltmp1: # Block address taken
 ; CHECK-NEXT:  .LBB2_1: # %label01
 ; CHECK-NEXT:    # =>This Loop Header: Depth=1
 ; CHECK-NEXT:    # Child Loop BB2_2 Depth 2
 ; CHECK-NEXT:    # Child Loop BB2_3 Depth 3
 ; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
 ; CHECK-NEXT:  .Ltmp2: # Block address taken
 ; CHECK-NEXT:  .LBB2_2: # %label02
 ; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
 ; CHECK-NEXT:    # => This Loop Header: Depth=2
 ; CHECK-NEXT:    # Child Loop BB2_3 Depth 3
 ; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
 ; CHECK-NEXT:    addl $4, {{[0-9]+}}(%esp)
 ; CHECK-NEXT:  .Ltmp3: # Block address taken
 ; CHECK-NEXT:  .LBB2_3: # %label03
 ; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB2_2 Depth=2
 ; CHECK-NEXT:    # => This Loop Header: Depth=3
 ; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
-; CHECK-NEXT:    .p2align 4, 0x90
 ; CHECK-NEXT:  .Ltmp4: # Block address taken
 ; CHECK-NEXT:  .LBB2_4: # %label04
 ; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
 ; CHECK-NEXT:    # Parent Loop BB2_2 Depth=2
 ; CHECK-NEXT:    # Parent Loop BB2_3 Depth=3
 ; CHECK-NEXT:    # => This Inner Loop Header: Depth=4
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    jmp .Ltmp1

 normal1:                                          ; preds = %normal0
   %1 = load i32, i32* %a.addr, align 4
   ret i32 %1
 }
 
 ; Test 4 - asm-goto referenced with the 'l' (ell) modifier and not.
 define void @test4() {
 ; CHECK-LABEL: test4:
-; CHECK:       # %bb.0:
+; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    #APP
-; CHECK-NOT:     ja .Ltmp50
-; CHECK-NEXT:    ja .Ltmp5
+; CHECK-NEXT:    ja .Ltmp5{{$}}
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:  .LBB3_1:
+; CHECK-NEXT:  .LBB3_1: # %asm.fallthrough
 ; CHECK-NEXT:    #APP
-; CHECK-NOT:     ja .Ltmp50
-; CHECK-NEXT:    ja .Ltmp5
+; CHECK-NEXT:    ja .Ltmp5{{$}}
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    jmp .LBB3_3
+; CHECK-NEXT:  .Ltmp5: # Block address taken
+; CHECK-NEXT:  .LBB3_3: # %quux
+; CHECK-NEXT:    retl
 entry:
   callbr void asm sideeffect "ja $0", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test4, %quux))
           to label %asm.fallthrough [label %quux]
 
 asm.fallthrough:                                  ; preds = %entry
   callbr void asm sideeffect "ja ${0:l}", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test4, %quux))
           to label %cleanup [label %quux]
 
 quux:                                             ; preds = %asm.fallthrough, %entry
   br label %cleanup
 
 cleanup:                                          ; preds = %asm.fallthrough, %quux
   ret void
 }