[AArch64][GlobalISel] Optimize compare and branch cases with G_INTTOPTR and unknown values

[aemerson]

Summary

Since we have distinct types for pointers and scalars, G_INTTOPTRs can sometimes obstruct attempts to find constant source values. These usually come about when try to do some kind of null pointer check. Teaching getConstantVRegValWithLookThrough about this operation allows the CBZ/CBNZ optimization to catch more cases.

This change also improves the case where we can't find a constant source at all. Previously we would emit a cmp, cset and tbnz for that. Now we try to just emit a cmp and conditional branch, saving an instruction.

The cumulative code size improvement of this change plus D64354 is 5.5% geomean on arm64 CTMark -O0.

[paquette]

LGTM

llvm/lib/CodeGen/GlobalISel/Utils.cpp

case TargetOpcode::G_ZEXT:
           MRI.getType(MI->getOperand(0).getReg()).getSizeInBits()));
       VReg = MI->getOperand(1).getReg();
       break;
     case TargetOpcode::COPY:
       VReg = MI->getOperand(1).getReg();
       if (TargetRegisterInfo::isPhysicalRegister(VReg))
         return None;
       break;
+    case TargetOpcode::G_INTTOPTR:
+      VReg = MI->getOperand(1).getReg();
+      break;
     default:
       return None;
     }
   }
   if (!MI || MI->getOpcode() != TargetOpcode::G_CONSTANT ||
       (!MI->getOperand(1).isImm() && !MI->getOperand(1).isCImm()))
     return None;
 

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

bool AArch64InstructionSelector::selectCompareBranch(
   MachineInstr *CCMI = MRI.getVRegDef(CondReg);
   if (CCMI->getOpcode() == TargetOpcode::G_TRUNC)
     CCMI = MRI.getVRegDef(CCMI->getOperand(1).getReg());
   if (CCMI->getOpcode() != TargetOpcode::G_ICMP)
     return false;
 
   Register LHS = CCMI->getOperand(2).getReg();
   Register RHS = CCMI->getOperand(3).getReg();
-  if (!getConstantVRegVal(RHS, MRI))
+  auto VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
+  if (!VRegAndVal)
     std::swap(RHS, LHS);
 
-  const auto RHSImm = getConstantVRegVal(RHS, MRI);
-  if (!RHSImm || *RHSImm != 0)
-    return false;
+  VRegAndVal = getConstantVRegValWithLookThrough(RHS, MRI);
+  if (!VRegAndVal || VRegAndVal->Value != 0) {
+    MachineIRBuilder MIB(I);
+    // If we can't select a CBZ then emit a cmp + Bcc.
+    if (!emitIntegerCompare(CCMI->getOperand(2), CCMI->getOperand(3),
+                            CCMI->getOperand(1), MIB))
+      return false;
+    const AArch64CC::CondCode CC = changeICMPPredToAArch64CC(
+        (CmpInst::Predicate)CCMI->getOperand(1).getPredicate());
+    MIB.buildInstr(AArch64::Bcc, {}, {}).addImm(CC).addMBB(DestMBB);
+    I.eraseFromParent();
+    return true;
+  }
 
   const RegisterBank &RB = *RBI.getRegBank(LHS, MRI, TRI);
   if (RB.getID() != AArch64::GPRRegBankID)
     return false;
 
   const auto Pred = (CmpInst::Predicate)CCMI->getOperand(1).getPredicate();
   if (Pred != CmpInst::ICMP_NE && Pred != CmpInst::ICMP_EQ)
     return false;

llvm/test/CodeGen/AArch64/GlobalISel/select-cbz.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
 
 --- |
   define void @cbz_s32() { ret void }
   define void @cbz_s64() { ret void }
   define void @cbnz_s32() { ret void }
   define void @cbnz_s64() { ret void }
+  define hidden void @test_rhs_inttoptr(i64* %p) { ret void }
+  define hidden void @test_rhs_unknown(i64* %p) { ret void }
 ...
 
 ---
-# CHECK-LABEL: name: cbz_s32
 name:            cbz_s32
 legalized:       true
 regBankSelected: true
 
-# CHECK:  body:
-# CHECK:   bb.0:
-# CHECK:    %0:gpr32 = COPY $w0
-# CHECK:    CBZW %0, %bb.1
-# CHECK:    B %bb.0
 body:             |
+  ; CHECK-LABEL: name: cbz_s32
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+  ; CHECK:   CBZW [[COPY]], %bb.1
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
   bb.0:
     liveins: $w0
     successors: %bb.0, %bb.1
 
     %0:gpr(s32) = COPY $w0
     %1:gpr(s32) = G_CONSTANT i32 0
     %2:gpr(s32) = G_ICMP intpred(eq), %0, %1
     %3:gpr(s1) = G_TRUNC %2(s32)
     G_BRCOND %3(s1), %bb.1
     G_BR %bb.0
 
   bb.1:
 ...
 
 ---
-# CHECK-LABEL: name: cbz_s64
 name:            cbz_s64
 legalized:       true
 regBankSelected: true
 
-# CHECK:  body:
-# CHECK:   bb.0:
-# CHECK:    %0:gpr64 = COPY $x0
-# CHECK:    CBZX %0, %bb.1
-# CHECK:    B %bb.0
 body:             |
+  ; CHECK-LABEL: name: cbz_s64
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK:   CBZX [[COPY]], %bb.1
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
   bb.0:
     liveins: $x0
     successors: %bb.0, %bb.1
 
     %0:gpr(s64) = COPY $x0
     %1:gpr(s64) = G_CONSTANT i64 0
     %2:gpr(s32) = G_ICMP intpred(eq), %0, %1
     %3:gpr(s1) = G_TRUNC %2(s32)
     G_BRCOND %3(s1), %bb.1
     G_BR %bb.0
 
   bb.1:
 ...
 
 ---
-# CHECK-LABEL: name: cbnz_s32
 name:            cbnz_s32
 legalized:       true
 regBankSelected: true
 
-# CHECK:  body:
-# CHECK:   bb.0:
-# CHECK:    %0:gpr32 = COPY $w0
-# CHECK:    CBNZW %0, %bb.1
-# CHECK:    B %bb.0
 body:             |
+  ; CHECK-LABEL: name: cbnz_s32
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+  ; CHECK:   CBNZW [[COPY]], %bb.1
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
   bb.0:
     liveins: $w0
     successors: %bb.0, %bb.1
 
     %0:gpr(s32) = COPY $w0
     %1:gpr(s32) = G_CONSTANT i32 0
     %2:gpr(s32) = G_ICMP intpred(ne), %0, %1
     %3:gpr(s1) = G_TRUNC %2(s32)
     G_BRCOND %3(s1), %bb.1
     G_BR %bb.0
 
   bb.1:
 ...
 
 ---
-# CHECK-LABEL: name: cbnz_s64
 name:            cbnz_s64
 legalized:       true
 regBankSelected: true
 
-# CHECK:  body:
-# CHECK:   bb.0:
-# CHECK:    %0:gpr64 = COPY $x0
-# CHECK:    CBNZX %0, %bb.1
-# CHECK:    B %bb.0
 body:             |
+  ; CHECK-LABEL: name: cbnz_s64
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.0(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+  ; CHECK:   CBNZX [[COPY]], %bb.1
+  ; CHECK:   B %bb.0
+  ; CHECK: bb.1:
   bb.0:
     liveins: $x0
     successors: %bb.0, %bb.1
 
     %0:gpr(s64) = COPY $x0
     %1:gpr(s64) = G_CONSTANT i64 0
     %2:gpr(s32) = G_ICMP intpred(ne), %0, %1
     %3:gpr(s1) = G_TRUNC %2(s32)
     G_BRCOND %3(s1), %bb.1
     G_BR %bb.0
 
   bb.1:
 ...
+---
+name:            test_rhs_inttoptr
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: test_rhs_inttoptr
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
+  ; CHECK:   CBZX [[COPY]], %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   STRXui $xzr, [[COPY]], 0 :: (store 8 into %ir.p)
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.1:
+    successors: %bb.2, %bb.3
+    liveins: $x0
+
+    %0:gpr(p0) = COPY $x0
+    %2:gpr(s64) = G_CONSTANT i64 0
+    %1:gpr(p0) = G_INTTOPTR %2(s64)
+    %4:gpr(s32) = G_ICMP intpred(eq), %0(p0), %1
+    %3:gpr(s1) = G_TRUNC %4(s32)
+    G_BRCOND %3(s1), %bb.3
+
+  bb.2:
+    %5:gpr(s64) = G_CONSTANT i64 0
+    G_STORE %5(s64), %0(p0) :: (store 8 into %ir.p)
+
+  bb.3:
+    RET_ReallyLR
+
+...
+---
+name:            test_rhs_unknown
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: test_rhs_unknown
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $x0
+  ; CHECK:   [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+  ; CHECK:   [[LDRXui:%[0-9]+]]:gpr64common = LDRXui [[COPY]], 0 :: (load 8 from %ir.p)
+  ; CHECK:   $xzr = SUBSXri [[LDRXui]], 42, 0, implicit-def $nzcv
+  ; CHECK:   Bcc 0, %bb.2, implicit $nzcv
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   STRXui $xzr, [[COPY]], 0 :: (store 8 into %ir.p)
+  ; CHECK: bb.2:
+  ; CHECK:   RET_ReallyLR
+  bb.1:
+    successors: %bb.2, %bb.3
+    liveins: $x0
+
+    %0:gpr(p0) = COPY $x0
+    %2:gpr(s64) = G_CONSTANT i64 42
+    %4:gpr(s64) = G_CONSTANT i64 0
+    %1:gpr(s64) = G_LOAD %0(p0) :: (load 8 from %ir.p)
+    %5:gpr(s32) = G_ICMP intpred(eq), %1(s64), %2
+    %3:gpr(s1) = G_TRUNC %5(s32)
+    G_BRCOND %3(s1), %bb.3
+
+  bb.2:
+    %6:gpr(s64) = G_CONSTANT i64 0
+    G_STORE %6(s64), %0(p0) :: (store 8 into %ir.p)
+
+  bb.3:
+    RET_ReallyLR
+
+...

llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt.mir

body:             |
   ; CHECK:   successors: %bb.4(0x40000000), %bb.1(0x40000000)
   ; CHECK:   liveins: $w0
   ; CHECK:   [[COPY:%[0-9]+]]:gpr32common = COPY $w0
   ; CHECK:   [[MOVi32imm:%[0-9]+]]:gpr32 = MOVi32imm 0
   ; CHECK:   [[SUBSWri:%[0-9]+]]:gpr32 = SUBSWri [[COPY]], 4, 0, implicit-def $nzcv
   ; CHECK:   [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[SUBSWri]], %subreg.sub_32
   ; CHECK:   [[UBFMXri:%[0-9]+]]:gpr64common = UBFMXri [[SUBREG_TO_REG]], 0, 31
   ; CHECK:   $xzr = SUBSXri [[UBFMXri]], 71, 0, implicit-def $nzcv
-  ; CHECK:   [[CSINCWr:%[0-9]+]]:gpr32 = CSINCWr $wzr, $wzr, 9, implicit $nzcv
-  ; CHECK:   TBNZW [[CSINCWr]], 0, %bb.4
+  ; CHECK:   Bcc 8, %bb.4, implicit $nzcv
   ; CHECK: bb.1.entry:
   ; CHECK:   successors: %bb.3(0x2aaaaaab), %bb.4(0x2aaaaaab), %bb.2(0x2aaaaaab)
   ; CHECK:   [[MOVi32imm1:%[0-9]+]]:gpr32 = MOVi32imm 0
   ; CHECK:   [[MOVaddrJT:%[0-9]+]]:gpr64 = MOVaddrJT target-flags(aarch64-page) %jump-table.0, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0
   ; CHECK:   early-clobber %18:gpr64, early-clobber %19:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[UBFMXri]], %jump-table.0
   ; CHECK:   BR %18
   ; CHECK: bb.2.sw.bb:
   ; CHECK:   successors: %bb.4(0x80000000)

llvm/test/CodeGen/AArch64/GlobalISel/swifterror.ll

entry:
   store i8 1, i8* %tmp
   ret float 1.0
 }
 
 ; "caller" calls "foo" that takes a swifterror parameter.
 define float @caller(i8* %error_ref) {
 ; CHECK-LABEL: caller:
 ; CHECK: mov [[ID:x[0-9]+]], x0
-; CHECK: mov [[ZERO:x[0-9]+]], #0
-; CHECK: mov x21, #0
 ; CHECK: bl {{.*}}foo
 ; CHECK: mov x0, x21
-; CHECK: cmp x21, [[ZERO]]
-; CHECK: b.ne
+; CHECK: cbnz x21
 ; Access part of the error object and save it to error_ref
 ; CHECK: ldrb [[CODE:w[0-9]+]], [x0, #8]
 ; CHECK: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK: bl {{.*}}free
 
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref
handler:
   call void @free(i8* %tmp)
   ret float 1.0
 }
 
 ; "caller2" is the caller of "foo", it calls "foo" inside a loop.
 define float @caller2(i8* %error_ref) {
 ; CHECK-LABEL: caller2:
 ; CHECK: mov [[ID:x[0-9]+]], x0
-; CHECK: mov [[ZERO:x[0-9]+]], #0
 ; CHECK: fmov [[CMP:s[0-9]+]], #1.0
 ; CHECK: mov x21, #0
 ; CHECK: bl {{.*}}foo
-; CHECK: cmp x21, [[ZERO]]
-; CHECK: b.ne
+; CHECK: cbnz x21
 ; CHECK: fcmp s0, [[CMP]]
 ; CHECK: b.le
 ; Access part of the error object and save it to error_ref
 ; CHECK: ldrb [[CODE:w[0-9]+]], [x21, #8]
 ; CHECK: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK: mov x0, x21
 ; CHECK: bl {{.*}}free
 
entry:
   ret void
 }
 
 ; "caller3" calls "foo_sret" that takes a swifterror parameter.
 define float @caller3(i8* %error_ref) {
 ; CHECK-LABEL: caller3:
 ; CHECK: mov [[ID:x[0-9]+]], x0
 ; CHECK: mov [[ZERO:x[0-9]+]], #0
-; CHECK: mov x21, #0
 ; CHECK: bl {{.*}}foo_sret
 ; CHECK: mov x0, x21
-; CHECK: cmp x21, [[ZERO]]
-; CHECK: b.ne
+; CHECK: cbnz x21
 ; Access part of the error object and save it to error_ref
 ; CHECK: ldrb [[CODE:w[0-9]+]], [x0, #8]
 ; CHECK: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK: bl {{.*}}free
 
 entry:
   %s = alloca %struct.S, align 8
   %error_ptr_ref = alloca swifterror %swift_error*
entry:
   ret float 1.0
 }
 
 ; "caller4" calls "foo_vararg" that takes a swifterror parameter.
 define float @caller4(i8* %error_ref) {
 ; CHECK-LABEL: caller4:
 
 ; CHECK: mov [[ID:x[0-9]+]], x0
-; CHECK: mov [[ZERO:x[0-9]+]], #0
 ; CHECK: stp {{x[0-9]+}}, {{x[0-9]+}}, [sp]
 ; CHECK: mov x21, #0
 ; CHECK: str {{x[0-9]+}}, [sp, #16]
 
 ; CHECK: bl {{.*}}foo_vararg
 ; CHECK: mov x0, x21
-; CHECK: cmp x21, [[ZERO]]
-; CHECK: b.ne
+; CHECK: cbnz x21
 ; Access part of the error object and save it to error_ref
 ; CHECK: ldrb [[CODE:w[0-9]+]], [x0, #8]
 ; CHECK: strb [[CODE]], [{{.*}}[[ID]]]
 ; CHECK: bl {{.*}}free
 entry:
   %error_ptr_ref = alloca swifterror %swift_error*
   store %swift_error* null, %swift_error** %error_ptr_ref