[AArch64] Rewrite (add, csel) to cinc

[SjoerdMeijer]

Summary

Don't rewrite an add instruction with 2 SET_CC operands into a csel instruction. The total instruction sequence uses an extra instruction and register. Preventing this, allows us to match a (add, csel) pattern and rewrite this into a cinc, see the new test case in arm64-csel.ll.

[dmgreen]

Should we be generating:

cmp
cset
cmp
csinc

That lets the csinc do the add, if it's only going to be adding 0/1. Maybe that's possible with an extra isel pattern from add(cset(..))?

[SjoerdMeijer]

Thanks for the suggestion Dave. Just did the pen and paper exercise and agree that:

cmp   w8, #3
cset  w8, hi
cmp   w9, #33
csinc w0, w8, w8, <=

is even better. :-) Also agreed that a match pattern would be nice, so looking into that now.

[SjoerdMeijer]

This now generates cinc, which is even better.

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

13193

I had to keep this, otherwise this lowering will happen first and the pattern, see below, doesn't match,.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

2167

I had to add an zext for this pattern, that's because the way setcc is legalised from an i1 to i32 first, then zext'ed to i64.

[dmgreen]

Thanks. LGTM

The lowering/optimization of CSEL/CSINC/etc feels a bit weak to me, so any improvements are good to see. Tablegen improvements especially as (as far as I understand) they can help in both SelectionDag and GlobalISel.

[SjoerdMeijer]

Yeah, thanks Dave, I think I will be looking a bit more in this area, but this is a start...

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

 //
 // The latter will get matched to a CSINC instruction.
 static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
   assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!");
   SDValue LHS = Op->getOperand(0);
   SDValue RHS = Op->getOperand(1);
   SetCCInfoAndKind InfoAndKind;
 
+  // If both operands are a SET_CC, then we don't want to perform this
+  // folding and create another csel as this results in more instructions
+  // (and higher register usage).
+  if (isSetCCOrZExtSetCC(LHS, InfoAndKind) &&
+      isSetCCOrZExtSetCC(RHS, InfoAndKind))
+    return SDValue();
+
   // If neither operand is a SET_CC, give up.
   if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) {
     std::swap(LHS, RHS);
     if (!isSetCCOrZExtSetCC(LHS, InfoAndKind))
       return SDValue();
   }
 
   // FIXME: This could be generatized to work for FP comparisons.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

def : Pat<(AArch64csel GPR32:$tval, (i32 -1), (i32 imm:$cc), NZCV),
           (CSINVWr GPR32:$tval, WZR, (i32 imm:$cc))>;
 def : Pat<(AArch64csel GPR64:$tval, (i64 -1), (i32 imm:$cc), NZCV),
           (CSINVXr GPR64:$tval, XZR, (i32 imm:$cc))>;
 def : Pat<(AArch64csel (i32 -1), GPR32:$fval, (i32 imm:$cc), NZCV),
           (CSINVWr GPR32:$fval, WZR, (i32 (inv_cond_XFORM imm:$cc)))>;
 def : Pat<(AArch64csel (i64 -1), GPR64:$fval, (i32 imm:$cc), NZCV),
           (CSINVXr GPR64:$fval, XZR, (i32 (inv_cond_XFORM imm:$cc)))>;
 
+def : Pat<(add GPR32:$val, (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV)),
+          (CSINCWr GPR32:$val, GPR32:$val, (i32 imm:$cc))>;
+def : Pat<(add GPR64:$val, (zext (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV))),
+          (CSINCXr GPR64:$val, GPR64:$val, (i32 imm:$cc))>;
+
 // The inverse of the condition code from the alias instruction is what is used
 // in the aliased instruction. The parser all ready inverts the condition code
 // for these aliases.
 def : InstAlias<"cset $dst, $cc",
                 (CSINCWr GPR32:$dst, WZR, WZR, inv_ccode:$cc)>;
 def : InstAlias<"cset $dst, $cc",
                 (CSINCXr GPR64:$dst, XZR, XZR, inv_ccode:$cc)>;
 

llvm/test/CodeGen/AArch64/arm64-csel.ll

 ; CHECK-LABEL: foo23:
 ; CHECK: cmp x0, #5
 ; CHECK: mov w[[REG:[0-9]+]], #6
 ; CHECK: csinc x0, x[[REG]], xzr, ne
   %cmp = icmp eq i64 %x, 5
   %res = select i1 %cmp, i64 1, i64 6
   ret i64 %res
 }
+
+define i16 @foo24(i8* nocapture readonly %A, i8* nocapture readonly %B) {
+; CHECK-LABEL: foo24:
+; CHECK:       ldrb    w[[W8:[0-9]+]], [x1]
+; CHECK-NEXT:  ldrb    w[[W9:[0-9]+]], [x0]
+; CHECK-NEXT:  cmp     w[[W8]], #33
+; CHECK-NEXT:  cset    w[[W8]], hi
+; CHECK-NEXT:  cmp     w[[W9]], #3
+; CHECK-NEXT:  cinc    w0, w[[W8]], hi
+; CHECK-NEXT:  ret
+entry:
+  %0 = load i8, i8* %A, align 1
+  %cmp = icmp ugt i8 %0, 3
+  %conv1 = zext i1 %cmp to i16
+  %1 = load i8, i8* %B, align 1
+  %cmp4 = icmp ugt i8 %1, 33
+  %conv5 = zext i1 %cmp4 to i16
+  %add = add nuw nsw i16 %conv5, %conv1
+  ret i16 %add
+}
+
+define i64 @foo25(i64* nocapture readonly %A, i64* nocapture readonly %B) {
+; CHECK-LABEL: foo25:
+; CHECK:       ldr    x[[X8:[0-9]+]], [x1]
+; CHECK-NEXT:  ldr    x[[X9:[0-9]+]], [x0]
+; CHECK-NEXT:  cmp    x[[X8]], #33
+; CHECK-NEXT:  cset   w[[W8]], hi
+; CHECK-NEXT:  cmp    x[[X9]], #3
+; CHECK-NEXT:  cinc   x0, x[[X8]], hi
+; CHECK-NEXT:  ret
+entry:
+  %0 = load i64, i64* %A, align 1
+  %cmp = icmp ugt i64 %0, 3
+  %conv1 = zext i1 %cmp to i64
+  %1 = load i64, i64* %B, align 1
+  %cmp4 = icmp ugt i64 %1, 33
+  %conv5 = zext i1 %cmp4 to i64
+  %add = add nuw nsw i64 %conv5, %conv1
+  ret i64 %add
+}
+

llvm/test/CodeGen/AArch64/half.ll

 ; CHECK-NEXT:    mov w8, #24576
 ; CHECK-NEXT:    fmov s0, w1
 ; CHECK-NEXT:    movk w8, #15974, lsl #16
 ; CHECK-NEXT:    mov w9, #16384
 ; CHECK-NEXT:    fcvt s0, h0
 ; CHECK-NEXT:    movk w9, #15428, lsl #16
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fmov s1, w9
-; CHECK-NEXT:    cset w8, pl
-; CHECK-NEXT:    fccmp s0, s1, #8, pl
-; CHECK-NEXT:    mov w9, #4
-; CHECK-NEXT:    csinc w9, w9, wzr, mi
-; CHECK-NEXT:    add w0, w8, w9
+; CHECK-NEXT:    fmov s2, w9
+; CHECK-NEXT:    mov w10, #4
+; CHECK-NEXT:    fccmp s0, s2, #8, pl
+; CHECK-NEXT:    csinc w8, w10, wzr, mi
+; CHECK-NEXT:    fcmp s0, s1
+; CHECK-NEXT:    cinc w0, w8, pl
 ; CHECK-NEXT:    ret
   %f16 = bitcast i16 %in to half
   %cmp0 = fcmp ogt half 0xH3333, %f16
   %cmp1 = fcmp ogt half 0xH2222, %f16
   %x = select i1 %cmp0, i16 0, i16 1
   %or = or i1 %cmp1, %cmp0
   %y = select i1 %or, i16 4, i16 1
   %r = add i16 %x, %y
   ret i16 %r
 }