[NVPTX] Add add.cc/addc.cc/sub.cc/subc.cc for i64

[slydiman]

Summary

PTX supports those instructions for i64 starting from 4.3.
The patch also marks corresponding DAG nodes legal for both i32 and i64.

[tra]

LGTM.
Nice. We could use a bit more tests, though.

llvm/test/CodeGen/NVPTX/add-sub-128bit.ll

4

We should add ptxas test for ptx43+ here, too.

8

CHECK -> COMMON
DEFAULT -> NOCARRY
PTX34 -> CARRY

9

Can you think of a way to test 32-bit sub/add-with-carry that the patch makes legal?
We can't use the same approach as we do here with i128 as we'd just end up with a regular 64-bit add/sub.

[tra]

llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp

490

BTW, ADDC/SUBC are apparently deprecated in favor of ADDCARRY/SUBCARRY
https://github.com/llvm/llvm-project/blob/a83f4b9cda57c5b3d414ec3bcf9ac891b2ec27e1/llvm/include/llvm/CodeGen/ISDOpcodes.h#L269

Perhaps those should be added as well.

[tra]

Any thoughts on the comments I've posted?

[slydiman]

llvm/test/CodeGen/NVPTX/add-sub-128bit.ll

4

I will collaborate with @asavonic to do this.

9

Sure. I need some time for it.

llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp

for (const auto& Ty : {MVT::i16, MVT::i32, MVT::i64}) {
     setOperationAction(ISD::SMAX, Ty, Legal);
     setOperationAction(ISD::UMIN, Ty, Legal);
     setOperationAction(ISD::UMAX, Ty, Legal);
 
     setOperationAction(ISD::CTPOP, Ty, Legal);
     setOperationAction(ISD::CTLZ, Ty, Legal);
   }
 
+  setOperationAction(ISD::ADDC, MVT::i32, Legal);
+  setOperationAction(ISD::ADDE, MVT::i32, Legal);
+  setOperationAction(ISD::SUBC, MVT::i32, Legal);
+  setOperationAction(ISD::SUBE, MVT::i32, Legal);
+  if (STI.getPTXVersion() >= 43) {
+    setOperationAction(ISD::ADDC, MVT::i64, Legal);
+    setOperationAction(ISD::ADDE, MVT::i64, Legal);
+    setOperationAction(ISD::SUBC, MVT::i64, Legal);
+    setOperationAction(ISD::SUBE, MVT::i64, Legal);
+  }
+
   setOperationAction(ISD::CTTZ, MVT::i16, Expand);
   setOperationAction(ISD::CTTZ, MVT::i32, Expand);
   setOperationAction(ISD::CTTZ, MVT::i64, Expand);
 
   // PTX does not directly support SELP of i1, so promote to i32 first
   setOperationAction(ISD::SELECT, MVT::i1, Custom);
 
   // PTX cannot multiply two i64s in a single instruction.

llvm/lib/Target/NVPTX/NVPTXInstrInfo.td

 
 def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;
 def noHWROT32 : Predicate<"!Subtarget->hasHWROT32()">;
 
 def True : Predicate<"true">;
 
 def hasPTX31 : Predicate<"Subtarget->getPTXVersion() >= 31">;
 def hasPTX42 : Predicate<"Subtarget->getPTXVersion() >= 42">;
+def hasPTX43 : Predicate<"Subtarget->getPTXVersion() >= 43">;
 def hasPTX60 : Predicate<"Subtarget->getPTXVersion() >= 60">;
 def hasPTX61 : Predicate<"Subtarget->getPTXVersion() >= 61">;
 def hasPTX63 : Predicate<"Subtarget->getPTXVersion() >= 63">;
 def hasPTX64 : Predicate<"Subtarget->getPTXVersion() >= 64">;
 def hasPTX65 : Predicate<"Subtarget->getPTXVersion() >= 65">;
 def hasPTX70 : Predicate<"Subtarget->getPTXVersion() >= 70">;
 def hasPTX71 : Predicate<"Subtarget->getPTXVersion() >= 71">;
 def hasPTX72 : Predicate<"Subtarget->getPTXVersion() >= 72">;
NVPTXInst<(outs Int16Regs:$dst), (ins Int16Regs:$a, Int16Regs:$b),
               !strconcat(OpcStr, "16 \t$dst, $a, $b;"),
               [(set Int16Regs:$dst, (OpNode Int16Regs:$a, Int16Regs:$b))]>;
   def i16ri :
     NVPTXInst<(outs Int16Regs:$dst), (ins Int16Regs:$a, i16imm:$b),
               !strconcat(OpcStr, "16 \t$dst, $a, $b;"),
               [(set Int16Regs:$dst, (OpNode Int16Regs:$a, (imm):$b))]>;
 }
 
-// Template for instructions which take 3 int32 args.  The instructions are
+// Template for instructions which take 3 int args.  The instructions are
 // named "<OpcStr>.s32" (e.g. "addc.cc.s32").
-multiclass ADD_SUB_INT_32<string OpcStr, SDNode OpNode> {
+multiclass ADD_SUB_INT_CARRY<string OpcStr, SDNode OpNode> {
+  let hasSideEffects = 1 in {
     def i32rr :
       NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$a, Int32Regs:$b),
                 !strconcat(OpcStr, ".s32 \t$dst, $a, $b;"),
                 [(set Int32Regs:$dst, (OpNode Int32Regs:$a, Int32Regs:$b))]>;
     def i32ri :
       NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$a, i32imm:$b),
                 !strconcat(OpcStr, ".s32 \t$dst, $a, $b;"),
                 [(set Int32Regs:$dst, (OpNode Int32Regs:$a, imm:$b))]>;
+    def i64rr :
+      NVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$a, Int64Regs:$b),
+                !strconcat(OpcStr, ".s64 \t$dst, $a, $b;"),
+                [(set Int64Regs:$dst, (OpNode Int64Regs:$a, Int64Regs:$b))]>,
+      Requires<[hasPTX43]>;
+    def i64ri :
+      NVPTXInst<(outs Int64Regs:$dst), (ins Int64Regs:$a, i64imm:$b),
+                !strconcat(OpcStr, ".s64 \t$dst, $a, $b;"),
+                [(set Int64Regs:$dst, (OpNode Int64Regs:$a, imm:$b))]>,
+      Requires<[hasPTX43]>;
+  }
 }
 
 // Template for instructions which take three fp64 or fp32 args.  The
 // instructions are named "<OpcStr>.f<Width>" (e.g. "min.f64").
 //
 // Also defines ftz (flush subnormal inputs and results to sign-preserving
 // zero) variants for fp32 functions.
 //

 defm ADD_i1 : ADD_SUB_i1<add>;
 defm SUB_i1 : ADD_SUB_i1<sub>;
 
 // int16, int32, and int64 signed addition.  Since nvptx is 2's complement, we
 // also use these for unsigned arithmetic.
 defm ADD : I3<"add.s", add>;
 defm SUB : I3<"sub.s", sub>;
 
-// int32 addition and subtraction with carry-out.
-// FIXME: PTX 4.3 adds a 64-bit add.cc (and maybe also 64-bit addc.cc?).
-defm ADDCC : ADD_SUB_INT_32<"add.cc", addc>;
-defm SUBCC : ADD_SUB_INT_32<"sub.cc", subc>;
-
-// int32 addition and subtraction with carry-in and carry-out.
-defm ADDCCC : ADD_SUB_INT_32<"addc.cc", adde>;
-defm SUBCCC : ADD_SUB_INT_32<"subc.cc", sube>;
+// in32 and int64 addition and subtraction with carry-out.
+defm ADDCC : ADD_SUB_INT_CARRY<"add.cc", addc>;
+defm SUBCC : ADD_SUB_INT_CARRY<"sub.cc", subc>;
+
+// int32 and int64 addition and subtraction with carry-in and carry-out.
+defm ADDCCC : ADD_SUB_INT_CARRY<"addc.cc", adde>;
+defm SUBCCC : ADD_SUB_INT_CARRY<"subc.cc", sube>;
 
 defm MULT : I3<"mul.lo.s", mul>;
 
 defm MULTHS : I3<"mul.hi.s", mulhs>;
 defm MULTHU : I3<"mul.hi.u", mulhu>;
 
 defm SDIV : I3<"div.s", sdiv>;
 defm UDIV : I3<"div.u", udiv>;

llvm/test/CodeGen/NVPTX/add-128bit.ll

-; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck %s
-; RUN: %if ptxas %{ llc < %s -march=nvptx -mcpu=sm_20 | %ptxas-verify %}
-
-target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
-
-
-
-define void @foo(i64 %a, i64 %add, i128* %retptr) {
-; CHECK:        add.s64
-; CHECK:        setp.lt.u64
-; CHECK:        setp.lt.u64
-; CHECK:        selp.u64
-; CHECK:        selp.b64
-; CHECK:        add.s64
-  %t1 = sext i64 %a to i128
-  %add2 = zext i64 %add to i128
-  %val = add i128 %t1, %add2
-  store i128 %val, i128* %retptr
-  ret void
-}

llvm/test/CodeGen/NVPTX/add-sub-128bit.ll

+; RUN: llc < %s -march=nvptx -mcpu=sm_20 | FileCheck %s --check-prefixes=COMMON,NOCARRY
+; RUN: llc < %s -march=nvptx -mcpu=sm_20 -mattr=+ptx43 | FileCheck %s --check-prefixes=COMMON,CARRY
+; RUN: %if ptxas %{ llc < %s -march=nvptx -mcpu=sm_20 | %ptxas-verify %}
+
+target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64"
+
+; COMMON-LABEL: test_add
+define i128 @test_add(i128 %a, i128 %b) {
+; NOCARRY:        add.s64
+; NOCARRY-NEXT:   setp.lt.u64
+; NOCARRY-NEXT:   setp.lt.u64
+; NOCARRY-NEXT:   selp.u64
+; NOCARRY-NEXT:   selp.b64
+; NOCARRY-NEXT:   add.s64
+
+; CARRY:          add.cc.s64
+; CARRY-NEXT:     addc.cc.s64
+
+  %1 = add i128 %a, %b
+  ret i128 %1
+}
+
+; COMMON-LABEL: test_sub
+define i128 @test_sub(i128 %a, i128 %b) {
+; NOCARRY:        sub.s64
+; NOCARRY-NEXT:   setp.lt.u64
+; NOCARRY-NEXT:   selp.s64
+; NOCARRY-NEXT:   add.s64
+; NOCARRY-NEXT:   sub.s64
+
+; CARRY:          sub.cc.s64
+; CARRY-NEXT:     subc.cc.s64
+
+  %1 = sub i128 %a, %b
+  ret i128 %1
+}