[NVPTX] Eliminate StoreRetval instructions with undef operand

[kovdan01]

Summary

Previously a lot of StoreRetval instructions with undef operand were generated
on NVPTX target when a big struct was returned by value. It resulted in a lot
of unneeded st.param.* instructions in final assembly. The patch solves the
issue by implementing the logic in NVPTX-specific part of DAG combiner.

[tra]

LGTM in principle, modulo tests as they need to actually check that we do not produce unwanted stores.

Also, the change is unlikely to affect the actual GPU code, only PTX, as ptxas is smart enough to eliminate undef stores: https://godbolt.org/z/fdb1hGdxW

llvm/test/CodeGen/NVPTX/store-retval.ll

33–38

This only verifies that the known values were stored, but the test would still pass on the current code which also stores the undef values.
You do need a handful of CHECK-NOT for st-param before/after the stores of valid values.

[kovdan01]

@tra Thanks for your review! Fixed tests.

Also, the change is unlikely to affect the actual GPU code, only PTX, as ptxas is smart enough to eliminate undef stores: https://godbolt.org/z/fdb1hGdxW

Hmm, it's an interesting notice, however, I suppose that enhancing PTX code also looks valuable.

llvm/test/CodeGen/NVPTX/store-retval.ll

33–38

Fixed, thanks!

llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp

   return SDValue();
 }
 
+static SDValue PerformStoreRetvalCombine(SDNode *N) {
+  // Operands from the 2nd to the last one are the values to be stored
+  for (std::size_t I = 2, OpsCount = N->ops().size(); I != OpsCount; ++I)
+    if (!N->getOperand(I).isUndef())
+      return SDValue();
+
+  // Operand 0 is the previous value in the chain. Cannot return EntryToken
+  // as the previous value will become unused and eliminated later.
+  return N->getOperand(0);
+}
+
 /// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
 ///
 static SDValue PerformADDCombine(SDNode *N,

       return PerformREMCombine(N, DCI, OptLevel);
     case ISD::SETCC:
       return PerformSETCCCombine(N, DCI);
+    case NVPTXISD::StoreRetval:
+    case NVPTXISD::StoreRetvalV2:
+    case NVPTXISD::StoreRetvalV4:
+      return PerformStoreRetvalCombine(N);
   }
   return SDValue();
 }

llvm/test/CodeGen/NVPTX/store-retval.ll

+; RUN: llc < %s --mtriple=nvptx-unknown-unknown | FileCheck %s
+;
+; This is IR generated with clang using -O3 optimization level
+; and nvptx-unknown-unknown target from the following C code.
+;
+; struct StNoalign {              unsigned int field[73]; };
+; struct StAlign8  { _Alignas(8)  unsigned int field[73]; };
+; struct StAlign16 { _Alignas(16) unsigned int field[73]; };
+;
+; #define DECLARE_FUNC(StName)                        \
+; struct StName func_##StName(struct StName in) {     \
+;   struct StName ret;                                \
+;   ret.field[8]  = in.field[12];                     \
+;   ret.field[9]  = in.field[13];                     \
+;   ret.field[10] = in.field[14];                     \
+;   ret.field[11] = in.field[15];                     \
+;   ret.field[27] = 0;                                \
+;   return ret;                                       \
+; }                                                   \
+;
+; DECLARE_FUNC(StNoalign)
+; DECLARE_FUNC(StAlign8)
+; DECLARE_FUNC(StAlign16)
+
+%struct.StNoalign = type { [73 x i32] }
+
+define %struct.StNoalign @func_StNoalign(%struct.StNoalign* nocapture noundef readonly byval(%struct.StNoalign) align 4 %in) {
+  ; CHECK-LABEL: .func{{.*}}func_StNoalign
+  ; CHECK:       ld.param.u32 [[A:%r[0-9]+]], [func_StNoalign_param_0+48];
+  ; CHECK:       ld.param.u32 [[B:%r[0-9]+]], [func_StNoalign_param_0+52];
+  ; CHECK:       ld.param.u32 [[C:%r[0-9]+]], [func_StNoalign_param_0+56];
+  ; CHECK:       ld.param.u32 [[D:%r[0-9]+]], [func_StNoalign_param_0+60];
+  ; CHECK:       st.param.b32 [func_retval0+32],  [[A]];
+  ; CHECK:       st.param.b32 [func_retval0+36],  [[B]];
+  ; CHECK:       st.param.b32 [func_retval0+40],  [[C]];
+  ; CHECK:       st.param.b32 [func_retval0+44],  [[D]];
+  ; CHECK:       mov.u32 [[E:%r[0-9]+]], 0;
+  ; CHECK:       st.param.b32 [func_retval0+108], [[E]];
+  ; CHECK-NEXT:  ret;
+  %arrayidx = getelementptr inbounds %struct.StNoalign, %struct.StNoalign* %in, i32 0, i32 0, i32 12
+  %1 = load i32, i32* %arrayidx, align 4
+  %arrayidx4 = getelementptr inbounds %struct.StNoalign, %struct.StNoalign* %in, i32 0, i32 0, i32 13
+  %2 = load i32, i32* %arrayidx4, align 4
+  %arrayidx8 = getelementptr inbounds %struct.StNoalign, %struct.StNoalign* %in, i32 0, i32 0, i32 14
+  %3 = load i32, i32* %arrayidx8, align 4
+  %arrayidx12 = getelementptr inbounds %struct.StNoalign, %struct.StNoalign* %in, i32 0, i32 0, i32 15
+  %4 = load i32, i32* %arrayidx12, align 4
+  %.fca.0.8.insert = insertvalue %struct.StNoalign { [73 x i32] [i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison] }, i32 %1, 0, 8
+  %.fca.0.9.insert = insertvalue %struct.StNoalign %.fca.0.8.insert, i32 %2, 0, 9
+  %.fca.0.10.insert = insertvalue %struct.StNoalign %.fca.0.9.insert, i32 %3, 0, 10
+  %.fca.0.11.insert = insertvalue %struct.StNoalign %.fca.0.10.insert, i32 %4, 0, 11
+  %.fca.0.27.insert = insertvalue %struct.StNoalign %.fca.0.11.insert, i32 0, 0, 27
+  ret %struct.StNoalign %.fca.0.27.insert
+}
+
+%struct.StAlign8 = type { [73 x i32], [4 x i8] }
+
+define %struct.StAlign8 @func_StAlign8(%struct.StAlign8* nocapture noundef readonly byval(%struct.StAlign8) align 8 %in) {
+  ; CHECK-LABEL: .func{{.*}}func_StAlign8
+  ; CHECK:       ld.param.v2.u32 {[[A:%r[0-9]+]], [[B:%r[0-9]+]]}, [func_StAlign8_param_0+48];
+  ; CHECK:       ld.param.v2.u32 {[[C:%r[0-9]+]], [[D:%r[0-9]+]]}, [func_StAlign8_param_0+56];
+  ; CHECK:       st.param.b32 [func_retval0+32],  [[A]];
+  ; CHECK:       st.param.b32 [func_retval0+36],  [[B]];
+  ; CHECK:       st.param.b32 [func_retval0+40],  [[C]];
+  ; CHECK:       st.param.b32 [func_retval0+44],  [[D]];
+  ; CHECK:       mov.u32 [[E:%r[0-9]+]], 0;
+  ; CHECK:       st.param.b32 [func_retval0+108], [[E]];
+  ; CHECK-NEXT:  ret;
+  %arrayidx = getelementptr inbounds %struct.StAlign8, %struct.StAlign8* %in, i32 0, i32 0, i32 12
+  %1 = load i32, i32* %arrayidx, align 8
+  %arrayidx4 = getelementptr inbounds %struct.StAlign8, %struct.StAlign8* %in, i32 0, i32 0, i32 13
+  %2 = load i32, i32* %arrayidx4, align 4
+  %arrayidx8 = getelementptr inbounds %struct.StAlign8, %struct.StAlign8* %in, i32 0, i32 0, i32 14
+  %3 = load i32, i32* %arrayidx8, align 8
+  %arrayidx12 = getelementptr inbounds %struct.StAlign8, %struct.StAlign8* %in, i32 0, i32 0, i32 15
+  %4 = load i32, i32* %arrayidx12, align 4
+  %.fca.0.8.insert = insertvalue %struct.StAlign8 { [73 x i32] [i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison], [4 x i8] poison }, i32 %1, 0, 8
+  %.fca.0.9.insert = insertvalue %struct.StAlign8 %.fca.0.8.insert, i32 %2, 0, 9
+  %.fca.0.10.insert = insertvalue %struct.StAlign8 %.fca.0.9.insert, i32 %3, 0, 10
+  %.fca.0.11.insert = insertvalue %struct.StAlign8 %.fca.0.10.insert, i32 %4, 0, 11
+  %.fca.0.27.insert = insertvalue %struct.StAlign8 %.fca.0.11.insert, i32 0, 0, 27
+  ret %struct.StAlign8 %.fca.0.27.insert
+}
+
+%struct.StAlign16 = type { [73 x i32], [12 x i8] }
+
+define %struct.StAlign16 @func_StAlign16(%struct.StAlign16* nocapture noundef readonly byval(%struct.StAlign16) align 16 %in) {
+  ; CHECK-LABEL: .func{{.*}}func_StAlign16
+  ; CHECK:       ld.param.v4.u32 {[[A:%r[0-9]+]], [[B:%r[0-9]+]], [[C:%r[0-9]+]], [[D:%r[0-9]+]]}, [func_StAlign16_param_0+48];
+  ; CHECK:       st.param.b32 [func_retval0+32],  [[A]];
+  ; CHECK:       st.param.b32 [func_retval0+36],  [[B]];
+  ; CHECK:       st.param.b32 [func_retval0+40],  [[C]];
+  ; CHECK:       st.param.b32 [func_retval0+44],  [[D]];
+  ; CHECK:       mov.u32 [[E:%r[0-9]+]], 0;
+  ; CHECK:       st.param.b32 [func_retval0+108], [[E]];
+  ; CHECK-NEXT:  ret;
+  %arrayidx = getelementptr inbounds %struct.StAlign16, %struct.StAlign16* %in, i32 0, i32 0, i32 12
+  %1 = load i32, i32* %arrayidx, align 16
+  %arrayidx4 = getelementptr inbounds %struct.StAlign16, %struct.StAlign16* %in, i32 0, i32 0, i32 13
+  %2 = load i32, i32* %arrayidx4, align 4
+  %arrayidx8 = getelementptr inbounds %struct.StAlign16, %struct.StAlign16* %in, i32 0, i32 0, i32 14
+  %3 = load i32, i32* %arrayidx8, align 8
+  %arrayidx12 = getelementptr inbounds %struct.StAlign16, %struct.StAlign16* %in, i32 0, i32 0, i32 15
+  %4 = load i32, i32* %arrayidx12, align 4
+  %.fca.0.8.insert = insertvalue %struct.StAlign16 { [73 x i32] [i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison], [12 x i8] poison }, i32 %1, 0, 8
+  %.fca.0.9.insert = insertvalue %struct.StAlign16 %.fca.0.8.insert, i32 %2, 0, 9
+  %.fca.0.10.insert = insertvalue %struct.StAlign16 %.fca.0.9.insert, i32 %3, 0, 10
+  %.fca.0.11.insert = insertvalue %struct.StAlign16 %.fca.0.10.insert, i32 %4, 0, 11
+  %.fca.0.27.insert = insertvalue %struct.StAlign16 %.fca.0.11.insert, i32 0, 0, 27
+  ret %struct.StAlign16 %.fca.0.27.insert
+}