[AArch64][GlobalISel] Legalize 32-bit + narrow G_SMULO + G_UMULO

[paquette]

Summary

SDAG lowers 32-bit and 64-bit G_SMULO + G_UMULO. We were missing the 32-bit case.

For other sizes, make the 0th type a power of 2 and clamp it to either 32 bits or 64 bits.

Right now, this will allow us to handle narrow types (e.g. s4, s24, etc.). The LegalizerHelper doesn't support narrowing G_SMULO or G_UMULO right now. I think we want clamping behaviour either way, so we might as well include it now to be explicit.

[aemerson]

llvm/test/CodeGen/AArch64/GlobalISel/legalize-mul.mir

245–265

How about we move this to arm64-fallback.ll and keep the abort=1 in this file.

[paquette]

Move test to arm64-fallback.ll + keep the abort.

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder({G_SDIV, G_UDIV})
       .libcallFor({s128})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
       .scalarize(0);
 
   getActionDefinitionsBuilder({G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
       .lowerFor({s1, s8, s16, s32, s64});
 
-  getActionDefinitionsBuilder({G_SMULO, G_UMULO}).lowerFor({{s64, s1}});
+  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+      .widenScalarToNextPow2(0, /*Min = */ 32)
+      .clampScalar(0, s32, s64)
+      .lowerIf(typeIs(1, s1));
 
   getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64});
 
   getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
       .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
       .clampNumElements(0, v8s8, v16s8)
       .clampNumElements(0, v4s16, v8s16)
       .clampNumElements(0, v2s32, v4s32)

llvm/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll

 define void @asm_input_ls64(%struct.foo* %input, i8* %addr) #2 {
 entry:
   %incast = bitcast %struct.foo* %input to i512*
   %val = load i512, i512* %incast, align 8
   call void asm sideeffect "st64b $0,[$1]", "r,r,~{memory}"(i512 %val, i8* %addr)
   ret void
 }
 
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %4:_(s128), %5:_(s1) = G_UMULO %0:_, %6:_ (in function: umul_s128)
+; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for umul_s128
+; FALLBACK-WITH-REPORT-OUT-LABEL: umul_s128
+declare {i128, i1} @llvm.umul.with.overflow.i128(i128, i128) nounwind readnone
+define zeroext i1 @umul_s128(i128 %v1, i128* %res) {
+entry:
+  %t = call {i128, i1} @llvm.umul.with.overflow.i128(i128 %v1, i128 2)
+  %val = extractvalue {i128, i1} %t, 0
+  %obit = extractvalue {i128, i1} %t, 1
+  store i128 %val, i128* %res
+  ret i1 %obit
+}
+
 attributes #1 = { "target-features"="+sve" }
 attributes #2 = { "target-features"="+ls64" }
 
 declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 %pattern)
 declare <vscale x 16 x i8> @llvm.aarch64.sve.ld1.nxv16i8(<vscale x 16 x i1>, i8*)

llvm/test/CodeGen/AArch64/GlobalISel/legalize-mul.mir

bb.0:
     %1:_(s64) = COPY $x1
     %2:_(s64), %3:_(s1) = G_UMULO %0, %1
     $x0 = COPY %2(s64)
     %4:_(s32) = G_ANYEXT %3(s1)
     $w0 = COPY %4(s32)
 
 ...
 ---
+name:            test_smul_overflow_s32
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_smul_overflow_s32
+    ; CHECK: %lhs:_(s32) = COPY $w0
+    ; CHECK: %rhs:_(s32) = COPY $w1
+    ; CHECK: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH %lhs, %rhs
+    ; CHECK: %mul:_(s32) = G_MUL %lhs, %rhs
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 31
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR %mul, [[C]](s64)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[SMULH]](s32), [[ASHR]]
+    ; CHECK: $w0 = COPY %mul(s32)
+    ; CHECK: %ext_overflow:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: $w0 = COPY %ext_overflow(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %lhs:_(s32) = COPY $w0
+    %rhs:_(s32) = COPY $w1
+    %mul:_(s32), %overflow:_(s1) = G_SMULO %lhs, %rhs
+    $w0 = COPY %mul(s32)
+    %ext_overflow:_(s32) = G_ANYEXT %overflow(s1)
+    $w0 = COPY %ext_overflow(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            test_umul_overflow_s32
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_umul_overflow_s32
+    ; CHECK: %lhs:_(s32) = COPY $w0
+    ; CHECK: %rhs:_(s32) = COPY $w1
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH %lhs, %rhs
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: %mul:_(s32) = G_MUL %lhs, %rhs
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s32), [[C]]
+    ; CHECK: $w0 = COPY %mul(s32)
+    ; CHECK: %ext_overflow:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: $w0 = COPY %ext_overflow(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %lhs:_(s32) = COPY $w0
+    %rhs:_(s32) = COPY $w1
+    %mul:_(s32), %overflow:_(s1) = G_UMULO %lhs, %rhs
+    $w0 = COPY %mul(s32)
+    %ext_overflow:_(s32) = G_ANYEXT %overflow(s1)
+    $w0 = COPY %ext_overflow(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
+name:            test_umul_overflow_s24
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_umul_overflow_s24
+    ; CHECK: %lhs_wide:_(s32) = COPY $w0
+    ; CHECK: %rhs_wide:_(s32) = COPY $w1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %lhs_wide(s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %rhs_wide(s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[AND]], [[AND1]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[AND]], [[AND1]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[UMULH]](s32), [[C1]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(ne), [[MUL]](s32), [[AND2]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ICMP1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[COPY3]]
+    ; CHECK: %ext_mul:_(s32) = COPY [[MUL]](s32)
+    ; CHECK: $w0 = COPY %ext_mul(s32)
+    ; CHECK: %ext_overflow:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $w0 = COPY %ext_overflow(s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %lhs_wide:_(s32) = COPY $w0
+    %rhs_wide:_(s32) = COPY $w1
+    %lhs:_(s24) = G_TRUNC %lhs_wide
+    %rhs:_(s24) = G_TRUNC %rhs_wide
+    %mul:_(s24), %overflow:_(s1) = G_UMULO %lhs, %rhs
+    %ext_mul:_(s32) = G_ANYEXT %mul
+    $w0 = COPY %ext_mul(s32)
+    %ext_overflow:_(s32) = G_ANYEXT %overflow(s1)
+    $w0 = COPY %ext_overflow(s32)
+    RET_ReallyLR implicit $w0
+
+...
+---
 name:            vector_mul_scalarize
 liveins:
   - { reg: '$q0' }
   - { reg: '$q1' }
 body:             |
   bb.1:
     liveins: $q0, $q1
 
bb.1:
     %9:_(s64), %10:_(s1) = G_UMULO %7, %8
     %31:_(s64) = G_CONSTANT i64 0
     G_STORE %31(s64), %6(p0) :: (store (s64), align 1)
     %16:_(s64) = G_ZEXT %10(s1)
     $x0 = COPY %9(s64)
     $x1 = COPY %16(s64)
     RET_ReallyLR implicit $x0
 
 ...

llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_SSUBE (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 2, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_UMULO (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_SMULO (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
 # DEBUG-NEXT: G_UMULH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_SMULH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
 # DEBUG-NEXT: .. the first uncovered type index: 1, OK
 # DEBUG-NEXT: .. the first uncovered imm index: 0, OK
 # DEBUG-NEXT: G_UADDSAT (opcode {{[0-9]+}}): 1 type index, 0 imm indices