[AArch64][FastISel] Handle CRC32 intrinsics

[aengelke]

Summary

With a similar reason as D148023; some applications make heavy use of
the CRC32 intrinsic (e.g., as part of a hash function) and therefore
benefit from avoiding frequent SelectionDAG fallbacks. In our
application, we get a 2% compile-time improvement.

[aengelke]

(format changes)

[aengelke]

Note that I'm not entirely happy with duplicating the test cases; changing the existing cases in arm64-crc32.ll could also work, but currently generates an extra uxtb/uxth with FastISel. Let me know what you think.

[aengelke]

Add check whether subtarget actually supports CRC.

[efriedma]

There's been very little development of AArch64 FastISel recently because we've turned on GlobalISel by default for AArch64. Is there some reason you're not using it? Or are you getting globalisel fallbacks so frequently that hacking fastisel is somehow worthwhile?

[aengelke]

We care a lot about compile times and in our application, GlobalISel (when it succeeds, about 10% fallback rate) increases compile times by ~50% compared to FastISel. We do plan to look at GlobalISel performance at some point, but it seems like there's a lot of work to be done to bring it at least on-par with FastISel wrt. compile times.

[efriedma]

Okay. I'm bringing it up because I'd like to make sure fastisel remains limited in scope; we don't have all the GlobalISel infrastructure, so reviewing any additions is significantly more work. (For comparison, globalisel supports these intrinsics with zero globalisel-specific lines of code.)

llvm/lib/Target/AArch64/AArch64FastISel.cpp

3792

You can assume intrinsics have the type declared in IntrinsicsAArch64.td; I don't think checking the type is legal does anything here.

3831

I'm pretty sure fastEmitInst_rr can't fail, so there isn't any reason to null-check the return value. Is this pattern copied from somewhere?

[aengelke]

Remove unnecessary checks

[aengelke]

In D148917#4288074, @efriedma wrote:

Okay. I'm bringing it up because I'd like to make sure fastisel remains limited in scope; we don't have all the GlobalISel infrastructure, so reviewing any additions is significantly more work.

I totally understand, and I don't plan further substantial additions (but crc32 and struct returns are fairly important for us).

llvm/lib/Target/AArch64/AArch64FastISel.cpp

3831

I probably slightly mixed this up with fastEmit_* which can fail (see sqrt intrinsic) and two occurences of fastEmitInst have checks (selectSDiv)/assertions (Intrinsic::frameaddress).

[aengelke]

(no change; re-trigger CI)

[efriedma]

LGTM

llvm/lib/Target/AArch64/AArch64FastISel.cpp

 #include "llvm/IR/Function.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsAArch64.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/AtomicOrdering.h"
ResultReg2 = fastEmitInst_rri(AArch64::CSINCWr, &AArch64::GPR32RegClass,
                                   AArch64::WZR, AArch64::WZR,
                                   getInvertedCondCode(CC));
     (void)ResultReg2;
     assert((ResultReg1 + 1) == ResultReg2 &&
            "Nonconsecutive result registers.");
     updateValueMap(II, ResultReg1, 2);
     return true;
   }
+  case Intrinsic::aarch64_crc32b:
+  case Intrinsic::aarch64_crc32h:
+  case Intrinsic::aarch64_crc32w:
+  case Intrinsic::aarch64_crc32x:
+  case Intrinsic::aarch64_crc32cb:
+  case Intrinsic::aarch64_crc32ch:
+  case Intrinsic::aarch64_crc32cw:
+  case Intrinsic::aarch64_crc32cx: {
+    if (!Subtarget->hasCRC())
+      return false;
+
+    unsigned Opc;
+    switch (II->getIntrinsicID()) {
+    default:
+      llvm_unreachable("Unexpected intrinsic!");
+    case Intrinsic::aarch64_crc32b:
+      Opc = AArch64::CRC32Brr;
+      break;
+    case Intrinsic::aarch64_crc32h:
+      Opc = AArch64::CRC32Hrr;
+      break;
+    case Intrinsic::aarch64_crc32w:
+      Opc = AArch64::CRC32Wrr;
+      break;
+    case Intrinsic::aarch64_crc32x:
+      Opc = AArch64::CRC32Xrr;
+      break;
+    case Intrinsic::aarch64_crc32cb:
+      Opc = AArch64::CRC32CBrr;
+      break;
+    case Intrinsic::aarch64_crc32ch:
+      Opc = AArch64::CRC32CHrr;
+      break;
+    case Intrinsic::aarch64_crc32cw:
+      Opc = AArch64::CRC32CWrr;
+      break;
+    case Intrinsic::aarch64_crc32cx:
+      Opc = AArch64::CRC32CXrr;
+      break;
+    }
+
+    Register LHSReg = getRegForValue(II->getArgOperand(0));
+    Register RHSReg = getRegForValue(II->getArgOperand(1));
+    if (!LHSReg || !RHSReg)
+      return false;
+
+    Register ResultReg =
+        fastEmitInst_rr(Opc, &AArch64::GPR32RegClass, LHSReg, RHSReg);
+    updateValueMap(II, ResultReg);
+    return true;
+  }
   }
   return false;
 }
 
 bool AArch64FastISel::selectRet(const Instruction *I) {
   const ReturnInst *Ret = cast<ReturnInst>(I);
   const Function &F = *I->getParent()->getParent();
 
   if (!FuncInfo.CanLowerReturn)

llvm/test/CodeGen/AArch64/arm64-fast-isel-crc32.ll

+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mattr=+crc -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mattr=+v8r -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-eabi -fast-isel -fast-isel-abort=3 -mcpu=cortex-a53 -mattr=+crc -o - %s | FileCheck %s
+
+; Note: tests are a copy of arm64-crc32.ll
+
+define i32 @test_crc32b(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32b:
+; CHECK: crc32b w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32b(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32h(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32h:
+; CHECK: crc32h w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32h(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32w(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32w:
+; CHECK: crc32w w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32w(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32x(i32 %cur, i64 %next) {
+; CHECK-LABEL: test_crc32x:
+; CHECK: crc32x w0, w0, x1
+  %val = call i32 @llvm.aarch64.crc32x(i32 %cur, i64 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cb(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32cb:
+; CHECK: crc32cb w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32cb(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32ch(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32ch:
+; CHECK: crc32ch w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32ch(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cw(i32 %cur, i32 %next) {
+; CHECK-LABEL: test_crc32cw:
+; CHECK: crc32cw w0, w0, w1
+  %val = call i32 @llvm.aarch64.crc32cw(i32 %cur, i32 %next)
+  ret i32 %val
+}
+
+define i32 @test_crc32cx(i32 %cur, i64 %next) {
+; CHECK-LABEL: test_crc32cx:
+; CHECK: crc32cx w0, w0, x1
+  %val = call i32 @llvm.aarch64.crc32cx(i32 %cur, i64 %next)
+  ret i32 %val
+}
+
+declare i32 @llvm.aarch64.crc32b(i32, i32)
+declare i32 @llvm.aarch64.crc32h(i32, i32)
+declare i32 @llvm.aarch64.crc32w(i32, i32)
+declare i32 @llvm.aarch64.crc32x(i32, i64)
+
+declare i32 @llvm.aarch64.crc32cb(i32, i32)
+declare i32 @llvm.aarch64.crc32ch(i32, i32)
+declare i32 @llvm.aarch64.crc32cw(i32, i32)
+declare i32 @llvm.aarch64.crc32cx(i32, i64)