[AArch64][GlobalISel] Select @llvm.aarch64.ldaxr.* intrinsics

[paquette]

Summary

Add a GISelPredicateCode to ldaxr_*. This allows us to import the patterns for @llvm.aarch64.ldaxr.*, and thus select them.

Add isLoadStoreOfNumBytes for the GISelPredicateCode, since each of these intrinsics involves the same check.

Add select-ldaxr-intrin.mir, and update arm64-ldxr-stxr.ll.

[aemerson]

The new test can be made more concise, but it's also a bit redundant given we're adding checks to an existing one. I don't mind either way.

llvm/test/CodeGen/AArch64/GlobalISel/select-ldaxr-intrin.mir

32

Just use a pointer parameter here instead of a global?

37

And return the value here to reduce the test size.

[paquette]

Simplify MIR test before committing

llvm/lib/Target/AArch64/AArch64InstrAtomics.td

def : Pat<(and (ldxr_2 GPR64sp:$addr), 0xffff),
           (SUBREG_TO_REG (i64 0), (LDXRH GPR64sp:$addr), sub_32)>;
 def : Pat<(and (ldxr_4 GPR64sp:$addr), 0xffffffff),
           (SUBREG_TO_REG (i64 0), (LDXRW GPR64sp:$addr), sub_32)>;
 
 // Load-exclusives.
 
 def ldaxr_1 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 1); }];
+}
 
 def ldaxr_2 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 2); }];
+}
 
 def ldaxr_4 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 4); }];
+}
 
 def ldaxr_8 : PatFrag<(ops node:$ptr), (int_aarch64_ldaxr node:$ptr), [{
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
-}]>;
+}]> {
+  let GISelPredicateCode = [{ return isLoadStoreOfNumBytes(MI, 8); }];
+}
 
 def : Pat<(ldaxr_1 GPR64sp:$addr),
           (SUBREG_TO_REG (i64 0), (LDAXRB GPR64sp:$addr), sub_32)>;
 def : Pat<(ldaxr_2 GPR64sp:$addr),
           (SUBREG_TO_REG (i64 0), (LDAXRH GPR64sp:$addr), sub_32)>;
 def : Pat<(ldaxr_4 GPR64sp:$addr),
           (SUBREG_TO_REG (i64 0), (LDAXRW GPR64sp:$addr), sub_32)>;
 def : Pat<(ldaxr_8 GPR64sp:$addr), (LDAXRX GPR64sp:$addr)>;

llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp

private:
   // Optimization methods.
   bool tryOptVectorShuffle(MachineInstr &I) const;
   bool tryOptVectorDup(MachineInstr &MI) const;
   bool tryOptSelect(MachineInstr &MI) const;
   MachineInstr *tryFoldIntegerCompare(MachineOperand &LHS, MachineOperand &RHS,
                                       MachineOperand &Predicate,
                                       MachineIRBuilder &MIRBuilder) const;
 
+  /// Return true if \p MI is a load or store of \p NumBytes bytes.
+  bool isLoadStoreOfNumBytes(const MachineInstr &MI, unsigned NumBytes) const;
+
   const AArch64TargetMachine &TM;
   const AArch64Subtarget &STI;
   const AArch64InstrInfo &TII;
   const AArch64RegisterInfo &TRI;
   const AArch64RegisterBankInfo &RBI;
 
   bool ProduceNonFlagSettingCondBr = false;
 

 void AArch64InstructionSelector::renderLogicalImm64(
     MachineInstrBuilder &MIB, const MachineInstr &I) const {
   assert(I.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
   uint64_t CstVal = I.getOperand(1).getCImm()->getZExtValue();
   uint64_t Enc = AArch64_AM::encodeLogicalImmediate(CstVal, 64);
   MIB.addImm(Enc);
 }
 
+bool AArch64InstructionSelector::isLoadStoreOfNumBytes(
+    const MachineInstr &MI, unsigned NumBytes) const {
+  if (!MI.mayLoadOrStore())
+    return false;
+  assert(MI.hasOneMemOperand() &&
+         "Expected load/store to have only one mem op!");
+  return (*MI.memoperands_begin())->getSize() == NumBytes;
+}
+
 namespace llvm {
 InstructionSelector *
 createAArch64InstructionSelector(const AArch64TargetMachine &TM,
                                  AArch64Subtarget &Subtarget,
                                  AArch64RegisterBankInfo &RBI) {
   return new AArch64InstructionSelector(TM, Subtarget, RBI);
 }
 }

llvm/test/CodeGen/AArch64/GlobalISel/select-ldaxr-intrin.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-unknown-unknown -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  define void @test_load_acquire_i8(i8* %addr) { ret void }
+  define void @test_load_acquire_i16(i16* %addr) { ret void }
+  define void @test_load_acquire_i32(i32* %addr) { ret void }
+  define void @test_load_acquire_i64(i64* %addr) { ret void }
+  @var = global i64 0, align 8
+...
+---
+
+name:            test_load_acquire_i8
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: test_load_acquire_i8
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[MOVaddr:%[0-9]+]]:gpr64common = MOVaddr target-flags(aarch64-page) @var, target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[LDAXRB:%[0-9]+]]:gpr32 = LDAXRB [[COPY]] :: (volatile load 1 from %ir.addr)
+    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64all = SUBREG_TO_REG 0, [[LDAXRB]], %subreg.sub_32
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY [[SUBREG_TO_REG]]
+    ; CHECK: [[ANDXri:%[0-9]+]]:gpr64common = ANDXri [[COPY1]], 4103
+    ; CHECK: STRXui [[ANDXri]], [[MOVaddr]], 0 :: (store 8 into @var)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %4:gpr(p0) = G_GLOBAL_VALUE @var
+    %1:gpr(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.ldaxr), %0(p0) :: (volatile load 1 from %ir.addr)
+    %5:gpr(s64) = G_CONSTANT i64 255
+    %6:gpr(s64) = COPY %1(s64)
+    %3:gpr(s64) = G_AND %6, %5
+    G_STORE %3(s64), %4(p0) :: (store 8 into @var)
+    RET_ReallyLR
+
+...
+---
+name:            test_load_acquire_i16
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: test_load_acquire_i16
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[MOVaddr:%[0-9]+]]:gpr64common = MOVaddr target-flags(aarch64-page) @var, target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[LDAXRH:%[0-9]+]]:gpr32 = LDAXRH [[COPY]] :: (volatile load 2 from %ir.addr)
+    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64all = SUBREG_TO_REG 0, [[LDAXRH]], %subreg.sub_32
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY [[SUBREG_TO_REG]]
+    ; CHECK: [[ANDXri:%[0-9]+]]:gpr64common = ANDXri [[COPY1]], 4111
+    ; CHECK: STRXui [[ANDXri]], [[MOVaddr]], 0 :: (store 8 into @var)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %4:gpr(p0) = G_GLOBAL_VALUE @var
+    %1:gpr(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.ldaxr), %0(p0) :: (volatile load 2 from %ir.addr)
+    %5:gpr(s64) = G_CONSTANT i64 65535
+    %6:gpr(s64) = COPY %1(s64)
+    %3:gpr(s64) = G_AND %6, %5
+    G_STORE %3(s64), %4(p0) :: (store 8 into @var)
+    RET_ReallyLR
+
+...
+---
+name:            test_load_acquire_i32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: test_load_acquire_i32
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[MOVaddr:%[0-9]+]]:gpr64common = MOVaddr target-flags(aarch64-page) @var, target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[LDAXRW:%[0-9]+]]:gpr32 = LDAXRW [[COPY]] :: (volatile load 4 from %ir.addr)
+    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64all = SUBREG_TO_REG 0, [[LDAXRW]], %subreg.sub_32
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr64 = COPY [[SUBREG_TO_REG]]
+    ; CHECK: [[ANDXri:%[0-9]+]]:gpr64common = ANDXri [[COPY1]], 4127
+    ; CHECK: STRXui [[ANDXri]], [[MOVaddr]], 0 :: (store 8 into @var)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %4:gpr(p0) = G_GLOBAL_VALUE @var
+    %1:gpr(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.ldaxr), %0(p0) :: (volatile load 4 from %ir.addr)
+    %5:gpr(s64) = G_CONSTANT i64 4294967295
+    %6:gpr(s64) = COPY %1(s64)
+    %3:gpr(s64) = G_AND %6, %5
+    G_STORE %3(s64), %4(p0) :: (store 8 into @var)
+    RET_ReallyLR
+
+...
+---
+name:            test_load_acquire_i64
+alignment:       2
+legalized:       true
+regBankSelected: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: name: test_load_acquire_i64
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64sp = COPY $x0
+    ; CHECK: [[MOVaddr:%[0-9]+]]:gpr64common = MOVaddr target-flags(aarch64-page) @var, target-flags(aarch64-pageoff, aarch64-nc) @var
+    ; CHECK: [[LDAXRX:%[0-9]+]]:gpr64 = LDAXRX [[COPY]] :: (volatile load 8 from %ir.addr)
+    ; CHECK: STRXui [[LDAXRX]], [[MOVaddr]], 0 :: (store 8 into @var)
+    ; CHECK: RET_ReallyLR
+    %0:gpr(p0) = COPY $x0
+    %2:gpr(p0) = G_GLOBAL_VALUE @var
+    %1:gpr(s64) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.ldaxr), %0(p0) :: (volatile load 8 from %ir.addr)
+    G_STORE %1(s64), %2(p0) :: (store 8 into @var)
+    RET_ReallyLR

llvm/test/CodeGen/AArch64/arm64-ldxr-stxr.ll

entry:
   %tmp7 = trunc i128 %tmp6 to i64
   %strexd = tail call i32 @llvm.aarch64.stlxp(i64 %tmp4, i64 %tmp7, i8* %ptr)
   ret i32 %strexd
 }
 
 declare %0 @llvm.aarch64.ldaxp(i8*) nounwind
 declare i32 @llvm.aarch64.stlxp(i64, i64, i8*) nounwind
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i8
 define void @test_load_acquire_i8(i8* %addr) {
 ; CHECK-LABEL: test_load_acquire_i8:
 ; CHECK: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtb
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; FIXME: GlobalISel doesn't fold ands/adds into load/store addressing modes
+; right now/ So, we won't get the :lo12:var.
+; GISEL-LABEL: test_load_acquire_i8:
+; GISEL: ldaxrb w[[LOADVAL:[0-9]+]], [x0]
+; GISEL-DAG: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i8(i8* %addr)
   %shortval = trunc i64 %val to i8
   %extval = zext i8 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i16
 define void @test_load_acquire_i16(i16* %addr) {
 ; CHECK-LABEL: test_load_acquire_i16:
 ; CHECK: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxth
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i16:
+; GISEL: ldaxrh w[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i16(i16* %addr)
   %shortval = trunc i64 %val to i16
   %extval = zext i16 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i32
 define void @test_load_acquire_i32(i32* %addr) {
 ; CHECK-LABEL: test_load_acquire_i32:
 ; CHECK: ldaxr w[[LOADVAL:[0-9]+]], [x0]
 ; CHECK-NOT: uxtw
 ; CHECK-NOT: and
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i32:
+; GISEL: ldaxr w[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i32(i32* %addr)
   %shortval = trunc i64 %val to i32
   %extval = zext i32 %shortval to i64
   store i64 %extval, i64* @var, align 8
   ret void
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_load_acquire_i64
 define void @test_load_acquire_i64(i64* %addr) {
 ; CHECK-LABEL: test_load_acquire_i64:
 ; CHECK: ldaxr x[[LOADVAL:[0-9]+]], [x0]
 ; CHECK: str x[[LOADVAL]], [{{x[0-9]+}}, :lo12:var]
 
+; GISEL-LABEL: test_load_acquire_i64:
+; GISEL: ldaxr x[[LOADVAL:[0-9]+]], [x0]
+; GISEL: str x[[LOADVAL]], [{{x[0-9]+}}]
   %val = call i64 @llvm.aarch64.ldaxr.p0i64(i64* %addr)
   store i64 %val, i64* @var, align 8
   ret void
 }
 
 
 declare i64 @llvm.aarch64.ldaxr.p0i8(i8*) nounwind
 declare i64 @llvm.aarch64.ldaxr.p0i16(i16*) nounwind