[MVE] Fixup order of gather writeback intrinsic outputs.

[dmgreen]

Summary

The MVE_VLDRWU32_qi_pre, like the other _pre/_post mve loads returns the writeback as result 0, the value as result 1. The llvm ir intrinsic seems to have this the other way around though, and so when lowering from one to the other we need to switch the first two outputs.

I've also fixed up the types of _pre/_post on normal MVE loads. There we were already getting the values the right way around, just not for the types. I don't believe this was causing anything to go wrong, it was just very confusing to read in the debug output.

[simon_tatham]

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

2525

This line looks redundant?

[dmgreen]

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

2525

The ReplaceUses? It's attempting to replace the chain operand of N with New. We have to create a new node now (I think), as opposed to morphing the existing value, so need to transfer over the chain too.

[simon_tatham]

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

2525

Oh yes. Blame Monday morning. I somehow misread it as the common idiom of updating a node in place.

llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
            SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0))
     Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
   else
     return false;
 
   SDValue Ops[] = {Base, NewOffset,
                    CurDAG->getTargetConstant(Pred, SDLoc(N), MVT::i32), PredReg,
                    Chain};
-  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), N->getValueType(0),
-                                       MVT::i32, MVT::Other, Ops);
+  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
+                                       N->getValueType(0), MVT::Other, Ops);
   transferMemOperands(N, New);
   ReplaceUses(SDValue(N, 0), SDValue(New, 1));
   ReplaceUses(SDValue(N, 1), SDValue(New, 0));
   ReplaceUses(SDValue(N, 2), SDValue(New, 2));
   CurDAG->RemoveDeadNode(N);
   return true;
 }
 
void ARMDAGToDAGISel::SelectMVE_WB(SDNode *N, const uint16_t *Opcodes,
 
   if (Predicated)
     AddMVEPredicateToOps(Ops, Loc, N->getOperand(4));
   else
     AddEmptyMVEPredicateToOps(Ops, Loc);
 
   Ops.push_back(N->getOperand(0)); // chain
 
-  CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
+  SmallVector<EVT, 8> VTs;
+  VTs.push_back(N->getValueType(1));
+  VTs.push_back(N->getValueType(0));
+  VTs.push_back(N->getValueType(2));
+
+  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), VTs, Ops);
+  ReplaceUses(SDValue(N, 0), SDValue(New, 1));
+  ReplaceUses(SDValue(N, 1), SDValue(New, 0));
+  ReplaceUses(SDValue(N, 2), SDValue(New, 2));
+  CurDAG->RemoveDeadNode(N);
 }
 
 void ARMDAGToDAGISel::SelectMVE_LongShift(SDNode *N, uint16_t Opcode,
                                           bool Immediate,
                                           bool HasSaturationOperand) {
   SDLoc Loc(N);
   SmallVector<SDValue, 8> Ops;
 

llvm/test/CodeGen/Thumb2/mve-intrinsics/scatter-gather.ll

 entry:
   %0 = call <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64> %addr, i32 -336)
   ret <2 x i64> %0
 }
 
 define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_s64(<2 x i64>* %addr) {
 ; CHECK-LABEL: test_vldrdq_gather_base_wb_s64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrd.u64 q1, [q0, #576]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrd.u64 q0, [q1, #576]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <2 x i64>, <2 x i64>* %addr, align 8
   %1 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> %0, i32 576)
   %2 = extractvalue { <2 x i64>, <2 x i64> } %1, 1
   store <2 x i64> %2, <2 x i64>* %addr, align 8
   %3 = extractvalue { <2 x i64>, <2 x i64> } %1, 0
   ret <2 x i64> %3
 }
 
 declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64>, i32)
 
 define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_u64(<2 x i64>* %addr) {
 ; CHECK-LABEL: test_vldrdq_gather_base_wb_u64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrd.u64 q1, [q0, #-328]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrd.u64 q0, [q1, #-328]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <2 x i64>, <2 x i64>* %addr, align 8
   %1 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> %0, i32 -328)
   %2 = extractvalue { <2 x i64>, <2 x i64> } %1, 1
   store <2 x i64> %2, <2 x i64>* %addr, align 8
   %3 = extractvalue { <2 x i64>, <2 x i64> } %1, 0
   ret <2 x i64> %3
 }
 
 define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_s64(<2 x i64>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrdq_gather_base_wb_z_s64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrdt.u64 q1, [q0, #664]!
+; CHECK-NEXT:    vldrdt.u64 q0, [q1, #664]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <2 x i64>, <2 x i64>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 664, <4 x i1> %2)
   %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1
   store <2 x i64> %4, <2 x i64>* %addr, align 8
   %5 = extractvalue { <2 x i64>, <2 x i64> } %3, 0
   ret <2 x i64> %5
 }
 
 declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <4 x i1>)
 
 define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_u64(<2 x i64>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrdq_gather_base_wb_z_u64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrdt.u64 q1, [q0, #656]!
+; CHECK-NEXT:    vldrdt.u64 q0, [q1, #656]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <2 x i64>, <2 x i64>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 656, <4 x i1> %2)
   %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1

 entry:
   %0 = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> %addr, i32 284)
   ret <4 x i32> %0
 }
 
 define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_f32(<4 x i32>* %addr) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [q0, #-64]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1, #-64]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %0, i32 -64)
   %2 = extractvalue { <4 x float>, <4 x i32> } %1, 1
   store <4 x i32> %2, <4 x i32>* %addr, align 8
   %3 = extractvalue { <4 x float>, <4 x i32> } %1, 0
   ret <4 x float> %3
 }
 
 declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32>, i32)
 
 define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_s32(<4 x i32>* %addr) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [q0, #80]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1, #80]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 80)
   %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
   store <4 x i32> %2, <4 x i32>* %addr, align 8
   %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
   ret <4 x i32> %3
 }
 
 declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32>, i32)
 
 define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_u32(<4 x i32>* %addr) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_u32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [q0, #480]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1, #480]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 480)
   %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
   store <4 x i32> %2, <4 x i32>* %addr, align 8
   %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
   ret <4 x i32> %3
 }
 
 define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_z_f32(<4 x i32>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_z_f32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q1, [q0, #-352]!
+; CHECK-NEXT:    vldrwt.u32 q0, [q1, #-352]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %0, i32 -352, <4 x i1> %2)
   %4 = extractvalue { <4 x float>, <4 x i32> } %3, 1
   store <4 x i32> %4, <4 x i32>* %addr, align 8
   %5 = extractvalue { <4 x float>, <4 x i32> } %3, 0
   ret <4 x float> %5
 }
 
 declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
 
 define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_z_s32(<4 x i32>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_z_s32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q1, [q0, #276]!
+; CHECK-NEXT:    vldrwt.u32 q0, [q1, #276]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 276, <4 x i1> %2)
   %4 = extractvalue { <4 x i32>, <4 x i32> } %3, 1
   store <4 x i32> %4, <4 x i32>* %addr, align 8
   %5 = extractvalue { <4 x i32>, <4 x i32> } %3, 0
   ret <4 x i32> %5
 }
 
 declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>)
 
 define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_z_u32(<4 x i32>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_z_u32:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrwt.u32 q1, [q0, #88]!
+; CHECK-NEXT:    vldrwt.u32 q0, [q1, #88]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> %0, i32 88, <4 x i1> %2)
   %4 = extractvalue { <4 x i32>, <4 x i32> } %3, 1

llvm/test/CodeGen/Thumb2/mve-intrinsics/vldr.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp -verify-machineinstrs -o - %s | FileCheck %s
 
 define arm_aapcs_vfpcc <4 x i32> @test_vldrwq_gather_base_wb_s32(<4 x i32>* %addr) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_s32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [q0, #80]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1, #80]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = tail call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> %0, i32 80)
   %2 = extractvalue { <4 x i32>, <4 x i32> } %1, 1
   store <4 x i32> %2, <4 x i32>* %addr, align 8
   %3 = extractvalue { <4 x i32>, <4 x i32> } %1, 0
   ret <4 x i32> %3
 }
 
 declare { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32>, i32)
 
 define arm_aapcs_vfpcc <4 x float> @test_vldrwq_gather_base_wb_f32(<4 x i32>* %addr) {
 ; CHECK-LABEL: test_vldrwq_gather_base_wb_f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
-; CHECK-NEXT:    vldrw.u32 q1, [q0, #64]!
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q0, [q1, #64]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %addr, align 8
   %1 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> %0, i32 64)
   %2 = extractvalue { <4 x float>, <4 x i32> } %1, 1
   store <4 x i32> %2, <4 x i32>* %addr, align 8
   %3 = extractvalue { <4 x float>, <4 x i32> } %1, 0
   ret <4 x float> %3
 }
 
 declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32>, i32)
 
 define arm_aapcs_vfpcc <2 x i64> @test_vldrdq_gather_base_wb_z_u64(<2 x i64>* %addr, i16 zeroext %p) {
 ; CHECK-LABEL: test_vldrdq_gather_base_wb_z_u64:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    vmsr p0, r1
-; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vpst
-; CHECK-NEXT:    vldrdt.u64 q1, [q0, #656]!
+; CHECK-NEXT:    vldrdt.u64 q0, [q1, #656]!
 ; CHECK-NEXT:    vstrw.32 q1, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <2 x i64>, <2 x i64>* %addr, align 8
   %1 = zext i16 %p to i32
   %2 = tail call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 %1)
   %3 = tail call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64> %0, i32 656, <4 x i1> %2)
   %4 = extractvalue { <2 x i64>, <2 x i64> } %3, 1
   store <2 x i64> %4, <2 x i64>* %addr, align 8
   %5 = extractvalue { <2 x i64>, <2 x i64> } %3, 0
   ret <2 x i64> %5
 }
 
 declare <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32)
 
 declare { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v4i1(<2 x i64>, i32, <4 x i1>)