diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp --- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp +++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp @@ -336,31 +336,58 @@ { ARM::VST1LNq8Pseudo_UPD, ARM::VST1LNd8_UPD, false, true, true, EvenDblSpc, 1, 8 ,true}, { ARM::VST1d16QPseudo, ARM::VST1d16Q, false, false, false, SingleSpc, 4, 4 ,false}, +{ ARM::VST1d16QPseudoWB_fixed, ARM::VST1d16Qwb_fixed, false, true, false, SingleSpc, 4, 4 ,false}, +{ ARM::VST1d16QPseudoWB_register, ARM::VST1d16Qwb_register, false, true, true, SingleSpc, 4, 4 ,false}, { ARM::VST1d16TPseudo, ARM::VST1d16T, false, false, false, SingleSpc, 3, 4 ,false}, +{ ARM::VST1d16TPseudoWB_fixed, ARM::VST1d16Twb_fixed, false, true, false, SingleSpc, 3, 4 ,false}, +{ ARM::VST1d16TPseudoWB_register, ARM::VST1d16Twb_register, false, true, true, SingleSpc, 3, 4 ,false}, + { ARM::VST1d32QPseudo, ARM::VST1d32Q, false, false, false, SingleSpc, 4, 2 ,false}, +{ ARM::VST1d32QPseudoWB_fixed, ARM::VST1d32Qwb_fixed, false, true, false, SingleSpc, 4, 2 ,false}, +{ ARM::VST1d32QPseudoWB_register, ARM::VST1d32Qwb_register, false, true, true, SingleSpc, 4, 2 ,false}, { ARM::VST1d32TPseudo, ARM::VST1d32T, false, false, false, SingleSpc, 3, 2 ,false}, +{ ARM::VST1d32TPseudoWB_fixed, ARM::VST1d32Twb_fixed, false, true, false, SingleSpc, 3, 2 ,false}, +{ ARM::VST1d32TPseudoWB_register, ARM::VST1d32Twb_register, false, true, true, SingleSpc, 3, 2 ,false}, + { ARM::VST1d64QPseudo, ARM::VST1d64Q, false, false, false, SingleSpc, 4, 1 ,false}, { ARM::VST1d64QPseudoWB_fixed, ARM::VST1d64Qwb_fixed, false, true, false, SingleSpc, 4, 1 ,false}, { ARM::VST1d64QPseudoWB_register, ARM::VST1d64Qwb_register, false, true, true, SingleSpc, 4, 1 ,false}, { ARM::VST1d64TPseudo, ARM::VST1d64T, false, false, false, SingleSpc, 3, 1 ,false}, { ARM::VST1d64TPseudoWB_fixed, ARM::VST1d64Twb_fixed, false, true, false, SingleSpc, 3, 1 ,false}, -{ ARM::VST1d64TPseudoWB_register, ARM::VST1d64Twb_register, false, true, true, SingleSpc, 3, 1 ,false}, +{ ARM::VST1d64TPseudoWB_register, ARM::VST1d64Twb_register, false, true, true, SingleSpc, 3, 1 ,false}, + { ARM::VST1d8QPseudo, ARM::VST1d8Q, false, false, false, SingleSpc, 4, 8 ,false}, +{ ARM::VST1d8QPseudoWB_fixed, ARM::VST1d8Qwb_fixed, false, true, false, SingleSpc, 4, 8 ,false}, +{ ARM::VST1d8QPseudoWB_register, ARM::VST1d8Qwb_register, false, true, true, SingleSpc, 4, 8 ,false}, { ARM::VST1d8TPseudo, ARM::VST1d8T, false, false, false, SingleSpc, 3, 8 ,false}, -{ ARM::VST1q16HighQPseudo, ARM::VST1d16Q, false, false, false, SingleHighQSpc, 4, 4 ,false}, -{ ARM::VST1q16HighTPseudo, ARM::VST1d16T, false, false, false, SingleHighTSpc, 3, 4 ,false}, +{ ARM::VST1d8TPseudoWB_fixed, ARM::VST1d8Twb_fixed, false, true, false, SingleSpc, 3, 8 ,false}, +{ ARM::VST1d8TPseudoWB_register, ARM::VST1d8Twb_register, false, true, true, SingleSpc, 3, 8 ,false}, + +{ ARM::VST1q16HighQPseudo, ARM::VST1d16Q, false, false, false, SingleHighQSpc, 4, 4 ,false}, +{ ARM::VST1q16HighQPseudo_UPD, ARM::VST1d16Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false}, +{ ARM::VST1q16HighTPseudo, ARM::VST1d16T, false, false, false, SingleHighTSpc, 3, 4 ,false}, +{ ARM::VST1q16HighTPseudo_UPD, ARM::VST1d16Twb_fixed, false, true, true, SingleHighTSpc, 3, 4 ,false}, { ARM::VST1q16LowQPseudo_UPD, ARM::VST1d16Qwb_fixed, false, true, true, SingleLowSpc, 4, 4 ,false}, { ARM::VST1q16LowTPseudo_UPD, ARM::VST1d16Twb_fixed, false, true, true, SingleLowSpc, 3, 4 ,false}, -{ ARM::VST1q32HighQPseudo, ARM::VST1d32Q, false, false, false, SingleHighQSpc, 4, 2 ,false}, -{ ARM::VST1q32HighTPseudo, ARM::VST1d32T, false, false, false, SingleHighTSpc, 3, 2 ,false}, + +{ ARM::VST1q32HighQPseudo, ARM::VST1d32Q, false, false, false, SingleHighQSpc, 4, 2 ,false}, +{ ARM::VST1q32HighQPseudo_UPD, ARM::VST1d32Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false}, +{ ARM::VST1q32HighTPseudo, ARM::VST1d32T, false, false, false, SingleHighTSpc, 3, 2 ,false}, +{ ARM::VST1q32HighTPseudo_UPD, ARM::VST1d32Twb_fixed, false, true, true, SingleHighTSpc, 3, 2 ,false}, { ARM::VST1q32LowQPseudo_UPD, ARM::VST1d32Qwb_fixed, false, true, true, SingleLowSpc, 4, 2 ,false}, { ARM::VST1q32LowTPseudo_UPD, ARM::VST1d32Twb_fixed, false, true, true, SingleLowSpc, 3, 2 ,false}, -{ ARM::VST1q64HighQPseudo, ARM::VST1d64Q, false, false, false, SingleHighQSpc, 4, 1 ,false}, -{ ARM::VST1q64HighTPseudo, ARM::VST1d64T, false, false, false, SingleHighTSpc, 3, 1 ,false}, + +{ ARM::VST1q64HighQPseudo, ARM::VST1d64Q, false, false, false, SingleHighQSpc, 4, 1 ,false}, +{ ARM::VST1q64HighQPseudo_UPD, ARM::VST1d64Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false}, +{ ARM::VST1q64HighTPseudo, ARM::VST1d64T, false, false, false, SingleHighTSpc, 3, 1 ,false}, +{ ARM::VST1q64HighTPseudo_UPD, ARM::VST1d64Twb_fixed, false, true, true, SingleHighTSpc, 3, 1 ,false}, { ARM::VST1q64LowQPseudo_UPD, ARM::VST1d64Qwb_fixed, false, true, true, SingleLowSpc, 4, 1 ,false}, { ARM::VST1q64LowTPseudo_UPD, ARM::VST1d64Twb_fixed, false, true, true, SingleLowSpc, 3, 1 ,false}, + { ARM::VST1q8HighQPseudo, ARM::VST1d8Q, false, false, false, SingleHighQSpc, 4, 8 ,false}, +{ ARM::VST1q8HighQPseudo_UPD, ARM::VST1d8Qwb_fixed, false, true, true, SingleHighQSpc, 4, 8 ,false}, { ARM::VST1q8HighTPseudo, ARM::VST1d8T, false, false, false, SingleHighTSpc, 3, 8 ,false}, +{ ARM::VST1q8HighTPseudo_UPD, ARM::VST1d8Twb_fixed, false, true, true, SingleHighTSpc, 3, 8 ,false}, { ARM::VST1q8LowQPseudo_UPD, ARM::VST1d8Qwb_fixed, false, true, true, SingleLowSpc, 4, 8 ,false}, { ARM::VST1q8LowTPseudo_UPD, ARM::VST1d8Twb_fixed, false, true, true, SingleLowSpc, 3, 8 ,false}, @@ -2651,14 +2678,20 @@ case ARM::VST3d16Pseudo: case ARM::VST3d32Pseudo: case ARM::VST1d8TPseudo: + case ARM::VST1d8TPseudoWB_fixed: + case ARM::VST1d8TPseudoWB_register: case ARM::VST1d16TPseudo: + case ARM::VST1d16TPseudoWB_fixed: + case ARM::VST1d16TPseudoWB_register: case ARM::VST1d32TPseudo: + case ARM::VST1d32TPseudoWB_fixed: + case ARM::VST1d32TPseudoWB_register: case ARM::VST1d64TPseudo: + case ARM::VST1d64TPseudoWB_fixed: + case ARM::VST1d64TPseudoWB_register: case ARM::VST3d8Pseudo_UPD: case ARM::VST3d16Pseudo_UPD: case ARM::VST3d32Pseudo_UPD: - case ARM::VST1d64TPseudoWB_fixed: - case ARM::VST1d64TPseudoWB_register: case ARM::VST3q8Pseudo_UPD: case ARM::VST3q16Pseudo_UPD: case ARM::VST3q32Pseudo_UPD: @@ -2672,14 +2705,20 @@ case ARM::VST4d16Pseudo: case ARM::VST4d32Pseudo: case ARM::VST1d8QPseudo: + case ARM::VST1d8QPseudoWB_fixed: + case ARM::VST1d8QPseudoWB_register: case ARM::VST1d16QPseudo: + case ARM::VST1d16QPseudoWB_fixed: + case ARM::VST1d16QPseudoWB_register: case ARM::VST1d32QPseudo: + case ARM::VST1d32QPseudoWB_fixed: + case ARM::VST1d32QPseudoWB_register: case ARM::VST1d64QPseudo: + case ARM::VST1d64QPseudoWB_fixed: + case ARM::VST1d64QPseudoWB_register: case ARM::VST4d8Pseudo_UPD: case ARM::VST4d16Pseudo_UPD: case ARM::VST4d32Pseudo_UPD: - case ARM::VST1d64QPseudoWB_fixed: - case ARM::VST1d64QPseudoWB_register: case ARM::VST1q8HighQPseudo: case ARM::VST1q8LowQPseudo_UPD: case ARM::VST1q8HighTPseudo: @@ -2696,6 +2735,14 @@ case ARM::VST1q64LowQPseudo_UPD: case ARM::VST1q64HighTPseudo: case ARM::VST1q64LowTPseudo_UPD: + case ARM::VST1q8HighTPseudo_UPD: + case ARM::VST1q16HighTPseudo_UPD: + case ARM::VST1q32HighTPseudo_UPD: + case ARM::VST1q64HighTPseudo_UPD: + case ARM::VST1q8HighQPseudo_UPD: + case ARM::VST1q16HighQPseudo_UPD: + case ARM::VST1q32HighQPseudo_UPD: + case ARM::VST1q64HighQPseudo_UPD: case ARM::VST4q8Pseudo_UPD: case ARM::VST4q16Pseudo_UPD: case ARM::VST4q32Pseudo_UPD: diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp --- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -1977,7 +1977,13 @@ case ARM::VST1q16wb_fixed : return true; case ARM::VST1q32wb_fixed : return true; case ARM::VST1q64wb_fixed : return true; + case ARM::VST1d8TPseudoWB_fixed : return true; + case ARM::VST1d16TPseudoWB_fixed : return true; + case ARM::VST1d32TPseudoWB_fixed : return true; case ARM::VST1d64TPseudoWB_fixed : return true; + case ARM::VST1d8QPseudoWB_fixed : return true; + case ARM::VST1d16QPseudoWB_fixed : return true; + case ARM::VST1d32QPseudoWB_fixed : return true; case ARM::VST1d64QPseudoWB_fixed : return true; case ARM::VST2d8wb_fixed : return true; case ARM::VST2d16wb_fixed : return true; @@ -2022,7 +2028,13 @@ case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register; case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register; case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register; + case ARM::VST1d8TPseudoWB_fixed: return ARM::VST1d8TPseudoWB_register; + case ARM::VST1d16TPseudoWB_fixed: return ARM::VST1d16TPseudoWB_register; + case ARM::VST1d32TPseudoWB_fixed: return ARM::VST1d32TPseudoWB_register; case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register; + case ARM::VST1d8QPseudoWB_fixed: return ARM::VST1d8QPseudoWB_register; + case ARM::VST1d16QPseudoWB_fixed: return ARM::VST1d16QPseudoWB_register; + case ARM::VST1d32QPseudoWB_fixed: return ARM::VST1d32QPseudoWB_register; case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register; case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register; @@ -4354,6 +4366,61 @@ break; } + case ARMISD::VST1x2_UPD: { + if (Subtarget->hasNEON()) { + static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed, + ARM::VST1q16wb_fixed, + ARM::VST1q32wb_fixed, + ARM::VST1q64wb_fixed}; + static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed, + ARM::VST1d16QPseudoWB_fixed, + ARM::VST1d32QPseudoWB_fixed, + ARM::VST1d64QPseudoWB_fixed }; + SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr); + return; + } + break; + } + + case ARMISD::VST1x3_UPD: { + if (Subtarget->hasNEON()) { + static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed, + ARM::VST1d16TPseudoWB_fixed, + ARM::VST1d32TPseudoWB_fixed, + ARM::VST1d64TPseudoWB_fixed }; + static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD, + ARM::VST1q16LowTPseudo_UPD, + ARM::VST1q32LowTPseudo_UPD, + ARM::VST1q64LowTPseudo_UPD }; + static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD, + ARM::VST1q16HighTPseudo_UPD, + ARM::VST1q32HighTPseudo_UPD, + ARM::VST1q64HighTPseudo_UPD }; + SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1); + return; + } + break; + } + + case ARMISD::VST1x4_UPD: { + if (Subtarget->hasNEON()) { + static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed, + ARM::VST1d16QPseudoWB_fixed, + ARM::VST1d32QPseudoWB_fixed, + ARM::VST1d64QPseudoWB_fixed }; + static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD, + ARM::VST1q16LowQPseudo_UPD, + ARM::VST1q32LowQPseudo_UPD, + ARM::VST1q64LowQPseudo_UPD }; + static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD, + ARM::VST1q16HighQPseudo_UPD, + ARM::VST1q32HighQPseudo_UPD, + ARM::VST1q64HighQPseudo_UPD }; + SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1); + return; + } + break; + } case ARMISD::VST2LN_UPD: { static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD, ARM::VST2LNd16Pseudo_UPD, diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -339,6 +339,9 @@ VST2LN_UPD, VST3LN_UPD, VST4LN_UPD, + VST1x2_UPD, + VST1x3_UPD, + VST1x4_UPD, // Load/Store of dual registers LDRD, diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -1792,6 +1792,9 @@ MAKE_CASE(ARMISD::VST2_UPD) MAKE_CASE(ARMISD::VST3_UPD) MAKE_CASE(ARMISD::VST4_UPD) + MAKE_CASE(ARMISD::VST1x2_UPD) + MAKE_CASE(ARMISD::VST1x3_UPD) + MAKE_CASE(ARMISD::VST1x4_UPD) MAKE_CASE(ARMISD::VST2LN_UPD) MAKE_CASE(ARMISD::VST3LN_UPD) MAKE_CASE(ARMISD::VST4LN_UPD) @@ -14624,6 +14627,8 @@ // Find the new opcode for the updating load/store. bool isLoadOp = true; bool isLaneOp = false; + // Workaround for vst1x and vld1x which do not have alignment operand. + bool hasAlignment = true; unsigned NewOpc = 0; unsigned NumVecs = 0; if (isIntrinsic) { @@ -14641,9 +14646,6 @@ case Intrinsic::arm_neon_vld1x2: case Intrinsic::arm_neon_vld1x3: case Intrinsic::arm_neon_vld1x4: - case Intrinsic::arm_neon_vst1x2: - case Intrinsic::arm_neon_vst1x3: - case Intrinsic::arm_neon_vst1x4: case Intrinsic::arm_neon_vld2dup: case Intrinsic::arm_neon_vld3dup: case Intrinsic::arm_neon_vld4dup: @@ -14670,6 +14672,12 @@ NumVecs = 3; isLoadOp = false; isLaneOp = true; break; case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD; NumVecs = 4; isLoadOp = false; isLaneOp = true; break; + case Intrinsic::arm_neon_vst1x2: NewOpc = ARMISD::VST1x2_UPD; + NumVecs = 2; isLoadOp = false; hasAlignment = false; break; + case Intrinsic::arm_neon_vst1x3: NewOpc = ARMISD::VST1x3_UPD; + NumVecs = 3; isLoadOp = false; hasAlignment = false; break; + case Intrinsic::arm_neon_vst1x4: NewOpc = ARMISD::VST1x4_UPD; + NumVecs = 4; isLoadOp = false; hasAlignment = false; break; } } else { isLaneOp = true; @@ -14773,7 +14781,9 @@ } else { // Loads (and of course intrinsics) match the intrinsics' signature, // so just add all but the alignment operand. - for (unsigned i = AddrOpIdx + 1; i < N->getNumOperands() - 1; ++i) + unsigned LastOperand = + hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands(); + for (unsigned i = AddrOpIdx + 1; i < LastOperand; ++i) Ops.push_back(N->getOperand(i)); } diff --git a/llvm/lib/Target/ARM/ARMInstrNEON.td b/llvm/lib/Target/ARM/ARMInstrNEON.td --- a/llvm/lib/Target/ARM/ARMInstrNEON.td +++ b/llvm/lib/Target/ARM/ARMInstrNEON.td @@ -1776,19 +1776,31 @@ defm VST1d64Twb : VST1D3WB<{1,1,0,?}, "64", addrmode6align64>; def VST1d8TPseudo : VSTQQPseudo, Sched<[WriteVST3]>; +def VST1d8TPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST3]>; +def VST1d8TPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST3]>; def VST1d16TPseudo : VSTQQPseudo, Sched<[WriteVST3]>; +def VST1d16TPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST3]>; +def VST1d16TPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST3]>; def VST1d32TPseudo : VSTQQPseudo, Sched<[WriteVST3]>; +def VST1d32TPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST3]>; +def VST1d32TPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST3]>; def VST1d64TPseudo : VSTQQPseudo, Sched<[WriteVST3]>; def VST1d64TPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST3]>; def VST1d64TPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST3]>; def VST1q8HighTPseudo : VSTQQQQPseudo, Sched<[WriteVST3]>; -def VST1q8LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; def VST1q16HighTPseudo : VSTQQQQPseudo, Sched<[WriteVST3]>; -def VST1q16LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; def VST1q32HighTPseudo : VSTQQQQPseudo, Sched<[WriteVST3]>; -def VST1q32LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; def VST1q64HighTPseudo : VSTQQQQPseudo, Sched<[WriteVST3]>; + +def VST1q8HighTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; +def VST1q16HighTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; +def VST1q32HighTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; +def VST1q64HighTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; + +def VST1q8LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; +def VST1q16LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; +def VST1q32LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; def VST1q64LowTPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST3]>; // ...with 4 registers @@ -1831,19 +1843,31 @@ defm VST1d64Qwb : VST1D4WB<{1,1,?,?}, "64", addrmode6align64or128or256>; def VST1d8QPseudo : VSTQQPseudo, Sched<[WriteVST4]>; +def VST1d8QPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST4]>; +def VST1d8QPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST4]>; def VST1d16QPseudo : VSTQQPseudo, Sched<[WriteVST4]>; +def VST1d16QPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST4]>; +def VST1d16QPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST4]>; def VST1d32QPseudo : VSTQQPseudo, Sched<[WriteVST4]>; +def VST1d32QPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST4]>; +def VST1d32QPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST4]>; def VST1d64QPseudo : VSTQQPseudo, Sched<[WriteVST4]>; def VST1d64QPseudoWB_fixed : VSTQQWBfixedPseudo, Sched<[WriteVST4]>; def VST1d64QPseudoWB_register : VSTQQWBPseudo, Sched<[WriteVST4]>; def VST1q8HighQPseudo : VSTQQQQPseudo, Sched<[WriteVST4]>; -def VST1q8LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; def VST1q16HighQPseudo : VSTQQQQPseudo, Sched<[WriteVST4]>; -def VST1q16LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; def VST1q32HighQPseudo : VSTQQQQPseudo, Sched<[WriteVST4]>; -def VST1q32LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; def VST1q64HighQPseudo : VSTQQQQPseudo, Sched<[WriteVST4]>; + +def VST1q8HighQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; +def VST1q16HighQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; +def VST1q32HighQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; +def VST1q64HighQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; + +def VST1q8LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; +def VST1q16LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; +def VST1q32LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; def VST1q64LowQPseudo_UPD : VSTQQQQWBPseudo, Sched<[WriteVST4]>; // VST2 : Vector Store (multiple 2-element structures) diff --git a/llvm/test/CodeGen/ARM/arm-vst1.ll b/llvm/test/CodeGen/ARM/arm-vst1.ll --- a/llvm/test/CodeGen/ARM/arm-vst1.ll +++ b/llvm/test/CodeGen/ARM/arm-vst1.ll @@ -386,92 +386,535 @@ ret void } -define void @postinc_1x2(i8* nocapture %0, i8* %1) { -; CHECK-LABEL: postinc_1x2: -; CHECK: vld1.8 {d16, d17, d18, d19}, [r1:256] -; CHECK-NEXT: add r1, r1, #32 -; CHECK-NEXT: vst1.8 {d16, d17, d18, d19}, [r0:256] -; CHECK-NEXT: add r0, r0, #32 -; CHECK-NEXT: vld1.8 {d16, d17, d18, d19}, [r1:256] -; CHECK-NEXT: vst1.8 {d16, d17, d18, d19}, [r0:256] -; CHECK-NEXT: bx lr - %3 = tail call { <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x2.v16i8.p0i8(i8* %1) - %4 = extractvalue { <16 x i8>, <16 x i8> } %3, 0 - %5 = extractvalue { <16 x i8>, <16 x i8> } %3, 1 - tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* %0, <16 x i8> %4, <16 x i8> %5) - %6 = getelementptr inbounds i8, i8* %1, i32 32 - %7 = tail call { <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x2.v16i8.p0i8(i8* nonnull %6) - %8 = extractvalue { <16 x i8>, <16 x i8> } %7, 0 - %9 = extractvalue { <16 x i8>, <16 x i8> } %7, 1 - %10 = getelementptr inbounds i8, i8* %0, i32 32 - tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* nonnull %10, <16 x i8> %8, <16 x i8> %9) - ret void +; Post increment + +define arm_aapcs_vfpcc i8* @test_vst1_u8_x2_post_imm(i8* %a, %struct.uint8x8x2_t %b) nounwind { +; CHECK-LABEL: test_vst1_u8_x2_post_imm: +; CHECK: vst1.8 {d0, d1}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1) + %tmp = getelementptr i8, i8* %a, i32 16 + ret i8* %tmp } -declare { <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x2.v16i8.p0i8(i8*) - -define void @postinc_1x3(i8* nocapture %0, i8* %1) { -; CHECK-LABEL: postinc_1x3: -; CHECK: add r2, r1, #48 -; CHECK-NEXT: vld1.8 {d16, d17, d18}, [r1:64]! -; CHECK-NEXT: vld1.8 {d19, d20, d21}, [r1:64] -; CHECK-NEXT: add r1, r0, #48 -; CHECK-NEXT: vst1.8 {d16, d17, d18}, [r0:64]! -; CHECK-NEXT: vst1.8 {d19, d20, d21}, [r0:64] -; CHECK-NEXT: vld1.8 {d16, d17, d18}, [r2:64]! -; CHECK-NEXT: vld1.8 {d19, d20, d21}, [r2:64] -; CHECK-NEXT: vst1.8 {d16, d17, d18}, [r1:64]! -; CHECK-NEXT: vst1.8 {d19, d20, d21}, [r1:64] -; CHECK-NEXT: bx lr - %3 = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x3.v16i8.p0i8(i8* %1) - %4 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %3, 0 - %5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %3, 1 - %a5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %3, 2 - tail call void @llvm.arm.neon.vst1x3.p0i8.v16i8(i8* %0, <16 x i8> %4, <16 x i8> %5, <16 x i8> %a5) - %6 = getelementptr inbounds i8, i8* %1, i32 48 - %7 = tail call { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x3.v16i8.p0i8(i8* nonnull %6) - %8 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %7, 0 - %9 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %7, 1 - %a9 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8> } %7, 2 - %10 = getelementptr inbounds i8, i8* %0, i32 48 - tail call void @llvm.arm.neon.vst1x3.p0i8.v16i8(i8* nonnull %10, <16 x i8> %8, <16 x i8> %9, <16 x i8> %a9) - ret void +define arm_aapcs_vfpcc i8* @test_vst1_u8_x2_post_reg(i8* %a, %struct.uint8x8x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u8_x2_post_reg: +; CHECK: vst1.8 {d0, d1}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1) + %tmp = getelementptr i8, i8* %a, i32 %inc + ret i8* %tmp } -declare { <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x3.v16i8.p0i8(i8*) - -define void @postinc_1x4(i8* nocapture %0, i8* %1) { -; CHECK-LABEL: postinc_1x4: -; CHECK: add r2, r1, #64 -; CHECK-NEXT: vld1.8 {d16, d17, d18, d19}, [r1:256]! -; CHECK-NEXT: vld1.8 {d20, d21, d22, d23}, [r1:256] -; CHECK-NEXT: add r1, r0, #64 -; CHECK-NEXT: vst1.8 {d16, d17, d18, d19}, [r0:256]! -; CHECK-NEXT: vst1.8 {d20, d21, d22, d23}, [r0:256] -; CHECK-NEXT: vld1.8 {d16, d17, d18, d19}, [r2:256]! -; CHECK-NEXT: vld1.8 {d20, d21, d22, d23}, [r2:256] -; CHECK-NEXT: vorr q15, q11, q11 -; CHECK-NEXT: vorr q14, q10, q10 -; CHECK-NEXT: vorr q13, q9, q9 -; CHECK-NEXT: vorr q12, q8, q8 -; CHECK-NEXT: vst1.8 {d24, d25, d26, d27}, [r1:256]! -; CHECK-NEXT: vst1.8 {d28, d29, d30, d31}, [r1:256] -; CHECK-NEXT: bx lr - %3 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x4.v16i8.p0i8(i8* %1) - %4 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %3, 0 - %5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %3, 1 - %6 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %3, 2 - %7 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %3, 3 - tail call void @llvm.arm.neon.vst1x4.p0i8.v16i8(i8* %0, <16 x i8> %4, <16 x i8> %5, <16 x i8> %6, <16 x i8> %7) - %8 = getelementptr inbounds i8, i8* %1, i32 64 - %9 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x4.v16i8.p0i8(i8* nonnull %8) - %10 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %9, 0 - %11 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %9, 1 - %12 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %9, 2 - %13 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %9, 3 - %14 = getelementptr inbounds i8, i8* %0, i32 64 - tail call void @llvm.arm.neon.vst1x4.p0i8.v16i8(i8* nonnull %14, <16 x i8> %10, <16 x i8> %11, <16 x i8> %12, <16 x i8> %13) - ret void +define arm_aapcs_vfpcc i16* @test_vst1_u16_x2_post_imm(i16* %a, %struct.uint16x4x2_t %b) nounwind { +; CHECK-LABEL: test_vst1_u16_x2_post_imm: +; CHECK: vst1.16 {d0, d1}, [r0:64]! +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1) + %tmp = getelementptr i16, i16* %a, i32 8 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1_u16_x2_post_reg(i16* %a, %struct.uint16x4x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u16_x2_post_reg: +; CHECK: lsl r1, r1, #1 +; CHECK-NEXT: vst1.16 {d0, d1}, [r0:64], r1 +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1) + %tmp = getelementptr i16, i16* %a, i32 %inc + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x2_post_imm(i32* %a, %struct.uint32x2x2_t %b) nounwind { +; CHECK-LABEL: test_vst1_u32_x2_post_imm: +; CHECK: vst1.32 {d0, d1}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1) + %tmp = getelementptr i32, i32* %a, i32 4 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x2_post_reg(i32* %a, %struct.uint32x2x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u32_x2_post_reg: +; CHECK: lsl r1, r1, #2 +; CHECK-NEXT: vst1.32 {d0, d1}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1) + %tmp = getelementptr i32, i32* %a, i32 %inc + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x2_post_imm(i64* %a, %struct.uint64x1x2_t %b) nounwind { +; CHECK-LABEL: test_vst1_u64_x2_post_imm: +; CHECK: vst1.64 {d0, d1}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1) + %tmp = getelementptr i64, i64* %a, i32 2 + ret i64* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x2_post_reg(i64* %a, %struct.uint64x1x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u64_x2_post_reg: +; CHECK: lsl r1, r1, #3 +; CHECK-NEXT: vst1.64 {d0, d1}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1) + %tmp = getelementptr i64, i64* %a, i32 %inc + ret i64* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1q_u8_x2_post_imm(i8* %a, %struct.uint8x16x2_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u8_x2_post_imm: +; CHECK: vst1.8 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x16x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x16x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1) + %tmp = getelementptr i8, i8* %a, i32 32 + ret i8* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1q_u8_x2_post_reg(i8* %a, %struct.uint8x16x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1q_u8_x2_post_reg: +; CHECK: vst1.8 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x16x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x16x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1) + %tmp = getelementptr i8, i8* %a, i32 %inc + ret i8* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1q_u16_x2_post_imm(i16* %a, %struct.uint16x8x2_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u16_x2_post_imm: +; CHECK: vst1.16 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint16x8x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x8x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1) + %tmp = getelementptr i16, i16* %a, i32 16 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1q_u16_x2_post_reg(i16* %a, %struct.uint16x8x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1q_u16_x2_post_reg: +; CHECK: lsl r1, r1, #1 +; CHECK-NEXT: vst1.16 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint16x8x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x8x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1) + %tmp = getelementptr i16, i16* %a, i32 %inc + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1q_u32_x2_post_imm(i32* %a, %struct.uint32x4x2_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u32_x2_post_imm: +; CHECK: vst1.32 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x4x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x4x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1) + %tmp = getelementptr i32, i32* %a, i32 8 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1q_u32_x2_post_reg(i32* %a, %struct.uint32x4x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1q_u32_x2_post_reg: +; CHECK: lsl r1, r1, #2 +; CHECK-NEXT: vst1.32 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x4x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x4x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1) + %tmp = getelementptr i32, i32* %a, i32 %inc + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1q_u64_x2_post_imm(i64* %a, %struct.uint64x2x2_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u64_x2_post_imm: +; CHECK: vst1.64 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x2x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x2x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1) + %tmp = getelementptr i64, i64* %a, i32 4 + ret i64* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1q_u64_x2_post_reg(i64* %a, %struct.uint64x2x2_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1q_u64_x2_post_reg: +; CHECK: lsl r1, r1, #3 +; CHECK-NEXT: vst1.64 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x2x2_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x2x2_t %b, 0, 1 + tail call void @llvm.arm.neon.vst1x2.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1) + %tmp = getelementptr i64, i64* %a, i32 %inc + ret i64* %tmp +} + + +define arm_aapcs_vfpcc i8* @test_vst1_u8_x3_post_imm(i8* %a, %struct.uint8x8x3_t %b) nounwind { +; CHECK-LABEL: test_vst1_u8_x3_post_imm: +; CHECK: vst1.8 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x8x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2) + %tmp = getelementptr i8, i8* %a, i32 24 + ret i8* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1_u8_x3_post_reg(i8* %a, %struct.uint8x8x3_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u8_x3_post_reg: +; CHECK: vst1.8 {d0, d1, d2}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x8x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2) + %tmp = getelementptr i8, i8* %a, i32 %inc + ret i8* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1_u16_x3_post_imm(i16* %a, %struct.uint16x4x3_t %b) nounwind { +; CHECK-LABEL: test_vst1_u16_x3_post_imm: +; CHECK: vst1.16 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x4x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2) + %tmp = getelementptr i16, i16* %a, i32 12 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1_u16_x3_post_reg(i16* %a, %struct.uint16x4x3_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u16_x3_post_reg: +; CHECK: lsl r1, r1, #1 +; CHECK-NEXT: vst1.16 {d0, d1, d2}, [r0:64], r1 +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x4x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2) + %tmp = getelementptr i16, i16* %a, i32 %inc + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x3_post_imm(i32* %a, %struct.uint32x2x3_t %b) nounwind { +; CHECK-LABEL: test_vst1_u32_x3_post_imm: +; CHECK: vst1.32 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x2x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2) + %tmp = getelementptr i32, i32* %a, i32 6 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x3_post_reg(i32* %a, %struct.uint32x2x3_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u32_x3_post_reg: +; CHECK: lsl r1, r1, #2 +; CHECK-NEXT: vst1.32 {d0, d1, d2}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x2x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2) + %tmp = getelementptr i32, i32* %a, i32 %inc + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x3_post_imm(i64* %a, %struct.uint64x1x3_t %b) nounwind { +; CHECK-LABEL: test_vst1_u64_x3_post_imm: +; CHECK: vst1.64 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x1x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2) + %tmp = getelementptr i64, i64* %a, i32 3 + ret i64* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x3_post_reg(i64* %a, %struct.uint64x1x3_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u64_x3_post_reg: +; CHECK: lsl r1, r1, #3 +; CHECK-NEXT: vst1.64 {d0, d1, d2}, [r0:64], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x1x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2) + %tmp = getelementptr i64, i64* %a, i32 %inc + ret i64* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1q_u8_x3_post_imm(i8* %a, %struct.uint8x16x3_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u8_x3_post_imm: +; CHECK: vst1.8 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: vst1.8 {d3, d4, d5}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x16x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x16x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x16x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1, <16 x i8> %b2) + %tmp = getelementptr i8, i8* %a, i32 48 + ret i8* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1q_u16_x3_post_imm(i16* %a, %struct.uint16x8x3_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u16_x3_post_imm: +; CHECK: vst1.16 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: vst1.16 {d3, d4, d5}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint16x8x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x8x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x8x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1, <8 x i16> %b2) + %tmp = getelementptr i16, i16* %a, i32 24 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1q_u32_x3_post_imm(i32* %a, %struct.uint32x4x3_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u32_x3_post_imm: +; CHECK: vst1.32 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: vst1.32 {d3, d4, d5}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x4x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x4x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x4x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1, <4 x i32> %b2) + %tmp = getelementptr i32, i32* %a, i32 12 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1q_u64_x3_post_imm(i64* %a, %struct.uint64x2x3_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u64_x3_post_imm: +; CHECK: vst1.64 {d0, d1, d2}, [r0:64]! +; CHECK-NEXT: vst1.64 {d3, d4, d5}, [r0:64]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x2x3_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x2x3_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x2x3_t %b, 0, 2 + tail call void @llvm.arm.neon.vst1x3.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1, <2 x i64> %b2) + %tmp = getelementptr i64, i64* %a, i32 6 + ret i64* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1_u8_x4_post_imm(i8* %a, %struct.uint8x8x4_t %b) nounwind { +; CHECK-LABEL: test_vst1_u8_x4_post_imm: +; CHECK: vst1.8 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x8x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint8x8x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2, <8 x i8> %b3) + %tmp = getelementptr i8, i8* %a, i32 32 + ret i8* %tmp +} + +define arm_aapcs_vfpcc i8* @test_vst1_u8_x4_post_reg(i8* %a, %struct.uint8x8x4_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u8_x4_post_reg: +; CHECK: vst1.8 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x8x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x8x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x8x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint8x8x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i8.v8i8(i8* %a, <8 x i8> %b0, <8 x i8> %b1, <8 x i8> %b2, <8 x i8> %b3) + %tmp = getelementptr i8, i8* %a, i32 %inc + ret i8* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1_u16_x4_post_imm(i16* %a, %struct.uint16x4x4_t %b) nounwind { +; CHECK-LABEL: test_vst1_u16_x4_post_imm: +; CHECK: vst1.16 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x4x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint16x4x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2, <4 x i16> %b3) + %tmp = getelementptr i16, i16* %a, i32 16 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1_u16_x4_post_reg(i16* %a, %struct.uint16x4x4_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u16_x4_post_reg: +; CHECK: lsl r1, r1, #1 +; CHECK-NEXT: vst1.16 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr + %b0 = extractvalue %struct.uint16x4x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x4x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x4x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint16x4x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i16.v4i16(i16* %a, <4 x i16> %b0, <4 x i16> %b1, <4 x i16> %b2, <4 x i16> %b3) + %tmp = getelementptr i16, i16* %a, i32 %inc + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x4_post_imm(i32* %a, %struct.uint32x2x4_t %b) nounwind { +; CHECK-LABEL: test_vst1_u32_x4_post_imm: +; CHECK: vst1.32 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x2x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint32x2x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2, <2 x i32> %b3) + %tmp = getelementptr i32, i32* %a, i32 8 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1_u32_x4_post_reg(i32* %a, %struct.uint32x2x4_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u32_x4_post_reg: +; CHECK: lsl r1, r1, #2 +; CHECK-NEXT: vst1.32 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x2x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x2x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x2x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint32x2x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i32.v2i32(i32* %a, <2 x i32> %b0, <2 x i32> %b1, <2 x i32> %b2, <2 x i32> %b3) + %tmp = getelementptr i32, i32* %a, i32 %inc + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x4_post_imm(i64* %a, %struct.uint64x1x4_t %b) nounwind { +; CHECK-LABEL: test_vst1_u64_x4_post_imm: +; CHECK: vst1.64 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x1x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint64x1x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2, <1 x i64> %b3) + %tmp = getelementptr i64, i64* %a, i32 4 + ret i64* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1_u64_x4_post_reg(i64* %a, %struct.uint64x1x4_t %b, i32 %inc) nounwind { +; CHECK-LABEL: test_vst1_u64_x4_post_reg: +; CHECK: lsl r1, r1, #3 +; CHECK-NEXT: vst1.64 {d0, d1, d2, d3}, [r0:256], r1 +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x1x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x1x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x1x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint64x1x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i64.v1i64(i64* %a, <1 x i64> %b0, <1 x i64> %b1, <1 x i64> %b2, <1 x i64> %b3) + %tmp = getelementptr i64, i64* %a, i32 %inc + ret i64* %tmp } -declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.arm.neon.vld1x4.v16i8.p0i8(i8*) +define arm_aapcs_vfpcc i8* @test_vst1q_u8_x4_post_imm(i8* %a, %struct.uint8x16x4_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u8_x4_post_imm: +; CHECK: vst1.8 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: vst1.8 {d4, d5, d6, d7}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint8x16x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint8x16x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint8x16x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint8x16x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i8.v16i8(i8* %a, <16 x i8> %b0, <16 x i8> %b1, <16 x i8> %b2, <16 x i8> %b3) + %tmp = getelementptr i8, i8* %a, i32 64 + ret i8* %tmp +} + +define arm_aapcs_vfpcc i16* @test_vst1q_u16_x4_post_imm(i16* %a, %struct.uint16x8x4_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u16_x4_post_imm: +; CHECK: vst1.16 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: vst1.16 {d4, d5, d6, d7}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint16x8x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint16x8x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint16x8x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint16x8x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i16.v8i16(i16* %a, <8 x i16> %b0, <8 x i16> %b1, <8 x i16> %b2, <8 x i16> %b3) + %tmp = getelementptr i16, i16* %a, i32 32 + ret i16* %tmp +} + +define arm_aapcs_vfpcc i32* @test_vst1q_u32_x4_post_imm(i32* %a, %struct.uint32x4x4_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u32_x4_post_imm: +; CHECK: vst1.32 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: vst1.32 {d4, d5, d6, d7}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint32x4x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint32x4x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint32x4x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint32x4x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i32.v4i32(i32* %a, <4 x i32> %b0, <4 x i32> %b1, <4 x i32> %b2, <4 x i32> %b3) + %tmp = getelementptr i32, i32* %a, i32 16 + ret i32* %tmp +} + +define arm_aapcs_vfpcc i64* @test_vst1q_u64_x4_post_imm(i64* %a, %struct.uint64x2x4_t %b) nounwind { +; CHECK-LABEL: test_vst1q_u64_x4_post_imm: +; CHECK: vst1.64 {d0, d1, d2, d3}, [r0:256]! +; CHECK-NEXT: vst1.64 {d4, d5, d6, d7}, [r0:256]! +; CHECK-NEXT: bx lr +entry: + %b0 = extractvalue %struct.uint64x2x4_t %b, 0, 0 + %b1 = extractvalue %struct.uint64x2x4_t %b, 0, 1 + %b2 = extractvalue %struct.uint64x2x4_t %b, 0, 2 + %b3 = extractvalue %struct.uint64x2x4_t %b, 0, 3 + tail call void @llvm.arm.neon.vst1x4.p0i64.v2i64(i64* %a, <2 x i64> %b0, <2 x i64> %b1, <2 x i64> %b2, <2 x i64> %b3) + %tmp = getelementptr i64, i64* %a, i32 8 + ret i64* %tmp +}