diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td --- a/llvm/lib/Target/ARM/ARMInstrMVE.td +++ b/llvm/lib/Target/ARM/ARMInstrMVE.td @@ -5288,65 +5288,7 @@ // Patterns //===----------------------------------------------------------------------===// -class MVE_vector_store_typed - : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7:$addr), - (RegImmInst (Ty MQPR:$val), t2addrmode_imm7:$addr)>; -class MVE_vector_maskedstore_typed - : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7:$addr, VCCR:$pred), - (RegImmInst (Ty MQPR:$val), t2addrmode_imm7:$addr, (i32 1), VCCR:$pred)>; - -multiclass MVE_vector_store { - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; - def : MVE_vector_store_typed; -} - -class MVE_vector_load_typed - : Pat<(Ty (LoadKind t2addrmode_imm7:$addr)), - (Ty (RegImmInst t2addrmode_imm7:$addr))>; -class MVE_vector_maskedload_typed - : Pat<(Ty (LoadKind t2addrmode_imm7:$addr, VCCR:$pred, (Ty NEONimmAllZerosV))), - (Ty (RegImmInst t2addrmode_imm7:$addr, (i32 1), VCCR:$pred))>; - -multiclass MVE_vector_load { - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; - def : MVE_vector_load_typed; -} - -class MVE_vector_offset_store_typed - : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr), - (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr)>; -class MVE_vector_offset_maskedstore_typed - : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr, VCCR:$pred), - (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr, (i32 1), VCCR:$pred)>; - -multiclass MVE_vector_offset_store { - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; - def : MVE_vector_offset_store_typed; -} +// PatFrags for loads and stores. Often trying to keep semi-consistent names. def aligned32_pre_store : PatFrag<(ops node:$val, node:$ptr, node:$offset), (pre_store node:$val, node:$ptr, node:$offset), [{ @@ -5366,79 +5308,69 @@ }]>; -def maskedload8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ +def aligned_maskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ auto *Ld = cast(N); return Ld->getMemoryVT().getScalarType() == MVT::i8; }]>; -def sextmaskedload8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (maskedload8 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_sextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{ return cast(N)->getExtensionType() == ISD::SEXTLOAD; }]>; -def zextmaskedload8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (maskedload8 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_zextmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{ return cast(N)->getExtensionType() == ISD::ZEXTLOAD; }]>; -def extmaskedload8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (maskedload8 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_extmaskedloadvi8 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi8 node:$ptr, node:$pred, node:$passthru), [{ auto *Ld = cast(N); EVT ScalarVT = Ld->getMemoryVT().getScalarType(); return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD; }]>; -def alignedmaskedload16: PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ +def aligned_maskedloadvi16: PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ auto *Ld = cast(N); EVT ScalarVT = Ld->getMemoryVT().getScalarType(); return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && Ld->getAlignment() >= 2; }]>; -def sextmaskedload16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (alignedmaskedload16 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_sextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{ return cast(N)->getExtensionType() == ISD::SEXTLOAD; }]>; -def zextmaskedload16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (alignedmaskedload16 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_zextmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{ return cast(N)->getExtensionType() == ISD::ZEXTLOAD; }]>; -def extmaskedload16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (alignedmaskedload16 node:$ptr, node:$pred, node:$passthru), [{ +def aligned_extmaskedloadvi16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (aligned_maskedloadvi16 node:$ptr, node:$pred, node:$passthru), [{ auto *Ld = cast(N); EVT ScalarVT = Ld->getMemoryVT().getScalarType(); return ScalarVT.isInteger() && Ld->getExtensionType() == ISD::EXTLOAD; }]>; -def alignedmaskedload32: PatFrag<(ops node:$ptr, node:$pred, node:$passthru), - (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ +def aligned_maskedloadvi32: PatFrag<(ops node:$ptr, node:$pred, node:$passthru), + (masked_ld node:$ptr, undef, node:$pred, node:$passthru), [{ auto *Ld = cast(N); EVT ScalarVT = Ld->getMemoryVT().getScalarType(); return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && Ld->getAlignment() >= 4; }]>; -def maskedstore8 : PatFrag<(ops node:$val, node:$ptr, node:$pred), - (masked_st node:$val, node:$ptr, undef, node:$pred), [{ +def aligned_maskedstvi8 : PatFrag<(ops node:$val, node:$ptr, node:$pred), + (masked_st node:$val, node:$ptr, undef, node:$pred), [{ return cast(N)->getMemoryVT().getScalarType() == MVT::i8; }]>; -def truncatingmaskedstore8 : PatFrag<(ops node:$val, node:$ptr, node:$pred), - (maskedstore8 node:$val, node:$ptr, node:$pred), [{ - return cast(N)->isTruncatingStore(); -}]>; -def maskedstore16 : PatFrag<(ops node:$val, node:$ptr, node:$pred), - (masked_st node:$val, node:$ptr, undef, node:$pred), [{ +def aligned_maskedstvi16 : PatFrag<(ops node:$val, node:$ptr, node:$pred), + (masked_st node:$val, node:$ptr, undef, node:$pred), [{ auto *St = cast(N); EVT ScalarVT = St->getMemoryVT().getScalarType(); return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; }]>; - -def truncatingmaskedstore16 : PatFrag<(ops node:$val, node:$ptr, node:$pred), - (maskedstore16 node:$val, node:$ptr, node:$pred), [{ - return cast(N)->isTruncatingStore(); -}]>; -def maskedstore32 : PatFrag<(ops node:$val, node:$ptr, node:$pred), - (masked_st node:$val, node:$ptr, undef, node:$pred), [{ +def aligned_maskedstvi32 : PatFrag<(ops node:$val, node:$ptr, node:$pred), + (masked_st node:$val, node:$ptr, undef, node:$pred), [{ auto *St = cast(N); EVT ScalarVT = St->getMemoryVT().getScalarType(); return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4; }]>; - def pre_maskedstore : PatFrag<(ops node:$val, node:$base, node:$offset, node:$mask), (masked_st node:$val, node:$base, node:$offset, node:$mask), [{ ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); @@ -5449,24 +5381,177 @@ ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); return AM == ISD::POST_INC || AM == ISD::POST_DEC; }]>; -def aligned32_pre_maskedstore : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), - (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ - return cast(N)->getAlignment() >= 4; +def aligned_pre_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + return cast(N)->getMemoryVT().getScalarType() == MVT::i8; }]>; -def aligned32_post_maskedstore : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), - (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ - return cast(N)->getAlignment() >= 4; +def aligned_post_maskedstorevi8 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + return cast(N)->getMemoryVT().getScalarType() == MVT::i8; }]>; -def aligned16_pre_maskedstore : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), - (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ - return cast(N)->getAlignment() >= 2; +def aligned_pre_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; }]>; -def aligned16_post_maskedstore : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), - (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ - return cast(N)->getAlignment() >= 2; +def aligned_post_maskedstorevi16 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; +}]>; +def aligned_pre_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (pre_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4; +}]>; +def aligned_post_maskedstorevi32 : PatFrag<(ops node:$val, node:$ptr, node:$offset, node:$mask), + (post_maskedstore node:$val, node:$ptr, node:$offset, node:$mask), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i32 || ScalarVT == MVT::f32) && St->getAlignment() >= 4; }]>; +// PatFrags for "Aligned" extending / truncating + +def aligned_extloadvi8 : PatFrag<(ops node:$ptr), (extloadvi8 node:$ptr)>; +def aligned_sextloadvi8 : PatFrag<(ops node:$ptr), (sextloadvi8 node:$ptr)>; +def aligned_zextloadvi8 : PatFrag<(ops node:$ptr), (zextloadvi8 node:$ptr)>; + +def aligned_truncstvi8 : PatFrag<(ops node:$val, node:$ptr), + (truncstorevi8 node:$val, node:$ptr)>; +def aligned_post_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset), + (post_truncstvi8 node:$val, node:$base, node:$offset)>; +def aligned_pre_truncstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset), + (pre_truncstvi8 node:$val, node:$base, node:$offset)>; + +let MinAlignment = 2 in { + def aligned_extloadvi16 : PatFrag<(ops node:$ptr), (extloadvi16 node:$ptr)>; + def aligned_sextloadvi16 : PatFrag<(ops node:$ptr), (sextloadvi16 node:$ptr)>; + def aligned_zextloadvi16 : PatFrag<(ops node:$ptr), (zextloadvi16 node:$ptr)>; + + def aligned_truncstvi16 : PatFrag<(ops node:$val, node:$ptr), + (truncstorevi16 node:$val, node:$ptr)>; + def aligned_post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset), + (post_truncstvi16 node:$val, node:$base, node:$offset)>; + def aligned_pre_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset), + (pre_truncstvi16 node:$val, node:$base, node:$offset)>; +} + +def truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$pred), + (masked_st node:$val, node:$base, undef, node:$pred), [{ + return cast(N)->isTruncatingStore(); +}]>; +def aligned_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$pred), + (truncmaskedst node:$val, node:$base, node:$pred), [{ + return cast(N)->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def aligned_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$pred), + (truncmaskedst node:$val, node:$base, node:$pred), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; +}]>; +def pre_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), + (masked_st node:$val, node:$base, node:$offset, node:$pred), [{ + ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); + return cast(N)->isTruncatingStore() && (AM == ISD::PRE_INC || AM == ISD::PRE_DEC); +}]>; +def aligned_pre_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), + (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{ + return cast(N)->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def aligned_pre_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), + (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; +}]>; +def post_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), + (masked_st node:$val, node:$base, node:$offset, node:$postd), [{ + ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); + return cast(N)->isTruncatingStore() && (AM == ISD::POST_INC || AM == ISD::POST_DEC); +}]>; +def aligned_post_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), + (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{ + return cast(N)->getMemoryVT().getScalarType() == MVT::i8; +}]>; +def aligned_post_truncmaskedstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), + (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{ + auto *St = cast(N); + EVT ScalarVT = St->getMemoryVT().getScalarType(); + return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; +}]>; + +// Load/store patterns + +class MVE_vector_store_typed + : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7:$addr), + (RegImmInst (Ty MQPR:$val), t2addrmode_imm7:$addr)>; + +class MVE_vector_maskedstore_typed + : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7:$addr, VCCR:$pred), + (RegImmInst (Ty MQPR:$val), t2addrmode_imm7:$addr, (i32 1), VCCR:$pred)>; + +multiclass MVE_vector_store { + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; + def : MVE_vector_store_typed; +} + +class MVE_vector_load_typed + : Pat<(Ty (LoadKind t2addrmode_imm7:$addr)), + (Ty (RegImmInst t2addrmode_imm7:$addr))>; + +class MVE_vector_maskedload_typed + : Pat<(Ty (LoadKind t2addrmode_imm7:$addr, VCCR:$pred, (Ty NEONimmAllZerosV))), + (Ty (RegImmInst t2addrmode_imm7:$addr, (i32 1), VCCR:$pred))>; + +multiclass MVE_vector_load { + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; + def : MVE_vector_load_typed; +} + +class MVE_vector_offset_store_typed + : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr), + (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr)>; + +class MVE_vector_offset_maskedstore_typed + : Pat<(StoreKind (Ty MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr, VCCR:$pred), + (Opcode MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr, (i32 1), VCCR:$pred)>; + +multiclass MVE_vector_offset_store { + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; + def : MVE_vector_offset_store_typed; +} + + let Predicates = [HasMVEInt, IsLE] in { // Stores defm : MVE_vector_store; @@ -5543,175 +5628,73 @@ let Predicates = [HasMVEInt] in { // Aligned masked store, shared between LE and BE - def : MVE_vector_maskedstore_typed; - def : MVE_vector_maskedstore_typed; - def : MVE_vector_maskedstore_typed; - def : MVE_vector_maskedstore_typed; - def : MVE_vector_maskedstore_typed; + def : MVE_vector_maskedstore_typed; + def : MVE_vector_maskedstore_typed; + def : MVE_vector_maskedstore_typed; + def : MVE_vector_maskedstore_typed; + def : MVE_vector_maskedstore_typed; // Pre/Post inc masked stores - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; - def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; + def : MVE_vector_offset_maskedstore_typed; // Aligned masked loads - def : MVE_vector_maskedload_typed; - def : MVE_vector_maskedload_typed; - def : MVE_vector_maskedload_typed; - def : MVE_vector_maskedload_typed; - def : MVE_vector_maskedload_typed; - - // Extending masked loads. - def : Pat<(v8i16 (sextmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v8i16 NEONimmAllZerosV))), - (v8i16 (MVE_VLDRBS16 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (sextmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRBS32 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v8i16 (zextmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v8i16 NEONimmAllZerosV))), - (v8i16 (MVE_VLDRBU16 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (zextmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRBU32 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v8i16 (extmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v8i16 NEONimmAllZerosV))), - (v8i16 (MVE_VLDRBU16 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (extmaskedload8 t2addrmode_imm7<0>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRBU32 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (sextmaskedload16 t2addrmode_imm7<1>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRHS32 t2addrmode_imm7<1>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (zextmaskedload16 t2addrmode_imm7<1>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRHU32 t2addrmode_imm7<1>:$addr, (i32 1), VCCR:$pred))>; - def : Pat<(v4i32 (extmaskedload16 t2addrmode_imm7<1>:$addr, VCCR:$pred, - (v4i32 NEONimmAllZerosV))), - (v4i32 (MVE_VLDRHU32 t2addrmode_imm7<1>:$addr, (i32 1), VCCR:$pred))>; + def : MVE_vector_maskedload_typed; + def : MVE_vector_maskedload_typed; + def : MVE_vector_maskedload_typed; + def : MVE_vector_maskedload_typed; + def : MVE_vector_maskedload_typed; } // Widening/Narrowing Loads/Stores -let MinAlignment = 2 in { - def truncstorevi16_align2 : PatFrag<(ops node:$val, node:$ptr), - (truncstorevi16 node:$val, node:$ptr)>; - def post_truncstvi16_align2 : PatFrag<(ops node:$val, node:$base, node:$offset), - (post_truncstvi16 node:$val, node:$base, node:$offset)>; - def pre_truncstvi16_align2 : PatFrag<(ops node:$val, node:$base, node:$offset), - (pre_truncstvi16 node:$val, node:$base, node:$offset)>; -} - -def pre_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), - (masked_st node:$val, node:$base, node:$offset, node:$pred), [{ - ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); - return AM == ISD::PRE_INC || AM == ISD::PRE_DEC; -}]>; -def pre_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), - (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{ - return cast(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; -def pre_truncmaskedstvi16_align2 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$pred), - (pre_truncmaskedst node:$val, node:$base, node:$offset, node:$pred), [{ - auto *St = cast(N); - EVT ScalarVT = St->getMemoryVT().getScalarType(); - return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; -}]>; -def post_truncmaskedst : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), - (masked_st node:$val, node:$base, node:$offset, node:$postd), [{ - ISD::MemIndexedMode AM = cast(N)->getAddressingMode(); - return AM == ISD::POST_INC || AM == ISD::POST_DEC; -}]>; -def post_truncmaskedstvi8 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), - (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{ - return cast(N)->getMemoryVT().getScalarType() == MVT::i8; -}]>; -def post_truncmaskedstvi16_align2 : PatFrag<(ops node:$val, node:$base, node:$offset, node:$postd), - (post_truncmaskedst node:$val, node:$base, node:$offset, node:$postd), [{ - auto *St = cast(N); - EVT ScalarVT = St->getMemoryVT().getScalarType(); - return (ScalarVT == MVT::i16 || ScalarVT == MVT::f16) && St->getAlignment() >= 2; -}]>; - -let Predicates = [HasMVEInt] in { - def : Pat<(truncstorevi8 (v8i16 MQPR:$val), taddrmode_imm7<0>:$addr), - (MVE_VSTRB16 MQPR:$val, taddrmode_imm7<0>:$addr)>; - def : Pat<(truncstorevi8 (v4i32 MQPR:$val), taddrmode_imm7<0>:$addr), - (MVE_VSTRB32 MQPR:$val, taddrmode_imm7<0>:$addr)>; - def : Pat<(truncstorevi16_align2 (v4i32 MQPR:$val), taddrmode_imm7<1>:$addr), - (MVE_VSTRH32 MQPR:$val, taddrmode_imm7<1>:$addr)>; - - def : Pat<(post_truncstvi8 (v8i16 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr), - (MVE_VSTRB16_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr)>; - def : Pat<(post_truncstvi8 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr), - (MVE_VSTRB32_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr)>; - def : Pat<(post_truncstvi16_align2 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<1>:$addr), - (MVE_VSTRH32_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<1>:$addr)>; - - def : Pat<(pre_truncstvi8 (v8i16 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr), - (MVE_VSTRB16_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr)>; - def : Pat<(pre_truncstvi8 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr), - (MVE_VSTRB32_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr)>; - def : Pat<(pre_truncstvi16_align2 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<1>:$addr), - (MVE_VSTRH32_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<1>:$addr)>; - - def : Pat<(truncatingmaskedstore8 (v8i16 MQPR:$val), taddrmode_imm7<0>:$addr, VCCR:$pred), - (MVE_VSTRB16 MQPR:$val, taddrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(truncatingmaskedstore8 (v4i32 MQPR:$val), taddrmode_imm7<0>:$addr, VCCR:$pred), - (MVE_VSTRB32 MQPR:$val, taddrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(truncatingmaskedstore16 (v4i32 MQPR:$val), taddrmode_imm7<1>:$addr, VCCR:$pred), - (MVE_VSTRH32 MQPR:$val, taddrmode_imm7<1>:$addr, (i32 1), VCCR:$pred)>; - - def : Pat<(post_truncmaskedstvi8 (v8i16 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr, VCCR:$pred), - (MVE_VSTRB16_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(post_truncmaskedstvi8 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr, VCCR:$pred), - (MVE_VSTRB32_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(post_truncmaskedstvi16_align2 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<1>:$addr, VCCR:$pred), - (MVE_VSTRH32_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<1>:$addr, (i32 1), VCCR:$pred)>; - - def : Pat<(pre_truncmaskedstvi8 (v8i16 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr, VCCR:$pred), - (MVE_VSTRB16_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(pre_truncmaskedstvi8 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr, VCCR:$pred), - (MVE_VSTRB32_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr, (i32 1), VCCR:$pred)>; - def : Pat<(pre_truncmaskedstvi16_align2 (v4i32 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<1>:$addr, VCCR:$pred), - (MVE_VSTRH32_pre MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<1>:$addr, (i32 1), VCCR:$pred)>; -} - - -let MinAlignment = 2 in { - def extloadvi16_align2 : PatFrag<(ops node:$ptr), (extloadvi16 node:$ptr)>; - def sextloadvi16_align2 : PatFrag<(ops node:$ptr), (sextloadvi16 node:$ptr)>; - def zextloadvi16_align2 : PatFrag<(ops node:$ptr), (zextloadvi16 node:$ptr)>; -} - -multiclass MVEExtLoad { - def _Any : Pat<(!cast("v" # DestLanes # "i" # DestElemBits) - (!cast("extloadvi" # SrcElemBits # Align) am:$addr)), - (!cast("MVE_VLDR" # SrcElemType # "U" # DestElemBits) - am:$addr)>; - def _Z : Pat<(!cast("v" # DestLanes # "i" # DestElemBits) - (!cast("zextloadvi" # SrcElemBits # Align) am:$addr)), - (!cast("MVE_VLDR" # SrcElemType # "U" # DestElemBits) - am:$addr)>; - def _S : Pat<(!cast("v" # DestLanes # "i" # DestElemBits) - (!cast("sextloadvi" # SrcElemBits # Align) am:$addr)), - (!cast("MVE_VLDR" # SrcElemType # "S" # DestElemBits) - am:$addr)>; +multiclass MVEExtLoadStore { + // Trunc stores + def : Pat<(!cast("aligned_truncst"#Amble) (VT MQPR:$val), taddrmode_imm7:$addr), + (!cast(StoreInst) MQPR:$val, taddrmode_imm7:$addr)>; + def : Pat<(!cast("aligned_post_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr), + (!cast(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr)>; + def : Pat<(!cast("aligned_pre_truncst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr), + (!cast(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr)>; + + // Masked trunc stores + def : Pat<(!cast("aligned_truncmaskedst"#Amble) (VT MQPR:$val), taddrmode_imm7:$addr, VCCR:$pred), + (!cast(StoreInst) MQPR:$val, taddrmode_imm7:$addr, (i32 1), VCCR:$pred)>; + def : Pat<(!cast("aligned_post_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr, VCCR:$pred), + (!cast(StoreInst#"_post") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr, (i32 1), VCCR:$pred)>; + def : Pat<(!cast("aligned_pre_truncmaskedst"#Amble) (VT MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset:$addr, VCCR:$pred), + (!cast(StoreInst#"_pre") MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset:$addr, (i32 1), VCCR:$pred)>; + + // Ext loads + def : Pat<(VT (!cast("aligned_extload"#Amble) taddrmode_imm7:$addr)), + (VT (LoadUInst taddrmode_imm7:$addr))>; + def : Pat<(VT (!cast("aligned_sextload"#Amble) taddrmode_imm7:$addr)), + (VT (LoadSInst taddrmode_imm7:$addr))>; + def : Pat<(VT (!cast("aligned_zextload"#Amble) taddrmode_imm7:$addr)), + (VT (LoadUInst taddrmode_imm7:$addr))>; + + // Masked ext loads + def : Pat<(VT (!cast("aligned_extmaskedload"#Amble) taddrmode_imm7:$addr, VCCR:$pred, (VT NEONimmAllZerosV))), + (VT (LoadUInst taddrmode_imm7:$addr, (i32 1), VCCR:$pred))>; + def : Pat<(VT (!cast("aligned_sextmaskedload"#Amble) taddrmode_imm7:$addr, VCCR:$pred, (VT NEONimmAllZerosV))), + (VT (LoadSInst taddrmode_imm7:$addr, (i32 1), VCCR:$pred))>; + def : Pat<(VT (!cast("aligned_zextmaskedload"#Amble) taddrmode_imm7:$addr, VCCR:$pred, (VT NEONimmAllZerosV))), + (VT (LoadUInst taddrmode_imm7:$addr, (i32 1), VCCR:$pred))>; } let Predicates = [HasMVEInt] in { - defm : MVEExtLoad<"4", "32", "8", "B", "", taddrmode_imm7<0>>; - defm : MVEExtLoad<"8", "16", "8", "B", "", taddrmode_imm7<0>>; - defm : MVEExtLoad<"4", "32", "16", "H", "_align2", taddrmode_imm7<1>>; + defm : MVEExtLoadStore; + defm : MVEExtLoadStore; + defm : MVEExtLoadStore; }