Index: include/llvm/Target/TargetSelectionDAG.td =================================================================== --- include/llvm/Target/TargetSelectionDAG.td +++ include/llvm/Target/TargetSelectionDAG.td @@ -1015,30 +1015,56 @@ def setne : PatFrag<(ops node:$lhs, node:$rhs), (setcc node:$lhs, node:$rhs, SETNE)>; -def atomic_cmp_swap_8 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast(N)->getMemoryVT() == MVT::i8; -}]>; -def atomic_cmp_swap_16 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast(N)->getMemoryVT() == MVT::i16; -}]>; -def atomic_cmp_swap_32 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast(N)->getMemoryVT() == MVT::i32; -}]>; -def atomic_cmp_swap_64 : - PatFrag<(ops node:$ptr, node:$cmp, node:$swap), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{ - return cast(N)->getMemoryVT() == MVT::i64; -}]>; +multiclass binary_atomic_op_ord { + def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), + (!cast(#NAME) node:$ptr, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Monotonic; + }]>; + def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$val), + (!cast(#NAME) node:$ptr, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Acquire; + }]>; + def #NAME#_release : PatFrag<(ops node:$ptr, node:$val), + (!cast(#NAME) node:$ptr, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Release; + }]>; + def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$val), + (!cast(#NAME) node:$ptr, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::AcquireRelease; + }]>; + def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$val), + (!cast(#NAME) node:$ptr, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent; + }]>; +} -multiclass binary_atomic_op { +multiclass ternary_atomic_op_ord { + def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(#NAME) node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Monotonic; + }]>; + def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(#NAME) node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Acquire; + }]>; + def #NAME#_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(#NAME) node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::Release; + }]>; + def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(#NAME) node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::AcquireRelease; + }]>; + def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (!cast(#NAME) node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent; + }]>; +} + +multiclass binary_atomic_op : + binary_atomic_op_ord { def _8 : PatFrag<(ops node:$ptr, node:$val), - (atomic_op node:$ptr, node:$val), [{ + (atomic_op node:$ptr, node:$val), [{ return cast(N)->getMemoryVT() == MVT::i8; }]>; def _16 : PatFrag<(ops node:$ptr, node:$val), @@ -1055,6 +1081,26 @@ }]>; } +multiclass ternary_atomic_op : + ternary_atomic_op_ord { + def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getMemoryVT() == MVT::i8; + }]>; + def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getMemoryVT() == MVT::i16; + }]>; + def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getMemoryVT() == MVT::i32; + }]>; + def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val), + (atomic_op node:$ptr, node:$cmp, node:$val), [{ + return cast(N)->getMemoryVT() == MVT::i64; + }]>; +} + defm atomic_load_add : binary_atomic_op; defm atomic_swap : binary_atomic_op; defm atomic_load_sub : binary_atomic_op; @@ -1067,6 +1113,377 @@ defm atomic_load_umin : binary_atomic_op; defm atomic_load_umax : binary_atomic_op; defm atomic_store : binary_atomic_op; +defm atomic_cmp_swap : ternary_atomic_op; + +defm atomic_load_add_8 : binary_atomic_op_ord; +defm atomic_load_add_16 : binary_atomic_op_ord; +defm atomic_load_add_32 : binary_atomic_op_ord; +defm atomic_load_add_64 : binary_atomic_op_ord; + +defm atomic_load_sub_8 : binary_atomic_op_ord; +defm atomic_load_sub_16 : binary_atomic_op_ord; +defm atomic_load_sub_32 : binary_atomic_op_ord; +defm atomic_load_sub_64 : binary_atomic_op_ord; + +defm atomic_load_and_8 : binary_atomic_op_ord; +defm atomic_load_and_16 : binary_atomic_op_ord; +defm atomic_load_and_32 : binary_atomic_op_ord; +defm atomic_load_and_64 : binary_atomic_op_ord; + +defm atomic_load_or_8 : binary_atomic_op_ord; +defm atomic_load_or_16 : binary_atomic_op_ord; +defm atomic_load_or_32 : binary_atomic_op_ord; +defm atomic_load_or_64 : binary_atomic_op_ord; + +defm atomic_load_xor_8 : binary_atomic_op_ord; +defm atomic_load_xor_16 : binary_atomic_op_ord; +defm atomic_load_xor_32 : binary_atomic_op_ord; +defm atomic_load_xor_64 : binary_atomic_op_ord; + +defm atomic_load_nand_8 : binary_atomic_op_ord; +defm atomic_load_nand_16 : binary_atomic_op_ord; +defm atomic_load_nand_32 : binary_atomic_op_ord; +defm atomic_load_nand_64 : binary_atomic_op_ord; + +defm atomic_load_max_8 : binary_atomic_op_ord; +defm atomic_load_max_16 : binary_atomic_op_ord; +defm atomic_load_max_32 : binary_atomic_op_ord; +defm atomic_load_max_64 : binary_atomic_op_ord; + +defm atomic_load_min_8 : binary_atomic_op_ord; +defm atomic_load_min_16 : binary_atomic_op_ord; +defm atomic_load_min_32 : binary_atomic_op_ord; +defm atomic_load_min_64 : binary_atomic_op_ord; + +defm atomic_load_umax_8 : binary_atomic_op_ord; +defm atomic_load_umax_16 : binary_atomic_op_ord; +defm atomic_load_umax_32 : binary_atomic_op_ord; +defm atomic_load_umax_64 : binary_atomic_op_ord; + +defm atomic_load_umin_8 : binary_atomic_op_ord; +defm atomic_load_umin_16 : binary_atomic_op_ord; +defm atomic_load_umin_32 : binary_atomic_op_ord; +defm atomic_load_umin_64 : binary_atomic_op_ord; + +defm atomic_load_add_8_monotonic : binary_atomic_op_ord; +defm atomic_load_add_16_monotonic : binary_atomic_op_ord; +defm atomic_load_add_32_monotonic : binary_atomic_op_ord; +defm atomic_load_add_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_sub_8_monotonic : binary_atomic_op_ord; +defm atomic_load_sub_16_monotonic : binary_atomic_op_ord; +defm atomic_load_sub_32_monotonic : binary_atomic_op_ord; +defm atomic_load_sub_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_and_8_monotonic : binary_atomic_op_ord; +defm atomic_load_and_16_monotonic : binary_atomic_op_ord; +defm atomic_load_and_32_monotonic : binary_atomic_op_ord; +defm atomic_load_and_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_or_8_monotonic : binary_atomic_op_ord; +defm atomic_load_or_16_monotonic : binary_atomic_op_ord; +defm atomic_load_or_32_monotonic : binary_atomic_op_ord; +defm atomic_load_or_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_xor_8_monotonic : binary_atomic_op_ord; +defm atomic_load_xor_16_monotonic : binary_atomic_op_ord; +defm atomic_load_xor_32_monotonic : binary_atomic_op_ord; +defm atomic_load_xor_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_nand_8_monotonic : binary_atomic_op_ord; +defm atomic_load_nand_16_monotonic : binary_atomic_op_ord; +defm atomic_load_nand_32_monotonic : binary_atomic_op_ord; +defm atomic_load_nand_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_max_8_monotonic : binary_atomic_op_ord; +defm atomic_load_max_16_monotonic : binary_atomic_op_ord; +defm atomic_load_max_32_monotonic : binary_atomic_op_ord; +defm atomic_load_max_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_min_8_monotonic : binary_atomic_op_ord; +defm atomic_load_min_16_monotonic : binary_atomic_op_ord; +defm atomic_load_min_32_monotonic : binary_atomic_op_ord; +defm atomic_load_min_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_umax_8_monotonic : binary_atomic_op_ord; +defm atomic_load_umax_16_monotonic : binary_atomic_op_ord; +defm atomic_load_umax_32_monotonic : binary_atomic_op_ord; +defm atomic_load_umax_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_umin_8_monotonic : binary_atomic_op_ord; +defm atomic_load_umin_16_monotonic : binary_atomic_op_ord; +defm atomic_load_umin_32_monotonic : binary_atomic_op_ord; +defm atomic_load_umin_64_monotonic : binary_atomic_op_ord; + +defm atomic_load_add_8_acquire : binary_atomic_op_ord; +defm atomic_load_add_16_acquire : binary_atomic_op_ord; +defm atomic_load_add_32_acquire : binary_atomic_op_ord; +defm atomic_load_add_64_acquire : binary_atomic_op_ord; + +defm atomic_load_sub_8_acquire : binary_atomic_op_ord; +defm atomic_load_sub_16_acquire : binary_atomic_op_ord; +defm atomic_load_sub_32_acquire : binary_atomic_op_ord; +defm atomic_load_sub_64_acquire : binary_atomic_op_ord; + +defm atomic_load_and_8_acquire : binary_atomic_op_ord; +defm atomic_load_and_16_acquire : binary_atomic_op_ord; +defm atomic_load_and_32_acquire : binary_atomic_op_ord; +defm atomic_load_and_64_acquire : binary_atomic_op_ord; + +defm atomic_load_or_8_acquire : binary_atomic_op_ord; +defm atomic_load_or_16_acquire : binary_atomic_op_ord; +defm atomic_load_or_32_acquire : binary_atomic_op_ord; +defm atomic_load_or_64_acquire : binary_atomic_op_ord; + +defm atomic_load_xor_8_acquire : binary_atomic_op_ord; +defm atomic_load_xor_16_acquire : binary_atomic_op_ord; +defm atomic_load_xor_32_acquire : binary_atomic_op_ord; +defm atomic_load_xor_64_acquire : binary_atomic_op_ord; + +defm atomic_load_nand_8_acquire : binary_atomic_op_ord; +defm atomic_load_nand_16_acquire : binary_atomic_op_ord; +defm atomic_load_nand_32_acquire : binary_atomic_op_ord; +defm atomic_load_nand_64_acquire : binary_atomic_op_ord; + +defm atomic_load_max_8_acquire : binary_atomic_op_ord; +defm atomic_load_max_16_acquire : binary_atomic_op_ord; +defm atomic_load_max_32_acquire : binary_atomic_op_ord; +defm atomic_load_max_64_acquire : binary_atomic_op_ord; + +defm atomic_load_min_8_acquire : binary_atomic_op_ord; +defm atomic_load_min_16_acquire : binary_atomic_op_ord; +defm atomic_load_min_32_acquire : binary_atomic_op_ord; +defm atomic_load_min_64_acquire : binary_atomic_op_ord; + +defm atomic_load_umax_8_acquire : binary_atomic_op_ord; +defm atomic_load_umax_16_acquire : binary_atomic_op_ord; +defm atomic_load_umax_32_acquire : binary_atomic_op_ord; +defm atomic_load_umax_64_acquire : binary_atomic_op_ord; + +defm atomic_load_umin_8_acquire : binary_atomic_op_ord; +defm atomic_load_umin_16_acquire : binary_atomic_op_ord; +defm atomic_load_umin_32_acquire : binary_atomic_op_ord; +defm atomic_load_umin_64_acquire : binary_atomic_op_ord; + +defm atomic_load_add_8_release : binary_atomic_op_ord; +defm atomic_load_add_16_release : binary_atomic_op_ord; +defm atomic_load_add_32_release : binary_atomic_op_ord; +defm atomic_load_add_64_release : binary_atomic_op_ord; + +defm atomic_load_sub_8_release : binary_atomic_op_ord; +defm atomic_load_sub_16_release : binary_atomic_op_ord; +defm atomic_load_sub_32_release : binary_atomic_op_ord; +defm atomic_load_sub_64_release : binary_atomic_op_ord; + +defm atomic_load_and_8_release : binary_atomic_op_ord; +defm atomic_load_and_16_release : binary_atomic_op_ord; +defm atomic_load_and_32_release : binary_atomic_op_ord; +defm atomic_load_and_64_release : binary_atomic_op_ord; + +defm atomic_load_or_8_release : binary_atomic_op_ord; +defm atomic_load_or_16_release : binary_atomic_op_ord; +defm atomic_load_or_32_release : binary_atomic_op_ord; +defm atomic_load_or_64_release : binary_atomic_op_ord; + +defm atomic_load_xor_8_release : binary_atomic_op_ord; +defm atomic_load_xor_16_release : binary_atomic_op_ord; +defm atomic_load_xor_32_release : binary_atomic_op_ord; +defm atomic_load_xor_64_release : binary_atomic_op_ord; + +defm atomic_load_nand_8_release : binary_atomic_op_ord; +defm atomic_load_nand_16_release : binary_atomic_op_ord; +defm atomic_load_nand_32_release : binary_atomic_op_ord; +defm atomic_load_nand_64_release : binary_atomic_op_ord; + +defm atomic_load_max_8_release : binary_atomic_op_ord; +defm atomic_load_max_16_release : binary_atomic_op_ord; +defm atomic_load_max_32_release : binary_atomic_op_ord; +defm atomic_load_max_64_release : binary_atomic_op_ord; + +defm atomic_load_min_8_release : binary_atomic_op_ord; +defm atomic_load_min_16_release : binary_atomic_op_ord; +defm atomic_load_min_32_release : binary_atomic_op_ord; +defm atomic_load_min_64_release : binary_atomic_op_ord; + +defm atomic_load_umax_8_release : binary_atomic_op_ord; +defm atomic_load_umax_16_release : binary_atomic_op_ord; +defm atomic_load_umax_32_release : binary_atomic_op_ord; +defm atomic_load_umax_64_release : binary_atomic_op_ord; + +defm atomic_load_umin_8_release : binary_atomic_op_ord; +defm atomic_load_umin_16_release : binary_atomic_op_ord; +defm atomic_load_umin_32_release : binary_atomic_op_ord; +defm atomic_load_umin_64_release : binary_atomic_op_ord; + +defm atomic_load_add_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_add_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_add_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_add_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_sub_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_sub_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_sub_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_sub_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_and_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_and_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_and_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_and_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_or_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_or_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_or_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_or_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_xor_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_xor_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_xor_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_xor_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_nand_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_nand_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_nand_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_nand_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_max_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_max_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_max_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_max_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_min_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_min_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_min_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_min_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_umax_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_umax_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_umax_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_umax_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_umin_8_acq_rel : binary_atomic_op_ord; +defm atomic_load_umin_16_acq_rel : binary_atomic_op_ord; +defm atomic_load_umin_32_acq_rel : binary_atomic_op_ord; +defm atomic_load_umin_64_acq_rel : binary_atomic_op_ord; + +defm atomic_load_add_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_add_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_add_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_add_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_sub_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_sub_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_sub_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_sub_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_and_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_and_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_and_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_and_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_or_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_or_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_or_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_or_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_xor_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_xor_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_xor_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_xor_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_max_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_max_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_max_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_max_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_min_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_min_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_min_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_min_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_umax_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_umax_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_umax_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_umax_64_seq_cst : binary_atomic_op_ord; + +defm atomic_load_umin_8_seq_cst : binary_atomic_op_ord; +defm atomic_load_umin_16_seq_cst : binary_atomic_op_ord; +defm atomic_load_umin_32_seq_cst : binary_atomic_op_ord; +defm atomic_load_umin_64_seq_cst : binary_atomic_op_ord; + +defm atomic_swap_8 : binary_atomic_op; +defm atomic_swap_16 : binary_atomic_op; +defm atomic_swap_32 : binary_atomic_op; +defm atomic_swap_64 : binary_atomic_op; + +defm atomic_swap_8_monotonic : binary_atomic_op; +defm atomic_swap_16_monotonic : binary_atomic_op; +defm atomic_swap_32_monotonic : binary_atomic_op; +defm atomic_swap_64_monotonic : binary_atomic_op; + +defm atomic_swap_8_acquire : binary_atomic_op; +defm atomic_swap_16_acquire : binary_atomic_op; +defm atomic_swap_32_acquire : binary_atomic_op; +defm atomic_swap_64_acquire : binary_atomic_op; + +defm atomic_swap_8_release : binary_atomic_op; +defm atomic_swap_16_release : binary_atomic_op; +defm atomic_swap_32_release : binary_atomic_op; +defm atomic_swap_64_release : binary_atomic_op; + +defm atomic_swap_8_acq_rel : binary_atomic_op; +defm atomic_swap_16_acq_rel : binary_atomic_op; +defm atomic_swap_32_acq_rel : binary_atomic_op; +defm atomic_swap_64_acq_rel : binary_atomic_op; + +defm atomic_swap_8_seq_cst : binary_atomic_op; +defm atomic_swap_16_seq_cst : binary_atomic_op; +defm atomic_swap_32_seq_cst : binary_atomic_op; +defm atomic_swap_64_seq_cst : binary_atomic_op; + +defm atomic_store_8 : binary_atomic_op; +defm atomic_store_16 : binary_atomic_op; +defm atomic_store_32 : binary_atomic_op; +defm atomic_store_64 : binary_atomic_op; + +defm atomic_store_8_monotonic : binary_atomic_op; +defm atomic_store_16_monotonic : binary_atomic_op; +defm atomic_store_32_monotonic : binary_atomic_op; +defm atomic_store_64_monotonic : binary_atomic_op; + +defm atomic_store_8_release : binary_atomic_op; +defm atomic_store_16_release : binary_atomic_op; +defm atomic_store_32_release : binary_atomic_op; +defm atomic_store_64_release : binary_atomic_op; + +defm atomic_cmp_swap_8 : ternary_atomic_op_ord; +defm atomic_cmp_swap_16 : ternary_atomic_op_ord; +defm atomic_cmp_swap_32 : ternary_atomic_op_ord; +defm atomic_cmp_swap_64 : ternary_atomic_op_ord; + +defm atomic_cmp_swap_8_monotonic : ternary_atomic_op_ord; +defm atomic_cmp_swap_16_monotonic : ternary_atomic_op_ord; +defm atomic_cmp_swap_32_monotonic : ternary_atomic_op_ord; +defm atomic_cmp_swap_64_monotonic : ternary_atomic_op_ord; + +defm atomic_cmp_swap_8_acquire : ternary_atomic_op_ord; +defm atomic_cmp_swap_16_acquire : ternary_atomic_op_ord; +defm atomic_cmp_swap_32_acquire : ternary_atomic_op_ord; +defm atomic_cmp_swap_64_acquire : ternary_atomic_op_ord; + +defm atomic_cmp_swap_8_release : ternary_atomic_op_ord; +defm atomic_cmp_swap_16_release : ternary_atomic_op_ord; +defm atomic_cmp_swap_32_release : ternary_atomic_op_ord; +defm atomic_cmp_swap_64_release : ternary_atomic_op_ord; + +defm atomic_cmp_swap_8_acq_rel : ternary_atomic_op_ord; +defm atomic_cmp_swap_16_acq_rel : ternary_atomic_op_ord; +defm atomic_cmp_swap_32_acq_rel : ternary_atomic_op_ord; +defm atomic_cmp_swap_64_acq_rel : ternary_atomic_op_ord; + +defm atomic_cmp_swap_8_seq_cst : ternary_atomic_op_ord; +defm atomic_cmp_swap_16_seq_cst : ternary_atomic_op_ord; +defm atomic_cmp_swap_32_seq_cst : ternary_atomic_op_ord; +defm atomic_cmp_swap_64_seq_cst : ternary_atomic_op_ord; def atomic_load_8 : PatFrag<(ops node:$ptr), Index: lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp =================================================================== --- lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp +++ lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp @@ -55,7 +55,10 @@ AU.setPreservesCFG(); MachineFunctionPass::getAnalysisUsage(AU); } + + bool ShouldSkip(const MachineInstr &MI, const MachineFunction &MF) const; }; + char AArch64DeadRegisterDefinitions::ID = 0; } // end anonymous namespace @@ -69,6 +72,66 @@ return false; } +bool +AArch64DeadRegisterDefinitions::ShouldSkip(const MachineInstr &MI, + const MachineFunction &MF) const { + if (!MF.getSubtarget().hasLSE()) + return false; + +#define CASE_AARCH64_ATOMIC_(PREFIX) \ + case AArch64::PREFIX##X: \ + case AArch64::PREFIX##W: \ + case AArch64::PREFIX##H: \ + case AArch64::PREFIX##B + + // ARM Errata D11904 05/12/2017: + // The Acquire semantics are present when (A == 1 && Rt != 11111). + // The Release semantics are present when (R == 1). + for (const MachineMemOperand *MMO : MI.memoperands()) { + if (MMO->isAtomic()) { + unsigned Opcode = MI.getOpcode(); + switch (Opcode) { + default: + return false; + break; + + CASE_AARCH64_ATOMIC_(LDADDA): + CASE_AARCH64_ATOMIC_(LDADDAL): + + CASE_AARCH64_ATOMIC_(LDCLRA): + CASE_AARCH64_ATOMIC_(LDCLRAL): + + CASE_AARCH64_ATOMIC_(LDEORA): + CASE_AARCH64_ATOMIC_(LDEORAL): + + CASE_AARCH64_ATOMIC_(LDSETA): + CASE_AARCH64_ATOMIC_(LDSETAL): + + CASE_AARCH64_ATOMIC_(LDSMAXA): + CASE_AARCH64_ATOMIC_(LDSMAXAL): + + CASE_AARCH64_ATOMIC_(LDSMINA): + CASE_AARCH64_ATOMIC_(LDSMINAL): + + CASE_AARCH64_ATOMIC_(LDUMAXA): + CASE_AARCH64_ATOMIC_(LDUMAXAL): + + CASE_AARCH64_ATOMIC_(LDUMINA): + CASE_AARCH64_ATOMIC_(LDUMINAL): + + CASE_AARCH64_ATOMIC_(SWPA): + CASE_AARCH64_ATOMIC_(SWPAL): + return true; + break; + } + } + } + +#undef CASE_AARCH64_ATOMIC_ + + return false; +} + void AArch64DeadRegisterDefinitions::processMachineBasicBlock( MachineBasicBlock &MBB) { const MachineFunction &MF = *MBB.getParent(); @@ -86,55 +149,12 @@ DEBUG(dbgs() << " Ignoring, XZR or WZR already used by the instruction\n"); continue; } - if (MF.getSubtarget().hasLSE()) { - // XZ/WZ for LSE can only be used when acquire semantics are not used, - // LDOPAL WZ is an invalid opcode. - switch (MI.getOpcode()) { - case AArch64::CASALb: - case AArch64::CASALh: - case AArch64::CASALs: - case AArch64::CASALd: - case AArch64::SWPALb: - case AArch64::SWPALh: - case AArch64::SWPALs: - case AArch64::SWPALd: - case AArch64::LDADDALb: - case AArch64::LDADDALh: - case AArch64::LDADDALs: - case AArch64::LDADDALd: - case AArch64::LDCLRALb: - case AArch64::LDCLRALh: - case AArch64::LDCLRALs: - case AArch64::LDCLRALd: - case AArch64::LDEORALb: - case AArch64::LDEORALh: - case AArch64::LDEORALs: - case AArch64::LDEORALd: - case AArch64::LDSETALb: - case AArch64::LDSETALh: - case AArch64::LDSETALs: - case AArch64::LDSETALd: - case AArch64::LDSMINALb: - case AArch64::LDSMINALh: - case AArch64::LDSMINALs: - case AArch64::LDSMINALd: - case AArch64::LDSMAXALb: - case AArch64::LDSMAXALh: - case AArch64::LDSMAXALs: - case AArch64::LDSMAXALd: - case AArch64::LDUMINALb: - case AArch64::LDUMINALh: - case AArch64::LDUMINALs: - case AArch64::LDUMINALd: - case AArch64::LDUMAXALb: - case AArch64::LDUMAXALh: - case AArch64::LDUMAXALs: - case AArch64::LDUMAXALd: - continue; - default: - break; - } + + if (ShouldSkip(MI, MF)) { + DEBUG(dbgs() << " Ignoring, Atomic instruction with acquire semantics using WZR/XZR\n"); + continue; } + const MCInstrDesc &Desc = MI.getDesc(); for (int I = 0, E = Desc.getNumDefs(); I != E; ++I) { MachineOperand &MO = MI.getOperand(I); Index: lib/Target/AArch64/AArch64InstrAtomics.td =================================================================== --- lib/Target/AArch64/AArch64InstrAtomics.td +++ lib/Target/AArch64/AArch64InstrAtomics.td @@ -407,57 +407,310 @@ Sched<[WriteAtomic]>; // v8.1 Atomic instructions: -def : Pat<(atomic_load_add_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_add_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_add_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_add_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_or_8 GPR64:$Rn, GPR32:$Rs), (LDSETALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_or_16 GPR64:$Rn, GPR32:$Rs), (LDSETALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_or_32 GPR64:$Rn, GPR32:$Rs), (LDSETALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_or_64 GPR64:$Rn, GPR64:$Rs), (LDSETALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_xor_8 GPR64:$Rn, GPR32:$Rs), (LDEORALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_xor_16 GPR64:$Rn, GPR32:$Rs), (LDEORALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_xor_32 GPR64:$Rn, GPR32:$Rs), (LDEORALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_xor_64 GPR64:$Rn, GPR64:$Rs), (LDEORALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_max_8 GPR64:$Rn, GPR32:$Rs), (LDSMAXALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_max_16 GPR64:$Rn, GPR32:$Rs), (LDSMAXALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_max_32 GPR64:$Rn, GPR32:$Rs), (LDSMAXALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_max_64 GPR64:$Rn, GPR64:$Rs), (LDSMAXALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_umax_8 GPR64:$Rn, GPR32:$Rs), (LDUMAXALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umax_16 GPR64:$Rn, GPR32:$Rs), (LDUMAXALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umax_32 GPR64:$Rn, GPR32:$Rs), (LDUMAXALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umax_64 GPR64:$Rn, GPR64:$Rs), (LDUMAXALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_min_8 GPR64:$Rn, GPR32:$Rs), (LDSMINALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_min_16 GPR64:$Rn, GPR32:$Rs), (LDSMINALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_min_32 GPR64:$Rn, GPR32:$Rs), (LDSMINALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_min_64 GPR64:$Rn, GPR64:$Rs), (LDSMINALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_umin_8 GPR64:$Rn, GPR32:$Rs), (LDUMINALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umin_16 GPR64:$Rn, GPR32:$Rs), (LDUMINALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umin_32 GPR64:$Rn, GPR32:$Rs), (LDUMINALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_load_umin_64 GPR64:$Rn, GPR64:$Rs), (LDUMINALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_cmp_swap_8 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALb GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; -def : Pat<(atomic_cmp_swap_16 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALh GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; -def : Pat<(atomic_cmp_swap_32 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALs GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; -def : Pat<(atomic_cmp_swap_64 GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALd GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; - -def : Pat<(atomic_swap_8 GPR64:$Rn, GPR32:$Rs), (SWPALb GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_swap_16 GPR64:$Rn, GPR32:$Rs), (SWPALh GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_swap_32 GPR64:$Rn, GPR32:$Rs), (SWPALs GPR32:$Rs, GPR64sp:$Rn)>; -def : Pat<(atomic_swap_64 GPR64:$Rn, GPR64:$Rs), (SWPALd GPR64:$Rs, GPR64sp:$Rn)>; - -def : Pat<(atomic_load_sub_8 GPR64:$Rn, GPR32:$Rs), (LDADDALb (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_sub_16 GPR64:$Rn, GPR32:$Rs), (LDADDALh (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_sub_32 GPR64:$Rn, GPR32:$Rs), (LDADDALs (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_sub_64 GPR64:$Rn, GPR64:$Rs), (LDADDALd (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; - -def : Pat<(atomic_load_and_8 GPR64:$Rn, GPR32:$Rs), (LDCLRALb (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_and_16 GPR64:$Rn, GPR32:$Rs), (LDCLRALh (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_and_32 GPR64:$Rn, GPR32:$Rs), (LDCLRALs (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; -def : Pat<(atomic_load_and_64 GPR64:$Rn, GPR64:$Rs), (LDCLRALd (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_add_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDADDX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_8_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64_acquire GPR64:$Rn, GPR64:$Rs), (LDADDAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_8_release GPR64:$Rn, GPR32:$Rs), (LDADDLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16_release GPR64:$Rn, GPR32:$Rs), (LDADDLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32_release GPR64:$Rn, GPR32:$Rs), (LDADDLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64_release GPR64:$Rn, GPR64:$Rs), (LDADDLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDADDALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDADDALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_8 GPR64:$Rn, GPR32:$Rs), (LDADDALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_16 GPR64:$Rn, GPR32:$Rs), (LDADDALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_32 GPR64:$Rn, GPR32:$Rs), (LDADDALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_add_64 GPR64:$Rn, GPR64:$Rs), (LDADDALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_sub_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDADDW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDADDX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_8_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32_acquire GPR64:$Rn, GPR32:$Rs), (LDADDAW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64_acquire GPR64:$Rn, GPR64:$Rs), (LDADDAX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_8_release GPR64:$Rn, GPR32:$Rs), (LDADDLB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16_release GPR64:$Rn, GPR32:$Rs), (LDADDLH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32_release GPR64:$Rn, GPR32:$Rs), (LDADDLW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64_release GPR64:$Rn, GPR64:$Rs), (LDADDLX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDADDALW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDADDALX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDADDALW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDADDALX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_8 GPR64:$Rn, GPR32:$Rs), (LDADDALB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_16 GPR64:$Rn, GPR32:$Rs), (LDADDALH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_32 GPR64:$Rn, GPR32:$Rs), (LDADDALW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_sub_64 GPR64:$Rn, GPR64:$Rs), (LDADDALX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_and_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDCLRB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDCLRH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDCLRW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDCLRX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_8_acquire GPR64:$Rn, GPR32:$Rs), (LDCLRAB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16_acquire GPR64:$Rn, GPR32:$Rs), (LDCLRAH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32_acquire GPR64:$Rn, GPR32:$Rs), (LDCLRAW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64_acquire GPR64:$Rn, GPR64:$Rs), (LDCLRAX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_8_release GPR64:$Rn, GPR32:$Rs), (LDCLRLB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16_release GPR64:$Rn, GPR32:$Rs), (LDCLRLH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32_release GPR64:$Rn, GPR32:$Rs), (LDCLRLW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64_release GPR64:$Rn, GPR64:$Rs), (LDCLRLX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDCLRALB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDCLRALH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDCLRALW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDCLRALX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDCLRALB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDCLRALH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDCLRALW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDCLRALX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_8 GPR64:$Rn, GPR32:$Rs), (LDCLRALB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_16 GPR64:$Rn, GPR32:$Rs), (LDCLRALH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_32 GPR64:$Rn, GPR32:$Rs), (LDCLRALW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>; + def : Pat<(atomic_load_and_64 GPR64:$Rn, GPR64:$Rs), (LDCLRALX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_xor_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDEORB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDEORH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDEORW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDEORX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_8_acquire GPR64:$Rn, GPR32:$Rs), (LDEORAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16_acquire GPR64:$Rn, GPR32:$Rs), (LDEORAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32_acquire GPR64:$Rn, GPR32:$Rs), (LDEORAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64_acquire GPR64:$Rn, GPR64:$Rs), (LDEORAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_8_release GPR64:$Rn, GPR32:$Rs), (LDEORLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16_release GPR64:$Rn, GPR32:$Rs), (LDEORLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32_release GPR64:$Rn, GPR32:$Rs), (LDEORLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64_release GPR64:$Rn, GPR64:$Rs), (LDEORLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDEORALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDEORALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDEORALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDEORALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDEORALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDEORALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDEORALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDEORALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_8 GPR64:$Rn, GPR32:$Rs), (LDEORALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_16 GPR64:$Rn, GPR32:$Rs), (LDEORALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_32 GPR64:$Rn, GPR32:$Rs), (LDEORALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_xor_64 GPR64:$Rn, GPR64:$Rs), (LDEORALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_or_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDSETB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDSETH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDSETW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDSETX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_8_acquire GPR64:$Rn, GPR32:$Rs), (LDSETAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16_acquire GPR64:$Rn, GPR32:$Rs), (LDSETAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32_acquire GPR64:$Rn, GPR32:$Rs), (LDSETAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64_acquire GPR64:$Rn, GPR64:$Rs), (LDSETAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_8_release GPR64:$Rn, GPR32:$Rs), (LDSETLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16_release GPR64:$Rn, GPR32:$Rs), (LDSETLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32_release GPR64:$Rn, GPR32:$Rs), (LDSETLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64_release GPR64:$Rn, GPR64:$Rs), (LDSETLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSETALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSETALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSETALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDSETALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSETALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSETALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSETALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDSETALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_8 GPR64:$Rn, GPR32:$Rs), (LDSETALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_16 GPR64:$Rn, GPR32:$Rs), (LDSETALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_32 GPR64:$Rn, GPR32:$Rs), (LDSETALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_or_64 GPR64:$Rn, GPR64:$Rs), (LDSETALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_max_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMAXB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMAXH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMAXW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDSMAXX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_8_acquire GPR64:$Rn, GPR32:$Rs), (LDSMAXAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16_acquire GPR64:$Rn, GPR32:$Rs), (LDSMAXAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32_acquire GPR64:$Rn, GPR32:$Rs), (LDSMAXAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64_acquire GPR64:$Rn, GPR64:$Rs), (LDSMAXAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_8_release GPR64:$Rn, GPR32:$Rs), (LDSMAXLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16_release GPR64:$Rn, GPR32:$Rs), (LDSMAXLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32_release GPR64:$Rn, GPR32:$Rs), (LDSMAXLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64_release GPR64:$Rn, GPR64:$Rs), (LDSMAXLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDSMAXALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDSMAXALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_8 GPR64:$Rn, GPR32:$Rs), (LDSMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_16 GPR64:$Rn, GPR32:$Rs), (LDSMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_32 GPR64:$Rn, GPR32:$Rs), (LDSMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_max_64 GPR64:$Rn, GPR64:$Rs), (LDSMAXALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_min_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMINB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMINH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDSMINW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDSMINX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_8_acquire GPR64:$Rn, GPR32:$Rs), (LDSMINAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16_acquire GPR64:$Rn, GPR32:$Rs), (LDSMINAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32_acquire GPR64:$Rn, GPR32:$Rs), (LDSMINAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64_acquire GPR64:$Rn, GPR64:$Rs), (LDSMINAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_8_release GPR64:$Rn, GPR32:$Rs), (LDSMINLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16_release GPR64:$Rn, GPR32:$Rs), (LDSMINLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32_release GPR64:$Rn, GPR32:$Rs), (LDSMINLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64_release GPR64:$Rn, GPR64:$Rs), (LDSMINLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDSMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDSMINALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDSMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDSMINALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_8 GPR64:$Rn, GPR32:$Rs), (LDSMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_16 GPR64:$Rn, GPR32:$Rs), (LDSMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_32 GPR64:$Rn, GPR32:$Rs), (LDSMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_min_64 GPR64:$Rn, GPR64:$Rs), (LDSMINALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_umax_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMAXB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMAXH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMAXW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDUMAXX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_8_acquire GPR64:$Rn, GPR32:$Rs), (LDUMAXAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16_acquire GPR64:$Rn, GPR32:$Rs), (LDUMAXAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32_acquire GPR64:$Rn, GPR32:$Rs), (LDUMAXAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64_acquire GPR64:$Rn, GPR64:$Rs), (LDUMAXAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_8_release GPR64:$Rn, GPR32:$Rs), (LDUMAXLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16_release GPR64:$Rn, GPR32:$Rs), (LDUMAXLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32_release GPR64:$Rn, GPR32:$Rs), (LDUMAXLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64_release GPR64:$Rn, GPR64:$Rs), (LDUMAXLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDUMAXALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDUMAXALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_8 GPR64:$Rn, GPR32:$Rs), (LDUMAXALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_16 GPR64:$Rn, GPR32:$Rs), (LDUMAXALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_32 GPR64:$Rn, GPR32:$Rs), (LDUMAXALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umax_64 GPR64:$Rn, GPR64:$Rs), (LDUMAXALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_load_umin_8_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMINB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMINH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32_monotonic GPR64:$Rn, GPR32:$Rs), (LDUMINW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64_monotonic GPR64:$Rn, GPR64:$Rs), (LDUMINX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_8_acquire GPR64:$Rn, GPR32:$Rs), (LDUMINAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16_acquire GPR64:$Rn, GPR32:$Rs), (LDUMINAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32_acquire GPR64:$Rn, GPR32:$Rs), (LDUMINAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64_acquire GPR64:$Rn, GPR64:$Rs), (LDUMINAX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_8_release GPR64:$Rn, GPR32:$Rs), (LDUMINLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16_release GPR64:$Rn, GPR32:$Rs), (LDUMINLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32_release GPR64:$Rn, GPR32:$Rs), (LDUMINLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64_release GPR64:$Rn, GPR64:$Rs), (LDUMINLX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_8_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32_acq_rel GPR64:$Rn, GPR32:$Rs), (LDUMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64_acq_rel GPR64:$Rn, GPR64:$Rs), (LDUMINALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_8_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32_seq_cst GPR64:$Rn, GPR32:$Rs), (LDUMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64_seq_cst GPR64:$Rn, GPR64:$Rs), (LDUMINALX GPR64:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_8 GPR64:$Rn, GPR32:$Rs), (LDUMINALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_16 GPR64:$Rn, GPR32:$Rs), (LDUMINALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_32 GPR64:$Rn, GPR32:$Rs), (LDUMINALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_load_umin_64 GPR64:$Rn, GPR64:$Rs), (LDUMINALX GPR64:$Rs, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_cmp_swap_8_monotonic GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16_monotonic GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32_monotonic GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64_monotonic GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; + + def : Pat<(atomic_cmp_swap_8_acquire GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASAB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16_acquire GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASAH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32_acquire GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASAW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64_acquire GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASAX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; + + def : Pat<(atomic_cmp_swap_8_release GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASLB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16_release GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASLH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32_release GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASLW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64_release GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASLX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; + + def : Pat<(atomic_cmp_swap_8_acq_rel GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16_acq_rel GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32_acq_rel GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64_acq_rel GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; + + def : Pat<(atomic_cmp_swap_8_seq_cst GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16_seq_cst GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32_seq_cst GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64_seq_cst GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; + + def : Pat<(atomic_cmp_swap_8 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_16 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_32 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>; + def : Pat<(atomic_cmp_swap_64 GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>; +} + +let Predicates = [HasLSE] in { + def : Pat<(atomic_swap_8_monotonic GPR64:$Rn, GPR32:$Rs), (SWPB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16_monotonic GPR64:$Rn, GPR32:$Rs), (SWPH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32_monotonic GPR64:$Rn, GPR32:$Rs), (SWPW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64_monotonic GPR64:$Rn, GPR64:$Rs), (SWPX GPR64:$Rs, GPR64sp:$Rn)>; + + def : Pat<(atomic_swap_8_acquire GPR64:$Rn, GPR32:$Rs), (SWPAB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16_acquire GPR64:$Rn, GPR32:$Rs), (SWPAH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32_acquire GPR64:$Rn, GPR32:$Rs), (SWPAW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64_acquire GPR64:$Rn, GPR64:$Rs), (SWPAX GPR64:$Rs, GPR64sp:$Rn)>; + + def : Pat<(atomic_swap_8_release GPR64:$Rn, GPR32:$Rs), (SWPLB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16_release GPR64:$Rn, GPR32:$Rs), (SWPLH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32_release GPR64:$Rn, GPR32:$Rs), (SWPLW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64_release GPR64:$Rn, GPR64:$Rs), (SWPLX GPR64:$Rs, GPR64sp:$Rn)>; + + def : Pat<(atomic_swap_8_acq_rel GPR64:$Rn, GPR32:$Rs), (SWPALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16_acq_rel GPR64:$Rn, GPR32:$Rs), (SWPALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32_acq_rel GPR64:$Rn, GPR32:$Rs), (SWPALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64_acq_rel GPR64:$Rn, GPR64:$Rs), (SWPALX GPR64:$Rs, GPR64sp:$Rn)>; + + def : Pat<(atomic_swap_8_seq_cst GPR64:$Rn, GPR32:$Rs), (SWPALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16_seq_cst GPR64:$Rn, GPR32:$Rs), (SWPALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32_seq_cst GPR64:$Rn, GPR32:$Rs), (SWPALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64_seq_cst GPR64:$Rn, GPR64:$Rs), (SWPALX GPR64:$Rs, GPR64sp:$Rn)>; + + def : Pat<(atomic_swap_8 GPR64:$Rn, GPR32:$Rs), (SWPALB GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_16 GPR64:$Rn, GPR32:$Rs), (SWPALH GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_32 GPR64:$Rn, GPR32:$Rs), (SWPALW GPR32:$Rs, GPR64sp:$Rn)>; + def : Pat<(atomic_swap_64 GPR64:$Rn, GPR64:$Rs), (SWPALX GPR64:$Rs, GPR64sp:$Rn)>; +} + Index: lib/Target/AArch64/AArch64InstrFormats.td =================================================================== --- lib/Target/AArch64/AArch64InstrFormats.td +++ lib/Target/AArch64/AArch64InstrFormats.td @@ -9393,37 +9393,37 @@ : BaseCASEncoding<(outs RC:$out),(ins RC:$Rs, RC:$Rt, GPR64sp:$Rn), "cas" # order # size, "\t$Rs, $Rt, [$Rn]", "$out = $Rs",[]>, - Sched<[WriteAtomic]> { + Sched<[WriteAtomic, WriteLD, WriteST]> { let NP = 1; } multiclass CompareAndSwap Acq, bits<1> Rel, string order> { - let Sz = 0b00, Acq = Acq, Rel = Rel in def b : BaseCAS; - let Sz = 0b01, Acq = Acq, Rel = Rel in def h : BaseCAS; - let Sz = 0b10, Acq = Acq, Rel = Rel in def s : BaseCAS; - let Sz = 0b11, Acq = Acq, Rel = Rel in def d : BaseCAS; + let Sz = 0b00, Acq = Acq, Rel = Rel in def B : BaseCAS; + let Sz = 0b01, Acq = Acq, Rel = Rel in def H : BaseCAS; + let Sz = 0b10, Acq = Acq, Rel = Rel in def W : BaseCAS; + let Sz = 0b11, Acq = Acq, Rel = Rel in def X : BaseCAS; } class BaseCASP : BaseCASEncoding<(outs RC:$out),(ins RC:$Rs, RC:$Rt, GPR64sp:$Rn), "casp" # order # size, "\t$Rs, $Rt, [$Rn]", "$out = $Rs",[]>, - Sched<[WriteAtomic]> { + Sched<[WriteAtomic, WriteLD, WriteST]> { let NP = 0; } multiclass CompareAndSwapPair Acq, bits<1> Rel, string order> { - let Sz = 0b00, Acq = Acq, Rel = Rel in - def s : BaseCASP; - let Sz = 0b01, Acq = Acq, Rel = Rel in - def d : BaseCASP; + let Sz = 0b00, Acq = Acq, Rel = Rel in + def W : BaseCASP; + let Sz = 0b01, Acq = Acq, Rel = Rel in + def X : BaseCASP; } let Predicates = [HasLSE] in class BaseSWP : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "swp" # order # size, "\t$Rs, $Rt, [$Rn]","",[]>, - Sched<[WriteAtomic]> { + Sched<[WriteAtomic, WriteLD, WriteST]> { bits<2> Sz; bit Acq; bit Rel; @@ -9446,17 +9446,17 @@ } multiclass Swap Acq, bits<1> Rel, string order> { - let Sz = 0b00, Acq = Acq, Rel = Rel in def b : BaseSWP; - let Sz = 0b01, Acq = Acq, Rel = Rel in def h : BaseSWP; - let Sz = 0b10, Acq = Acq, Rel = Rel in def s : BaseSWP; - let Sz = 0b11, Acq = Acq, Rel = Rel in def d : BaseSWP; + let Sz = 0b00, Acq = Acq, Rel = Rel in def B : BaseSWP; + let Sz = 0b01, Acq = Acq, Rel = Rel in def H : BaseSWP; + let Sz = 0b10, Acq = Acq, Rel = Rel in def W : BaseSWP; + let Sz = 0b11, Acq = Acq, Rel = Rel in def X : BaseSWP; } let Predicates = [HasLSE], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in class BaseLDOPregister : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "ld" # op # order # size, "\t$Rs, $Rt, [$Rn]","",[]>, - Sched<[WriteAtomic]> { + Sched<[WriteAtomic, WriteLD, WriteST]> { bits<2> Sz; bit Acq; bit Rel; @@ -9478,16 +9478,16 @@ let Predicates = [HasLSE]; } -multiclass LDOPregister opc, string op, bits<1> Acq, bits<1> Rel, +multiclass LDOPregister opc, string op, bits<1> Acq, bits<1> Rel, string order> { - let Sz = 0b00, Acq = Acq, Rel = Rel, opc = opc in - def b : BaseLDOPregister; - let Sz = 0b01, Acq = Acq, Rel = Rel, opc = opc in - def h : BaseLDOPregister; - let Sz = 0b10, Acq = Acq, Rel = Rel, opc = opc in - def s : BaseLDOPregister; - let Sz = 0b11, Acq = Acq, Rel = Rel, opc = opc in - def d : BaseLDOPregister; + let Sz = 0b00, Acq = Acq, Rel = Rel, opc = opc in + def B : BaseLDOPregister; + let Sz = 0b01, Acq = Acq, Rel = Rel, opc = opc in + def H : BaseLDOPregister; + let Sz = 0b10, Acq = Acq, Rel = Rel, opc = opc in + def W : BaseLDOPregister; + let Sz = 0b11, Acq = Acq, Rel = Rel, opc = opc in + def X : BaseLDOPregister; } let Predicates = [HasLSE] in @@ -9496,22 +9496,42 @@ InstAlias; multiclass STOPregister { - def : BaseSTOPregister(instr # "Lb")>; - def : BaseSTOPregister(instr # "Lh")>; - def : BaseSTOPregister(instr # "Ls")>; - def : BaseSTOPregister(instr # "Ld")>; - def : BaseSTOPregister(instr # "b")>; - def : BaseSTOPregister(instr # "h")>; - def : BaseSTOPregister(instr # "s")>; - def : BaseSTOPregister(instr # "d")>; + def : BaseSTOPregister(instr # "LB")>; + def : BaseSTOPregister(instr # "LH")>; + def : BaseSTOPregister(instr # "LW")>; + def : BaseSTOPregister(instr # "LX")>; + def : BaseSTOPregister(instr # "B")>; + def : BaseSTOPregister(instr # "H")>; + def : BaseSTOPregister(instr # "W")>; + def : BaseSTOPregister(instr # "X")>; +} + +multiclass STOPregister_patterns_ord { + def : Pat<(!cast(op#"_"#size#"_monotonic") GPR64sp:$Rn, SRHS), + (!cast(inst#suffix) DRHS, GPR64sp:$Rn)>; + def : Pat<(!cast(op#"_"#size#"_release") GPR64sp:$Rn, SRHS), + (!cast(inst#suffix) DRHS, GPR64sp:$Rn)>; +} + +multiclass STOPregister_patterns { + defm : STOPregister_patterns_ord; + defm : STOPregister_patterns_ord; + defm : STOPregister_patterns_ord; + defm : STOPregister_patterns_ord; } //---------------------------------------------------------------------------- Index: lib/Target/AArch64/AArch64RegisterInfo.td =================================================================== --- lib/Target/AArch64/AArch64RegisterInfo.td +++ lib/Target/AArch64/AArch64RegisterInfo.td @@ -602,17 +602,18 @@ // ARMv8.1a atomic CASP register operands -def WSeqPairs : RegisterTuples<[sube32, subo32], +def WSeqPairs : RegisterTuples<[sube32, subo32], [(rotl GPR32, 0), (rotl GPR32, 1)]>; -def XSeqPairs : RegisterTuples<[sube64, subo64], +def XSeqPairs : RegisterTuples<[sube64, subo64], [(rotl GPR64, 0), (rotl GPR64, 1)]>; -def WSeqPairsClass : RegisterClass<"AArch64", [untyped], 32, - (add WSeqPairs)>{ +def WSeqPairsClass : RegisterClass<"AArch64", [untyped], 32, + (add WSeqPairs)> { let Size = 64; } -def XSeqPairsClass : RegisterClass<"AArch64", [untyped], 64, - (add XSeqPairs)>{ + +def XSeqPairsClass : RegisterClass<"AArch64", [untyped], 64, + (add XSeqPairs)> { let Size = 128; } Index: lib/Target/AArch64/AArch64SchedThunderX2T99.td =================================================================== --- lib/Target/AArch64/AArch64SchedThunderX2T99.td +++ lib/Target/AArch64/AArch64SchedThunderX2T99.td @@ -315,6 +315,36 @@ let NumMicroOps = 3; } +// 8 cycles on LS0 or LS1 and I0, I1, or I2. +def THX2T99Write_8Cyc_I012 : SchedWriteRes<[THX2T99LS01, THX2T99I012]> { + let Latency = 8; + let NumMicroOps = 4; +} + +// 12 cycles on LS0 or LS1 and I0, I1, or I2. +def THX2T99Write_12Cyc_I012 : SchedWriteRes<[THX2T99LS01, THX2T99I012]> { + let Latency = 12; + let NumMicroOps = 6; +} + +// 16 cycles on LS0 or LS1 and I0, I1, or I2. +def THX2T99Write_16Cyc_I012 : SchedWriteRes<[THX2T99LS01, THX2T99I012]> { + let Latency = 16; + let NumMicroOps = 8; +} + +// 24 cycles on LS0 or LS1 and I0, I1, or I2. +def THX2T99Write_24Cyc_I012 : SchedWriteRes<[THX2T99LS01, THX2T99I012]> { + let Latency = 24; + let NumMicroOps = 12; +} + +// 32 cycles on LS0 or LS1 and I0, I1, or I2. +def THX2T99Write_32Cyc_I012 : SchedWriteRes<[THX2T99LS01, THX2T99I012]> { + let Latency = 32; + let NumMicroOps = 16; +} + // Define commonly used read types. // No forwarding is provided for these types. @@ -1741,5 +1771,108 @@ def : InstRW<[THX2T99Write_1Cyc_LS01_F01, WriteAdr], (instregex "^ST4i(8|16|32|64)_POST$")>; +// V8.1a Atomics (LSE) +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs CASB, CASH, CASW, CASX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs CASAB, CASAH, CASAW, CASAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs CASLB, CASLH, CASLW, CASLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs CASALB, CASALH, CASALW, CASALX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDLARB, LDLARH, LDLARW, LDLARX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDADDB, LDADDH, LDADDW, LDADDX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDADDAB, LDADDAH, LDADDAW, LDADDAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDADDLB, LDADDLH, LDADDLW, LDADDLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs LDADDALB, LDADDALH, LDADDALW, LDADDALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDCLRB, LDCLRH, LDCLRW, LDCLRX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDCLRAB, LDCLRAH, LDCLRAW, LDCLRAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDCLRLB, LDCLRLH, LDCLRLW, LDCLRLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs LDCLRALB, LDCLRALH, LDCLRALW, LDCLRALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDEORB, LDEORH, LDEORW, LDEORX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDEORAB, LDEORAH, LDEORAW, LDEORAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDEORLB, LDEORLH, LDEORLW, LDEORLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs LDEORALB, LDEORALH, LDEORALW, LDEORALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDSETB, LDSETH, LDSETW, LDSETX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDSETAB, LDSETAH, LDSETAW, LDSETAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs LDSETLB, LDSETLH, LDSETLW, LDSETLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs LDSETALB, LDSETALH, LDSETALW, LDSETALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDSMAXB, LDSMAXH, LDSMAXW, LDSMAXX, + LDSMAXAB, LDSMAXAH, LDSMAXAW, LDSMAXAX, + LDSMAXLB, LDSMAXLH, LDSMAXLW, LDSMAXLX, + LDSMAXALB, LDSMAXALH, LDSMAXALW, LDSMAXALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDSMINB, LDSMINH, LDSMINW, LDSMINX, + LDSMINAB, LDSMINAH, LDSMINAW, LDSMINAX, + LDSMINLB, LDSMINLH, LDSMINLW, LDSMINLX, + LDSMINALB, LDSMINALH, LDSMINALW, LDSMINALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDUMAXB, LDUMAXH, LDUMAXW, LDUMAXX, + LDUMAXAB, LDUMAXAH, LDUMAXAW, LDUMAXAX, + LDUMAXLB, LDUMAXLH, LDUMAXLW, LDUMAXLX, + LDUMAXALB, LDUMAXALH, LDUMAXALW, LDUMAXALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs LDUMINB, LDUMINH, LDUMINW, LDUMINX, + LDUMINAB, LDUMINAH, LDUMINAW, LDUMINAX, + LDUMINLB, LDUMINLH, LDUMINLW, LDUMINLX, + LDUMINALB, LDUMINALH, LDUMINALW, LDUMINALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs SWPB, SWPH, SWPW, SWPX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs SWPAB, SWPAH, SWPAW, SWPAX)>; + +def : InstRW<[THX2T99Write_12Cyc_I012, WriteAtomic], + (instrs SWPLB, SWPLH, SWPLW, SWPLX)>; + +def : InstRW<[THX2T99Write_16Cyc_I012, WriteAtomic], + (instrs SWPALB, SWPALH, SWPALW, SWPALX)>; + +def : InstRW<[THX2T99Write_8Cyc_I012, WriteAtomic], + (instrs STLLRB, STLLRH, STLLRW, STLLRX)>; + } // SchedModel = ThunderX2T99Model