[LoongArch] Support finer-grained DBAR hints for LA664+

[SixWeining]

Summary

These are treated as DBAR 0 on older uarchs, so we can start to
unconditionally emit the new hints right away.

[SixWeining]

Only a few hints (i.e. 0x0, 0x10, 0x12, 0x14) are emitted. Is it possible to emit more?

Add @hev who is more experienced on dbar usage and generating especially on JVM.

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

264–278

Seems that tests are missing.

[hev]

Grateful for your contribution.

llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll

116

Is it better to use dbar 18 + store for LA664 and later micro-architectures?

269

We should use dbar with ordering kind here.

[SixWeining]

Revised by @hev and landed as https://github.com/llvm/llvm-project/commit/956482de13107b640cffedd08610fcccd98f708f.

This can be closed now.

llvm/lib/Target/LoongArch/LoongArchISelLowering.h

private:
   SDValue lowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerShiftRightParts(SDValue Op, SelectionDAG &DAG, bool IsSRA) const;
 
   MachineBasicBlock *
   EmitInstrWithCustomInserter(MachineInstr &MI,
                               MachineBasicBlock *BB) const override;
+  SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp

if (Subtarget.is64Bit()) {
     setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
     setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
     setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
     setOperationAction(ISD::INTRINSIC_VOID, MVT::i64, Custom);
   }
 
+  setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
+
   static const ISD::CondCode FPCCToExpand[] = {
       ISD::SETOGT, ISD::SETOGE, ISD::SETUGT, ISD::SETUGE,
       ISD::SETGE,  ISD::SETNE,  ISD::SETGT};
 
   if (Subtarget.hasBasicF()) {
     setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
     setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
     setOperationAction(ISD::BR_CC, MVT::f32, Expand);
bool LoongArchTargetLowering::isOffsetFoldingLegal(
   // it in the global address node. Later peephole optimisations may choose to
   // fold it back in when profitable.
   return false;
 }
 
 SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
                                                 SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
+  case ISD::ATOMIC_FENCE:
+    return lowerATOMIC_FENCE(Op, DAG);
   case ISD::EH_DWARF_CFA:
     return lowerEH_DWARF_CFA(Op, DAG);
   case ISD::GlobalAddress:
     return lowerGlobalAddress(Op, DAG);
   case ISD::GlobalTLSAddress:
     return lowerGlobalTLSAddress(Op, DAG);
   case ISD::INTRINSIC_WO_CHAIN:
     return lowerINTRINSIC_WO_CHAIN(Op, DAG);
SDValue LoongArchTargetLowering::LowerOperation(SDValue Op,
   case ISD::RETURNADDR:
     return lowerRETURNADDR(Op, DAG);
   case ISD::WRITE_REGISTER:
     return lowerWRITE_REGISTER(Op, DAG);
   }
   return SDValue();
 }
 
+SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  SyncScope::ID FenceSSID =
+      static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
+
+  // singlethread fences only synchronize with signal handlers on the same
+  // thread and thus only need to preserve instruction order, not actually
+  // enforce memory ordering.
+  if (FenceSSID == SyncScope::SingleThread)
+    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
+    return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
+
+  return Op;
+}
+
 SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
                                                      SelectionDAG &DAG) const {
 
   if (Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i32) {
     DAG.getContext()->emitError(
         "On LA64, only 64-bit registers can be written.");
     return Op.getOperand(0);
   }

llvm/lib/Target/LoongArch/LoongArchInstrInfo.td

 def : RegRegStPat<truncstorei8, STX_B, GPR, i64>;
 def : RegRegStPat<truncstorei16, STX_H, GPR, i64>;
 def : RegRegStPat<truncstorei32, STX_W, GPR, i64>;
 def : RegRegStPat<store, STX_D, GPR, i64>;
 } // Predicates = [IsLA64]
 
 /// Atomic loads and stores
 
-def : Pat<(atomic_fence timm, timm), (DBAR 0)>;
+// DBAR hint encoding for LA664 and later micro-architectures, paraphrased from
+// the Linux patch revealing it [1]:
+//
+// - Bit 4: kind of constraint (0: completion, 1: ordering)
+// - Bit 3: barrier for previous read (0: true, 1: false)
+// - Bit 2: barrier for previous write (0: true, 1: false)
+// - Bit 1: barrier for succeeding read (0: true, 1: false)
+// - Bit 0: barrier for succeeding write (0: true, 1: false)
+//
+// Hint 0x700: barrier for "read after read" from the same address, which is
+// e.g. needed by LL-SC loops on older models. (DBAR 0x700 behaves the same as
+// nop if such reordering is disabled on supporting newer models.)
+//
+// [1]: https://lore.kernel.org/loongarch/[e-mail]/
+//
+// Implementations without support for the finer-granularity hints simply treat
+// all as the full barrier (DBAR 0), so we can unconditionally start emiting the
+// more precise hints right away.
+
+def : Pat<(atomic_fence 4, timm), (DBAR 0b10100)>; // acquire
+def : Pat<(atomic_fence 5, timm), (DBAR 0b10010)>; // release
+def : Pat<(atomic_fence 6, timm), (DBAR 0b10000)>; // acqrel
+def : Pat<(atomic_fence 7, timm), (DBAR 0)>; // seqcst
 
 defm : LdPat<atomic_load_8, LD_B>;
 defm : LdPat<atomic_load_16, LD_H>;
 defm : LdPat<atomic_load_32, LD_W>;
 
 class release_seqcst_store<PatFrag base>
     : PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val), [{
   AtomicOrdering Ordering = cast<AtomicSDNode>(N)->getSuccessOrdering();

llvm/test/CodeGen/LoongArch/ir-instruction/fence.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
 ; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
 
 define void @fence_acquire() nounwind {
 ; LA32-LABEL: fence_acquire:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_acquire:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   fence acquire
   ret void
 }
 
 define void @fence_release() nounwind {
 ; LA32-LABEL: fence_release:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_release:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    ret
   fence release
   ret void
 }
 
 define void @fence_acq_rel() nounwind {
 ; LA32-LABEL: fence_acq_rel:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: fence_acq_rel:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 16
 ; LA64-NEXT:    ret
   fence acq_rel
   ret void
 }
 
 define void @fence_seq_cst() nounwind {
 ; LA32-LABEL: fence_seq_cst:
 ; LA32:       # %bb.0:

llvm/test/CodeGen/LoongArch/ir-instruction/load-store-atomic.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc --mtriple=loongarch32 < %s | FileCheck %s --check-prefix=LA32
 ; RUN: llc --mtriple=loongarch64 < %s | FileCheck %s --check-prefix=LA64
 
 define i8 @load_acquire_i8(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.b $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i8:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.b $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i8, ptr %ptr acquire, align 1
   ret i8 %val
 }
 
 define i16 @load_acquire_i16(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i16:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.h $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i16:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.h $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i16, ptr %ptr acquire, align 2
   ret i16 %val
 }
 
 define i32 @load_acquire_i32(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i32:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    ld.w $a0, $a0, 0
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 20
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i32:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.w $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i32, ptr %ptr acquire, align 4
   ret i32 %val
 }
 
 define i64 @load_acquire_i64(ptr %ptr) {
 ; LA32-LABEL: load_acquire_i64:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    addi.w $sp, $sp, -16
 ; LA32-NEXT:    .cfi_def_cfa_offset 16
 ; LA32-NEXT:    st.w $ra, $sp, 12 # 4-byte Folded Spill
 ; LA32-NEXT:    .cfi_offset 1, -4
 ; LA32-NEXT:    ori $a1, $zero, 2
 ; LA32-NEXT:    bl %plt(__atomic_load_8)
 ; LA32-NEXT:    ld.w $ra, $sp, 12 # 4-byte Folded Reload
 ; LA32-NEXT:    addi.w $sp, $sp, 16
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: load_acquire_i64:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    ld.d $a0, $a0, 0
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 20
 ; LA64-NEXT:    ret
   %val = load atomic i64, ptr %ptr acquire, align 8
   ret i64 %val
 }
 
 define void @store_release_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_release_i8:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.b $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_release_i8:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    st.b $a1, $a0, 0
 ; LA64-NEXT:    ret
   store atomic i8 %v, ptr %ptr release, align 1
   ret void
 }
 
 define void @store_release_i16(ptr %ptr, i16 signext %v) {
 ; LA32-LABEL: store_release_i16:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.h $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_release_i16:
 ; LA64:       # %bb.0:
-; LA64-NEXT:    dbar 0
+; LA64-NEXT:    dbar 18
 ; LA64-NEXT:    st.h $a1, $a0, 0
 ; LA64-NEXT:    ret
   store atomic i16 %v, ptr %ptr release, align 2
   ret void
 }
 
 define void @store_release_i32(ptr %ptr, i32 signext %v) {
 ; LA32-LABEL: store_release_i32:
 ; LA32:       # %bb.0:
-; LA32-NEXT:    dbar 0
+; LA32-NEXT:    dbar 18
 ; LA32-NEXT:    st.w $a1, $a0, 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_release_i32:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    amswap_db.w $zero, $a1, $a0
 ; LA64-NEXT:    ret
   store atomic i32 %v, ptr %ptr release, align 4
   ret void
 }
 
 define void @store_release_i64(ptr %ptr, i64 %v) {
 ; LA32-LABEL: store_release_i64:
 ; LA32:       # %bb.0:

 ; LA64-LABEL: store_monotonic_i64:
 ; LA64:       # %bb.0:
 ; LA64-NEXT:    st.d $a1, $a0, 0
 ; LA64-NEXT:    ret
   store atomic i64 %v, ptr %ptr monotonic, align 8
   ret void
 }
 
 define void @store_seq_cst_i8(ptr %ptr, i8 signext %v) {
 ; LA32-LABEL: store_seq_cst_i8:
 ; LA32:       # %bb.0:
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    st.b $a1, $a0, 0
 ; LA32-NEXT:    dbar 0
 ; LA32-NEXT:    ret
 ;
 ; LA64-LABEL: store_seq_cst_i8: