[X86] Prefer `lock or` over mfence.

[vchuravy]

Summary

LLVM currently emits mfence for __atomic_thread_fence(seq_cst). On
modern CPUs lock or is more efficient and provides the same sequential
consistency. GCC 11 made this switch as well (see https://gcc.gnu.org/pipermail/gcc-cvs/2020-July/314418.html)
and https://reviews.llvm.org/D61863 and https://reviews.llvm.org/D58632
moved into this direction as well, but didn't touch fence seq_cst.

[loladiro]

I just looked at this again also, since this was requested in https://github.com/JuliaLang/julia/pull/48123 and came to the same conclusion, so LGTM, but it would be good to know from @craig.topper or @reames if there was any reason this wasn't done in D61863.

[vchuravy]

We talked at LLVMdev and the reason why it wasn't done where non-temporal memory ops. The LLVM langref and the C standard says nothing about them, but currently this is the only way to obtain a fence operation that affects them.

They asked me to do a bit of canvasing to find out if folks rely on this and/or wait to see how the GCC change shake out.

[loladiro]

Hmm, both semantics seems reasonable, and I don't think we can just make that decision for the frontend here. Perhaps at the IR level, we need a different syncscope property that declares whether it's expected to synchronize with non-temporal operations or not and then in clang we set it properly to match GCC (potentially with a matching __builtin_nontemporal_fence()?). I can see that frontends that make use of !nontemporal would expect fence to synchronize it.

llvm/lib/Target/X86/X86.td

 def TuningUseSLMArithCosts
     : SubtargetFeature<"use-slm-arith-costs", "UseSLMArithCosts", "true",
         "Use Silvermont specific arithmetic costs">;
 
 def TuningUseGLMDivSqrtCosts
     : SubtargetFeature<"use-glm-div-sqrt-costs", "UseGLMDivSqrtCosts", "true",
         "Use Goldmont specific floating point div/sqrt costs">;
 
+def TuningAvoidMFENCE
+   : SubtargetFeature<"avoid-mfence", "AvoidMFence", "true",
+        "Avoid MFENCE for fence seq_cst, and instead use lock or">;
+
 //===----------------------------------------------------------------------===//
 // X86 CPU Families
 // TODO: Remove these - use general tuning features to determine codegen.
 //===----------------------------------------------------------------------===//
 
 // Bonnell
 def ProcIntelAtom : SubtargetFeature<"", "IsAtom", "true", "Is Intel Atom processor">;
 
list<SubtargetFeature> X86_64V4Features = !listconcat(X86_64V3Features, [
     FeatureCDI,
     FeatureDQI,
     FeatureVLX,
   ]);
 
   // Nehalem
   list<SubtargetFeature> NHMFeatures = X86_64V2Features;
   list<SubtargetFeature> NHMTuning = [TuningMacroFusion,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
 
   // Westmere
   list<SubtargetFeature> WSMAdditionalFeatures = [FeaturePCLMUL];
   list<SubtargetFeature> WSMTuning = NHMTuning;
   list<SubtargetFeature> WSMFeatures =
     !listconcat(NHMFeatures, WSMAdditionalFeatures);
 
   // Sandybridge
   list<SubtargetFeature> SNBAdditionalFeatures = [FeatureAVX,
                                                   FeatureXSAVE,
                                                   FeatureXSAVEOPT];
   list<SubtargetFeature> SNBTuning = [TuningMacroFusion,
                                       TuningSlow3OpsLEA,
                                       TuningSlowDivide64,
                                       TuningSlowUAMem32,
                                       TuningFastScalarFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningPOPCNTFalseDeps,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> SNBFeatures =
     !listconcat(WSMFeatures, SNBAdditionalFeatures);
 
   // Ivybridge
   list<SubtargetFeature> IVBAdditionalFeatures = [FeatureRDRAND,
                                                   FeatureF16C,
                                                   FeatureFSGSBase];
   list<SubtargetFeature> IVBTuning = SNBTuning;
list<SubtargetFeature> HSWTuning = [TuningMacroFusion,
                                       TuningSlowDivide64,
                                       TuningFastScalarFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningFastVariableCrossLaneShuffle,
                                       TuningFastVariablePerLaneShuffle,
                                       TuningPOPCNTFalseDeps,
                                       TuningLZCNTFalseDeps,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> HSWFeatures =
     !listconcat(IVBFeatures, HSWAdditionalFeatures);
 
   // Broadwell
   list<SubtargetFeature> BDWAdditionalFeatures = [FeatureADX,
                                                   FeatureRDSEED,
                                                   FeaturePRFCHW];
   list<SubtargetFeature> BDWTuning = HSWTuning;
list<SubtargetFeature> SKLTuning = [TuningFastGather,
                                       TuningSlowDivide64,
                                       TuningFastScalarFSQRT,
                                       TuningFastVectorFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningFastVariableCrossLaneShuffle,
                                       TuningFastVariablePerLaneShuffle,
                                       TuningPOPCNTFalseDeps,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> SKLFeatures =
     !listconcat(BDWFeatures, SKLAdditionalFeatures);
 
   // Skylake-AVX512
   list<SubtargetFeature> SKXAdditionalFeatures = [FeatureAES,
                                                   FeatureXSAVEC,
                                                   FeatureXSAVES,
                                                   FeatureCLFLUSHOPT,
list<SubtargetFeature> SKXTuning = [TuningFastGather,
                                       TuningFastScalarFSQRT,
                                       TuningFastVectorFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningFastVariableCrossLaneShuffle,
                                       TuningFastVariablePerLaneShuffle,
                                       TuningPrefer256Bit,
                                       TuningPOPCNTFalseDeps,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> SKXFeatures =
     !listconcat(BDWFeatures, SKXAdditionalFeatures);
 
   // Cascadelake
   list<SubtargetFeature> CLXAdditionalFeatures = [FeatureVNNI];
   list<SubtargetFeature> CLXTuning = SKXTuning;
   list<SubtargetFeature> CLXFeatures =
     !listconcat(SKXFeatures, CLXAdditionalFeatures);
list<SubtargetFeature> CNLTuning = [TuningFastGather,
                                       TuningSlowDivide64,
                                       TuningFastScalarFSQRT,
                                       TuningFastVectorFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningFastVariableCrossLaneShuffle,
                                       TuningFastVariablePerLaneShuffle,
                                       TuningPrefer256Bit,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> CNLFeatures =
     !listconcat(SKLFeatures, CNLAdditionalFeatures);
 
   // Icelake
   list<SubtargetFeature> ICLAdditionalFeatures = [FeatureBITALG,
                                                   FeatureVAES,
                                                   FeatureVBMI2,
                                                   FeatureVNNI,
                                                   FeatureVPCLMULQDQ,
                                                   FeatureVPOPCNTDQ,
                                                   FeatureGFNI,
                                                   FeatureRDPID,
                                                   FeatureFSRM];
   list<SubtargetFeature> ICLTuning = [TuningFastGather,
                                       TuningMacroFusion,
                                       TuningSlow3OpsLEA,
                                       TuningSlowDivide64,
                                       TuningFastScalarFSQRT,
                                       TuningFastVectorFSQRT,
                                       TuningFastSHLDRotate,
                                       TuningFast15ByteNOP,
                                       TuningFastVariableCrossLaneShuffle,
                                       TuningFastVariablePerLaneShuffle,
                                       TuningPrefer256Bit,
-                                      TuningInsertVZEROUPPER];
+                                      TuningInsertVZEROUPPER,
+                                      TuningAvoidMFENCE];
   list<SubtargetFeature> ICLFeatures =
     !listconcat(CNLFeatures, ICLAdditionalFeatures);
 
   // Icelake Server
   list<SubtargetFeature> ICXAdditionalFeatures = [FeaturePCONFIG,
                                                   FeatureCLWB,
                                                   FeatureWBNOINVD];
   list<SubtargetFeature> ICXTuning = ICLTuning;
list<SubtargetFeature> GLPTuning = [TuningUseGLMDivSqrtCosts,
                                       TuningFastMOVBE,
                                       TuningInsertVZEROUPPER];
   list<SubtargetFeature> GLPFeatures =
     !listconcat(GLMFeatures, GLPAdditionalFeatures);
 
   // Tremont
   list<SubtargetFeature> TRMAdditionalFeatures = [FeatureCLWB,
                                                   FeatureGFNI];
-  list<SubtargetFeature> TRMTuning = GLPTuning;
+  list<SubtargetFeature> TRMAdditionalTuning = [TuningAvoidMFENCE];
+  list<SubtargetFeature> TRMTuning = !listconcat(GLPTuning, TRMAdditionalTuning);
   list<SubtargetFeature> TRMFeatures =
     !listconcat(GLPFeatures, TRMAdditionalFeatures);
 
   // Alderlake
   list<SubtargetFeature> ADLAdditionalFeatures = [FeatureSERIALIZE,
                                                   FeaturePCONFIG,
                                                   FeatureSHSTK,
                                                   FeatureWIDEKL,
list<SubtargetFeature> BdVer1Features = [FeatureX87,
                                            FeatureXSAVE,
                                            FeatureLWP,
                                            FeatureLAHFSAHF64];
   list<SubtargetFeature> BdVer1Tuning = [TuningSlowSHLD,
                                          TuningFast11ByteNOP,
                                          TuningFastScalarShiftMasks,
                                          TuningBranchFusion,
                                          TuningSBBDepBreaking,
-                                         TuningInsertVZEROUPPER];
+                                         TuningInsertVZEROUPPER,
+                                         TuningAvoidMFENCE];
 
   // PileDriver
   list<SubtargetFeature> BdVer2AdditionalFeatures = [FeatureF16C,
                                                      FeatureBMI,
                                                      FeatureTBM,
                                                      FeatureFMA];
   list<SubtargetFeature> BdVer2AdditionalTuning = [TuningFastBEXTR,
                                                    TuningFastMOVBE];
list<SubtargetFeature> ZNTuning = [TuningFastLZCNT,
                                      TuningBranchFusion,
                                      TuningFastScalarFSQRT,
                                      TuningFastVectorFSQRT,
                                      TuningFastScalarShiftMasks,
                                      TuningFastVariablePerLaneShuffle,
                                      TuningFastMOVBE,
                                      TuningSlowSHLD,
                                      TuningSBBDepBreaking,
-                                     TuningInsertVZEROUPPER];
+                                     TuningInsertVZEROUPPER,
+                                     TuningAvoidMFENCE];
   list<SubtargetFeature> ZN2AdditionalFeatures = [FeatureCLWB,
                                                   FeatureRDPID,
                                                   FeatureRDPRU,
                                                   FeatureWBNOINVD];
   list<SubtargetFeature> ZN2Tuning = ZNTuning;
   list<SubtargetFeature> ZN2Features =
     !listconcat(ZNFeatures, ZN2AdditionalFeatures);
   list<SubtargetFeature> ZN3AdditionalFeatures = [FeatureFSRM,
def : ProcModel<"core2", SandyBridgeModel, [
   FeatureNOPL,
   FeatureX86_64,
   FeatureCX16,
   FeatureLAHFSAHF64
 ],
 [
   TuningMacroFusion,
   TuningSlowUAMem16,
-  TuningInsertVZEROUPPER
+  TuningInsertVZEROUPPER,
+  TuningAvoidMFENCE
 ]>;
 def : ProcModel<"penryn", SandyBridgeModel, [
   FeatureX87,
   FeatureCX8,
   FeatureCMOV,
   FeatureMMX,
   FeatureSSE41,
   FeatureFXSR,
   FeatureNOPL,
   FeatureX86_64,
   FeatureCX16,
   FeatureLAHFSAHF64
 ],
 [
   TuningMacroFusion,
   TuningSlowUAMem16,
-  TuningInsertVZEROUPPER
+  TuningInsertVZEROUPPER,
+  TuningAvoidMFENCE
 ]>;
 
 // Atom CPUs.
 foreach P = ["bonnell", "atom"] in {
   def : ProcModel<P, AtomModel, ProcessorFeatures.AtomFeatures,
                   ProcessorFeatures.AtomTuning>;
 }
 

llvm/lib/Target/X86/X86ISelLowering.cpp

AtomicOrdering FenceOrdering =
       static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
   SyncScope::ID FenceSSID =
       static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
 
   // The only fence that needs an instruction is a sequentially-consistent
   // cross-thread fence.
   if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
       FenceSSID == SyncScope::System) {
-    if (Subtarget.hasMFence())
+    if (!Subtarget.avoidMFence() && Subtarget.hasMFence())
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
     SDValue Chain = Op.getOperand(0);
     return emitLockedStackOp(DAG, Subtarget, Chain, dl);
   }
 
   // MEMBARRIER is a compiler barrier; it codegens to a no-op.
   return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));

llvm/lib/Target/X86/X86TargetTransformInfo.h

const FeatureBitset InlineFeatureIgnoreList = {
       X86::TuningSlowPMULLD,
       X86::TuningSlowSHLD,
       X86::TuningSlowTwoMemOps,
       X86::TuningSlowUAMem16,
       X86::TuningPreferMaskRegisters,
       X86::TuningInsertVZEROUPPER,
       X86::TuningUseSLMArithCosts,
       X86::TuningUseGLMDivSqrtCosts,
+      X86::TuningAvoidMFENCE,
 
       // Perf-tuning flags.
       X86::TuningFastGather,
       X86::TuningSlowUAMem32,
 
       // Based on whether user set the -mprefer-vector-width command line.
       X86::TuningPrefer128Bit,
       X86::TuningPrefer256Bit,

llvm/test/CodeGen/X86/atomic-unordered.ll

 ;; part of folding, but is technically legal to lift the add above the fence.
 ;; If that were to happen, please rewrite the test to ensure load movement
 ;; isn't violated.
 
 define i64 @nofold_fence(i64* %p) {
 ; CHECK-LABEL: nofold_fence:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq (%rdi), %rax
-; CHECK-NEXT:    mfence
+; CHECK-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    addq $15, %rax
 ; CHECK-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8
   fence seq_cst
   %ret = add i64 %v, 15
   ret i64 %ret
 }
 
; CHECK-O3-EX-NEXT:    retq
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
 define i64 @fold_constant_fence(i64 %arg) {
 ; CHECK-O0-LABEL: fold_constant_fence:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq Constant(%rip), %rax
-; CHECK-O0-NEXT:    mfence
+; CHECK-O0-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    addq %rdi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-CUR-LABEL: fold_constant_fence:
 ; CHECK-O3-CUR:       # %bb.0:
 ; CHECK-O3-CUR-NEXT:    movq Constant(%rip), %rax
-; CHECK-O3-CUR-NEXT:    mfence
+; CHECK-O3-CUR-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O3-CUR-NEXT:    addq %rdi, %rax
 ; CHECK-O3-CUR-NEXT:    retq
 ;
 ; CHECK-O3-EX-LABEL: fold_constant_fence:
 ; CHECK-O3-EX:       # %bb.0:
 ; CHECK-O3-EX-NEXT:    movq %rdi, %rax
 ; CHECK-O3-EX-NEXT:    addq Constant(%rip), %rax
-; CHECK-O3-EX-NEXT:    mfence
+; CHECK-O3-EX-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* @Constant unordered, align 8
   fence seq_cst
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
 define i64 @fold_invariant_clobber(i64* dereferenceable(8) %p, i64 %arg) {
; CHECK-O3-EX-NEXT:    retq
   ret i64 %ret
 }
 
 
 define i64 @fold_invariant_fence(i64* dereferenceable(8) %p, i64 %arg) {
 ; CHECK-O0-LABEL: fold_invariant_fence:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movq (%rdi), %rax
-; CHECK-O0-NEXT:    mfence
+; CHECK-O0-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    addq %rsi, %rax
 ; CHECK-O0-NEXT:    retq
 ;
 ; CHECK-O3-CUR-LABEL: fold_invariant_fence:
 ; CHECK-O3-CUR:       # %bb.0:
 ; CHECK-O3-CUR-NEXT:    movq (%rdi), %rax
-; CHECK-O3-CUR-NEXT:    mfence
+; CHECK-O3-CUR-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O3-CUR-NEXT:    addq %rsi, %rax
 ; CHECK-O3-CUR-NEXT:    retq
 ;
 ; CHECK-O3-EX-LABEL: fold_invariant_fence:
 ; CHECK-O3-EX:       # %bb.0:
 ; CHECK-O3-EX-NEXT:    movq %rsi, %rax
 ; CHECK-O3-EX-NEXT:    addq (%rdi), %rax
-; CHECK-O3-EX-NEXT:    mfence
+; CHECK-O3-EX-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O3-EX-NEXT:    retq
   %v = load atomic i64, i64* %p unordered, align 8, !invariant.load !{}
   fence seq_cst
   %ret = add i64 %v, %arg
   ret i64 %ret
 }
 
 
; CHECK-O3-NEXT:    retq
   %res = or i16 %v1.ext, %v2.sht
   ret i16 %res
 }
 
 define i1 @fold_cmp_over_fence(i32* %p, i32 %v1) {
 ; CHECK-O0-LABEL: fold_cmp_over_fence:
 ; CHECK-O0:       # %bb.0:
 ; CHECK-O0-NEXT:    movl (%rdi), %eax
-; CHECK-O0-NEXT:    mfence
+; CHECK-O0-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
 ; CHECK-O0-NEXT:    cmpl %eax, %esi
 ; CHECK-O0-NEXT:    jne .LBB116_2
 ; CHECK-O0-NEXT:  # %bb.1: # %taken
 ; CHECK-O0-NEXT:    movb $1, %al
 ; CHECK-O0-NEXT:    retq
 ; CHECK-O0-NEXT:  .LBB116_2: # %untaken
 ; CHECK-O0-NEXT:    xorl %eax, %eax
 ; CHECK-O0-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-O0-NEXT:    retq
 ;
-; CHECK-O3-CUR-LABEL: fold_cmp_over_fence:
-; CHECK-O3-CUR:       # %bb.0:
-; CHECK-O3-CUR-NEXT:    movl (%rdi), %eax
-; CHECK-O3-CUR-NEXT:    mfence
-; CHECK-O3-CUR-NEXT:    cmpl %eax, %esi
-; CHECK-O3-CUR-NEXT:    jne .LBB116_2
-; CHECK-O3-CUR-NEXT:  # %bb.1: # %taken
-; CHECK-O3-CUR-NEXT:    movb $1, %al
-; CHECK-O3-CUR-NEXT:    retq
-; CHECK-O3-CUR-NEXT:  .LBB116_2: # %untaken
-; CHECK-O3-CUR-NEXT:    xorl %eax, %eax
-; CHECK-O3-CUR-NEXT:    retq
-;
-; CHECK-O3-EX-LABEL: fold_cmp_over_fence:
-; CHECK-O3-EX:       # %bb.0:
-; CHECK-O3-EX-NEXT:    cmpl (%rdi), %esi
-; CHECK-O3-EX-NEXT:    mfence
-; CHECK-O3-EX-NEXT:    jne .LBB116_2
-; CHECK-O3-EX-NEXT:  # %bb.1: # %taken
-; CHECK-O3-EX-NEXT:    movb $1, %al
-; CHECK-O3-EX-NEXT:    retq
-; CHECK-O3-EX-NEXT:  .LBB116_2: # %untaken
-; CHECK-O3-EX-NEXT:    xorl %eax, %eax
-; CHECK-O3-EX-NEXT:    retq
+; CHECK-O3-LABEL: fold_cmp_over_fence:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl (%rdi), %eax
+; CHECK-O3-NEXT:    lock orl $0, -{{[0-9]+}}(%rsp)
+; CHECK-O3-NEXT:    cmpl %eax, %esi
+; CHECK-O3-NEXT:    jne .LBB116_2
+; CHECK-O3-NEXT:  # %bb.1: # %taken
+; CHECK-O3-NEXT:    movb $1, %al
+; CHECK-O3-NEXT:    retq
+; CHECK-O3-NEXT:  .LBB116_2: # %untaken
+; CHECK-O3-NEXT:    xorl %eax, %eax
+; CHECK-O3-NEXT:    retq
   %v2 = load atomic i32, i32* %p unordered, align 4
   fence seq_cst
   %cmp = icmp eq i32 %v1, %v2
   br i1 %cmp, label %taken, label %untaken
 taken:
   ret i1 true
 untaken:
   ret i1 false
 }