[AtomicExpand][X86] Let atomic expand generate inline sequence for unaligned load/store of atomic primitive integer types on x86_64

[wmi]

Summary

The patch is to fix PR34347.

Now llvm generates library calls of atomic_load/store for unaligned load/store of atomic primitive integer type vars on x86_64. Since x86_64 support unaligned atomic load/store for primitive integers, this is not efficient. The patch fixes that.

To fix PR34347, change in clang is also needed. That will be included in another patch.

include/llvm/Target/TargetLowering.h

public:
   }
 
   /// Returns how the IR-level AtomicExpand pass should expand the given
   /// AtomicRMW, if at all. Default is to never expand.
   virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const {
     return AtomicExpansionKind::None;
   }
 
+  /// Returns true if the given target supports lock-free atomic operations at
+  /// the specified width and alignment.
+  virtual bool hasBuiltinAtomic(uint64_t Size, uint64_t Align) const {
+    return Align >= Size && Size <= getMaxAtomicSizeInBitsSupported() / 8;
+  }
+
   /// On some platforms, an AtomicRMW that never actually modifies the value
   /// (such as fetch_add of 0) can be turned into a fence followed by an
   /// atomic load. This may sound useless, but it makes it possible for the
   /// processor to keep the cacheline shared, dramatically improving
   /// performance. And such idempotent RMWs are useful for implementing some
   /// kinds of locks, see for example (justification + benchmarks):
   /// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
   /// This method tries doing that transformation, returning the atomic load if

lib/CodeGen/AtomicExpandPass.cpp

 
 // Determine if a particular atomic operation has a supported size,
 // and is of appropriate alignment, to be passed through for target
 // lowering. (Versus turning into a __atomic libcall)
 template <typename Inst>
 bool atomicSizeSupported(const TargetLowering *TLI, Inst *I) {
   unsigned Size = getAtomicOpSize(I);
   unsigned Align = getAtomicOpAlign(I);
-  return Align >= Size && Size <= TLI->getMaxAtomicSizeInBitsSupported() / 8;
+  return TLI->hasBuiltinAtomic(Size, Align);
 }
 
 } // end anonymous namespace
 
 bool AtomicExpand::runOnFunction(Function &F) {
   auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
   if (!TPC)
     return false;

lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
   AtomicOrdering Order = I.getOrdering();
   SyncScope::ID SSID = I.getSyncScopeID();
 
   SDValue InChain = getRoot();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
 
-  if (I.getAlignment() < VT.getSizeInBits() / 8)
-    report_fatal_error("Cannot generate unaligned atomic load");
+  if (!TLI.hasBuiltinAtomic(VT.getSizeInBits() / 8, I.getAlignment()))
+    report_fatal_error("Cannot generate atomic load for the instruction");
 
   MachineMemOperand *MMO =
       DAG.getMachineFunction().
       getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
                            MachineMemOperand::MOVolatile |
                            MachineMemOperand::MOLoad,
                            VT.getStoreSize(),
                            I.getAlignment() ? I.getAlignment() :
void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
   SyncScope::ID SSID = I.getSyncScopeID();
 
   SDValue InChain = getRoot();
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT =
       TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
 
-  if (I.getAlignment() < VT.getSizeInBits() / 8)
-    report_fatal_error("Cannot generate unaligned atomic store");
+  if (!TLI.hasBuiltinAtomic(VT.getSizeInBits() / 8, I.getAlignment()))
+    report_fatal_error("Cannot generate atomic store for the instruction");
 
   SDValue OutChain =
     DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
                   InChain,
                   getValue(I.getPointerOperand()),
                   getValue(I.getValueOperand()),
                   I.getPointerOperand(), I.getAlignment(),
                   Order, SSID);

lib/Target/X86/X86ISelLowering.h

private:
     const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
 
     TargetLoweringBase::AtomicExpansionKind
     shouldExpandAtomicLoadInIR(LoadInst *SI) const override;
     bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
     TargetLoweringBase::AtomicExpansionKind
     shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
 
+    bool hasBuiltinAtomic(uint64_t Size, uint64_t Align) const override;
     LoadInst *
     lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
 
     bool needsCmpXchgNb(Type *MemType) const;
 
     void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
                                 MachineBasicBlock *DispatchBB, int FI) const;
 

lib/Target/X86/X86ISelLowering.cpp

for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
     setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
     setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
   }
 
   if (Subtarget.hasCmpxchg16b()) {
     setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
   }
+  setMaxAtomicSizeInBitsSupported(128);
 
   // FIXME - use subtarget debug flags
   if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
       !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
       TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
     setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
   }
 
X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   case AtomicRMWInst::UMax:
   case AtomicRMWInst::UMin:
     // These always require a non-trivial set of data operations on x86. We must
     // use a cmpxchg loop.
     return AtomicExpansionKind::CmpXChg;
   }
 }
 
+bool X86TargetLowering::hasBuiltinAtomic(uint64_t Size, uint64_t Align) const {
+  return Size <= getMaxAtomicSizeInBitsSupported() / 8;
+}
+
 LoadInst *
 X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
   unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
   Type *MemType = AI->getType();
   // Accesses larger than the native width are turned into cmpxchg/libcalls, so
   // there is no benefit in turning such RMWs into loads, and it is actually
   // harmful as it introduces a mfence.
   if (MemType->getPrimitiveSizeInBits() > NativeWidth)

test/Transforms/AtomicExpand/X86/expand-unaligned-expansion.ll

+; RUN: opt -S %s -atomic-expand -mtriple=x86_64-linux-gnu | FileCheck %s
+; RUN: opt -S %s -atomic-expand -mtriple=x86_64-linux-gnu -mattr=cx16 | FileCheck %s --check-prefix=CHECK-CX16
+; X86_64 supports unaligned atomic load/store for primitive integer types.
+; Check the unaligned atomic load/store are properly expanded.
+
+define i16 @test_unaligned_load_i16(i16* %ptr) {
+; CHECK-LABEL: @test_unaligned_load_i16
+; CHECK: %t0 = load atomic i16, i16* %ptr monotonic, align 1
+; CHECK: ret i16 %t0
+  %t0 = load atomic i16, i16* %ptr monotonic, align 1
+  ret i16 %t0
+}
+
+define i32 @test_unaligned_load_i32(i32* %ptr) {
+; CHECK-LABEL: @test_unaligned_load_i32
+; CHECK: %t0 = load atomic i32, i32* %ptr monotonic, align 2
+; CHECK: ret i32 %t0
+  %t0 = load atomic i32, i32* %ptr monotonic, align 2
+  ret i32 %t0
+}
+
+define i64 @test_unaligned_load_i64(i64* %ptr) {
+; CHECK-LABEL: @test_unaligned_load_i64
+; CHECK: %t0 = load atomic i64, i64* %ptr monotonic, align 4
+; CHECK: ret i64 %t0
+  %t0 = load atomic i64, i64* %ptr monotonic, align 4
+  ret i64 %t0
+}
+
+define i128 @test_unaligned_load_i128(i128* %ptr) {
+; CHECK-LABEL: @test_unaligned_load_i128
+; CHECK: %t0 = load atomic i128, i128* %ptr monotonic, align 8
+; CHECK: ret i128 %t0
+; CHECK-CX16: %[[XCHG:.*]] = cmpxchg i128* %ptr, i128 0, i128 0 monotonic monotonic
+; CHECK-CX16: %[[EXTR:.*]] = extractvalue { i128, i1 } %[[XCHG]], 0
+; CHECK-CX16: ret i128 %[[EXTR]]
+  %t0 = load atomic i128, i128* %ptr monotonic, align 8
+  ret i128 %t0
+}
+
+define void @test_unaligned_store_i16(i16 %val, i16* %ptr) {
+; CHECK-LABEL: @test_unaligned_store_i16
+; CHECK: store atomic i16 %val, i16* %ptr monotonic, align 1
+; CHECK: ret void
+  store atomic i16 %val, i16* %ptr monotonic, align 1
+  ret void
+}
+
+define void @test_unaligned_store_i32(i32 %val, i32* %ptr) {
+; CHECK-LABEL: @test_unaligned_store_i32
+; CHECK: store atomic i32 %val, i32* %ptr monotonic, align 2
+; CHECK: ret void
+  store atomic i32 %val, i32* %ptr monotonic, align 2
+  ret void
+}
+
+define void @test_unaligned_store_i64(i64 %val, i64* %ptr) {
+; CHECK-LABEL: @test_unaligned_store_i64
+; CHECK: store atomic i64 %val, i64* %ptr monotonic, align 4
+; CHECK: ret void
+  store atomic i64 %val, i64* %ptr monotonic, align 4
+  ret void
+}
+
+define void @test_unaligned_store_i128(i128 %val, i128* %ptr) {
+; CHECK-LABEL: @test_unaligned_store_i128
+; CHECK: store atomic i128 %val, i128* %ptr monotonic, align 4
+; CHECK: ret void
+; CHECK-CX16:   %[[LOAD:.*]] = load i128, i128* %ptr, align 16
+; CHECK-CX16:   br label %atomicrmw.start
+; CHECK-CX16: atomicrmw.start:
+; CHECK-CX16:   %loaded = phi i128 [ %[[LOAD]], %0 ], [ %newloaded, %atomicrmw.start ]
+; CHECK-CX16:   %[[XCHG:.*]] = cmpxchg i128* %ptr, i128 %loaded, i128 %val monotonic monotonic
+; CHECK-CX16:   %[[SUCC:.*]] = extractvalue { i128, i1 } %[[XCHG]], 1
+; CHECK-CX16:   %newloaded = extractvalue { i128, i1 } %[[XCHG]], 0
+; CHECK-CX16:   br i1 %[[SUCC]], label %atomicrmw.end, label %atomicrmw.start
+  store atomic i128 %val, i128* %ptr monotonic, align 4
+  ret void
+}