[SPIR-V] Add atomic_init builtin and fix atomic explicit lowering

[mpaszkowski]

Summary

This change:

• Adds implementation for atomic_init builtin
• Fixes atomic_exchange_explicit lowering
• Fixes atomic_store_explicit lowering
• Adds a new line char to LLVM_DEBUG messages printed from SPIRVBuiltins.cpp

The AtomicsBuiltinsFloat.ll test comes from the SPIR-V translator and was extended with additional checks.

[iliya-diyachkov]

llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll

9

As I know we use %[[#]] instead of {{%.+}}.

[mpaszkowski]

Thanks @iliya-diyachkov! The formatting is now fixed.

[iliya-diyachkov]

It looks good to me. Thank you for the patch, Michal.

[andreytr]

llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll

9

Just small indentation fix, please.

[mpaszkowski]

llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll

9

Sure, will fix before committing the change!

llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp

 
 static Register buildConstantIntReg(uint64_t Val, MachineIRBuilder &MIRBuilder,
                                     SPIRVGlobalRegistry *GR,
                                     unsigned BitWidth = 32) {
   SPIRVType *IntType = GR->getOrCreateSPIRVIntegerType(BitWidth, MIRBuilder);
   return GR->buildConstantInt(Val, MIRBuilder, IntType);
 }
 
+/// Helper function for translating atomic init to OpStore.
+static bool buildAtomicInitInst(const SPIRV::IncomingCall *Call,
+                                MachineIRBuilder &MIRBuilder) {
+  assert(Call->Arguments.size() == 2 &&
+         "Need 2 arguments for atomic init translation");
+
+  MIRBuilder.buildInstr(SPIRV::OpStore)
+      .addUse(Call->Arguments[0])
+      .addUse(Call->Arguments[1]);
+  return true;
+}
+
 /// Helper function for building an atomic load instruction.
 static bool buildAtomicLoadInst(const SPIRV::IncomingCall *Call,
                                 MachineIRBuilder &MIRBuilder,
                                 SPIRVGlobalRegistry *GR) {
   Register PtrRegister = Call->Arguments[0];
   // TODO: if true insert call to __translate_ocl_memory_sccope before
   // OpAtomicLoad and the function implementation. We can use Translator's
   // output for transcoding/atomic_explicit_arguments.cl as an example.
static bool buildAtomicRMWInst(const SPIRV::IncomingCall *Call, unsigned Opcode,
                                MachineIRBuilder &MIRBuilder,
                                SPIRVGlobalRegistry *GR) {
   const MachineRegisterInfo *MRI = MIRBuilder.getMRI();
   Register ScopeRegister;
   SPIRV::Scope::Scope Scope = SPIRV::Scope::Workgroup;
   if (Call->Arguments.size() >= 4) {
     assert(Call->Arguments.size() == 4 && "Extra args for explicit atomic RMW");
     auto CLScope = static_cast<SPIRV::CLMemoryScope>(
-        getIConstVal(Call->Arguments[5], MRI));
+        getIConstVal(Call->Arguments[3], MRI));
     Scope = getSPIRVScope(CLScope);
     if (CLScope == static_cast<unsigned>(Scope))
-      ScopeRegister = Call->Arguments[5];
+      ScopeRegister = Call->Arguments[3];
   }
   if (!ScopeRegister.isValid())
     ScopeRegister = buildConstantIntReg(Scope, MIRBuilder, GR);
 
   Register PtrRegister = Call->Arguments[0];
   Register MemSemanticsReg;
   unsigned Semantics = SPIRV::MemorySemantics::None;
   if (Call->Arguments.size() >= 3) {
     std::memory_order Order =
         static_cast<std::memory_order>(getIConstVal(Call->Arguments[2], MRI));
     Semantics =
         getSPIRVMemSemantics(Order) |
         getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister));
     if (Order == Semantics)
-      MemSemanticsReg = Call->Arguments[3];
+      MemSemanticsReg = Call->Arguments[2];
   }
   if (!MemSemanticsReg.isValid())
     MemSemanticsReg = buildConstantIntReg(Semantics, MIRBuilder, GR);
 
   MIRBuilder.buildInstr(Opcode)
       .addDef(Call->ReturnRegister)
       .addUse(GR->getSPIRVTypeID(Call->ReturnType))
       .addUse(PtrRegister)
static bool generateAtomicInst(const SPIRV::IncomingCall *Call,
                                MachineIRBuilder &MIRBuilder,
                                SPIRVGlobalRegistry *GR) {
   // Lookup the instruction opcode in the TableGen records.
   const SPIRV::DemangledBuiltin *Builtin = Call->Builtin;
   unsigned Opcode =
       SPIRV::lookupNativeBuiltin(Builtin->Name, Builtin->Set)->Opcode;
 
   switch (Opcode) {
+  case SPIRV::OpStore:
+    return buildAtomicInitInst(Call, MIRBuilder);
   case SPIRV::OpAtomicLoad:
     return buildAtomicLoadInst(Call, MIRBuilder, GR);
   case SPIRV::OpAtomicStore:
     return buildAtomicStoreInst(Call, MIRBuilder, GR);
   case SPIRV::OpAtomicCompareExchange:
   case SPIRV::OpAtomicCompareExchangeWeak:
     return buildAtomicCompareExchangeInst(Call, MIRBuilder, GR);
   case SPIRV::OpAtomicIAdd:
   case SPIRV::OpAtomicISub:
   case SPIRV::OpAtomicOr:
   case SPIRV::OpAtomicXor:
   case SPIRV::OpAtomicAnd:
+  case SPIRV::OpAtomicExchange:
     return buildAtomicRMWInst(Call, Opcode, MIRBuilder, GR);
   case SPIRV::OpMemoryBarrier:
     return buildBarrierInst(Call, SPIRV::OpMemoryBarrier, MIRBuilder, GR);
   default:
     return false;
   }
 }
 
if (OrigRetTy && !OrigRetTy->isVoidTy()) {
     ReturnType = GR->assignTypeToVReg(OrigRetTy, ReturnRegister, MIRBuilder);
   }
 
   // Lookup the builtin in the TableGen records.
   std::unique_ptr<const IncomingCall> Call =
       lookupBuiltin(DemangledCall, Set, ReturnRegister, ReturnType, Args);
 
   if (!Call) {
-    LLVM_DEBUG(dbgs() << "Builtin record was not found!");
-    return {};
+    LLVM_DEBUG(dbgs() << "Builtin record was not found!\n");
+    return None;
   }
 
   // TODO: check if the provided args meet the builtin requirments.
   assert(Args.size() >= Call->Builtin->MinNumArgs &&
          "Too few arguments to generate the builtin");
-  if (Call->Builtin->MaxNumArgs && Args.size() <= Call->Builtin->MaxNumArgs)
-    LLVM_DEBUG(dbgs() << "More arguments provided than required!");
+  if (Call->Builtin->MaxNumArgs && Args.size() > Call->Builtin->MaxNumArgs)
+    LLVM_DEBUG(dbgs() << "More arguments provided than required!\n");
 
   // Match the builtin with implementation based on the grouping.
   switch (Call->Builtin->Group) {
   case SPIRV::Extended:
     return generateExtInst(Call.get(), MIRBuilder, GR);
   case SPIRV::Relational:
     return generateRelationalInst(Call.get(), MIRBuilder, GR);
   case SPIRV::Group:

llvm/lib/Target/SPIRV/SPIRVBuiltins.td

 defm : DemangledNativeBuiltin<"signbit", OpenCL_std, Relational, 1, 1, OpSignBitSet>;
 defm : DemangledNativeBuiltin<"__spirv_SignBitSet", OpenCL_std, Relational, 1, 1, OpSignBitSet>;
 defm : DemangledNativeBuiltin<"any", OpenCL_std, Relational, 1, 1, OpAny>;
 defm : DemangledNativeBuiltin<"__spirv_Any", OpenCL_std, Relational, 1, 1, OpAny>;
 defm : DemangledNativeBuiltin<"all", OpenCL_std, Relational, 1, 1, OpAll>;
 defm : DemangledNativeBuiltin<"__spirv_All", OpenCL_std, Relational, 1, 1, OpAll>;
 
 // Atomic builtin records:
+defm : DemangledNativeBuiltin<"atomic_init", OpenCL_std, Atomic, 2, 2, OpStore>;
 defm : DemangledNativeBuiltin<"atomic_load", OpenCL_std, Atomic, 1, 1, OpAtomicLoad>;
 defm : DemangledNativeBuiltin<"atomic_load_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicLoad>;
 defm : DemangledNativeBuiltin<"atomic_store", OpenCL_std, Atomic, 2, 2, OpAtomicStore>;
-defm : DemangledNativeBuiltin<"atomic_store_explicit", OpenCL_std, Atomic, 2, 2, OpAtomicStore>;
+defm : DemangledNativeBuiltin<"atomic_store_explicit", OpenCL_std, Atomic, 2, 4, OpAtomicStore>;
 defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
 defm : DemangledNativeBuiltin<"atomic_compare_exchange_strong_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchange>;
 defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchangeWeak>;
 defm : DemangledNativeBuiltin<"atomic_compare_exchange_weak_explicit", OpenCL_std, Atomic, 5, 6, OpAtomicCompareExchangeWeak>;
 defm : DemangledNativeBuiltin<"atom_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
 defm : DemangledNativeBuiltin<"atomic_cmpxchg", OpenCL_std, Atomic, 3, 6, OpAtomicCompareExchange>;
 defm : DemangledNativeBuiltin<"atom_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;
 defm : DemangledNativeBuiltin<"atomic_add", OpenCL_std, Atomic, 2, 4, OpAtomicIAdd>;

llvm/test/CodeGen/SPIRV/AtomicBuiltinsFloat.ll

+; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s
+
+;; Types:
+; CHECK:         %[[#F32:]] = OpTypeFloat 32
+;; Constants:
+; CHECK:         %[[#CONST:]] = OpConstant %[[#F32]] 1065353216
+;; Atomic instructions:
+; CHECK:         OpStore %[[#]] %[[#CONST]]
+; CHECK-COUNT-3: OpAtomicStore
+; CHECK-COUNT-3: OpAtomicLoad
+; CHECK-COUNT-3: OpAtomicExchange
+
+define spir_kernel void @test_atomic_kernel(float addrspace(3)* %ff) local_unnamed_addr #0 !kernel_arg_addr_space !3 !kernel_arg_access_qual !4 !kernel_arg_type !5 !kernel_arg_base_type !6 !kernel_arg_type_qual !7 {
+entry:
+  %0 = addrspacecast float addrspace(3)* %ff to float addrspace(4)*
+  tail call spir_func void @_Z11atomic_initPU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2
+  tail call spir_func void @_Z12atomic_storePU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2
+  tail call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)* %0, float 1.000000e+00, i32 0) #2
+  tail call spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)* %0, float 1.000000e+00, i32 0, i32 1) #2
+  %call = tail call spir_func float @_Z11atomic_loadPU3AS4VU7_Atomicf(float addrspace(4)* %0) #2
+  %call1 = tail call spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order(float addrspace(4)* %0, i32 0) #2
+  %call2 = tail call spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order12memory_scope(float addrspace(4)* %0, i32 0, i32 1) #2
+  %call3 = tail call spir_func float @_Z15atomic_exchangePU3AS4VU7_Atomicff(float addrspace(4)* %0, float 1.000000e+00) #2
+  %call4 = tail call spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)* %0, float 1.000000e+00, i32 0) #2
+  %call5 = tail call spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)* %0, float 1.000000e+00, i32 0, i32 1) #2
+  ret void
+}
+
+declare spir_func void @_Z11atomic_initPU3AS4VU7_Atomicff(float addrspace(4)*, float)
+
+declare spir_func void @_Z12atomic_storePU3AS4VU7_Atomicff(float addrspace(4)*, float)
+
+declare spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)*, float, i32)
+
+declare spir_func void @_Z21atomic_store_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)*, float, i32, i32)
+
+declare spir_func float @_Z11atomic_loadPU3AS4VU7_Atomicf(float addrspace(4)*)
+
+declare spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order(float addrspace(4)*, i32)
+
+declare spir_func float @_Z20atomic_load_explicitPU3AS4VU7_Atomicf12memory_order12memory_scope(float addrspace(4)*, i32, i32)
+
+declare spir_func float @_Z15atomic_exchangePU3AS4VU7_Atomicff(float addrspace(4)*, float)
+
+declare spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order(float addrspace(4)*, float, i32)
+
+declare spir_func float @_Z24atomic_exchange_explicitPU3AS4VU7_Atomicff12memory_order12memory_scope(float addrspace(4)*, float, i32, i32)
+
+!3 = !{i32 3}
+!4 = !{!"none"}
+!5 = !{!"atomic_float*"}
+!6 = !{!"_Atomic(float)*"}
+!7 = !{!"volatile"}