Index: llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -622,6 +622,56 @@ return true; } +/// Helper function for building atomic flag instructions (e.g. +/// OpAtomicFlagTestAndSet). +static bool buildAtomicFlagInst(const SPIRV::IncomingCall *Call, + unsigned Opcode, MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + const MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + + Register PtrRegister = Call->Arguments[0]; + + Register MemSemanticsReg; + unsigned Semantics = SPIRV::MemorySemantics::SequentiallyConsistent; + if (Call->Arguments.size() >= 2) { + std::memory_order Order = + static_cast(getIConstVal(Call->Arguments[1], MRI)); + + Semantics = + getSPIRVMemSemantics(Order) | + getMemSemanticsForStorageClass(GR->getPointerStorageClass(PtrRegister)); + if (Order == Semantics) + MemSemanticsReg = Call->Arguments[1]; + } + if (!MemSemanticsReg.isValid()) + MemSemanticsReg = buildConstantIntReg(Semantics, MIRBuilder, GR); + + if (Opcode == SPIRV::OpAtomicFlagClear) + assert((Semantics != SPIRV::MemorySemantics::Acquire && + Semantics != SPIRV::MemorySemantics::AcquireRelease) && + "Invalid memory order argument!"); + + Register ScopeRegister; + SPIRV::Scope::Scope Scope = SPIRV::Scope::Device; + if (Call->Arguments.size() >= 3) { + auto CLScope = static_cast( + getIConstVal(Call->Arguments[2], MRI)); + Scope = getSPIRVScope(CLScope); + if (CLScope == static_cast(Scope)) + ScopeRegister = Call->Arguments[2]; + } + if (!ScopeRegister.isValid()) + ScopeRegister = buildConstantIntReg(Scope, MIRBuilder, GR); + + auto MIB = MIRBuilder.buildInstr(Opcode); + if (Opcode == SPIRV::OpAtomicFlagTestAndSet) + MIB.addDef(Call->ReturnRegister) + .addUse(GR->getSPIRVTypeID(Call->ReturnType)); + + MIB.addUse(PtrRegister).addUse(ScopeRegister).addUse(MemSemanticsReg); + return true; +} + /// Helper function for building barriers, i.e., memory/control ordering /// operations. static bool buildBarrierInst(const SPIRV::IncomingCall *Call, unsigned Opcode, @@ -991,6 +1041,9 @@ return buildAtomicRMWInst(Call, Opcode, MIRBuilder, GR); case SPIRV::OpMemoryBarrier: return buildBarrierInst(Call, SPIRV::OpMemoryBarrier, MIRBuilder, GR); + case SPIRV::OpAtomicFlagTestAndSet: + case SPIRV::OpAtomicFlagClear: + return buildAtomicFlagInst(Call, Opcode, MIRBuilder, GR); default: return false; } Index: llvm/lib/Target/SPIRV/SPIRVBuiltins.td =================================================================== --- llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -527,6 +527,10 @@ defm : DemangledNativeBuiltin<"atomic_fetch_or_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicOr>; defm : DemangledNativeBuiltin<"atomic_fetch_xor_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicXor>; defm : DemangledNativeBuiltin<"atomic_fetch_and_explicit", OpenCL_std, Atomic, 4, 6, OpAtomicAnd>; +defm : DemangledNativeBuiltin<"atomic_flag_test_and_set", OpenCL_std, Atomic, 1, 1, OpAtomicFlagTestAndSet>; +defm : DemangledNativeBuiltin<"atomic_flag_test_and_set_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagTestAndSet>; +defm : DemangledNativeBuiltin<"atomic_flag_clear", OpenCL_std, Atomic, 1, 1, OpAtomicFlagClear>; +defm : DemangledNativeBuiltin<"atomic_flag_clear_explicit", OpenCL_std, Atomic, 2, 3, OpAtomicFlagClear>; // Barrier builtin records: defm : DemangledNativeBuiltin<"barrier", OpenCL_std, Barrier, 1, 3, OpControlBarrier>; Index: llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll =================================================================== --- /dev/null +++ llvm/test/CodeGen/SPIRV/transcoding/atomic_flag.ll @@ -0,0 +1,60 @@ +; RUN: llc -O0 -mtriple=spirv64-unknown-unknown %s -o - | FileCheck %s + +;; Types: +; CHECK-DAG: %[[#INT:]] = OpTypeInt 32 +; CHECK-DAG: %[[#BOOL:]] = OpTypeBool +; CHECK-DAG: %[[#PTR:]] = OpTypePointer Generic %[[#]] +;; Constants: +; CHECK-DAG: %[[#SEQ_CONSI_SEMAN:]] = OpConstant %[[#INT]] 16 +; CHECK-DAG: %[[#DEVICE_SCOPE:]] = OpConstant %[[#INT]] 1 +; CHECK-DAG: %[[#WORKGROUP_SCOPE:]] = OpConstant %[[#INT]] 2 +;; Instructions: +; CHECK-DAG: %[[#CAST:]] = OpPtrCastToGeneric %[[#PTR]] %[[#]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#CAST]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#CAST]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: %[[#]] = OpAtomicFlagTestAndSet %[[#BOOL]] %[[#CAST]] %[[#WORKGROUP_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#CAST]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#CAST]] %[[#DEVICE_SCOPE]] %[[#SEQ_CONSI_SEMAN]] +; CHECK: OpAtomicFlagClear %[[#CAST]] %[[#WORKGROUP_SCOPE]] %[[#SEQ_CONSI_SEMAN]] + +define dso_local spir_kernel void @testAtomicFlag(i32 addrspace(1)* nocapture noundef %res) { +entry: + %f = alloca i32, align 4 + %0 = bitcast i32* %f to i8* + call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %0) + %f.ascast = addrspacecast i32* %f to i32 addrspace(4)* + %call = call spir_func zeroext i1 @_Z24atomic_flag_test_and_setPU3AS4VU7_Atomici(i32 addrspace(4)* noundef %f.ascast) + %conv = zext i1 %call to i32 + store i32 %conv, i32 addrspace(1)* %res, align 4 + %call2 = call spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order(i32 addrspace(4)* noundef %f.ascast, i32 noundef 5) + %conv3 = zext i1 %call2 to i32 + %1 = load i32, i32 addrspace(1)* %res, align 4 + %add = add nsw i32 %1, %conv3 + store i32 %add, i32 addrspace(1)* %res, align 4 + %call5 = call spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(i32 addrspace(4)* noundef %f.ascast, i32 noundef 5, i32 noundef 1) + %conv6 = zext i1 %call5 to i32 + %2 = load i32, i32 addrspace(1)* %res, align 4 + %add7 = add nsw i32 %2, %conv6 + store i32 %add7, i32 addrspace(1)* %res, align 4 + call spir_func void @_Z17atomic_flag_clearPU3AS4VU7_Atomici(i32 addrspace(4)* noundef %f.ascast) + call spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order(i32 addrspace(4)* noundef %f.ascast, i32 noundef 5) + call spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(i32 addrspace(4)* noundef %f.ascast, i32 noundef 5, i32 noundef 1) + call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %0) + ret void +} + +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) + +declare spir_func zeroext i1 @_Z24atomic_flag_test_and_setPU3AS4VU7_Atomici(i32 addrspace(4)* noundef) + +declare spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order(i32 addrspace(4)* noundef, i32 noundef) + +declare spir_func zeroext i1 @_Z33atomic_flag_test_and_set_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(i32 addrspace(4)* noundef, i32 noundef, i32 noundef) + +declare spir_func void @_Z17atomic_flag_clearPU3AS4VU7_Atomici(i32 addrspace(4)* noundef) + +declare spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order(i32 addrspace(4)* noundef, i32 noundef) + +declare spir_func void @_Z26atomic_flag_clear_explicitPU3AS4VU7_Atomici12memory_order12memory_scope(i32 addrspace(4)* noundef, i32 noundef, i32 noundef) + +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)