This is an archive of the discontinued LLVM Phabricator instance.

Differential D62826

AMDGPU: Expand < 32-bit atomics
ClosedPublic

Authored by arsenm on Jun 3 2019, 3:18 PM.

Details

Reviewers
rampitec
Summary

Also fix AtomicExpand asserting on atomicrmw fadd/fsub.

Diff Detail

Event Timeline

arsenm created this revision.Jun 3 2019, 3:18 PM
Herald added subscribers: jfb, t-tye, tpr and 6 others. · View Herald TranscriptJun 3 2019, 3:18 PM
arsenm added a comment.Jun 10 2019, 3:03 PM

ping

rampitec accepted this revision.Jun 10 2019, 4:35 PM

LGTM

This revision is now accepted and ready to land.Jun 10 2019, 4:35 PM
arsenm closed this revision.Jun 10 2019, 6:32 PM

r363021

Revision Contents

PathSize
lib/
CodeGen/
4 lines
Target/
AMDGPU/
2 lines
test/
Transforms/
AtomicExpand/
AMDGPU/
208 lines
208 lines
51 lines

Diff 202806

lib/CodeGen/AtomicExpandPass.cpp

Show First 20 Lines • Show All 579 Lines • ▼ Show 20 Lines if (ValueSize < MinCASSize) {
AI->getOrdering(), PerformOp); AI->getOrdering(), PerformOp);
} }
return true; return true;
} }
case TargetLoweringBase::AtomicExpansionKind::CmpXChg: { case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned ValueSize = getAtomicOpSize(AI); unsigned ValueSize = getAtomicOpSize(AI);
if (ValueSize < MinCASSize) { if (ValueSize < MinCASSize) {
// TODO: Handle atomicrmw fadd/fsub
if (AI->getType()->isFloatingPointTy())
return false;
expandPartwordAtomicRMW(AI, expandPartwordAtomicRMW(AI,
TargetLoweringBase::AtomicExpansionKind::CmpXChg); TargetLoweringBase::AtomicExpansionKind::CmpXChg);
} else { } else {
expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun); expandAtomicRMWToCmpXchg(AI, createCmpXchgInstFun);
} }
return true; return true;
} }
case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: { case TargetLoweringBase::AtomicExpansionKind::MaskedIntrinsic: {
▲ Show 20 Lines • Show All 1,205 Lines • Show Last 20 Lines

lib/Target/AMDGPU/AMDGPUISelLowering.cpp

Show First 20 Lines • Show All 517 Lines • ▼ Show 20 LinesAMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
// FIXME: This is only partially true. If we have to do vector compares, any // FIXME: This is only partially true. If we have to do vector compares, any
// SGPR pair can be a condition register. If we have a uniform condition, we // SGPR pair can be a condition register. If we have a uniform condition, we
// are better off doing SALU operations, where there is only one SCC. For now, // are better off doing SALU operations, where there is only one SCC. For now,
// we don't have a way of knowing during instruction selection if a condition // we don't have a way of knowing during instruction selection if a condition
// will be uniform and we always use vector compares. Assume we are using // will be uniform and we always use vector compares. Assume we are using
// vector compares until that is fixed. // vector compares until that is fixed.
setHasMultipleConditionRegisters(true); setHasMultipleConditionRegisters(true);
setMinCmpXchgSizeInBits(32);
PredictableSelectIsExpensive = false; PredictableSelectIsExpensive = false;
// We want to find all load dependencies for long chains of stores to enable // We want to find all load dependencies for long chains of stores to enable
// merging into very wide vectors. The problem is with vectors with > 4 // merging into very wide vectors. The problem is with vectors with > 4
// elements. MergeConsecutiveStores will attempt to merge these because x8/x16 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16
// vectors are a legal type, even though we have to split the loads // vectors are a legal type, even though we have to split the loads
// usually. When we can more precisely specify load legality per address // usually. When we can more precisely specify load legality per address
// space, we should be able to make FindBetterChain/MergeConsecutiveStores // space, we should be able to make FindBetterChain/MergeConsecutiveStores
▲ Show 20 Lines • Show All 4,204 Lines • Show Last 20 Lines

test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i16.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -atomic-expand %s | FileCheck %s
; RUN: opt -mtriple=r600-mesa-mesa3d -S -atomic-expand %s | FileCheck %s
define i16 @test_atomicrmw_xchg_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw xchg i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_add_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw add i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw add i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_sub_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_sub_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw sub i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw sub i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_and_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_and_i16_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i16 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[ANDOPERAND:%.*]] = or i32 [[INV_MASK]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw and i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[ANDOPERAND]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i16
; CHECK-NEXT: ret i16 [[TMP7]]
;
%res = atomicrmw and i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_nand_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_nand_i16_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i16 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP6]], -1
; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[NEW]], [[MASK]]
; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP9:%.*]] = or i32 [[TMP8]], [[TMP7]]
; CHECK-NEXT: [[TMP10:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP9]] seq_cst seq_cst
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP10]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[TMP11:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i16
; CHECK-NEXT: ret i16 [[TMP12]]
;
%res = atomicrmw nand i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_or_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_or_i16_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i16 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw or i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i16
; CHECK-NEXT: ret i16 [[TMP7]]
;
%res = atomicrmw or i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_xor_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_xor_i16_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i16 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw xor i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i16
; CHECK-NEXT: ret i16 [[TMP7]]
;
%res = atomicrmw xor i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_max_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_max_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw max i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw max i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_min_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_min_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw min i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw min i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_umax_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_umax_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw umax i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw umax i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_atomicrmw_umin_i16_global(i16 addrspace(1)* %ptr, i16 %value) {
; CHECK-LABEL: @test_atomicrmw_umin_i16_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw umin i16 addrspace(1)* [[PTR:%.*]], i16 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i16 [[RES]]
;
%res = atomicrmw umin i16 addrspace(1)* %ptr, i16 %value seq_cst
ret i16 %res
}
define i16 @test_cmpxchg_i16_global(i16 addrspace(1)* %out, i16 %in, i16 %old) {
; CHECK-LABEL: @test_cmpxchg_i16_global(
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i16, i16 addrspace(1)* [[OUT:%.*]], i64 4
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i16 addrspace(1)* [[GEP]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 65535, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i16 [[IN:%.*]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP6:%.*]] = zext i16 [[OLD:%.*]] to i32
; CHECK-NEXT: [[TMP7:%.*]] = shl i32 [[TMP6]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]]
; CHECK-NEXT: [[TMP9:%.*]] = and i32 [[TMP8]], [[INV_MASK]]
; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]
; CHECK: partword.cmpxchg.loop:
; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP9]], [[TMP0:%.*]] ], [ [[TMP16:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]
; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[TMP10]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = or i32 [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP13:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[TMP12]], i32 [[TMP11]] seq_cst seq_cst
; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP13]], 0
; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { i32, i1 } [[TMP13]], 1
; CHECK-NEXT: br i1 [[TMP15]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]
; CHECK: partword.cmpxchg.failure:
; CHECK-NEXT: [[TMP16]] = and i32 [[TMP14]], [[INV_MASK]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i32 [[TMP10]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]
; CHECK: partword.cmpxchg.end:
; CHECK-NEXT: [[TMP18:%.*]] = lshr i32 [[TMP14]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP18]] to i16
; CHECK-NEXT: [[TMP20:%.*]] = insertvalue { i16, i1 } undef, i16 [[TMP19]], 0
; CHECK-NEXT: [[TMP21:%.*]] = insertvalue { i16, i1 } [[TMP20]], i1 [[TMP15]], 1
; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i16, i1 } [[TMP21]], 0
; CHECK-NEXT: ret i16 [[EXTRACT]]
;
%gep = getelementptr i16, i16 addrspace(1)* %out, i64 4
%res = cmpxchg i16 addrspace(1)* %gep, i16 %old, i16 %in seq_cst seq_cst
%extract = extractvalue {i16, i1} %res, 0
ret i16 %extract
}

test/Transforms/AtomicExpand/AMDGPU/expand-atomic-i8.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -mtriple=amdgcn-amd-amdhsa -S -atomic-expand %s | FileCheck %s
; RUN: opt -mtriple=r600-mesa-mesa3d -S -atomic-expand %s | FileCheck %s
define i8 @test_atomicrmw_xchg_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_xchg_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw xchg i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw xchg i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_add_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_add_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw add i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw add i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_sub_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_sub_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw sub i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw sub i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_and_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_and_i8_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[ANDOPERAND:%.*]] = or i32 [[INV_MASK]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw and i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[ANDOPERAND]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; CHECK-NEXT: ret i8 [[TMP7]]
;
%res = atomicrmw and i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_nand_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_nand_i8_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]], align 4
; CHECK-NEXT: br label [[ATOMICRMW_START:%.*]]
; CHECK: atomicrmw.start:
; CHECK-NEXT: [[LOADED:%.*]] = phi i32 [ [[TMP5]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
; CHECK-NEXT: [[TMP6:%.*]] = and i32 [[LOADED]], [[VALOPERAND_SHIFTED]]
; CHECK-NEXT: [[NEW:%.*]] = xor i32 [[TMP6]], -1
; CHECK-NEXT: [[TMP7:%.*]] = and i32 [[NEW]], [[MASK]]
; CHECK-NEXT: [[TMP8:%.*]] = and i32 [[LOADED]], [[INV_MASK]]
; CHECK-NEXT: [[TMP9:%.*]] = or i32 [[TMP8]], [[TMP7]]
; CHECK-NEXT: [[TMP10:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[LOADED]], i32 [[TMP9]] seq_cst seq_cst
; CHECK-NEXT: [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP10]], 1
; CHECK-NEXT: [[NEWLOADED]] = extractvalue { i32, i1 } [[TMP10]], 0
; CHECK-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; CHECK: atomicrmw.end:
; CHECK-NEXT: [[TMP11:%.*]] = lshr i32 [[NEWLOADED]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP12:%.*]] = trunc i32 [[TMP11]] to i8
; CHECK-NEXT: ret i8 [[TMP12]]
;
%res = atomicrmw nand i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_or_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_or_i8_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw or i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; CHECK-NEXT: ret i8 [[TMP7]]
;
%res = atomicrmw or i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_xor_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_xor_i8_global(
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[PTR:%.*]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[VALUE:%.*]] to i32
; CHECK-NEXT: [[VALOPERAND_SHIFTED:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP5:%.*]] = atomicrmw xor i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[VALOPERAND_SHIFTED]] seq_cst
; CHECK-NEXT: [[TMP6:%.*]] = lshr i32 [[TMP5]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP7:%.*]] = trunc i32 [[TMP6]] to i8
; CHECK-NEXT: ret i8 [[TMP7]]
;
%res = atomicrmw xor i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_max_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_max_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw max i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw max i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_min_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_min_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw min i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw min i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_umax_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_umax_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw umax i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw umax i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_atomicrmw_umin_i8_global(i8 addrspace(1)* %ptr, i8 %value) {
; CHECK-LABEL: @test_atomicrmw_umin_i8_global(
; CHECK-NEXT: [[RES:%.*]] = atomicrmw umin i8 addrspace(1)* [[PTR:%.*]], i8 [[VALUE:%.*]] seq_cst
; CHECK-NEXT: ret i8 [[RES]]
;
%res = atomicrmw umin i8 addrspace(1)* %ptr, i8 %value seq_cst
ret i8 %res
}
define i8 @test_cmpxchg_i8_global(i8 addrspace(1)* %out, i8 %in, i8 %old) {
; CHECK-LABEL: @test_cmpxchg_i8_global(
; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, i8 addrspace(1)* [[OUT:%.*]], i64 4
; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[GEP]] to i64
; CHECK-NEXT: [[TMP2:%.*]] = and i64 [[TMP1]], -4
; CHECK-NEXT: [[ALIGNEDADDR:%.*]] = inttoptr i64 [[TMP2]] to i32 addrspace(1)*
; CHECK-NEXT: [[PTRLSB:%.*]] = and i64 [[TMP1]], 3
; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[PTRLSB]], 3
; CHECK-NEXT: [[SHIFTAMT:%.*]] = trunc i64 [[TMP3]] to i32
; CHECK-NEXT: [[MASK:%.*]] = shl i32 255, [[SHIFTAMT]]
; CHECK-NEXT: [[INV_MASK:%.*]] = xor i32 [[MASK]], -1
; CHECK-NEXT: [[TMP4:%.*]] = zext i8 [[IN:%.*]] to i32
; CHECK-NEXT: [[TMP5:%.*]] = shl i32 [[TMP4]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP6:%.*]] = zext i8 [[OLD:%.*]] to i32
; CHECK-NEXT: [[TMP7:%.*]] = shl i32 [[TMP6]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP8:%.*]] = load i32, i32 addrspace(1)* [[ALIGNEDADDR]]
; CHECK-NEXT: [[TMP9:%.*]] = and i32 [[TMP8]], [[INV_MASK]]
; CHECK-NEXT: br label [[PARTWORD_CMPXCHG_LOOP:%.*]]
; CHECK: partword.cmpxchg.loop:
; CHECK-NEXT: [[TMP10:%.*]] = phi i32 [ [[TMP9]], [[TMP0:%.*]] ], [ [[TMP16:%.*]], [[PARTWORD_CMPXCHG_FAILURE:%.*]] ]
; CHECK-NEXT: [[TMP11:%.*]] = or i32 [[TMP10]], [[TMP5]]
; CHECK-NEXT: [[TMP12:%.*]] = or i32 [[TMP10]], [[TMP7]]
; CHECK-NEXT: [[TMP13:%.*]] = cmpxchg i32 addrspace(1)* [[ALIGNEDADDR]], i32 [[TMP12]], i32 [[TMP11]] seq_cst seq_cst
; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { i32, i1 } [[TMP13]], 0
; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { i32, i1 } [[TMP13]], 1
; CHECK-NEXT: br i1 [[TMP15]], label [[PARTWORD_CMPXCHG_END:%.*]], label [[PARTWORD_CMPXCHG_FAILURE]]
; CHECK: partword.cmpxchg.failure:
; CHECK-NEXT: [[TMP16]] = and i32 [[TMP14]], [[INV_MASK]]
; CHECK-NEXT: [[TMP17:%.*]] = icmp ne i32 [[TMP10]], [[TMP16]]
; CHECK-NEXT: br i1 [[TMP17]], label [[PARTWORD_CMPXCHG_LOOP]], label [[PARTWORD_CMPXCHG_END]]
; CHECK: partword.cmpxchg.end:
; CHECK-NEXT: [[TMP18:%.*]] = lshr i32 [[TMP14]], [[SHIFTAMT]]
; CHECK-NEXT: [[TMP19:%.*]] = trunc i32 [[TMP18]] to i8
; CHECK-NEXT: [[TMP20:%.*]] = insertvalue { i8, i1 } undef, i8 [[TMP19]], 0
; CHECK-NEXT: [[TMP21:%.*]] = insertvalue { i8, i1 } [[TMP20]], i1 [[TMP15]], 1
; CHECK-NEXT: [[EXTRACT:%.*]] = extractvalue { i8, i1 } [[TMP21]], 0
; CHECK-NEXT: ret i8 [[EXTRACT]]
;
%gep = getelementptr i8, i8 addrspace(1)* %out, i64 4
%res = cmpxchg i8 addrspace(1)* %gep, i8 %old, i8 %in seq_cst seq_cst
%extract = extractvalue {i8, i1} %res, 0
ret i8 %extract
}

test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll

Show First 20 Lines • Show All 64 Lines • ▼ Show 20 Lines
; GCN-NEXT: ret float [[TMP6]] ; GCN-NEXT: ret float [[TMP6]]
; ;
%res = atomicrmw fsub float addrspace(3)* %ptr, float %value seq_cst %res = atomicrmw fsub float addrspace(3)* %ptr, float %value seq_cst
ret float %res ret float %res
} }
define half @test_atomicrmw_fsub_f16_flat(half* %ptr, half %value) { define half @test_atomicrmw_fsub_f16_flat(half* %ptr, half %value) {
; GCN-LABEL: @test_atomicrmw_fsub_f16_flat( ; GCN-LABEL: @test_atomicrmw_fsub_f16_flat(
; GCN-NEXT: [[TMP1:%.*]] = load half, half* [[PTR:%.*]], align 2 ; GCN-NEXT: [[RES:%.*]] = atomicrmw fsub half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GCN-NEXT: br label [[ATOMICRMW_START:%.*]] ; GCN-NEXT: ret half [[RES]]
; GCN: atomicrmw.start:
; GCN-NEXT: [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GCN-NEXT: [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
; GCN-NEXT: [[TMP2:%.*]] = bitcast half* [[PTR]] to i16*
; GCN-NEXT: [[TMP3:%.*]] = bitcast half [[NEW]] to i16
; GCN-NEXT: [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
; GCN-NEXT: [[TMP5:%.*]] = cmpxchg i16* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
; GCN-NEXT: [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GCN: atomicrmw.end:
; GCN-NEXT: ret half [[TMP6]]
; ;
%res = atomicrmw fsub half* %ptr, half %value seq_cst %res = atomicrmw fsub half* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define half @test_atomicrmw_fsub_f16_global(half addrspace(1)* %ptr, half %value) { define half @test_atomicrmw_fsub_f16_global(half addrspace(1)* %ptr, half %value) {
; GCN-LABEL: @test_atomicrmw_fsub_f16_global( ; GCN-LABEL: @test_atomicrmw_fsub_f16_global(
; GCN-NEXT: [[TMP1:%.*]] = load half, half addrspace(1)* [[PTR:%.*]], align 2 ; GCN-NEXT: [[RES:%.*]] = atomicrmw fsub half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GCN-NEXT: br label [[ATOMICRMW_START:%.*]] ; GCN-NEXT: ret half [[RES]]
; GCN: atomicrmw.start:
; GCN-NEXT: [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GCN-NEXT: [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
; GCN-NEXT: [[TMP2:%.*]] = bitcast half addrspace(1)* [[PTR]] to i16 addrspace(1)*
; GCN-NEXT: [[TMP3:%.*]] = bitcast half [[NEW]] to i16
; GCN-NEXT: [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
; GCN-NEXT: [[TMP5:%.*]] = cmpxchg i16 addrspace(1)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
; GCN-NEXT: [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GCN: atomicrmw.end:
; GCN-NEXT: ret half [[TMP6]]
; ;
%res = atomicrmw fsub half addrspace(1)* %ptr, half %value seq_cst %res = atomicrmw fsub half addrspace(1)* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define half @test_atomicrmw_fsub_f16_local(half addrspace(3)* %ptr, half %value) { define half @test_atomicrmw_fsub_f16_local(half addrspace(3)* %ptr, half %value) {
; GCN-LABEL: @test_atomicrmw_fsub_f16_local( ; GCN-LABEL: @test_atomicrmw_fsub_f16_local(
; GCN-NEXT: [[TMP1:%.*]] = load half, half addrspace(3)* [[PTR:%.*]], align 2 ; GCN-NEXT: [[RES:%.*]] = atomicrmw fsub half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
; GCN-NEXT: br label [[ATOMICRMW_START:%.*]] ; GCN-NEXT: ret half [[RES]]
; GCN: atomicrmw.start:
; GCN-NEXT: [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
; GCN-NEXT: [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
; GCN-NEXT: [[TMP2:%.*]] = bitcast half addrspace(3)* [[PTR]] to i16 addrspace(3)*
; GCN-NEXT: [[TMP3:%.*]] = bitcast half [[NEW]] to i16
; GCN-NEXT: [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
; GCN-NEXT: [[TMP5:%.*]] = cmpxchg i16 addrspace(3)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
; GCN-NEXT: [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
; GCN-NEXT: [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
; GCN-NEXT: [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
; GCN-NEXT: br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
; GCN: atomicrmw.end:
; GCN-NEXT: ret half [[TMP6]]
; ;
%res = atomicrmw fsub half addrspace(3)* %ptr, half %value seq_cst %res = atomicrmw fsub half addrspace(3)* %ptr, half %value seq_cst
ret half %res ret half %res
} }
define double @test_atomicrmw_fsub_f64_flat(double* %ptr, double %value) { define double @test_atomicrmw_fsub_f64_flat(double* %ptr, double %value) {
; GCN-LABEL: @test_atomicrmw_fsub_f64_flat( ; GCN-LABEL: @test_atomicrmw_fsub_f64_flat(
; GCN-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8 ; GCN-NEXT: [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
▲ Show 20 Lines • Show All 62 Lines • Show Last 20 Lines