[AMDGPU][InferAddressSpaces] Only rewrite address-spaces that can be trivially casted to flat for llvm.amdgcn.flat.atomic.{fadd,fmax,fmin} (2/2)

[jmmartinez]

Summary

The intrinsic @llvm.amdgcn.flat.atomic.{fadd,fmax,fmin} can only be
selected for flat address spaces (constant, flat and global).

This patch restricts the cases over which GCNTTIImpl::rewriteIntrinsicWithAddressSpace
rewrites the intrinsic.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

1089–1090

But the not-noop address space casts are the cases we're interested in handling. Do you mean to check for valid casts?

llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat-fadd-fmin-fmax-intrinsics.ll

4

Can you precommit the test to show the diff?

166

Add a test with a random number, other unknown address spaces should generally act like global.

Also test the new fat pointer address spaces?

[arsenm]

llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat-fadd-fmin-fmax-intrinsics.ll

166

Also constant 32 bit

[jmmartinez]

• Taking into account remarks

[jmmartinez]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

1089–1090

The problem is that currently we cannot lower this builtin for other address spaces other than flat.

I agree that the rewrite of this intrinsic loses almost all interest for this intrinsics. Maybe we shouldn't even rewrite them at all, which would simplify the code.

Handling CONSTANT_ADDRESS_32BIT gets awkward since it's not handled in isFlatGlobalAddrSpace. I modified isFlatGlobalAddrSpace to take it into account but I've got several fails in other tests (I haven't looked at the details yet).

llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat-fadd-fmin-fmax-intrinsics.ll

4

I've added https://reviews.llvm.org/D150259 as a parent patch.

[jmmartinez]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

1089–1090

Ping

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

1086–1087

Is this not just isExtendedGlobalAddrSpace?

1091

I don't understand why you are checking both address spaces. The old address space had to have been flat?

[jmmartinez]

• Remarks taken into account.

[jmmartinez]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

1091

You're right. I fixed the code to match.

[jmmartinez]

• Reduced diff

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

case Intrinsic::ptrmask: {
     }
 
     return B.CreateIntrinsic(Intrinsic::ptrmask, {NewV->getType(), MaskTy},
                              {NewV, MaskOp});
   }
   case Intrinsic::amdgcn_flat_atomic_fadd:
   case Intrinsic::amdgcn_flat_atomic_fmax:
   case Intrinsic::amdgcn_flat_atomic_fmin: {
-    Module *M = II->getParent()->getParent()->getParent();
     Type *DestTy = II->getType();
     Type *SrcTy = NewV->getType();
+    unsigned NewAS = SrcTy->getPointerAddressSpace();
+    if (!AMDGPU::isExtendedGlobalAddrSpace(NewAS))
+      return nullptr;
+    Module *M = II->getModule();
     Function *NewDecl = Intrinsic::getDeclaration(M, II->getIntrinsicID(),
                                                   {DestTy, SrcTy, DestTy});
     II->setArgOperand(0, NewV);
     II->setCalledFunction(NewDecl);
     return II;
   }
   default:
     return nullptr;
   }

llvm/test/Transforms/InferAddressSpaces/AMDGPU/flat-fadd-fmin-fmax-intrinsics.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=infer-address-spaces %s | FileCheck %s
 
 declare float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %ptr, float %data)
 declare float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %ptr, float %data)
 declare float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %ptr, float %data)
 
 define amdgpu_kernel void @flat_atomic_fadd_f32_p1(ptr addrspace(1) %ptr, float %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f32_p1
 ; CHECK-SAME: (ptr addrspace(1) [[PTR:%.*]], float [[DATA:%.*]]) {
 ; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p1.f32(ptr addrspace(1) [[PTR]], float [[DATA]])
 ; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p1.f32(ptr addrspace(1) [[PTR]], float [[DATA]])
 ; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p1.f32(ptr addrspace(1) [[PTR]], float [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(1) %ptr to ptr
   %add = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %cast, float %data)
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f32_p2(ptr addrspace(2) %ptr, float %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f32_p2
 ; CHECK-SAME: (ptr addrspace(2) [[PTR:%.*]], float [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p2.f32(ptr addrspace(2) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p2.f32(ptr addrspace(2) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p2.f32(ptr addrspace(2) [[PTR]], float [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr [[CAST]], float [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(2) %ptr to ptr
   %add = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %cast, float %data)
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f32_p3(ptr addrspace(3) %ptr, float %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f32_p3
 ; CHECK-SAME: (ptr addrspace(3) [[PTR:%.*]], float [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p3.f32(ptr addrspace(3) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p3.f32(ptr addrspace(3) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p3.f32(ptr addrspace(3) [[PTR]], float [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr [[CAST]], float [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(3) %ptr to ptr
   %add = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %cast, float %data)
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
;
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f32_p5(ptr addrspace(5) %ptr, float %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f32_p5
 ; CHECK-SAME: (ptr addrspace(5) [[PTR:%.*]], float [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p5.f32(ptr addrspace(5) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p5.f32(ptr addrspace(5) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p5.f32(ptr addrspace(5) [[PTR]], float [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr [[CAST]], float [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(5) %ptr to ptr
   %add = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %cast, float %data)
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
;
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f32_p7(ptr addrspace(7) %ptr, float %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f32_p7
 ; CHECK-SAME: (ptr addrspace(7) [[PTR:%.*]], float [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p7.f32(ptr addrspace(7) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p7.f32(ptr addrspace(7) [[PTR]], float [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p7.f32(ptr addrspace(7) [[PTR]], float [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(7) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr [[CAST]], float [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr [[CAST]], float [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(7) %ptr to ptr
   %add = call float @llvm.amdgcn.flat.atomic.fadd.f32.p0.f32(ptr %cast, float %data)
   %max = call float @llvm.amdgcn.flat.atomic.fmax.f32.p0.f32(ptr %cast, float %data)
   %min = call float @llvm.amdgcn.flat.atomic.fmin.f32.p0.f32(ptr %cast, float %data)
   ret void
 }
;
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f64_p2(ptr addrspace(2) %ptr, double %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f64_p2
 ; CHECK-SAME: (ptr addrspace(2) [[PTR:%.*]], double [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p2.f64(ptr addrspace(2) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p2.f64(ptr addrspace(2) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p2.f64(ptr addrspace(2) [[PTR]], double [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(2) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr [[CAST]], double [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(2) %ptr to ptr
   %add = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %cast, double %data)
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f64_p3(ptr addrspace(3) %ptr, double %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f64_p3
 ; CHECK-SAME: (ptr addrspace(3) [[PTR:%.*]], double [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p3.f64(ptr addrspace(3) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p3.f64(ptr addrspace(3) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p3.f64(ptr addrspace(3) [[PTR]], double [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(3) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr [[CAST]], double [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(3) %ptr to ptr
   %add = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %cast, double %data)
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
;
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f64_p5(ptr addrspace(5) %ptr, double %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f64_p5
 ; CHECK-SAME: (ptr addrspace(5) [[PTR:%.*]], double [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p5.f64(ptr addrspace(5) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p5.f64(ptr addrspace(5) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p5.f64(ptr addrspace(5) [[PTR]], double [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(5) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr [[CAST]], double [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(5) %ptr to ptr
   %add = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %cast, double %data)
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
;
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }
 
 define amdgpu_kernel void @flat_atomic_fadd_f64_p7(ptr addrspace(7) %ptr, double %data) {
 ; CHECK-LABEL: define amdgpu_kernel void @flat_atomic_fadd_f64_p7
 ; CHECK-SAME: (ptr addrspace(7) [[PTR:%.*]], double [[DATA:%.*]]) {
-; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p7.f64(ptr addrspace(7) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p7.f64(ptr addrspace(7) [[PTR]], double [[DATA]])
-; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p7.f64(ptr addrspace(7) [[PTR]], double [[DATA]])
+; CHECK-NEXT:    [[CAST:%.*]] = addrspacecast ptr addrspace(7) [[PTR]] to ptr
+; CHECK-NEXT:    [[ADD:%.*]] = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MAX:%.*]] = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr [[CAST]], double [[DATA]])
+; CHECK-NEXT:    [[MIN:%.*]] = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr [[CAST]], double [[DATA]])
 ; CHECK-NEXT:    ret void
 ;
   %cast = addrspacecast ptr addrspace(7) %ptr to ptr
   %add = call double @llvm.amdgcn.flat.atomic.fadd.f64.p0.f64(ptr %cast, double %data)
   %max = call double @llvm.amdgcn.flat.atomic.fmax.f64.p0.f64(ptr %cast, double %data)
   %min = call double @llvm.amdgcn.flat.atomic.fmin.f64.p0.f64(ptr %cast, double %data)
   ret void
 }