Index: llvm/lib/Target/AMDGPU/AMDGPUCombine.td =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUCombine.td +++ llvm/lib/Target/AMDGPU/AMDGPUCombine.td @@ -45,6 +45,12 @@ [{ return PreLegalizerHelper.matchClampI64ToI16(*${clamp_i64_to_i16}, MRI, *MF, ${matchinfo}); }]), (apply [{ PreLegalizerHelper.applyClampI64ToI16(*${clamp_i64_to_i16}, ${matchinfo}); }])>; +def remove_fcanonicalize : GICombineRule< + (defs root:$fcanonicalize), + (match (wip_match_opcode G_FCANONICALIZE):$fcanonicalize, + [{ return PostLegalizerHelper.matchRemoveFcanonicalize(*${fcanonicalize}); }]), + (apply [{ PostLegalizerHelper.applyRemoveFcanonicalize(*${fcanonicalize}); }])>; + // Combines which should only apply on SI/VI def gfx6gfx7_combines : GICombineGroup<[fcmp_select_to_fmin_fmax_legacy]>; @@ -57,7 +63,7 @@ def AMDGPUPostLegalizerCombinerHelper: GICombinerHelper< "AMDGPUGenPostLegalizerCombinerHelper", [all_combines, gfx6gfx7_combines, - uchar_to_float, cvt_f32_ubyteN]> { + uchar_to_float, cvt_f32_ubyteN, remove_fcanonicalize]> { let DisableRuleOption = "amdgpupostlegalizercombiner-disable-rule"; let StateClass = "AMDGPUPostLegalizerCombinerHelperState"; let AdditionalArguments = []; Index: llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp =================================================================== --- llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp +++ llvm/lib/Target/AMDGPU/AMDGPUPostLegalizerCombiner.cpp @@ -66,6 +66,9 @@ bool matchCvtF32UByteN(MachineInstr &MI, CvtF32UByteMatchInfo &MatchInfo); void applyCvtF32UByteN(MachineInstr &MI, const CvtF32UByteMatchInfo &MatchInfo); + + bool matchRemoveFcanonicalize(MachineInstr &MI); + void applyRemoveFcanonicalize(MachineInstr &MI); }; bool AMDGPUPostLegalizerCombinerHelper::matchFMinFMaxLegacy( @@ -245,6 +248,20 @@ MI.eraseFromParent(); } +bool AMDGPUPostLegalizerCombinerHelper::matchRemoveFcanonicalize( + MachineInstr &MI) { + unsigned SrcOpcode = MRI.getVRegDef(MI.getOperand(1).getReg())->getOpcode(); + return SrcOpcode == AMDGPU::G_FMINNUM_IEEE || + SrcOpcode == AMDGPU::G_FMAXNUM_IEEE; +} + +void AMDGPUPostLegalizerCombinerHelper::applyRemoveFcanonicalize( + MachineInstr &MI) { + Helper.replaceRegWith(MRI, MI.getOperand(0).getReg(), + MI.getOperand(1).getReg()); + MI.eraseFromParent(); +} + class AMDGPUPostLegalizerCombinerHelperState { protected: CombinerHelper &Helper; Index: llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir =================================================================== --- /dev/null +++ llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fcanonicalize.mir @@ -0,0 +1,169 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=amdgpu-postlegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s + +--- +name: test_fminnum_with_fminnum_argument_s32_ieee_mode_on +tracksRegLiveness: true +legalized: true +machineFunctionInfo: + mode: + ieee: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr2 + + ; CHECK-LABEL: name: test_fminnum_with_fminnum_argument_s32_ieee_mode_on + ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY]] + ; CHECK: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY1]] + ; CHECK: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 + ; CHECK: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY2]] + ; CHECK: [[FMINNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FMINNUM_IEEE]], [[FCANONICALIZE2]] + ; CHECK: $vgpr0 = COPY [[FMINNUM_IEEE1]](s32) + %0:_(s32) = COPY $vgpr0 + %1:_(s32) = COPY $vgpr1 + %7:_(s32) = G_FCANONICALIZE %0 + %8:_(s32) = G_FCANONICALIZE %1 + %2:_(s32) = G_FMINNUM_IEEE %7, %8 + %3:_(s32) = COPY $vgpr2 + %5:_(s32) = G_FCANONICALIZE %2 + %6:_(s32) = G_FCANONICALIZE %3 + %4:_(s32) = G_FMINNUM_IEEE %5, %6 + $vgpr0 = COPY %4(s32) +... + +--- +name: test_fminnum_with_fmaxnum_argument_s32_ieee_mode_on +tracksRegLiveness: true +legalized: true +machineFunctionInfo: + mode: + ieee: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr2 + + ; CHECK-LABEL: name: test_fminnum_with_fmaxnum_argument_s32_ieee_mode_on + ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY]] + ; CHECK: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY1]] + ; CHECK: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 + ; CHECK: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY2]] + ; CHECK: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FMAXNUM_IEEE]], [[FCANONICALIZE2]] + ; CHECK: $vgpr0 = COPY [[FMINNUM_IEEE]](s32) + %0:_(s32) = COPY $vgpr0 + %1:_(s32) = COPY $vgpr1 + %7:_(s32) = G_FCANONICALIZE %0 + %8:_(s32) = G_FCANONICALIZE %1 + %2:_(s32) = G_FMAXNUM_IEEE %7, %8 + %3:_(s32) = COPY $vgpr2 + %5:_(s32) = G_FCANONICALIZE %2 + %6:_(s32) = G_FCANONICALIZE %3 + %4:_(s32) = G_FMINNUM_IEEE %5, %6 + $vgpr0 = COPY %4(s32) +... + +--- +name: test_fmaxnum_with_fmaxnum_argument_s32_ieee_mode_on +tracksRegLiveness: true +legalized: true +machineFunctionInfo: + mode: + ieee: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr2 + + ; CHECK-LABEL: name: test_fmaxnum_with_fmaxnum_argument_s32_ieee_mode_on + ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY]] + ; CHECK: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY1]] + ; CHECK: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 + ; CHECK: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY2]] + ; CHECK: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FMAXNUM_IEEE]], [[FCANONICALIZE2]] + ; CHECK: $vgpr0 = COPY [[FMAXNUM_IEEE1]](s32) + %0:_(s32) = COPY $vgpr0 + %1:_(s32) = COPY $vgpr1 + %7:_(s32) = G_FCANONICALIZE %0 + %8:_(s32) = G_FCANONICALIZE %1 + %2:_(s32) = G_FMAXNUM_IEEE %7, %8 + %3:_(s32) = COPY $vgpr2 + %5:_(s32) = G_FCANONICALIZE %2 + %6:_(s32) = G_FCANONICALIZE %3 + %4:_(s32) = G_FMAXNUM_IEEE %5, %6 + $vgpr0 = COPY %4(s32) +... + +--- +name: test_fmaxnum_with_fminnum_argument_s32_ieee_mode_on +tracksRegLiveness: true +legalized: true +machineFunctionInfo: + mode: + ieee: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1, $vgpr2 + + ; CHECK-LABEL: name: test_fmaxnum_with_fminnum_argument_s32_ieee_mode_on + ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY]] + ; CHECK: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY1]] + ; CHECK: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]] + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2 + ; CHECK: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY2]] + ; CHECK: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FMINNUM_IEEE]], [[FCANONICALIZE2]] + ; CHECK: $vgpr0 = COPY [[FMAXNUM_IEEE]](s32) + %0:_(s32) = COPY $vgpr0 + %1:_(s32) = COPY $vgpr1 + %7:_(s32) = G_FCANONICALIZE %0 + %8:_(s32) = G_FCANONICALIZE %1 + %2:_(s32) = G_FMINNUM_IEEE %7, %8 + %3:_(s32) = COPY $vgpr2 + %5:_(s32) = G_FCANONICALIZE %2 + %6:_(s32) = G_FCANONICALIZE %3 + %4:_(s32) = G_FMAXNUM_IEEE %5, %6 + $vgpr0 = COPY %4(s32) +... + +--- +name: test_multiple_uses +tracksRegLiveness: true +legalized: true +machineFunctionInfo: + mode: + ieee: true +body: | + bb.0: + liveins: $vgpr0, $vgpr1 + + ; CHECK-LABEL: name: test_multiple_uses + ; CHECK: liveins: $vgpr0, $vgpr1 + ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY]] + ; CHECK: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[COPY1]] + ; CHECK: [[FMINNUM_IEEE:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[FCANONICALIZE1]] + ; CHECK: [[FMAXNUM_IEEE:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FMINNUM_IEEE]], [[FMINNUM_IEEE]] + ; CHECK: $vgpr0 = COPY [[FMAXNUM_IEEE]](s32) + %0:_(s32) = COPY $vgpr0 + %1:_(s32) = COPY $vgpr1 + %6:_(s32) = G_FCANONICALIZE %0 + %7:_(s32) = G_FCANONICALIZE %1 + %2:_(s32) = G_FMINNUM_IEEE %6, %7 + %4:_(s32) = G_FCANONICALIZE %2 + %5:_(s32) = G_FCANONICALIZE %2 + %3:_(s32) = G_FMAXNUM_IEEE %4, %5 + $vgpr0 = COPY %3(s32) +... Index: llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll =================================================================== --- llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll +++ llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll @@ -115,11 +115,8 @@ ; SI-NEXT: v_min_f32_e32 v5, v2, v3 ; SI-NEXT: v_max_f32_e32 v2, v2, v3 ; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; SI-NEXT: v_min_f32_e32 v2, v2, v3 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_max_f32_e32 v2, v3, v2 +; SI-NEXT: v_max_f32_e32 v2, v5, v2 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11] ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; SI-NEXT: s_endpgm @@ -157,11 +154,8 @@ ; VI-NEXT: v_min_f32_e32 v5, v4, v2 ; VI-NEXT: v_max_f32_e32 v2, v4, v2 ; VI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; VI-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; VI-NEXT: v_min_f32_e32 v2, v2, v3 -; VI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; VI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; VI-NEXT: v_max_f32_e32 v2, v3, v2 +; VI-NEXT: v_max_f32_e32 v2, v5, v2 ; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -181,12 +175,9 @@ ; GFX9-NEXT: v_max_f32_e32 v1, v1, v1 ; GFX9-NEXT: v_min_f32_e32 v4, v1, v2 ; GFX9-NEXT: v_max_f32_e32 v1, v1, v2 -; GFX9-NEXT: v_max_f32_e32 v3, v3, v3 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v1 -; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_max_f32_e32 v2, v4, v4 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v1 -; GFX9-NEXT: v_max_f32_e32 v1, v2, v1 +; GFX9-NEXT: v_max_f32_e32 v2, v3, v3 +; GFX9-NEXT: v_min_f32_e32 v1, v1, v2 +; GFX9-NEXT: v_max_f32_e32 v1, v4, v1 ; GFX9-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() @@ -541,11 +532,8 @@ ; SI-NEXT: v_mul_f32_e32 v3, 1.0, v4 ; SI-NEXT: buffer_store_dword v5, off, s[0:3], 0 ; SI-NEXT: s_waitcnt vmcnt(0) -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; SI-NEXT: v_min_f32_e32 v2, v2, v3 -; SI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; SI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; SI-NEXT: v_max_f32_e32 v2, v3, v2 +; SI-NEXT: v_max_f32_e32 v2, v5, v2 ; SI-NEXT: s_mov_b64 s[2:3], s[10:11] ; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; SI-NEXT: s_endpgm @@ -580,13 +568,10 @@ ; VI-NEXT: v_mul_f32_e32 v4, 1.0, v7 ; VI-NEXT: v_mul_f32_e32 v2, 1.0, v2 ; VI-NEXT: v_min_f32_e32 v5, v4, v2 -; VI-NEXT: v_max_f32_e32 v2, v4, v2 ; VI-NEXT: v_mul_f32_e32 v3, 1.0, v3 -; VI-NEXT: v_mul_f32_e32 v2, 1.0, v2 +; VI-NEXT: v_max_f32_e32 v2, v4, v2 ; VI-NEXT: v_min_f32_e32 v2, v2, v3 -; VI-NEXT: v_mul_f32_e32 v3, 1.0, v5 -; VI-NEXT: v_mul_f32_e32 v2, 1.0, v2 -; VI-NEXT: v_max_f32_e32 v2, v3, v2 +; VI-NEXT: v_max_f32_e32 v2, v5, v2 ; VI-NEXT: flat_store_dword v[0:1], v5 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: flat_store_dword v[0:1], v2 @@ -607,14 +592,11 @@ ; GFX9-NEXT: v_max_f32_e32 v2, v2, v2 ; GFX9-NEXT: v_min_f32_e32 v4, v1, v2 ; GFX9-NEXT: v_max_f32_e32 v1, v1, v2 +; GFX9-NEXT: v_max_f32_e32 v3, v3, v3 ; GFX9-NEXT: global_store_dword v[0:1], v4, off ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_max_f32_e32 v3, v3, v3 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v1 ; GFX9-NEXT: v_min_f32_e32 v1, v1, v3 -; GFX9-NEXT: v_max_f32_e32 v2, v4, v4 -; GFX9-NEXT: v_max_f32_e32 v1, v1, v1 -; GFX9-NEXT: v_max_f32_e32 v1, v2, v1 +; GFX9-NEXT: v_max_f32_e32 v1, v4, v1 ; GFX9-NEXT: global_store_dword v0, v1, s[0:1] ; GFX9-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x()