[GVN] Freeze SrcVal if materializes load to a smaller type load

[StephenFan]

Summary

Fixes: https://github.com/llvm/llvm-project/issues/63968

[StephenFan]

Fix PhaseOrdering test.

[nikic]

I don't think this fixes the issue. Let's say we have in memory i32 poison, i32 0. Then we person a load i64 on this and a load i32. Then trunc(freeze(load i64)) will not result in zero, but rather in freeze(poison). The load already propagates poison to the full value, freezing afterwards is too late.

llvm/include/llvm/Transforms/Utils/VNCoercion.h

 int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
                                      MemIntrinsic *DepMI, const DataLayout &DL);
 
 /// If analyzeLoadFromClobberingStore/Load returned an offset, this function
 /// can be used to actually perform the extraction of the bits from the store.
 /// It inserts instructions to do so at InsertPt, and returns the extracted
 /// value.
 Value *getValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
-                            Instruction *InsertPt, const DataLayout &DL);
+                       Instruction *InsertPt, const DataLayout &DL,
+                       bool SrcValIsLoadValue = false);
 // This is the same as getValueForLoad, except it performs no insertion.
 // It only allows constant inputs.
 Constant *getConstantValueForLoad(Constant *SrcVal, unsigned Offset,
                                   Type *LoadTy, const DataLayout &DL);
 
 /// If analyzeLoadFromClobberingMemInst returned an offset, this function can be
 /// used to actually perform the extraction of the bits from the memory
 /// intrinsic.  It inserts instructions to do so at InsertPt, and returns the

llvm/lib/Transforms/Scalar/GVN.cpp

if (Res->getType() != LoadTy) {
                         << "\n\n\n");
     }
   } else if (isCoercedLoadValue()) {
     LoadInst *CoercedLoad = getCoercedLoadValue();
     if (CoercedLoad->getType() == LoadTy && Offset == 0) {
       Res = CoercedLoad;
       combineMetadataForCSE(CoercedLoad, Load, false);
     } else {
-      Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, DL);
+      Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, DL,
+                            /*SrcValIsLoadValue*/ true);
       // We are adding a new user for this load, for which the original
       // metadata may not hold. Additionally, the new load may have a different
       // size and type, so their metadata cannot be combined in any
       // straightforward way.
       // Drop all metadata that is not known to cause immediate UB on violation,
       // unless the load has !noundef, in which case all metadata violations
       // will be promoted to UB.
       // TODO: We can combine noalias/alias.scope metadata here, because it is

llvm/lib/Transforms/Utils/VNCoercion.cpp

int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
   unsigned IndexSize = DL.getIndexTypeSizeInBits(Src->getType());
   if (ConstantFoldLoadFromConstPtr(Src, LoadTy, APInt(IndexSize, Offset), DL))
     return Offset;
   return -1;
 }
 
 static Value *getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset,
                                          Type *LoadTy, IRBuilderBase &Builder,
-                                         const DataLayout &DL) {
+                                         const DataLayout &DL,
+                                         bool SrcValIsLoadValue) {
   LLVMContext &Ctx = SrcVal->getType()->getContext();
 
   // If two pointers are in the same address space, they have the same size,
   // so we don't need to do any truncation, etc. This avoids introducing
   // ptrtoint instructions for pointers that may be non-integral.
   if (SrcVal->getType()->isPointerTy() && LoadTy->isPointerTy() &&
       cast<PointerType>(SrcVal->getType())->getAddressSpace() ==
           cast<PointerType>(LoadTy)->getAddressSpace()) {
static Value *getStoreValueForLoadHelper(Value *SrcVal, unsigned Offset,
   if (DL.isLittleEndian())
     ShiftAmt = Offset * 8;
   else
     ShiftAmt = (StoreSize - LoadSize - Offset) * 8;
   if (ShiftAmt)
     SrcVal = Builder.CreateLShr(SrcVal,
                                 ConstantInt::get(SrcVal->getType(), ShiftAmt));
 
-  if (LoadSize != StoreSize)
+  if (LoadSize != StoreSize) {
+    if (SrcValIsLoadValue && !isGuaranteedNotToBePoison(SrcVal))
+      SrcVal = Builder.CreateFreeze(SrcVal);
     SrcVal = Builder.CreateTruncOrBitCast(SrcVal,
                                           IntegerType::get(Ctx, LoadSize * 8));
+  }
   return SrcVal;
 }
 
 Value *getValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
-                       Instruction *InsertPt, const DataLayout &DL) {
+                       Instruction *InsertPt, const DataLayout &DL,
+                       bool SrcValIsLoadValue) {
 
 #ifndef NDEBUG
   unsigned SrcValSize = DL.getTypeStoreSize(SrcVal->getType()).getFixedValue();
   unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedValue();
   assert(Offset + LoadSize <= SrcValSize);
 #endif
   IRBuilder<> Builder(InsertPt);
-  SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, Builder, DL);
+  SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, Builder, DL,
+                                      SrcValIsLoadValue);
   return coerceAvailableValueToLoadType(SrcVal, LoadTy, Builder, DL);
 }
 
 Constant *getConstantValueForLoad(Constant *SrcVal, unsigned Offset,
                                   Type *LoadTy, const DataLayout &DL) {
 #ifndef NDEBUG
   unsigned SrcValSize = DL.getTypeStoreSize(SrcVal->getType()).getFixedValue();
   unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedValue();

llvm/test/Transforms/GVN/PRE/atomic.ll

;
   ret i64 %res
 }
 
 ; narrowing does work
 define i64 @narrow(ptr %P1) {
 ; CHECK-LABEL: define i64 @narrow
 ; CHECK-SAME: (ptr [[P1:%.*]]) {
 ; CHECK-NEXT:    [[A64:%.*]] = load atomic i64, ptr [[P1]] unordered, align 4
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A64]] to i32
-; CHECK-NEXT:    [[B64:%.*]] = sext i32 [[TMP1]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[A64]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[B64:%.*]] = sext i32 [[TMP2]] to i64
 ; CHECK-NEXT:    [[RES:%.*]] = sub i64 [[A64]], [[B64]]
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
   %a64 = load atomic i64, ptr %P1 unordered, align 4
   %b = load atomic i32, ptr %P1 unordered, align 4
   %b64 = sext i32 %b to i64
   %res = sub i64 %a64, %b64
   ret i64 %res

llvm/test/Transforms/GVN/PRE/rle.ll

 ;; Load -> Load forwarding in partial alias case.
 ;;===----------------------------------------------------------------------===;;
 
 define i32 @load_load_partial_alias(ptr %P) nounwind ssp {
 ; LE-LABEL: @load_load_partial_alias(
 ; LE-NEXT:  entry:
 ; LE-NEXT:    [[TTMP2:%.*]] = load i32, ptr [[P:%.*]], align 4
 ; LE-NEXT:    [[TMP0:%.*]] = lshr i32 [[TTMP2]], 8
-; LE-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
-; LE-NEXT:    [[CONV:%.*]] = zext i8 [[TMP1]] to i32
+; LE-NEXT:    [[TMP1:%.*]] = freeze i32 [[TMP0]]
+; LE-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+; LE-NEXT:    [[CONV:%.*]] = zext i8 [[TMP2]] to i32
 ; LE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TTMP2]], [[CONV]]
 ; LE-NEXT:    ret i32 [[ADD]]
 ;
 ; BE-LABEL: @load_load_partial_alias(
 ; BE-NEXT:  entry:
 ; BE-NEXT:    [[TTMP2:%.*]] = load i32, ptr [[P:%.*]], align 4
 ; BE-NEXT:    [[TMP0:%.*]] = lshr i32 [[TTMP2]], 16
-; BE-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
-; BE-NEXT:    [[CONV:%.*]] = zext i8 [[TMP1]] to i32
+; BE-NEXT:    [[TMP1:%.*]] = freeze i32 [[TMP0]]
+; BE-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
+; BE-NEXT:    [[CONV:%.*]] = zext i8 [[TMP2]] to i32
 ; BE-NEXT:    [[ADD:%.*]] = add nsw i32 [[TTMP2]], [[CONV]]
 ; BE-NEXT:    ret i32 [[ADD]]
 ;
 entry:
   %ttmp2 = load i32, ptr %P
   %add.ptr = getelementptr inbounds i8, ptr %P, i64 1
   %ttmp5 = load i8, ptr %add.ptr
   %conv = zext i8 %ttmp5 to i32

 define void @load_load_partial_alias_loop(ptr %P) {
 ; LE-LABEL: @load_load_partial_alias_loop(
 ; LE-NEXT:  entry:
 ; LE-NEXT:    [[P_1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1
 ; LE-NEXT:    [[V_1:%.*]] = load i8, ptr [[P_1]], align 1
 ; LE-NEXT:    call void @use.i8(i8 [[V_1]])
 ; LE-NEXT:    [[V_1_32:%.*]] = load i32, ptr [[P_1]], align 4
 ; LE-NEXT:    call void @use.i32(i32 [[V_1_32]])
-; LE-NEXT:    [[TMP0:%.*]] = trunc i32 [[V_1_32]] to i8
+; LE-NEXT:    [[TMP0:%.*]] = freeze i32 [[V_1_32]]
+; LE-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
 ; LE-NEXT:    br label [[LOOP:%.*]]
 ; LE:       loop:
-; LE-NEXT:    [[V_I:%.*]] = phi i8 [ [[TMP0]], [[ENTRY:%.*]] ], [ [[V_I_PRE:%.*]], [[LOOP_LOOP_CRIT_EDGE:%.*]] ]
+; LE-NEXT:    [[V_I:%.*]] = phi i8 [ [[TMP1]], [[ENTRY:%.*]] ], [ [[V_I_PRE:%.*]], [[LOOP_LOOP_CRIT_EDGE:%.*]] ]
 ; LE-NEXT:    [[I:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[I_INC:%.*]], [[LOOP_LOOP_CRIT_EDGE]] ]
 ; LE-NEXT:    [[P_I:%.*]] = getelementptr i8, ptr [[P]], i64 [[I]]
 ; LE-NEXT:    call void @use.i8(i8 [[V_I]])
 ; LE-NEXT:    [[V_I_32:%.*]] = load i32, ptr [[P_I]], align 4
 ; LE-NEXT:    call void @use.i32(i32 [[V_I_32]])
 ; LE-NEXT:    [[I_INC]] = add i64 [[I]], 1
 ; LE-NEXT:    [[CMP:%.*]] = icmp ne i64 [[I_INC]], 64
 ; LE-NEXT:    br i1 [[CMP]], label [[LOOP_LOOP_CRIT_EDGE]], label [[EXIT:%.*]]
 ; LE:       loop.loop_crit_edge:
 ; LE-NEXT:    [[P_I_PHI_TRANS_INSERT:%.*]] = getelementptr i8, ptr [[P]], i64 [[I_INC]]
 ; LE-NEXT:    [[V_I_PRE]] = load i8, ptr [[P_I_PHI_TRANS_INSERT]], align 1
 ; LE-NEXT:    br label [[LOOP]]
 ; LE:       exit:
 ; LE-NEXT:    ret void
 ;
 ; BE-LABEL: @load_load_partial_alias_loop(
 ; BE-NEXT:  entry:
 ; BE-NEXT:    [[P_1:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 1
 ; BE-NEXT:    [[V_1:%.*]] = load i8, ptr [[P_1]], align 1
 ; BE-NEXT:    call void @use.i8(i8 [[V_1]])
 ; BE-NEXT:    [[V_1_32:%.*]] = load i32, ptr [[P_1]], align 4
 ; BE-NEXT:    call void @use.i32(i32 [[V_1_32]])
 ; BE-NEXT:    [[TMP0:%.*]] = lshr i32 [[V_1_32]], 24
-; BE-NEXT:    [[TMP1:%.*]] = trunc i32 [[TMP0]] to i8
+; BE-NEXT:    [[TMP1:%.*]] = freeze i32 [[TMP0]]
+; BE-NEXT:    [[TMP2:%.*]] = trunc i32 [[TMP1]] to i8
 ; BE-NEXT:    br label [[LOOP:%.*]]
 ; BE:       loop:
-; BE-NEXT:    [[V_I:%.*]] = phi i8 [ [[TMP1]], [[ENTRY:%.*]] ], [ [[V_I_PRE:%.*]], [[LOOP_LOOP_CRIT_EDGE:%.*]] ]
+; BE-NEXT:    [[V_I:%.*]] = phi i8 [ [[TMP2]], [[ENTRY:%.*]] ], [ [[V_I_PRE:%.*]], [[LOOP_LOOP_CRIT_EDGE:%.*]] ]
 ; BE-NEXT:    [[I:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[I_INC:%.*]], [[LOOP_LOOP_CRIT_EDGE]] ]
 ; BE-NEXT:    [[P_I:%.*]] = getelementptr i8, ptr [[P]], i64 [[I]]
 ; BE-NEXT:    call void @use.i8(i8 [[V_I]])
 ; BE-NEXT:    [[V_I_32:%.*]] = load i32, ptr [[P_I]], align 4
 ; BE-NEXT:    call void @use.i32(i32 [[V_I_32]])
 ; BE-NEXT:    [[I_INC]] = add i64 [[I]], 1
 ; BE-NEXT:    [[CMP:%.*]] = icmp ne i64 [[I_INC]], 64
 ; BE-NEXT:    br i1 [[CMP]], label [[LOOP_LOOP_CRIT_EDGE]], label [[EXIT:%.*]]

 
 define void @load_load_partial_alias_atomic(ptr %arg) {
 ; LE-LABEL: @load_load_partial_alias_atomic(
 ; LE-NEXT:  bb:
 ; LE-NEXT:    [[TMP2_1:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 1
 ; LE-NEXT:    [[TMP2_3:%.*]] = load i64, ptr [[TMP2_1]], align 4
 ; LE-NEXT:    [[TMP3_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 2
 ; LE-NEXT:    [[TMP0:%.*]] = lshr i64 [[TMP2_3]], 8
-; LE-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8
+; LE-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
+; LE-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
 ; LE-NEXT:    br label [[BB5:%.*]]
 ; LE:       bb5:
-; LE-NEXT:    [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], [[BB5]] ], [ [[TMP1]], [[BB:%.*]] ]
+; LE-NEXT:    [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], [[BB5]] ], [ [[TMP2]], [[BB:%.*]] ]
 ; LE-NEXT:    [[TMP6_1:%.*]] = load atomic i8, ptr @global acquire, align 4
 ; LE-NEXT:    [[TMP7_1:%.*]] = add i8 [[TMP6_1]], [[TMP4_1]]
 ; LE-NEXT:    store i8 [[TMP7_1]], ptr [[ARG]], align 1
 ; LE-NEXT:    [[TMP4_1_PRE]] = load i8, ptr [[TMP3_1]], align 4
 ; LE-NEXT:    br label [[BB5]]
 ;
 ; BE-LABEL: @load_load_partial_alias_atomic(
 ; BE-NEXT:  bb:
 ; BE-NEXT:    [[TMP2_1:%.*]] = getelementptr inbounds i8, ptr [[ARG:%.*]], i64 1
 ; BE-NEXT:    [[TMP2_3:%.*]] = load i64, ptr [[TMP2_1]], align 4
 ; BE-NEXT:    [[TMP3_1:%.*]] = getelementptr inbounds i8, ptr [[ARG]], i64 2
 ; BE-NEXT:    [[TMP0:%.*]] = lshr i64 [[TMP2_3]], 48
-; BE-NEXT:    [[TMP1:%.*]] = trunc i64 [[TMP0]] to i8
+; BE-NEXT:    [[TMP1:%.*]] = freeze i64 [[TMP0]]
+; BE-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
 ; BE-NEXT:    br label [[BB5:%.*]]
 ; BE:       bb5:
-; BE-NEXT:    [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], [[BB5]] ], [ [[TMP1]], [[BB:%.*]] ]
+; BE-NEXT:    [[TMP4_1:%.*]] = phi i8 [ [[TMP4_1_PRE:%.*]], [[BB5]] ], [ [[TMP2]], [[BB:%.*]] ]
 ; BE-NEXT:    [[TMP6_1:%.*]] = load atomic i8, ptr @global acquire, align 4
 ; BE-NEXT:    [[TMP7_1:%.*]] = add i8 [[TMP6_1]], [[TMP4_1]]
 ; BE-NEXT:    store i8 [[TMP7_1]], ptr [[ARG]], align 1
 ; BE-NEXT:    [[TMP4_1_PRE]] = load i8, ptr [[TMP3_1]], align 4
 ; BE-NEXT:    br label [[BB5]]
 ;
 bb:
   %tmp2.1 = getelementptr inbounds i8, ptr %arg, i64 1

llvm/test/Transforms/GVN/metadata.ll

;
   ret void
 }
 
 define void @load_ptr_nonnull_to_i32(ptr %p) {
 ; CHECK-LABEL: define void @load_ptr_nonnull_to_i32
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[VAL:%.*]] = load ptr, ptr [[P]], align 8
 ; CHECK-NEXT:    [[VAL_INT:%.*]] = ptrtoint ptr [[VAL]] to i64
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[VAL_INT]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[VAL_INT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
 ; CHECK-NEXT:    call void @use.i64(i64 [[VAL_INT]])
-; CHECK-NEXT:    call void @use.i32(i32 [[TMP1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
   %val = load ptr, ptr %p, align 8, !nonnull !{}
   %val.int = ptrtoint ptr %val to i64
   %val2 = load i32, ptr %p, align 8
   call void @use.i64(i64 %val.int)
   call void @use.i32(i32 %val2)
   ret void
 }
 
 define void @load_i64_range_to_i32_range(ptr %p) {
 ; CHECK-LABEL: define void @load_i64_range_to_i32_range
 ; CHECK-SAME: (ptr [[P:%.*]]) {
 ; CHECK-NEXT:    [[VAL:%.*]] = load i64, ptr [[P]], align 8
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[VAL]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = freeze i64 [[VAL]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
 ; CHECK-NEXT:    call void @use.i64(i64 [[VAL]])
-; CHECK-NEXT:    call void @use.i32(i32 [[TMP1]])
+; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    ret void
 ;
   %val = load i64, ptr %p, align 8, !range !{i64 0, i64 10}
   %val2 = load i32, ptr %p, align 8, !range !{i32 0, i32 10}
   call void @use.i64(i64 %val)
   call void @use.i32(i32 %val2)
   ret void
 }

llvm/test/Transforms/PhaseOrdering/X86/vec-load-combine.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes="default<O3>" -S            < %s | FileCheck %s --check-prefix=SSE
 ; RUN: opt -passes="default<O3>" -S -mattr=avx < %s | FileCheck %s --check-prefix=AVX
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64--"
 
 %union.ElementWiseAccess = type { <4 x float> }
 
 $getAt = comdat any
 
 define dso_local noundef <4 x float> @ConvertVectors_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %0) #0 {
 ; SSE-LABEL: @ConvertVectors_ByRef(
 ; SSE-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; SSE-NEXT:    [[DOTFR4:%.*]] = freeze <4 x float> [[TMP2]]
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[DOTFR4]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
 ; SSE-NEXT:    ret <4 x float> [[TMP3]]
 ;
 ; AVX-LABEL: @ConvertVectors_ByRef(
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x float>, ptr [[TMP0:%.*]], align 16
-; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
+; AVX-NEXT:    [[DOTFR4:%.*]] = freeze <4 x float> [[TMP2]]
+; AVX-NEXT:    [[TMP3:%.*]] = shufflevector <4 x float> [[DOTFR4]], <4 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 2>
 ; AVX-NEXT:    ret <4 x float> [[TMP3]]
 ;
   %2 = alloca ptr, align 8
   %3 = alloca <4 x float>, align 16
   store ptr %0, ptr %2, align 8
   %4 = load ptr, ptr %2, align 8
   %5 = call noundef nonnull align 16 dereferenceable(16) ptr @castToElementWiseAccess_ByRef(ptr noundef nonnull align 16 dereferenceable(16) %4)
   %6 = call noundef float @getAt(ptr noundef nonnull align 16 dereferenceable(16) %5, i32 noundef 0)