Index: llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -1375,8 +1375,9 @@ unsigned AddressSpace, TTI::TargetCostKind CostKind, const Instruction *I) { + EVT VT = TLI->getValueType(DL, Ty, true); // Type legalization can't handle structs - if (TLI->getValueType(DL, Ty, true) == MVT::Other) + if (VT == MVT::Other) return BaseT::getMemoryOpCost(Opcode, Ty, Alignment, AddressSpace, CostKind); @@ -1403,25 +1404,35 @@ return LT.first * 2 * AmortizationCost; } - if (useNeonVector(Ty) && + bool IsTruncOrExt = VT != LT.second; + if (IsTruncOrExt && useNeonVector(Ty) && cast(Ty)->getElementType()->isIntegerTy(8)) { - unsigned ProfitableNumElements; - if (Opcode == Instruction::Store) - // We use a custom trunc store lowering so v.4b should be profitable. - ProfitableNumElements = 4; - else - // We scalarize the loads because there is not v.4b register and we - // have to promote the elements to v.2. - ProfitableNumElements = 8; - - if (cast(Ty)->getNumElements() < ProfitableNumElements) { - unsigned NumVecElts = cast(Ty)->getNumElements(); - unsigned NumVectorizableInstsToAmortize = NumVecElts * 2; - // We generate 2 instructions per vector element. - return NumVectorizableInstsToAmortize * NumVecElts * 2; - } - } + static const CostTblEntry CostTblTruncExt[] { + // Truncating i32/i16 stores, see test/CodeGen/AArch64/neon-truncstore.ll. + {ISD::STORE, MVT::v2i16, 4}, + {ISD::STORE, MVT::v4i16, 2}, + {ISD::STORE, MVT::v8i16, 2}, + {ISD::STORE, MVT::v16i16, 3}, + {ISD::STORE, MVT::v2i32, 4}, + {ISD::STORE, MVT::v4i32, 3}, + {ISD::STORE, MVT::v8i32, 4}, + {ISD::STORE, MVT::v16i32, 7}, + {ISD::STORE, MVT::v32i32, 13}, + // Extending i8 loads, see test/CodeGen/AArch65/aarch64-load-ext.ll. + {ISD::LOAD, MVT::v2i16, 4}, + {ISD::LOAD, MVT::v4i16, 2}, + {ISD::LOAD, MVT::v8i16, 2}, + {ISD::LOAD, MVT::v16i16, 3}, + {ISD::LOAD, MVT::v2i32, 4}, + {ISD::LOAD, MVT::v4i32, 3}, + {ISD::LOAD, MVT::v8i32, 3}, + }; + int ISD = Opcode == Instruction::Store ? ISD::STORE : ISD::LOAD; + if (const auto *Entry = CostTableLookup(CostTblTruncExt, ISD, LT.second)) + return LT.first * Entry->Cost; + return cast(Ty)->getNumElements() * 2; + } return LT.first; } Index: llvm/test/Analysis/CostModel/AArch64/cast.ll =================================================================== --- llvm/test/Analysis/CostModel/AArch64/cast.ll +++ llvm/test/Analysis/CostModel/AArch64/cast.ll @@ -701,8 +701,8 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadi8 = load i8, i8* undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadi16 = load i16, i16* undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadi32 = load i32, i32* undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef -; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %loadv2i8 = load <2 x i8>, <2 x i8>* undef +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %loadv4i8 = load <4 x i8>, <4 x i8>* undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadv8i8 = load <8 x i8>, <8 x i8>* undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadv2i16 = load <2 x i16>, <2 x i16>* undef ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %loadv4i16 = load <4 x i16>, <4 x i16>* undef Index: llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll =================================================================== --- llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll +++ llvm/test/Analysis/CostModel/AArch64/mem-op-cost-model.ll @@ -49,8 +49,8 @@ define <4 x i8> @load4(<4 x i8>* %ptr) { ; CHECK: 'Cost Model Analysis' for function 'load4': -; CHECK-NEON: Cost Model: Found an estimated cost of 64 for instruction: -; CHECK-SVE-128: Cost Model: Found an estimated cost of 64 for instruction: +; CHECK-NEON: Cost Model: Found an estimated cost of 2 for instruction: +; CHECK-SVE-128: Cost Model: Found an estimated cost of 2 for instruction: ; CHECK-SVE-256: Cost Model: Found an estimated cost of 1 for instruction: ; CHECK-SVE-512: Cost Model: Found an estimated cost of 1 for instruction: %out = load <4 x i8>, <4 x i8>* %ptr @@ -59,8 +59,8 @@ define void @store4(<4 x i8>* %ptr, <4 x i8> %val) { ; CHECK: 'Cost Model Analysis' for function 'store4': -; CHECK-NEON: Cost Model: Found an estimated cost of 1 for instruction: -; CHECK-SVE-128: Cost Model: Found an estimated cost of 1 for instruction: +; CHECK-NEON: Cost Model: Found an estimated cost of 2 for instruction: +; CHECK-SVE-128: Cost Model: Found an estimated cost of 2 for instruction: ; CHECK-SVE-256: Cost Model: Found an estimated cost of 1 for instruction: ; CHECK-SVE-512: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %val, <4 x i8>* %ptr Index: llvm/test/Analysis/CostModel/AArch64/store.ll =================================================================== --- llvm/test/Analysis/CostModel/AArch64/store.ll +++ llvm/test/Analysis/CostModel/AArch64/store.ll @@ -12,7 +12,7 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i64> undef, <4 x i64>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x i32> undef, <8 x i32>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <16 x i16> undef, <16 x i16>* undef, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <32 x i8> undef, <32 x i8>* undef, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: store <32 x i8> undef, <32 x i8>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x double> undef, <4 x double>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <8 x float> undef, <8 x float>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <16 x half> undef, <16 x half>* undef, align 4 @@ -23,10 +23,10 @@ ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x double> undef, <2 x double>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x float> undef, <4 x float>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x half> undef, <8 x half>* undef, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2 -; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4 -; CHECK-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2 -; CHECK-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4 +; CHECK-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2 +; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4 ; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; ; SIZE-LABEL: 'getMemoryOpCost' @@ -65,10 +65,10 @@ ; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <2 x double> undef, <2 x double>* undef, align 4 ; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <4 x float> undef, <4 x float>* undef, align 4 ; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <8 x half> undef, <8 x half>* undef, align 4 -; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2 -; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4 -; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2 -; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4 +; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i8> undef, <2 x i8>* undef, align 2 +; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> undef, <4 x i8>* undef, align 4 +; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %1 = load <2 x i8>, <2 x i8>* undef, align 2 +; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <4 x i8>, <4 x i8>* undef, align 4 ; SLOW_MISALIGNED_128_STORE-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; store <4 x i64> undef, <4 x i64> * undef Index: llvm/test/Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/extend-vectorization-factor-for-unprofitable-memops.ll @@ -4,12 +4,11 @@ ; are not profitable. ; Test with a loop that contains memory accesses of i8 and i32 types. The -; default maximum VF for NEON is 4, but vectorizing 4 x i8 is not -; profitable. But we can extend to VF to 8 or 16, at which point the -; i8 memory accesses become profitable. +; default maximum VF for NEON is 4. And while we don't have an instruction to +; load 4 x i8, vectorization might still be profitable. define void @test_load_i8_store_i32(i8* noalias %src, i32* noalias %dst, i32 %off, i64 %N) { ; CHECK-LABEL: @test_load_i8_store_i32( -; CHECK-NOT: x i8> +; CHECK: x i8> ; entry: br label %loop Index: llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll +++ llvm/test/Transforms/LoopVectorize/AArch64/interleaved-vs-scalar.ll @@ -11,10 +11,10 @@ %pair = type { i8, i8 } ; CHECK-LABEL: test -; CHECK: Found an estimated cost of 20 for VF 2 For instruction: {{.*}} load i8 +; CHECK: Found an estimated cost of 17 for VF 2 For instruction: {{.*}} load i8 ; CHECK: Found an estimated cost of 0 for VF 2 For instruction: {{.*}} load i8 ; CHECK: vector.body -; CHECK: load i8 +; CHECK: load <4 x i8> ; CHECK: br i1 {{.*}}, label %middle.block, label %vector.body define void @test(%pair* %p, i64 %n) { Index: llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll =================================================================== --- llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll +++ llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll @@ -171,10 +171,8 @@ ; ; MAX-COST-LABEL: @PR32038( ; MAX-COST-NEXT: entry: -; MAX-COST-NEXT: [[TMP0:%.*]] = load <2 x i8>, <2 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <2 x i8>*), align 1 -; MAX-COST-NEXT: [[TMP1:%.*]] = icmp eq <2 x i8> [[TMP0]], zeroinitializer -; MAX-COST-NEXT: [[TMP2:%.*]] = load <2 x i8>, <2 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 3) to <2 x i8>*), align 1 -; MAX-COST-NEXT: [[TMP3:%.*]] = icmp eq <2 x i8> [[TMP2]], zeroinitializer +; MAX-COST-NEXT: [[TMP0:%.*]] = load <4 x i8>, <4 x i8>* bitcast (i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 1) to <4 x i8>*), align 1 +; MAX-COST-NEXT: [[TMP1:%.*]] = icmp eq <4 x i8> [[TMP0]], zeroinitializer ; MAX-COST-NEXT: [[P8:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 5), align 1 ; MAX-COST-NEXT: [[P9:%.*]] = icmp eq i8 [[P8]], 0 ; MAX-COST-NEXT: [[P10:%.*]] = load i8, i8* getelementptr inbounds ([80 x i8], [80 x i8]* @a, i64 0, i64 6), align 2 @@ -186,21 +184,17 @@ ; MAX-COST-NEXT: br label [[FOR_BODY:%.*]] ; MAX-COST: for.body: ; MAX-COST-NEXT: [[P17:%.*]] = phi i32 [ [[P34:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ] -; MAX-COST-NEXT: [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1 -; MAX-COST-NEXT: [[TMP5:%.*]] = shufflevector <2 x i1> [[TMP1]], <2 x i1> poison, <4 x i32> -; MAX-COST-NEXT: [[TMP6:%.*]] = shufflevector <4 x i1> poison, <4 x i1> [[TMP5]], <4 x i32> -; MAX-COST-NEXT: [[TMP7:%.*]] = shufflevector <2 x i1> [[TMP3]], <2 x i1> poison, <4 x i32> -; MAX-COST-NEXT: [[TMP8:%.*]] = shufflevector <4 x i1> [[TMP6]], <4 x i1> [[TMP7]], <4 x i32> -; MAX-COST-NEXT: [[TMP9:%.*]] = select <4 x i1> [[TMP8]], <4 x i32> , <4 x i32> -; MAX-COST-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0 -; MAX-COST-NEXT: [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1 -; MAX-COST-NEXT: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 +; MAX-COST-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3 +; MAX-COST-NEXT: [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> , <4 x i32> +; MAX-COST-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2 +; MAX-COST-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1 +; MAX-COST-NEXT: [[TMP6:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0 ; MAX-COST-NEXT: [[P27:%.*]] = select i1 [[P9]], i32 -720, i32 -80 ; MAX-COST-NEXT: [[P29:%.*]] = select i1 [[P11]], i32 -720, i32 -80 -; MAX-COST-NEXT: [[TMP13:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]]) -; MAX-COST-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], [[P27]] -; MAX-COST-NEXT: [[TMP15:%.*]] = add i32 [[TMP14]], [[P29]] -; MAX-COST-NEXT: [[OP_EXTRA:%.*]] = add i32 [[TMP15]], -5 +; MAX-COST-NEXT: [[TMP7:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP3]]) +; MAX-COST-NEXT: [[TMP8:%.*]] = add i32 [[TMP7]], [[P27]] +; MAX-COST-NEXT: [[TMP9:%.*]] = add i32 [[TMP8]], [[P29]] +; MAX-COST-NEXT: [[OP_EXTRA:%.*]] = add i32 [[TMP9]], -5 ; MAX-COST-NEXT: [[P31:%.*]] = select i1 [[P13]], i32 -720, i32 -80 ; MAX-COST-NEXT: [[P32:%.*]] = add i32 [[OP_EXTRA]], [[P31]] ; MAX-COST-NEXT: [[P33:%.*]] = select i1 [[P15]], i32 -720, i32 -80 Index: llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll =================================================================== --- llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll +++ llvm/test/Transforms/SLPVectorizer/AArch64/loadi8.ll @@ -13,52 +13,32 @@ ; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SCALE]], align 16 ; CHECK-NEXT: [[OFFSET:%.*]] = getelementptr inbounds [[STRUCT_WEIGHT_T]], %struct.weight_t* [[W]], i64 0, i32 1 ; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[OFFSET]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = load i8, i8* [[SRC:%.*]], align 1 -; CHECK-NEXT: [[CONV:%.*]] = zext i8 [[TMP2]] to i32 -; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP0]], [[CONV]] -; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[MUL]], [[TMP1]] -; CHECK-NEXT: [[TOBOOL_NOT_I:%.*]] = icmp ult i32 [[ADD]], 256 -; CHECK-NEXT: [[TMP3:%.*]] = icmp sgt i32 [[ADD]], 0 -; CHECK-NEXT: [[SHR_I:%.*]] = sext i1 [[TMP3]] to i32 -; CHECK-NEXT: [[COND_I:%.*]] = select i1 [[TOBOOL_NOT_I]], i32 [[ADD]], i32 [[SHR_I]] -; CHECK-NEXT: [[CONV_I:%.*]] = trunc i32 [[COND_I]] to i8 -; CHECK-NEXT: store i8 [[CONV_I]], i8* [[DST:%.*]], align 1 -; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 1 -; CHECK-NEXT: [[TMP4:%.*]] = load i8, i8* [[ARRAYIDX_1]], align 1 -; CHECK-NEXT: [[CONV_1:%.*]] = zext i8 [[TMP4]] to i32 -; CHECK-NEXT: [[MUL_1:%.*]] = mul nsw i32 [[TMP0]], [[CONV_1]] -; CHECK-NEXT: [[ADD_1:%.*]] = add nsw i32 [[MUL_1]], [[TMP1]] -; CHECK-NEXT: [[TOBOOL_NOT_I_1:%.*]] = icmp ult i32 [[ADD_1]], 256 -; CHECK-NEXT: [[TMP5:%.*]] = icmp sgt i32 [[ADD_1]], 0 -; CHECK-NEXT: [[SHR_I_1:%.*]] = sext i1 [[TMP5]] to i32 -; CHECK-NEXT: [[COND_I_1:%.*]] = select i1 [[TOBOOL_NOT_I_1]], i32 [[ADD_1]], i32 [[SHR_I_1]] -; CHECK-NEXT: [[CONV_I_1:%.*]] = trunc i32 [[COND_I_1]] to i8 -; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 1 -; CHECK-NEXT: store i8 [[CONV_I_1]], i8* [[ARRAYIDX2_1]], align 1 +; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds i8, i8* [[SRC:%.*]], i64 1 +; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds i8, i8* [[DST:%.*]], i64 1 ; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 2 -; CHECK-NEXT: [[TMP6:%.*]] = load i8, i8* [[ARRAYIDX_2]], align 1 -; CHECK-NEXT: [[CONV_2:%.*]] = zext i8 [[TMP6]] to i32 -; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[TMP0]], [[CONV_2]] -; CHECK-NEXT: [[ADD_2:%.*]] = add nsw i32 [[MUL_2]], [[TMP1]] -; CHECK-NEXT: [[TOBOOL_NOT_I_2:%.*]] = icmp ult i32 [[ADD_2]], 256 -; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt i32 [[ADD_2]], 0 -; CHECK-NEXT: [[SHR_I_2:%.*]] = sext i1 [[TMP7]] to i32 -; CHECK-NEXT: [[COND_I_2:%.*]] = select i1 [[TOBOOL_NOT_I_2]], i32 [[ADD_2]], i32 [[SHR_I_2]] -; CHECK-NEXT: [[CONV_I_2:%.*]] = trunc i32 [[COND_I_2]] to i8 ; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 2 -; CHECK-NEXT: store i8 [[CONV_I_2]], i8* [[ARRAYIDX2_2]], align 1 ; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds i8, i8* [[SRC]], i64 3 -; CHECK-NEXT: [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX_3]], align 1 -; CHECK-NEXT: [[CONV_3:%.*]] = zext i8 [[TMP8]] to i32 -; CHECK-NEXT: [[MUL_3:%.*]] = mul nsw i32 [[TMP0]], [[CONV_3]] -; CHECK-NEXT: [[ADD_3:%.*]] = add nsw i32 [[MUL_3]], [[TMP1]] -; CHECK-NEXT: [[TOBOOL_NOT_I_3:%.*]] = icmp ult i32 [[ADD_3]], 256 -; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt i32 [[ADD_3]], 0 -; CHECK-NEXT: [[SHR_I_3:%.*]] = sext i1 [[TMP9]] to i32 -; CHECK-NEXT: [[COND_I_3:%.*]] = select i1 [[TOBOOL_NOT_I_3]], i32 [[ADD_3]], i32 [[SHR_I_3]] -; CHECK-NEXT: [[CONV_I_3:%.*]] = trunc i32 [[COND_I_3]] to i8 +; CHECK-NEXT: [[TMP2:%.*]] = bitcast i8* [[SRC]] to <4 x i8>* +; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, <4 x i8>* [[TMP2]], align 1 +; CHECK-NEXT: [[TMP4:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32> +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[TMP0]], i32 1 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP0]], i32 2 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP0]], i32 3 +; CHECK-NEXT: [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP4]] +; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> poison, i32 [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[TMP1]], i32 1 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[TMP1]], i32 2 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP1]], i32 3 +; CHECK-NEXT: [[TMP14:%.*]] = add nsw <4 x i32> [[TMP9]], [[TMP13]] +; CHECK-NEXT: [[TMP15:%.*]] = icmp ult <4 x i32> [[TMP14]], +; CHECK-NEXT: [[TMP16:%.*]] = icmp sgt <4 x i32> [[TMP14]], zeroinitializer +; CHECK-NEXT: [[TMP17:%.*]] = sext <4 x i1> [[TMP16]] to <4 x i32> +; CHECK-NEXT: [[TMP18:%.*]] = select <4 x i1> [[TMP15]], <4 x i32> [[TMP14]], <4 x i32> [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = trunc <4 x i32> [[TMP18]] to <4 x i8> ; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds i8, i8* [[DST]], i64 3 -; CHECK-NEXT: store i8 [[CONV_I_3]], i8* [[ARRAYIDX2_3]], align 1 +; CHECK-NEXT: [[TMP20:%.*]] = bitcast i8* [[DST]] to <4 x i8>* +; CHECK-NEXT: store <4 x i8> [[TMP19]], <4 x i8>* [[TMP20]], align 1 ; CHECK-NEXT: ret void ; entry: