diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -6443,7 +6443,6 @@ bool CodeGenPrepare::optimizeExtUses(Instruction *I) { BasicBlock *DefBB = I->getParent(); - // If the result of a {s|z}ext and its source are both live out, rewrite all // other uses of the source with result of extension. Value *Src = I->getOperand(0); @@ -8048,8 +8047,9 @@ if (OptimizeNoopCopyExpression(CI, *TLI, *DL)) return true; - if (isa(I) && TLI->optimizeExtendOrTruncateConversion( - I, LI->getLoopFor(I->getParent()))) + if ((isa(I) || isa(I)) && + TLI->optimizeExtendOrTruncateConversion(I, + LI->getLoopFor(I->getParent()))) return true; if (isa(I) || isa(I)) { @@ -8068,6 +8068,7 @@ return MadeChange | optimizeExtUses(I); } } + return false; } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -13239,6 +13239,43 @@ ZExt->eraseFromParent(); } +static void createTblForTrunc(TruncInst *TI) { + IRBuilder<> Builder(TI); + SmallVector Parts; + Type *VecTy = FixedVectorType::get(Builder.getInt8Ty(), 16); + Parts.push_back(Builder.CreateBitCast( + Builder.CreateShuffleVector(TI->getOperand(0), {0, 1, 2, 3}), VecTy)); + Parts.push_back(Builder.CreateBitCast( + Builder.CreateShuffleVector(TI->getOperand(0), {4, 5, 6, 7}), VecTy)); + + Intrinsic::ID TblID = Intrinsic::aarch64_neon_tbl2; + unsigned NumElements = cast(TI->getType())->getNumElements(); + if (NumElements == 16) { + Parts.push_back(Builder.CreateBitCast( + Builder.CreateShuffleVector(TI->getOperand(0), {8, 9, 10, 11}), VecTy)); + Parts.push_back(Builder.CreateBitCast( + Builder.CreateShuffleVector(TI->getOperand(0), {12, 13, 14, 15}), + VecTy)); + TblID = Intrinsic::aarch64_neon_tbl4; + } + SmallVector MaskConst; + for (unsigned Idx = 0; Idx < NumElements * 4; Idx += 4) + MaskConst.push_back(ConstantInt::get(Builder.getInt8Ty(), Idx)); + + for (unsigned Idx = NumElements * 4; Idx < 64; Idx += 4) + MaskConst.push_back(ConstantInt::get(Builder.getInt8Ty(), 255)); + + Parts.push_back(ConstantVector::get(MaskConst)); + auto *F = + Intrinsic::getDeclaration(TI->getModule(), TblID, Parts[0]->getType()); + Value *Res = Builder.CreateCall(F, Parts); + + if (NumElements == 8) + Res = Builder.CreateShuffleVector(Res, {0, 1, 2, 3, 4, 5, 6, 7}); + TI->replaceAllUsesWith(Res); + TI->eraseFromParent(); +} + bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(Instruction *I, Loop *L) const { // Try to optimize conversions using tbl. This requires materializing constant @@ -13284,7 +13321,20 @@ I->replaceAllUsesWith(UI); I->eraseFromParent(); createTblShuffleForZExt(ZExt); + return true; } + + // Convert 'trunc <(8|16) x i32> %x to <(8|16) x i32>' to a single tbl.4 + // instruction selecting the lowest 8 bits per lane of the input interpreted + // as 2 or 4 <4 x i32> vectors. + auto *TI = dyn_cast(I); + if (TI && (SrcTy->getNumElements() == 8 || SrcTy->getNumElements() == 16) && + SrcTy->getElementType()->isIntegerTy(32) && + DstTy->getElementType()->isIntegerTy(8)) { + createTblForTrunc(TI); + return true; + } + return false; } diff --git a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll --- a/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll +++ b/llvm/test/CodeGen/AArch64/trunc-to-tbl.ll @@ -4,25 +4,47 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" target triple = "arm64-apple-ios" +; CHECK-LABEL: lCPI0_0: +; CHECK-NEXT: .byte 0 ; 0x0 +; CHECK-NEXT: .byte 4 ; 0x4 +; CHECK-NEXT: .byte 8 ; 0x8 +; CHECK-NEXT: .byte 12 ; 0xc +; CHECK-NEXT: .byte 16 ; 0x10 +; CHECK-NEXT: .byte 20 ; 0x14 +; CHECK-NEXT: .byte 24 ; 0x18 +; CHECK-NEXT: .byte 28 ; 0x1c +; CHECK-NEXT: .byte 32 ; 0x20 +; CHECK-NEXT: .byte 36 ; 0x24 +; CHECK-NEXT: .byte 40 ; 0x28 +; CHECK-NEXT: .byte 44 ; 0x2c +; CHECK-NEXT: .byte 48 ; 0x30 +; CHECK-NEXT: .byte 52 ; 0x34 +; CHECK-NEXT: .byte 56 ; 0x38 +; CHECK-NEXT: .byte 60 ; 0x3c + + ; It's profitable to use a single tbl.4 instruction to lower the truncate. define void @trunc_v16i32_to_v16i8_in_loop(ptr %A, ptr %dst) { ; CHECK-LABEL: trunc_v16i32_to_v16i8_in_loop: ; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: Lloh0: +; CHECK-NEXT: adrp x9, lCPI0_0@PAGE ; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: Lloh1: +; CHECK-NEXT: ldr q0, [x9, lCPI0_0@PAGEOFF] ; CHECK-NEXT: LBB0_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: add x9, x0, x8, lsl #6 -; CHECK-NEXT: ldp q1, q0, [x9, #32] -; CHECK-NEXT: ldp q3, q2, [x9] -; CHECK-NEXT: uzp1.8h v0, v1, v0 -; CHECK-NEXT: uzp1.8h v1, v3, v2 -; CHECK-NEXT: uzp1.16b v0, v1, v0 -; CHECK-NEXT: str q0, [x1, x8, lsl #4] +; CHECK-NEXT: ldp q1, q2, [x9] +; CHECK-NEXT: ldp q3, q4, [x9, #32] +; CHECK-NEXT: tbl.16b v1, { v1, v2, v3, v4 }, v0 +; CHECK-NEXT: str q1, [x1, x8, lsl #4] ; CHECK-NEXT: add x8, x8, #1 ; CHECK-NEXT: cmp x8, #1000 ; CHECK-NEXT: b.eq LBB0_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret +; CHECK-NEXT: .loh AdrpLdr Lloh0, Lloh1 entry: br label %loop @@ -60,23 +82,46 @@ ret void } + +; CHECK-LABEL: lCPI2_0: +; CHECK-NEXT: .byte 0 ; 0x0 +; CHECK-NEXT: .byte 4 ; 0x4 +; CHECK-NEXT: .byte 8 ; 0x8 +; CHECK-NEXT: .byte 12 ; 0xc +; CHECK-NEXT: .byte 16 ; 0x10 +; CHECK-NEXT: .byte 20 ; 0x14 +; CHECK-NEXT: .byte 24 ; 0x18 +; CHECK-NEXT: .byte 28 ; 0x1c +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff +; CHECK-NEXT: .byte 255 ; 0xff + ; It's profitable to use a single tbl.2 instruction to lower the truncate. define void @trunc_v8i32_to_v8i8_in_loop(ptr %A, ptr %dst) { ; CHECK-LABEL: trunc_v8i32_to_v8i8_in_loop: ; CHECK: ; %bb.0: ; %entry +; CHECK-NEXT: Lloh2: +; CHECK-NEXT: adrp x9, lCPI2_0@PAGE ; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: Lloh3: +; CHECK-NEXT: ldr q0, [x9, lCPI2_0@PAGEOFF] ; CHECK-NEXT: LBB2_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: add x9, x0, x8, lsl #5 -; CHECK-NEXT: ldp q1, q0, [x9] -; CHECK-NEXT: uzp1.8h v0, v1, v0 -; CHECK-NEXT: xtn.8b v0, v0 -; CHECK-NEXT: str d0, [x1, x8, lsl #3] +; CHECK-NEXT: ldp q1, q2, [x9] +; CHECK-NEXT: tbl.16b v1, { v1, v2 }, v0 +; CHECK-NEXT: str d1, [x1, x8, lsl #3] ; CHECK-NEXT: add x8, x8, #1 ; CHECK-NEXT: cmp x8, #1000 ; CHECK-NEXT: b.eq LBB2_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret +; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh3 entry: br label %loop