Index: lib/Target/X86/X86TargetTransformInfo.cpp =================================================================== --- lib/Target/X86/X86TargetTransformInfo.cpp +++ lib/Target/X86/X86TargetTransformInfo.cpp @@ -951,19 +951,33 @@ { ISD::BITREVERSE, MVT::v4i64, 5 }, { ISD::BITREVERSE, MVT::v8i32, 5 }, { ISD::BITREVERSE, MVT::v16i16, 5 }, - { ISD::BITREVERSE, MVT::v32i8, 5 } + { ISD::BITREVERSE, MVT::v32i8, 5 }, + { ISD::BSWAP, MVT::v4i64, 1 }, + { ISD::BSWAP, MVT::v8i32, 1 }, + { ISD::BSWAP, MVT::v16i16, 1 } }; static const CostTblEntry AVX1CostTbl[] = { { ISD::BITREVERSE, MVT::v4i64, 10 }, { ISD::BITREVERSE, MVT::v8i32, 10 }, { ISD::BITREVERSE, MVT::v16i16, 10 }, - { ISD::BITREVERSE, MVT::v32i8, 10 } + { ISD::BITREVERSE, MVT::v32i8, 10 }, + { ISD::BSWAP, MVT::v4i64, 4 }, + { ISD::BSWAP, MVT::v8i32, 4 }, + { ISD::BSWAP, MVT::v16i16, 4 } }; static const CostTblEntry SSSE3CostTbl[] = { { ISD::BITREVERSE, MVT::v2i64, 5 }, { ISD::BITREVERSE, MVT::v4i32, 5 }, { ISD::BITREVERSE, MVT::v8i16, 5 }, - { ISD::BITREVERSE, MVT::v16i8, 5 } + { ISD::BITREVERSE, MVT::v16i8, 5 }, + { ISD::BSWAP, MVT::v2i64, 1 }, + { ISD::BSWAP, MVT::v4i32, 1 }, + { ISD::BSWAP, MVT::v8i16, 1 } + }; + static const CostTblEntry SSE2CostTbl[] = { + { ISD::BSWAP, MVT::v2i64, 7 }, + { ISD::BSWAP, MVT::v4i32, 7 }, + { ISD::BSWAP, MVT::v8i16, 7 } }; unsigned ISD = ISD::DELETED_NODE; @@ -973,6 +987,9 @@ case Intrinsic::bitreverse: ISD = ISD::BITREVERSE; break; + case Intrinsic::bswap: + ISD = ISD::BSWAP; + break; } // Legalize the type. @@ -996,6 +1013,10 @@ if (const auto *Entry = CostTableLookup(SSSE3CostTbl, ISD, MTy)) return LT.first * Entry->Cost; + if (ST->hasSSE2()) + if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy)) + return LT.first * Entry->Cost; + return BaseT::getIntrinsicInstrCost(IID, RetTy, Tys, FMF); } Index: test/Analysis/CostModel/X86/bswap.ll =================================================================== --- test/Analysis/CostModel/X86/bswap.ll +++ test/Analysis/CostModel/X86/bswap.ll @@ -1,8 +1,8 @@ ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=pentium4 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=SSE2 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=SSE42 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core-avx2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX2 -; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=corei7-avx -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -check-prefix=AVX1 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core-avx2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=AVX -check-prefix=AVX2 +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX1 ; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver4 -cost-model -analyze | FileCheck %s -check-prefix=CHECK -check-prefix=XOP -check-prefix=XOPAVX2 ; Verify the cost of vector bswap instructions. @@ -17,66 +17,66 @@ define <2 x i64> @var_bswap_v2i64(<2 x i64> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v2i64': -; SSE2: Found an estimated cost of 6 for instruction: %bswap -; SSE42: Found an estimated cost of 6 for instruction: %bswap -; AVX: Found an estimated cost of 6 for instruction: %bswap -; AVX2: Found an estimated cost of 6 for instruction: %bswap -; XOP: Found an estimated cost of 6 for instruction: %bswap +; SSE2: Found an estimated cost of 7 for instruction: %bswap +; SSE42: Found an estimated cost of 1 for instruction: %bswap +; AVX: Found an estimated cost of 1 for instruction: %bswap +; XOP: Found an estimated cost of 1 for instruction: %bswap %bswap = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %a) ret <2 x i64> %bswap } define <4 x i64> @var_bswap_v4i64(<4 x i64> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v4i64': -; SSE2: Found an estimated cost of 12 for instruction: %bswap -; SSE42: Found an estimated cost of 12 for instruction: %bswap -; AVX: Found an estimated cost of 12 for instruction: %bswap -; AVX2: Found an estimated cost of 12 for instruction: %bswap -; XOP: Found an estimated cost of 12 for instruction: %bswap +; SSE2: Found an estimated cost of 14 for instruction: %bswap +; SSE42: Found an estimated cost of 2 for instruction: %bswap +; AVX1: Found an estimated cost of 4 for instruction: %bswap +; AVX2: Found an estimated cost of 1 for instruction: %bswap +; XOPAVX1: Found an estimated cost of 4 for instruction: %bswap +; XOPAVX2: Found an estimated cost of 1 for instruction: %bswap %bswap = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> %a) ret <4 x i64> %bswap } define <4 x i32> @var_bswap_v4i32(<4 x i32> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v4i32': -; SSE2: Found an estimated cost of 12 for instruction: %bswap -; SSE42: Found an estimated cost of 12 for instruction: %bswap -; AVX: Found an estimated cost of 12 for instruction: %bswap -; AVX2: Found an estimated cost of 12 for instruction: %bswap -; XOP: Found an estimated cost of 12 for instruction: %bswap +; SSE2: Found an estimated cost of 7 for instruction: %bswap +; SSE42: Found an estimated cost of 1 for instruction: %bswap +; AVX: Found an estimated cost of 1 for instruction: %bswap +; XOP: Found an estimated cost of 1 for instruction: %bswap %bswap = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %a) ret <4 x i32> %bswap } define <8 x i32> @var_bswap_v8i32(<8 x i32> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v8i32': -; SSE2: Found an estimated cost of 24 for instruction: %bswap -; SSE42: Found an estimated cost of 24 for instruction: %bswap -; AVX: Found an estimated cost of 24 for instruction: %bswap -; AVX2: Found an estimated cost of 24 for instruction: %bswap -; XOP: Found an estimated cost of 24 for instruction: %bswap +; SSE2: Found an estimated cost of 14 for instruction: %bswap +; SSE42: Found an estimated cost of 2 for instruction: %bswap +; AVX1: Found an estimated cost of 4 for instruction: %bswap +; AVX2: Found an estimated cost of 1 for instruction: %bswap +; XOPAVX1: Found an estimated cost of 4 for instruction: %bswap +; XOPAVX2: Found an estimated cost of 1 for instruction: %bswap %bswap = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> %a) ret <8 x i32> %bswap } define <8 x i16> @var_bswap_v8i16(<8 x i16> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v8i16': -; SSE2: Found an estimated cost of 24 for instruction: %bswap -; SSE42: Found an estimated cost of 24 for instruction: %bswap -; AVX: Found an estimated cost of 24 for instruction: %bswap -; AVX2: Found an estimated cost of 24 for instruction: %bswap -; XOP: Found an estimated cost of 24 for instruction: %bswap +; SSE2: Found an estimated cost of 7 for instruction: %bswap +; SSE42: Found an estimated cost of 1 for instruction: %bswap +; AVX: Found an estimated cost of 1 for instruction: %bswap +; XOP: Found an estimated cost of 1 for instruction: %bswap %bswap = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %a) ret <8 x i16> %bswap } define <16 x i16> @var_bswap_v16i16(<16 x i16> %a) { ; CHECK: 'Cost Model Analysis' for function 'var_bswap_v16i16': -; SSE2: Found an estimated cost of 48 for instruction: %bswap -; SSE42: Found an estimated cost of 48 for instruction: %bswap -; AVX: Found an estimated cost of 48 for instruction: %bswap -; AVX2: Found an estimated cost of 48 for instruction: %bswap -; XOP: Found an estimated cost of 48 for instruction: %bswap +; SSE2: Found an estimated cost of 14 for instruction: %bswap +; SSE42: Found an estimated cost of 2 for instruction: %bswap +; AVX1: Found an estimated cost of 4 for instruction: %bswap +; AVX2: Found an estimated cost of 1 for instruction: %bswap +; XOPAVX1: Found an estimated cost of 4 for instruction: %bswap +; XOPAVX2: Found an estimated cost of 1 for instruction: %bswap %bswap = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> %a) ret <16 x i16> %bswap -} \ No newline at end of file +} Index: test/Analysis/CostModel/X86/scalarize.ll =================================================================== --- test/Analysis/CostModel/X86/scalarize.ll +++ test/Analysis/CostModel/X86/scalarize.ll @@ -21,11 +21,11 @@ define void @test_scalarized_intrinsics() { %r1 = add %i8 undef, undef -; CHECK32: cost of 12 {{.*}}bswap.v4i32 -; CHECK64: cost of 12 {{.*}}bswap.v4i32 +; CHECK32: cost of 1 {{.*}}bswap.v4i32 +; CHECK64: cost of 1 {{.*}}bswap.v4i32 %r2 = call %i4 @llvm.bswap.v4i32(%i4 undef) -; CHECK32: cost of 10 {{.*}}bswap.v2i64 -; CHECK64: cost of 6 {{.*}}bswap.v2i64 +; CHECK32: cost of 1 {{.*}}bswap.v2i64 +; CHECK64: cost of 1 {{.*}}bswap.v2i64 %r3 = call %i8 @llvm.bswap.v2i64(%i8 undef) ; CHECK32: cost of 12 {{.*}}cttz.v4i32 Index: test/Transforms/SLPVectorizer/X86/bswap.ll =================================================================== --- test/Transforms/SLPVectorizer/X86/bswap.ll +++ test/Transforms/SLPVectorizer/X86/bswap.ll @@ -17,14 +17,20 @@ declare i16 @llvm.bswap.i16(i16) define void @bswap_2i64() #0 { -; CHECK-LABEL: @bswap_2i64( -; CHECK-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8 -; CHECK-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]]) -; CHECK-NEXT: store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8 -; CHECK-NEXT: store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8 -; CHECK-NEXT: ret void +; SSE-LABEL: @bswap_2i64( +; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8 +; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8 +; SSE-NEXT: [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]]) +; SSE-NEXT: [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]]) +; SSE-NEXT: store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 0), align 8 +; SSE-NEXT: store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i32 0, i64 1), align 8 +; SSE-NEXT: ret void +; +; AVX-LABEL: @bswap_2i64( +; AVX-NEXT: [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([4 x i64]* @src64 to <2 x i64>*), align 8 +; AVX-NEXT: [[TMP2:%.*]] = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> [[TMP1]]) +; AVX-NEXT: store <2 x i64> [[TMP2]], <2 x i64>* bitcast ([4 x i64]* @dst64 to <2 x i64>*), align 8 +; AVX-NEXT: ret void ; %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 0), align 8 %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i32 0, i64 1), align 8 @@ -36,20 +42,26 @@ } define void @bswap_4i64() #0 { -; CHECK-LABEL: @bswap_4i64( -; CHECK-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4 -; CHECK-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4 -; CHECK-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4 -; CHECK-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]]) -; CHECK-NEXT: [[BSWAP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD2]]) -; CHECK-NEXT: [[BSWAP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD3]]) -; CHECK-NEXT: store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4 -; CHECK-NEXT: store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4 -; CHECK-NEXT: store i64 [[BSWAP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4 -; CHECK-NEXT: store i64 [[BSWAP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4 -; CHECK-NEXT: ret void +; SSE-LABEL: @bswap_4i64( +; SSE-NEXT: [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4 +; SSE-NEXT: [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4 +; SSE-NEXT: [[LD2:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 2), align 4 +; SSE-NEXT: [[LD3:%.*]] = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 3), align 4 +; SSE-NEXT: [[BSWAP0:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD0]]) +; SSE-NEXT: [[BSWAP1:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD1]]) +; SSE-NEXT: [[BSWAP2:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD2]]) +; SSE-NEXT: [[BSWAP3:%.*]] = call i64 @llvm.bswap.i64(i64 [[LD3]]) +; SSE-NEXT: store i64 [[BSWAP0]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 0), align 4 +; SSE-NEXT: store i64 [[BSWAP1]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 1), align 4 +; SSE-NEXT: store i64 [[BSWAP2]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 2), align 4 +; SSE-NEXT: store i64 [[BSWAP3]], i64* getelementptr inbounds ([4 x i64], [4 x i64]* @dst64, i64 0, i64 3), align 4 +; SSE-NEXT: ret void +; +; AVX-LABEL: @bswap_4i64( +; AVX-NEXT: [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([4 x i64]* @src64 to <4 x i64>*), align 4 +; AVX-NEXT: [[TMP2:%.*]] = call <4 x i64> @llvm.bswap.v4i64(<4 x i64> [[TMP1]]) +; AVX-NEXT: store <4 x i64> [[TMP2]], <4 x i64>* bitcast ([4 x i64]* @dst64 to <4 x i64>*), align 4 +; AVX-NEXT: ret void ; %ld0 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 0), align 4 %ld1 = load i64, i64* getelementptr inbounds ([4 x i64], [4 x i64]* @src64, i64 0, i64 1), align 4 @@ -68,18 +80,9 @@ define void @bswap_4i32() #0 { ; CHECK-LABEL: @bswap_4i32( -; CHECK-NEXT: [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4 -; CHECK-NEXT: [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 4 -; CHECK-NEXT: [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 4 -; CHECK-NEXT: [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 4 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD1]]) -; CHECK-NEXT: [[BSWAP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD2]]) -; CHECK-NEXT: [[BSWAP3:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD3]]) -; CHECK-NEXT: store i32 [[BSWAP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 4 -; CHECK-NEXT: store i32 [[BSWAP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 4 -; CHECK-NEXT: store i32 [[BSWAP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 4 -; CHECK-NEXT: store i32 [[BSWAP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 4 +; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]]) +; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 4 ; CHECK-NEXT: ret void ; %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 4 @@ -98,32 +101,20 @@ } define void @bswap_8i32() #0 { -; CHECK-LABEL: @bswap_8i32( -; CHECK-NEXT: [[LD0:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2 -; CHECK-NEXT: [[LD1:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2 -; CHECK-NEXT: [[LD2:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 2), align 2 -; CHECK-NEXT: [[LD3:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 3), align 2 -; CHECK-NEXT: [[LD4:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4), align 2 -; CHECK-NEXT: [[LD5:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 5), align 2 -; CHECK-NEXT: [[LD6:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 6), align 2 -; CHECK-NEXT: [[LD7:%.*]] = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 7), align 2 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD1]]) -; CHECK-NEXT: [[BSWAP2:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD2]]) -; CHECK-NEXT: [[BSWAP3:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD3]]) -; CHECK-NEXT: [[BSWAP4:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD4]]) -; CHECK-NEXT: [[BSWAP5:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD5]]) -; CHECK-NEXT: [[BSWAP6:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD6]]) -; CHECK-NEXT: [[BSWAP7:%.*]] = call i32 @llvm.bswap.i32(i32 [[LD7]]) -; CHECK-NEXT: store i32 [[BSWAP0]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 0), align 2 -; CHECK-NEXT: store i32 [[BSWAP1]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 1), align 2 -; CHECK-NEXT: store i32 [[BSWAP2]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 2), align 2 -; CHECK-NEXT: store i32 [[BSWAP3]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 3), align 2 -; CHECK-NEXT: store i32 [[BSWAP4]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4), align 2 -; CHECK-NEXT: store i32 [[BSWAP5]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 5), align 2 -; CHECK-NEXT: store i32 [[BSWAP6]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 6), align 2 -; CHECK-NEXT: store i32 [[BSWAP7]], i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 7), align 2 -; CHECK-NEXT: ret void +; SSE-LABEL: @bswap_8i32( +; SSE-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([8 x i32]* @src32 to <4 x i32>*), align 2 +; SSE-NEXT: [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 4) to <4 x i32>*), align 2 +; SSE-NEXT: [[TMP3:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP1]]) +; SSE-NEXT: [[TMP4:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP2]]) +; SSE-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* bitcast ([8 x i32]* @dst32 to <4 x i32>*), align 2 +; SSE-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x i32], [8 x i32]* @dst32, i32 0, i64 4) to <4 x i32>*), align 2 +; SSE-NEXT: ret void +; +; AVX-LABEL: @bswap_8i32( +; AVX-NEXT: [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([8 x i32]* @src32 to <8 x i32>*), align 2 +; AVX-NEXT: [[TMP2:%.*]] = call <8 x i32> @llvm.bswap.v8i32(<8 x i32> [[TMP1]]) +; AVX-NEXT: store <8 x i32> [[TMP2]], <8 x i32>* bitcast ([8 x i32]* @dst32 to <8 x i32>*), align 2 +; AVX-NEXT: ret void ; %ld0 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 0), align 2 %ld1 = load i32, i32* getelementptr inbounds ([8 x i32], [8 x i32]* @src32, i32 0, i64 1), align 2 @@ -154,30 +145,9 @@ define void @bswap_8i16() #0 { ; CHECK-LABEL: @bswap_8i16( -; CHECK-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2 -; CHECK-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2 -; CHECK-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2 -; CHECK-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2 -; CHECK-NEXT: [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2 -; CHECK-NEXT: [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2 -; CHECK-NEXT: [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2 -; CHECK-NEXT: [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD1]]) -; CHECK-NEXT: [[BSWAP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD2]]) -; CHECK-NEXT: [[BSWAP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD3]]) -; CHECK-NEXT: [[BSWAP4:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD4]]) -; CHECK-NEXT: [[BSWAP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD5]]) -; CHECK-NEXT: [[BSWAP6:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD6]]) -; CHECK-NEXT: [[BSWAP7:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD7]]) -; CHECK-NEXT: store i16 [[BSWAP0]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2 -; CHECK-NEXT: store i16 [[BSWAP1]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2 -; CHECK-NEXT: store i16 [[BSWAP2]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2 -; CHECK-NEXT: store i16 [[BSWAP3]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2 -; CHECK-NEXT: store i16 [[BSWAP4]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2 -; CHECK-NEXT: store i16 [[BSWAP5]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2 -; CHECK-NEXT: store i16 [[BSWAP6]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2 -; CHECK-NEXT: store i16 [[BSWAP7]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2 +; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2 +; CHECK-NEXT: [[TMP2:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]]) +; CHECK-NEXT: store <8 x i16> [[TMP2]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2 ; CHECK-NEXT: ret void ; %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2 @@ -208,56 +178,20 @@ } define void @bswap_16i16() #0 { -; CHECK-LABEL: @bswap_16i16( -; CHECK-NEXT: [[LD0:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2 -; CHECK-NEXT: [[LD1:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2 -; CHECK-NEXT: [[LD2:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 2), align 2 -; CHECK-NEXT: [[LD3:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 3), align 2 -; CHECK-NEXT: [[LD4:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 4), align 2 -; CHECK-NEXT: [[LD5:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 5), align 2 -; CHECK-NEXT: [[LD6:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 6), align 2 -; CHECK-NEXT: [[LD7:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 7), align 2 -; CHECK-NEXT: [[LD8:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8), align 2 -; CHECK-NEXT: [[LD9:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 9), align 2 -; CHECK-NEXT: [[LD10:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 10), align 2 -; CHECK-NEXT: [[LD11:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 11), align 2 -; CHECK-NEXT: [[LD12:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 12), align 2 -; CHECK-NEXT: [[LD13:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 13), align 2 -; CHECK-NEXT: [[LD14:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 14), align 2 -; CHECK-NEXT: [[LD15:%.*]] = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 15), align 2 -; CHECK-NEXT: [[BSWAP0:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD0]]) -; CHECK-NEXT: [[BSWAP1:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD1]]) -; CHECK-NEXT: [[BSWAP2:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD2]]) -; CHECK-NEXT: [[BSWAP3:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD3]]) -; CHECK-NEXT: [[BSWAP4:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD4]]) -; CHECK-NEXT: [[BSWAP5:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD5]]) -; CHECK-NEXT: [[BSWAP6:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD6]]) -; CHECK-NEXT: [[BSWAP7:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD7]]) -; CHECK-NEXT: [[BSWAP8:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD8]]) -; CHECK-NEXT: [[BSWAP9:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD9]]) -; CHECK-NEXT: [[BSWAP10:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD10]]) -; CHECK-NEXT: [[BSWAP11:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD11]]) -; CHECK-NEXT: [[BSWAP12:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD12]]) -; CHECK-NEXT: [[BSWAP13:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD13]]) -; CHECK-NEXT: [[BSWAP14:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD14]]) -; CHECK-NEXT: [[BSWAP15:%.*]] = call i16 @llvm.bswap.i16(i16 [[LD15]]) -; CHECK-NEXT: store i16 [[BSWAP0]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 0), align 2 -; CHECK-NEXT: store i16 [[BSWAP1]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 1), align 2 -; CHECK-NEXT: store i16 [[BSWAP2]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 2), align 2 -; CHECK-NEXT: store i16 [[BSWAP3]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 3), align 2 -; CHECK-NEXT: store i16 [[BSWAP4]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 4), align 2 -; CHECK-NEXT: store i16 [[BSWAP5]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 5), align 2 -; CHECK-NEXT: store i16 [[BSWAP6]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 6), align 2 -; CHECK-NEXT: store i16 [[BSWAP7]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 7), align 2 -; CHECK-NEXT: store i16 [[BSWAP8]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8), align 2 -; CHECK-NEXT: store i16 [[BSWAP9]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 9), align 2 -; CHECK-NEXT: store i16 [[BSWAP10]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 10), align 2 -; CHECK-NEXT: store i16 [[BSWAP11]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 11), align 2 -; CHECK-NEXT: store i16 [[BSWAP12]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 12), align 2 -; CHECK-NEXT: store i16 [[BSWAP13]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 13), align 2 -; CHECK-NEXT: store i16 [[BSWAP14]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 14), align 2 -; CHECK-NEXT: store i16 [[BSWAP15]], i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 15), align 2 -; CHECK-NEXT: ret void +; SSE-LABEL: @bswap_16i16( +; SSE-NEXT: [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([16 x i16]* @src16 to <8 x i16>*), align 2 +; SSE-NEXT: [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 8) to <8 x i16>*), align 2 +; SSE-NEXT: [[TMP3:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP1]]) +; SSE-NEXT: [[TMP4:%.*]] = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> [[TMP2]]) +; SSE-NEXT: store <8 x i16> [[TMP3]], <8 x i16>* bitcast ([16 x i16]* @dst16 to <8 x i16>*), align 2 +; SSE-NEXT: store <8 x i16> [[TMP4]], <8 x i16>* bitcast (i16* getelementptr inbounds ([16 x i16], [16 x i16]* @dst16, i16 0, i64 8) to <8 x i16>*), align 2 +; SSE-NEXT: ret void +; +; AVX-LABEL: @bswap_16i16( +; AVX-NEXT: [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([16 x i16]* @src16 to <16 x i16>*), align 2 +; AVX-NEXT: [[TMP2:%.*]] = call <16 x i16> @llvm.bswap.v16i16(<16 x i16> [[TMP1]]) +; AVX-NEXT: store <16 x i16> [[TMP2]], <16 x i16>* bitcast ([16 x i16]* @dst16 to <16 x i16>*), align 2 +; AVX-NEXT: ret void ; %ld0 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 0), align 2 %ld1 = load i16, i16* getelementptr inbounds ([16 x i16], [16 x i16]* @src16, i16 0, i64 1), align 2