Index: include/llvm/CodeGen/Passes.h =================================================================== --- include/llvm/CodeGen/Passes.h +++ include/llvm/CodeGen/Passes.h @@ -379,6 +379,11 @@ /// FunctionPass *createInterleavedAccessPass(); + /// InterleavedLoadCombines Pass - This pass identifies interleaved loads and + /// combines them into wide loads detectable by InterleavedAccessPass + /// + FunctionPass *createInterleavedLoadCombinePass(); + /// LowerEmuTLS - This pass generates __emutls_[vt].xyz variables for all /// TLS variables for the emulated TLS model. /// Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -182,6 +182,7 @@ void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstructionSelectPass(PassRegistry&); void initializeInterleavedAccessPass(PassRegistry&); +void initializeInterleavedLoadCombinePass(PassRegistry &); void initializeInternalizeLegacyPassPass(PassRegistry&); void initializeIntervalPartitionPass(PassRegistry&); void initializeJumpThreadingPass(PassRegistry&); Index: lib/CodeGen/CMakeLists.txt =================================================================== --- lib/CodeGen/CMakeLists.txt +++ lib/CodeGen/CMakeLists.txt @@ -39,6 +39,7 @@ InlineSpiller.cpp InterferenceCache.cpp InterleavedAccessPass.cpp + InterleavedLoadCombinePass.cpp IntrinsicLowering.cpp LatencyPriorityQueue.cpp LazyMachineBlockFrequencyInfo.cpp Index: lib/CodeGen/CodeGen.cpp =================================================================== --- lib/CodeGen/CodeGen.cpp +++ lib/CodeGen/CodeGen.cpp @@ -42,6 +42,7 @@ initializeIfConverterPass(Registry); initializeImplicitNullChecksPass(Registry); initializeIndirectBrExpandPassPass(Registry); + initializeInterleavedLoadCombinePass(Registry); initializeInterleavedAccessPass(Registry); initializeLiveDebugValuesPass(Registry); initializeLiveDebugVariablesPass(Registry); Index: lib/CodeGen/InterleavedLoadCombinePass.cpp =================================================================== --- /dev/null +++ lib/CodeGen/InterleavedLoadCombinePass.cpp @@ -0,0 +1,1366 @@ +//===- InterleavedLoadCombine.cpp - Combine Interleaved Loads ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// \file +// +// This file defines the interleaved-load-combine pass. The pass searches for +// ShuffleVectorInstruction that execute interleaving loads. If a matching +// pattern is found, it adds a combined load and further instructions in a +// pattern that is detectable by InterleavedAccesPass. The old instructions are +// left dead to be removed later. The pass is specifically designed to be +// executed just before InterleavedAccesPass to find any left-over instances +// that are not detected within former passes. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/MemoryLocation.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" +#include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Pass.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/IPO/PassManagerBuilder.h" + +#include +#include +#include + +using namespace llvm; + +#define DEBUG_TYPE "interleaved-load-combine" + +namespace { + +/// Statistic counter +STATISTIC(NumInterleavedLoadCombine, "Number of combined loads"); + +/// Option to disable the pass +static cl::opt DisableInterleavedLoadCombine( + "disable-" DEBUG_TYPE, cl::init(false), cl::Hidden, + cl::desc("Disable combining of interleaved loads")); + +struct VectorInfo; + +struct InterleavedLoadCombineImpl { +public: + InterleavedLoadCombineImpl(Function &F, DominatorTree &DT, MemorySSA &MSSA, + TargetMachine &TM) + : F(F), DT(DT), MSSA(MSSA), + TLI(*TM.getSubtargetImpl(F)->getTargetLowering()), + TTI(TM.getTargetTransformInfo(F)) {} + + /// Scan the function for interleaved load candidates and execute the + /// replacement if applicable. + bool run(); + +private: + /// Function this pass is working on + Function &F; + + /// Dominator Tree Analysis + DominatorTree &DT; + + /// Memory Alias Analyses + MemorySSA &MSSA; + + /// Target Lowering Information + const TargetLowering &TLI; + + /// Target Transform Information + const TargetTransformInfo TTI; + + /// Find the instruction in sets LIs that dominates all others, return nullptr + /// if there is none. + LoadInst *findFirstLoad(const std::set &LIs); + + /// Replace interleaved load candidates. It does additional + /// analyses if this makes sense. Returns true on success and false + /// of nothing has been changed. + bool combine(std::list &InterleavedLoad, + OptimizationRemarkEmitter &ORE); + + /// Given a set of VectorInfo containing candidates for a given interleave + /// factor, find a set that represents a 'factor' interleaved load. + bool findPattern(std::list &Candidates, + std::list &InterleavedLoad, unsigned Factor, + const DataLayout &DL); +}; // InterleavedLoadCombine + +/// First Order Polynomial on an n-Bit Integer Value +/// +/// Polynomial(Value) = Value * B + A + E*2^(n-e) +/// +/// A and B are the coefficients. E*2^(n-e) is an error within 'e' most +/// significant bits. It is introduced if an exact computation cannot be proven +/// (e.q. division by 2). +/// +/// As part of this optimization multiple loads will be combined. It necessary +/// to prove that loads are within some relative offset to each other. This +/// class is used to prove relative offsets of values loaded from memory. +/// +/// Representing an integer in this form is sound since addition in two's +/// complement is associative (trivial) and multiplication distributes over the +/// addition (see Proof(1) in Polynomial::mul). Further, both operations +/// commute. +// +// Example: +// declare @fn(i64 %IDX, <4 x float>* %PTR) { +// %Pa1 = add i64 %IDX, 2 +// %Pa2 = lshr i64 %Pa1, 1 +// %Pa3 = getelementptr inbounds <4 x float>, <4 x float>* %PTR, i64 %Pa2 +// %Va = load <4 x float>, <4 x float>* %Pa3 +// +// %Pb1 = add i64 %IDX, 4 +// %Pb2 = lshr i64 %Pb1, 1 +// %Pb3 = getelementptr inbounds <4 x float>, <4 x float>* %PTR, i64 %Pb2 +// %Vb = load <4 x float>, <4 x float>* %Pb3 +// ... } +// +// The goal is to prove that two loads load consecutive addresses. +// +// In this case the polynomials are constructed by the following +// steps. +// +// The number tag #e specifies the error bits. +// +// Pa_0 = %IDX #0 +// Pa_1 = %IDX + 2 #0 | add 2 +// Pa_2 = %IDX/2 + 1 #1 | lshr 1 +// Pa_3 = %IDX/2 + 1 #1 | GEP, step signext to i64 +// Pa_4 = (%IDX/2)*16 + 16 #0 | GEP, multiply index by sizeof(4) for floats +// Pa_5 = (%IDX/2)*16 + 16 #0 | GEP, add offset of leading components +// +// Pb_0 = %IDX #0 +// Pb_1 = %IDX + 4 #0 | add 2 +// Pb_2 = %IDX/2 + 2 #1 | lshr 1 +// Pb_3 = %IDX/2 + 2 #1 | GEP, step signext to i64 +// Pb_4 = (%IDX/2)*16 + 32 #0 | GEP, multiply index by sizeof(4) for floats +// Pb_5 = (%IDX/2)*16 + 16 #0 | GEP, add offset of leading components +// +// Pb_5 - Pa_5 = 16 #0 | subtract to get the offset +// +// Remark: %PTR is not maintained within this class. So in this instance the +// offset of 16 can only be assumed if the pointers are equal. +// +class Polynomial { + /// Operations on B + enum BOps { + LShr, + Mul, + SExt, + Trunc, + }; + + /// Number of Error Bits e + unsigned ErrorMSBs; + + /// Value + Value *V; + + /// Coefficient B + SmallVector, 4> B; + + /// Coefficient A + APInt A; + +public: + Polynomial(Value *V) : ErrorMSBs((unsigned)-1), V(V), B(), A() { + IntegerType *Ty = dyn_cast(V->getType()); + if (Ty) { + ErrorMSBs = 0; + this->V = V; + A = APInt(Ty->getBitWidth(), 0); + } + } + + Polynomial(const APInt &A, unsigned ErrorMSBs = 0) + : ErrorMSBs(ErrorMSBs), V(NULL), B(), A(A) {} + + Polynomial(unsigned BitWidth, uint64_t A, unsigned ErrorMSBs = 0) + : ErrorMSBs(ErrorMSBs), V(NULL), B(), A(BitWidth, A) {} + + Polynomial() : ErrorMSBs((unsigned)-1), V(NULL), B(), A() {} + + /// Increment and clamp the number of undefined bits. + void incErrorMSBs(unsigned amt) { + if (ErrorMSBs == (unsigned)-1) + return; + + ErrorMSBs += amt; + if (ErrorMSBs > A.getBitWidth()) + ErrorMSBs = A.getBitWidth(); + } + + /// Decrement and clamp the number of undefined bits. + void decErrorMSBs(unsigned amt) { + if (ErrorMSBs == (unsigned)-1) + return; + + if (ErrorMSBs > amt) + ErrorMSBs -= amt; + else + ErrorMSBs = 0; + } + + /// Apply an add on the polynomial + Polynomial &add(const APInt &C) { + // Note: Addition is associative in two's complement even when in case of + // signed overflow. + // + // Error bits can only propagate into higher significant bits. As these are + // already regarded as undefined, there is no change. + // + // Theorem: Adding a constant to a polynomial does not change the error + // term. + // + // Proof: + // + // Since the addition is associative and commutes: + // + // (B + A + E*2^(n-e)) + C = B + (A + C) + E*2^(n-e) + // [qed] + + if (C.getBitWidth() != A.getBitWidth()) { + ErrorMSBs = (unsigned)-1; + return *this; + } + + A += C; + return *this; + } + + /// Apply a multiplication onto the polynomial. + Polynomial &mul(const APInt &C) { + // Note: Multiplication distributes over the addition + // + // Theorem: Multiplication distributes over the addition + // + // Proof(1): + // + // (B+A)*C =- + // = (B + A) + (B + A) + .. {C Times} + // addition is associative and commutes, hence + // = B + B + .. {C Times} .. + A + A + .. {C times} + // = B*C + A*C + // (see (function add) for signed values and overflows) + // [qed] + // + // Theorem: If C has c trailing zeros, errors bits in A or B are shifted out + // to the left. + // + // Proof(2): + // + // Let B' and A' be the n-Bit inputs with some unknown errors EA, + // EB at e leading bits. B' and A' can be written down as: + // + // B' = B + 2^(n-e)*EB + // A' = A + 2^(n-e)*EA + // + // Let C' be an input with c trailing zero bits. C' can be written as + // + // C' = C*2^c + // + // Therefore we can compute the result by using distributivity and + // commutativity. + // + // (B'*C' + A'*C') = [B + 2^(n-e)*EB] * C' + [A + 2^(n-e)*EA] * C' = + // = [B + 2^(n-e)*EB + A + 2^(n-e)*EA] * C' = + // = (B'+A') * C' = + // = [B + 2^(n-e)*EB + A + 2^(n-e)*EA] * C' = + // = [B + A + 2^(n-e)*EB + 2^(n-e)*EA] * C' = + // = (B + A) * C' + [2^(n-e)*EB + 2^(n-e)*EA)] * C' = + // = (B + A) * C' + [2^(n-e)*EB + 2^(n-e)*EA)] * C*2^c = + // = (B + A) * C' + C*(EB + EA)*2^(n-e)*2^c = + // + // Let EC be the final error with EC = C*(EB + EA) + // + // = (B + A)*C' + EC*2^(n-e)*2^c = + // = (B + A)*C' + EC*2^(n-(e-c)) + // + // Since EC is multiplied by 2^(n-(e-c)) the resulting error contains c + // less error bits than the input. c bits are shifted out to the left. + // [qed] + + if (C.getBitWidth() != A.getBitWidth()) { + ErrorMSBs = (unsigned)-1; + return *this; + } + + // Multiplying by one is a no-op. + if (C.isOneValue()) { + return *this; + } + + // Multiplying by zero removes the coefficient B and defines all bits. + if (C.isNullValue()) { + ErrorMSBs = 0; + deleteB(); + } + + // See Proof(2): Trailing zero bits indicate a left shift. This removes + // leading bits from the result even if they are undefined. + decErrorMSBs(C.countTrailingZeros()); + + A *= C; + pushBOperation(Mul, C); + return *this; + } + + /// Apply a logical shift right on the polynomial + Polynomial &lshr(const APInt &C) { + // Theorem(1): (B + A + E*2^(n-e)) >> 1 => (B >> 1) + (A >> 1) + E'*2^(n-e') + // where + // e' = e + 1, + // E is a e-bit number, + // E' is a e'-bit number, + // holds under the following precondition: + // pre(1): A % 2 = 0 + // pre(2): e < n, (see Theorem(2) for the trivial case with e=n) + // where >> expresses a logical shift to the right, with adding zeros. + // + // We need to show that for every, E there is a E' + // + // B = b_h * 2^(n-1) + b_m * 2 + b_l + // A = a_h * 2^(n-1) + a_m * 2 (pre(1)) + // + // where a_h, b_h, b_l are single bits, and a_m, b_m are (n-2) bit numbers + // + // Let X = (B + A + E*2^(n-e)) >> 1 + // Let Y = (B >> 1) + (A >> 1) + E*2^(n-e) >> 1 + // + // X = [B + A + E*2^(n-e)] >> 1 = + // = [ b_h * 2^(n-1) + b_m * 2 + b_l + + // + a_h * 2^(n-1) + a_m * 2 + + // + E * 2^(n-e) ] >> 1 = + // + // The sum is built by putting the overflow of [a_m + b+n] into the term + // 2^(n-1). As there are no more bits beyond 2^(n-1) the overflow within + // this bit is discarded. This is expressed by % 2. + // + // The bit in position 0 cannot overflow into the term (b_m + a_m). + // + // = [ ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-1) + + // + ((b_m + a_m) % 2^(n-2)) * 2 + + // + b_l + E * 2^(n-e) ] >> 1 = + // + // The shift is computed by dividing the terms by 2 and by cutting off + // b_l. + // + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + E * 2^(n-(e+1)) = + // + // by the definition in the Theorem e+1 = e' + // + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + E * 2^(n-e') = + // + // Compute Y by applying distributivity first + // + // Y = (B >> 1) + (A >> 1) + E*2^(n-e') = + // = (b_h * 2^(n-1) + b_m * 2 + b_l) >> 1 + + // + (a_h * 2^(n-1) + a_m * 2) >> 1 + + // + E * 2^(n-e) >> 1 = + // + // Again, the shift is computed by dividing the terms by 2 and by cutting + // off b_l. + // + // = b_h * 2^(n-2) + b_m + + // + a_h * 2^(n-2) + a_m + + // + E * 2^(n-(e+1)) = + // + // Again, the sum is built by putting the overflow of [a_m + b+n] into + // the term 2^(n-1). But this time there is room for a second bit in the + // term 2^(n-2) we add this bit to a new term and denote it o_h in a + // second step. + // + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] >> 1) * 2^(n-1) + + // + ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + E * 2^(n-(e+1)) = + // + // Let o_h = [b_h + a_h + (b_m + a_m) >> (n-2)] >> 1 + // Further replace e+1 by e'. + // + // = o_h * 2^(n-1) + + // + ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + E * 2^(n-e') = + // + // Move o_h into the error term and construct E'. To ensure that there is + // no 2^x with negative x, this step requires pre(2) (e < n). + // + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + o_h * 2^(e'-1) * 2^(n-e') + | pre(2), move 2^(e'-1) + // | out of the old exponent + // + E * 2^(n-e') = + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + [o_h * 2^(e'-1) + E] * 2^(n-e') + | move 2^(e'-1) out of + // | the old exponent + // + // Let E' = o_h * 2^(e'-1) + E + // + // = ([b_h + a_h + (b_m + a_m) >> (n-2)] % 2) * 2^(n-2) + + // + ((b_m + a_m) % 2^(n-2)) + + // + E' * 2^(n-e') + // + // Because X and Y are distinct only in there error terms and E' can be + // constructed as shown the theorem holds. + // [qed] + // + // For completeness in case of the case e=n it is also required to show that + // distributivity can be applied. + // + // In this case Theorem(1) transforms to (the pre-condition on A can also be + // dropped) + // + // Theorem(2): (B + A + E) >> 1 => (B >> 1) + (A >> 1) + E' + // where + // A, B, E, E' are two's complement numbers with the same bit + // width + // + // Let A + B + E = X + // Let (B >> 1) + (A >> 1) = Y + // + // Therefore we need to show that for every X and Y there is an E' which + // makes the equation + // + // X = Y + E' + // + // hold. This is trivially the case for E' = X - Y. + // + // [qed] + // + // Remark: Distributing lshr with and arbitrary number n can be expressed as + // ((((B + A) lshr 1) lshr 1) ... ) {n times}. + // This construction induces n additional error bits at the left. + + if (C.getBitWidth() != A.getBitWidth()) { + ErrorMSBs = (unsigned)-1; + return *this; + } + + if (C.isNullValue()) + return *this; + + // Test if the result will be zero + unsigned shiftAmt = C.getZExtValue(); + if (shiftAmt >= C.getBitWidth()) + return mul(APInt(C.getBitWidth(), 0)); + + // The proof that shiftAmt LSBs are zero for at least one summand is only + // possible for the constant number. + // + // If this can be proven add shiftAmt to the error counter + // `ErrorMSBs`. Otherwise set all bits as undefined. + if (A.countTrailingZeros() < shiftAmt) + ErrorMSBs = A.getBitWidth(); + else + incErrorMSBs(shiftAmt); + + // Apply the operation. + pushBOperation(LShr, C); + A = A.lshr(shiftAmt); + + return *this; + } + + /// Apply a sign-extend or truncate operation on the polynomial. + Polynomial &sextOrTrunc(unsigned n) { + if (n < A.getBitWidth()) { + // Truncate: Clearly undefined Bits on the MSB side are removed + // if there are any. + decErrorMSBs(A.getBitWidth() - n); + A = A.trunc(n); + pushBOperation(Trunc, APInt(sizeof(n) * 8, n)); + } + if (n > A.getBitWidth()) { + // Extend: Clearly extending first and adding later is different + // to adding first and extending later in all extended bits. + incErrorMSBs(n - A.getBitWidth()); + A = A.sext(n); + pushBOperation(SExt, APInt(sizeof(n) * 8, n)); + } + + return *this; + } + + /// Test if there is a coefficient B. + bool isFirstOrder() const { return V != nullptr; } + + /// Test coefficient B of two Polynomials are equal. + bool isCompatibleTo(const Polynomial &o) const { + // The polynomial use different bit width. + if (A.getBitWidth() != o.A.getBitWidth()) + return false; + + // If neither Polynomial has the Coefficient B. + if (!isFirstOrder() && !o.isFirstOrder()) + return true; + + // The index variable is different. + if (V != o.V) + return false; + + // Check the operations. + if (B.size() != o.B.size()) + return false; + + auto ob = o.B.begin(); + for (auto &b : B) { + if (b != *ob) + return false; + ob++; + } + + return true; + } + + /// Subtract two polynomials, return an undefined polynomial if + /// subtraction is not possible. + Polynomial operator-(const Polynomial &o) const { + // Return an undefined polynomial if incompatible. + if (!isCompatibleTo(o)) + return Polynomial(); + + // If the polynomials are compatible (meaning they have the same + // coefficient on B), B is eliminated. Thus a polynomial solely + // containing A is returned + return Polynomial(A - o.A, std::max(ErrorMSBs, o.ErrorMSBs)); + } + + /// Subtract a constant from a polynomial, + Polynomial operator-(uint64_t C) const { + Polynomial Result(*this); + Result.A -= C; + return Result; + } + + /// Add a constant to a polynomial, + Polynomial operator+(uint64_t C) const { + Polynomial Result(*this); + Result.A += C; + return Result; + } + + /// Returns true if it can be proven that two Polynomials are equal. + bool isProvenEqualTo(const Polynomial &o) { + // Subtract both polynomials and test if it is fully defined and zero. + Polynomial r = *this - o; + return (r.ErrorMSBs == 0) && (!r.isFirstOrder()) && (r.A.isNullValue()); + } + + /// Print the polynomial into a stream. + void print(raw_ostream &OS) const { + OS << "[{#ErrBits:" << ErrorMSBs << "} "; + + if (V) { + for (auto b : B) + OS << "("; + OS << "(" << *V << ") "; + + for (auto b : B) { + switch (b.first) { + case LShr: + OS << "LShr "; + break; + case Mul: + OS << "Mul "; + break; + case SExt: + OS << "SExt "; + break; + case Trunc: + OS << "Trunc "; + break; + } + + OS << b.second << ") "; + } + } + + OS << "+ " << A << "]"; + } + +private: + void deleteB() { + V = nullptr; + B.clear(); + } + + void pushBOperation(const BOps Op, const APInt &C) { + if (isFirstOrder()) { + B.push_back(std::make_pair(Op, C)); + return; + } + } +}; + +static raw_ostream &operator<<(raw_ostream &OS, const Polynomial &P) { + P.print(OS); + return OS; +} + +/// VectorInfo stores abstract the following information for each vector +/// element: +/// +/// 1) The the memory address loaded into the element as Polynomial +/// 2) a set of load instruction necessary to construct the vector, +/// 3) a set of all other instructions that are necessary to create the vector and +/// 4) a pointer value that can be used as relative base for all elements. +struct VectorInfo { +private: + VectorInfo(const VectorInfo &c) : VTy(c.VTy) { + llvm_unreachable( + "Copying VectorInfo is neither implemented nor necessary,"); + } + +public: + /// Information of a Vector Element + struct ElementInfo { + /// Offset Polynomial. + Polynomial Ofs; + + /// The Load Instruction used to Load the entry. LI is null if the pointer + /// of the load instruction does not point on to the entry + LoadInst *LI; + + ElementInfo(Polynomial Offset = Polynomial(), LoadInst *LI = nullptr) + : Ofs(Offset), LI(LI) {} + }; + + /// Basic-block the load instructions are within + BasicBlock *BB; + + /// Pointer value of all participation load instructions + Value *PV; + + /// Participating load instructions + std::set LIs; + + /// Participating instructions + std::set Is; + + /// Final shuffle-vector instruction + ShuffleVectorInst *SVI; + + /// Information of the offset for each vector element + ElementInfo *EI; + + /// Vector Type + VectorType *const VTy; + + VectorInfo(VectorType *VTy) + : BB(nullptr), PV(nullptr), LIs(), Is(), SVI(nullptr), VTy(VTy) { + EI = new ElementInfo[VTy->getNumElements()]; + } + + virtual ~VectorInfo() { delete[] EI; } + + unsigned getDimension() const { return VTy->getNumElements(); } + + /// Test if the VectorInfo can be part of an interleaved load with the + /// specified factor. + /// + /// \param Factor of the interleave + /// \param DL Targets Datalayout + /// + /// \returns true if this is possible and false if not + bool isInterleaved(unsigned Factor, const DataLayout &DL) const { + unsigned Size = DL.getTypeAllocSize(VTy->getElementType()); + for (unsigned i = 1; i < getDimension(); i++) { + if (!EI[i].Ofs.isProvenEqualTo(EI[0].Ofs + i * Factor * Size)) { + return false; + } + } + return true; + } + + /// Recursively computes the vector information stored in V. + /// + /// This function delegates the work to specialized implementations + /// + /// \param V Value to operate on + /// \param Result Result of the computation + /// + /// \returns false if no sensible information can be gathered. + static bool compute(Value *V, VectorInfo &Result, const DataLayout &DL) { + ShuffleVectorInst *SVI = dyn_cast(V); + if (SVI) + return computeFromSVI(SVI, Result, DL); + LoadInst *LI = dyn_cast(V); + if (LI) + return computeFromLI(LI, Result, DL); + BitCastInst *BCI = dyn_cast(V); + if (BCI) + return computeFromBCI(BCI, Result, DL); + return false; + } + + /// BitCastInst specialization to compute the vector information. + /// + /// \param BCI BitCastInst to operate on + /// \param Result Result of the computation + /// + /// \returns false if no sensible information can be gathered. + static bool computeFromBCI(BitCastInst *BCI, VectorInfo &Result, + const DataLayout &DL) { + Instruction *Op = dyn_cast(BCI->getOperand(0)); + + if (!Op) + return false; + + VectorType *VTy = dyn_cast(Op->getType()); + if (!VTy) + return false; + + // We can only cast from large to smaller vectors + if (Result.VTy->getNumElements() % VTy->getNumElements()) + return false; + + unsigned Factor = Result.VTy->getNumElements() / VTy->getNumElements(); + unsigned NewSize = DL.getTypeAllocSize(Result.VTy->getElementType()); + unsigned OldSize = DL.getTypeAllocSize(VTy->getElementType()); + + if (NewSize * Factor != OldSize) + return false; + + VectorInfo Old(VTy); + if (!compute(Op, Old, DL)) + return false; + + for (unsigned i = 0; i < Result.VTy->getNumElements(); i += Factor) { + for (unsigned j = 0; j < Factor; j++) { + Result.EI[i + j] = + ElementInfo(Old.EI[i / Factor].Ofs + j * NewSize, + j == 0 ? Old.EI[i / Factor].LI : nullptr); + } + } + + Result.BB = Old.BB; + Result.PV = Old.PV; + Result.LIs.insert(Old.LIs.begin(), Old.LIs.end()); + Result.Is.insert(Old.Is.begin(), Old.Is.end()); + Result.Is.insert(BCI); + Result.SVI = nullptr; + + return true; + } + + /// ShuffleVectorInst specialization to compute vector information. + /// + /// \param SVI ShuffleVectorInst to operate on + /// \param Result Result of the computation + /// + /// Compute the left and the right side vector information and merge them by + /// applying the shuffle operation. This function also ensures that the left + /// and right side have compatible loads. This means that all loads are with + /// in the same basic block and are based on the same pointer. + /// + /// \returns false if no sensible information can be gathered. + static bool computeFromSVI(ShuffleVectorInst *SVI, VectorInfo &Result, + const DataLayout &DL) { + VectorType *ArgTy = dyn_cast(SVI->getOperand(0)->getType()); + assert(ArgTy && "ShuffleVector Operand is not a VectorType"); + + // Compute the left hand vector information. + VectorInfo LHS(ArgTy); + if (!compute(SVI->getOperand(0), LHS, DL)) + LHS.BB = nullptr; + + // Compute the right hand vector information. + VectorInfo RHS(ArgTy); + if (!compute(SVI->getOperand(1), RHS, DL)) + RHS.BB = nullptr; + + // Neither operand produced sensible results? + if (!LHS.BB && !RHS.BB) + return false; + // Only RHS produced sensible results? + else if (!LHS.BB) { + Result.BB = RHS.BB; + Result.PV = RHS.PV; + } + // Only LHS produced sensible results? + else if (!RHS.BB) { + Result.BB = LHS.BB; + Result.PV = LHS.PV; + } + // Both operands produced sensible results? + else if ((LHS.BB == RHS.BB) && (LHS.PV == LHS.PV)) { + Result.BB = LHS.BB; + Result.PV = LHS.PV; + } + // Both operands produced sensible results but they are incompatible. + else { + return false; + } + + // Merge and apply the operation on the offset information. + if (LHS.BB) { + Result.LIs.insert(LHS.LIs.begin(), LHS.LIs.end()); + Result.Is.insert(LHS.Is.begin(), LHS.Is.end()); + } + if (RHS.BB) { + Result.LIs.insert(RHS.LIs.begin(), RHS.LIs.end()); + Result.Is.insert(RHS.Is.begin(), RHS.Is.end()); + } + Result.Is.insert(SVI); + Result.SVI = SVI; + + int j = 0; + for (int i : SVI->getShuffleMask()) { + assert((i < 2 * (signed)ArgTy->getNumElements()) && + "Invalid ShuffleVectorInst (index out of bounds)"); + + if (i < 0) + Result.EI[j] = ElementInfo(); + else if (i < (signed)ArgTy->getNumElements()) { + if (LHS.BB) + Result.EI[j] = LHS.EI[i]; + else + Result.EI[j] = ElementInfo(); + } else { + if (RHS.BB) + Result.EI[j] = RHS.EI[i - ArgTy->getNumElements()]; + else + Result.EI[j] = ElementInfo(); + } + j++; + } + + return true; + } + + /// LoadInst specialization to compute vector information. + /// + /// This function also acts as abort condition to the recursion. + /// + /// \param LI LoadInst to operate on + /// \param Result Result of the computation + /// + /// \returns false if no sensible information can be gathered. + static bool computeFromLI(LoadInst *LI, VectorInfo &Result, + const DataLayout &DL) { + Value *BasePtr; + Polynomial Offset; + + if (LI->isVolatile()) + return false; + + if (LI->isAtomic()) + return false; + + // Get the base polynomial + computePolynomialFromPointer(*LI->getPointerOperand(), Offset, BasePtr, DL); + + Result.BB = LI->getParent(); + Result.PV = BasePtr; + Result.LIs.insert(LI); + Result.Is.insert(LI); + + for (unsigned i = 0; i < Result.getDimension(); i++) { + Value *Idx[2] = { + ConstantInt::get(Type::getInt32Ty(LI->getContext()), 0), + ConstantInt::get(Type::getInt32Ty(LI->getContext()), i), + }; + int64_t Ofs = DL.getIndexedOffsetInType(Result.VTy, makeArrayRef(Idx, 2)); + Result.EI[i] = ElementInfo(Offset + Ofs, i == 0 ? LI : nullptr); + } + + return true; + } + + /// Recursively compute polynomial of a value. + /// + /// \param BO Input binary operation + /// \param Result Result polynomial + static void computePolynomialBinOp(BinaryOperator &BO, Polynomial &Result) { + Value *LHS = BO.getOperand(0); + Value *RHS = BO.getOperand(1); + + // Find the RHS Constant if any + ConstantInt *C = dyn_cast(RHS); + if ((!C) && BO.isCommutative()) { + C = dyn_cast(LHS); + if (C) + std::swap(LHS, RHS); + } + + switch (BO.getOpcode()) { + case Instruction::Add: + if (!C) + break; + + computePolynomial(*LHS, Result); + Result.add(C->getValue()); + return; + + case Instruction::LShr: + if (!C) + break; + + computePolynomial(*LHS, Result); + Result.lshr(C->getValue()); + return; + + default: + break; + } + + Result = Polynomial(&BO); + } + + /// Recursively compute polynomial of a value + /// + /// \param V input value + /// \param Result result polynomial + static void computePolynomial(Value &V, Polynomial &Result) { + if (isa(&V)) + computePolynomialBinOp(*dyn_cast(&V), Result); + else + Result = Polynomial(&V); + } + + /// Compute the Polynomial representation of a Pointer type. + /// + /// \param Ptr input pointer value + /// \param Result result polynomial + /// \param BasePtr pointer the polynomial is based on + /// \param DL Datalayout of the target machine + static void computePolynomialFromPointer(Value &Ptr, Polynomial &Result, + Value *&BasePtr, + const DataLayout &DL) { + // Not a pointer type? Return an undefined polynomial + PointerType *PtrTy = dyn_cast(Ptr.getType()); + if (!PtrTy) { + Result = Polynomial(); + BasePtr = nullptr; + } + unsigned PointerBits = + DL.getIndexSizeInBits(PtrTy->getPointerAddressSpace()); + + /// Skip pointer casts. Return Zero polynomial otherwise + if (isa(&Ptr)) { + CastInst &CI = *cast(&Ptr); + switch (CI.getOpcode()) { + case Instruction::BitCast: + computePolynomialFromPointer(*CI.getOperand(0), Result, BasePtr, DL); + break; + default: + BasePtr = &Ptr; + Polynomial(PointerBits, 0); + break; + } + } + /// Resolve GetElementPtrInst. + else if (isa(&Ptr)) { + GetElementPtrInst &GEP = *cast(&Ptr); + + APInt BaseOffset(PointerBits, 0); + + // Check if we can compute the Offset with accumulateConstantOffset + if (GEP.accumulateConstantOffset(DL, BaseOffset)) { + Result = Polynomial(BaseOffset); + BasePtr = GEP.getPointerOperand(); + return; + } else { + // Otherwise we allow that the last index operand of the GEP is + // non-constant. + unsigned idxOperand, e; + SmallVector Indices; + for (idxOperand = 1, e = GEP.getNumOperands(); idxOperand < e; + idxOperand++) { + ConstantInt *IDX = dyn_cast(GEP.getOperand(idxOperand)); + if (!IDX) + break; + Indices.push_back(IDX); + } + + // It must also be the last operand. + if (idxOperand + 1 != e) { + Result = Polynomial(); + BasePtr = nullptr; + return; + } + + // Compute the polynomial of the index operand. + computePolynomial(*GEP.getOperand(idxOperand), Result); + + // Compute base offset from zero based index, excluding the last + // variable operand. + BaseOffset = + DL.getIndexedOffsetInType(GEP.getSourceElementType(), Indices); + + // Apply the operations of GEP to the polynomial. + unsigned ResultSize = DL.getTypeAllocSize(GEP.getResultElementType()); + Result.sextOrTrunc(PointerBits); + Result.mul(APInt(PointerBits, ResultSize)); + Result.add(BaseOffset); + BasePtr = GEP.getPointerOperand(); + } + } + // All other instructions are handled by using the value as base pointer and + // a zero polynomial. + else { + BasePtr = &Ptr; + Polynomial(DL.getIndexSizeInBits(PtrTy->getPointerAddressSpace()), 0); + } + } + +#ifndef NDEBUG + void print(raw_ostream &OS) const { + if (PV) + OS << *PV; + else + OS << "(none)"; + OS << " + "; + for (unsigned i = 0; i < getDimension(); i++) + OS << ((i == 0) ? "[" : ", ") << EI[i].Ofs; + OS << "]"; + } +#endif +}; + +#ifndef NDEBUG +static raw_ostream &operator<<(raw_ostream &OS, const VectorInfo &S) { + S.print(OS); + return OS; +} +#endif +} // anonymous namespace + +bool InterleavedLoadCombineImpl::findPattern( + std::list &Candidates, std::list &InterleavedLoad, + unsigned Factor, const DataLayout &DL) { + for (auto C0 = Candidates.begin(), E0 = Candidates.end(); C0 != E0; ++C0) { + unsigned i; + // Try to find an interleaved load using the front of Worklist as first line + unsigned Size = DL.getTypeAllocSize(C0->VTy->getElementType()); + + // List containing iterators pointing to the VectorInfos of the candidates + std::vector::iterator> Res(Factor, Candidates.end()); + + for (auto C = Candidates.begin(), E = Candidates.end(); C != E; C++) { + if (C->VTy != C0->VTy) + continue; + if (C->BB != C0->BB) + continue; + if (C->PV != C0->PV) + continue; + + // Check the current value matches any of factor - 1 remaining lines + for (i = 1; i < Factor; i++) { + if (C->EI[0].Ofs.isProvenEqualTo(C0->EI[0].Ofs + i * Size)) { + Res[i] = C; + } + } + + for (i = 1; i < Factor; i++) { + if (Res[i] == Candidates.end()) + break; + } + if (i == Factor) { + Res[0] = C0; + break; + } + } + + if (Res[0] != Candidates.end()) { + // Move the result into the output + for (unsigned i = 0; i < Factor; i++) { + InterleavedLoad.splice(InterleavedLoad.end(), Candidates, Res[i]); + } + + return true; + } + } + return false; +} + +LoadInst * +InterleavedLoadCombineImpl::findFirstLoad(const std::set &LIs) { + assert(!LIs.empty() && "No load instructions given."); + + // All LIs are within the same BB. Select the first for a reference. + BasicBlock *BB = (*LIs.begin())->getParent(); + BasicBlock::iterator FLI = + std::find_if(BB->begin(), BB->end(), [&LIs](Instruction &I) -> bool { + return is_contained(LIs, &I); + }); + assert(FLI != BB->end()); + + return cast(FLI); +} + +bool InterleavedLoadCombineImpl::combine(std::list &InterleavedLoad, + OptimizationRemarkEmitter &ORE) { + LLVM_DEBUG(dbgs() << "Checking interleaved load\n"); + for (auto &VI : InterleavedLoad) + LLVM_DEBUG(dbgs() << VI << "\n"); + + // The insertion point is the LoadInst which loads the first values. The + // following tests are used to proof that the combined load can be inserted + // just before InsertionPoint. + LoadInst *InsertionPoint = InterleavedLoad.front().EI[0].LI; + + // Test if the offset is computed + if (!InsertionPoint) + return false; + + std::set LIs; + std::set Is; + std::set SVIs; + + unsigned InterleavedCost; + unsigned InstructionCost = 0; + + // Get the interleave factor + unsigned Factor = InterleavedLoad.size(); + + // Merge all input sets used in analysis + for (auto &VI : InterleavedLoad) { + // Generate a set of all load instructions to be combined + LIs.insert(VI.LIs.begin(), VI.LIs.end()); + + // Generate a set of all instructions taking part in load + // interleaved. This list excludes the instructions necessary for the + // polynomial construction. + Is.insert(VI.Is.begin(), VI.Is.end()); + + // Generate the set of the final ShuffleVectorInst. + SVIs.insert(VI.SVI); + } + + // There is nothing to combine. + if (LIs.size() < 2) + return false; + + // Test if all participating instruction will be dead after the + // transformation. If intermediate results are used, no performance gain can + // be expected. Also sum the cost of the Instructions beeing left dead. + for (auto &I : Is) { + // Compute the old cost + InstructionCost += + TTI.getInstructionCost(I, TargetTransformInfo::TCK_Latency); + + // The final SVIs are allowed not to be dead, all uses will be replaced + if (SVIs.find(I) != SVIs.end()) + continue; + + // If there are users outside the set to be eliminated, we abort the + // transformation. No gain can be expected. + for (const auto &U : I->users()) { + if (Is.find(dyn_cast(U)) == Is.end()) + return false; + } + } + + // We know that all LoadInst are within the same BB. This guarantees that + // either everything or nothing is loaded. + LoadInst *First = findFirstLoad(LIs); + + // To be safe that the loads can be combined, iterate over all loads and test + // that the corresponding defining access dominates first LI. This guarantees + // that there are no aliasing stores in between the loads. + auto FMA = MSSA.getMemoryAccess(First); + for (auto LI : LIs) { + auto MADef = MSSA.getMemoryAccess(LI)->getDefiningAccess(); + if (!MSSA.dominates(MADef, FMA)) + return false; + } + assert(!LIs.empty() && "There are no LoadInst to combine"); + + // It is necessary that insertion point dominates all final ShuffleVectorInst. + for (auto &VI : InterleavedLoad) { + if (!DT.dominates(InsertionPoint, VI.SVI)) + return false; + } + + // All checks are done. Add instructions detectable by InterleavedAccessPass + // The old instruction will are left dead. + IRBuilder<> Builder(InsertionPoint); + Type *ETy = InterleavedLoad.front().SVI->getType()->getElementType(); + unsigned ElementsPerSVI = + InterleavedLoad.front().SVI->getType()->getNumElements(); + VectorType *ILTy = VectorType::get(ETy, Factor * ElementsPerSVI); + + SmallVector Indices; + for (unsigned i = 0; i < Factor; i++) + Indices.push_back(i); + InterleavedCost = TTI.getInterleavedMemoryOpCost( + Instruction::Load, ILTy, Factor, Indices, InsertionPoint->getAlignment(), + InsertionPoint->getPointerAddressSpace()); + + if (InterleavedCost >= InstructionCost) { + return false; + } + + // Create a pointer cast for the wide load. + auto CI = Builder.CreatePointerCast(InsertionPoint->getOperand(0), + ILTy->getPointerTo(), + "interleaved.wide.ptrcast"); + + // Create the wide load and update the MemorySSA. + auto LI = Builder.CreateAlignedLoad(CI, InsertionPoint->getAlignment(), + "interleaved.wide.load"); + auto MSSAU = MemorySSAUpdater(&MSSA); + MemoryUse *MSSALoad = cast(MSSAU.createMemoryAccessBefore( + LI, nullptr, MSSA.getMemoryAccess(InsertionPoint))); + MSSAU.insertUse(MSSALoad); + + // Create the final SVIs and replace all uses. + int i = 0; + for (auto &VI : InterleavedLoad) { + SmallVector Mask; + for (unsigned j = 0; j < ElementsPerSVI; j++) + Mask.push_back(i + j * Factor); + + Builder.SetInsertPoint(VI.SVI); + auto SVI = Builder.CreateShuffleVector(LI, UndefValue::get(LI->getType()), + Mask, "interleaved.shuffle"); + VI.SVI->replaceAllUsesWith(SVI); + i++; + } + + NumInterleavedLoadCombine++; + ORE.emit([&]() { + return OptimizationRemark(DEBUG_TYPE, "Combined Interleaved Load", LI) + << "Load interleaved combined with factor " + << ore::NV("Factor", Factor); + }); + + return true; +} + +bool InterleavedLoadCombineImpl::run() { + OptimizationRemarkEmitter ORE(&F); + bool changed = false; + unsigned MaxFactor = TLI.getMaxSupportedInterleaveFactor(); + + auto &DL = F.getParent()->getDataLayout(); + + // Start with the highest factor to avoid combining and recombining. + for (unsigned Factor = MaxFactor; Factor >= 2; Factor--) { + std::list Candidates; + + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { + if (auto SVI = dyn_cast(&I)) { + + Candidates.emplace_back(SVI->getType()); + + if (!VectorInfo::computeFromSVI(SVI, Candidates.back(), DL)) { + Candidates.pop_back(); + continue; + } + + if (!Candidates.back().isInterleaved(Factor, DL)) { + Candidates.pop_back(); + } + } + } + } + + std::list InterleavedLoad; + while (findPattern(Candidates, InterleavedLoad, Factor, DL)) { + if (combine(InterleavedLoad, ORE)) { + changed = true; + } else { + // Remove the first element of the Interleaved Load but put the others + // back on the list and continue searching + Candidates.splice(Candidates.begin(), InterleavedLoad, + std::next(InterleavedLoad.begin()), + InterleavedLoad.end()); + } + InterleavedLoad.clear(); + } + } + + return changed; +} + +namespace { +/// This pass combines interleaved loads into a pattern detectable by +/// InterleavedAccessPass. +struct InterleavedLoadCombine : public FunctionPass { + static char ID; + + InterleavedLoadCombine() : FunctionPass(ID) { + initializeInterleavedLoadCombinePass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "Interleaved Load Combine Pass"; + } + + bool runOnFunction(Function &F) override { + if (DisableInterleavedLoadCombine) + return false; + + auto *TPC = getAnalysisIfAvailable(); + if (!TPC) + return false; + + LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() + << "\n"); + + return InterleavedLoadCombineImpl( + F, getAnalysis().getDomTree(), + getAnalysis().getMSSA(), + TPC->getTM()) + .run(); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + FunctionPass::getAnalysisUsage(AU); + } + +private: +}; +} // anonymous namespace + +char InterleavedLoadCombine::ID = 0; + +INITIALIZE_PASS_BEGIN( + InterleavedLoadCombine, DEBUG_TYPE, + "Combine interleaved loads into wide loads and shufflevector instructions", + false, false) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass) +INITIALIZE_PASS_END( + InterleavedLoadCombine, DEBUG_TYPE, + "Combine interleaved loads into wide loads and shufflevector instructions", + false, false) + +FunctionPass * +llvm::createInterleavedLoadCombinePass() { + auto P = new InterleavedLoadCombine(); + return P; +} Index: lib/Target/AArch64/AArch64TargetMachine.cpp =================================================================== --- lib/Target/AArch64/AArch64TargetMachine.cpp +++ lib/Target/AArch64/AArch64TargetMachine.cpp @@ -419,8 +419,10 @@ TargetPassConfig::addIRPasses(); // Match interleaved memory accesses to ldN/stN intrinsics. - if (TM->getOptLevel() != CodeGenOpt::None) + if (TM->getOptLevel() != CodeGenOpt::None) { + addPass(createInterleavedLoadCombinePass()); addPass(createInterleavedAccessPass()); + } if (TM->getOptLevel() == CodeGenOpt::Aggressive && EnableGEPOpt) { // Call SeparateConstOffsetFromGEP pass to extract constants within indices Index: test/CodeGen/AArch64/O3-pipeline.ll =================================================================== --- test/CodeGen/AArch64/O3-pipeline.ll +++ test/CodeGen/AArch64/O3-pipeline.ll @@ -49,6 +49,11 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Dominator Tree Construction +; CHECK-NEXT: Basic Alias Analysis (stateless AA impl) +; CHECK-NEXT: Function Alias Analysis Results +; CHECK-NEXT: Memory SSA +; CHECK-NEXT: Interleaved Load Combine Pass +; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Interleaved Access Pass ; CHECK-NEXT: Natural Loop Information ; CHECK-NEXT: CodeGen Prepare Index: test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll =================================================================== --- /dev/null +++ test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll @@ -0,0 +1,406 @@ +; RUN: llc < %s | FileCheck --check-prefix AS %s +; RUN: opt -S -interleaved-load-combine < %s | FileCheck %s + +; ModuleID = 'aarch64_interleaved-ld-combine.bc' +target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" +target triple = "arm64--linux-gnu" + +; This should be lowered into LD4 +define void @aarch64_ilc_const(<4 x float>* %ptr) { +entry: + +;;; Check LLVM transformation +; CHECK-LABEL: @aarch64_ilc_const( +; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 2 +; CHECK-DAG: [[CAST:%.+]] = bitcast <4 x float>* [[GEP]] to <16 x float>* +; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, <16 x float>* [[CAST]], align 16 +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK: ret void + +;;; Check if it gets lowerd +; AS-LABEL: aarch64_ilc_const +; AS: ld4 +; AS: ret + + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 2 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 3 + %gep3 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 4 + %gep4 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 5 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %ld3 = load <4 x float>, <4 x float>* %gep3, align 16 + %ld4 = load <4 x float>, <4 x float>* %gep4, align 16 + %sv1 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv2 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv3 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %sv4 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %m0_3 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m4_7 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m8_11 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + %m12_15 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + store <4 x float> %m8_11, <4 x float>* %gep3, align 16 + store <4 x float> %m12_15, <4 x float>* %gep4, align 16 + ret void +} + +; This should be lowered into LD4 +define void @aarch64_ilc_idx(<4 x float>* %ptr, i64 %idx) { +entry: + +;;; Check LLVM transformation +; CHECK-LABEL: @aarch64_ilc_idx( +; CHECK-DAG: [[ADD:%.+]] = add i64 %idx, 16 +; CHECK-DAG: [[LSHR:%.+]] = lshr i64 [[ADD]], 2 +; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 [[LSHR]] +; CHECK-DAG: [[CAST:%.+]] = bitcast <4 x float>* [[GEP]] to <16 x float>* +; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, <16 x float>* [[CAST]], align 16 +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK: ret void + +; AS-LABEL: aarch64_ilc_idx +; AS-DAG: lsl [[LSL:x[0-9]+]], x1, #2 +; AS-DAG: add [[ADD:x[0-9]+]], [[LSL]], #64 +; AS-DAG: and [[AND:x[0-9]+]], [[ADD]], #0xfffffffffffffff0 +; AS-DAG: add [[ADR:x[0-9]+]], x0, [[AND]] +; AS-DAG: ld4 { v[[V0:[0-9]+]].4s, v[[V1:[0-9]+]].4s, v[[V2:[0-9]+]].4s, v[[V3:[0-9]+]].4s }, {{\[}}[[ADR]]{{\]}} +; AS-DAG: str q[[V0]] +; AS-DAG: str q[[V1]] +; AS-DAG: str q[[V2]] +; AS-DAG: str q[[V3]] +; AS: ret + + %a2 = add i64 %idx, 20 + %idx2 = lshr i64 %a2, 2 + %a3 = add i64 %idx, 24 + %a1 = add i64 %idx, 16 + %idx1 = lshr i64 %a1, 2 + %idx3 = lshr i64 %a3, 2 + %a4 = add i64 %idx, 28 + %idx4 = lshr i64 %a4, 2 + + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx2 + %gep4 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx4 + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx1 + %gep3 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx3 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %ld3 = load <4 x float>, <4 x float>* %gep3, align 16 + %ld4 = load <4 x float>, <4 x float>* %gep4, align 16 + %sv1 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv2 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv3 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %sv4 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %m0_3 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m4_7 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m8_11 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + %m12_15 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + store <4 x float> %m8_11, <4 x float>* %gep3, align 16 + store <4 x float> %m12_15, <4 x float>* %gep4, align 16 + ret void +} + +; This should be lowered into LD4, a offset of has to be taken into account +%struct.ilc = type <{ float, [0 x <4 x float>] }> +define void @aarch64_ilc_struct(%struct.ilc* %ptr, i64 %idx) { +entry: + +;;; Check LLVM transformation +; CHECK-LABEL: @aarch64_ilc_struct( +; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2 +; CHECK-DAG: [[GEP:%.+]] = getelementptr %struct.ilc, %struct.ilc* %ptr, i32 0, i32 1, i64 [[LSHR]] +; CHECK-DAG: [[CAST:%.+]] = bitcast <4 x float>* [[GEP]] to <16 x float>* +; CHECK-DAG: [[LOAD:%.+]] = load <16 x float>, <16 x float>* [[CAST]], align 4 +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK-DAG: %{{.* }}= shufflevector <16 x float> [[LOAD]], <16 x float> undef, <4 x i32> +; CHECK: ret void + +; AS-LABEL: aarch64_ilc_struct +; AS-DAG: lsl [[LSL:x[0-9]+]], x1, #2 +; AS-DAG: add [[ADD:x[0-9]+]], x0, #4 +; AS-DAG: and [[AND:x[0-9]+]], [[LSL]], #0xfffffffffffffff0 +; AS-DAG: add [[ADR:x[0-9]+]], [[ADD]], [[AND]] +; AS-DAG: ld4 { v[[V0:[0-9]+]].4s, v[[V1:[0-9]+]].4s, v[[V2:[0-9]+]].4s, v[[V3:[0-9]+]].4s }, {{\[}}[[ADR]]{{\]}} +; AS-DAG: str q[[V0]] +; AS-DAG: str q[[V1]] +; AS-DAG: str q[[V2]] +; AS-DAG: str q[[V3]] +; AS: ret + + %a1 = add i64 %idx, 4 + %idx2 = lshr i64 %a1, 2 + %a2 = add i64 %idx, 8 + %idx3 = lshr i64 %a2, 2 + %a3 = add i64 %idx, 12 + %idx4 = lshr i64 %a3, 2 + + %gep2 = getelementptr %struct.ilc, %struct.ilc* %ptr, i32 0, i32 1, i64 %idx2 + %gep3 = getelementptr %struct.ilc, %struct.ilc* %ptr, i32 0, i32 1, i64 %idx3 + %gep4 = getelementptr %struct.ilc, %struct.ilc* %ptr, i32 0, i32 1, i64 %idx4 + %idx1 = lshr i64 %idx, 2 + %gep1 = getelementptr %struct.ilc, %struct.ilc* %ptr, i32 0, i32 1, i64 %idx1 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 4 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 4 + %ld3 = load <4 x float>, <4 x float>* %gep3, align 4 + %ld4 = load <4 x float>, <4 x float>* %gep4, align 4 + %sv1 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv2 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv3 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %sv4 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + %m0_3 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m4_7 = shufflevector <4 x float> %sv1, <4 x float> %sv3, <4 x i32> + %m8_11 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + %m12_15 = shufflevector <4 x float> %sv2, <4 x float> %sv4, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + store <4 x float> %m8_11, <4 x float>* %gep3, align 16 + store <4 x float> %m12_15, <4 x float>* %gep4, align 16 + ret void +} + +; This should be lowered into LD2 +define void @aarch64_ilc_idx_ld2(<4 x float>* %ptr, i64 %idx) { +entry: +; CHECK-LABEL: @aarch64_ilc_idx_ld2( +; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2 +; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 [[LSHR]] +; CHECK-DAG: [[CAST:%.+]] = bitcast <4 x float>* [[GEP]] to <8 x float>* +; CHECK-DAG: [[LOAD:%.+]] = load <8 x float>, <8 x float>* [[CAST]], align 16 +; CHECK: %{{.* }}= shufflevector <8 x float> [[LOAD]], <8 x float> undef, <4 x i32> +; CHECK: %{{.* }}= shufflevector <8 x float> [[LOAD]], <8 x float> undef, <4 x i32> +; CHECK-DAG: ret void + +; AS-LABEL: aarch64_ilc_idx_ld2 +; AS: ld2 +; AS: ret + + %idx1 = lshr i64 %idx, 2 + %a1 = add i64 %idx, 4 + %idx2 = lshr i64 %a1, 2 + + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx1 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx2 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1 + store <4 x float> %m4_7, <4 x float>* %gep2 + ret void +} + +; This should be lowered into LD3 +define void @aarch64_ilc_idx_ld3(<4 x float>* %ptr, i64 %idx) { +entry: +; CHECK-LABEL: @aarch64_ilc_idx_ld3( +; CHECK-DAG: [[LSHR:%.+]] = lshr i64 %idx, 2 +; CHECK-DAG: [[GEP:%.+]] = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 [[LSHR]] +; CHECK-DAG: [[CAST:%.+]] = bitcast <4 x float>* [[GEP]] to <12 x float>* +; CHECK-DAG: [[LOAD:%.+]] = load <12 x float>, <12 x float>* [[CAST]], align 16 +; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> undef, <4 x i32> +; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> undef, <4 x i32> +; CHECK: %{{.* }}= shufflevector <12 x float> [[LOAD]], <12 x float> undef, <4 x i32> +; CHECK-DAG: ret void + +; AS-LABEL: aarch64_ilc_idx_ld3 +; AS: ld3 +; AS: ret + + %idx1 = lshr i64 %idx, 2 + %a1 = add i64 %idx, 4 + %idx2 = lshr i64 %a1, 2 + %a2 = add i64 %idx, 8 + %idx3 = lshr i64 %a2, 2 + + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx1 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx2 + %gep3 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i64 %idx3 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %ld3 = load <4 x float>, <4 x float>* %gep3, align 16 + + %sv1 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv2 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %sv3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %m0_3 = shufflevector <4 x float> %sv1, <4 x float> %ld3, <4 x i32> + %m4_7 = shufflevector <4 x float> %sv2, <4 x float> %ld3, <4 x i32> + %m8_11 = shufflevector <4 x float> %sv3, <4 x float> %ld3, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + store <4 x float> %m8_11, <4 x float>* %gep3, align 16 + ret void +} +; %sv3 = shufflevector <4 x float> %ld3, <4 x float> %ld4, <4 x i32> + +; This must not be lowered +define void @aarch64_ilc_i32_idx(<4 x float>* %ptr, i32 %idx) { +; CHECK-LABEL: @aarch64_ilc_i32_idx( +; CHECK: %idx1 = lshr i32 %idx, 2 +; CHECK-NEXT: %a1 = add i32 %idx, 4 +; CHECK-NEXT: %idx2 = lshr i32 %a1, 2 +; CHECK-NEXT: %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 %idx1 +; CHECK-NEXT: %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 %idx2 +; CHECK-NEXT: %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 +; CHECK-NEXT: %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 +; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: store <4 x float> %m0_3, <4 x float>* %gep1, align 16 +; CHECK-NEXT: store <4 x float> %m4_7, <4 x float>* %gep2, align 16 +; CHECK-NEXT: ret void + +; AS-LABEL: aarch64_ilc_i32_idx +; AS-DAG: @function +; AS-NOT: ld2 +; AS-NOT: ld3 +; AS-NOT: ld4 +; AS-DAG: ret + +entry: + %idx1 = lshr i32 %idx, 2 + %a1 = add i32 %idx, 4 + %idx2 = lshr i32 %a1, 2 + + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 %idx1 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 %idx2 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + ret void +} + +; Volatile loads must not be lowered +define void @aarch64_ilc_volatile(<4 x float>* %ptr) { +; CHECK-LABEL: @aarch64_ilc_volatile( +; CHECK: %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 0 +; CHECK-NEXT: %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 1 +; CHECK-NEXT: %ld1 = load volatile <4 x float>, <4 x float>* %gep1, align 16 +; CHECK-NEXT: %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 +; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: store <4 x float> %m0_3, <4 x float>* %gep1, align 16 +; CHECK-NEXT: store <4 x float> %m4_7, <4 x float>* %gep2, align 16 +; CHECK-NEXT: ret void + +; AS-LABEL: aarch64_ilc_volatile +; AS-DAG: @function +; AS-NOT: ld2 +; AS-NOT: ld3 +; AS-NOT: ld4 +; AS-DAG: ret + +entry: + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 0 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 1 + %ld1 = load volatile <4 x float>, <4 x float>* %gep1, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + ret void +} + +; This must not be lowered +define void @aarch64_ilc_depmem(<4 x float>* %ptr, i32 %idx) { +entry: +; CHECK-LABEL: @aarch64_ilc_depmem( +; CHECK: %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 0 +; CHECK-NEXT: %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 1 +; CHECK-NEXT: %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 +; CHECK-NEXT: store <4 x float> %ld1, <4 x float>* %gep2, align 16 +; CHECK-NEXT: %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 +; CHECK-NEXT: %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> +; CHECK-NEXT: store <4 x float> %m0_3, <4 x float>* %gep1, align 16 +; CHECK-NEXT: store <4 x float> %m4_7, <4 x float>* %gep2, align 16 +; CHECK-NEXT: ret void + +; AS-LABEL: aarch64_ilc_depmem +; AS-DAG: @function +; AS-NOT: ld2 +; AS-NOT: ld3 +; AS-NOT: ld4 +; AS-DAG: ret + + %gep1 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 0 + %gep2 = getelementptr inbounds <4 x float>, <4 x float>* %ptr, i32 1 + %ld1 = load <4 x float>, <4 x float>* %gep1, align 16 + store <4 x float> %ld1, <4 x float>* %gep2, align 16 + %ld2 = load <4 x float>, <4 x float>* %gep2, align 16 + %m0_3 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + %m4_7 = shufflevector <4 x float> %ld1, <4 x float> %ld2, <4 x i32> + + store <4 x float> %m0_3, <4 x float>* %gep1, align 16 + store <4 x float> %m4_7, <4 x float>* %gep2, align 16 + ret void +} + +; This cannot be converted - insertion position cannot be determined +define void @aarch64_no_insertion_pos(float* %ptr) { +entry: +; CHECK-LABEL: @aarch64_no_insertion_pos( +; CHECK: %p0 = getelementptr inbounds float, float* %ptr, i32 0 +; CHECK-NEXT: %p1 = getelementptr inbounds float, float* %ptr, i32 4 +; CHECK-NEXT: %b0 = bitcast float* %p0 to <5 x float>* +; CHECK-NEXT: %b1 = bitcast float* %p1 to <5 x float>* +; CHECK-NEXT: %l0 = load <5 x float>, <5 x float>* %b0 +; CHECK-NEXT: %l1 = load <5 x float>, <5 x float>* %b1 +; CHECK-NEXT: %s0 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> +; CHECK-NEXT: %s1 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> +; CHECK-NEXT: ret void + + %p0 = getelementptr inbounds float, float* %ptr, i32 0 + %p1 = getelementptr inbounds float, float* %ptr, i32 4 + %b0 = bitcast float* %p0 to <5 x float>* + %b1 = bitcast float* %p1 to <5 x float>* + %l0 = load <5 x float>, <5 x float>* %b0 + %l1 = load <5 x float>, <5 x float>* %b1 + %s0 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> + %s1 = shufflevector <5 x float> %l0, <5 x float> %l1, <4 x i32> + ret void +} + +; This cannot be converted - the insertion position does not dominate all +; uses +define void @aarch64_insertpos_does_not_dominate(float* %ptr) { +entry: +; CHECK-LABEL: @aarch64_insertpos_does_not_dominate( +; CHECK: %p0 = getelementptr inbounds float, float* %ptr, i32 0 +; CHECK-NEXT: %p1 = getelementptr inbounds float, float* %ptr, i32 1 +; CHECK-NEXT: %b0 = bitcast float* %p0 to <7 x float>* +; CHECK-NEXT: %b1 = bitcast float* %p1 to <7 x float>* +; CHECK-NEXT: %l1 = load <7 x float>, <7 x float>* %b1 +; CHECK-NEXT: %s1 = shufflevector <7 x float> %l1, <7 x float> undef, <4 x i32> +; CHECK-NEXT: %l0 = load <7 x float>, <7 x float>* %b0 +; CHECK-NEXT: %s0 = shufflevector <7 x float> %l0, <7 x float> undef, <4 x i32> +; CHECK-NEXT: ret void + %p0 = getelementptr inbounds float, float* %ptr, i32 0 + %p1 = getelementptr inbounds float, float* %ptr, i32 1 + %b0 = bitcast float* %p0 to <7 x float>* + %b1 = bitcast float* %p1 to <7 x float>* + %l1 = load <7 x float>, <7 x float>* %b1 + %s1 = shufflevector <7 x float> %l1, <7 x float> undef, <4 x i32> + %l0 = load <7 x float>, <7 x float>* %b0 + %s0 = shufflevector <7 x float> %l0, <7 x float> undef, <4 x i32> + ret void +} Index: tools/opt/opt.cpp =================================================================== --- tools/opt/opt.cpp +++ tools/opt/opt.cpp @@ -463,6 +463,7 @@ initializePreISelIntrinsicLoweringLegacyPassPass(Registry); initializeGlobalMergePass(Registry); initializeIndirectBrExpandPassPass(Registry); + initializeInterleavedLoadCombinePass(Registry); initializeInterleavedAccessPass(Registry); initializeEntryExitInstrumenterPass(Registry); initializePostInlineEntryExitInstrumenterPass(Registry);