Index: llvm/trunk/include/llvm/IR/Intrinsics.h =================================================================== --- llvm/trunk/include/llvm/IR/Intrinsics.h +++ llvm/trunk/include/llvm/IR/Intrinsics.h @@ -116,7 +116,8 @@ AK_AnyInteger, AK_AnyFloat, AK_AnyVector, - AK_AnyPointer + AK_AnyPointer, + AK_MatchType = 7 }; unsigned getArgumentNumber() const { @@ -161,14 +162,21 @@ /// of IITDescriptors. void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl &T); - /// Match the specified type (which comes from an intrinsic argument or return - /// value) with the type constraints specified by the .td file. If the given - /// type is an overloaded type it is pushed to the ArgTys vector. + enum MatchIntrinsicTypesResult { + MatchIntrinsicTypes_Match = 0, + MatchIntrinsicTypes_NoMatchRet = 1, + MatchIntrinsicTypes_NoMatchArg = 2, + }; + + /// Match the specified function type with the type constraints specified by + /// the .td file. If the given type is an overloaded type it is pushed to the + /// ArgTys vector. /// /// Returns false if the given type matches with the constraints, true /// otherwise. - bool matchIntrinsicType(Type *Ty, ArrayRef &Infos, - SmallVectorImpl &ArgTys); + MatchIntrinsicTypesResult + matchIntrinsicSignature(FunctionType *FTy, ArrayRef &Infos, + SmallVectorImpl &ArgTys); /// Verify if the intrinsic has variable arguments. This method is intended to /// be called after all the fixed arguments have been matched first. Index: llvm/trunk/lib/IR/Function.cpp =================================================================== --- llvm/trunk/lib/IR/Function.cpp +++ llvm/trunk/lib/IR/Function.cpp @@ -1047,12 +1047,26 @@ #include "llvm/IR/IntrinsicImpl.inc" #undef GET_LLVM_INTRINSIC_FOR_MS_BUILTIN -bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef &Infos, - SmallVectorImpl &ArgTys) { +using DeferredIntrinsicMatchPair = + std::pair>; + +static bool matchIntrinsicType( + Type *Ty, ArrayRef &Infos, + SmallVectorImpl &ArgTys, + SmallVectorImpl &DeferredChecks, + bool IsDeferredCheck) { using namespace Intrinsic; // If we ran out of descriptors, there are too many arguments. if (Infos.empty()) return true; + + // Do this before slicing off the 'front' part + auto InfosRef = Infos; + auto DeferCheck = [&DeferredChecks, &InfosRef](Type *T) { + DeferredChecks.emplace_back(T, InfosRef); + return false; + }; + IITDescriptor D = Infos.front(); Infos = Infos.slice(1); @@ -1070,12 +1084,14 @@ case IITDescriptor::Vector: { VectorType *VT = dyn_cast(Ty); return !VT || VT->getNumElements() != D.Vector_Width || - matchIntrinsicType(VT->getElementType(), Infos, ArgTys); + matchIntrinsicType(VT->getElementType(), Infos, ArgTys, + DeferredChecks, IsDeferredCheck); } case IITDescriptor::Pointer: { PointerType *PT = dyn_cast(Ty); return !PT || PT->getAddressSpace() != D.Pointer_AddressSpace || - matchIntrinsicType(PT->getElementType(), Infos, ArgTys); + matchIntrinsicType(PT->getElementType(), Infos, ArgTys, + DeferredChecks, IsDeferredCheck); } case IITDescriptor::Struct: { @@ -1084,20 +1100,24 @@ return true; for (unsigned i = 0, e = D.Struct_NumElements; i != e; ++i) - if (matchIntrinsicType(ST->getElementType(i), Infos, ArgTys)) + if (matchIntrinsicType(ST->getElementType(i), Infos, ArgTys, + DeferredChecks, IsDeferredCheck)) return true; return false; } case IITDescriptor::Argument: - // Two cases here - If this is the second occurrence of an argument, verify - // that the later instance matches the previous instance. + // If this is the second occurrence of an argument, + // verify that the later instance matches the previous instance. if (D.getArgumentNumber() < ArgTys.size()) return Ty != ArgTys[D.getArgumentNumber()]; - // Otherwise, if this is the first instance of an argument, record it and - // verify the "Any" kind. - assert(D.getArgumentNumber() == ArgTys.size() && "Table consistency error"); + if (D.getArgumentNumber() > ArgTys.size() || + D.getArgumentKind() == IITDescriptor::AK_MatchType) + return IsDeferredCheck || DeferCheck(Ty); + + assert(D.getArgumentNumber() == ArgTys.size() && !IsDeferredCheck && + "Table consistency error"); ArgTys.push_back(Ty); switch (D.getArgumentKind()) { @@ -1106,13 +1126,14 @@ case IITDescriptor::AK_AnyFloat: return !Ty->isFPOrFPVectorTy(); case IITDescriptor::AK_AnyVector: return !isa(Ty); case IITDescriptor::AK_AnyPointer: return !isa(Ty); + default: break; } llvm_unreachable("all argument kinds not covered"); case IITDescriptor::ExtendArgument: { - // This may only be used when referring to a previous vector argument. + // If this is a forward reference, defer the check for later. if (D.getArgumentNumber() >= ArgTys.size()) - return true; + return IsDeferredCheck || DeferCheck(Ty); Type *NewTy = ArgTys[D.getArgumentNumber()]; if (VectorType *VTy = dyn_cast(NewTy)) @@ -1125,9 +1146,9 @@ return Ty != NewTy; } case IITDescriptor::TruncArgument: { - // This may only be used when referring to a previous vector argument. + // If this is a forward reference, defer the check for later. if (D.getArgumentNumber() >= ArgTys.size()) - return true; + return IsDeferredCheck || DeferCheck(Ty); Type *NewTy = ArgTys[D.getArgumentNumber()]; if (VectorType *VTy = dyn_cast(NewTy)) @@ -1140,14 +1161,17 @@ return Ty != NewTy; } case IITDescriptor::HalfVecArgument: - // This may only be used when referring to a previous vector argument. + // If this is a forward reference, defer the check for later. return D.getArgumentNumber() >= ArgTys.size() || !isa(ArgTys[D.getArgumentNumber()]) || VectorType::getHalfElementsVectorType( cast(ArgTys[D.getArgumentNumber()])) != Ty; case IITDescriptor::SameVecWidthArgument: { - if (D.getArgumentNumber() >= ArgTys.size()) - return true; + if (D.getArgumentNumber() >= ArgTys.size()) { + // Defer check and subsequent check for the vector element type. + Infos = Infos.slice(1); + return IsDeferredCheck || DeferCheck(Ty); + } auto *ReferenceType = dyn_cast(ArgTys[D.getArgumentNumber()]); auto *ThisArgType = dyn_cast(Ty); // Both must be vectors of the same number of elements or neither. @@ -1160,18 +1184,19 @@ return true; EltTy = ThisArgType->getVectorElementType(); } - return matchIntrinsicType(EltTy, Infos, ArgTys); + return matchIntrinsicType(EltTy, Infos, ArgTys, DeferredChecks, + IsDeferredCheck); } case IITDescriptor::PtrToArgument: { if (D.getArgumentNumber() >= ArgTys.size()) - return true; + return IsDeferredCheck || DeferCheck(Ty); Type * ReferenceType = ArgTys[D.getArgumentNumber()]; PointerType *ThisArgType = dyn_cast(Ty); return (!ThisArgType || ThisArgType->getElementType() != ReferenceType); } case IITDescriptor::PtrToElt: { if (D.getArgumentNumber() >= ArgTys.size()) - return true; + return IsDeferredCheck || DeferCheck(Ty); VectorType * ReferenceType = dyn_cast (ArgTys[D.getArgumentNumber()]); PointerType *ThisArgType = dyn_cast(Ty); @@ -1181,15 +1206,20 @@ } case IITDescriptor::VecOfAnyPtrsToElt: { unsigned RefArgNumber = D.getRefArgNumber(); + if (RefArgNumber >= ArgTys.size()) { + if (IsDeferredCheck) + return true; + // If forward referencing, already add the pointer-vector type and + // defer the checks for later. + ArgTys.push_back(Ty); + return DeferCheck(Ty); + } - // This may only be used when referring to a previous argument. - if (RefArgNumber >= ArgTys.size()) - return true; - - // Record the overloaded type - assert(D.getOverloadArgNumber() == ArgTys.size() && - "Table consistency error"); - ArgTys.push_back(Ty); + if (!IsDeferredCheck){ + assert(D.getOverloadArgNumber() == ArgTys.size() && + "Table consistency error"); + ArgTys.push_back(Ty); + } // Verify the overloaded type "matches" the Ref type. // i.e. Ty is a vector with the same width as Ref. @@ -1211,6 +1241,32 @@ llvm_unreachable("unhandled"); } +Intrinsic::MatchIntrinsicTypesResult +Intrinsic::matchIntrinsicSignature(FunctionType *FTy, + ArrayRef &Infos, + SmallVectorImpl &ArgTys) { + SmallVector DeferredChecks; + if (matchIntrinsicType(FTy->getReturnType(), Infos, ArgTys, DeferredChecks, + false)) + return MatchIntrinsicTypes_NoMatchRet; + + unsigned NumDeferredReturnChecks = DeferredChecks.size(); + + for (auto Ty : FTy->params()) + if (matchIntrinsicType(Ty, Infos, ArgTys, DeferredChecks, false)) + return MatchIntrinsicTypes_NoMatchArg; + + for (unsigned I = 0, E = DeferredChecks.size(); I != E; ++I) { + DeferredIntrinsicMatchPair &Check = DeferredChecks[I]; + if (matchIntrinsicType(Check.first, Check.second, ArgTys, DeferredChecks, + true)) + return I < NumDeferredReturnChecks ? MatchIntrinsicTypes_NoMatchRet + : MatchIntrinsicTypes_NoMatchArg; + } + + return MatchIntrinsicTypes_Match; +} + bool Intrinsic::matchIntrinsicVarArg(bool isVarArg, ArrayRef &Infos) { @@ -1244,13 +1300,8 @@ getIntrinsicInfoTableEntries(ID, Table); ArrayRef TableRef = Table; - // If we encounter any problems matching the signature with the descriptor - // just give up remangling. It's up to verifier to report the discrepancy. - if (Intrinsic::matchIntrinsicType(FTy->getReturnType(), TableRef, ArgTys)) + if (Intrinsic::matchIntrinsicSignature(FTy, TableRef, ArgTys)) return None; - for (auto Ty : FTy->params()) - if (Intrinsic::matchIntrinsicType(Ty, TableRef, ArgTys)) - return None; if (Intrinsic::matchIntrinsicVarArg(FTy->isVarArg(), TableRef)) return None; } Index: llvm/trunk/lib/IR/Verifier.cpp =================================================================== --- llvm/trunk/lib/IR/Verifier.cpp +++ llvm/trunk/lib/IR/Verifier.cpp @@ -4154,14 +4154,14 @@ getIntrinsicInfoTableEntries(ID, Table); ArrayRef TableRef = Table; + // Walk the descriptors to extract overloaded types. SmallVector ArgTys; - Assert(!Intrinsic::matchIntrinsicType(IFTy->getReturnType(), - TableRef, ArgTys), + Intrinsic::MatchIntrinsicTypesResult Res = + Intrinsic::matchIntrinsicSignature(IFTy, TableRef, ArgTys); + Assert(Res != Intrinsic::MatchIntrinsicTypes_NoMatchRet, "Intrinsic has incorrect return type!", IF); - for (unsigned i = 0, e = IFTy->getNumParams(); i != e; ++i) - Assert(!Intrinsic::matchIntrinsicType(IFTy->getParamType(i), - TableRef, ArgTys), - "Intrinsic has incorrect argument type!", IF); + Assert(Res != Intrinsic::MatchIntrinsicTypes_NoMatchArg, + "Intrinsic has incorrect argument type!", IF); // Verify if the intrinsic call matches the vararg property. if (IsVarArg) Index: llvm/trunk/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp =================================================================== --- llvm/trunk/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ llvm/trunk/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1019,13 +1019,12 @@ getIntrinsicInfoTableEntries(IID, Table); ArrayRef TableRef = Table; + // Validate function argument and return types, extracting overloaded types + // along the way. FunctionType *FTy = II->getCalledFunction()->getFunctionType(); SmallVector OverloadTys; - Intrinsic::matchIntrinsicType(FTy->getReturnType(), TableRef, OverloadTys); - for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) - Intrinsic::matchIntrinsicType(FTy->getParamType(i), TableRef, OverloadTys); + Intrinsic::matchIntrinsicSignature(FTy, TableRef, OverloadTys); - // Get the new return type overload of the intrinsic. Module *M = II->getParent()->getParent()->getParent(); Type *EltTy = II->getType()->getVectorElementType(); Type *NewTy = (NewNumElts == 1) ? EltTy : VectorType::get(EltTy, NewNumElts); Index: llvm/trunk/utils/TableGen/CodeGenTarget.cpp =================================================================== --- llvm/trunk/utils/TableGen/CodeGenTarget.cpp +++ llvm/trunk/utils/TableGen/CodeGenTarget.cpp @@ -591,9 +591,29 @@ TargetPrefix + ".'!"); } - // Parse the list of return types. + ListInit *RetTypes = R->getValueAsListInit("RetTypes"); + ListInit *ParamTypes = R->getValueAsListInit("ParamTypes"); + + // First collate a list of overloaded types. std::vector OverloadedVTs; - ListInit *TypeList = R->getValueAsListInit("RetTypes"); + for (ListInit *TypeList : {RetTypes, ParamTypes}) { + for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { + Record *TyEl = TypeList->getElementAsRecord(i); + assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); + + if (TyEl->isSubClassOf("LLVMMatchType")) + continue; + + MVT::SimpleValueType VT = getValueType(TyEl->getValueAsDef("VT")); + if (MVT(VT).isOverloaded()) { + OverloadedVTs.push_back(VT); + isOverloaded = true; + } + } + } + + // Parse the list of return types. + ListInit *TypeList = RetTypes; for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { Record *TyEl = TypeList->getElementAsRecord(i); assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); @@ -613,10 +633,6 @@ } else { VT = getValueType(TyEl->getValueAsDef("VT")); } - if (MVT(VT).isOverloaded()) { - OverloadedVTs.push_back(VT); - isOverloaded = true; - } // Reject invalid types. if (VT == MVT::isVoid) @@ -628,7 +644,7 @@ } // Parse the list of parameter types. - TypeList = R->getValueAsListInit("ParamTypes"); + TypeList = ParamTypes; for (unsigned i = 0, e = TypeList->size(); i != e; ++i) { Record *TyEl = TypeList->getElementAsRecord(i); assert(TyEl->isSubClassOf("LLVMType") && "Expected a type!"); @@ -654,11 +670,6 @@ } else VT = getValueType(TyEl->getValueAsDef("VT")); - if (MVT(VT).isOverloaded()) { - OverloadedVTs.push_back(VT); - isOverloaded = true; - } - // Reject invalid types. if (VT == MVT::isVoid && i != e-1 /*void at end means varargs*/) PrintFatalError(DefLoc, "Intrinsic '" + DefName + Index: llvm/trunk/utils/TableGen/IntrinsicEmitter.cpp =================================================================== --- llvm/trunk/utils/TableGen/IntrinsicEmitter.cpp +++ llvm/trunk/utils/TableGen/IntrinsicEmitter.cpp @@ -258,10 +258,12 @@ #endif static void EncodeFixedType(Record *R, std::vector &ArgCodes, - std::vector &Sig) { + unsigned &NextArgCode, + std::vector &Sig, + ArrayRef Mapping) { if (R->isSubClassOf("LLVMMatchType")) { - unsigned Number = R->getValueAsInt("Number"); + unsigned Number = Mapping[R->getValueAsInt("Number")]; assert(Number < ArgCodes.size() && "Invalid matching number!"); if (R->isSubClassOf("LLVMExtendedType")) Sig.push_back(IIT_EXTEND_ARG); @@ -280,10 +282,8 @@ Sig.push_back(IIT_PTR_TO_ARG); else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT); - unsigned ArgNo = ArgCodes.size(); - ArgCodes.push_back(3 /*vAny*/); // Encode overloaded ArgNo - Sig.push_back(ArgNo); + Sig.push_back(NextArgCode++); // Encode LLVMMatchType ArgNo Sig.push_back(Number); return; @@ -291,7 +291,7 @@ Sig.push_back(IIT_PTR_TO_ELT); else Sig.push_back(IIT_ARG); - return Sig.push_back((Number << 3) | ArgCodes[Number]); + return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/); } MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT")); @@ -309,8 +309,9 @@ Sig.push_back(IIT_ARG); // Figure out what arg # this is consuming, and remember what kind it was. - unsigned ArgNo = ArgCodes.size(); - ArgCodes.push_back(Tmp); + assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp && + "Invalid or no ArgCode associated with overloaded VT!"); + unsigned ArgNo = NextArgCode++; // Encode what sort of argument it must be in the low 3 bits of the ArgNo. return Sig.push_back((ArgNo << 3) | Tmp); @@ -328,7 +329,8 @@ } else { Sig.push_back(IIT_PTR); } - return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, Sig); + return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig, + Mapping); } } @@ -353,6 +355,42 @@ EncodeFixedValueType(VT, Sig); } +static void UpdateArgCodes(Record *R, std::vector &ArgCodes, + unsigned int &NumInserted, + SmallVectorImpl &Mapping) { + if (R->isSubClassOf("LLVMMatchType")) { + if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { + ArgCodes.push_back(3 /*vAny*/); + ++NumInserted; + } + return; + } + + unsigned Tmp = 0; + switch (getValueType(R->getValueAsDef("VT"))) { + default: break; + case MVT::iPTRAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::vAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::fAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::iAny: + ++Tmp; + LLVM_FALLTHROUGH; + case MVT::Any: + unsigned OriginalIdx = ArgCodes.size() - NumInserted; + assert(OriginalIdx >= Mapping.size()); + Mapping.resize(OriginalIdx+1); + Mapping[OriginalIdx] = ArgCodes.size(); + ArgCodes.push_back(Tmp); + break; + } +} + #if defined(_MSC_VER) && !defined(__clang__) #pragma optimize("",on) #endif @@ -363,6 +401,17 @@ std::vector &TypeSig) { std::vector ArgCodes; + // Add codes for any overloaded result VTs. + unsigned int NumInserted = 0; + SmallVector ArgMapping; + for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) + UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping); + + // Add codes for any overloaded operand VTs. + for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) + UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping); + + unsigned NextArgCode = 0; if (Int.IS.RetVTs.empty()) TypeSig.push_back(IIT_Done); else if (Int.IS.RetVTs.size() == 1 && @@ -382,11 +431,13 @@ } for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) - EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, TypeSig); + EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig, + ArgMapping); } for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) - EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, TypeSig); + EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig, + ArgMapping); } static void printIITEntry(raw_ostream &OS, unsigned char X) {