[Lexer] Allow LLLexer to be used as an API

[wsmoses]

Summary

Explose LLVM Lexer for usage externally as an API

[jdoerfert]

Moving the 3 files into the include directory seems fine to me.

llvm/include/llvm/AsmParser/LLLexer.h

 
 #include "LLToken.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/Support/SMLoc.h"
 #include <string>
 
 namespace llvm {
   class Type;
   class SMDiagnostic;
   class SourceMgr;
   class LLVMContext;
 
   class LLLexer {
     const char *CurPtr;
     StringRef CurBuf;
     SMDiagnostic &ErrorInfo;

llvm/include/llvm/AsmParser/LLParser.h

 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include <map>
 
 namespace llvm {
   class Module;
   class Function;
   class Value;
   class BasicBlock;
   class Instruction;
   class Constant;
   class GlobalValue;
   class Comdat;
   class MDString;
struct ValID {
     ValID() = default;
     ValID(const ValID &RHS)
         : Kind(RHS.Kind), Loc(RHS.Loc), UIntVal(RHS.UIntVal), FTy(RHS.FTy),
           StrVal(RHS.StrVal), StrVal2(RHS.StrVal2), APSIntVal(RHS.APSIntVal),
           APFloatVal(RHS.APFloatVal), ConstantVal(RHS.ConstantVal) {
       assert(!RHS.ConstantStructElts);
     }
 
     bool operator<(const ValID &RHS) const {
       if (Kind == t_LocalID || Kind == t_GlobalID)
         return UIntVal < RHS.UIntVal;
       assert((Kind == t_LocalName || Kind == t_GlobalName ||
               Kind == t_ConstantStruct || Kind == t_PackedConstantStruct) &&
              "Ordering not defined for this ValID kind yet");
       return StrVal < RHS.StrVal;
     }
   };
 
   class LLParser {
   public:
     typedef LLLexer::LocTy LocTy;
   private:
     LLVMContext &Context;
     LLLexer Lex;
     // Module being parsed, null if we are only parsing summary index.
     Module *M;
private:
 
     /// Only the llvm-as tool may set this to false to bypass
     /// UpgradeDebuginfo so it can generate broken bitcode.
     bool UpgradeDebugInfo;
 
     std::string SourceFileName;
 
   public:
     LLParser(StringRef F, SourceMgr &SM, SMDiagnostic &Err, Module *M,
              ModuleSummaryIndex *Index, LLVMContext &Context,
              SlotMapping *Slots = nullptr)
         : Context(Context), Lex(F, SM, Err, Context), M(M), Index(Index),
           Slots(Slots), BlockAddressPFS(nullptr) {}
     bool Run(
         bool UpgradeDebugInfo, DataLayoutCallbackTy DataLayoutCallback =
                                    [](StringRef) { return None; });
 
     bool parseStandaloneConstantValue(Constant *&C, const SlotMapping *Slots);
 
     bool parseTypeAtBeginning(Type *&Ty, unsigned &Read,
                               const SlotMapping *Slots);
 
     LLVMContext &getContext() { return Context; }
 
   private:
     bool error(LocTy L, const Twine &Msg) const { return Lex.Error(L, Msg); }
     bool tokError(const Twine &Msg) const { return error(Lex.getLoc(), Msg); }
 
     /// Restore the internal name and slot mappings using the mappings that
     /// were created at an earlier parsing stage.
     void restoreParsingState(const SlotMapping *Slots);
 
     /// getGlobalVal - Get a value with the specified name or ID, creating a
     /// forward reference record if needed.  This can return null if the value
     /// exists but does not have the right type.
     GlobalValue *getGlobalVal(const std::string &N, Type *Ty, LocTy Loc,
                               bool IsCall);
     GlobalValue *getGlobalVal(unsigned ID, Type *Ty, LocTy Loc, bool IsCall);
 
     /// Get a Comdat with the specified name, creating a forward reference
     /// record if needed.
     Comdat *getComdat(const std::string &Name, LocTy Loc);
 
     // Helper Routines.
     bool parseToken(lltok::Kind T, const char *ErrMsg);
     bool EatIfPresent(lltok::Kind T) {
       if (Lex.getKind() != T) return false;
       Lex.Lex();
       return true;
     }
 
     FastMathFlags EatFastMathFlagsIfPresent() {
       FastMathFlags FMF;
       while (true)
         switch (Lex.getKind()) {
         case lltok::kw_fast: FMF.setFast();            Lex.Lex(); continue;
         case lltok::kw_nnan: FMF.setNoNaNs();          Lex.Lex(); continue;
         case lltok::kw_ninf: FMF.setNoInfs();          Lex.Lex(); continue;
         case lltok::kw_nsz:  FMF.setNoSignedZeros();   Lex.Lex(); continue;
         case lltok::kw_arcp: FMF.setAllowReciprocal(); Lex.Lex(); continue;
         case lltok::kw_contract:
           FMF.setAllowContract(true);
           Lex.Lex();
           continue;
         case lltok::kw_reassoc: FMF.setAllowReassoc(); Lex.Lex(); continue;
         case lltok::kw_afn:     FMF.setApproxFunc();   Lex.Lex(); continue;
         default: return FMF;
         }
       return FMF;
     }
 
     bool parseOptionalToken(lltok::Kind T, bool &Present,
                             LocTy *Loc = nullptr) {
       if (Lex.getKind() != T) {
         Present = false;
       } else {
         if (Loc)
           *Loc = Lex.getLoc();
         Lex.Lex();
         Present = true;
Value *checkValidVariableType(LocTy Loc, const Twine &Name, Type *Ty,
                                   Value *Val, bool IsCall);
 
     bool parseConstantValue(Type *Ty, Constant *&C);
     bool parseValue(Type *Ty, Value *&V, PerFunctionState *PFS);
     bool parseValue(Type *Ty, Value *&V, PerFunctionState &PFS) {
       return parseValue(Ty, V, &PFS);
     }
 
     bool parseValue(Type *Ty, Value *&V, LocTy &Loc, PerFunctionState &PFS) {
       Loc = Lex.getLoc();
       return parseValue(Ty, V, &PFS);
     }
 
     bool parseTypeAndValue(Value *&V, PerFunctionState *PFS);
     bool parseTypeAndValue(Value *&V, PerFunctionState &PFS) {
       return parseTypeAndValue(V, &PFS);
     }
     bool parseTypeAndValue(Value *&V, LocTy &Loc, PerFunctionState &PFS) {
       Loc = Lex.getLoc();
       return parseTypeAndValue(V, PFS);
     }
     bool parseTypeAndBasicBlock(BasicBlock *&BB, LocTy &Loc,
                                 PerFunctionState &PFS);
     bool parseTypeAndBasicBlock(BasicBlock *&BB, PerFunctionState &PFS) {
       LocTy Loc;
       return parseTypeAndBasicBlock(BB, Loc, PFS);
     }
 
     struct ParamInfo {
       LocTy Loc;
       Value *V;
       AttributeSet Attrs;
       ParamInfo(LocTy loc, Value *v, AttributeSet attrs)
           : Loc(loc), V(v), Attrs(attrs) {}
     };
     bool parseParameterList(SmallVectorImpl<ParamInfo> &ArgList,
                             PerFunctionState &PFS, bool IsMustTailCall = false,
private:
     template <class ParserTy> bool parseMDFieldsImplBody(ParserTy ParseField);
     template <class ParserTy>
     bool parseMDFieldsImpl(ParserTy ParseField, LocTy &ClosingLoc);
     bool parseSpecializedMDNode(MDNode *&N, bool IsDistinct = false);
 
 #define HANDLE_SPECIALIZED_MDNODE_LEAF(CLASS)                                  \
   bool parse##CLASS(MDNode *&Result, bool IsDistinct);
 #include "llvm/IR/Metadata.def"
     bool parseDIArgList(MDNode *&Result, bool IsDistinct,
                         PerFunctionState *PFS);
 
     // Function Parsing.
     struct ArgInfo {
       LocTy Loc;
       Type *Ty;
       AttributeSet Attrs;
       std::string Name;
#include "llvm/IR/Metadata.def"
     bool parseFreeze(Instruction *&I, PerFunctionState &PFS);
 
     // Use-list order directives.
     bool parseUseListOrder(PerFunctionState *PFS = nullptr);
     bool parseUseListOrderBB();
     bool parseUseListOrderIndexes(SmallVectorImpl<unsigned> &Indexes);
     bool sortUseListOrder(Value *V, ArrayRef<unsigned> Indexes, SMLoc Loc);
   };
 } // End llvm namespace
 
 #endif

llvm/include/llvm/AsmParser/LLToken.h

 //===- LLToken.h - Token Codes for LLVM Assembly Files ----------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the enums for the .ll lexer.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_ASMPARSER_LLTOKEN_H
 #define LLVM_LIB_ASMPARSER_LLTOKEN_H
 
 namespace llvm {
 namespace lltok {
 enum Kind {
   // Markers
   Eof,
   Error,
 
   // Tokens with no info.
   dotdotdot, // ...
   equal,
   comma, // =  ,
   star,  // *
   lsquare,
   rsquare, // [  ]
   lbrace,
   rbrace, // {  }
   less,
   greater, // <  >
   lparen,
   rparen,  // (  )
   exclaim, // !
   bar,     // |
   colon,   // :
 
   kw_vscale,
   kw_x,
   kw_true,
   kw_false,
   kw_declare,
   kw_define,
   kw_global,
   kw_constant,
 
   kw_dso_local,
   kw_dso_preemptable,
 
   kw_private,
   kw_internal,
   kw_linkonce,
   kw_linkonce_odr,
   kw_weak, // Used as a linkage, and a modifier for "cmpxchg".
   kw_weak_odr,
   kw_appending,
   kw_dllimport,
   kw_dllexport,
   kw_common,
   kw_available_externally,
   kw_default,
   kw_hidden,
   kw_protected,
   kw_unnamed_addr,
   kw_local_unnamed_addr,
   kw_externally_initialized,
   kw_extern_weak,
   kw_external,
   kw_thread_local,
   kw_localdynamic,
   kw_initialexec,
   kw_localexec,
   kw_zeroinitializer,
   kw_undef,
   kw_poison,
   kw_null,
   kw_none,
   kw_to,
   kw_caller,
   kw_within,
   kw_from,
   kw_tail,
   kw_musttail,
   kw_notail,
   kw_target,
   kw_triple,
   kw_source_filename,
   kw_unwind,
   kw_deplibs, // FIXME: Remove in 4.0
   kw_datalayout,
   kw_volatile,
   kw_atomic,
   kw_unordered,
   kw_monotonic,
   kw_acquire,
   kw_release,
   kw_acq_rel,
   kw_seq_cst,
   kw_syncscope,
   kw_nnan,
   kw_ninf,
   kw_nsz,
   kw_arcp,
   kw_contract,
   kw_reassoc,
   kw_afn,
   kw_fast,
   kw_nuw,
   kw_nsw,
   kw_exact,
   kw_inbounds,
   kw_inrange,
   kw_align,
   kw_addrspace,
   kw_section,
   kw_partition,
   kw_alias,
   kw_ifunc,
   kw_module,
   kw_asm,
   kw_sideeffect,
   kw_alignstack,
   kw_inteldialect,
   kw_gc,
   kw_prefix,
   kw_prologue,
   kw_c,
 
   kw_cc,
   kw_ccc,
   kw_fastcc,
   kw_coldcc,
   kw_intel_ocl_bicc,
   kw_cfguard_checkcc,
   kw_x86_stdcallcc,
   kw_x86_fastcallcc,
   kw_x86_thiscallcc,
   kw_x86_vectorcallcc,
   kw_x86_regcallcc,
   kw_arm_apcscc,
   kw_arm_aapcscc,
   kw_arm_aapcs_vfpcc,
   kw_aarch64_vector_pcs,
   kw_aarch64_sve_vector_pcs,
   kw_msp430_intrcc,
   kw_avr_intrcc,
   kw_avr_signalcc,
   kw_ptx_kernel,
   kw_ptx_device,
   kw_spir_kernel,
   kw_spir_func,
   kw_x86_64_sysvcc,
   kw_win64cc,
   kw_webkit_jscc,
   kw_anyregcc,
   kw_swiftcc,
   kw_preserve_mostcc,
   kw_preserve_allcc,
   kw_ghccc,
   kw_x86_intrcc,
   kw_hhvmcc,
   kw_hhvm_ccc,
   kw_cxx_fast_tlscc,
   kw_amdgpu_vs,
   kw_amdgpu_ls,
   kw_amdgpu_hs,
   kw_amdgpu_es,
   kw_amdgpu_gs,
   kw_amdgpu_ps,
   kw_amdgpu_cs,
   kw_amdgpu_kernel,
   kw_amdgpu_gfx,
   kw_tailcc,
 
   // Attributes:
   kw_attributes,
   kw_allocsize,
   kw_alwaysinline,
   kw_argmemonly,
   kw_sanitize_address,
   kw_sanitize_hwaddress,
   kw_sanitize_memtag,
   kw_builtin,
   kw_byval,
   kw_inalloca,
   kw_cold,
   kw_convergent,
   kw_dereferenceable,
   kw_dereferenceable_or_null,
   kw_inaccessiblememonly,
   kw_inaccessiblemem_or_argmemonly,
   kw_inlinehint,
   kw_inreg,
   kw_jumptable,
   kw_minsize,
   kw_naked,
   kw_nest,
   kw_noalias,
   kw_noundef,
   kw_nobuiltin,
   kw_nocallback,
   kw_nocapture,
   kw_noduplicate,
   kw_nofree,
   kw_noimplicitfloat,
   kw_noinline,
   kw_norecurse,
   kw_nonlazybind,
   kw_nomerge,
   kw_nonnull,
   kw_noprofile,
   kw_noredzone,
   kw_noreturn,
   kw_nosync,
   kw_nocf_check,
   kw_nounwind,
   kw_null_pointer_is_valid,
   kw_optforfuzzing,
   kw_optnone,
   kw_optsize,
   kw_preallocated,
   kw_readnone,
   kw_readonly,
   kw_returned,
   kw_returns_twice,
   kw_signext,
   kw_speculatable,
   kw_ssp,
   kw_sspreq,
   kw_sspstrong,
   kw_safestack,
   kw_shadowcallstack,
   kw_sret,
   kw_sanitize_thread,
   kw_sanitize_memory,
   kw_speculative_load_hardening,
   kw_strictfp,
   kw_swifterror,
   kw_swiftself,
   kw_uwtable,
   kw_vscale_range,
   kw_willreturn,
   kw_writeonly,
   kw_zeroext,
   kw_immarg,
   kw_byref,
   kw_mustprogress,
 
   kw_type,
   kw_opaque,
 
   kw_comdat,
 
   // Comdat types
   kw_any,
   kw_exactmatch,
   kw_largest,
   kw_noduplicates,
   kw_samesize,
 
   kw_eq,
   kw_ne,
   kw_slt,
   kw_sgt,
   kw_sle,
   kw_sge,
   kw_ult,
   kw_ugt,
   kw_ule,
   kw_uge,
   kw_oeq,
   kw_one,
   kw_olt,
   kw_ogt,
   kw_ole,
   kw_oge,
   kw_ord,
   kw_uno,
   kw_ueq,
   kw_une,
 
   // atomicrmw operations that aren't also instruction keywords.
   kw_xchg,
   kw_nand,
   kw_max,
   kw_min,
   kw_umax,
   kw_umin,
 
   // Instruction Opcodes (Opcode in UIntVal).
   kw_fneg,
   kw_add,
   kw_fadd,
   kw_sub,
   kw_fsub,
   kw_mul,
   kw_fmul,
   kw_udiv,
   kw_sdiv,
   kw_fdiv,
   kw_urem,
   kw_srem,
   kw_frem,
   kw_shl,
   kw_lshr,
   kw_ashr,
   kw_and,
   kw_or,
   kw_xor,
   kw_icmp,
   kw_fcmp,
 
   kw_phi,
   kw_call,
   kw_trunc,
   kw_zext,
   kw_sext,
   kw_fptrunc,
   kw_fpext,
   kw_uitofp,
   kw_sitofp,
   kw_fptoui,
   kw_fptosi,
   kw_inttoptr,
   kw_ptrtoint,
   kw_bitcast,
   kw_addrspacecast,
   kw_select,
   kw_va_arg,
 
   kw_landingpad,
   kw_personality,
   kw_cleanup,
   kw_catch,
   kw_filter,
 
   kw_ret,
   kw_br,
   kw_switch,
   kw_indirectbr,
   kw_invoke,
   kw_resume,
   kw_unreachable,
   kw_cleanupret,
   kw_catchswitch,
   kw_catchret,
   kw_catchpad,
   kw_cleanuppad,
   kw_callbr,
 
   kw_alloca,
   kw_load,
   kw_store,
   kw_fence,
   kw_cmpxchg,
   kw_atomicrmw,
   kw_getelementptr,
 
   kw_extractelement,
   kw_insertelement,
   kw_shufflevector,
   kw_extractvalue,
   kw_insertvalue,
   kw_blockaddress,
   kw_dso_local_equivalent,
 
   kw_freeze,
 
   // Metadata types.
   kw_distinct,
 
   // Use-list order directives.
   kw_uselistorder,
   kw_uselistorder_bb,
 
   // Summary index keywords
   kw_path,
   kw_hash,
   kw_gv,
   kw_guid,
   kw_name,
   kw_summaries,
   kw_flags,
   kw_blockcount,
   kw_linkage,
   kw_visibility,
   kw_notEligibleToImport,
   kw_live,
   kw_dsoLocal,
   kw_canAutoHide,
   kw_function,
   kw_insts,
   kw_funcFlags,
   kw_readNone,
   kw_readOnly,
   kw_noRecurse,
   kw_returnDoesNotAlias,
   kw_noInline,
   kw_alwaysInline,
   kw_calls,
   kw_callee,
   kw_params,
   kw_param,
   kw_hotness,
   kw_unknown,
   kw_hot,
   kw_critical,
   kw_relbf,
   kw_variable,
   kw_vTableFuncs,
   kw_virtFunc,
   kw_aliasee,
   kw_refs,
   kw_typeIdInfo,
   kw_typeTests,
   kw_typeTestAssumeVCalls,
   kw_typeCheckedLoadVCalls,
   kw_typeTestAssumeConstVCalls,
   kw_typeCheckedLoadConstVCalls,
   kw_vFuncId,
   kw_offset,
   kw_args,
   kw_typeid,
   kw_typeidCompatibleVTable,
   kw_summary,
   kw_typeTestRes,
   kw_kind,
   kw_unsat,
   kw_byteArray,
   kw_inline,
   kw_single,
   kw_allOnes,
   kw_sizeM1BitWidth,
   kw_alignLog2,
   kw_sizeM1,
   kw_bitMask,
   kw_inlineBits,
   kw_vcall_visibility,
   kw_wpdResolutions,
   kw_wpdRes,
   kw_indir,
   kw_singleImpl,
   kw_branchFunnel,
   kw_singleImplName,
   kw_resByArg,
   kw_byArg,
   kw_uniformRetVal,
   kw_uniqueRetVal,
   kw_virtualConstProp,
   kw_info,
   kw_byte,
   kw_bit,
   kw_varFlags,
 
   // Unsigned Valued tokens (UIntVal).
   LabelID,    // 42:
   GlobalID,   // @42
   LocalVarID, // %42
   AttrGrpID,  // #42
   SummaryID,  // ^42
 
   // String valued tokens (StrVal).
   LabelStr,         // foo:
   GlobalVar,        // @foo @"foo"
   ComdatVar,        // $foo
   LocalVar,         // %foo %"foo"
   MetadataVar,      // !foo
   StringConstant,   // "foo"
   DwarfTag,         // DW_TAG_foo
   DwarfAttEncoding, // DW_ATE_foo
   DwarfVirtuality,  // DW_VIRTUALITY_foo
   DwarfLang,        // DW_LANG_foo
   DwarfCC,          // DW_CC_foo
   EmissionKind,     // lineTablesOnly
   NameTableKind,    // GNU
   DwarfOp,          // DW_OP_foo
   DIFlag,           // DIFlagFoo
   DISPFlag,         // DISPFlagFoo
   DwarfMacinfo,     // DW_MACINFO_foo
   ChecksumKind,     // CSK_foo
 
   // Type valued tokens (TyVal).
   Type,
 
   APFloat, // APFloatVal
   APSInt   // APSInt
 };
 } // end namespace lltok
 } // end namespace llvm
 
 #endif

llvm/lib/AsmParser/LLLexer.cpp

 //===- LLLexer.cpp - Lexer for .ll Files ----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // Implement the Lexer for .ll files.
 //
 //===----------------------------------------------------------------------===//
 
-#include "LLLexer.h"
+#include "llvm/AsmParser/LLLexer.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/SourceMgr.h"

llvm/lib/AsmParser/LLParser.cpp

 //===-- LLParser.cpp - Parser Class ---------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file defines the parser class for .ll files.
 //
 //===----------------------------------------------------------------------===//
 
-#include "LLParser.h"
-#include "LLToken.h"
+#include "llvm/AsmParser/LLParser.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/AsmParser/LLToken.h"
 #include "llvm/AsmParser/SlotMapping.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/ConstantRange.h"

llvm/lib/AsmParser/Parser.cpp

 //===- Parser.cpp - Main dispatch module for the Parser library -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This library implements the functionality defined in llvm/AsmParser/Parser.h
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/AsmParser/Parser.h"
-#include "LLParser.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/AsmParser/LLParser.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSummaryIndex.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
 #include <cstring>
 #include <system_error>
 
 using namespace llvm;