diff --git a/libc/CMakeLists.txt b/libc/CMakeLists.txt --- a/libc/CMakeLists.txt +++ b/libc/CMakeLists.txt @@ -22,3 +22,4 @@ add_subdirectory(include) add_subdirectory(src) add_subdirectory(lib) +add_subdirectory(utils) diff --git a/libc/cmake/modules/LLVMLibCRules.cmake b/libc/cmake/modules/LLVMLibCRules.cmake --- a/libc/cmake/modules/LLVMLibCRules.cmake +++ b/libc/cmake/modules/LLVMLibCRules.cmake @@ -54,7 +54,7 @@ "ADD_GEN_HDR" "" # No optional arguments "DEF_FILE;GEN_HDR" # Single value arguments - "PARAMS;DATA_FILES" # Multi value arguments + "PARAMS;DATA_FILES;DEPENDS" # Multi value arguments ${ARGN} ) if(NOT ADD_GEN_HDR_DEF_FILE) @@ -76,21 +76,21 @@ set(replacement_params "") if(ADD_GEN_HDR_PARAMS) - list(APPEND replacement_params "-P" ${ADD_GEN_HDR_PARAMS}) + list(APPEND replacement_params "--args" ${ADD_GEN_HDR_PARAMS}) endif() set(gen_hdr_script "${LIBC_BUILD_SCRIPTS_DIR}/gen_hdr.py") add_custom_command( OUTPUT ${out_file} - COMMAND ${gen_hdr_script} -o ${out_file} ${in_file} ${replacement_params} + COMMAND $ -o ${out_file} --header ${ADD_GEN_HDR_GEN_HDR} --def ${in_file} ${replacement_params} -I ${LIBC_SOURCE_DIR} ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - DEPENDS ${in_file} ${fq_data_files} ${gen_hdr_script} + DEPENDS ${in_file} ${fq_data_files} ${LIBC_SOURCE_DIR}/config/${LIBC_TARGET_OS}/api.td libc-hdrgen ) add_custom_target( ${target_name} - DEPENDS ${out_file} + DEPENDS ${out_file} ${ADD_GEN_HDR_DEPENDS} ) endfunction(add_gen_header) diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td new file mode 100644 --- /dev/null +++ b/libc/config/linux/api.td @@ -0,0 +1,85 @@ +include "config/public_api.td" + +include "spec/stdc.td" + +def FloatT : TypeDecl<"float_t"> { + let Decl = [{ + #if __FLT_EVAL_METHOD__ == 1 + typedef float float_t + #elif __FLT_EVAL_METHOD__ == 2 + ... + #else + ... + #endif + }]; // This is only an example and not exactly how it will appear +} + +def SizeT : TypeDecl<"size_t"> { + let Decl = [{ + #define __need_size_t + #include + }]; +} + +def NullMacro : MacroDef<"NULL"> { + let Defn = [{ + #define __need_NULL + #include + }]; +} + +def MathAPI : PublicAPI<"math.h"> { + let Functions = [ + "acos", + "acosl", + ]; + + let TypeDeclarations = [ + FloatT, + ]; +} + +def StringAPI : PublicAPI<"string.h"> { + let Functions = [ + "memcpy", + "memmove", + "memcmp", + "memchr", + "memset", + "strcpy", + "strncpy", + "strcat", + "strncat", + "strcmp", + "strcoll", + "strncmp", + "strxfrm", + "strchr", + "strcspn", + "strpbrk", + "strrchr", + "strspn", + "strstr", + "strtok", + "strerror", + "strlen", + ]; + + let TypeDeclarations = [ + SizeT, + ]; + + let Macros = [ + NullMacro, + ]; +} + +def StdIoAPI : PublicAPI<"stdio.h"> { + let TypeDeclarations = [ + SizeT, + ]; + + let Functions = [ + "snprintf", + ]; +} diff --git a/libc/config/public_api.td b/libc/config/public_api.td new file mode 100644 --- /dev/null +++ b/libc/config/public_api.td @@ -0,0 +1,19 @@ +include "spec/spec.td" + +class TypeDecl { + string Name = name; + string Decl = ""; +} + +class MacroDef { + string Name = name; + string Defn = ""; +} + +class PublicAPI { + string HeaderName = name; + list Macros = []; + list TypeDeclarations = []; + list Structs = []; + list Functions = []; +} diff --git a/libc/docs/ground_truth_specification.rst b/libc/docs/ground_truth_specification.rst new file mode 100644 --- /dev/null +++ b/libc/docs/ground_truth_specification.rst @@ -0,0 +1,11 @@ +The ground truth of standards +============================= + +Like any modern libc, LLVM libc also supports a wide number of standards and +extensions. To avoid developing headers, wrappers and sources in a disjointed +fashion, LLVM libc employs ground truth files. These files live under the +``spec`` directory and list ground truth corresponding the ISO C standanrd, the +POSIX extension standard, etc. For example, the path to the ground truth file +for the ISO C standard is ``spec/stdc.td``. Tools like the header generator +(described in the header generation document), docs generator, etc. use the +ground truth files to generate headers, docs etc. diff --git a/libc/docs/header_gen_scheme.svg b/libc/docs/header_gen_scheme.svg new file mode 100644 --- /dev/null +++ b/libc/docs/header_gen_scheme.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/libc/docs/header_generation.rst b/libc/docs/header_generation.rst --- a/libc/docs/header_generation.rst +++ b/libc/docs/header_generation.rst @@ -96,3 +96,21 @@ The header generator will only include content starting from the line after the line on which this command is listed. + +``public_api`` +~~~~~~~~~~~~~~ + +This is a replacement command which should be listed in an input ``.h.def`` +file. The header file generator will replace this command with the public API of +the target platform. See the build system document for more information on the +relevant build rules. Also, see "Mechanics of public_api" to learn the mechanics +of how the header generator replaces this command with the public API. + +Arguments + + None. + +Action + + The header generator will replace this command with the public API to be exposed + from the generated header file. diff --git a/libc/docs/mechanics_of_public_api.rst b/libc/docs/mechanics_of_public_api.rst new file mode 100644 --- /dev/null +++ b/libc/docs/mechanics_of_public_api.rst @@ -0,0 +1,29 @@ +The mechanics of the ``public_api`` command +=========================================== + +The build system, in combination with the header generation mechanism, +facilitates the fine grained ability to pick and choose the public API one wants +to expose on their platform. The public header files are always generated from +the corresponding ``.h.def`` files. A header generation command ``%%public_api`` +is listed in these files. In the generated header file, the header generator +replaces this command with the public API relevant for the target platform. + +Under the hood +-------------- + +When the header generator sees the ``%%public_api`` command, it looks up the +API config file for the platform in the path ``config//api.td``. +The API config file lists two kinds of items: + +1. The list of standards from which the public entities available on the platform + are derived from. +2. For each header file exposed on the platfrom, the list of public members + provided in that header file. + +Note that, the header generator only learns the names of the public entities +from the header config file (the 2nd item from above.) The exact manner in which +the entities are to be declared is got from the standards (the 1st item from +above.) + +See the ground truth document for more information on how the standards are +formally listed in LLVM libc using LLVM table-gen files. diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt --- a/libc/include/CMakeLists.txt +++ b/libc/include/CMakeLists.txt @@ -21,10 +21,10 @@ llvm_libc_common_h ) -add_header( +add_gen_header( string_h - HDR - string.h + DEF_FILE string.h.def + GEN_HDR string.h DEPENDS llvm_libc_common_h ) diff --git a/libc/include/string.h b/libc/include/string.h deleted file mode 100644 --- a/libc/include/string.h +++ /dev/null @@ -1,66 +0,0 @@ -//===---------------- C standard library header string.h ------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIBC_STRING_H -#define LLVM_LIBC_STRING_H - -#include <__llvm-libc-common.h> - -#define __need_size_t // To get only size_t from stddef.h -#define __need_NULL // To get only NULL from stddef.h -#include - -__BEGIN_C_DECLS - -void *memcpy(void *__restrict, const void *__restrict, size_t); - -void *memmove(void *, const void *, size_t); - -int memcmp(const void *, const void *, size_t); - -void *memchr(const void *, int, size_t); - -void *memset(void *, int, size_t); - -char *strcpy(char *__restrict, const char *__restrict); - -char *strncpy(char *__restrict, const char *__restrict, size_t); - -char *strcat(char *__restrict, const char *__restrict); - -char *strncat(char *, const char *, size_t); - -int strcmp(const char *, const char *); - -int strcoll(const char *, const char *); - -int strncmp(const char *, const char *, size_t); - -size_t strxfrm(char *__restrict, const char *__restrict, size_t); - -char *strchr(const char *, int); - -size_t strcspn(const char *, const char *); - -char *strpbrk(const char *, const char *); - -char *strrchr(const char *, int c); - -size_t strspn(const char *, const char *); - -char *strstr(const char *, const char *); - -char *strtok(char *__restrict, const char *__restrict); - -char *strerror(int); - -size_t strlen(const char *); - -__END_C_DECLS - -#endif // LLVM_LIBC_STRING_H diff --git a/libc/include/string.h.def b/libc/include/string.h.def new file mode 100644 --- /dev/null +++ b/libc/include/string.h.def @@ -0,0 +1,16 @@ +//===---------------- C standard library header string.h ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_STRING_H +#define LLVM_LIBC_STRING_H + +#include <__llvm-libc-common.h> + +%%public_api() + +#endif // LLVM_LIBC_STRING_H diff --git a/libc/spec/spec.td b/libc/spec/spec.td new file mode 100644 --- /dev/null +++ b/libc/spec/spec.td @@ -0,0 +1,74 @@ +class Type {} + +class NamedType : Type { + string Name = name; +} + +class Field { + string Name = name; + Type FieldType = type; +} + +// Class to describe concrete structs specified by a standard. +class Struct : NamedType { + list Fields; +} + +class PtrType : Type { + Type PointeeType = type; +} + +class ConstType : Type { + Type UnqualifiedType = type; +} + +class RestrictedPtrType : Type { + Type PointeeType = type; +} + +// Builtin types. +def VarArgType : Type {} +def VoidType : NamedType<"void">; +def IntType : NamedType<"int">; +def FloatType : NamedType<"float">; +def DoubleType : NamedType<"double">; +def LongDoubleType : NamedType<"long double">; +def CharType : NamedType<"char">; + +class Macro { + string Name = name; +} + +class Annotation {} + +class RetValSpec annotations = []> { + Type ReturnType = type; + list Annotations = annotations; +} + +class ArgSpec annotations = [], string name = ""> { + Type ArgType = type; + list Annotations = annotations; + string Name = name; +} + +class FunctionSpec args> { + string Name = name; + RetValSpec Return = return; + list Args = args; +} + +class HeaderSpec macros, + list types, + list functions> { + string Name = name; + list Functions = functions; + list Types = types; + list Macros = macros; +} + +class StandardSpec { + string Name = name; + list Headers; +} diff --git a/libc/spec/stdc.td b/libc/spec/stdc.td new file mode 100644 --- /dev/null +++ b/libc/spec/stdc.td @@ -0,0 +1,178 @@ +def StdC : StandardSpec<"stdc"> { + PtrType VoidPtr = PtrType; + ConstType ConstVoidPtr = ConstType; + RestrictedPtrType VoidRestrictedPtr = RestrictedPtrType; + ConstType ConstVoidRestrictedPtr = ConstType; + + PtrType CharPtr = PtrType; + ConstType ConstCharPtr = ConstType; + RestrictedPtrType CharRestrictedPtr = RestrictedPtrType; + ConstType ConstCharRestrictedPtr = ConstType; + + NamedType SizeTType = NamedType<"size_t">; + + HeaderSpec String = HeaderSpec< + "string.h", + [ + Macro<"NULL">, + ], + [ + SizeTType, + ], + [ + FunctionSpec< + "memcpy", + RetValSpec, + [ArgSpec, + ArgSpec, + ArgSpec] + >, + FunctionSpec< + "memmove", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "memcmp", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "memchr", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "memset", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "strcpy", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strncpy", + RetValSpec, + [ArgSpec, + ArgSpec, + ArgSpec] + >, + FunctionSpec< + "strcat", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strncat", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "strcmp", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strcoll", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strncmp", + RetValSpec, + [ArgSpec, ArgSpec, ArgSpec] + >, + FunctionSpec< + "strxfrm", + RetValSpec, + [ArgSpec, + ArgSpec, + ArgSpec] + >, + FunctionSpec< + "strchr", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strcspn", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strpbrk", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strrchr", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strspn", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strstr", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strtok", + RetValSpec, + [ArgSpec, ArgSpec] + >, + FunctionSpec< + "strerror", + RetValSpec, + [ArgSpec] + >, + FunctionSpec< + "strlen", + RetValSpec, + [ArgSpec] + >, + ] + >; + + HeaderSpec Math = HeaderSpec< + "math.h", + [], // Macros + [ + NamedType<"float_t">, + NamedType<"double_t">, + ], + [ + FunctionSpec<"acos", RetValSpec, [ArgSpec]>, + FunctionSpec<"acosl", RetValSpec, [ArgSpec]>, + ] + >; + + HeaderSpec StdIO = HeaderSpec< + "stdio.h", + [], // Macros + [ // Types + SizeTType, + ], + [ + FunctionSpec< + "snprintf", + RetValSpec, + [ArgSpec, + ArgSpec, + ArgSpec, + ArgSpec] + >, + ] + >; + + let Headers = [ + Math, + String, + StdIO, + ]; +} diff --git a/libc/utils/CMakeLists.txt b/libc/utils/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/libc/utils/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(HdrGen) diff --git a/libc/utils/HdrGen/CMakeLists.txt b/libc/utils/HdrGen/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/CMakeLists.txt @@ -0,0 +1,10 @@ +add_tablegen(libc-hdrgen llvm-libc + Command.h + Generator.cpp + Generator.h + IncludeFileCommand.cpp + IncludeFileCommand.h + Main.cpp + PublicAPICommand.cpp + PublicAPICommand.h +) diff --git a/libc/utils/HdrGen/Command.h b/libc/utils/HdrGen/Command.h new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/Command.h @@ -0,0 +1,52 @@ +//===-------- Base class for header generation commands ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_UTILS_HDRGEN_COMMAND_H +#define LLVM_LIBC_UTILS_HDRGEN_COMMAND_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/SourceMgr.h" + +#include + +namespace llvm { + +class raw_ostream; +class RecordKeeper; + +} // namespace llvm + +namespace llvm_libc { + +typedef llvm::SmallVector ArgVector; + +class Command { +public: + class ErrorReporter { + llvm::SMLoc Loc; + const llvm::SourceMgr &SrcMgr; + + public: + ErrorReporter(llvm::SMLoc L, llvm::SourceMgr &SM) : Loc(L), SrcMgr(SM) {} + + void printFatalError(llvm::Twine Msg) const { + SrcMgr.PrintMessage(Loc, llvm::SourceMgr::DK_Error, Msg); + std::exit(1); + } + }; + + virtual void run(llvm::raw_ostream &OS, const ArgVector &Args, + llvm::StringRef StdHeader, llvm::RecordKeeper &Records, + const ErrorReporter &Reporter) const = 0; +}; + +} // namespace llvm_libc + +#endif // LLVM_LIBC_UTILS_HDRGEN_COMMAND_H diff --git a/libc/utils/HdrGen/Generator.h b/libc/utils/HdrGen/Generator.h new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/Generator.h @@ -0,0 +1,56 @@ +//===------------- - The main header generation class -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H +#define LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H + +#include "Command.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +#include +#include +#include + +namespace llvm { + +class raw_ostream; +class RecordKeeper; + +} // namespace llvm + +namespace llvm_libc { + +class Command; + +class Generator { + llvm::StringRef HeaderDefFile; + llvm::StringRef StdHeader; + std::unordered_map &ArgMap; + + std::unique_ptr IncludeFileCmd; + std::unique_ptr PublicAPICmd; + + Command *getCommandHandler(llvm::StringRef CommandName); + + void parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args); + + void printError(llvm::StringRef Msg); + +public: + Generator(const std::string &DefFile, const std::string &Header, + std::unordered_map &Map) + : HeaderDefFile(DefFile), StdHeader(Header), ArgMap(Map) {} + + void generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records); +}; + +} // namespace llvm_libc + +#endif // LLVM_LIBC_UTILS_HDRGEN_GENERATOR_H diff --git a/libc/utils/HdrGen/Generator.cpp b/libc/utils/HdrGen/Generator.cpp new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/Generator.cpp @@ -0,0 +1,122 @@ +//===---- Implementation of the main header generation class -----*- C++ -*===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Generator.h" + +#include "IncludeFileCommand.h" +#include "PublicAPICommand.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include + +static const char CommandPrefix[] = "%%"; +static const size_t CommandPrefixSize = llvm::StringRef(CommandPrefix).size(); + +static const char CommentPrefix[] = ""; + +static const char ParamNamePrefix[] = "${"; +static const size_t ParamNamePrefixSize = + llvm::StringRef(ParamNamePrefix).size(); +static const char ParamNameSuffix[] = "}"; +static const size_t ParamNameSuffixSize = + llvm::StringRef(ParamNameSuffix).size(); + +namespace llvm_libc { + +Command *Generator::getCommandHandler(llvm::StringRef CommandName) { + if (CommandName == IncludeFileCommand::Name) { + if (!IncludeFileCmd) { + IncludeFileCmd = std::make_unique(); + } + return IncludeFileCmd.get(); + } else if (CommandName == PublicAPICommand::Name) { + if (!PublicAPICmd) { + PublicAPICmd = std::make_unique(); + } + return PublicAPICmd.get(); + } else { + return nullptr; + } +} + +void Generator::parseCommandArgs(llvm::StringRef ArgStr, ArgVector &Args) { + if (!ArgStr.contains(',') && ArgStr.trim(' ').trim('\t').size() == 0) { + // If it is just space between the parenthesis + return; + } + + ArgStr.split(Args, ","); + for (llvm::StringRef &A : Args) { + A = A.trim(' '); + if (A.startswith(ParamNamePrefix) and A.endswith(ParamNameSuffix)) { + A = A.drop_front(ParamNamePrefixSize).drop_back(ParamNameSuffixSize); + A = ArgMap[A]; + } + } +} + +void Generator::generate(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) { + auto DefFileBuffer = llvm::MemoryBuffer::getFile(HeaderDefFile); + if (!DefFileBuffer) { + llvm::errs() << "Unable to open " << HeaderDefFile << ".\n"; + std::exit(1); + } + llvm::SourceMgr SrcMgr; + unsigned DefFileID = SrcMgr.AddNewSourceBuffer( + std::move(DefFileBuffer.get()), llvm::SMLoc::getFromPointer(nullptr)); + + llvm::StringRef Content = SrcMgr.getMemoryBuffer(DefFileID)->getBuffer(); + while (true) { + std::pair P = Content.split('\n'); + Content = P.second; + + llvm::StringRef Line = P.first.trim(' '); + if (Line.startswith(CommandPrefix)) { + Line = Line.drop_front(CommandPrefixSize); + + P = Line.split("("); + if (P.second.empty() || P.second[P.second.size() - 1] != ')') { + SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(P.second.data()), + llvm::SourceMgr::DK_Error, + "Command argument list should begin with '(' " + "and end with ')'."); + std::exit(1); + } + llvm::StringRef CommandName = P.first; + Command *Cmd = getCommandHandler(CommandName); + if (Cmd == nullptr) { + SrcMgr.PrintMessage(llvm::SMLoc::getFromPointer(CommandName.data()), + llvm::SourceMgr::DK_Error, + "Unknown command '%%" + CommandName + "'."); + std::exit(1); + } + + llvm::StringRef ArgStr = P.second.drop_back(1); + ArgVector Args; + parseCommandArgs(ArgStr, Args); + + Command::ErrorReporter Reporter( + llvm::SMLoc::getFromPointer(CommandName.data()), SrcMgr); + Cmd->run(OS, Args, StdHeader, Records, Reporter); + } else if (!Line.startswith(CommentPrefix)) { + // There is no comment or command on this line so we just write it as is. + OS << P.first << "\n"; + } + + if (P.second.empty()) { + break; + } + } +} + +} // namespace llvm_libc diff --git a/libc/utils/HdrGen/IncludeFileCommand.h b/libc/utils/HdrGen/IncludeFileCommand.h new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/IncludeFileCommand.h @@ -0,0 +1,32 @@ +//===-------- Class which implements the %%include_file command -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H +#define LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H + +#include "Command.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" + +#include + +namespace llvm_libc { + +class IncludeFileCommand : public Command { +public: + static const char Name[]; + + void run(llvm::raw_ostream &OS, const ArgVector &Args, + llvm::StringRef StdHeader, llvm::RecordKeeper &Records, + const Command::ErrorReporter &Reporter) const override; +}; + +} // namespace llvm_libc + +#endif // LLVM_LIBC_UTILS_HDRGEN_INCLUDE_COMMAND_H diff --git a/libc/utils/HdrGen/IncludeFileCommand.cpp b/libc/utils/HdrGen/IncludeFileCommand.cpp new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/IncludeFileCommand.cpp @@ -0,0 +1,51 @@ +//===----------- Implementation of IncludeFileCommand -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "IncludeFileCommand.h" + +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SourceMgr.h" + +#include + +namespace llvm_libc { + +const char IncludeFileCommand::Name[] = "include_file"; + +void IncludeFileCommand::run(llvm::raw_ostream &OS, const ArgVector &Args, + llvm::StringRef StdHeader, + llvm::RecordKeeper &Records, + const Command::ErrorReporter &Reporter) const { + if (Args.size() != 1) { + Reporter.printFatalError( + "%%include_file command takes exactly 1 argument."); + } + + llvm::StringRef IncludeFile = Args[0]; + auto Buffer = llvm::MemoryBuffer::getFileAsStream(IncludeFile); + if (!Buffer) { + Reporter.printFatalError(llvm::StringRef("Unable to open ") + IncludeFile); + } + + llvm::StringRef Content = Buffer.get()->getBuffer(); + + // If the included file has %%begin() command listed, then we want to write + // only the content after the begin command. + // TODO: The way the content is split below does not allow space within the + // the parentheses and, before and after the command. This probably is too + // strict and should be relaxed. + auto P = Content.split("\n%%begin()\n"); + if (P.second.empty()) { + // There was no %%begin in the content. + OS << P.first; + } else { + OS << P.second; + } +} + +} // namespace llvm_libc diff --git a/libc/utils/HdrGen/Main.cpp b/libc/utils/HdrGen/Main.cpp new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/Main.cpp @@ -0,0 +1,56 @@ +//===---------------- "main" function of libc-hdrgen ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Generator.h" + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/TableGen/Main.h" + +#include +#include + +namespace { + +llvm::cl::opt + HeaderDefFile("def", llvm::cl::desc("Path to the .h.def file."), + llvm::cl::value_desc(""), llvm::cl::Required); +llvm::cl::opt StandardHeader( + "header", + llvm::cl::desc("The standard header file which is to be generated."), + llvm::cl::value_desc("
")); +llvm::cl::list ReplacementValues( + "args", llvm::cl::desc("Command seperated = pairs."), + llvm::cl::value_desc("[,name=value]")); + +void ParseArgValuePairs(std::unordered_map &Map) { + for (std::string &R : ReplacementValues) { + auto Pair = llvm::StringRef(R).split('='); + Map[Pair.first] = Pair.second; + } +} + +} // anonymous namespace + +namespace llvm_libc { + +bool HeaderGeneratorMain(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) { + std::unordered_map ArgMap; + ParseArgValuePairs(ArgMap); + Generator G(HeaderDefFile, StandardHeader, ArgMap); + G.generate(OS, Records); + + return false; +} + +} // namespace llvm_libc + +int main(int argc, char *argv[]) { + llvm::cl::ParseCommandLineOptions(argc, argv); + return TableGenMain(argv[0], &llvm_libc::HeaderGeneratorMain); +} diff --git a/libc/utils/HdrGen/PublicAPICommand.h b/libc/utils/HdrGen/PublicAPICommand.h new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/PublicAPICommand.h @@ -0,0 +1,36 @@ +//===---------- Implementation of PublicAPICommand --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Command.h" + +#include "llvm/ADT/StringRef.h" + +#include +#include +#include + +namespace llvm { + +class raw_ostream; +class Record; +class RecordKeeper; + +} // namespace llvm + +namespace llvm_libc { + +class PublicAPICommand : public Command { +public: + static const char Name[]; + + void run(llvm::raw_ostream &OS, const ArgVector &Args, + llvm::StringRef StdHeader, llvm::RecordKeeper &Records, + const Command::ErrorReporter &Reporter) const override; +}; + +} // namespace llvm_libc diff --git a/libc/utils/HdrGen/PublicAPICommand.cpp b/libc/utils/HdrGen/PublicAPICommand.cpp new file mode 100644 --- /dev/null +++ b/libc/utils/HdrGen/PublicAPICommand.cpp @@ -0,0 +1,272 @@ +//===--------------- Implementation of PublicAPICommand ----------*-C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "PublicAPICommand.h" + +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +static const char NamedTypeClassName[] = "NamedType"; +static const char PtrTypeClassName[] = "PtrType"; +static const char RestrictedPtrTypeClassName[] = "RestrictedPtrType"; +static const char ConstTypeClassName[] = "ConstType"; +static const char StructTypeClassName[] = "Struct"; + +static const char StandardSpecClassName[] = "StandardSpec"; +static const char PublicAPIClassName[] = "PublicAPI"; + +static bool isa(llvm::Record *Def, llvm::Record *TypeClass) { + llvm::RecordRecTy *RecordType = Def->getType(); + llvm::ArrayRef Classes = RecordType->getClasses(); + // We want exact types. That is, we don't want the classes listed in + // spec.td to be subclassed. Hence, we do not want the record |Def| + // to be of more than one class type.. + if (Classes.size() != 1) { + return false; + } + return Classes[0] == TypeClass; +} + +// Text blocks for macro definitions and type decls can be indented to +// suit the surrounding tablegen listing. We need to dedent such blocks +// before writing them out. +static void dedentAndWrite(llvm::StringRef Text, llvm::raw_ostream &OS) { + llvm::SmallVector Lines; + llvm::SplitString(Text, Lines, "\n\r"); + size_t shortest_indent = 1024; + for (llvm::StringRef L : Lines) { + llvm::StringRef Indent = L.take_while([](char c) { return c == ' '; }); + size_t IndentSize = Indent.size(); + if (Indent.size() == L.size()) { + // Line is all spaces so no point noting the indent. + continue; + } + if (IndentSize < shortest_indent) { + shortest_indent = IndentSize; + } + } + for (llvm::StringRef L : Lines) { + if (L.size() >= shortest_indent) { + OS << L.drop_front(shortest_indent) << "\n"; + } else { + OS << L << "\n"; + } + } +} + +class APIGenerator { + llvm::StringRef StdHeader; + + // TableGen classes in spec.td. + llvm::Record *NamedTypeClass; + llvm::Record *PtrTypeClass; + llvm::Record *RestrictedPtrTypeClass; + llvm::Record *ConstTypeClass; + llvm::Record *StructClass; + llvm::Record *StandardSpecClass; + llvm::Record *PublicAPIClass; + + using NameToRecordMapping = std::unordered_map; + using NameSet = std::unordered_set; + + // Mapping from names to records defining them. + NameToRecordMapping MacroSpecMap; + NameToRecordMapping TypeSpecMap; + NameToRecordMapping FunctionSpecMap; + NameToRecordMapping MacroDefsMap; + NameToRecordMapping TypeDeclsMap; + + NameSet Structs; + NameSet Functions; + + bool isaNamedType(llvm::Record *Def) { return isa(Def, NamedTypeClass); } + + bool isaStructType(llvm::Record *Def) { return isa(Def, StructClass); } + + bool isaPtrType(llvm::Record *Def) { return isa(Def, PtrTypeClass); } + + bool isaConstType(llvm::Record *Def) { return isa(Def, ConstTypeClass); } + + bool isaRestrictedPtrType(llvm::Record *Def) { + return isa(Def, RestrictedPtrTypeClass); + } + + bool isaStandardSpec(llvm::Record *Def) { + return isa(Def, StandardSpecClass); + } + + bool isaPublicAPI(llvm::Record *Def) { return isa(Def, PublicAPIClass); } + + std::string getTypeAsString(llvm::Record *TypeRecord) { + if (isaNamedType(TypeRecord) || isaStructType(TypeRecord)) { + return TypeRecord->getValueAsString("Name"); + } else if (isaPtrType(TypeRecord)) { + return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + " *"; + } else if (isaConstType(TypeRecord)) { + return std::string("const ") + + getTypeAsString(TypeRecord->getValueAsDef("UnqualifiedType")); + } else if (isaRestrictedPtrType(TypeRecord)) { + return getTypeAsString(TypeRecord->getValueAsDef("PointeeType")) + + " *__restrict"; + } else { + llvm::PrintFatalError(TypeRecord->getLoc(), "Invalid type.\n"); + } + } + + void indexStandardSpecDef(llvm::Record *StandardSpec) { + auto HeaderSpecList = StandardSpec->getValueAsListOfDefs("Headers"); + for (llvm::Record *HeaderSpec : HeaderSpecList) { + if (HeaderSpec->getValueAsString("Name") == StdHeader) { + auto MacroSpecList = HeaderSpec->getValueAsListOfDefs("Macros"); + // TODO: Trigger a fatal error on duplicate specs. + for (llvm::Record *MacroSpec : MacroSpecList) { + MacroSpecMap[MacroSpec->getValueAsString("Name")] = MacroSpec; + } + + auto TypeSpecList = HeaderSpec->getValueAsListOfDefs("Types"); + for (llvm::Record *TypeSpec : TypeSpecList) { + TypeSpecMap[TypeSpec->getValueAsString("Name")] = TypeSpec; + } + + auto FunctionSpecList = HeaderSpec->getValueAsListOfDefs("Functions"); + for (llvm::Record *FunctionSpec : FunctionSpecList) { + FunctionSpecMap[FunctionSpec->getValueAsString("Name")] = + FunctionSpec; + } + } + } + } + + void indexPublicAPIDef(llvm::Record *PublicAPI) { + // While indexing the public API, we do not check if any of the entities + // requested is from an included standard. Such a check is done while + // generating the API. + auto MacroDefList = PublicAPI->getValueAsListOfDefs("Macros"); + for (llvm::Record *MacroDef : MacroDefList) { + MacroDefsMap[MacroDef->getValueAsString("Name")] = MacroDef; + } + + auto TypeDeclList = PublicAPI->getValueAsListOfDefs("TypeDeclarations"); + for (llvm::Record *TypeDecl : TypeDeclList) { + TypeDeclsMap[TypeDecl->getValueAsString("Name")] = TypeDecl; + } + + auto StructList = PublicAPI->getValueAsListOfStrings("Structs"); + for (llvm::StringRef StructName : StructList) { + Structs.insert(StructName); + } + + auto FunctionList = PublicAPI->getValueAsListOfStrings("Functions"); + for (llvm::StringRef FunctionName : FunctionList) { + Functions.insert(FunctionName); + } + } + + void index(llvm::RecordKeeper &Records) { + NamedTypeClass = Records.getClass(NamedTypeClassName); + PtrTypeClass = Records.getClass(PtrTypeClassName); + RestrictedPtrTypeClass = Records.getClass(RestrictedPtrTypeClassName); + StructClass = Records.getClass(StructTypeClassName); + ConstTypeClass = Records.getClass(ConstTypeClassName); + StandardSpecClass = Records.getClass(StandardSpecClassName); + PublicAPIClass = Records.getClass(PublicAPIClassName); + + const auto &DefsMap = Records.getDefs(); + for (auto &Pair : DefsMap) { + llvm::Record *Def = Pair.second.get(); + if (isaStandardSpec(Def)) { + indexStandardSpecDef(Def); + } + if (isaPublicAPI(Def)) { + if (Def->getValueAsString("HeaderName") == StdHeader) { + indexPublicAPIDef(Def); + } + } + } + } + +public: + APIGenerator(llvm::StringRef Header, llvm::RecordKeeper &Records) + : StdHeader(Header) { + index(Records); + } + + void write(llvm::raw_ostream &OS) { + for (auto &Pair : MacroDefsMap) { + const std::string &Name = Pair.first; + if (MacroSpecMap.find(Name) == MacroSpecMap.end()) { + llvm::PrintFatalError(Name + " not found in any standard spec.\n"); + } + + llvm::Record *MacroDef = Pair.second; + dedentAndWrite(MacroDef->getValueAsString("Defn"), OS); + + OS << "\n"; + } + + for (auto &Pair : TypeDeclsMap) { + const std::string &Name = Pair.first; + if (TypeSpecMap.find(Name) == TypeSpecMap.end()) { + llvm::PrintFatalError(Name + " not found in any standard spec.\n"); + } + + llvm::Record *TypeDecl = Pair.second; + dedentAndWrite(TypeDecl->getValueAsString("Decl"), OS); + + OS << "\n"; + } + + OS << "__BEGIN_C_DECLS\n\n"; + for (auto &Name : Functions) { + if (FunctionSpecMap.find(Name) == FunctionSpecMap.end()) { + llvm::PrintFatalError(Name + " not found in any standard spec.\n"); + } + + llvm::Record *FunctionSpec = FunctionSpecMap[Name]; + llvm::Record *RetValSpec = FunctionSpec->getValueAsDef("Return"); + llvm::Record *ReturnType = RetValSpec->getValueAsDef("ReturnType"); + + OS << getTypeAsString(ReturnType) << " " << Name << "("; + + auto ArgsList = FunctionSpec->getValueAsListOfDefs("Args"); + for (size_t i = 0; i < ArgsList.size(); ++i) { + llvm::Record *ArgType = ArgsList[i]->getValueAsDef("ArgType"); + OS << getTypeAsString(ArgType); + if (i < ArgsList.size() - 1) { + OS << ", "; + } + } + + OS << ");\n\n"; + } + OS << "__END_C_DECLS\n"; + } +}; + +namespace llvm_libc { + +void writePublicAPI(llvm::raw_ostream &OS, llvm::RecordKeeper &Records) {} + +const char PublicAPICommand::Name[] = "public_api"; + +void PublicAPICommand::run(llvm::raw_ostream &OS, const ArgVector &Args, + llvm::StringRef StdHeader, + llvm::RecordKeeper &Records, + const Command::ErrorReporter &Reporter) const { + if (Args.size() != 0) { + Reporter.printFatalError("public_api command does not take any arguments."); + } + + APIGenerator G(StdHeader, Records); + G.write(OS); +} + +} // namespace llvm_libc diff --git a/libc/utils/build_scripts/gen_hdr.py b/libc/utils/build_scripts/gen_hdr.py deleted file mode 100755 --- a/libc/utils/build_scripts/gen_hdr.py +++ /dev/null @@ -1,188 +0,0 @@ -#! /usr/bin/python -#===---------------- Script to generate header files ----------------------===# -# -# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -# See https:#llvm.org/LICENSE.txt for license information. -# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# -#===-----------------------------------------------------------------------===# -# -# This script takes a .h.def file and generates a .h header file. -# See docs/header_generation.md for more information. -# -#===-----------------------------------------------------------------------===# - -import argparse -import contextlib -import os -import sys - -COMMAND_PREFIX = "%%" -COMMENT_PREFIX = "" - -BEGIN_COMMAND = "begin" -COMMENT_COMMAND = "comment" -INCLUDE_FILE_COMMAND = "include_file" - - -class _Location(object): - def __init__(self, filename, line_number): - self.filename = filename - self.line_number = line_number - - def __str__(self): - return "%s:%s" % (self.filename, self.line_number) - - -@contextlib.contextmanager -def output_stream_manager(filename): - if filename is None: - try: - yield sys.stdout - finally: - pass - else: - output_stream = open(filename, "w") - try: - yield output_stream - finally: - output_stream.close() - - -def _parse_command(loc, line): - open_paren = line.find("(") - if open_paren < 0 or line[-1] != ")": - return _fatal_error(loc, "Incorrect header generation command syntax.") - command_name = line[len(COMMAND_PREFIX):open_paren] - args = line[open_paren + 1:-1].split(",") - args = [a.strip() for a in args] - if len(args) == 1 and not args[0]: - # There are no args, so we will make the args list an empty list. - args = [] - return command_name.strip(), args - - -def _is_named_arg(token): - if token.startswith("${") and token.endswith("}"): - return True - else: - return False - - -def _get_arg_name(token): - return token[2:-1] - - -def _fatal_error(loc, msg): - sys.exit("ERROR:%s: %s" % (loc, msg)) - - -def _is_begin_command(line): - if line.startswith(COMMAND_PREFIX + BEGIN_COMMAND): - return True - - -def include_file_command(out_stream, loc, args, values): - if len(args) != 1: - _fatal_error(loc, "`%%include_file` command takes exactly one " - "argument. %d given." % len(args)) - include_file_path = args[0] - if _is_named_arg(include_file_path): - arg_name = _get_arg_name(include_file_path) - include_file_path = values.get(arg_name) - if not include_file_path: - _fatal_error( - loc, - "No value specified for argument '%s'." % arg_name) - if not os.path.exists(include_file_path): - _fatal_error( - loc, - "Include file %s not found." % include_file_path) - with open(include_file_path, "r") as include_file: - begin = False - for line in include_file.readlines(): - line = line.strip() - if _is_begin_command(line): - # Parse the command to make sure there are no errors. - command_name, args = _parse_command(loc, line) - if args: - _fatal_error(loc, "Begin command does not take any args.") - begin = True - # Skip the line on which %%begin() is listed. - continue - if begin: - out_stream.write(line + "\n") - - -def begin_command(out_stream, loc, args, values): - # "begin" command can only occur in a file included with %%include_file - # command. It is not a replacement command. Hence, we just fail with - # a fatal error. - _fatal_error(loc, "Begin command cannot be listed in an input file.") - - -# Mapping from a command name to its implementation function. -REPLACEMENT_COMMANDS = { - INCLUDE_FILE_COMMAND: include_file_command, - BEGIN_COMMAND: begin_command, -} - - -def apply_replacement_command(out_stream, loc, line, values): - if not line.startswith(COMMAND_PREFIX): - # This line is not a replacement command. - return line - command_name, args = _parse_command(loc, line) - command = REPLACEMENT_COMMANDS.get(command_name.strip()) - if not command: - _fatal_error(loc, "Unknown replacement command `%`", command_name) - command(out_stream, loc, args, values) - - -def parse_options(): - parser = argparse.ArgumentParser( - description="Script to generate header files from .def files.") - parser.add_argument("def_file", metavar="DEF_FILE", - help="Path to the .def file.") - parser.add_argument("--args", "-P", nargs= "*", default=[], - help="NAME=VALUE pairs for command arguments in the " - "input .def file.") - # The output file argument is optional. If not specified, the generated - # header file content will be written to stdout. - parser.add_argument("--out-file", "-o", - help="Path to the generated header file. Defaults to " - "stdout") - opts = parser.parse_args() - if not all(["=" in arg for arg in opts.args]): - # We want all args to be specified in the form "name=value". - _fatal_error( - __file__ + ":" + "[command line]", - "Command arguments should be listed in the form NAME=VALUE") - return opts - - -def main(): - opts = parse_options() - arg_values = {} - for name_value_pair in opts.args: - name, value = name_value_pair.split("=") - arg_values[name] = value - with open(opts.def_file, "r") as def_file: - loc = _Location(opts.def_file, 0) - with output_stream_manager(opts.out_file) as out_stream: - for line in def_file: - loc.line_number += 1 - line = line.strip() - if line.startswith(COMMAND_PREFIX): - replacement_text = apply_replacement_command( - out_stream, loc, line, arg_values) - out_stream.write("\n") - elif line.startswith(COMMENT_PREFIX): - # Ignore comment line - continue - else: - out_stream.write(line + "\n") - - -if __name__ == "__main__": - main()