diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt --- a/lld/test/CMakeLists.txt +++ b/lld/test/CMakeLists.txt @@ -28,7 +28,7 @@ FileCheck count llc llvm-ar llvm-as llvm-bcanalyzer llvm-config llvm-cvtres llvm-dis llvm-dwarfdump llvm-lib llvm-lipo llvm-mc llvm-nm llvm-objcopy llvm-objdump llvm-pdbutil llvm-readelf llvm-readobj llvm-strip not obj2yaml - opt yaml2obj + opt split-file yaml2obj ) endif() diff --git a/lld/test/ELF/linkerscript/noload.s b/lld/test/ELF/linkerscript/noload.s --- a/lld/test/ELF/linkerscript/noload.s +++ b/lld/test/ELF/linkerscript/noload.s @@ -1,12 +1,8 @@ # REQUIRES: x86 -# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o -# RUN: echo "SECTIONS { \ -# RUN: .data_noload_a (NOLOAD) : { *(.data_noload_a) } \ -# RUN: .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } \ -# RUN: .no_input_sec_noload (NOLOAD) : { . += 1; } \ -# RUN: .text (0x20000) : { *(.text) } };" > %t.script -# RUN: ld.lld -o %t --script %t.script %t.o -# RUN: llvm-readelf -S -l %t | FileCheck %s +# RUN: split-file %s %t +# RUN: llvm-mc -filetype=obj -triple=x86_64 %t/asm -o %t.o +# RUN: ld.lld --script %t/lds %t.o -o %t/out +# RUN: llvm-readelf -S -l %t/out | FileCheck %s # CHECK: Name Type Address Off Size # CHECK: .data_noload_a NOBITS 0000000000000000 [[OFF:[0-9a-f]+]] 001000 @@ -16,6 +12,7 @@ # CHECK: Type Offset VirtAddr PhysAddr # CHECK-NEXT: LOAD 0x001000 0x0000000000020000 0x0000000000020000 +#--- asm .section .text,"ax",@progbits nop @@ -24,3 +21,11 @@ .section .data_noload_b,"aw",@progbits .zero 4096 + +#--- lds +SECTIONS { + .data_noload_a (NOLOAD) : { *(.data_noload_a) } + .data_noload_b (0x10000) (NOLOAD) : { *(.data_noload_b) } + .no_input_sec_noload (NOLOAD) : { . += 1; } + .text (0x20000) : { *(.text) } +} diff --git a/llvm/docs/TestingGuide.rst b/llvm/docs/TestingGuide.rst --- a/llvm/docs/TestingGuide.rst +++ b/llvm/docs/TestingGuide.rst @@ -271,8 +271,27 @@ Extra files ----------- -If your test requires extra files besides the file containing the ``RUN:`` -lines, the idiomatic place to put them is in a subdirectory ``Inputs``. +If your test requires extra files besides the file containing the ``RUN:`` lines +and the extra files are small, consider specifying them in the same file and +using ``split-file`` to extract them. For example, + +.. code-block:: llvm + + ; RUN: split-file %s %t + ; RUN: llvm-link -S %t/a.ll %t/b.ll | FileCheck %s + + ; CHECK: ... + + ;--- a.ll + ... + ;--- b.ll + ... + +The parts are separated by the regex ``^(.|//)--- ``. By default the +extracted content has leading empty lines to preserve line numbers. Specify +``--no-leading-lines`` to drop leading lines. + +If the extra files are large, the idiomatic place to put them is in a subdirectory ``Inputs``. You can then refer to the extra files as ``%S/Inputs/foo.bar``. For example, consider ``test/Linker/ident.ll``. The directory structure is diff --git a/llvm/test/CMakeLists.txt b/llvm/test/CMakeLists.txt --- a/llvm/test/CMakeLists.txt +++ b/llvm/test/CMakeLists.txt @@ -119,6 +119,7 @@ opt sancov sanstats + split-file verify-uselistorder yaml-bench yaml2obj diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py --- a/llvm/test/lit.cfg.py +++ b/llvm/test/lit.cfg.py @@ -141,6 +141,7 @@ ToolSubst('%llvm-objcopy', FindTool('llvm-objcopy')), ToolSubst('%llvm-strip', FindTool('llvm-strip')), ToolSubst('%llvm-install-name-tool', FindTool('llvm-install-name-tool')), + ToolSubst('%split-file', FindTool('split-file')), ] # FIXME: Why do we have both `lli` and `%lli` that do slightly different things? diff --git a/llvm/test/tools/gold/X86/multiple-sections.ll b/llvm/test/tools/gold/X86/multiple-sections.ll --- a/llvm/test/tools/gold/X86/multiple-sections.ll +++ b/llvm/test/tools/gold/X86/multiple-sections.ll @@ -1,10 +1,8 @@ -; RUN: echo ".text.tin" > %t_order_lto.txt -; RUN: echo ".text._start" >> %t_order_lto.txt -; RUN: echo ".text.pat" >> %t_order_lto.txt -; RUN: llvm-as %s -o %t.o +; RUN: split-file %s %t +; RUN: llvm-as %t/a.ll -o %t.o ; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \ ; RUN: -m elf_x86_64 -o %t.exe %t.o \ -; RUN: --section-ordering-file=%t_order_lto.txt +; RUN: --section-ordering-file=%t/order ; RUN: llvm-readelf -s %t.exe | FileCheck %s ; Check that the order of the sections is tin -> _start -> pat. @@ -13,6 +11,12 @@ ; CHECK: 00000000004000b0 1 FUNC LOCAL DEFAULT 1 tin ; CHECK: 00000000004000c0 15 FUNC GLOBAL DEFAULT 1 _start +;--- order +.text.tin +.text._start +.text.pat + +;--- a.ll target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/tools/llvm-strings/radix.test b/llvm/test/tools/llvm-strings/radix.test --- a/llvm/test/tools/llvm-strings/radix.test +++ b/llvm/test/tools/llvm-strings/radix.test @@ -1,29 +1,32 @@ ## Show that llvm-strings can handle the -t/--radix switch properly. -RUN: echo one > %t -RUN: echo two >> %t -RUN: echo three >> %t -RUN: echo four >> %t -RUN: echo five >> %t -RUN: echo six >> %t -RUN: echo seven >> %t -RUN: echo eight >> %t -RUN: echo nine >> %t -RUN: echo ten >> %t +RUN: split-file --no-leading-lines %s %t +#--- a.txt +one +two +three +four +five +six +seven +eight +nine +ten +#--- end -RUN: llvm-strings %t | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}} -RUN: llvm-strings -t d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}} -RUN: llvm-strings -t o %t | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace --implicit-check-not={{.}} -RUN: llvm-strings -t x %t | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace --implicit-check-not={{.}} +RUN: llvm-strings %t/a.txt | FileCheck %s -check-prefix CHECK-NONE --implicit-check-not={{.}} +RUN: llvm-strings -t d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace --implicit-check-not={{.}} +RUN: llvm-strings -t o %t/a.txt | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace --implicit-check-not={{.}} +RUN: llvm-strings -t x %t/a.txt | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace --implicit-check-not={{.}} ## Show --radix works too. -RUN: llvm-strings --radix d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace -RUN: llvm-strings --radix o %t | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace -RUN: llvm-strings --radix x %t | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace +RUN: llvm-strings --radix d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace +RUN: llvm-strings --radix o %t/a.txt | FileCheck %s -check-prefix CHECK-OCT --strict-whitespace +RUN: llvm-strings --radix x %t/a.txt | FileCheck %s -check-prefix CHECK-HEX --strict-whitespace ## Show different syntaxes work. -RUN: llvm-strings --radix=d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace -RUN: llvm-strings -t=d %t | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace +RUN: llvm-strings --radix=d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace +RUN: llvm-strings -t=d %t/a.txt | FileCheck %s -check-prefix CHECK-DEC --strict-whitespace CHECK-NONE: {{^}}three CHECK-NONE: {{^}}four @@ -54,5 +57,5 @@ CHECK-HEX: {{^}} 28 nine ## Show that an invalid value is rejected. -RUN: not llvm-strings --radix z %t 2>&1 | FileCheck %s --check-prefix=INVALID +RUN: not llvm-strings --radix z %t/a.txt 2>&1 | FileCheck %s --check-prefix=INVALID INVALID: llvm-strings{{.*}}: for the --radix option: Cannot find option named 'z'! diff --git a/llvm/test/tools/split-file/Inputs/basic-aa.txt b/llvm/test/tools/split-file/Inputs/basic-aa.txt new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/Inputs/basic-aa.txt @@ -0,0 +1,2 @@ + +aa diff --git a/llvm/test/tools/split-file/Inputs/basic-bb.txt b/llvm/test/tools/split-file/Inputs/basic-bb.txt new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/Inputs/basic-bb.txt @@ -0,0 +1,6 @@ + + + +; Comments are preserved. +bb + diff --git a/llvm/test/tools/split-file/Inputs/basic-cc.txt b/llvm/test/tools/split-file/Inputs/basic-cc.txt new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/Inputs/basic-cc.txt @@ -0,0 +1,8 @@ + + + + + + + +cc diff --git a/llvm/test/tools/split-file/basic.test b/llvm/test/tools/split-file/basic.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/basic.test @@ -0,0 +1,40 @@ +#--- aa +aa +;--- bb +; Comments are preserved. +bb + +//--- subdir/cc +cc +//--- end + +# RUN: rm -rf %t +# RUN: split-file %s %t +# RUN: diff %S/Inputs/basic-aa.txt %t/aa +# RUN: diff %S/Inputs/basic-bb.txt %t/bb +# RUN: diff %S/Inputs/basic-cc.txt %t/subdir/cc +# RUN: FileCheck %s --check-prefix=END < %t/end + +## Can be called on a non-empty directory. +# RUN: split-file %s %t +# RUN: diff %S/Inputs/basic-aa.txt %t/aa + +## Test that we will delete the output if it is a file, so that we can create +## a directory. +# RUN: rm -rf %t && touch %t +# RUN: split-file %s %t +# RUN: diff %S/Inputs/basic-aa.txt %t/aa + +# END: RUN: split-file %s %t + +# RUN: not %split-file 2>&1 | FileCheck %s --check-prefix=NO_INPUT + +# NO_INPUT: split-file: error: input filename is not specified + +# RUN: not %split-file %s '' 2>&1 | FileCheck %s --check-prefix=NO_OUTPUT + +# NO_OUTPUT: split-file: error: output directory is not specified + +# RUN: not %split-file %S/Inputs/basic-aa.txt %t 2>&1 | FileCheck %s --check-prefix=NOT_EXIST + +# NOT_EXIST: split-file: error: {{.*}}.txt: no part separator was found diff --git a/llvm/test/tools/split-file/empty.test b/llvm/test/tools/split-file/empty.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/empty.test @@ -0,0 +1,4 @@ +# RUN: split-file --no-leading-lines %s %t +# RUN: count 0 < %t/empty + +#--- empty diff --git a/llvm/test/tools/split-file/error.test b/llvm/test/tools/split-file/error.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/error.test @@ -0,0 +1,16 @@ +# RUN: not %split-file %s %t 2>&1 | FileCheck %s +# RUN: not ls %t/dup + +# CHECK: {{.*}}.test:[[#@LINE+1]]: error: empty part name +//--- + +# CHECK: {{.*}}.test:[[#@LINE+1]]: error: part name cannot have leading or trailing space +//--- leading_space + +# CHECK: {{.*}}.test:[[#@LINE+1]]: error: part name cannot have leading or trailing space +//--- trailing_space + +;--- dup + +# CHECK: {{.*}}.test:[[#@LINE+1]]: error: ';--- dup' occurs more than once +;--- dup diff --git a/llvm/test/tools/split-file/help.test b/llvm/test/tools/split-file/help.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/help.test @@ -0,0 +1,6 @@ +RUN: split-file --help 2>&1 | FileCheck --implicit-check-not='General Options:' %s +CHECK: OVERVIEW: Split input {{.*}} +CHECK: USAGE: split-file [options] filename directory +CHECK: Generic Options: +CHECK: split-file Options: +CHECK: --no-leading-lines diff --git a/llvm/test/tools/split-file/no-leading-lines.test b/llvm/test/tools/split-file/no-leading-lines.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/no-leading-lines.test @@ -0,0 +1,10 @@ +## With --no-leading-lines, don't add leading lines (which is used to preserve line numbers). + +# RUN: split-file --no-leading-lines %s %t +# RUN: count 1 < %t/a.txt +# RUN: FileCheck %s < %t/a.txt + +# CHECK: input + +#--- a.txt +input diff --git a/llvm/test/tools/split-file/output-is-special.test b/llvm/test/tools/split-file/output-is-special.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/split-file/output-is-special.test @@ -0,0 +1,8 @@ +# UNSUPPORTED: system-windows +# REQUIRES: shell + +## Don't set the permission of a character special file, otherwise there will +## be an EPERM error (or worse: root may change the permission). +# RUN: not split-file %s /dev/null 2>&1 | FileCheck %s + +# CHECK: error: /dev/null: output cannot be a special file diff --git a/llvm/tools/split-file/.clang-tidy b/llvm/tools/split-file/.clang-tidy new file mode 100644 --- /dev/null +++ b/llvm/tools/split-file/.clang-tidy @@ -0,0 +1,19 @@ +# Almost identical to the top-level .clang-tidy, except that {Member,Parameter,Variable}Case use camelBack. +Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming' +CheckOptions: + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: camelBack + - key: readability-identifier-naming.MemberCase + value: camelBack + - key: readability-identifier-naming.ParameterCase + value: camelBack + - key: readability-identifier-naming.UnionCase + value: CamelCase + - key: readability-identifier-naming.VariableCase + value: camelBack + - key: readability-identifier-naming.IgnoreMainLikeFunctions + value: 1 diff --git a/llvm/tools/split-file/CMakeLists.txt b/llvm/tools/split-file/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/llvm/tools/split-file/CMakeLists.txt @@ -0,0 +1,7 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_llvm_tool(split-file + split-file.cpp + ) diff --git a/llvm/tools/split-file/split-file.cpp b/llvm/tools/split-file/split-file.cpp new file mode 100644 --- /dev/null +++ b/llvm/tools/split-file/split-file.cpp @@ -0,0 +1,172 @@ +//===- split-file.cpp - Input splitting utility ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Split input into multipe parts separated by regex '^(.|//)--- ' and extract +// the specified part. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileOutputBuffer.h" +#include "llvm/Support/LineIterator.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/WithColor.h" +#include +#include + +using namespace llvm; + +static cl::OptionCategory cat("split-file Options"); + +static cl::opt input(cl::Positional, cl::desc("filename"), + cl::cat(cat)); + +static cl::opt output(cl::Positional, cl::desc("directory"), + cl::value_desc("directory"), cl::cat(cat)); + +static cl::opt noLeadingLines("no-leading-lines", + cl::desc("Don't preserve line numbers"), + cl::cat(cat)); + +static StringRef toolName; +static int errorCount; + +LLVM_ATTRIBUTE_NORETURN static void fatal(StringRef filename, + const Twine &message) { + if (filename.empty()) + WithColor::error(errs(), toolName) << message << '\n'; + else + WithColor::error(errs(), toolName) << filename << ": " << message << '\n'; + exit(1); +} + +static void error(StringRef filename, int line, const Twine &message) { + ++errorCount; + errs() << filename << ':' << line << ": "; + WithColor::error(errs()) << message << '\n'; +} + +namespace { +struct Part { + const char *begin = nullptr; + const char *end = nullptr; + int leadingLines = 0; +}; +} // namespace + +static int handle(MemoryBuffer &inputBuf, StringRef input) { + DenseMap partToBegin; + StringRef lastPart, separator; + for (line_iterator i(inputBuf, /*SkipBlanks=*/false, '\0'); !i.is_at_eof();) { + const int lineNo = i.line_number(); + const StringRef line = *i++; + const size_t markerLen = line.startswith("//") ? 6 : 5; + if (!(line.size() >= markerLen && + line.substr(markerLen - 4).startswith("--- "))) + continue; + separator = line.substr(0, markerLen); + const StringRef partName = line.substr(markerLen); + if (partName.empty()) { + error(input, lineNo, "empty part name"); + continue; + } + if (isSpace(partName.front()) || isSpace(partName.back())) { + error(input, lineNo, "part name cannot have leading or trailing space"); + continue; + } + + auto res = partToBegin.try_emplace(partName); + if (!res.second) { + error(input, lineNo, + "'" + separator + partName + "' occurs more than once"); + continue; + } + if (!lastPart.empty()) + partToBegin[lastPart].end = line.data(); + Part &cur = res.first->second; + if (!i.is_at_eof()) + cur.begin = i->data(); + // If --no-leading-lines is not specified, numEmptyLines is 0. Append + // newlines so that the extracted part preserves line numbers. + cur.leadingLines = noLeadingLines ? 0 : i.line_number() - 1; + + lastPart = partName; + } + if (lastPart.empty()) + fatal(input, "no part separator was found"); + if (errorCount) + return 1; + partToBegin[lastPart].end = inputBuf.getBufferEnd(); + + std::vector> outputFiles; + SmallString<256> partPath; + for (auto &keyValue : partToBegin) { + partPath.clear(); + sys::path::append(partPath, output, keyValue.first); + std::error_code ec = + sys::fs::create_directories(sys::path::parent_path(partPath)); + if (ec) + fatal(input, ec.message()); + auto f = std::make_unique(partPath.str(), ec, + llvm::sys::fs::OF_None); + if (!f) + fatal(input, ec.message()); + + Part &part = keyValue.second; + for (int i = 0; i != part.leadingLines; ++i) + (*f).os().write('\n'); + if (part.begin) + (*f).os().write(part.begin, part.end - part.begin); + outputFiles.push_back(std::move(f)); + } + + for (std::unique_ptr &outputFile : outputFiles) + outputFile->keep(); + return 0; +} + +int main(int argc, const char **argv) { + toolName = sys::path::stem(argv[0]); + cl::HideUnrelatedOptions({&cat}); + cl::ParseCommandLineOptions( + argc, argv, + "Split input into multiple parts separated by regex '^(.|//)--- ' and " + "extract the part specified by '^(.|//)--- '\n", + nullptr, + /*EnvVar=*/nullptr, + /*LongOptionsUseDoubleDash=*/true); + + if (input.empty()) + fatal("", "input filename is not specified"); + if (output.empty()) + fatal("", "output directory is not specified"); + ErrorOr> bufferOrErr = + MemoryBuffer::getFileOrSTDIN(input); + if (std::error_code ec = bufferOrErr.getError()) + fatal(input, ec.message()); + + // Delete output if it is a file or an empty directory, so that we can create + // a directory. + sys::fs::file_status status; + if (std::error_code ec = sys::fs::status(output, status)) + if (ec.value() != static_cast(std::errc::no_such_file_or_directory)) + fatal(output, ec.message()); + if (status.type() != sys::fs::file_type::file_not_found && + status.type() != sys::fs::file_type::directory_file && + status.type() != sys::fs::file_type::regular_file) + fatal(output, "output cannot be a special file"); + if (std::error_code ec = sys::fs::remove(output, /*IgnoreNonExisting=*/true)) + if (ec.value() != static_cast(std::errc::directory_not_empty)) + fatal(output, ec.message()); + return handle(**bufferOrErr, input); +} diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn --- a/llvm/utils/gn/secondary/lld/test/BUILD.gn +++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn @@ -94,6 +94,7 @@ "//llvm/tools/llvm-readobj:symlinks", "//llvm/tools/obj2yaml", "//llvm/tools/opt", + "//llvm/tools/split-file", "//llvm/tools/yaml2obj", "//llvm/utils/FileCheck", "//llvm/utils/count", diff --git a/llvm/utils/gn/secondary/llvm/test/BUILD.gn b/llvm/utils/gn/secondary/llvm/test/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/test/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/test/BUILD.gn @@ -260,6 +260,7 @@ "//llvm/tools/opt", "//llvm/tools/sancov", "//llvm/tools/sanstats", + "//llvm/tools/split-file", "//llvm/tools/verify-uselistorder", "//llvm/tools/yaml2obj", "//llvm/unittests", diff --git a/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn new file mode 100644 --- /dev/null +++ b/llvm/utils/gn/secondary/llvm/tools/split-file/BUILD.gn @@ -0,0 +1,4 @@ +executable("split-file") { + deps = [ "//llvm/lib/Support" ] + sources = [ "split-file.cpp" ] +}