Index: compiler-rt/trunk/lib/fuzzer/CMakeLists.txt =================================================================== --- compiler-rt/trunk/lib/fuzzer/CMakeLists.txt +++ compiler-rt/trunk/lib/fuzzer/CMakeLists.txt @@ -1,5 +1,6 @@ set(LIBFUZZER_SOURCES FuzzerClangCounters.cpp + FuzzerCommand.cpp FuzzerCrossOver.cpp FuzzerDriver.cpp FuzzerExtFunctionsDlsym.cpp Index: compiler-rt/trunk/lib/fuzzer/FuzzerCommand.h =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerCommand.h +++ compiler-rt/trunk/lib/fuzzer/FuzzerCommand.h @@ -0,0 +1,180 @@ +//===- FuzzerCommand.h - Interface representing a process -------*- C++ -* ===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// FuzzerCommand represents a command to run in a subprocess. It allows callers +// to manage command line arguments and output and error streams. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_FUZZER_COMMAND_H +#define LLVM_FUZZER_COMMAND_H + +#include "FuzzerDefs.h" +#include "FuzzerIO.h" + +#include +#include +#include +#include + +namespace fuzzer { + +class Command final { +public: + // This command line flag is used to indicate that the remaining command line + // is immutable, meaning this flag effectively marks the end of the mutable + // argument list. + static inline const char *const ignoreRemainingArgs() { + static const char *const kIgnoreRemaining = "-ignore_remaining_args=1"; + return kIgnoreRemaining; + } + + Command() : CombinedOutAndErr(false) {} + + explicit Command(const Vector &ArgsToAdd) + : Args(ArgsToAdd), CombinedOutAndErr(false) {} + + explicit Command(const Command &Other) + : Args(Other.Args), CombinedOutAndErr(Other.CombinedOutAndErr), + OutputFile(Other.OutputFile) {} + + Command &operator=(const Command &Other) { + Args = Other.Args; + CombinedOutAndErr = Other.CombinedOutAndErr; + OutputFile = Other.OutputFile; + return *this; + } + + ~Command() {} + + // Returns true if the given Arg is present in Args. Only checks up to + // "-ignore_remaining_args=1". + bool hasArgument(const std::string &Arg) const { + auto i = endMutableArgs(); + return std::find(Args.begin(), i, Arg) != i; + } + + // Gets all of the current command line arguments, **including** those after + // "-ignore-remaining-args=1". + const Vector &getArguments() const { return Args; } + + // Adds the given argument before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArgument(const std::string &Arg) { + Args.insert(endMutableArgs(), Arg); + } + + // Adds all given arguments before "-ignore_remaining_args=1", or at the end + // if that flag isn't present. + void addArguments(const Vector &ArgsToAdd) { + Args.insert(endMutableArgs(), ArgsToAdd.begin(), ArgsToAdd.end()); + } + + // Removes the given argument from the command argument list. Ignores any + // occurrences after "-ignore_remaining_args=1", if present. + void removeArgument(const std::string &Arg) { + auto i = endMutableArgs(); + Args.erase(std::remove(Args.begin(), i, Arg), i); + } + + // Like hasArgument, but checks for "-[Flag]=...". + bool hasFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + return std::any_of(Args.begin(), endMutableArgs(), IsMatch); + } + + // Returns the value of the first instance of a given flag, or an empty string + // if the flag isn't present. Ignores any occurrences after + // "-ignore_remaining_args=1", if present. + std::string getFlagValue(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + auto j = std::find_if(Args.begin(), i, IsMatch); + std::string result; + if (j != i) { + result = j->substr(Arg.length()); + } + return result; + } + + // Like AddArgument, but adds "-[Flag]=[Value]". + void addFlag(const std::string &Flag, const std::string &Value) { + addArgument("-" + Flag + "=" + Value); + } + + // Like RemoveArgument, but removes "-[Flag]=...". + void removeFlag(const std::string &Flag) { + std::string Arg("-" + Flag + "="); + auto IsMatch = [&](const std::string &Other) { + return Arg.compare(0, std::string::npos, Other, 0, Arg.length()) == 0; + }; + auto i = endMutableArgs(); + Args.erase(std::remove_if(Args.begin(), i, IsMatch), i); + } + + // Returns whether the command's stdout is being written to an output file. + bool hasOutputFile() const { return !OutputFile.empty(); } + + // Returns the currently set output file. + const std::string &getOutputFile() const { return OutputFile; } + + // Configures the command to redirect its output to the name file. + void setOutputFile(const std::string &FileName) { OutputFile = FileName; } + + // Returns whether the command's stderr is redirected to stdout. + bool isOutAndErrCombined() const { return CombinedOutAndErr; } + + // Sets whether to redirect the command's stderr to its stdout. + void combineOutAndErr(bool combine = true) { CombinedOutAndErr = combine; } + + // Returns a string representation of the command. On many systems this will + // be the equivalent command line. + std::string toString() const { + std::stringstream SS; + for (auto arg : getArguments()) + SS << arg << " "; + if (isOutAndErrCombined()) + SS << "2>&1 "; + if (hasOutputFile()) + SS << ">" << getOutputFile() << " "; + std::string result = SS.str(); + if (!result.empty()) + result = result.substr(0, result.length() - 1); + return result; + } + +private: + Command(Command &&Other) = delete; + Command &operator=(Command &&Other) = delete; + + Vector::iterator endMutableArgs() { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + Vector::const_iterator endMutableArgs() const { + return std::find(Args.begin(), Args.end(), ignoreRemainingArgs()); + } + + // The command arguments. Args[0] is the command name. + Vector Args; + + // True indicates stderr is redirected to stdout. + bool CombinedOutAndErr; + + // If not empty, stdout is redirected to the named file. + std::string OutputFile; +}; + +} // namespace fuzzer + +#endif // LLVM_FUZZER_COMMAND_H Index: compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp +++ compiler-rt/trunk/lib/fuzzer/FuzzerDriver.cpp @@ -9,6 +9,7 @@ // FuzzerDriver and flag parsing. //===----------------------------------------------------------------------===// +#include "FuzzerCommand.h" #include "FuzzerCorpus.h" #include "FuzzerIO.h" #include "FuzzerInterface.h" @@ -206,16 +207,20 @@ } } -static void WorkerThread(const std::string &Cmd, std::atomic *Counter, +static void WorkerThread(const Command &BaseCmd, std::atomic *Counter, unsigned NumJobs, std::atomic *HasErrors) { while (true) { unsigned C = (*Counter)++; if (C >= NumJobs) break; std::string Log = "fuzz-" + std::to_string(C) + ".log"; - std::string ToRun = Cmd + " > " + Log + " 2>&1\n"; - if (Flags.verbosity) - Printf("%s", ToRun.c_str()); - int ExitCode = ExecuteCommand(ToRun); + Command Cmd(BaseCmd); + Cmd.setOutputFile(Log); + Cmd.combineOutAndErr(); + if (Flags.verbosity) { + std::string CommandLine = Cmd.toString(); + Printf("%s", CommandLine.c_str()); + } + int ExitCode = ExecuteCommand(Cmd); if (ExitCode != 0) *HasErrors = true; std::lock_guard Lock(Mu); @@ -240,12 +245,14 @@ unsigned NumWorkers, unsigned NumJobs) { std::atomic Counter(0); std::atomic HasErrors(false); - std::string Cmd = CloneArgsWithoutX(Args, "jobs", "workers"); + Command Cmd(Args); + Cmd.removeFlag("jobs"); + Cmd.removeFlag("workers"); Vector V; std::thread Pulse(PulseThread); Pulse.detach(); for (unsigned i = 0; i < NumWorkers; i++) - V.push_back(std::thread(WorkerThread, Cmd, &Counter, NumJobs, &HasErrors)); + V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); for (auto &T : V) T.join(); return HasErrors ? 1 : 0; @@ -303,20 +310,19 @@ } std::string InputFilePath = Inputs->at(0); std::string OutputFilePath = Flags.exact_artifact_path; - std::string BaseCmd = - CloneArgsWithoutX(Args, "cleanse_crash", "cleanse_crash"); + Command Cmd(Args); + Cmd.removeFlag("cleanse_crash"); - auto InputPos = BaseCmd.find(" " + InputFilePath + " "); - assert(InputPos != std::string::npos); - BaseCmd.erase(InputPos, InputFilePath.size() + 1); + assert(Cmd.hasArgument(InputFilePath)); + Cmd.removeArgument(InputFilePath); auto LogFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); auto TmpFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".repro"); - auto LogFileRedirect = " > " + LogFilePath + " 2>&1 "; - - auto Cmd = BaseCmd + " " + TmpFilePath + LogFileRedirect; + Cmd.addArgument(TmpFilePath); + Cmd.setOutputFile(LogFilePath); + Cmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; auto U = FileToVector(CurrentFilePath); @@ -361,22 +367,22 @@ exit(1); } std::string InputFilePath = Inputs->at(0); - auto BaseCmd = SplitBefore( - "-ignore_remaining_args=1", - CloneArgsWithoutX(Args, "minimize_crash", "exact_artifact_path")); - auto InputPos = BaseCmd.first.find(" " + InputFilePath + " "); - assert(InputPos != std::string::npos); - BaseCmd.first.erase(InputPos, InputFilePath.size() + 1); + Command BaseCmd(Args); + BaseCmd.removeFlag("minimize_crash"); + BaseCmd.removeFlag("exact_artifact_path"); + assert(BaseCmd.hasArgument(InputFilePath)); + BaseCmd.removeArgument(InputFilePath); if (Flags.runs <= 0 && Flags.max_total_time == 0) { Printf("INFO: you need to specify -runs=N or " "-max_total_time=N with -minimize_crash=1\n" "INFO: defaulting to -max_total_time=600\n"); - BaseCmd.first += " -max_total_time=600"; + BaseCmd.addFlag("max_total_time", "600"); } auto LogFilePath = DirPlusFile( TmpDir(), "libFuzzerTemp." + std::to_string(GetPid()) + ".txt"); - auto LogFileRedirect = " > " + LogFilePath + " 2>&1 "; + BaseCmd.setOutputFile(LogFilePath); + BaseCmd.combineOutAndErr(); std::string CurrentFilePath = InputFilePath; while (true) { @@ -384,10 +390,11 @@ Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", CurrentFilePath.c_str(), U.size()); - auto Cmd = BaseCmd.first + " " + CurrentFilePath + LogFileRedirect + " " + - BaseCmd.second; + Command Cmd(BaseCmd); + Cmd.addArgument(CurrentFilePath); - Printf("CRASH_MIN: executing: %s\n", Cmd.c_str()); + std::string CommandLine = Cmd.toString(); + Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); int ExitCode = ExecuteCommand(Cmd); if (ExitCode == 0) { Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); @@ -404,9 +411,10 @@ Flags.exact_artifact_path ? Flags.exact_artifact_path : Options.ArtifactPrefix + "minimized-from-" + Hash(U); - Cmd += " -minimize_crash_internal_step=1 -exact_artifact_path=" + - ArtifactPath; - Printf("CRASH_MIN: executing: %s\n", Cmd.c_str()); + Cmd.addFlag("minimize_crash_internal_step", "1"); + Cmd.addFlag("exact_artifact_path", ArtifactPath); + CommandLine = Cmd.toString(); + Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); ExitCode = ExecuteCommand(Cmd); CopyFileToErr(LogFilePath); if (ExitCode == 0) { Index: compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp +++ compiler-rt/trunk/lib/fuzzer/FuzzerMerge.cpp @@ -9,6 +9,7 @@ // Merging corpora. //===----------------------------------------------------------------------===// +#include "FuzzerCommand.h" #include "FuzzerMerge.h" #include "FuzzerIO.h" #include "FuzzerInternal.h" @@ -232,7 +233,7 @@ std::ostringstream StartedLine; // Write the pre-run marker. OF << "STARTED " << std::dec << i << " " << U.size() << "\n"; - OF.flush(); // Flush is important since ExecuteCommand may crash. + OF.flush(); // Flush is important since Command::Execute may crash. // Run. TPC.ResetMaps(); ExecuteCallback(U.data(), U.size()); @@ -332,15 +333,16 @@ // Execute the inner process until it passes. // Every inner process should execute at least one input. - auto BaseCmd = SplitBefore("-ignore_remaining_args=1", - CloneArgsWithoutX(Args, "merge")); + Command BaseCmd(Args); + BaseCmd.removeFlag("merge"); bool Success = false; for (size_t Attempt = 1; Attempt <= NumAttempts; Attempt++) { MaybeExitGracefully(); Printf("MERGE-OUTER: attempt %zd\n", Attempt); - auto ExitCode = - ExecuteCommand(BaseCmd.first + " -merge_control_file=" + CFPath + - " -merge_inner=1 " + BaseCmd.second); + Command Cmd(BaseCmd); + Cmd.addFlag("merge_control_file", CFPath); + Cmd.addFlag("merge_inner", "1"); + auto ExitCode = ExecuteCommand(Cmd); if (!ExitCode) { Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", Attempt); Success = true; Index: compiler-rt/trunk/lib/fuzzer/FuzzerUtil.h =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerUtil.h +++ compiler-rt/trunk/lib/fuzzer/FuzzerUtil.h @@ -13,6 +13,7 @@ #define LLVM_FUZZER_UTIL_H #include "FuzzerDefs.h" +#include "FuzzerCommand.h" namespace fuzzer { @@ -50,7 +51,7 @@ size_t GetPeakRSSMb(); -int ExecuteCommand(const std::string &Command); +int ExecuteCommand(const Command &Cmd); FILE *OpenProcessPipe(const char *Command, const char *Mode); Index: compiler-rt/trunk/lib/fuzzer/FuzzerUtilDarwin.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerUtilDarwin.cpp +++ compiler-rt/trunk/lib/fuzzer/FuzzerUtilDarwin.cpp @@ -10,7 +10,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_APPLE - +#include "FuzzerCommand.h" #include "FuzzerIO.h" #include #include @@ -38,7 +38,8 @@ // signal handlers when the first thread enters and restores them when the last // thread finishes execution of the function and ensures this is not racey by // using a mutex. -int ExecuteCommand(const std::string &Command) { +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); posix_spawnattr_t SpawnAttributes; if (posix_spawnattr_init(&SpawnAttributes)) return -1; @@ -98,7 +99,7 @@ pid_t Pid; char **Environ = environ; // Read from global - const char *CommandCStr = Command.c_str(); + const char *CommandCStr = CmdLine.c_str(); char *const Argv[] = { strdup("sh"), strdup("-c"), Index: compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp +++ compiler-rt/trunk/lib/fuzzer/FuzzerUtilLinux.cpp @@ -10,13 +10,15 @@ //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_LINUX || LIBFUZZER_NETBSD +#include "FuzzerCommand.h" #include namespace fuzzer { -int ExecuteCommand(const std::string &Command) { - return system(Command.c_str()); +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); } } // namespace fuzzer Index: compiler-rt/trunk/lib/fuzzer/FuzzerUtilWindows.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/FuzzerUtilWindows.cpp +++ compiler-rt/trunk/lib/fuzzer/FuzzerUtilWindows.cpp @@ -10,6 +10,7 @@ //===----------------------------------------------------------------------===// #include "FuzzerDefs.h" #if LIBFUZZER_WINDOWS +#include "FuzzerCommand.h" #include "FuzzerIO.h" #include "FuzzerInternal.h" #include @@ -151,8 +152,9 @@ return _popen(Command, Mode); } -int ExecuteCommand(const std::string &Command) { - return system(Command.c_str()); +int ExecuteCommand(const Command &Cmd) { + std::string CmdLine = Cmd.toString(); + return system(CmdLine.c_str()); } const void *SearchMemory(const void *Data, size_t DataLen, const void *Patt, Index: compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp =================================================================== --- compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp +++ compiler-rt/trunk/lib/fuzzer/tests/FuzzerUnittest.cpp @@ -766,6 +766,166 @@ EXPECT_EQ(Res, Expected); } +// FuzzerCommand unit tests. The arguments in the two helper methods below must +// match. +static void makeCommandArgs(Vector *ArgsToAdd) { + assert(ArgsToAdd); + ArgsToAdd->clear(); + ArgsToAdd->push_back("foo"); + ArgsToAdd->push_back("-bar=baz"); + ArgsToAdd->push_back("qux"); + ArgsToAdd->push_back(Command::ignoreRemainingArgs()); + ArgsToAdd->push_back("quux"); + ArgsToAdd->push_back("-grault=garply"); +} + +static std::string makeCmdLine(const char *separator, const char *suffix) { + std::string CmdLine("foo -bar=baz qux "); + if (strlen(separator) != 0) { + CmdLine += separator; + CmdLine += " "; + } + CmdLine += Command::ignoreRemainingArgs(); + CmdLine += " quux -grault=garply"; + if (strlen(suffix) != 0) { + CmdLine += " "; + CmdLine += suffix; + } + return CmdLine; +} + +TEST(FuzzerCommand, Create) { + std::string CmdLine; + + // Default constructor + Command DefaultCmd; + + CmdLine = DefaultCmd.toString(); + EXPECT_EQ(CmdLine, ""); + + // Explicit constructor + Vector ArgsToAdd; + makeCommandArgs(&ArgsToAdd); + Command InitializedCmd(ArgsToAdd); + + CmdLine = InitializedCmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); + + // Compare each argument + auto InitializedArgs = InitializedCmd.getArguments(); + auto i = ArgsToAdd.begin(); + auto j = InitializedArgs.begin(); + while (i != ArgsToAdd.end() && j != InitializedArgs.end()) { + EXPECT_EQ(*i++, *j++); + } + EXPECT_EQ(i, ArgsToAdd.end()); + EXPECT_EQ(j, InitializedArgs.end()); + + // Copy constructor + Command CopiedCmd(InitializedCmd); + + CmdLine = CopiedCmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); + + // Assignment operator + Command AssignedCmd; + AssignedCmd = CopiedCmd; + + CmdLine = AssignedCmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); +} + +TEST(FuzzerCommand, ModifyArguments) { + Vector ArgsToAdd; + makeCommandArgs(&ArgsToAdd); + Command Cmd; + std::string CmdLine; + + Cmd.addArguments(ArgsToAdd); + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); + + Cmd.addArgument("waldo"); + EXPECT_TRUE(Cmd.hasArgument("waldo")); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("waldo", "")); + + Cmd.removeArgument("waldo"); + EXPECT_FALSE(Cmd.hasArgument("waldo")); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); +} + +TEST(FuzzerCommand, ModifyFlags) { + Vector ArgsToAdd; + makeCommandArgs(&ArgsToAdd); + Command Cmd(ArgsToAdd); + std::string Value, CmdLine; + ASSERT_FALSE(Cmd.hasFlag("fred")); + + Value = Cmd.getFlagValue("fred"); + EXPECT_EQ(Value, ""); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); + + Cmd.addFlag("fred", "plugh"); + EXPECT_TRUE(Cmd.hasFlag("fred")); + + Value = Cmd.getFlagValue("fred"); + EXPECT_EQ(Value, "plugh"); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("-fred=plugh", "")); + + Cmd.removeFlag("fred"); + EXPECT_FALSE(Cmd.hasFlag("fred")); + + Value = Cmd.getFlagValue("fred"); + EXPECT_EQ(Value, ""); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "")); +} + +TEST(FuzzerCommand, SetOutput) { + Vector ArgsToAdd; + makeCommandArgs(&ArgsToAdd); + Command Cmd(ArgsToAdd); + std::string CmdLine; + ASSERT_FALSE(Cmd.hasOutputFile()); + ASSERT_FALSE(Cmd.isOutAndErrCombined()); + + Cmd.combineOutAndErr(true); + EXPECT_TRUE(Cmd.isOutAndErrCombined()); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "2>&1")); + + Cmd.combineOutAndErr(false); + EXPECT_FALSE(Cmd.isOutAndErrCombined()); + + Cmd.setOutputFile("xyzzy"); + EXPECT_TRUE(Cmd.hasOutputFile()); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", ">xyzzy")); + + Cmd.setOutputFile("thud"); + EXPECT_TRUE(Cmd.hasOutputFile()); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", ">thud")); + + Cmd.combineOutAndErr(); + EXPECT_TRUE(Cmd.isOutAndErrCombined()); + + CmdLine = Cmd.toString(); + EXPECT_EQ(CmdLine, makeCmdLine("", "2>&1 >thud")); +} + int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();