Index: clang-tools-extra/clangd/GlobalCompilationDatabase.cpp =================================================================== --- clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -17,8 +17,10 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" +#include "llvm/Support/ConvertUTF.h" #include "llvm/Support/FileSystem.h" #include "llvm/Support/Path.h" +#include "llvm/Support/StringSaver.h" #include #include #include @@ -27,6 +29,124 @@ namespace clangd { namespace { +bool expandResponseFile(llvm::StringRef FName, llvm::StringSaver &Saver, + llvm::cl::TokenizerCallback Tokenizer, + SmallVectorImpl &NewArgv) { + llvm::ErrorOr> MemBufOrErr = + llvm::MemoryBuffer::getFile(FName); + if (!MemBufOrErr) + return false; + llvm::MemoryBuffer &MemBuf = *MemBufOrErr.get(); + StringRef Str(MemBuf.getBufferStart(), MemBuf.getBufferSize()); + + // If we have a UTF-16 byte order mark, convert to UTF-8 for parsing. + ArrayRef BufRef(MemBuf.getBufferStart(), MemBuf.getBufferEnd()); + std::string UTF8Buf; + // It is called byte order marker but the UTF-8 BOM is actually not affected + // by the host system's endianness. + auto HasUtF8ByteOrderMark = [](ArrayRef S) { + return (S.size() >= 3 && S[0] == '\xef' && S[1] == '\xbb' && + S[2] == '\xbf'); + }; + if (llvm::hasUTF16ByteOrderMark(BufRef)) { + if (!convertUTF16ToUTF8String(BufRef, UTF8Buf)) + return false; + Str = StringRef(UTF8Buf); + } + // If we see UTF-8 BOM sequence at the beginning of a file, we shall remove + // these bytes before parsing. + // Reference: http://en.wikipedia.org/wiki/UTF-8#Byte_order_mark + else if (HasUtF8ByteOrderMark(BufRef)) + Str = StringRef(BufRef.data() + 3, BufRef.size() - 3); + // Tokenize the contents into NewArgv. + Tokenizer(Str, Saver, NewArgv, false); + return true; +} + +bool expandResponseFiles(tooling::CompileCommand &Cmd, + llvm::cl::TokenizerCallback Tokenizer) { + bool AllExpanded = true; + struct ResponseFileRecord { + llvm::StringRef File; + size_t End; + }; + std::vector &Argv = Cmd.CommandLine; + // To detect recursive response files, we maintain a stack of files and the + // position of the last argument in the file. This position is updated + // dynamically as we recursively expand files. + SmallVector FileStack; + + // Push a dummy entry that represents the initial command line, removing + // the need to check for an empty list. + FileStack.push_back({"", Argv.size()}); + + // Don't cache Argv.size() because it can change. + for (unsigned I = 0; I != Argv.size();) { + while (I == FileStack.back().End) { + // Passing the end of a file's argument list, so we can remove it from the + // stack. + FileStack.pop_back(); + } + + std::string &Arg = Argv[I]; + + if (Arg[0] != '@') { + ++I; + continue; + } + SmallString<128> ResponseFile; + if (llvm::sys::path::is_relative(&Arg[1])) { + llvm::sys::path::append(ResponseFile, Cmd.Directory, &Arg[1]); + } + llvm::sys::path::remove_dots(ResponseFile); + + auto IsEquivalent = [ResponseFile](const ResponseFileRecord &RFile) { + return llvm::sys::fs::equivalent(RFile.File, ResponseFile); + }; + + // Check for recursive response files. + if (std::any_of(FileStack.begin() + 1, FileStack.end(), IsEquivalent)) { + // This file is recursive, so we leave it in the argument stream and + // move on. + AllExpanded = false; + ++I; + continue; + } + + // Replace this response file argument with the tokenization of its + // contents. Nested response files are expanded in subsequent iterations. + SmallVector ExpandedArgv; + llvm::BumpPtrAllocator Alloc; + llvm::StringSaver Saver(Alloc); + llvm::SmallVector T; + if (!expandResponseFile(ResponseFile, Saver, Tokenizer, ExpandedArgv)) { + // We couldn't read this file, so we leave it in the argument stream and + // move on. + AllExpanded = false; + ++I; + continue; + } + + for (ResponseFileRecord &Record : FileStack) { + // Increase the end of all active records by the number of newly expanded + // arguments, minus the response file itself. + Record.End += ExpandedArgv.size() - 1; + } + + FileStack.push_back({ResponseFile, I + ExpandedArgv.size()}); + Argv.erase(Argv.begin() + I); + Argv.insert(Argv.begin() + I, ExpandedArgv.begin(), ExpandedArgv.end()); + } + + // If successful, the top of the file stack will mark the end of the Argv + // stream. A failure here indicates a bug in the stack popping logic above. + // Note that FileStack may have more than one element at this point because we + // don't have a chance to pop the stack when encountering recursive files at + // the end of the stream, so seeing that doesn't indicate a bug. + assert(FileStack.size() > 0 && Argv.size() == FileStack.back().End); + return AllExpanded; +} + void adjustArguments(tooling::CompileCommand &Cmd, llvm::StringRef ResourceDir) { tooling::ArgumentsAdjuster ArgsAdjuster = tooling::combineAdjusters( @@ -40,6 +160,14 @@ tooling::getClangSyntaxOnlyAdjuster())); Cmd.CommandLine = ArgsAdjuster(Cmd.CommandLine, Cmd.Filename); + + // ExpandResponseFile + auto Tokenizer = llvm::Triple(llvm::sys::getProcessTriple()).isOSWindows() + ? llvm::cl::TokenizeWindowsCommandLine + : llvm::cl::TokenizeGNUCommandLine; + if (!expandResponseFiles(Cmd, Tokenizer)) + log("Failed to expand response files for {0}", Cmd.Filename); + // Inject the resource dir. // FIXME: Don't overwrite it if it's already there. if (!ResourceDir.empty())