[lldb-vscode] Implement stderr/stdout on win32 and redirect lldb log to VSCode

[nidefawl]

Summary

This patch implements stderr/stdout forwarding on windows.
This was previously not implemented in D99974.
I added separate callbacks so the output can be sent to the different channels VSCode provides (OutputType::Stdout, OutputType::Stderr, OutputType::Console).

This patch also passes a log callback handler to SBDebugger::Create to be able to see logging output when it is enabled.

Since the output is now redirect on early startup I removed the calls to SetOutputFileHandle/SetErrorFileHandle, which set them to /dev/null.

I send the output of stderr/stdout/lldb log to OutputType::Console

[mstorsjo]

lldb/tools/lldb-vscode/OutputRedirector.cpp

15

Minor style issue - I guess it'd be less of double negation, if we'd change the ifdef to #if defined(_WIN32) .. #else

55

This change looks unrelated (although I'm not familiar with this piece of code), although it's probably correct

[wallace]

lldb/tools/lldb-vscode/OutputRedirector.cpp

15

feel free to refactor that part

lldb/tools/lldb-vscode/lldb-vscode.cpp

1444–1449

could you test by creating a .lldbinit file with garbage so that the debugger prints error messages during initialization, and then make sure that they are properly redirected to the console of the IDE?

[nidefawl]

Reorder OS check to remove negation

[nidefawl]

lldb/tools/lldb-vscode/OutputRedirector.cpp

55

The reference to the StringRef is never stored, so this avoids a copy.
VSCode::SendOutput takes a StringRef and explicitly converts it to std::string before emplacing it in the json message.

lldb/tools/lldb-vscode/lldb-vscode.cpp

1444–1449

Yes, I can see the error messages in the VSCode debugger console. Both on windows and linux.

[wallace]

thank you!

[wallace]

Let me know if you need help upstreaming this patch.

[nidefawl]

In D123025#3429896, @wallace wrote:

Let me know if you need help upstreaming this patch.

Yes I need some help.
This is my first contribution. I don't have commit access.

[wallace]

ok! I'll commit it for you

lldb/tools/lldb-vscode/OutputRedirector.cpp

 //===-- OutputRedirector.cpp -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
-#if !defined(_WIN32)
+#if defined(_WIN32)
+#include <fcntl.h>
+#include <io.h>
+#else
 #include <unistd.h>
 #endif
 
 #include "OutputRedirector.h"
+#include "llvm/ADT/StringRef.h"
 
 using namespace llvm;
 
 namespace lldb_vscode {
 
 Error RedirectFd(int fd, std::function<void(llvm::StringRef)> callback) {
-#if !defined(_WIN32)
   int new_fd[2];
+#if defined(_WIN32)
+  if (_pipe(new_fd, 4096, O_TEXT) == -1) {
+#else
   if (pipe(new_fd) == -1) {
+#endif
     int error = errno;
     return createStringError(inconvertibleErrorCode(),
                              "Couldn't create new pipe for fd %d. %s", fd,
                              strerror(error));
   }
 
   if (dup2(new_fd[1], fd) == -1) {
     int error = errno;
while (true) {
       ssize_t bytes_count = read(read_fd, &buffer, sizeof(buffer));
       if (bytes_count == 0)
         return;
       if (bytes_count == -1) {
         if (errno == EAGAIN || errno == EINTR)
           continue;
         break;
       }
-      callback(StringRef(buffer, bytes_count).str());
+      callback(StringRef(buffer, bytes_count));
     }
   });
   t.detach();
-#endif
   return Error::success();
 }
 
 } // namespace lldb_vscode

lldb/tools/lldb-vscode/lldb-vscode.cpp

 #include "LLDBUtils.h"
 #include "OutputRedirector.h"
 
 #if defined(_WIN32)
 #ifndef PATH_MAX
 #define PATH_MAX MAX_PATH
 #endif
 typedef int socklen_t;
-constexpr const char *dev_null_path = "nul";
-
-#else
-constexpr const char *dev_null_path = "/dev/null";
-
 #endif
 
 using namespace lldb_vscode;
 
 namespace {
 enum ID {
   OPT_INVALID = 0, // This is not an option ID.
 #define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, ALIASARGS, FLAGS, PARAM,  \

 //       "body": {
 //         "$ref": "#/definitions/Capabilities",
 //         "description": "The capabilities of this debug adapter."
 //       }
 //     }
 //   }]
 // }
 void request_initialize(const llvm::json::Object &request) {
-  g_vsc.debugger = lldb::SBDebugger::Create(true /*source_init_files*/);
+  auto log_cb = [](const char *buf, void *baton) -> void {
+    g_vsc.SendOutput(OutputType::Console, llvm::StringRef{buf});
+  };
+  g_vsc.debugger =
+      lldb::SBDebugger::Create(true /*source_init_files*/, log_cb, nullptr);
   g_vsc.progress_event_thread = std::thread(ProgressEventThreadFunction);
 
-  // Create an empty target right away since we might get breakpoint requests
-  // before we are given an executable to launch in a "launch" request, or a
-  // executable when attaching to a process by process ID in a "attach"
-  // request.
-  FILE *out = llvm::sys::RetryAfterSignal(nullptr, fopen, dev_null_path, "w");
-  if (out) {
-    // Set the output and error file handles to redirect into nothing otherwise
-    // if any code in LLDB prints to the debugger file handles, the output and
-    // error file handles are initialized to STDOUT and STDERR and any output
-    // will kill our debug session.
-    g_vsc.debugger.SetOutputFileHandle(out, true);
-    g_vsc.debugger.SetErrorFileHandle(out, false);
-  }
-
   // Start our event thread so we can receive events from the debugger, target,
   // process and more.
   g_vsc.event_thread = std::thread(EventThreadFunction);
 
   llvm::json::Object response;
   FillResponse(request, response);
   llvm::json::Object body;
   // The debug adapter supports the configurationDoneRequest.

 /// Redirect stdout and stderr fo the IDE's console output.
 ///
 /// Errors in this operation will be printed to the log file and the IDE's
 /// console output as well.
 ///
 /// \return
 ///     A fd pointing to the original stdout.
 int SetupStdoutStderrRedirection() {
-  int new_stdout_fd = dup(fileno(stdout));
-  auto stdout_err_redirector_callback = [&](llvm::StringRef data) {
-    g_vsc.SendOutput(OutputType::Console, data);
+  int stdoutfd = fileno(stdout);
+  int new_stdout_fd = dup(stdoutfd);
+  auto output_callback_stderr = [](llvm::StringRef data) {
+    g_vsc.SendOutput(OutputType::Stderr, data);
   };
-
-  for (int fd : {fileno(stdout), fileno(stderr)}) {
-    if (llvm::Error err = RedirectFd(fd, stdout_err_redirector_callback)) {
-      std::string error_message = llvm::toString(std::move(err));
-      if (g_vsc.log)
-        *g_vsc.log << error_message << std::endl;
-      stdout_err_redirector_callback(error_message);
-    }
+  auto output_callback_stdout = [](llvm::StringRef data) {
+    g_vsc.SendOutput(OutputType::Stdout, data);
+  };
+  if (llvm::Error err = RedirectFd(stdoutfd, output_callback_stdout)) {
+    output_callback_stderr(error_message);
+  if (llvm::Error err = RedirectFd(fileno(stderr), output_callback_stderr)) {
+    std::string error_message = llvm::toString(std::move(err));
+    if (g_vsc.log)
+      *g_vsc.log << error_message << std::endl;
+    output_callback_stderr(error_message);
   }
 
   /// used only by TestVSCode_redirection_to_console.py
   if (getenv("LLDB_VSCODE_TEST_STDOUT_STDERR_REDIRECTION") != nullptr)
     redirection_test();
   return new_stdout_fd;
 }