Support Windows

[kpdev42]

Summary

After this patch LNT can be installed on Windows with preinstalled C++ compiler, e.g. Microsoft Visual Studio.

OS Laboratory. Huawei Russian Research Institute. Saint-Petersburg

[danilaml]

lnt/testing/profile/cPerf.cpp

63

Would it work with lean and mean macro defined? It's preferable, if it would.

141

shouldn't it return ssize_t?

155

Why cast to unsigned? What if size == 0?

408

I think there is a potential mem leak here, but that's not critical.

[kpdev42]

Addressing review

[kpdev42]

lnt/testing/profile/cPerf.cpp

63

Sure, I have updated the patch.

141

Thanks! Fixed now.

155

Updated to ssize_t.
Note size = 0 must be provided only with lineprt = nullptr and then size is 128 after allocating bufptr. The behavior is undefined for non null lineprt and size = 0. Anyway, we have no such case here.

408

Please look at the code if (Line) free(Line); below.

[danilaml]

lnt/testing/profile/cPerf.cpp

155

If it's the behaviour of the POSIX getline this is trying to emulate, then I guess this is fine.

408

From the quick glance it looks like if realloc succeeds once inside the added getline function but fails the second time, -1 is returned with lineptr still pointing to the old non-realloced buffer, so there will potentially be a double free as well. I might be reading it wrong however.

[slydiman]

lnt/testing/profile/cPerf.cpp

408

Please look at the example on Microsoft doc https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/realloc

// Reallocate and show new size:
oldbuffer = buffer;     // save pointer in case realloc fails
if ( (buffer = realloc(buffer, size + (1000 * sizeof(long)) ) ) == NULL)
{
   free(oldbuffer);  // free original block
   exit(1);
}

So everything is correct according to this example.

[danilaml]

lnt/testing/profile/cPerf.cpp

408

I don't see it. The situation I'm talking about is akin to:

oldbuffer = buffer;     // save pointer in case realloc fails
if ( (buffer = realloc(buffer, size + (1000 * sizeof(long)) ) ) == NULL) // succeeds 
{
   free(oldbuffer); 
   exit(1);
}
// oldbuffer is pointing to the old, potentially freed buffer, buffer != oldbuffer
size += ...
if ( (buffer = realloc(buffer, size + (1000 * sizeof(long)) ) ) == NULL) // fails
{
   free(oldbuffer);  // buffer is not freed, oldbuffer is freed the second time
}

Here, oldbuffer would

[slydiman]

lnt/testing/profile/cPerf.cpp

157

It seems p may be invalid after realloc().

[kpdev42]

I have fixed and optimized the function getline() with all the comments in mind

[danilaml]

LGTM

lnt/testing/profile/cPerf.cpp

 // binaries like "perf", "bash", or libaries such as libdl. We don't care about
 // these, and these binaries and libraries are very large to nm and objdump.
 //
 // So we have a threshold - if a binary contains < 1% of all samples, don't
 // bother importing it.
 //
 //===----------------------------------------------------------------------===//
 
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#define _CRT_SECURE_NO_WARNINGS
+#define strtok_r strtok_s
+#include <windows.h>
+#include <BaseTsd.h>
+typedef SSIZE_T ssize_t;
+#define PROT_EXEC 4
+#endif
 #include <Python.h>
 #include <algorithm>
 #include <cassert>
 #include <cstring>
 #include <exception>
 #include <fcntl.h>
 #include <iostream>
 #include <map>
 #include <sstream>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
+#ifndef _WIN32
 #include <sys/mman.h>
-#include <sys/stat.h>
 #include <sys/wait.h>
 #include <unistd.h>
+#endif
+#include <sys/stat.h>
 #include <vector>
 
 //===----------------------------------------------------------------------===//
 // Helpers
 //===----------------------------------------------------------------------===//
 
 #ifndef STANDALONE
 #ifdef assert
uint64_t TakeU64(unsigned char *&Buf) {
   uint64_t X = * (uint64_t*) Buf;
   Buf += sizeof(uint64_t);
   return X;
 }
 
 // Forks, execs Cmd under a shell and returns
 // a file descriptor reading the command's stdout.
 FILE *ForkAndExec(std::string Cmd) {
+#ifdef _WIN32
+  Cmd = "cmd.exe /c " + Cmd;
+  FILE *Stream = _popen(Cmd.c_str(), "rt");
+#else
   int P[2];
   pipe(P);
 
   FILE *Stream = fdopen(P[0], "r");
   
   if (fork() == 0) {
     dup2(P[1], 1);
     close(P[0]);
     execl("/bin/sh", "sh", "-c", Cmd.c_str(), 0);
   } else {
     close(P[1]);
   }
-
+#endif
   return Stream;
 }
 
+#ifdef _WIN32
+ssize_t getline(char** lineptr, size_t* n, FILE* stream) {
+  if (lineptr == nullptr || stream == nullptr || n == nullptr) return -1;
+  int c = fgetc(stream);
+  if (c == EOF) return -1;
+  char* bufptr = *lineptr;
+  size_t size = *n;
+  if (bufptr == nullptr) {
+    size = 128;
+    bufptr = (char*)malloc(size);
+    if (bufptr == nullptr) return -1;
+    *lineptr = bufptr;
+  }
+  char* p = bufptr;
+  while (c != EOF) {
+    size_t offset = (size_t)(p - bufptr);
+    if ((offset + 2) > size)) {
+      size = size + 128;
+      bufptr = (char*)realloc(bufptr, size);
+      if (bufptr == nullptr) return -1;
+      *lineptr = bufptr;
+      p = bufptr + offset;
+    }
+    *p++ = c;
+    if (c == '\n') break;
+    c = fgetc(stream);
+  }
+  *p = '\0';
+  *n = size;
+  return p - bufptr;
+}
+#endif
+
 void Assert(bool Expr, const char *ExprStr, const char *File, int Line) {
   if (Expr)
     return;
   char Str[512];
   sprintf(Str, "%s (%s:%d)", ExprStr, File, Line);
   throw std::logic_error(Str);
 }
 

 struct perf_event_mmap {
   struct perf_event_header header;
 
   uint32_t pid, tid;
   uint64_t start, extent, pgoff;
   char filename[1];
 };
 
-#define PROT_EXEC 4
-
 struct perf_event_mmap2 {
   struct perf_event_header header;
 
   uint32_t pid, tid;
   uint64_t start, extent, pgoff;
   uint32_t maj, min;
   uint64_t ino, ino_generation;
   uint32_t prot, flags;
public:
 
   void fetchSymbols(Map *M, bool Dynamic) {
     std::string D = "-D";
     if (!Dynamic)
       // Don't fetch the dynamic symbols - instead fetch static ones.
       D = "";
     std::string Cmd = Nm + " " + D + " -S --defined-only " +
                       BinaryCacheRoot + std::string(M->Filename) +
+#ifdef _WIN32
+                      " 2> NUL";
+#else
                       " 2>/dev/null";
+#endif
     auto Stream = ForkAndExec(Cmd);
 
     char *Line = nullptr;
     size_t LineLen = 0;
     while (true) {
       ssize_t Len = getline(&Line, &LineLen, Stream);
       if (Len == -1)
         break;
 
       std::vector<std::string> SplittedLine;
       if (splitLine(std::string(Line), SplittedLine) < 4)
         continue; 
 
       const std::string& One = SplittedLine[0];
       const std::string& Two = SplittedLine[1];
while (true) {
       }
       if (Four.back() == '\n')
         Four.pop_back();
       push_back({Start, Start + Extent, Four});
     }
     if (Line)
       free(Line);
 
+#ifdef _WIN32
+    _pclose(Stream);
+#else
     fclose(Stream);
     wait(NULL);
+#endif
   }
 
   void reset(Map *M) {
     clear();
     // Fetch both dynamic and static symbols, sort and unique them.
     fetchSymbols(M, true);
     fetchSymbols(M, false);
     
public:
   char *Line;
   size_t LineLen;
 
   ObjdumpOutput(std::string Objdump, std::string BinaryCacheRoot)
     : Objdump(Objdump), BinaryCacheRoot(BinaryCacheRoot), Stream(nullptr),
       Line(NULL), LineLen(0) {}
   ~ObjdumpOutput() {
     if (Stream) {
+#ifdef _WIN32
+      _pclose(Stream);
+#else
       fclose(Stream);
       wait(NULL);
+#endif
     }
     if (Line)
       free(Line);
   }
 
   void reset(Map *M, uint64_t Start, uint64_t Stop) {
     ThisAddress = 0;
     ThisText = "";
     if (Stream) {
+#ifdef _WIN32
+      _pclose(Stream);
+#else
       fclose(Stream);
       wait(NULL);
+#endif
     }
 
     char buf1[32], buf2[32];
     sprintf(buf1, "%#llx", Start);
     sprintf(buf2, "%#llx", Stop + 4);
 
     std::string Cmd = Objdump + " -d --no-show-raw-insn --start-address=" +
                       std::string(buf1) + " --stop-address=" +
                       std::string(buf2) + " " +
                       BinaryCacheRoot + std::string(M->Filename) +
+#ifdef _WIN32
+                      " 2> NUL";
+#else
                       " 2>/dev/null";
+#endif
     Stream = ForkAndExec(Cmd);
 
     EndAddress = Stop;
   };
 
   std::string getText() { return ThisText; }
 
   uint64_t next() {
void emitSymbol(
       Symbol &Sym, Map &M,
       std::map<uint64_t, std::map<const char *, uint64_t>>::iterator Event,
       std::map<const char *, uint64_t> &SymEvents,
       uint64_t Adjust);
   PyObject *complete();
 
 private:
   unsigned char *Buffer;
+#ifdef _WIN32
+  HANDLE hFile;
+  HANDLE hMapFile;
+#else
   size_t BufferLen;
+#endif
 
   perf_header *Header;
   std::map<uint64_t, const char *> EventIDs;
   std::map<uint64_t, uint64_t> EventLayouts;
   std::map<size_t, std::map<uint64_t, std::map<const char *, uint64_t>>> Events;
   std::map<const char *, uint64_t> TotalEvents;
   std::map<uint64_t, std::map<const char *, uint64_t>> TotalEventsPerMap;
   std::vector<Map> Maps;
   std::map<uint64_t, std::map<uint64_t, size_t>> CurrentMaps;
 
   PyObject *Functions, *TopLevelCounters;
   std::vector<PyObject*> Lines;
   
   std::string Nm, Objdump, BinaryCacheRoot;
 };
 
 PerfReader::PerfReader(const std::string &Filename,
                        std::string Nm, std::string Objdump,
                        std::string BinaryCacheRoot)
     : Nm(Nm), Objdump(Objdump), BinaryCacheRoot(BinaryCacheRoot) {
   TopLevelCounters = PyDict_New();
   Functions = PyDict_New();
+#ifdef _WIN32
+  Buffer = nullptr;
+  hMapFile = nullptr;
+  hFile = ::CreateFileA(Filename.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,
+                        OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+  assert(hFile != INVALID_HANDLE_VALUE);
+  LARGE_INTEGER size;
+  size.QuadPart = 0;
+  assert(::GetFileSizeEx(hFile, &size) != FALSE);
+  hMapFile = ::CreateFileMapping(hFile, NULL, PAGE_READONLY, size.HighPart, size.LowPart, NULL);
+  assert(hMapFile != nullptr);
+  Buffer = (unsigned char*)::MapViewOfFile(hMapFile, FILE_MAP_READ, 0, 0, 0);
+  assert(Buffer != nullptr);
+#else
   int fd = open(Filename.c_str(), O_RDONLY);
   assert(fd > 0);
 
   struct stat sb;
   assert(fstat(fd, &sb) == 0);
   BufferLen = (size_t)sb.st_size;
 
   Buffer = (unsigned char *)mmap(NULL, BufferLen, PROT_READ, MAP_SHARED, fd, 0);
   assert(Buffer != MAP_FAILED);
+#endif
 }
 
-PerfReader::~PerfReader() { munmap(Buffer, BufferLen); }
+PerfReader::~PerfReader() {
+#ifdef _WIN32
+  if (hMapFile != NULL) {
+    if (Buffer != NULL) ::UnmapViewOfFile(Buffer);
+    ::CloseHandle(hMapFile);
+  }
+  if (hFile != INVALID_HANDLE_VALUE) ::CloseHandle(hFile);
+#else
+  munmap(Buffer, BufferLen);
+#endif
+}
 
 void PerfReader::readHeader() {
   Header = (perf_header *)&Buffer[0];
 
   assert(!strncmp(Header->magic, "PERFILE2", 8));
 }
 
 void PerfReader::readDataStream() {