[trace][intelpt] Support system-wide tracing [16] - Create threads automatically from context switch data in the post-mortem case

[wallace]

Summary

For some context, The context switch data contains information of which threads were
executed by each traced process, therefore it's not necessary to specify
them in the trace file.

So this diffs adds support for that automatic feature. Eventually we
could include it to live processes as well.

[wallace]

update

[jj10306]

lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionFileParser.cpp

167–170

So should this be an empty list or will the key not exist at all in the case that context switches are provided?

212–215

is this necessary? can't this be inferred by if the threads key isn't present/the list is empty?
also, is this even being used rn? It appears to me that you always invoke CreateThreadsFromContextSwitches() if session.cores is present?

lldb/test/API/commands/trace/intelpt-multi-core-trace/trace_missing_threads.json

32–36

shouldn't this be gone or just an empty list depending on how we want to implement it?

[wallace]

lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionFileParser.cpp

167–170

this should be an empty list. I've improved this documentaiton a bit

212–215

lol, i forgot to delete this

lldb/test/API/commands/trace/intelpt-multi-core-trace/trace_missing_threads.json

32–36

i want to here here the hybrid situation of having some explicit threads here and others inferred

[wallace]

This was improved in https://reviews.llvm.org/D127804

lldb/include/lldb/Target/Trace.h

protected:
   ///
   /// \return
   ///     \b Error::success() if this operation succeedes, or an actual error
   ///     otherwise.
   virtual llvm::Error
   DoRefreshLiveProcessState(TraceGetStateResponse state,
                             llvm::StringRef json_response) = 0;
 
+  /// Return the list of processes traced by this instance. None of the returned
+  /// pointers are invalid.
+  std::vector<Process *> GetTracedProcesses() const;
+
   /// Method to be invoked by the plug-in to refresh the live process state. It
   /// will invoked DoRefreshLiveProcessState at some point, which should be
   /// implemented by the plug-in for custom state handling.
   ///
   /// The result is cached through the same process stop. Even in the case of
   /// errors, it caches the error.
   ///
   /// \return

lldb/source/Plugins/Trace/intel-pt/TraceIntelPT.h

public:
 
   /// \return
   ///     The timer object for this trace.
   TaskTimer &GetTimer();
 
 private:
   friend class TraceIntelPTSessionFileParser;
 
+  /// Create post-mortem threads associated with the processes traced by this
+  /// instance using the context switch traces.
+  ///
+  /// This does nothing if the threads already exist.
+  ///
+  /// \return
+  ///   An \a llvm::Error in case of failures.
+  llvm::Error CreateThreadsFromContextSwitches();
+
   llvm::Expected<pt_cpu> GetCPUInfoForLiveProcess();
 
   /// Postmortem trace constructor
   ///
   /// \param[in] session
   ///     The definition file for the postmortem session.
   ///
   /// \param[in] traces_proceses
   ///     The processes traced in the live session.
   ///
   /// \param[in] trace_threads
   ///     The threads traced in the live session. They must belong to the
   ///     processes mentioned above.
-  TraceIntelPT(JSONTraceSession &session,
+  TraceIntelPT(JSONTraceSession &session, const FileSpec &session_file_dir,
                llvm::ArrayRef<lldb::ProcessSP> traced_processes,
                llvm::ArrayRef<lldb::ThreadPostMortemTraceSP> traced_threads);
 
   /// Constructor for live processes
   TraceIntelPT(Process &live_process)
       : Trace(live_process), m_thread_decoders(){};
 
   /// Decode the trace of the given thread that, i.e. recontruct the traced
private:
   std::map<lldb::tid_t, std::unique_ptr<ThreadDecoder>> m_thread_decoders;
   /// Helper variable used to track long running operations for telemetry.
   TaskTimer m_task_timer;
   /// It is provided by either a session file or a live process to convert TSC
   /// counters to and from nanos. It might not be available on all hosts.
   llvm::Optional<LinuxPerfZeroTscConversion> m_tsc_conversion;
 };
 
+using TraceIntelPTSP = std::shared_ptr<TraceIntelPT>;
+
 } // namespace trace_intel_pt
 } // namespace lldb_private
 
 #endif // LLDB_SOURCE_PLUGINS_TRACE_INTEL_PT_TRACEINTELPT_H

lldb/source/Plugins/Trace/intel-pt/TraceIntelPT.cpp

 
 Expected<TraceSP> TraceIntelPT::CreateInstanceForLiveProcess(Process &process) {
   TraceSP instance(new TraceIntelPT(process));
   process.GetTarget().SetTrace(instance);
   return instance;
 }
 
 TraceIntelPT::TraceIntelPT(JSONTraceSession &session,
+                           const FileSpec &session_file_dir,
                            ArrayRef<ProcessSP> traced_processes,
                            ArrayRef<ThreadPostMortemTraceSP> traced_threads)
     : Trace(traced_processes, session.GetCoreIds()),
       m_cpu_info(session.cpu_info),
       m_tsc_conversion(session.tsc_perf_zero_conversion) {
   for (const ThreadPostMortemTraceSP &thread : traced_threads) {
     m_thread_decoders.emplace(thread->GetID(),
                               std::make_unique<ThreadDecoder>(thread, *this));
     if (const Optional<FileSpec> &trace_file = thread->GetTraceFile()) {
       SetPostMortemThreadDataFile(thread->GetID(),
                                   IntelPTDataKinds::kTraceBuffer, *trace_file);
     }
   }
   if (session.cores) {
     std::vector<core_id_t> cores;
 
     for (const JSONCore &core : *session.cores) {
+      FileSpec trace_buffer(core.trace_buffer);
+      if (trace_buffer.IsRelative())
+        trace_buffer.PrependPathComponent(session_file_dir);
+
       SetPostMortemCoreDataFile(core.core_id, IntelPTDataKinds::kTraceBuffer,
-                                FileSpec(core.trace_buffer));
+                                trace_buffer);
+
+      FileSpec context_switch(core.context_switch_trace);
+      if (context_switch.IsRelative())
+        context_switch.PrependPathComponent(session_file_dir);
       SetPostMortemCoreDataFile(core.core_id,
                                 IntelPTDataKinds::kPerfContextSwitchTrace,
-                                FileSpec(core.context_switch_trace));
+                                context_switch);
       cores.push_back(core.core_id);
     }
 
     std::vector<tid_t> tids;
     for (const JSONProcess &process : session.processes)
       for (const JSONThread &thread : process.threads)
         tids.push_back(thread.tid);
 

 }
 
 Error TraceIntelPT::OnThreadBufferRead(lldb::tid_t tid,
                                        OnBinaryDataReadCallback callback) {
   return OnThreadBinaryDataRead(tid, IntelPTDataKinds::kTraceBuffer, callback);
 }
 
 TaskTimer &TraceIntelPT::GetTimer() { return m_task_timer; }
+
+Error TraceIntelPT::CreateThreadsFromContextSwitches() {
+  DenseMap<lldb::pid_t, DenseSet<lldb::tid_t>> pids_to_tids;
+
+  for (core_id_t core_id : GetTracedCores()) {
+    Error err = OnCoreBinaryDataRead(
+        core_id, IntelPTDataKinds::kPerfContextSwitchTrace,
+        [&](ArrayRef<uint8_t> data) -> Error {
+          Expected<std::vector<ThreadContinuousExecution>> executions =
+              DecodePerfContextSwitchTrace(data, core_id, *m_tsc_conversion);
+          if (!executions)
+            return executions.takeError();
+          for (const ThreadContinuousExecution &execution : *executions)
+            pids_to_tids[execution.pid].insert(execution.tid);
+          return Error::success();
+        });
+    if (err)
+      return err;
+  }
+
+  DenseMap<lldb::pid_t, Process *> processes;
+  for (Process *proc : GetTracedProcesses())
+    processes.try_emplace(proc->GetID(), proc);
+
+  for (const auto &pid_to_tids : pids_to_tids) {
+    lldb::pid_t pid = pid_to_tids.first;
+    auto it = processes.find(pid);
+    if (it == processes.end())
+      continue;
+
+    Process &process = *it->second;
+    ThreadList &thread_list = process.GetThreadList();
+
+    for (lldb::tid_t tid : pid_to_tids.second) {
+      if (!thread_list.FindThreadByID(tid)) {
+        thread_list.AddThread(std::make_shared<ThreadPostMortemTrace>(
+            process, tid, /*trace_file*/ None));
+      }
+    }
+  }
+  return Error::success();
+}

lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionFileParser.h

public:
   /// Parse the structured data trace session and create the corresponding \a
   /// Target objects. In case of an error, no targets are created.
   ///
   /// \return
   ///   A \a lldb::TraceSP instance with the trace session data. In case of
   ///   errors, return a null pointer.
   llvm::Expected<lldb::TraceSP> Parse();
 
-  lldb::TraceSP
+  llvm::Expected<lldb::TraceSP>
   CreateTraceIntelPTInstance(JSONTraceSession &session,
                              std::vector<ParsedProcess> &parsed_processes);
 
 private:
   /// Resolve non-absolute paths relative to the session file folder. It
   /// modifies the given file_spec.
   void NormalizePath(lldb_private::FileSpec &file_spec);
 
   lldb::ThreadPostMortemTraceSP ParseThread(lldb::ProcessSP &process_sp,
                                             const JSONThread &thread);
 
+  /// Create the corresponding Threads and Process objects given the JSON
+  /// process definition.
+  ///
+  /// \param[in] process
+  ///   The JSON process definition
   llvm::Expected<ParsedProcess> ParseProcess(const JSONProcess &process);
 
   llvm::Error ParseModule(lldb::TargetSP &target_sp, const JSONModule &module);
 
   /// Create a user-friendly error message upon a JSON-parsing failure using the
   /// \a json::ObjectMapper functionality.
   ///
   /// \param[in] root
   ///   The \a llvm::json::Path::Root used to parse the JSON \a value.
   ///
   /// \param[in] value
   ///   The json value that failed to parse.
   ///
   /// \return
   ///   An \a llvm::Error containing the user-friendly error message.
   llvm::Error CreateJSONError(llvm::json::Path::Root &root,
                               const llvm::json::Value &value);
 
+  /// Create the corresponding Process, Thread and Module objects given this
+  /// session file.
   llvm::Expected<std::vector<ParsedProcess>>
   ParseSessionFile(const JSONTraceSession &session);
 
   Debugger &m_debugger;
   const llvm::json::Value &m_trace_session_file;
   std::string m_session_file_dir;
 };
 
 } // namespace trace_intel_pt
 } // namespace lldb_private
 
 
 #endif // LLDB_SOURCE_PLUGINS_TRACE_INTEL_PT_TRACEINTELPTSESSIONFILEPARSER_H

lldb/source/Plugins/Trace/intel-pt/TraceIntelPTSessionFileParser.cpp

"cpuInfo": {
     "stepping": integer
   },
   "processes": [
     {
       "pid": integer,
       "triple"?: string,
           // Optional clang/llvm target triple.
       "threads": [
+          // A list of known threads for the given process. When context switch
+          // data is provided, LLDB will automatically create threads for the
+          // this process whenever it finds new threads when traversing the
+          // context switches.
         {
           "tid": integer,
           "traceBuffer"?: string
               // Path to the raw Intel PT buffer file for this thread.
         }
       ],
       "modules": [
         {
"tscPerfZeroConversion"?: {
     // Values used to convert between TSCs and nanoseconds. See the time_zero
     // section in https://man7.org/linux/man-pages/man2/perf_event_open.2.html
     // for for information.
 
     "timeMult": integer,
     "timeShift": integer,
     "timeZero": integer,
   }
+  "dontCreateThreadsFromContextSwitches"?: boolean,
+    // If this is true, then the automatic creation of threads from context switch
+    // data is disabled, and thus only the threads provided in the "processes.threads"
+    // section will be created.
 }
 
 Notes:
 
 - All paths are either absolute or relative to folder containing the session file.
 - "cores" is provided if and only if processes[].threads[].traceBuffer is not provided.
 - "tscPerfZeroConversion" must be provided if "cores" is provided.
  })";
   }
   return schema;
 }
 
-TraceSP TraceIntelPTSessionFileParser::CreateTraceIntelPTInstance(
+Expected<TraceSP> TraceIntelPTSessionFileParser::CreateTraceIntelPTInstance(
     JSONTraceSession &session, std::vector<ParsedProcess> &parsed_processes) {
   std::vector<ThreadPostMortemTraceSP> threads;
   std::vector<ProcessSP> processes;
   for (const ParsedProcess &parsed_process : parsed_processes) {
     processes.push_back(parsed_process.target_sp->GetProcessSP());
     threads.insert(threads.end(), parsed_process.threads.begin(),
                    parsed_process.threads.end());
   }
 
-  TraceSP trace_instance(new TraceIntelPT(session, processes, threads));
+  TraceIntelPTSP trace_instance(new TraceIntelPT(
+      session, FileSpec(m_session_file_dir), processes, threads));
   for (const ParsedProcess &parsed_process : parsed_processes)
     parsed_process.target_sp->SetTrace(trace_instance);
 
+  if (session.cores) {
+    if (Error err = trace_instance->CreateThreadsFromContextSwitches())
+      return std::move(err);
+  }
+
   return trace_instance;
 }
 
 Expected<TraceSP> TraceIntelPTSessionFileParser::Parse() {
   json::Path::Root root("traceSession");
   JSONTraceSession session;
   if (!fromJSON(m_trace_session_file, session, root))
     return CreateJSONError(root, m_trace_session_file);
 
   if (Expected<std::vector<ParsedProcess>> parsed_processes =
           ParseSessionFile(session))
     return CreateTraceIntelPTInstance(session, *parsed_processes);
   else
     return parsed_processes.takeError();
 }

lldb/source/Target/Trace.cpp

 }
 
 ArrayRef<lldb::core_id_t> Trace::GetTracedCores() {
   RefreshLiveProcessState();
   if (m_cores)
     return *m_cores;
   return {};
 }
+
+std::vector<Process *> Trace::GetTracedProcesses() const {
+  std::vector<Process *> processes;
+
+  for (Process *proc : m_postmortem_processes)
+    processes.push_back(proc);
+
+  if (m_live_process)
+    processes.push_back(m_live_process);
+  return processes;
+}

lldb/test/API/commands/trace/TestTraceLoad.py

def testLoadMultiCoreTrace(self):
         self.expect("thread trace dump instructions 2 -t",
           substrs=["19521: [tsc=0x008fb5211c143fd8] error: expected tracing enabled event",
                    "m.out`foo() + 65 at multi_thread.cpp:12:21",
                    "19520: [tsc=0x008fb5211bfbc69e] 0x0000000000400ba7    jg     0x400bb3"])
         self.expect("thread trace dump instructions 3 -t",
           substrs=["67910: [tsc=0x008fb5211bfdf270] 0x0000000000400bd7    addl   $0x1, -0x4(%rbp)",
                    "m.out`bar() + 26 at multi_thread.cpp:20:6"])
 
+    def testLoadMultiCoreTraceWithMissingThreads(self):
+        src_dir = self.getSourceDir()
+        trace_definition_file = os.path.join(src_dir, "intelpt-multi-core-trace", "trace_missing_threads.json")
+        self.expect("trace load -v " + trace_definition_file, substrs=["intel-pt"])
+        self.expect("thread trace dump instructions 3 -t",
+          substrs=["19521: [tsc=0x008fb5211c143fd8] error: expected tracing enabled event",
+                   "m.out`foo() + 65 at multi_thread.cpp:12:21",
+                   "19520: [tsc=0x008fb5211bfbc69e] 0x0000000000400ba7    jg     0x400bb3"])
+        self.expect("thread trace dump instructions 2 -t",
+          substrs=["67910: [tsc=0x008fb5211bfdf270] 0x0000000000400bd7    addl   $0x1, -0x4(%rbp)",
+                   "m.out`bar() + 26 at multi_thread.cpp:20:6"])
+
     def testLoadTrace(self):
         src_dir = self.getSourceDir()
         trace_definition_file = os.path.join(src_dir, "intelpt-trace", "trace.json")
         self.expect("trace load -v " + trace_definition_file, substrs=["intel-pt"])
 
         target = self.dbg.GetSelectedTarget()
         process = target.GetProcess()
         self.assertEqual(process.GetProcessID(), 1234)

lldb/test/API/commands/trace/intelpt-multi-core-trace/trace_missing_threads.json

+{
+  "cores": [
+    {
+      "contextSwitchTrace": "/tmp/trace8/cores/45.perf_context_switch_trace",
+      "coreId": 45,
+      "traceBuffer": "/tmp/trace8/cores/45.intelpt_trace"
+    },
+    {
+      "contextSwitchTrace": "/tmp/trace8/cores/51.perf_context_switch_trace",
+      "coreId": 51,
+      "traceBuffer": "/tmp/trace8/cores/51.intelpt_trace"
+    }
+  ],
+  "cpuInfo": {
+    "family": 6,
+    "model": 85,
+    "stepping": 4,
+    "vendor": "GenuineIntel"
+  },
+  "processes": [
+    {
+      "modules": [
+        {
+          "file": "modules/m.out",
+          "systemPath": "/tmp/m.out",
+          "loadAddress": 4194304,
+          "uuid": "AEFB0D59-233F-80FF-6D3C-4DED498534CF-11017B3B"
+        }
+      ],
+      "pid": 3497234,
+      "threads": [
+        {
+          "tid": 3497234
+        }
+      ]
+    }
+  ],
+  "tscPerfZeroConversion": {
+    "timeMult": 1076264588,
+    "timeShift": 31,
+    "timeZero": 18433473881008870804
+  },
+  "type": "intel-pt"
+}