[trace][intelpt] Support system-wide tracing [5] - Disable/enable per-core tracing based on the process state

[wallace]

Summary

When tracing on per-core mode, we are tracing all processes, which means
that after hitting a breakpoint, our process will stop running (thus
producing no more tracing data) but other processes will continue
writing to our trace buffers. This causes a big data loss for our trace.
As a way to remediate this, I'm adding some logic to pause and unpause
tracing based on the target's state. The earlier we do it the better,
however, I'm not adding the trigger at the earliest possible point for
simplicity of this diff. Later we can improve that part.

[wallace]

add better error handling

[jj10306]

lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp

271

What's the purpose of this new flush flag? When would you want to call this method with it set to false? I can only think of cases when you want to flush the buffer if you're trying to read its data

lldb/source/Plugins/Process/Linux/Perf.h

221–227

Nice, happy to see we are extending the mini perf API (:
In the future we also can update this to control whether or not the event is enabled at perf_event_open time because that's the default but there may be cases where you don't want the event to be enabled until you explicitly enable it w ioctl

[wallace]

lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp

271

This is nonsense, I shouldn't have created this flag =P

lldb/source/Plugins/Process/Linux/Perf.h

221–227

that's a good idea. I think I'll do it now

[wallace]

• remove the flush parameter
• use perf_event_attr.disabled to set the initial state of the collection

[jj10306]

lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp

211

nit: from the "Set" name, my first impression was this method was simply setting m_collection_state, but in reality it's doing some nontrivial operations, namely changing the state of the perf event via ioctl. Consider changing the name from "Set" to avoid giving the impression of a trivial operation

309

nice

lldb/include/lldb/Host/common/NativeProcessProtocol.h

protected:
   /// Notify the delegate that an exec occurred.
   ///
   /// Provide a mechanism for a delegate to clear out any exec-
   /// sensitive data.
   void NotifyDidExec();
 
   NativeThreadProtocol *GetThreadByIDUnlocked(lldb::tid_t tid);
 
+  /// Notify tracers that the state of the target process has changed.
+  virtual void NotifyTracersProcessStateChanged(lldb::StateType state) {}
+
 private:
   void SynchronouslyNotifyProcessStateChanged(lldb::StateType state);
   llvm::Expected<SoftwareBreakpoint>
   EnableSoftwareBreakpoint(lldb::addr_t addr, uint32_t size_hint);
 };
 } // namespace lldb_private
 
 #endif // LLDB_HOST_COMMON_NATIVEPROCESSPROTOCOL_H

lldb/source/Host/common/NativeProcessProtocol.cpp

Status NativeProcessProtocol::RemoveHardwareBreakpoint(lldb::addr_t addr) {
   return error;
 }
 
 void NativeProcessProtocol::SynchronouslyNotifyProcessStateChanged(
     lldb::StateType state) {
   Log *log = GetLog(LLDBLog::Process);
 
   m_delegate.ProcessStateChanged(this, state);
+  NotifyTracersProcessStateChanged(state);
 
   LLDB_LOG(log, "sent state notification [{0}] from process {1}", state,
            GetID());
 }
 
 void NativeProcessProtocol::NotifyDidExec() {
   Log *log = GetLog(LLDBLog::Process);
   LLDB_LOG(log, "process {0} exec()ed", GetID());

lldb/source/Plugins/Process/Linux/IntelPTCollector.h

public:
   void Clear();
 
   bool TracesThread(lldb::tid_t tid) const;
 
   size_t GetTotalBufferSize() const;
 
   std::vector<TraceThreadState> GetThreadStates() const;
 
-  llvm::Expected<const IntelPTSingleBufferTrace &>
-  GetTracedThread(lldb::tid_t tid) const;
+  llvm::Expected<IntelPTSingleBufferTrace &> GetTracedThread(lldb::tid_t tid);
 
   llvm::Error TraceStart(lldb::tid_t tid,
                          const TraceIntelPTStartRequest &request);
 
   llvm::Error TraceStop(lldb::tid_t tid);
 
   size_t GetTracedThreadsCount() const;
 
public:
   ///   An \a IntelPTMultiCoreTrace instance if tracing was successful, or
   ///   an \a llvm::Error otherwise.
   static llvm::Expected<IntelPTPerThreadProcessTraceUP>
   Start(const TraceIntelPTStartRequest &request,
         llvm::ArrayRef<lldb::tid_t> current_tids);
 
   bool TracesThread(lldb::tid_t tid) const;
 
-  const IntelPTThreadTraceCollection &GetThreadTraces() const;
+  IntelPTThreadTraceCollection &GetThreadTraces();
 
   llvm::Error TraceStart(lldb::tid_t tid);
 
   llvm::Error TraceStop(lldb::tid_t tid);
 
 private:
   IntelPTPerThreadProcessTrace(const TraceIntelPTStartRequest &request)
       : m_tracing_params(request) {}
 
   IntelPTThreadTraceCollection m_thread_traces;
   /// Params used to trace threads when the user started "process tracing".
   TraceIntelPTStartRequest m_tracing_params;
 };
 
 /// Main class that manages intel-pt process and thread tracing.
 class IntelPTCollector {
 public:
   /// \param[in] process
   ///     Process to be traced.
   IntelPTCollector(NativeProcessProtocol &process);
 
   static bool IsSupported();
 
+  /// To be invoked whenever the state of the target process has changed.
+  void OnProcessStateChanged(lldb::StateType state);
+
   /// If "process tracing" is enabled, then trace the given thread.
   llvm::Error OnThreadCreated(lldb::tid_t tid);
 
   /// Stops tracing a tracing upon a destroy event.
   llvm::Error OnThreadDestroyed(lldb::tid_t tid);
 
   /// Implementation of the jLLDBTraceStop packet
   llvm::Error TraceStop(const TraceStopRequest &request);
 
   /// Implementation of the jLLDBTraceStart packet
   llvm::Error TraceStart(const TraceIntelPTStartRequest &request);
 
   /// Implementation of the jLLDBTraceGetState packet
   llvm::Expected<llvm::json::Value> GetState() const;
 
   /// Implementation of the jLLDBTraceGetBinaryData packet
   llvm::Expected<std::vector<uint8_t>>
-  GetBinaryData(const TraceGetBinaryDataRequest &request) const;
+  GetBinaryData(const TraceGetBinaryDataRequest &request);
 
   /// Dispose of all traces
   void Clear();
 
 private:
   llvm::Error TraceStop(lldb::tid_t tid);
 
   /// Start tracing a specific thread.
   llvm::Error TraceStart(lldb::tid_t tid,
                          const TraceIntelPTStartRequest &request);
 
-  llvm::Expected<const IntelPTSingleBufferTrace &>
-  GetTracedThread(lldb::tid_t tid) const;
+  llvm::Expected<IntelPTSingleBufferTrace &> GetTracedThread(lldb::tid_t tid);
 
   bool IsProcessTracingEnabled() const;
 
   void ClearProcessTracing();
 
   NativeProcessProtocol &m_process;
   /// Threads traced due to "thread tracing"
   IntelPTThreadTraceCollection m_thread_traces;

lldb/source/Plugins/Process/Linux/IntelPTCollector.cpp

 
 Error IntelPTThreadTraceCollection::TraceStart(
     lldb::tid_t tid, const TraceIntelPTStartRequest &request) {
   if (TracesThread(tid))
     return createStringError(inconvertibleErrorCode(),
                              "Thread %" PRIu64 " already traced", tid);
 
   Expected<IntelPTSingleBufferTraceUP> trace_up =
-      IntelPTSingleBufferTrace::Start(request, tid);
+      IntelPTSingleBufferTrace::Start(request, tid, /*core_id=*/None,
+                                      TraceCollectionState::Running);
   if (!trace_up)
     return trace_up.takeError();
 
   m_total_buffer_size += (*trace_up)->GetTraceBufferSize();
   m_thread_traces.try_emplace(tid, std::move(*trace_up));
   return Error::success();
 }
 
 size_t IntelPTThreadTraceCollection::GetTotalBufferSize() const {
   return m_total_buffer_size;
 }
 
 std::vector<TraceThreadState>
 IntelPTThreadTraceCollection::GetThreadStates() const {
   std::vector<TraceThreadState> states;
   for (const auto &it : m_thread_traces)
     states.push_back({it.first,
                       {TraceBinaryData{IntelPTDataKinds::kTraceBuffer,
                                        it.second->GetTraceBufferSize()}}});
   return states;
 }
 
-Expected<const IntelPTSingleBufferTrace &>
-IntelPTThreadTraceCollection::GetTracedThread(lldb::tid_t tid) const {
+Expected<IntelPTSingleBufferTrace &>
+IntelPTThreadTraceCollection::GetTracedThread(lldb::tid_t tid) {
   auto it = m_thread_traces.find(tid);
   if (it == m_thread_traces.end())
     return createStringError(inconvertibleErrorCode(),
                              "Thread %" PRIu64 " not currently traced", tid);
   return *it->second.get();
 }
 
 void IntelPTThreadTraceCollection::Clear() {
return createStringError(
         "Thread %" PRIu64 " can't be traced as the process trace size limit "
         "has been reached. Consider retracing with a higher "
         "limit.",
         tid);
 
   return m_thread_traces.TraceStart(tid, m_tracing_params);
 }
 
-const IntelPTThreadTraceCollection &
-IntelPTPerThreadProcessTrace::GetThreadTraces() const {
+IntelPTThreadTraceCollection &IntelPTPerThreadProcessTrace::GetThreadTraces() {
   return m_thread_traces;
 }
 
 /// IntelPTCollector
 
 IntelPTCollector::IntelPTCollector(NativeProcessProtocol &process)
     : m_process(process) {
   if (Expected<LinuxPerfZeroTscConversion> tsc_conversion =
if (request.IsProcessTracing()) {
     Error error = Error::success();
     for (int64_t tid : *request.tids)
       error = joinErrors(std::move(error),
                          m_thread_traces.TraceStart(tid, request));
     return error;
   }
 }
 
+void IntelPTCollector::OnProcessStateChanged(lldb::StateType state) {
+  if (m_per_core_process_trace_up)
+    m_per_core_process_trace_up->OnProcessStateChanged(state);
+}
+
 Error IntelPTCollector::OnThreadCreated(lldb::tid_t tid) {
   if (m_per_thread_process_trace_up)
     return m_per_thread_process_trace_up->TraceStart(tid);
 
   return Error::success();
 }
 
 Error IntelPTCollector::OnThreadDestroyed(lldb::tid_t tid) {
m_per_core_process_trace_up->ForEachCore(
           state.cores->push_back({core_id,
                                   {{IntelPTDataKinds::kTraceBuffer,
                                     core_trace.GetTraceBufferSize()}}});
         });
   }
   return toJSON(state);
 }
 
-Expected<const IntelPTSingleBufferTrace &>
-IntelPTCollector::GetTracedThread(lldb::tid_t tid) const {
+Expected<IntelPTSingleBufferTrace &>
+IntelPTCollector::GetTracedThread(lldb::tid_t tid) {
   if (m_per_thread_process_trace_up &&
       m_per_thread_process_trace_up->TracesThread(tid))
     return m_per_thread_process_trace_up->GetThreadTraces().GetTracedThread(
         tid);
   return m_thread_traces.GetTracedThread(tid);
 }
 
 Expected<std::vector<uint8_t>>
-IntelPTCollector::GetBinaryData(const TraceGetBinaryDataRequest &request) const {
+IntelPTCollector::GetBinaryData(const TraceGetBinaryDataRequest &request) {
   if (request.kind == IntelPTDataKinds::kTraceBuffer) {
-    if (Expected<const IntelPTSingleBufferTrace &> trace =
+    if (Expected<IntelPTSingleBufferTrace &> trace =
             GetTracedThread(*request.tid))
       return trace->GetTraceBuffer(request.offset, request.size);
     else
       return trace.takeError();
   } else if (request.kind == IntelPTDataKinds::kProcFsCpuInfo) {
     return GetProcfsCpuInfo();
   }
   return createStringError(inconvertibleErrorCode(),

lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.h

public:
 
   /// Execute the provided callback on each core that is being traced.
   ///
   /// \param[in] callback.core_id
   ///   The core id that is being traced.
   ///
   /// \param[in] callback.core_trace
   ///   The single-buffer trace instance for the given core.
-  void
-  ForEachCore(std::function<void(lldb::core_id_t core_id,
-                                 const IntelPTSingleBufferTrace &core_trace)>
+  void ForEachCore(std::function<void(lldb::core_id_t core_id,
+                                      IntelPTSingleBufferTrace &core_trace)>
                        callback);
 
+  /// This method should be invoked as early as possible whenever the process
+  /// resumes or stops so that intel-pt collection is not enabled when
+  /// the process is not running. This is done to prevent polluting the core
+  /// traces with executions of unrelated processes, which increases the data
+  /// loss of the target process, given that core traces don't filter by
+  /// process.
+  /// A possible way to avoid this is to use CR3 filtering, which is equivalent
+  /// to process filtering, but the perf_event API doesn't support it.
+  void OnProcessStateChanged(lldb::StateType state);
+
 private:
   IntelPTMultiCoreTrace(
       llvm::DenseMap<lldb::core_id_t, IntelPTSingleBufferTraceUP>
           &&traces_per_core)
       : m_traces_per_core(std::move(traces_per_core)) {}
 
   llvm::DenseMap<lldb::core_id_t, IntelPTSingleBufferTraceUP> m_traces_per_core;
+
+  /// The initial state is stopped because tracing can only start when the
+  /// process is paused.
+  lldb::StateType m_process_state = lldb::StateType::eStateStopped;
 };
 
 } // namespace process_linux
 } // namespace lldb_private
 
 #endif // liblldb_IntelPTMultiCoreTrace_H_

lldb/source/Plugins/Process/Linux/IntelPTMultiCoreTrace.cpp

 //===-- IntelPTMultiCoreTrace.cpp -----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "IntelPTMultiCoreTrace.h"
 
 #include "Procfs.h"
 
+#include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
+
 using namespace lldb;
 using namespace lldb_private;
 using namespace process_linux;
 using namespace llvm;
 
 static bool IsTotalBufferLimitReached(ArrayRef<core_id_t> cores,
                                       const TraceIntelPTStartRequest &request) {
   uint64_t required = cores.size() * request.trace_buffer_size;
if (IsTotalBufferLimitReached(*core_ids, request))
     return createStringError(
         inconvertibleErrorCode(),
         "The process can't be traced because the process trace size limit "
         "has been reached. Consider retracing with a higher limit.");
 
   llvm::DenseMap<core_id_t, IntelPTSingleBufferTraceUP> buffers;
   for (core_id_t core_id : *core_ids) {
     if (Expected<IntelPTSingleBufferTraceUP> core_trace =
-            IntelPTSingleBufferTrace::Start(request, /*tid=*/None, core_id))
+            IntelPTSingleBufferTrace::Start(request, /*tid=*/None, core_id,
+                                            TraceCollectionState::Paused))
       buffers.try_emplace(core_id, std::move(*core_trace));
     else
       return IncludePerfEventParanoidMessageInError(core_trace.takeError());
   }
 
   return IntelPTMultiCoreTraceUP(new IntelPTMultiCoreTrace(std::move(buffers)));
 }
 
 void IntelPTMultiCoreTrace::ForEachCore(
-    std::function<void(core_id_t core_id,
-                       const IntelPTSingleBufferTrace &core_trace)>
+    std::function<void(core_id_t core_id, IntelPTSingleBufferTrace &core_trace)>
         callback) {
   for (auto &it : m_traces_per_core)
     callback(it.first, *it.second);
 }
+
+void IntelPTMultiCoreTrace::OnProcessStateChanged(lldb::StateType state) {
+  if (m_process_state == state)
+    return;
+  switch (state) {
+  case eStateStopped:
+  case eStateExited: {
+    ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
+      if (Error err =
+              core_trace.ChangeCollectionState(TraceCollectionState::Paused)) {
+        LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
+                       "Unable to pause the core trace for core {0}", core_id);
+      }
+    });
+    break;
+  }
+  case eStateRunning: {
+    ForEachCore([](core_id_t core_id, IntelPTSingleBufferTrace &core_trace) {
+      if (Error err =
+              core_trace.ChangeCollectionState(TraceCollectionState::Running)) {
+        LLDB_LOG_ERROR(GetLog(POSIXLog::Trace), std::move(err),
+                       "Unable to resume the core trace for core {0}", core_id);
+      }
+    });
+    break;
+  }
+  default:
+    break;
+  }
+}

lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.h

 namespace process_linux {
 
 llvm::Expected<uint32_t> GetIntelPTOSEventType();
 
 class IntelPTSingleBufferTrace;
 
 using IntelPTSingleBufferTraceUP = std::unique_ptr<IntelPTSingleBufferTrace>;
 
+enum class TraceCollectionState {
+  Running,
+  Paused,
+};
+
 /// This class wraps a single perf event collecting intel pt data in a single
 /// buffer.
 class IntelPTSingleBufferTrace {
 public:
   /// Start tracing using a single Intel PT trace buffer.
   ///
   /// \param[in] request
   ///     Intel PT configuration parameters.
   ///
   /// \param[in] tid
   ///     The tid of the thread to be traced. If \b None, then this traces all
   ///     threads of all processes.
   ///
   /// \param[in] core_id
   ///     The CPU core id where to trace. If \b None, then this traces all CPUs.
   ///
+  /// \param[in] initial_state
+  ///     The initial trace collection state.
+  ///
   /// \return
   ///   A \a IntelPTSingleBufferTrace instance if tracing was successful, or
   ///   an \a llvm::Error otherwise.
   static llvm::Expected<IntelPTSingleBufferTraceUP>
   Start(const TraceIntelPTStartRequest &request,
         llvm::Optional<lldb::tid_t> tid,
-        llvm::Optional<lldb::core_id_t> core_id = llvm::None);
+        llvm::Optional<lldb::core_id_t> core_id,
+        TraceCollectionState initial_state);
 
   /// \return
   ///    The bytes requested by a jLLDBTraceGetBinaryData packet that was routed
   ///    to this trace instace.
   llvm::Expected<std::vector<uint8_t>>
   GetBinaryData(const TraceGetBinaryDataRequest &request) const;
 
-  /// Read the trace buffer managed by this trace instance.
+  /// Read the trace buffer managed by this trace instance. To ensure that the
+  /// data is up-to-date and is not corrupted by read-write race conditions, the
+  /// underlying perf_event is paused during read, and later it's returned to
+  /// its initial state.
   ///
   /// \param[in] offset
   ///     Offset of the data to read.
   ///
   /// \param[in] size
   ///     Number of bytes to read.
   ///
   /// \return
   ///     A vector with the requested binary data. The vector will have the
   ///     size of the requested \a size. Non-available positions will be
   ///     filled with zeroes.
   llvm::Expected<std::vector<uint8_t>> GetTraceBuffer(size_t offset,
-                                                      size_t size) const;
+                                                      size_t size);
 
   /// \return
-  ///   The total the size in bytes used by the trace buffer managed by this
-  ///   trace instance.
+  ///     The total the size in bytes used by the trace buffer managed by this
+  ///     trace instance.
   size_t GetTraceBufferSize() const;
 
+  /// Change the collection state for this trace.
+  ///
+  /// This is a no-op if \p state is the same as the current state.
+  ///
+  /// \param[in] state
+  ///     The new state.
+  ///
+  /// \return
+  ///     An error if the state couldn't be changed.
+  llvm::Error ChangeCollectionState(TraceCollectionState state);
+
 private:
   /// Construct new \a IntelPTSingleBufferThreadTrace. Users are supposed to
   /// create instances of this class via the \a Start() method and not invoke
   /// this one directly.
   ///
   /// \param[in] perf_event
   ///   perf event configured for IntelPT.
-  IntelPTSingleBufferTrace(PerfEvent &&perf_event)
-      : m_perf_event(std::move(perf_event)) {}
+  ///
+  /// \param[in] collection_state
+  ///   The initial collection state for the provided perf_event.
+  IntelPTSingleBufferTrace(PerfEvent &&perf_event,
+                           TraceCollectionState collection_state)
+      : m_perf_event(std::move(perf_event)),
+        m_collection_state(collection_state) {}
 
   /// perf event configured for IntelPT.
   PerfEvent m_perf_event;
+
+  /// The initial state is stopped because tracing can only start when the
+  /// process is paused.
+  TraceCollectionState m_collection_state;
 };
 
 } // namespace process_linux
 } // namespace lldb_private
 
 #endif // liblldb_IntelPTSingleBufferTrace_H_

lldb/source/Plugins/Process/Linux/IntelPTSingleBufferTrace.cpp

 #include "lldb/Utility/StreamString.h"
 
 #include "llvm/Support/Host.h"
 #include "llvm/Support/MemoryBuffer.h"
 
 #include <sstream>
 
 #include <linux/perf_event.h>
-#include <sys/ioctl.h>
 #include <sys/syscall.h>
 #include <unistd.h>
 
 using namespace lldb;
 using namespace lldb_private;
 using namespace process_linux;
 using namespace llvm;
 
CreateIntelPTPerfEventConfiguration(bool enable_tsc,
   return attr;
 }
 #endif
 
 size_t IntelPTSingleBufferTrace::GetTraceBufferSize() const {
   return m_perf_event.GetAuxBuffer().size();
 }
 
+Error IntelPTSingleBufferTrace::ChangeCollectionState(
+    TraceCollectionState new_state) {
+  if (new_state == m_collection_state)
+    return Error::success();
+
+  switch (new_state) {
+  case TraceCollectionState::Paused:
+    if (Error err = m_perf_event.DisableWithIoctl())
+      return err;
+    break;
+  case TraceCollectionState::Running:
+    if (Error err = m_perf_event.EnableWithIoctl())
+      return err;
+    break;
+  }
+  m_collection_state = new_state;
+  return Error::success();
+}
+
 Expected<std::vector<uint8_t>>
-IntelPTSingleBufferTrace::GetTraceBuffer(size_t offset, size_t size) const {
-  auto fd = m_perf_event.GetFd();
-  perf_event_mmap_page &mmap_metadata = m_perf_event.GetMetadataPage();
+IntelPTSingleBufferTrace::GetTraceBuffer(size_t offset, size_t size) {
   // Disable the perf event to force a flush out of the CPU's internal buffer.
   // Besides, we can guarantee that the CPU won't override any data as we are
   // reading the buffer.
-  //
   // The Intel documentation says:
   //
-  // Packets are first buffered internally and then written out asynchronously.
-  // To collect packet output for postprocessing, a collector needs first to
-  // ensure that all packet data has been flushed from internal buffers.
-  // Software can ensure this by stopping packet generation by clearing
-  // IA32_RTIT_CTL.TraceEn (see “Disabling Packet Generation” in
+  // Packets are first buffered internally and then written out
+  // asynchronously. To collect packet output for postprocessing, a collector
+  // needs first to ensure that all packet data has been flushed from internal
+  // buffers. Software can ensure this by stopping packet generation by
+  // clearing IA32_RTIT_CTL.TraceEn (see “Disabling Packet Generation” in
   // Section 35.2.7.2).
   //
-  // This is achieved by the PERF_EVENT_IOC_DISABLE ioctl request, as mentioned
-  // in the man page of perf_event_open.
-  ioctl(fd, PERF_EVENT_IOC_DISABLE);
+  // This is achieved by the PERF_EVENT_IOC_DISABLE ioctl request, as
+  // mentioned in the man page of perf_event_open.
+  TraceCollectionState previous_state = m_collection_state;
+  if (Error err = ChangeCollectionState(TraceCollectionState::Paused))
+    return std::move(err);
 
+  std::vector<uint8_t> data(size, 0);
+  perf_event_mmap_page &mmap_metadata = m_perf_event.GetMetadataPage();
   Log *log = GetLog(POSIXLog::Trace);
-  Status error;
   uint64_t head = mmap_metadata.aux_head;
 
   LLDB_LOG(log, "Aux size -{0} , Head - {1}", mmap_metadata.aux_size, head);
 
   /**
    * When configured as ring buffer, the aux buffer keeps wrapping around
    * the buffer and its not possible to detect how many times the buffer
    * wrapped. Initially the buffer is filled with zeros,as shown below
    * so in order to get complete buffer we first copy firstpartsize, followed
    * by any left over part from beginning to aux_head
    *
    * aux_offset [d,d,d,d,d,d,d,d,0,0,0,0,0,0,0,0,0,0,0] aux_size
    *                 aux_head->||<- firstpartsize  ->|
    *
    * */
 
-  std::vector<uint8_t> data(size, 0);
   MutableArrayRef<uint8_t> buffer(data);
   ReadCyclicBuffer(buffer, m_perf_event.GetAuxBuffer(),
                    static_cast<size_t>(head), offset);
 
-  // Reenable tracing now we have read the buffer
-  ioctl(fd, PERF_EVENT_IOC_ENABLE);
+  if (Error err = ChangeCollectionState(previous_state))
+    return std::move(err);
+
   return data;
 }
 
-Expected<IntelPTSingleBufferTraceUP>
-IntelPTSingleBufferTrace::Start(const TraceIntelPTStartRequest &request,
-                                Optional<lldb::tid_t> tid,
-                                Optional<core_id_t> core_id) {
+Expected<IntelPTSingleBufferTraceUP> IntelPTSingleBufferTrace::Start(
+    const TraceIntelPTStartRequest &request, Optional<lldb::tid_t> tid,
+    Optional<core_id_t> core_id, TraceCollectionState initial_state) {
 #ifndef PERF_ATTR_SIZE_VER5
   return createStringError(inconvertibleErrorCode(),
                            "Intel PT Linux perf event not supported");
 #else
   Log *log = GetLog(POSIXLog::Trace);
 
   LLDB_LOG(log, "Will start tracing thread id {0} and cpu id {1}", tid,
            core_id);
uint64_t buffer_numpages = static_cast<uint64_t>(llvm::PowerOf2Floor(
       (request.trace_buffer_size + page_size - 1) / page_size));
 
   Expected<perf_event_attr> attr = CreateIntelPTPerfEventConfiguration(
       request.enable_tsc, request.psb_period.map([](int value) {
         return static_cast<uint64_t>(value);
       }));
   if (!attr)
     return attr.takeError();
+  attr->disabled = initial_state == TraceCollectionState::Paused;
 
   LLDB_LOG(log, "Will create trace buffer of size {0}",
            request.trace_buffer_size);
 
   if (Expected<PerfEvent> perf_event = PerfEvent::Init(*attr, tid, core_id)) {
     if (Error mmap_err = perf_event->MmapMetadataAndBuffers(buffer_numpages,
                                                             buffer_numpages)) {
       return std::move(mmap_err);
     }
-    return IntelPTSingleBufferTraceUP(
-        new IntelPTSingleBufferTrace(std::move(*perf_event)));
+    IntelPTSingleBufferTraceUP trace_up(
+        new IntelPTSingleBufferTrace(std::move(*perf_event), initial_state));
+    return trace_up;
   } else {
     return perf_event.takeError();
   }
 #endif
 }

lldb/source/Plugins/Process/Linux/NativeProcessLinux.h

private:
   Status NotifyTracersOfNewThread(lldb::tid_t tid);
 
   /// Stop tracing threads upon a destroy event.
   ///
   /// Trace mechanisms should modify this method to provide automatic trace
   /// stopping for threads being destroyed.
   Status NotifyTracersOfThreadDestroyed(lldb::tid_t tid);
 
+  void NotifyTracersProcessStateChanged(lldb::StateType state) override;
+
   /// Writes the raw event message code (vis-a-vis PTRACE_GETEVENTMSG)
   /// corresponding to the given thread ID to the memory pointed to by @p
   /// message.
   Status GetEventMessage(lldb::tid_t tid, unsigned long *message);
 
   void NotifyThreadDeath(lldb::tid_t tid);
 
   Status Detach(lldb::tid_t tid);

lldb/source/Plugins/Process/Linux/NativeProcessLinux.cpp

void NativeProcessLinux::StopTrackingThread(NativeThreadLinux &thread) {
   });
   assert(it != m_threads.end());
   m_threads.erase(it);
 
   NotifyTracersOfThreadDestroyed(thread_id);
   SignalIfAllThreadsStopped();
 }
 
+void NativeProcessLinux::NotifyTracersProcessStateChanged(
+    lldb::StateType state) {
+  m_intel_pt_collector.OnProcessStateChanged(state);
+}
+
 Status NativeProcessLinux::NotifyTracersOfNewThread(lldb::tid_t tid) {
   Log *log = GetLog(POSIXLog::Thread);
   Status error(m_intel_pt_collector.OnThreadCreated(tid));
   if (error.Fail())
     LLDB_LOG(log, "Failed to trace a new thread with intel-pt, tid = {0}. {1}",
              tid, error.AsCString());
   return error;
 }

lldb/source/Plugins/Process/Linux/Perf.h

public:
   ///
   /// This should be called only after \a PerfEvent::MmapMetadataAndBuffers,
   /// otherwise a failure might happen.
   ///
   /// \return
   ///   \a ArrayRef<uint8_t> extending \a aux_size bytes from \a aux_offset.
   llvm::ArrayRef<uint8_t> GetAuxBuffer() const;
 
+  /// Use the ioctl API to disable the perf event. This doesn't terminate the
+  /// perf event.
+  ///
+  /// \return
+  ///   An Error if the perf event couldn't be disabled.
+  llvm::Error DisableWithIoctl() const;
+
+  /// Use the ioctl API to enable the perf event.
+  ///
+  /// \return
+  ///   An Error if the perf event couldn't be enabled.
+  llvm::Error EnableWithIoctl() const;
+
 private:
   /// Create new \a PerfEvent.
   ///
   /// \param[in] fd
   ///   File descriptor of the perf event.
   PerfEvent(long fd)
       : m_fd(new long(fd), resource_handle::FileDescriptorDeleter()),
         m_metadata_data_base(), m_aux_base() {}

lldb/source/Plugins/Process/Linux/Perf.cpp

 
 #include "Plugins/Process/POSIX/ProcessPOSIXLog.h"
 #include "lldb/Host/linux/Support.h"
 
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
 
+#include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
 
 using namespace lldb_private;
 using namespace process_linux;
 using namespace llvm;
 
return {reinterpret_cast<uint8_t *>(m_metadata_data_base.get()) +
            static_cast<size_t>(mmap_metadata.data_size)};
 }
 
 ArrayRef<uint8_t> PerfEvent::GetAuxBuffer() const {
   perf_event_mmap_page &mmap_metadata = GetMetadataPage();
   return {reinterpret_cast<uint8_t *>(m_aux_base.get()),
            static_cast<size_t>(mmap_metadata.aux_size)};
 }
+
+Error PerfEvent::DisableWithIoctl() const {
+  if (ioctl(*m_fd, PERF_EVENT_IOC_DISABLE) < 0)
+    return createStringError(inconvertibleErrorCode(),
+                             "Can't disable perf event. %s",
+                             std::strerror(errno));
+  return Error::success();
+}
+
+Error PerfEvent::EnableWithIoctl() const {
+  if (ioctl(*m_fd, PERF_EVENT_IOC_ENABLE) < 0)
+    return createStringError(inconvertibleErrorCode(),
+                             "Can't disable perf event. %s",
+                             std::strerror(errno));
+  return Error::success();
+}