Cap size of the profiling file (#14)

* Profiling file can now be capped to a maximum size to prevent from
running out of memory in constrained systems (as described in #13).
* When capping is set, two rotating files are used instead of ones and
only the most recent events are stored.

Fix issue #13

Author: cedric-chedaleux

README.md

@@ -50,6 +50,14 @@ uprofile::timeBegin("my_custom_function");
 uprofile::timeEnd("my_custom_function");
 ```
 
+#### Limit the size of the profiling file
+
+```cpp
+uprofile::start("uprofile.log", 500000 /* max file size in bytes */);
+```
+
+It will generate two rotating files with the most recent events. With the above example, both files will be `uprofile_0.log` and `uprofile_1.log`.
+
 ### GPU monitoring
 
 The library also supports GPU metrics monitoring like usage and memory. Since GPU monitoring is specific to each vendor, an interface `IGPUMonitor` is available to abstract each vendor monitor system.
@@ -86,8 +94,8 @@ Here is the list of GPUs supported by `cppuprofile`
 The build process is based on CMake. Minimum version is 2.8.
 
 ```commandline
-$ cmake --configure . -B ../build-cppuprofile
-$ cmake --build ../build-cppuprofile
+$ cmake -Bbuild .
+$ cmake --build build
 ```
 
 ### Shared/dynamic library
@@ -106,7 +114,7 @@ If you want to disable profiling in Release mode or if you want to only enable p
 To disable the profiling:
 
 ```commandline
-$ cmake --configure . -B ../build-cppuprofile -DPROFILE_ENABLED=OFF
+$ cmake -Bbuild . -DPROFILE_ENABLED=OFF
 ```
 
 ## Tools
@@ -136,9 +144,9 @@ The project provides a C++ sample application called `uprof-sample`
 that shows how to use the `cppuprofile` library. You can build it with `SAMPLE_ENABLED` option:
 
 ```commandline
-$ cmake --configure . -B ../build-cppuprofile -DSAMPLE_ENABLED=ON
-$ cmake --build ../build-cppuprofile
-$ ../build-cppuprofile/sample/uprof-sample
+$ cmake -Bbuild . -DSAMPLE_ENABLED=ON
+$ cmake --build build
+$ ./build/sample/uprof-sample
 ```
 
 ## Windows support limitations

lib/CMakeLists.txt

@@ -41,6 +41,8 @@ SET(UProfile_PUBLIC_HEADERS
 SET(UProfile_IMPL
     uprofile.cpp
     uprofileimpl.cpp
+    eventsfile.h
+    eventsfile.cpp
     util/timer.cpp
     util/cpumonitor.cpp
 )

lib/eventsfile.cpp

@@ -0,0 +1,82 @@
+// Software Name : cppuprofile
+// SPDX-FileCopyrightText: Copyright (c) 2024 Orange
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// This software is distributed under the BSD License;
+// see the LICENSE file for more details.
+//
+// Author: Cédric CHEDALEUX <cedric.chedaleux@orange.com> et al.
+
+#include "eventsfile.h"
+
+#include <iostream>
+#include <sstream>
+
+using namespace std;
+
+namespace uprofile
+{
+
+EventsFile::EventsFile(const char* filepath, unsigned long long maxCapSize) :
+    m_maxCapSize(maxCapSize)
+{
+    if (m_maxCapSize > 0) {
+        string str = string(filepath);
+        auto found = str.find_last_of('.');
+        bool hasExtension = found != std::string::npos;
+        string basePath = hasExtension ? str.substr(0, found) : str;
+        string extension = hasExtension ? str.substr(found) : "";
+        for (int i = 0; i < ROTATING_FILES_NUMBER; ++i) {
+            m_filePaths.emplace_back(basePath + "_" + std::to_string(i) + extension);
+        }
+    } else {
+        m_filePaths.emplace_back(filepath);
+    }
+
+    m_file.open(m_filePaths[m_currentFileIdx], std::ios::out);
+    if (!m_file.is_open()) {
+        std::cerr << "Failed to open file: " << m_filePaths[m_currentFileIdx] << std::endl;
+    }
+}
+
+EventsFile::~EventsFile()
+{
+    m_file.close();
+}
+
+void EventsFile::write(const std::string& event, unsigned long long timestamp, const std::list<std::string>& data)
+{
+    string csvSeparator(";");
+    stringstream ss;
+    ss << event << csvSeparator << timestamp;
+    for (auto it = data.cbegin(); it != data.cend(); ++it) {
+        ss << csvSeparator << *it;
+    }
+    ss << "\n";
+    string line = ss.str();
+
+    // Total size of all rotating files should never exceed the defined max cap size
+    if (m_maxCapSize > 0 && m_currentFileSize + line.size() > m_maxCapSize / ROTATING_FILES_NUMBER) {
+        rotateFile();
+    }
+
+    std::lock_guard<std::mutex> guard(m_fileMutex);
+    m_file << line;
+    m_file.flush();
+    m_currentFileSize += line.size();
+}
+
+void EventsFile::rotateFile()
+{
+    m_file.close();
+    m_currentFileSize = 0;
+    if (++m_currentFileIdx >= m_filePaths.size()) {
+        m_currentFileIdx = 0;
+    }
+    m_file.open(m_filePaths[m_currentFileIdx], std::ios::out);
+    if (!m_file.is_open()) {
+        std::cerr << "Failed to open file: " << m_filePaths[m_currentFileIdx] << std::endl;
+    }
+}
+
+}

lib/eventsfile.h

@@ -0,0 +1,47 @@
+// Software Name : cppuprofile
+// SPDX-FileCopyrightText: Copyright (c) 2024 Orange
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// This software is distributed under the BSD License;
+// see the LICENSE file for more details.
+//
+// Author: Cédric CHEDALEUX <cedric.chedaleux@orange.com> et al.
+
+#ifndef EVENTSFILE_H_
+#define EVENTSFILE_H_
+
+#include <fstream>
+#include <list>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+
+namespace uprofile
+{
+
+class EventsFile
+{
+public:
+    EventsFile(const char* filepath, unsigned long long maxCapSize);
+    ~EventsFile();
+
+    void write(const std::string& event, unsigned long long timestamp, const std::list<std::string>& data);
+
+    static const int ROTATING_FILES_NUMBER = 2;
+
+private:
+    std::mutex m_fileMutex;
+    std::ofstream m_file;
+    std::vector<std::string> m_filePaths;
+    unsigned int m_currentFileIdx = 0;
+    unsigned long long m_currentFileSize = 0;
+    unsigned long long m_maxCapSize = 0;
+
+    void rotateFile();
+};
+using EventsFilePtr = std::shared_ptr<EventsFile>;
+
+}
+
+#endif /* EVENTSFILE_H_ */

lib/uprofile.cpp

@@ -34,9 +34,9 @@ using namespace std::chrono;
 namespace uprofile
 {
 
-void start(const char* file)
+void start(const char* filepath, unsigned long long maxCapSize)
 {
-    UPROFILE_INSTANCE_CALL(start, file);
+    UPROFILE_INSTANCE_CALL(start, filepath, maxCapSize);
 }
 
 void stop()

lib/uprofile.h

@@ -29,9 +29,15 @@ namespace uprofile
 /**
  * @ingroup uprofile
  * @brief Start profiling to periodically record monitored events to a file
- * @param file: file path where events will be saved
+ * @param filepath: file path where events will be saved
+ * @param maxCapSize: maximum file size in bytes
+ *
+ * By default, the file is unbounded.
+ *
+ * If you have some storage constraints, set maxCapSize parameter for generating two rotating files (<file>_0.<ext> and <file>_1.<ext>)
+ * Each file will have a maximum size of maxCapSize/2 (In this mode, more recent events will override older events)
  */
-UPROFAPI void start(const char* file);
+UPROFAPI void start(const char* filepath, unsigned long long maxCapSize = 0);
 
 /**
  * @ingroup uprofile

lib/uprofileimpl.cpp

@@ -52,12 +52,9 @@ UProfileImpl::~UProfileImpl()
     removeGPUMonitor();
 }
 
-void UProfileImpl::start(const char* file)
+void UProfileImpl::start(const char* filepath, unsigned long long maxCapSize)
 {
-    m_file.open(file, std::ios::out);
-    if (!m_file.is_open()) {
-        std::cerr << "Failed to open file: " << file << std::endl;
-    }
+    m_file = make_shared<EventsFile>(filepath, maxCapSize);
 }
 
 void UProfileImpl::addGPUMonitor(IGPUMonitor* monitor)
@@ -226,7 +223,7 @@ void UProfileImpl::stop()
     if (m_gpuMonitor) {
         m_gpuMonitor->stop();
     }
-    m_file.close();
+    m_file.reset();
 }
 
 void UProfileImpl::setTimestampUnit(TimestampUnit tsUnit)
@@ -236,43 +233,37 @@ void UProfileImpl::setTimestampUnit(TimestampUnit tsUnit)
 
 void UProfileImpl::write(ProfilingType type, const std::list<std::string>& data)
 {
-    if (m_file.is_open()) {
-        const char csvSeparator = ';';
-        std::string strType;
-        switch (type) {
-        case ProfilingType::TIME_EXEC:
-            strType = "time_exec";
-            break;
-        case ProfilingType::TIME_EVENT:
-            strType = "time_event";
-            break;
-        case ProfilingType::PROCESS_MEMORY:
-            strType = "proc_mem";
-            break;
-        case ProfilingType::SYSTEM_MEMORY:
-            strType = "sys_mem";
-            break;
-        case ProfilingType::CPU:
-            strType = "cpu";
-            break;
-        case ProfilingType::GPU_USAGE:
-            strType = "gpu";
-            break;
-        case ProfilingType::GPU_MEMORY:
-            strType = "gpu_mem";
-            break;
-        default:
-            strType = "undefined";
-            break;
-        }
-        std::lock_guard<std::mutex> guard(m_fileMutex);
-        m_file << strType.c_str() << csvSeparator << getTimestamp();
-        for (auto it = data.cbegin(); it != data.cend(); ++it) {
-            m_file << csvSeparator << *it;
-        }
-        m_file << "\n";
-        m_file.flush();
+    if (!m_file) {
+        return;
+    }
+    std::string strType;
+    switch (type) {
+    case ProfilingType::TIME_EXEC:
+        strType = "time_exec";
+        break;
+    case ProfilingType::TIME_EVENT:
+        strType = "time_event";
+        break;
+    case ProfilingType::PROCESS_MEMORY:
+        strType = "proc_mem";
+        break;
+    case ProfilingType::SYSTEM_MEMORY:
+        strType = "sys_mem";
+        break;
+    case ProfilingType::CPU:
+        strType = "cpu";
+        break;
+    case ProfilingType::GPU_USAGE:
+        strType = "gpu";
+        break;
+    case ProfilingType::GPU_MEMORY:
+        strType = "gpu_mem";
+        break;
+    default:
+        strType = "undefined";
+        break;
     }
+    m_file->write(strType, getTimestamp(), data);
 }
 
 unsigned long long UProfileImpl::getTimestamp() const

lib/uprofileimpl.h

@@ -10,6 +10,7 @@
 #ifndef UPROFILEIMPL_H_
 #define UPROFILEIMPL_H_
 
+#include "eventsfile.h"
 #include "igpumonitor.h"
 #include "timestampunit.h"
 #include "util/cpumonitor.h"
@@ -40,7 +41,7 @@ class UProfileImpl
     virtual ~UProfileImpl();
 
     // Implementation
-    void start(const char* file);
+    void start(const char* filepath, unsigned long long maxCapSize = 0);
     void stop();
     void addGPUMonitor(IGPUMonitor* monitor);
     void removeGPUMonitor();
@@ -73,7 +74,7 @@ class UProfileImpl
 
     TimestampUnit m_tsUnit;
     std::map<std::string, unsigned long long> m_steps; // Store steps (title, start time)
-    std::ofstream m_file;
+    EventsFilePtr m_file = nullptr;
     Timer m_processMemoryMonitorTimer;
     Timer m_systemMemoryMonitorTimer;
     Timer m_cpuMonitorTimer;

requirements.txt

@@ -1,3 +1,4 @@
 numpy
 plotly
 pandas
+pre-commit