Capture c++ stack traces from core dump files on linux (#38600)

* add pystack to workbench and developer environments

* Turn on core dumps on linux for the inner workbench process

Has to switch from multiprocessing to subprocess in order to do this (needed the preexec option).
Through some testing, this should preserve the desired spawn behaviour.
A new module 'workbench_process.py' has been created to use with subprocess.run (can't pass it a function)

* add module to get the compressed output from pystack on the latest worbench core file

* add tests for new run pystack module

* add cpp traces to the report sender

* base64 encode data to store as a string

* show pystack output in the show more details window

* add release note

* update existing tests

* fix pystack test

* add support for lz4 compressed core files, such as those on idaaas

* only import lz4 if on linux

* return Path type as intended

* use process id to identify core dump file

* move release note to 6.13

* pass missing parameter to errorreporter constructor

* only launch with preexec_func when not on windows to avoid crash

* add property to user properties default file

* Update comment in run_pystack

---------

Co-authored-by: thomashampson <thomas.hampson@stfc.ac.uk>

Author: jhaigh0

Framework/Kernel/inc/MantidKernel/ErrorReporter.h

@@ -30,7 +30,8 @@ class MANTID_KERNEL_DLL ErrorReporter {
                 const std::string &textBox);
   /// Constructor
   ErrorReporter(std::string application, Types::Core::time_duration startTime, std::string exitCode, bool share,
-                std::string name, std::string email, std::string textBox, std::string stacktrace);
+                std::string name, std::string email, std::string textBox, std::string stacktrace,
+                std::string cppTraces);
   /// Sends an error report
   Kernel::InternetHelper::HTTPStatus sendErrorReport();
   /// Generates an error string in json format
@@ -50,6 +51,7 @@ class MANTID_KERNEL_DLL ErrorReporter {
   const std::string m_textbox;
   std::string m_url;
   const std::string m_stacktrace;
+  const std::string m_cppTraces;
 };
 
 } // namespace Kernel

Framework/Kernel/src/ErrorReporter.cpp

@@ -33,21 +33,21 @@ Logger g_log("ErrorReporter");
  */
 ErrorReporter::ErrorReporter(const std::string &application, const Types::Core::time_duration &upTime,
                              const std::string &exitCode, const bool share)
-    : ErrorReporter(application, upTime, exitCode, share, "", "", "", "") {}
+    : ErrorReporter(application, upTime, exitCode, share, "", "", "", "", "") {}
 
 /** Constructor
  */
 ErrorReporter::ErrorReporter(const std::string &application, const Types::Core::time_duration &upTime,
                              const std::string &exitCode, const bool share, const std::string &name,
                              const std::string &email, const std::string &textBox)
-    : ErrorReporter(application, upTime, exitCode, share, name, email, textBox, "") {}
+    : ErrorReporter(application, upTime, exitCode, share, name, email, textBox, "", "") {}
 
 ErrorReporter::ErrorReporter(std::string application, Types::Core::time_duration upTime, std::string exitCode,
                              const bool share, std::string name, std::string email, std::string textBox,
-                             std::string traceback)
+                             std::string traceback, std::string cppTraces)
     : m_application(std::move(application)), m_exitCode(std::move(exitCode)), m_upTime(std::move(upTime)),
       m_share(share), m_name(std::move(name)), m_email(std::move(email)), m_textbox(std::move(textBox)),
-      m_stacktrace(std::move(traceback)) {
+      m_stacktrace(std::move(traceback)), m_cppTraces(std::move(cppTraces)) {
   auto url = Mantid::Kernel::ConfigService::Instance().getValue<std::string>("errorreports.rooturl");
   if (!url.has_value()) {
     g_log.debug() << "Failed to load error report url\n";
@@ -113,11 +113,13 @@ std::string ErrorReporter::generateErrorMessage() const {
     message["email"] = m_email;
     message["name"] = m_name;
     message["stacktrace"] = m_stacktrace;
+    message["cppCompressedTraces"] = m_cppTraces;
   } else {
     message["email"] = "";
     message["name"] = "";
     message["textBox"] = m_textbox;
     message["stacktrace"] = "";
+    message["cppCompressedTraces"] = "";
   }
 
   return Mantid::JsonHelpers::jsonToString(message);

Framework/Kernel/test/ErrorReporterTest.h

@@ -66,7 +66,7 @@ class ErrorReporterTest : public CxxTest::TestSuite {
     const std::string appName = "My testing application name";
     const Mantid::Types::Core::time_duration upTime(5, 0, 7, 0);
     const std::string stackTrace = "File \" C :\\file\\path\\file.py\", line 194, in broken_function";
-    TestableErrorReporter reporter(appName, upTime, "0", true, "name", "email", "textBox", stackTrace);
+    TestableErrorReporter reporter(appName, upTime, "0", true, "name", "email", "textBox", stackTrace, "");
     const std::string message = reporter.generateErrorMessage();
 
     ::Json::Value root;
@@ -102,7 +102,7 @@ class ErrorReporterTest : public CxxTest::TestSuite {
   void test_errorMessageWithShareAndRecoveryFileHash() {
     const std::string name = "My testing application name";
     const Mantid::Types::Core::time_duration upTime(5, 0, 7, 0);
-    TestableErrorReporter errorService(name, upTime, "0", true, "name", "email", "textBox", "stacktrace");
+    TestableErrorReporter errorService(name, upTime, "0", true, "name", "email", "textBox", "stacktrace", "cppTraces");
     const std::string message = errorService.generateErrorMessage();
 
     ::Json::Value root;
@@ -111,7 +111,7 @@ class ErrorReporterTest : public CxxTest::TestSuite {
     const std::vector<std::string> expectedMembers{
         "ParaView", "application", "host",      "mantidSha1", "mantidVersion", "osArch",
         "osName",   "osReadable",  "osVersion", "uid",        "facility",      "upTime",
-        "exitCode", "textBox",     "name",      "email",      "stacktrace"};
+        "exitCode", "textBox",     "name",      "email",      "stacktrace",    "cppCompressedTraces"};
     for (auto expectedMember : expectedMembers) {
       TSM_ASSERT(expectedMember + " not found",
                  std::find(members.begin(), members.end(), expectedMember) != members.end());
@@ -124,12 +124,13 @@ class ErrorReporterTest : public CxxTest::TestSuite {
     TS_ASSERT_EQUALS(root["email"].asString(), "email");
     TS_ASSERT_EQUALS(root["textBox"].asString(), "textBox");
     TS_ASSERT_EQUALS(root["stacktrace"].asString(), "stacktrace");
+    TS_ASSERT_EQUALS(root["cppCompressedTraces"].asString(), "cppTraces");
   }
 
   void test_errorMessageWithNoShareAndRecoveryFileHash() {
     const std::string name = "My testing application name";
     const Mantid::Types::Core::time_duration upTime(5, 0, 7, 0);
-    TestableErrorReporter errorService(name, upTime, "0", false, "name", "email", "textBox", "stacktrace");
+    TestableErrorReporter errorService(name, upTime, "0", false, "name", "email", "textBox", "stacktrace", "cppTraces");
     const std::string message = errorService.generateErrorMessage();
 
     ::Json::Value root;
@@ -138,7 +139,8 @@ class ErrorReporterTest : public CxxTest::TestSuite {
     const std::vector<std::string> expectedMembers{
         "ParaView", "application", "host",      "mantidSha1", "mantidVersion", "osArch",
         "osName",   "osReadable",  "osVersion", "uid",        "facility",      "upTime",
-        "exitCode", "textBox",     "name",      "email",      "stacktrace"};
+        "exitCode", "textBox",     "name",      "email",      "stacktrace",    "upTime",
+        "exitCode", "textBox",     "name",      "email",      "stacktrace",    "cppCompressedTraces"};
     for (auto expectedMember : expectedMembers) {
       TSM_ASSERT(expectedMember + " not found",
                  std::find(members.begin(), members.end(), expectedMember) != members.end());
@@ -151,5 +153,6 @@ class ErrorReporterTest : public CxxTest::TestSuite {
     TS_ASSERT_EQUALS(root["email"].asString(), "");
     TS_ASSERT_EQUALS(root["textBox"].asString(), "textBox");
     TS_ASSERT_EQUALS(root["stacktrace"].asString(), "");
+    TS_ASSERT_EQUALS(root["cppCompressedTraces"].asString(), "");
   }
 };

Framework/Properties/Mantid.properties.template

@@ -59,6 +59,9 @@ usagereports.enabled = @ENABLE_USAGE_REPORTS@
 errorreports.rooturl = https://errorreports.mantidproject.org
 usagereports.rooturl = https://reports.mantidproject.org
 
+# Location of core dump files (linux only feature)
+errorreports.core_dumps =
+
 # Where to load Grouping files (that are shipped with Mantid) from
 groupingFiles.directory = @MANTID_ROOT@/instrument/Grouping
 

Framework/PythonInterface/mantid/kernel/src/Exports/ErrorReporter.cpp

@@ -19,7 +19,7 @@ void export_ErrorReporter() {
                 std::string>())
 
       .def(init<std::string, Mantid::Types::Core::time_duration, std::string, bool, std::string, std::string,
-                std::string, std::string>())
+                std::string, std::string, std::string>())
 
       .def("sendErrorReport", &ErrorReporter::sendErrorReport, arg("self"), "Sends an error report")
 

Framework/PythonInterface/test/python/mantid/kernel/ConfigServiceTest.py

@@ -157,6 +157,7 @@ def test_properties_documented(self):
             "UpdateInstrumentDefinitions.URL",  # shouldn't be changed by users
             "docs.html.root",  # shouldn't be changed by users
             "errorreports.rooturl",  # shouldn't be changed by users
+            "errorreports.core_dumps",
             "usagereports.rooturl",  # shouldn't be changed by users
             "workspace.sendto.SansView.arguments",
             "workspace.sendto.SansView.saveusing",  # related to SASview in menu

conda/recipes/mantid-developer/meta.yaml

@@ -25,6 +25,7 @@ requirements:
     - libboost-python-devel {{ libboost_python_devel }}
     - libopenblas {{ libopenblas }} # [osx or linux]
     - librdkafka {{ librdkafka }}
+    - lz4  # [linux]
     - matplotlib {{ matplotlib }}
     - muparser {{ muparser }}
     - ninja {{ ninja }}
@@ -37,6 +38,7 @@ requirements:
     - pydantic {{ pydantic }}
     - pyqt {{ pyqt }}
     - pyqtwebengine
+    - pystack # [linux]
     - python-dateutil {{ python_dateutil }}
     - python {{ python }}
     - python.app # [osx]

conda/recipes/mantidworkbench/meta.yaml

@@ -39,11 +39,13 @@ requirements:
     - versioningit {{ versioningit }}
   run:
     - ipykernel
+    - lz4  # [linux]
     - psutil {{ psutil }}
     - {{ pin_compatible("python", max_pin="x.x") }}
     - matplotlib {{ matplotlib }}
     - mslice {{ mslice }}
     - python.app  # [osx]
+    - pystack  # [linux]
     - qtconsole {{ qtconsole }}
     - {{ pin_compatible("setuptools", max_pin="x.x") }}
     {% if environ.get('INCLUDE_MANTIDDOCS', 'True') != 'False' %}

docs/source/release/v6.13.0/Workbench/New_features/38405.rst

@@ -0,0 +1,3 @@
+- Added property ``errorreports.core_dumps``. Linux users can set this to the directory on their system where core dump files are put after a crash (e.g ``errorreports.core_dumps=/var/lib/apport/coredump``).
+  Workbench will then be able to use this property to extract useful information from the core dump file created after a crash and give that to the error reporting service.
+  This will help us to diagnose some problems where previously no stacktrace was available after a crash. On Linux, core dumps are now always turned on for the workbench process.