34850 systemtests with ctest

CMakeLists.txt

@@ -185,6 +185,7 @@ endif()
 
 if(BUILD_MANTIDFRAMEWORK)
   add_subdirectory(scripts)
+  add_subdirectory(Testing/SystemTests/tests)
 endif()
 
 # Docs requirements
@@ -209,7 +210,7 @@ if(ENABLE_DOCS)
   add_subdirectory(docs)
 endif()
 
-# System test data target
+# System test targets
 add_subdirectory(Testing/SystemTests/scripts)
 
 if(COVERAGE)

Testing/SystemTests/lib/systemtests/systemtesting.py

@@ -814,21 +814,7 @@ def execute(self, runner, exclude_in_pr_builds):
         self._result.date = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         self._result.addItem(["test_date", self._result.date])
 
-        if retcode == TestRunner.SUCCESS_CODE:
-            status = "success"
-        elif retcode == TestRunner.GENERIC_FAIL_CODE:
-            # This is most likely an algorithm failure, but it's not certain
-            status = "algorithm failure"
-        elif retcode == TestRunner.VALIDATION_FAIL_CODE:
-            status = "failed validation"
-        elif retcode == TestRunner.SEGFAULT_CODE:
-            status = "crashed"
-        elif retcode == TestRunner.SKIP_TEST:
-            status = "skipped"
-        elif retcode < 0:
-            status = "hung"
-        else:
-            status = "unknown"
+        status = exit_code_to_str(retcode)
 
         # Check return code and add result
         self._result.status = status if status in ["success", "skipped"] else "failed"
@@ -1161,7 +1147,7 @@ def loadTestsFromModule(self, filename):
             spec.loader.exec_module(mod)
 
             module_classes = dict(inspect.getmembers(mod, inspect.isclass))
-            module_classes = [x for x in module_classes if self.isValidTestClass(module_classes[x]) and x != "MantidSystemTest"]
+            module_classes = [x for x in module_classes if isValidTestClass(module_classes[x]) and x != "MantidSystemTest"]
             for test_name in module_classes:
                 tests.append(TestSuite(self._runner.getTestDir(), modname, test_name, filename))
             module_loaded = True
@@ -1176,21 +1162,6 @@ def loadTestsFromModule(self, filename):
 
         return module_loaded, tests
 
-    def isValidTestClass(self, class_obj):
-        """Returns true if the test is a valid test class. It is valid
-        if: the class subclassses MantidSystemTest and has no abstract methods
-        """
-        if not issubclass(class_obj, MantidSystemTest):
-            return False
-        # Check if the get_reference_file is abstract or not
-        if hasattr(class_obj, "__abstractmethods__"):
-            if len(class_obj.__abstractmethods__) == 0:
-                return True
-            else:
-                return False
-        else:
-            return True
-
 
 #########################################################################
 # Class to handle the environment
@@ -1321,6 +1292,42 @@ def restoreconfig(self):
         self.__moveFile(self.__userPropsFile, self.__userPropsFileSystest)
         self.__moveFile(self.__userPropsFileBackup, self.__userPropsFile)
 
+def exit_code_to_str(exit_code):
+    if exit_code < 0:
+         return "hung"
+    match exit_code:
+        case TestRunner.SUCCESS_CODE:
+            return "success"
+        case TestRunner.GENERIC_FAIL_CODE:
+            # This is most likely an algorithm failure, but it's not certain
+            return "algorithm failure"
+        case TestRunner.VALIDATION_FAIL_CODE:
+            return "failed validation"
+        case TestRunner.SEGFAULT_CODE:
+            return "crashed"
+        case TestRunner.SKIP_TEST:
+            return "skipped"
+        case _:
+            return"unknown"
+
+#########################################################################
+# Function to check if a given class object is a Mantid System Test
+# that can be run by the TestRunner class.
+#########################################################################
+def isValidTestClass(class_obj):
+    """Returns true if the test is a valid test class. It is valid
+    if: the class subclassses MantidSystemTest and has no abstract methods
+    """
+    if not issubclass(class_obj, MantidSystemTest):
+        return False
+    # Check if the get_reference_file is abstract or not
+    if hasattr(class_obj, "__abstractmethods__"):
+        if len(class_obj.__abstractmethods__) == 0:
+            return True
+        else:
+            return False
+    else:
+        return True
 
 #########################################################################
 # Function to return a string describing the environment

Testing/SystemTests/scripts/systestrunner.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python
+# Mantid Repository : https://github.com/mantidproject/mantid
+#
+# Copyright © 2025 ISIS Rutherford Appleton Laboratory UKRI,
+#   NScD Oak Ridge National Laboratory, European Spallation Source,
+#   Institut Laue - Langevin & CSNS, Institute of High Energy Physics, CAS
+# SPDX - License - Identifier: GPL - 3.0 +
+
+import inspect
+import os
+import sys
+import time
+import importlib.util
+
+from collections import OrderedDict
+
+# Prevents errors in systemtests that use matplotlib directly
+os.environ["MPLBACKEND"] = "Agg"
+
+#########################################################################
+# Set up the command line options
+#########################################################################
+
+start_time = time.time()
+
+DEFAULT_QT_API = "pyqt5"
+THIS_MODULE_DIR = os.path.dirname(os.path.realpath(__file__))
+DEFAULT_FRAMEWORK_LOC = os.path.realpath(os.path.join(THIS_MODULE_DIR, "..", "lib", "systemtests"))
+SANS_TEST_LOC = os.path.realpath(os.path.join(THIS_MODULE_DIR, "..", "tests", "framework", "ISIS", "SANS"))
+DATA_DIRS_LIST_PATH = os.path.join(THIS_MODULE_DIR, "datasearch-directories.txt")
+SAVE_DIR_LIST_PATH = os.path.join(THIS_MODULE_DIR, "defaultsave-directory.txt")
+DEFAULT_LOG_LEVEL = "information"
+DEFAULT_ARCHIVE_SEARCH = True
+DEFAULT_MAKE_PROP = True
+DEFAULT_EXECUTABLE = sys.executable
+
+
+def kill_children(processes):
+    for process in processes:
+        process.terminate()
+    # Exit with status 1 - NOT OK
+    sys.exit(1)
+
+
+def main(argv):
+    # Set the Qt version to use during the system tests
+    os.environ["QT_API"] = DEFAULT_QT_API
+
+    # import the system testing framework
+    sys.path.append(DEFAULT_FRAMEWORK_LOC)
+    import systemtesting
+
+    # allow PythonInterface/test to be discoverable
+    sys.path.append(systemtesting.FRAMEWORK_PYTHONINTERFACE_TEST_DIR)
+
+    sys.path.append(SANS_TEST_LOC)
+    #########################################################################
+    # Configure mantid
+    #########################################################################
+
+    # Parse files containing the search and save directories
+    with open(DATA_DIRS_LIST_PATH, "r") as f_handle:
+        data_paths = f_handle.read().strip()
+
+    with open(SAVE_DIR_LIST_PATH, "r") as f_handle:
+        save_dir = f_handle.read().strip()
+
+    # Configure properties file
+    mtdconf = systemtesting.MantidFrameworkConfig(
+        loglevel=DEFAULT_LOG_LEVEL, data_dirs=data_paths, save_dir=save_dir, archivesearch=DEFAULT_ARCHIVE_SEARCH
+    )
+    if DEFAULT_MAKE_PROP:
+        mtdconf.config()
+
+    #########################################################################
+    # Generate list of tests
+    #########################################################################
+
+    path_to_test = argv[1]
+    test_dir_name = os.path.dirname(path_to_test)
+    test_file_name = os.path.basename(path_to_test)
+    test_module_name = os.path.splitext(test_file_name)[0]
+    test_spec = importlib.util.spec_from_file_location(test_module_name, path_to_test)
+    test_module = importlib.util.module_from_spec(test_spec)
+    test_spec.loader.exec_module(test_module)
+    test_classes = dict(inspect.getmembers(test_module, inspect.isclass))
+
+    test_class_names = [test_class for test_class in test_classes if
+                    systemtesting.isValidTestClass(test_classes[test_class]) and test_class != "MantidSystemTest"]
+
+    if not test_class_names:
+        raise NameError(f"No test classes found in system test module: {test_module_name}.")
+
+    runner = systemtesting.TestRunner(
+        executable=DEFAULT_EXECUTABLE,
+        # see InstallerTests.py for why lstrip is required
+        exec_args="--classic",
+        escape_quotes=True,
+    )
+
+    results = OrderedDict()
+    for test_class_name in test_class_names:
+        script_obj = systemtesting.TestScript(test_dir_name, test_module_name, test_class_name, False)
+        results[test_class_name] = runner.start_in_current_process(script_obj)
+
+    #########################################################################
+    # Process Results
+    #########################################################################
+
+    failure = False
+    if any(exit_code is not systemtesting.TestRunner.SUCCESS_CODE and
+           exit_code is not systemtesting.TestRunner.SKIP_TEST
+           for (exit_code, _) in results.values()):
+        failure = True
+
+    #########################################################################
+    # Cleanup
+    #########################################################################
+
+    # Put the configuration back to its original state
+    mtdconf.restoreconfig()
+
+    if failure:
+        results_string = f"FAILURE REPORT FOR TESTS IN {test_file_name}:\n"
+        for class_name, (exit_code, stdout) in results.items():
+            results_string += f"{class_name}: {systemtesting.exit_code_to_str(exit_code)}\n"
+            if exit_code is not systemtesting.TestRunner.SUCCESS_CODE:
+                results_string += f"Stdout: {stdout}\n"
+        raise RuntimeError(results_string)
+
+
+if __name__ == "__main__":
+    main(sys.argv)

Testing/SystemTests/tests/CMakeLists.txt

@@ -0,0 +1,4 @@
+# System Test Subdirectories to be added to ctest.
+
+add_subdirectory(framework)
+add_subdirectory(qt)