changes to mantid's assert_almost_equal test fixture

Framework/PythonInterface/mantid/_testing/AssertAlmostEqual.py

@@ -8,6 +8,9 @@
 from mantid.simpleapi import CompareWorkspaces
 from mantid.kernel import Property
 
+# the absolute tolerance to be used if neither atol nor rtol specified
+DEFAULT_ABS_TOLERANCE = 1.0e-10
+
 
 def assert_almost_equal(Workspace1, Workspace2, rtol=Property.EMPTY_DBL, atol=Property.EMPTY_DBL, **kwargs):
     """
@@ -27,26 +30,52 @@ def assert_almost_equal(Workspace1, Workspace2, rtol=Property.EMPTY_DBL, atol=Pr
     AssertionError
         If Workspace1 and Workspace2 are not equal up to specified precision
     ValueError
-        If atol and rtol are both not provided
+        If atol or rtol are below zero
+    ValueError
+        If kwargs contains keys that cshould instead be set by atol or rtol
 
     """
     # check arguments
-    if len(set(kwargs.keys()).intersection({"Workspace1", "Workspace2", "Tolerance", "ToleranceRelErr"})):
-        raise ValueError("Workspace1, Workspace2, Tolerance, ToleranceRelErr cannot be passed as additional parameters")
+    if len(set(kwargs.keys()).intersection({"Tolerance", "ToleranceRelErr"})):
+        raise ValueError("Tolerance, ToleranceRelErr cannot be passed as additional parameters: set atol or rtol instead")
+    if atol < 0.0:
+        raise ValueError("Absolute tolerance must be nonnegative")
+    if rtol < 0.0:
+        raise ValueError("Relative tolerance must be nonnegative")
 
-    if rtol == Property.EMPTY_DBL and atol == Property.EMPTY_DBL:
-        raise ValueError("Specify rtol or atol")
+    use_relative = rtol != Property.EMPTY_DBL
+    use_absolute = atol != Property.EMPTY_DBL
 
-    if rtol > Property.EMPTY_DBL:
-        result, message = CompareWorkspaces(Workspace1, Workspace2, Tolerance=rtol, ToleranceRelErr=True, **kwargs)
-        msg_dict = message.toDict()
-        if not result:
-            msg = ", ".join(msg_dict["Message"]) + f", Workspaces are not within relative tolerance ({rtol})"
-            raise AssertionError(msg)
+    # perform the required comparisons
 
-    if atol > Property.EMPTY_DBL:
-        result, message = CompareWorkspaces(Workspace1, Workspace2, Tolerance=atol, ToleranceRelErr=False, **kwargs)
+    # if neither set, do simple absolute comparison
+    if not use_absolute and not use_relative:
+        res, message = CompareWorkspaces(Workspace1, Workspace2, Tolerance=DEFAULT_ABS_TOLERANCE, ToleranceRelErr=False, **kwargs)
         msg_dict = message.toDict()
-        if not result:
-            msg = ", ".join(msg_dict["Message"]) + f", Workspaces are not within absolute tolerance ({atol})"
+        if res:
+            return
+        else:
+            msg = ", ".join(msg_dict["Message"]) + f", Workspaces are not within default tolerance ({DEFAULT_ABS_TOLERANCE})"
             raise AssertionError(msg)
+
+    # if rtol set, perform relative comparison
+    rel_res, rel_msg = True, {}
+    if use_relative:
+        rel_res, message = CompareWorkspaces(Workspace1, Workspace2, Tolerance=rtol, ToleranceRelErr=True, **kwargs)
+        rel_msg = message.toDict()
+
+    # if atol set, perform absolute comparison
+    abs_res, abs_msg = True, {}
+    if use_absolute:
+        abs_res, message = CompareWorkspaces(Workspace1, Workspace2, Tolerance=atol, ToleranceRelErr=False, **kwargs)
+        abs_msg = message.toDict()
+
+    # validate the comparison results
+
+    msg = ""
+    if not rel_res:
+        msg += ", ".join(rel_msg["Message"]) + f", Workspaces are not within relative tolerance ({rtol})"
+    if not abs_res:
+        msg += ", ".join(abs_msg["Message"]) + f", Workspaces are not within absolute tolerance ({atol})"
+    if not rel_res or not abs_res:
+        raise AssertionError(msg)

Framework/PythonInterface/test/python/mantid/_testing/AssertAlmostEqualTest.py

@@ -8,7 +8,7 @@
 import unittest
 from unittest import mock
 from mantid.testing import assert_almost_equal
-from mantid.simpleapi import CreateWorkspace
+from mantid.simpleapi import CreateWorkspace, CreateSingleValuedWorkspace
 
 
 class AssertAlmostEqualTest(unittest.TestCase):
@@ -21,8 +21,16 @@ def setUpClass(self):
         self.ws2 = ws2
         self.ws3 = ws3
 
+    # SIMPLE CASES
+
+    def test_noargs(self):
+        # compare (ws1 - ws2) < 1e-10
+        another1 = self.ws1.clone()
+        assert_almost_equal(self.ws1, another1)
+
     def test_simple(self):
-        assert_almost_equal(self.ws1, self.ws2, atol=1, rtol=1)
+        # simple test with wide tolerances to always pass
+        assert_almost_equal(self.ws1, self.ws2, atol=1, rtol=2)
 
     def test_atol(self):
         # compare (ws1 - ws2) < atol
@@ -32,10 +40,82 @@ def test_rtol(self):
         # compare (ws1 - ws2) / (0.5 * (ws1 + ws2)) < rtol
         assert_almost_equal(self.ws1, self.ws3, rtol=0.7)
 
-    def test_raises(self):
+    # VALIDATION TESTS
+
+    def test_validate_forbidden_keys(self):
+        with self.assertRaises(TypeError):
+            assert_almost_equal(self.ws1, self.ws1, Workspace1=self.ws3)
+        with self.assertRaises(TypeError):
+            assert_almost_equal(self.ws1, self.ws1, Workspace2=self.ws3)
+        with self.assertRaises(ValueError):
+            assert_almost_equal(self.ws1, self.ws1, Tolerance=0.01)
+        with self.assertRaises(ValueError):
+            assert_almost_equal(self.ws1, self.ws1, ToleranceRelErr=True)
+
+    def test_validate_atol(self):
+        with self.assertRaises(ValueError):
+            assert_almost_equal(self.ws1, self.ws2, atol=-1.0, rtol=1.0)
+
+    def test_validate_rtol(self):
         with self.assertRaises(ValueError):
+            assert_almost_equal(self.ws1, self.ws2, ato1=1.0, rtol=-1.0)
+
+    # NEGATIVE TESTS
+
+    def test_fail_unequal(self):
+        with self.assertRaises(AssertionError):
             assert_almost_equal(self.ws1, self.ws2)
 
+    def test_fail_absolute(self):
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(self.ws1, self.ws2, atol=1e-3)
+
+    def test_fail_relative(self):
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(self.ws1, self.ws2, rtol=1.0e-4)
+
+    # MIXED CASES
+
+    def test_both_atol_fails(self):
+        atol = 0.01
+        rtol = 0.01
+        ts1 = CreateSingleValuedWorkspace(1000000.1)
+        ts2 = CreateSingleValuedWorkspace(1000000.0)
+        # ensure rtol passes, atol fails separately
+        assert_almost_equal(ts1, ts2, rtol=rtol)
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(ts1, ts2, atol=atol)
+        # ensure both fail
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(ts1, ts2, atol=atol, rtol=rtol)
+
+    def test_both_rtol_fails(self):
+        atol = 0.01
+        rtol = 0.01
+        tes1 = CreateSingleValuedWorkspace(0.00001)
+        tes2 = CreateSingleValuedWorkspace(0.00010)
+        # ensure atol passes, rtol fails separately
+        assert_almost_equal(tes1, tes2, atol=atol)
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(tes1, tes2, rtol=rtol)
+        # ensure both fail
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(tes1, tes2, atol=atol, rtol=rtol)
+
+    def test_both_both_fail(self):
+        atol = 0.01
+        rtol = 0.01
+        ts1 = CreateSingleValuedWorkspace(100.01)
+        ts2 = CreateSingleValuedWorkspace(200.02)
+        # ensure rtol, atol fail separately
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(ts1, ts2, atol=atol)
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(ts1, ts2, rtol=rtol)
+        # ensure both fail
+        with self.assertRaises(AssertionError):
+            assert_almost_equal(ts1, ts2, atol=atol, rtol=rtol)
+
 
 if __name__ == "__main__":
     unittest.main()

docs/source/api/python/mantid/testing/assert_almost_equal.rst

@@ -5,12 +5,12 @@
 =====================
 
 This is a Python function for testing if two modules are within a tolerance.
-At least ``rtol`` or ``atol`` parameters should be specified. Neither being
-specified generates a ``ValueError`` exception. Both being specified runs
-the underlying :ref:`algm-CompareWorkspaces` algorithm
-twice, with relative tolerance being first.
-
-
+If ``rtol`` is specified, will compare using the relative difference.
+If ``atol`` is specified, will compare using the absolute difference.
+One or both of ``rtol`` and ``atol`` may be specified.
+Will run the underlying :ref:`algm-CompareWorkspaces` algorithm for each
+case specified, and fail if any of the comparisons fail.
+If neither ``rtol`` nor ``atol``  is specified, will perform an absolute compatison to within 1.e-10.
 
 .. module:`mantid.testing`
 

docs/source/release/v6.11.0/Framework/Python/New_features/37878.rst

@@ -0,0 +1 @@
+- Fix python fuction ``assert_almost_equal`` to fail for non-equal workspaces