updating processing classes to inherit eachother

Author: kujaku11

mtpy/processing/aurora/process_aurora.py

@@ -16,8 +16,6 @@
 from aurora.config.config_creator import ConfigCreator
 from aurora.pipelines.process_mth5 import process_mth5
 
-from aurora.transfer_function.kernel_dataset import KernelDataset
-
 from mth5.helpers import close_open_files
 from mth5.mth5 import MTH5
 
@@ -29,5 +27,21 @@
 warnings.filterwarnings("ignore")
 # =============================================================================
 
+
 class AuroraProcessing(BaseProcessing):
-    
+    """
+    convenience class to process with Aurora
+
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def create_config(self):
+        pass
+
+    def build_kernel_dataset(self):
+        pass
+
+    def process(self):
+        pass

mtpy/processing/base.py

@@ -8,128 +8,42 @@
 # =============================================================================
 # Imports
 # =============================================================================
-from pathlib import Path
 
 from mtpy.processing.run_summary import RunSummary
 from mtpy.processing.kernel_dataset import KernelDataset
 
 # =============================================================================
 
 
-class BaseProcessing:
+class BaseProcessing(KernelDataset):
     """
     Base processing class contains path to various files
     """
 
     def __init__(self, **kwargs):
-        self.local_station_id = None
-        self.remote_station_id = None
-        self.local_mth5_path = None
-        self.remote_mth5_path = None
-        self.config = None
-        self.run_summary = RunSummary()
-        self.kernel_dataset = KernelDataset()
-
-        for key, value in kwargs.items():
-            setattr(self, key, value)
-
-    @property
-    def local_station_id(self):
-        return self._local_station_id
-
-    @local_station_id.setter
-    def local_station_id(self, value):
-        if value is None:
-            self._local_station_id = None
-        else:
-            try:
-                self._local_station_id = str(value)
-            except ValueError:
-                raise ValueError(
-                    f"Bad type {type(value)}. "
-                    "Cannot convert local_station_id value to string."
-                )
-
-    @property
-    def local_mth5_path(self):
-        return self._local_mth5_path
-
-    @local_mth5_path.setter
-    def local_mth5_path(self, value):
-        self._local_mth5_path = self.set_path(value)
-
-    def has_local_mth5(self):
-        """test if local mth5 exists"""
-        if self.local_mth5_path is None:
-            return False
-        else:
-            return self.local_mth5_path.exists()
-
-    @property
-    def remote_station_id(self):
-        return self._remote_station_id
-
-    @remote_station_id.setter
-    def remote_station_id(self, value):
-        if value is None:
-            self._remote_station_id = None
-        else:
-            try:
-                self._remote_station_id = str(value)
-            except ValueError:
-                raise ValueError(
-                    f"Bad type {type(value)}. "
-                    "Cannot convert remote_station_id value to string."
-                )
-
-    @property
-    def remote_mth5_path(self):
-        return self._remote_mth5_path
+        super().__init__(**kwargs)
 
-    @remote_mth5_path.setter
-    def remote_mth5_path(self, value):
-        self._remote_mth5_path = self.set_path(value)
-
-    def has_remote_mth5(self):
-        """test if remote mth5 exists"""
-        if self.remote_mth5_path is None:
-            return False
-        else:
-            return self.remote_mth5_path.exists()
+        self.config = None
 
     @property
     def mth5_list(self):
         """
-        list of mth5's as [local, remote]
 
-        :return: DESCRIPTION
-        :rtype: TYPE
+        :return: list of mth5 to get run summary from
+        :rtype: list
 
         """
         mth5_list = []
         if self.has_local_mth5():
             mth5_list.append(self.local_mth5_path)
-            if self.has_remote_mth5():
-                if self.local_mth5_path != self.remote_mth5_path:
-                    mth5_list.append(self.remote_mth5_path)
-        else:
-            raise IOError("Local MTH5 path must be set with a valid file path.")
-        return mth5_list
-
-    @classmethod
-    def set_path(self, value):
-        return_path = None
-        if value is not None:
-            if isinstance(value, (str, Path)):
-                return_path = Path(value)
-                if not return_path.exists():
-                    raise IOError(f"Cannot find file: {return_path}")
-            else:
-                raise ValueError(f"Cannot convert type{type(value)} to Path")
+        if self.has_remote_mth5():
+            mth5_list.append(self.remote_mth5_path)
 
-        return return_path
+        if len(mth5_list) == 0:
+            raise ValueError("No MTH5 file paths set. Return list is empty.")
+        return mth5_list
 
-    def get_run_summary(self, inplace: bool = True):
+    def get_run_summary(self):
         """
         Get the RunSummary object
 
@@ -142,47 +56,9 @@ def get_run_summary(self, inplace: bool = True):
 
         """
 
-        if inplace:
-            self.run_summary.from_mth5s(self.mth5_list)
-        else:
-            run_summary = RunSummary()
-            run_summary.from_mth5s(self.mth5_list)
-            return run_summary
-
-    def has_run_summary(self):
-        """
-        test if has run summary
-
-        :return: DESCRIPTION
-        :rtype: TYPE
-
-        """
-        if self.run_summary.df is not None:
-            return True
-        return False
-
-    def get_kernel_dataset(self, inplace: bool = True):
-        """
-
-        :param inplace: DESCRIPTION, defaults to True
-        :type inplace: bool, optional
-        :return: DESCRIPTION
-        :rtype: TYPE
-
-        """
-        if not self.has_run_summary():
-            self.get_run_summary()
-        if inplace:
-
-            self.kernel_dataset.from_run_summary(
-                self.run_summary, self.local_station_id, self.remote_station_id
-            )
-        else:
-            kds = KernelDataset()
-            kds.from_run_summary(
-                self.run_summary, self.local_station_id, self.remote_station_id
-            )
-            return kds
+        run_summary = RunSummary()
+        run_summary.from_mth5s(self.mth5_list)
+        return run_summary
 
     def has_kernel_dataset(self):
         """

mtpy/processing/kernel_dataset.py

@@ -132,6 +132,7 @@ def __init__(
         df: Optional[Union[pd.DataFrame, None]] = None,
         local_station_id: Optional[str] = "",
         remote_station_id: Optional[Union[str, None]] = None,
+        **kwargs,
     ):
         """
         Constructor.
@@ -154,6 +155,11 @@ def __init__(
         self.initialized = False
         self.local_mth5_obj = None
         self.remote_mth5_obj = None
+        self._local_mth5_path = None
+        self._remote_mth5_path = None
+
+        for key, value in kwargs.items():
+            setattr(self, key, value)
 
     def __str__(self):
         return str(self.mini_summary.head())
@@ -313,11 +319,11 @@ def local_mth5_path(self):
                 ].unique()[0]
             )
         else:
-            return None
+            return self._local_mth5_path
 
-    # @local_mth5_path.setter
-    # def local_mth5_path(self, value):
-    #     self._local_mth5_path = self.set_path(value)
+    @local_mth5_path.setter
+    def local_mth5_path(self, value):
+        self._local_mth5_path = self.set_path(value)
 
     def has_local_mth5(self):
         """test if local mth5 exists"""
@@ -365,9 +371,9 @@ def remote_mth5_path(self):
         else:
             return None
 
-    # @remote_mth5_path.setter
-    # def remote_mth5_path(self, value):
-    #     self._remote_mth5_path = self.set_path(value)
+    @remote_mth5_path.setter
+    def remote_mth5_path(self, value):
+        self._remote_mth5_path = self.set_path(value)
 
     def has_remote_mth5(self):
         """test if remote mth5 exists"""
@@ -392,7 +398,7 @@ def set_path(self, value):
     def from_run_summary(
         self,
         run_summary: RunSummary,
-        local_station_id: str,
+        local_station_id: Optional[Union(str, None)] = None,
         remote_station_id: Optional[Union[str, None]] = None,
     ) -> None:
         """
@@ -408,12 +414,14 @@ def from_run_summary(
             Label of the remote reference station
 
         """
-        self.local_station_id = local_station_id
-        self.remote_station_id = remote_station_id
+        if local_station_id is not None:
+            self.local_station_id = local_station_id
+        if remote_station_id is not None:
+            self.remote_station_id = remote_station_id
 
-        station_ids = [local_station_id]
+        station_ids = [self.local_station_id]
         if remote_station_id:
-            station_ids.append(remote_station_id)
+            station_ids.append(self.remote_station_id)
         df = restrict_to_station_list(
             run_summary.df, station_ids, inplace=False
         )
@@ -428,20 +436,21 @@ def from_run_summary(
         df = self._add_columns(df)
 
         # set remote reference
-        if remote_station_id:
-            cond = df.station == remote_station_id
+        if self.remote_station_id:
+            cond = df.station == self.remote_station_id
             df.remote = cond
 
         # be sure to set date time columns and restrict to simultaneous runs
         df = self._set_datetime_columns(df)
-        if remote_station_id:
+        if self.remote_station_id:
             df = self.restrict_run_intervals_to_simultaneous(df)
 
         # Again check df is non-empty
         if len(df) == 0:
             msg = (
-                f"Local: {local_station_id} and remote: {remote_station_id} do "
-                f"not overlap, Remote reference processing not a valid option."
+                f"Local: {self.local_station_id} and remote: "
+                f"{self.remote_station_id} do not overlap. Remote reference "
+                "processing not a valid option."
             )
             logger.error(msg)
             raise ValueError(msg)

tests/processing/aurora/test_aurora_synthetic.py

@@ -8,17 +8,13 @@
 # =============================================================================
 # Imports
 # =============================================================================
-from pathlib import Path
-import pandas as pd
+
 import unittest
 
 from mth5.data.make_mth5_from_asc import MTH5_PATH, create_test12rr_h5
 from mth5.utils.helpers import close_open_files
 
 from mtpy.processing.run_summary import RunSummary
-
-from mtpy.processing.kernel_dataset import intervals_overlap
-from mtpy.processing.kernel_dataset import overlap
 from mtpy.processing.kernel_dataset import KernelDataset
 from mtpy import MT