Merge pull request #161 from sassoftware/1.9.1

v1.9.1

Author: smlindauer

examples/data/hmeqModels/H2OBinaryGLM/glmFit.pickle

@@ -13,6 +13,6 @@
     },
     {
         "role": "scoreResource",
-        "name": "glmfit.pickle"
+        "name": "glmfit.mojo"
     }
 ]

examples/data/hmeqModels/H2OBinaryGLM/glmfit_binary.zip

@@ -13,18 +13,15 @@
 
 h2o.init()
 
-with gzip.open(Path(settings.pickle_path) / "{model_file_name}", "r") as fileIn, open(Path(settings.pickle_path) / "glmfit.zip", "wb") as fileOut:
-    shutil.copyfileobj(fileIn, fileOut)
-os.chmod(Path(settings.pickle_path) / "glmfit.zip", 0o777)
-model = h2o.import_mojo(Path(settings.pickle_path) / "glmfit.zip")
+model = h2o.import_mojo(Path(settings.pickle_path))
 
 def score(LOAN, MORTDUE, VALUE, REASON, JOB, YOJ, DEROG, DELINQ, CLAGE, NINQ, CLNO, DEBTINC):
     "Output: EM_CLASSIFICATION, EM_EVENTPROBABILITY"
 
     try:
         global model
     except NameError:
-        model = h2o.import_mojo(Path(settings.pickle_path) / "glmfit.zip")
+        model = h2o.import_mojo(Path(settings.pickle_path))
 
     try:
         if math.isnan(LOAN):

examples/data/hmeqModels/H2OBinaryGLM/glmfit_mojo.zip

@@ -648,7 +648,11 @@
     "\n",
     "# Serialize the models to a pickle format\n",
     "for (mod, prefix, path) in zip(model, model_prefix, zip_folder):\n",
-    "    pzmm.PickleModel.pickle_trained_model(trained_model=mod, model_prefix=prefix, pickle_path=path)"
+    "    pzmm.PickleModel.pickle_trained_model(\n",
+    "        model_prefix=prefix,\n",
+    "        trained_model=mod,\n",
+    "        pickle_path=path\n",
+    "    )"
    ]
   },
   {

examples/data/hmeqModels/H2OMOJOGLM/fileMetadata.json

@@ -204,7 +204,11 @@
    "source": [
     "model_prefix = \"MLFlowModel\"\n",
     "zip_folder = Path.cwd() / \"data/MLFlowModels/Model1/\"\n",
-    "pzmm.PickleModel.pickle_trained_model(trained_model=None, model_prefix=model_prefix, pickle_path=zip_folder, mlflow_details=metadata_dict)"
+    "pzmm.PickleModel.pickle_trained_model(\n",
+    "    model_prefix=model_prefix, \n",
+    "    pickle_path=zip_folder, \n",
+    "    mlflow_details=metadata_dict\n",
+    ")"
    ]
   },
   {

examples/data/hmeqModels/H2OMOJOGLM/glmfit.mojo

@@ -378,7 +378,11 @@
     "# Output variables expected in SAS Model Manager\n",
     "score_metrics = [\"EM_PREDICTION\"]\n",
     "\n",
-    "pzmm.PickleModel.pickle_trained_model(lrm, model_prefix, zip_folder)"
+    "pzmm.PickleModel.pickle_trained_model(\n",
+    "    model_prefix=model_prefix,\n",
+    "    trained_model=lrm,\n",
+    "    pickle_path=zip_folder\n",
+    ")"
    ]
   },
   {

examples/data/hmeqModels/H2OMOJOGLM/glmfit.zip

@@ -136,7 +136,7 @@ def import_model(
         model_prefix: str,
         project: Union[str, dict, RestObj],
         input_data: Optional[DataFrame] = None,
-        predict_method: [Callable[..., List], List[Any]] = None,
+        predict_method: Union[Callable[..., List], List[Any]] = None,
         score_metrics: Optional[List[str]] = None,
         pickle_type: str = "pickle",
         project_version: str = "latest",

examples/data/hmeqModels/H2OMOJOGLM/glmfit_mojo.zip

@@ -6,7 +6,12 @@
 import pickle
 import shutil
 from pathlib import Path
-from typing import Optional, Union
+from typing import Any, Optional, Union
+
+try:
+    import h2o
+except ImportError:
+    h2o = None
 
 from ..utils.misc import check_if_jupyter
 
@@ -20,8 +25,8 @@ class PickleModel:
     @classmethod
     def pickle_trained_model(
         cls,
-        trained_model,
-        model_prefix,
+        model_prefix: str,
+        trained_model: Optional[Any] = None,
         pickle_path: Union[str, Path, None] = None,
         is_h2o_model: bool = False,
         is_binary_model: bool = False,
@@ -40,12 +45,11 @@ def pickle_trained_model(
 
         Parameters
         ---------------
-        trained_model : model object, str, or Path
-            For non-H2O models, this argument contains the model variable. Otherwise,
-            this should be the file path of the MOJO file.
         model_prefix : str or Path
             Variable name for the model to be displayed in SAS Open Model Manager
             (i.e. hmeqClassTree + [Score.py || .pickle]).
+        trained_model : model object
+            The trained model to be exported.
         pickle_path : str, optional
             File location for the output pickle file. The default value is None.
         is_h2o_model : bool, optional
@@ -60,7 +64,7 @@ def pickle_trained_model(
             file. The default value is False.
         mlflow_details : dict, optional
             Model details from an MLFlow model. This dictionary is created by the
-            readMLModelFile function. The default value is None
+            readMLModelFile function. The default value is None.
 
         Returns
         -------
@@ -106,22 +110,29 @@ def pickle_trained_model(
                         )
                 else:
                     return {model_prefix + PICKLE: pickle.dumps(trained_model)}
-            # For binary H2O models, rename the binary file as a pickle file
-            elif is_binary_model and pickle_path:
-                binary_file = Path(pickle_path) / model_prefix
-                binary_file.rename(binary_file.with_suffix(PICKLE))
-            # For MOJO H2O models, gzip the model file and adjust the file extension
+            # For binary H2O models, save the binary file as a "pickle" file
+            elif is_h2o_model and is_binary_model and pickle_path:
+                if not h2o:
+                    raise RuntimeError(
+                        "The h2o package is required to save the model as a binary h2o"
+                        "model."
+                    )
+                h2o.save_model(
+                    model=trained_model,
+                    force=True,
+                    path=str(pickle_path),
+                    filename=f"{model_prefix}.pickle",
+                )
+            # For MOJO H2O models, save as a mojo file and adjust the extension to .mojo
             elif is_h2o_model and pickle_path:
-                with open(Path(trained_model), "rb") as fileIn, gzip.open(
-                    Path(pickle_path) / (model_prefix + ".mojo"), "wb"
-                ) as fileOut:
-                    fileOut.writelines(fileIn)
-                if cls.notebook_output:
-                    print(
-                        f"MOJO model {model_prefix} was successfully gzipped and saved "
-                        f"to {Path(pickle_path) / (model_prefix + '.mojo')}."
+                if not h2o:
+                    raise RuntimeError(
+                        "The h2o package is required to save the model as a mojo model."
                     )
-            else:
+                trained_model.save_mojo(
+                    force=True, path=str(pickle_path), filename=f"{model_prefix}.mojo"
+                )
+            elif is_binary_model or is_h2o_model:
                 raise ValueError(
                     "There is currently no support for file-less H2O.ai model handling."
                     " Please include a value for the pickle_path argument."

examples/data/hmeqModels/H2OBinaryGLM/score_glmfit_mojo.py renamed to examples/data/hmeqModels/H2OMOJOGLM/score_glmfit_mojo.py

@@ -23,6 +23,27 @@
 from ..utils.decorators import deprecated
 from ..utils.misc import check_if_jupyter
 
+try:
+    # noinspection PyPackageRequirements
+    import numpy as np
+
+    class NpEncoder(json.JSONEncoder):
+        def default(self, obj):
+            if isinstance(obj, np.integer):
+                return int(obj)
+            if isinstance(obj, np.floating):
+                return float(obj)
+            if isinstance(obj, np.ndarray):
+                return obj.tolist()
+            return json.JSONEncoder.default(self, obj)
+
+except ImportError:
+    np = None
+
+    class NpEncoder(json.JSONEncoder):
+        pass
+
+
 # TODO: add converter for any type of dataset (list, dataframe, numpy array)
 
 # Constants
@@ -125,17 +146,17 @@ def write_var_json(
                 file_name = OUTPUT
 
             with open(Path(json_path) / file_name, "w") as json_file:
-                json_file.write(json.dumps(dict_list, indent=4))
+                json_file.write(json.dumps(dict_list, indent=4, cls=NpEncoder))
             if cls.notebook_output:
                 print(
                     f"{file_name} was successfully written and saved to "
                     f"{Path(json_path) / file_name}"
                 )
         else:
             if is_input:
-                return {INPUT: json.dumps(dict_list)}
+                return {INPUT: json.dumps(dict_list, indent=4, cls=NpEncoder)}
             else:
-                return {OUTPUT: json.dumps(dict_list)}
+                return {OUTPUT: json.dumps(dict_list, indent=4, cls=NpEncoder)}
 
     @staticmethod
     def generate_variable_properties(
@@ -329,17 +350,17 @@ def write_model_properties_json(
                 )
 
         if not target_values:
-            model_function = "Prediction"
+            model_function = model_function if model_function else "Prediction"
             target_level = "INTERVAL"
             target_event = ""
             event_prob_var = ""
         elif isinstance(target_values, list) and len(target_values) == 2:
-            model_function = "Classification"
+            model_function = model_function if model_function else "Classification"
             target_level = "BINARY"
             target_event = str(target_values[0])
             event_prob_var = f"P_{target_values[0]}"
         elif isinstance(target_values, list) and len(target_values) > 2:
-            model_function = "Classification"
+            model_function = model_function if model_function else "Classification"
             target_level = "NOMINAL"
             target_event = ""
             event_prob_var = ""
@@ -577,14 +598,14 @@ def input_fit_statistics(
 
         if json_path:
             with open(Path(json_path) / FITSTAT, "w") as json_file:
-                json_file.write(json.dumps(json_dict, indent=4))
+                json_file.write(json.dumps(json_dict, indent=4, cls=NpEncoder))
             if cls.notebook_output:
                 print(
                     f"{FITSTAT} was successfully written and saved to "
                     f"{Path(json_path) / FITSTAT}"
                 )
         else:
-            return {FITSTAT: json.dumps(json_dict, indent=4)}
+            return {FITSTAT: json.dumps(json_dict, indent=4, cls=NpEncoder)}
 
     @classmethod
     def add_tuple_to_fitstat(
@@ -861,17 +882,17 @@ def calculate_model_statistics(
         if json_path:
             for name in [FITSTAT, ROC, LIFT]:
                 with open(Path(json_path) / name, "w") as json_file:
-                    json_file.write(json.dumps(json_dict, indent=4))
+                    json_file.write(json.dumps(json_dict, indent=4, cls=NpEncoder))
                 if cls.notebook_output:
                     print(
                         f"{name} was successfully written and saved to "
                         f"{Path(json_path) / name}"
                     )
         else:
             return {
-                FITSTAT: json.dumps(json_dict[0], indent=4),
-                ROC: json.dumps(json_dict[1], indent=4),
-                LIFT: json.dumps(json_dict[2], indent=4),
+                FITSTAT: json.dumps(json_dict[0], indent=4, cls=NpEncoder),
+                ROC: json.dumps(json_dict[1], indent=4, cls=NpEncoder),
+                LIFT: json.dumps(json_dict[2], indent=4, cls=NpEncoder),
             }
 
     @staticmethod
@@ -1020,11 +1041,11 @@ def apply_dataframe_to_json(
             values from the SAS CAS percentile action set added in.
         """
         for row_num in range(len(stat_df)):
-            row_dict = stat_df.iloc[row_num].to_dict()
+            row_dict = stat_df.iloc[row_num].replace(float("nan"), None).to_dict()
             json_dict[row_num + partition * len(stat_df)]["dataMap"].update(row_dict)
         return json_dict
 
-    # noinspection PyCallingNonCallable,PyNestedDecorators
+    # noinspection PyCallingNonCallable, PyNestedDecorators
     @deprecated(
         "Please use the calculate_model_statistics method instead.",
         version="1.9",

examples/pzmm_binary_classification_model_import.ipynb

@@ -24,7 +24,7 @@ def write_score_code(
         cls,
         model_prefix: str,
         input_data: Union[DataFrame, List[dict]],
-        predict_method: [Callable[..., List], List[Any]],
+        predict_method: Union[Callable[..., List], List[Any]],
         target_variable: Optional[str] = None,
         target_values: Optional[List] = None,
         score_metrics: Optional[List[str]] = None,
@@ -515,23 +515,8 @@ def _viya4_model_load(
         pickle_type = pickle_type if pickle_type else "pickle"
 
         if mojo_model:
-            cls.score_code += (
-                f"with gzip.open(Path(settings.pickle_path) / "
-                '"{model_file_name}", "r") as fileIn, '
-                "open(Path(settings.pickle_path) / "
-                f"\"{str(Path(model_file_name).with_suffix('.zip'))}\","
-                f" \"wb\") as fileOut:\n{'':4}shutil.copyfileobj(fileIn,"
-                " fileOut)\nos.chmod(Path(settings.pickle_path) / "
-                f"\"{str(Path(model_file_name).with_suffix('.zip'))}\""
-                ", 0o777)\nmodel = h2o.import_mojo("
-                "Path(settings.pickle_path) / "
-                f"\"{str(Path(model_file_name).with_suffix('.zip'))}\")"
-                "\n\n"
-            )
-            return (
-                f"{'':8}model = h2o.import_mojo(Path(settings.pickle_path) / "
-                f"\"{str(Path(model_file_name).with_suffix('.zip'))}\")\n\n"
-            )
+            cls.score_code += "model = h2o.import_mojo(Path(settings.pickle_path))\n\n"
+            return f"{'':8}model = h2o.import_mojo(Path(settings.pickle_path))\n\n"
         elif binary_h2o_model:
             cls.score_code += "model = h2o.load(Path(settings.pickle_path))\n\n"
             return f"{'':8}model = h2o.load(Path(settings.pickle_path))\n\n"

examples/pzmm_h2o_model_import.ipynb

@@ -5,7 +5,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import re
-from unittest.mock import patch, PropertyMock
+from unittest.mock import PropertyMock, patch
+
 
 def test_list_packages():
     from sasctl.utils.misc import installed_packages