Util: For loading code at runtime, use importlib instead of imp

The `imp` module is deprecated and got removed from Python 3.12.

Author: amotl

mqttwarn/util.py

@@ -1,7 +1,7 @@
 # (c) 2014-2023 The mqttwarn developers
 import functools
-import hashlib
-import imp
+import importlib.machinery
+import importlib.util
 import json
 import logging
 import os
@@ -15,6 +15,9 @@
 import pkg_resources
 from six import string_types
 
+if t.TYPE_CHECKING:
+    from importlib._bootstrap_external import FileLoader
+
 logger = logging.getLogger(__name__)
 
 
@@ -132,15 +135,20 @@ def load_module_from_file(path: str) -> types.ModuleType:
     :param path:
     :return:
     """
-    try:
-        fp = open(path, "rb")
-        digest = hashlib.md5(path.encode("utf-8")).hexdigest()
-        return imp.load_source(digest, path, fp)  # type: ignore[arg-type]
-    finally:
-        try:
-            fp.close()
-        except:
-            pass
+    name = Path(path).stem
+    loader: "FileLoader"
+    if path.endswith(".py"):
+        loader = importlib.machinery.SourceFileLoader(fullname=name, path=path)
+    elif path.endswith(".pyc"):
+        loader = importlib.machinery.SourcelessFileLoader(fullname=name, path=path)
+    else:
+        raise ImportError(f"Loading file failed (only .py and .pyc): {path}")
+    spec = importlib.util.spec_from_loader(loader.name, loader)
+    if spec is None:
+        raise ModuleNotFoundError(f"Failed loading module from file: {path}")
+    mod = importlib.util.module_from_spec(spec)
+    loader.exec_module(mod)
+    return mod
 
 
 def load_module_by_name(name: str) -> types.ModuleType:
@@ -150,11 +158,11 @@ def load_module_by_name(name: str) -> types.ModuleType:
     :param name:
     :return:
     """
-    module = import_module(name)
+    module = import_symbol(name)
     return module
 
 
-def import_module(name: str, path: t.Optional[t.List[str]] = None) -> types.ModuleType:
+def import_symbol(name: str, parent: t.Optional[types.ModuleType] = None) -> types.ModuleType:
     """
     Derived from `import_from_dotted_path`:
     https://chase-seibert.github.io/blog/2014/04/23/python-imp-examples.html
@@ -168,16 +176,38 @@ def import_module(name: str, path: t.Optional[t.List[str]] = None) -> types.Modu
         next_module = name
         remaining_names = None
 
-    fp, pathname, description = imp.find_module(next_module, path)
-    module = imp.load_module(next_module, fp, pathname, description)  # type: ignore[arg-type]
+    parent_name = None
+    next_module_real = next_module
+
+    if parent is not None:
+        next_module_real = "." + next_module
+        parent_name = parent.__name__
+    try:
+        spec = importlib.util.find_spec(next_module_real, parent_name)
+    except (AttributeError, ModuleNotFoundError):
+        module = parent
+        if module is None or module.__loader__ is None:
+            raise ImportError(f"Symbol not found: {name}")
+        if hasattr(module, next_module):
+            return getattr(module, next_module)
+        else:
+            raise ImportError(f"Symbol not found: {name}, module={module}")
+
+    if spec is None:
+        msg = f"Symbol not found: {name}"
+        if parent is not None:
+            msg += f", module={parent.__name__}"
+        raise ImportError(msg)
+    module = importlib.util.module_from_spec(spec)
+
+    # Actually load the module.
+    loader: FileLoader = module.__loader__
+    loader.exec_module(module)
 
     if remaining_names is None:
         return module
 
-    if hasattr(module, remaining_names):
-        return getattr(module, remaining_names)
-    else:
-        return import_module(remaining_names, path=list(module.__path__))
+    return import_symbol(remaining_names, parent=module)
 
 
 def load_functions(filepath: t.Optional[str] = None) -> t.Optional[types.ModuleType]:
@@ -188,17 +218,7 @@ def load_functions(filepath: t.Optional[str] = None) -> t.Optional[types.ModuleT
     if not os.path.isfile(filepath):
         raise IOError("'{}' not found".format(filepath))
 
-    mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
-
-    if file_ext.lower() == ".py":
-        py_mod = imp.load_source(mod_name, filepath)
-
-    elif file_ext.lower() == ".pyc":
-        py_mod = imp.load_compiled(mod_name, filepath)
-
-    else:
-        raise ValueError("'{}' does not have the .py or .pyc extension".format(filepath))
-
+    py_mod = load_module_from_file(filepath)
     return py_mod
 
 

tests/test_util.py

@@ -1,10 +1,11 @@
 # -*- coding: utf-8 -*-
-# (c) 2018-2022 The mqttwarn developers
+# (c) 2018-2023 The mqttwarn developers
 from __future__ import division
 
 import py_compile
 import re
 import time
+import types
 from builtins import str
 
 import pytest
@@ -13,7 +14,7 @@
     Formatter,
     asbool,
     get_resource_content,
-    import_module,
+    import_symbol,
     load_file,
     load_function,
     load_functions,
@@ -135,9 +136,9 @@ def test_load_functions():
     assert str(excinfo.value) == "'{}' not found".format("unknown.txt")
 
     # Load functions file that is not a python file
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(ImportError) as excinfo:
         load_functions(filepath=configfile_full)
-    assert str(excinfo.value) == "'{}' does not have the .py or .pyc extension".format(configfile_full)
+    assert re.match(r"Loading file failed \(only .py and .pyc\): .+full.ini", str(excinfo.value))
 
     # Load bad functions file
     with pytest.raises(Exception):
@@ -172,7 +173,7 @@ def test_load_function():
     with pytest.raises(AttributeError) as excinfo:
         load_function(name="unknown", py_mod=py_mod)
     assert re.match(
-        "Function 'unknown' does not exist in '.*{}c?'".format(funcfile_good),
+        "Function 'unknown' does not exist in '.+functions_good.py'",
         str(excinfo.value),
     )
 
@@ -182,15 +183,40 @@ def test_get_resource_content():
     assert "[defaults]" in payload
 
 
-def test_import_module():
+def test_import_symbol_module_success():
     """
-    Proof that the `import_module` function works as intended.
+    Proof that the `import_symbol` function works as intended.
     """
-    symbol = import_module("mqttwarn.services.log")
-    assert symbol.__name__ == "log"
+    symbol = import_symbol("mqttwarn.services.log")
+    assert symbol.__name__ == "mqttwarn.services.log"
+    assert isinstance(symbol, types.ModuleType)
+
+
+def test_import_symbol_module_fail():
+    """
+    Proof that the `import_symbol` function works as intended.
+    """
+    with pytest.raises(ImportError) as ex:
+        import_symbol("foo.bar.baz")
+    assert ex.match("Symbol not found: foo.bar.baz")
+
 
-    symbol = import_module("mqttwarn.services.log.plugin")
+def test_import_symbol_function_success():
+    """
+    Proof that the `import_symbol` function works as intended.
+    """
+    symbol = import_symbol("mqttwarn.services.log.plugin")
     assert symbol.__name__ == "plugin"
+    assert isinstance(symbol, types.FunctionType)
+
+
+def test_import_symbol_function_fail():
+    """
+    Proof that the `import_symbol` function works as intended.
+    """
+    with pytest.raises(ImportError) as ex:
+        import_symbol("mqttwarn.services.log.foo.bar.baz")
+    assert ex.match("Symbol not found: foo.bar.baz, module=<module 'mqttwarn.services.log' from")
 
 
 def test_load_file_success(tmp_path):