Remove deprecated pkg_resources in favor of importlib (#82)

* Use importlib if available and fallback to pkg_resources in python < 3.8

* style change

Author: VMRuiz

sh_scrapy/crawl.py

@@ -119,16 +119,23 @@ def _run_scrapy(argv, settings):
 
 
 def _run_pkgscript(argv):
-    import pkg_resources
     if argv[0].startswith('py:'):
         argv[0] = argv[0][3:]
     scriptname = argv[0]
     sys.argv = argv
 
     def get_distribution():
-        for ep in pkg_resources.WorkingSet().iter_entry_points('scrapy'):
+        try:
+            import importlib.metadata
+            eps = importlib.metadata.entry_points(group='scrapy')
+        except ImportError:
+            import pkg_resources
+            eps = pkg_resources.WorkingSet().iter_entry_points('scrapy')
+
+        for ep in eps:
             if ep.name == 'settings':
                 return ep.dist
+
     d = get_distribution()
     if not d:
         raise ValueError(SCRAPY_SETTINGS_ENTRYPOINT_NOT_FOUND)

tests/test_crawl.py

@@ -3,6 +3,7 @@
 import json
 import mock
 import pytest
+import unittest
 from scrapy.settings import Settings
 
 import sh_scrapy.crawl
@@ -120,6 +121,7 @@ def test_run_pkg_script(run_pkg_mock):
     assert run_pkg_mock.call_args[0] == (['py:script.py'],)
 
 
+@unittest.skipIf(sys.version_info > (3,7), "Requires Python 3.7 or lower")
 @mock.patch('pkg_resources.WorkingSet')
 def test_run_pkg_script_distribution_not_found(working_set_class):
     fake_set = mock.Mock()
@@ -128,6 +130,14 @@ def test_run_pkg_script_distribution_not_found(working_set_class):
     with pytest.raises(ValueError):
         _run(['py:script.py'], {'SETTING': 'VALUE'})
 
+@unittest.skipIf(sys.version_info < (3,8), "Requires Python 3.8 or higher")
+@mock.patch('importlib.metadata.entry_points')
+def test_run_pkg_script_distribution_not_found_python_3_8_plus(working_set_class):
+    fake_set = mock.Mock()
+    fake_set.iter_entry_points.return_value = iter(())
+    working_set_class.return_value = [fake_set]
+    with pytest.raises(ValueError):
+        _run(['py:script.py'], {'SETTING': 'VALUE'})
 
 @mock.patch('sh_scrapy.crawl._run_scrapy')
 def test_run_scrapy_spider(run_scrapy_mock):
@@ -155,6 +165,7 @@ def get_working_set(working_set_class):
     return working_set
 
 
+@unittest.skipIf(sys.version_info > (3,7), "Requires Python 3.7 or lower")
 @mock.patch('pkg_resources.WorkingSet')
 def test_run_pkgscript_base_usage(working_set_class):
     working_set = get_working_set(working_set_class)
@@ -167,6 +178,25 @@ def test_run_pkgscript_base_usage(working_set_class):
         'script.py', {'__name__': '__main__'})
     assert sys.argv == ['script.py', 'arg1', 'arg2']
 
+def get_entry_points_mock():
+    """Helper to configure a fake entry point"""
+    ep = mock.Mock()
+    ep.name = 'settings'
+    ep.dist.run_script = mock.Mock()
+    return [ep]
+
+@unittest.skipIf(sys.version_info < (3,8), "Requires Python 3.8 or higher")
+@mock.patch('importlib.metadata.entry_points')
+def test_run_pkgscript_base_usage_python_3_8_plus(entry_points_mock):
+    entry_points_mock.return_value = get_entry_points_mock()
+    _run_pkgscript(['py:script.py', 'arg1', 'arg2'])
+    assert entry_points_mock.called
+    assert entry_points_mock.call_args[1] == {'group': 'scrapy'}
+    ep = entry_points_mock.return_value[0]
+    assert ep.dist.run_script.called
+    assert ep.dist.run_script.call_args[0] == ('script.py', {'__name__': '__main__'})
+    assert sys.argv == ['script.py', 'arg1', 'arg2']
+
 
 @mock.patch.dict(os.environ, {
     'SHUB_SETTINGS': '{"project_settings": {"SETTING....'})