Merge pull request #117 from opendata-swiss/tests/update-tests-for-ckan-2.11-python-3

Tests/update tests for ckan 2.11 python 3

Author: bellisk

.coveragerc

@@ -0,0 +1,6 @@
+[report]
+omit =
+    */site-packages/*
+    */python?.?/*
+    ckan/*
+    */tests/*

.github/workflows/test.yml

@@ -19,16 +19,17 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [2.8]
+        ckan-version: ["2.10", "2.11"]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-version }}
+      options: --user root
     services:
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
@@ -37,7 +38,7 @@ jobs:
           POSTGRES_DB: postgres
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       redis:
-          image: redis:3
+        image: redis:3
     env:
       CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
       CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
@@ -47,25 +48,23 @@ jobs:
       CKAN_SITE_URL: http://test.ckan.net
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Install requirements
-      run: |
-        # Replace default path to CKAN core config file with the one on the container
-        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-        pip install -r requirements.txt
-        pip install -r dev-requirements.txt
-        pip install -e .
-        # Last commit before support for Python 2 was dropped
-        pip install -e git+https://github.com/ckan/ckanext-harvest.git@v1.4.2#egg=ckanext-harvest
-        pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/v1.4.2/requirements.txt
-        # Last commit before support for Python 2 was dropped
-        pip install -e git+https://github.com/ckan/ckanext-dcat.git@0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6#egg=ckanext-dcat
-        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6/requirements-py2.txt
-        pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6/dev-requirements-py2.txt
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Install requirements
+        run: |
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          pip install -e .
+          pip install -e git+https://github.com/ckan/ckanext-harvest.git#egg=ckanext-harvest
+          pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/master/requirements.txt
+          pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat
+          pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/master/requirements.txt
+          # Replace default path to CKAN core config file with the one on the container
+          sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
 
-    - name: Setup extension
-      run: |
-        paster --plugin=ckan db init -c test.ini
-        paster --plugin=ckanext-harvest harvester initdb -c test.ini
-    - name: Run tests
-      run: nosetests --ckan --nocapture --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.dcatapchharvest --cover-inclusive --cover-erase --cover-tests ckanext/dcatapchharvest
+      - name: Setup extension
+        run: |
+          ckan -c test.ini db init
+          ckan -c test.ini db pending-migrations --apply
+      - name: Run tests
+        run: pytest --ckan-ini=test.ini --disable-warnings  --cov=ckanext.dcatapchharvest ckanext/dcatapchharvest

ckanext/dcatapchharvest/harvesters.py

@@ -71,7 +71,7 @@ def before_download(self, url, harvest_job):
         url = url.replace("ogd.global.szh.loc", "data.stadt-zuerich.ch")
         return url, []
 
-    def _get_guid(self, dataset_dict, source_url=None):
+    def _get_guid(self, dataset_dict, source_url=None):  # noqa C901
         """
         Try to get a unique identifier for a harvested dataset
         It will be the first found of:
@@ -82,6 +82,8 @@ def _get_guid(self, dataset_dict, source_url=None):
          The last two are obviously not optimal, as depend on title, which
          might change.
          Returns None if no guid could be decided.
+
+         TODO: This method is too complex (flake8 says 16). Refactor it!
         """
         guid = None
 

ckanext/dcatapchharvest/profiles.py

@@ -78,7 +78,7 @@ def _add_multilang_value(
                     if values:
                         # the values can be either a multilang-dict or they are
                         # nested in another iterable (e.g. keywords)
-                        if not hasattr(values, "__iter__"):
+                        if not isinstance(values, list):
                             values = [values]
                         for value in values:
                             if value:
@@ -542,7 +542,8 @@ def _get_groups(self, subject):
         # Deduplicate group names before returning list of group dicts
         return [{"name": name} for name in list(set(group_names))]
 
-    def parse_dataset(self, dataset_dict, dataset_ref):
+    def parse_dataset(self, dataset_dict, dataset_ref):  # noqa C901
+        # TODO: This method is too complex (flake8 says 30). Refactor it!
         log.debug(f"Parsing dataset '{dataset_ref!r}'")
 
         dataset_dict["temporals"] = []
@@ -760,7 +761,8 @@ def parse_dataset(self, dataset_dict, dataset_ref):
 
         return dataset_dict
 
-    def graph_from_dataset(self, dataset_dict, dataset_ref):
+    def graph_from_dataset(self, dataset_dict, dataset_ref):  # noqa C901
+        # TODO: This method is too complex (flake8 says 49, I am amazed). Refactor it!
 
         log.debug(f"Create graph from dataset '{dataset_dict['name']}'")