Upgrade to python3 ckan2 11

.coveragerc

@@ -2,4 +2,5 @@
 omit =
     */site-packages/*
     */python?.?/*
-    ckan/*
+    ckan/*
+    */tests/*

.github/workflows/test.yml

@@ -3,20 +3,23 @@ on: [push, pull_request]
 jobs:
   lint:
     runs-on: ubuntu-latest
-    container:
-      image: python:2.7.18-buster
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
       - name: Install requirements
-        run: pip install flake8 pycodestyle
+        run: pip install flake8 pycodestyle black isort
       - name: Check syntax
-        run: flake8 --statistics --show-source --exclude ckanext/switzerland/tests,setup.py ckanext
+        run: flake8 . --count --select=C901,E721,E901,E999,F401,F821,F822,F823,F841 --show-source --statistics --extend-exclude ckan
+      - name: Check codestyle
+        run: |
+          isort --diff --check ckanext/
+          black --diff --check ckanext/
 
   test:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [2.8]
+        ckan-version: ["2.11"]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -41,41 +44,41 @@ jobs:
       CKAN_REDIS_URL: redis://redis:6379/1
       CKAN_SITE_URL: http://test.ckan.net
       WORKDIR: /__w/ckanext-switzerland-ng/ckanext-switzerland-ng
-      SOLR_CONFIG_CKAN_DIR: /opt/solr/server/solr/ckan/conf
+      SOLR_CONFIG_DIR: /opt/solr/server/solr/configsets
+      SOLR_CONFIG_CKAN_DIR: /opt/solr/server/solr/configsets/ckan/conf
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Create solr container
       run: |
-        docker create --name test_solr --network ${{ job.container.network }} --network-alias solr \
+        /usr/bin/docker create --name test_solr --network ${{ job.container.network }} --network-alias solr \
           --workdir $WORKDIR --publish 8983:8983 \
-          -e SOLR_HEAP=1024m -e GITHUB_ACTIONS=true -e CI=true \
-          -v "${{ github.workspace }}/solr/schema.xml":"$SOLR_CONFIG_CKAN_DIR/schema.xml" \
-          -v "${{ github.workspace }}/solr/english_stop.txt":"$SOLR_CONFIG_CKAN_DIR/english_stop.txt" \
-          -v "${{ github.workspace }}/solr/fr_elision.txt":"$SOLR_CONFIG_CKAN_DIR/fr_elision.txt" \
-          -v "${{ github.workspace }}/solr/french_stop.txt":"$SOLR_CONFIG_CKAN_DIR/french_stop.txt" \
+          -e "SOLR_HEAP=1024m" -e "SOLR_SCHEMA_FILE=$SOLR_CONFIG_CKAN_DIR/managed-schema" \
+          -e GITHUB_ACTIONS=true -e CI=true -v "${{ github.workspace }}/solr/schema.xml":"$SOLR_CONFIG_CKAN_DIR/managed-schema" \
           -v "${{ github.workspace }}/solr/german_dictionary.txt":"$SOLR_CONFIG_CKAN_DIR/german_dictionary.txt" \
-          -v "${{ github.workspace }}/solr/german_stop.txt":"$SOLR_CONFIG_CKAN_DIR/german_stop.txt" \
-          -v "${{ github.workspace }}/solr/italian_stop.txt":"$SOLR_CONFIG_CKAN_DIR/italian_stop.txt" \
           -v "${{ github.workspace }}/solr/solrconfig.xml":"$SOLR_CONFIG_CKAN_DIR/solrconfig.xml" \
-          ckan/ckan-solr:${{ matrix.ckan-version }}
+          -v "${{ github.workspace }}/solr/lang/fr_elision.txt":"$SOLR_CONFIG_DIR/ckan/lang/fr_elision.txt" \
+          -v "${{ github.workspace }}/solr/lang/stopwords_de.txt":"$SOLR_CONFIG_DIR/ckan/lang/stopwords_de.txt" \
+          -v "${{ github.workspace }}/solr/lang/stopwords_en.txt":"$SOLR_CONFIG_DIR/ckan/lang/stopwords_en.txt" \
+          -v "${{ github.workspace }}/solr/lang/stopwords_fr.txt":"$SOLR_CONFIG_DIR/ckan/lang/stopwords_fr.txt" \
+          -v "${{ github.workspace }}/solr/lang/stopwords_it.txt":"$SOLR_CONFIG_DIR/ckan/lang/stopwords_it.txt" \
+          ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
         docker start test_solr
     - name: Create ckan container
       run: |
-        docker create --name test_ckan --network ${{ job.container.network }} --network-alias ckan \
-          -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e CKAN_SITE_URL=http://test.ckan.net \
+        /usr/bin/docker create --name test_ckan --network ${{ job.container.network }} --network-alias ckan \
+          -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e CKAN_SITE_URL=http://test.ckan.net --user root \
           -v "/var/run/docker.sock":"/var/run/docker.sock" \
           -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" \
           -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" \
           -v "/home/runner/work/_temp/_github_home":"/github/home" \
           -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" \
-          --entrypoint "tail" openknowledge/ckan-dev:${{ matrix.ckan-version }} "-f" "/dev/null"
+          --entrypoint "tail" ckan/ckan-dev:${{ matrix.ckan-version }} "-f" "/dev/null"
         docker start test_ckan
     - name: Install requirements and set up database
       run: |
         docker exec test_ckan $WORKDIR/bin/install_test_requirements.sh ${{ matrix.ckan-version }}
     - name: Run tests
       run: | 
-        docker exec test_ckan nosetests --ckan --with-pylons=$WORKDIR/test.ini \
-          --with-coverage --cover-package=ckanext.switzerland --cover-inclusive --cover-erase --cover-tests \
-          $WORKDIR/ckanext/switzerland
+        docker exec test_ckan pytest --ckan-ini=$WORKDIR/test.ini --cov=ckanext.switzerland \
+          --cov-config=$WORKDIR/.coveragerc --disable-warnings $WORKDIR/ckanext/switzerland/tests

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+profile = black

MANIFEST.in

@@ -1,2 +1,3 @@
 include README.rst
-recursive-include ckanext/switzerland *.html *.json *.js *.less *.css
+recursive-include ckanext/switzerland *.html *.json *.js *.less *.css *.yml
+recursive-include ckanext/switzerland/migration *.ini *.py *.mako

bin/install_test_requirements.sh

@@ -13,46 +13,33 @@ echo "Install ckanext dependencies"
 pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming
 pip install -e git+https://github.com/ckan/ckanext-fluent.git#egg=ckanext-fluent
 pip install -e git+https://github.com/ckan/ckanext-hierarchy.git#egg=ckanext-hierarchy
-
-# Last commit before support for Python 2 was dropped
-pip install -e git+https://github.com/ckan/ckanext-harvest.git@v1.4.2#egg=ckanext-harvest
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/v1.4.2/requirements.txt
-
-# Last commit before support for Python 2 was dropped
-pip install -e git+https://github.com/ckan/ckanext-dcat.git@0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6#egg=ckanext-dcat
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/0c26bed5b7a3a7fca8e7b78e338aace096e0ebf6/requirements-py2.txt
-
-# Last commit before support for Python 2 was dropped
-pip install -e git+https://github.com/ckan/ckanext-showcase.git@v1.5.2#egg=ckanext-showcase
-
-# Last commit before support for Python 2 was dropped
-pip install -e git+https://github.com/ckan/ckanext-xloader.git@16b84175005435d579607638dfe2056dda61af70#egg=ckanext-xloader
-pip install -r https://raw.githubusercontent.com/ckan/ckanext-xloader/16b84175005435d579607638dfe2056dda61af70/requirements.txt
+pip install -e git+https://github.com/ckan/ckanext-harvest.git#egg=ckanext-harvest
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/master/requirements.txt
+pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-dcat/master/requirements-py2.txt
+pip install -e git+https://github.com/ckan/ckanext-showcase.git#egg=ckanext-showcase
+pip install -e git+https://github.com/ckan/ckanext-xloader.git#egg=ckanext-xloader
+pip install -r https://raw.githubusercontent.com/ckan/ckanext-xloader/master/requirements.txt
 
 # Our ckanexts
-pip install -e git+https://github.com/opendata-swiss/ckanext-dcatapchharvest.git#egg=ckanext-dcatapchharvest
-pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-dcatapchharvest/master/requirements.txt
+# TODO: require main branch of all of these once they are updated to Python 3 and CKAN 2.11
+pip install -e git+https://github.com/opendata-swiss/ckanext-dcatapchharvest.git@feat/upgrade_to_py3#egg=ckanext-dcatapchharvest
+pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-dcatapchharvest/feat/upgrade_to_py3/requirements.txt
 pip install -e git+https://github.com/opendata-swiss/ckanext-harvester_dashboard.git#egg=ckanext-harvester_dashboard
 pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-harvester_dashboard/master/requirements.txt
 pip install -e git+https://github.com/bellisk/ckanext-subscribe.git#egg=ckanext-subscribe
 pip install -r https://raw.githubusercontent.com/bellisk/ckanext-subscribe/master/requirements.txt
-pip install -e git+https://github.com/opendata-swiss/ckanext-password-policy.git#egg=ckanext-password-policy
-pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-password-policy/master/requirements.txt
+pip install -e git+https://github.com/opendata-swiss/ckanext-password-policy.git@feat/upgrade_to_py3#egg=ckanext-password-policy
+pip install -r https://raw.githubusercontent.com/opendata-swiss/ckanext-password-policy/feat/upgrade_to_py3/requirements.txt
 
 echo "Replace default path to CKAN core config file with the one on the container"
 sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' "$WORKDIR"/test.ini
 
 echo "Replace default database url with the one for the postgres service"
 sed -i -e 's/sqlalchemy.url = .*/sqlalchemy.url = postgresql:\/\/ckan_default:pass@postgres\/ckan_test/' "$WORKDIR"/test.ini
 
-echo "Remove plugins from CKAN config"
-paster --plugin=ckan config-tool "$WORKDIR"/test.ini "ckan.plugins = "
-
 echo "Init db"
-paster --plugin=ckan db init -c "$WORKDIR"/test.ini
-
-echo "Init harvester db"
-paster --plugin=ckanext-harvest harvester initdb -c "$WORKDIR"/test.ini
+ckan -c "$WORKDIR"/test.ini db init
 
-echo "Re-enable plugins in CKAN config"
-paster --plugin=ckan config-tool "$WORKDIR"/test.ini "ckan.plugins = ogdch ogdch_pkg ogdch_res ogdch_group ogdch_org ogdch_showcase ogdch_subscribe ogdch_middleware ogdch_dcat scheming_datasets scheming_groups scheming_organizations fluent hierarchy_display harvester_dashboard"
+echo "Apply pending migrations"
+ckan -c "$WORKDIR"/test.ini db pending-migrations --apply

ckanext/__init__.py

@@ -1,7 +1,9 @@
 # this is a namespace package
 try:
     import pkg_resources
+
     pkg_resources.declare_namespace(__name__)
 except ImportError:
     import pkgutil
+
     __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/switzerland/assets/css/ogdch_stats_box.css

@@ -0,0 +1,43 @@
+.homepage .module-stats {
+    /*margin-top: 1rem;*/
+    color: #444;
+}
+.homepage .module-stats .card {
+    padding: 1.5rem 1.75rem;
+}
+.homepage .module-stats .card ul {
+    margin: 0;
+    list-style: none;
+    padding: 0;
+}
+.homepage .module-stats .card ul::after {
+    display: block;
+    clear: both;
+    content: "";
+}
+.homepage .module-stats .card ul li {
+    float: left;
+    width: 25%;
+    font-weight: 300;
+}
+.homepage .module-stats .card ul li a {
+    display: block;
+}
+.homepage .module-stats .card ul li a strong {
+    font-size: 0.875rem;
+    line-height: 1.5;
+    font-weight: bold;
+}
+.homepage .module-stats .card ul li a:hover {
+    text-decoration: none;
+}
+.homepage .module-feeds .module-stats {
+    margin-top: 0;
+}
+
+@media (max-width: 767.98px) {
+    .homepage .module-stats {
+        margin-top: 0.5rem;
+        margin-bottom: 0.5rem;
+    }
+}

ckanext/switzerland/assets/webassets.yml

@@ -0,0 +1,21 @@
+switzerland-css:
+  output: switzerland/switzerland.css
+  contents:
+    - css/ogdch_file_upload.css
+    - css/ogdch_form_helper.css
+    - css/ogdch_organization_filter.css
+    - css/ogdch_stats_box.css
+    - vendor/bootstrap-datepicker/css/bootstrap-datepicker.css
+
+switzerland-js:
+  output: switzerland/switzerland.js
+  contents:
+    - js/ogdch_file_upload.js
+    - js/ogdch_form_helper_contact_points.js
+    - js/ogdch_info_box.js
+    - js/ogdch_form_helper_relations.js
+    - js/ogdch_form_helper_qualified_relations.js
+    - js/ogdch_form_helper_temporals.js
+    - js/ogdch_datepicker.js
+    - js/ogdch_organization_filter.js
+    - vendor/bootstrap-datepicker/js/bootstrap-datepicker.js

ckanext/switzerland/blueprints/organization.py

@@ -0,0 +1,114 @@
+import logging
+from typing import Any
+
+import ckan.lib.base as base
+import ckan.logic as logic
+from ckan.common import _, current_user, request
+from ckan.lib.helpers import Page
+from ckan.lib.helpers import helper_functions as h
+from ckan.plugins.toolkit import get_action, redirect_to, request
+from ckan.types import Context
+from ckan.views.group import _get_group_template
+from flask import Blueprint
+
+log = logging.getLogger(__name__)
+
+ValidationError = logic.ValidationError
+NotFound = logic.NotFound
+NotAuthorized = logic.NotAuthorized
+check_access = logic.check_access
+
+org = Blueprint(
+    "ogdch_organization",
+    __name__,
+    url_prefix="/organization",
+)
+
+
+def xml_upload(name):
+    org = get_action("organization_show")({}, {"id": name})
+
+    if request.files.get("file_upload") != "":
+        data_dict = {
+            "data": dict(request.files),
+            "organization": org["id"],
+        }
+        get_action("ogdch_xml_upload")({}, data_dict)
+    else:
+        h.flash_error("Error uploading file: no data received.")
+
+    return redirect_to("organization.read", id=name)
+
+
+def index() -> str:
+    """Copied from ckan.views.group.index to remove pagination on the organization
+    index page, as it doesn't work well with the ckanext-hierarchy display.
+    """
+    extra_vars: dict[str, Any] = {}
+    page = h.get_page_number(request.args) or 1
+    group_type = "organization"
+    is_organization = True
+
+    context: Context = {
+        "user": current_user.name,
+        "for_view": True,
+        "with_private": False,
+    }
+
+    try:
+        action_name = "organization_list" if is_organization else "group_list"
+        check_access(action_name, context)
+    except NotAuthorized:
+        base.abort(403, _("Not authorized to see this page"))
+
+    q = request.args.get("q", "")
+    sort_by = request.args.get("sort")
+
+    extra_vars["q"] = q
+    extra_vars["sort_by_selected"] = sort_by
+
+    # pass user info to context as needed to view private datasets of
+    # orgs correctly
+    if current_user.is_authenticated:
+        context["user_id"] = current_user.id
+        context["user_is_admin"] = current_user.sysadmin  # type: ignore
+
+    try:
+        data_dict_global_results: dict[str, Any] = {
+            "all_fields": True,
+            "q": q,
+            "sort": sort_by,
+            "type": group_type or "group",
+            "include_dataset_count": True,
+            "include_member_count": True,
+            "include_extras": True,
+        }
+
+        action_name = "organization_list" if is_organization else "group_list"
+        global_results = get_action(action_name)(context, data_dict_global_results)
+    except ValidationError as e:
+        if e.error_dict and e.error_dict.get("message"):
+            msg: Any = e.error_dict["message"]
+        else:
+            msg = str(e)
+        h.flash_error(msg)
+        extra_vars["page"] = Page([], 0)
+        extra_vars["group_type"] = group_type
+        return base.render(
+            _get_group_template("index_template", group_type), extra_vars
+        )
+
+    extra_vars["page"] = Page(
+        collection=global_results,
+        page=page,
+        url=h.pager_url,
+        items_per_page=len(global_results),
+    )
+
+    extra_vars["page"].items = global_results
+    extra_vars["group_type"] = group_type
+    return base.render(_get_group_template("index_template", group_type), extra_vars)
+
+
+org.add_url_rule("/xml_upload/<name>", view_func=xml_upload, methods=["POST"])
+org.add_url_rule("/", view_func=index, strict_slashes=False)

ckanext/switzerland/blueprints/perma.py

@@ -0,0 +1,29 @@
+import logging
+
+import ckan.logic as logic
+from ckan.plugins.toolkit import _, abort, get_action, redirect_to
+from flask import Blueprint
+
+log = logging.getLogger(__name__)
+
+NotFound = logic.NotFound
+
+perma = Blueprint("perma", __name__, url_prefix="/perma")
+
+
+def read(id):
+    """
+    This action redirects requests to /perma/{identifier} to
+    the corresponding /dataset/{slug} route
+    """
+    try:
+        dataset = get_action("ogdch_dataset_by_identifier")(
+            {"for_view": True}, {"identifier": id}
+        )
+        # redirect to dataset detail page
+        return redirect_to("dataset.read", id=dataset["name"])
+    except NotFound:
+        abort(404, _("Dataset not found"))
+
+
+perma.add_url_rule("/<id>", view_func=read)