Feat (upload/download): Implemented file versioning

Author: sagargg

ckanext/versions/plugin.py

@@ -1,8 +1,13 @@
-import os
 import ckan.plugins as plugins
 import ckan.plugins.toolkit as toolkit
+
 from ckanext.versions.logic import auth, action, validators
 from ckanext.versions import helpers, views
+from ckanext.versions.uploader import (
+    _get_stringified_date,
+    LocalResourceUpload,
+    S3ResourceUpload,
+)
 
 
 class VersionsPlugin(plugins.SingletonPlugin):
@@ -12,14 +17,15 @@ class VersionsPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IValidators, inherit=True)
     plugins.implements(plugins.ITemplateHelpers)
     plugins.implements(plugins.IBlueprint)
+    plugins.implements(plugins.IUploader, inherit=True)
+    plugins.implements(plugins.IResourceController, inherit=True)
 
     # IConfigurer
     def update_config(self, config_):
         toolkit.add_template_directory(config_, "templates")
         toolkit.add_public_directory(config_, "public")
         toolkit.add_resource("assets", "versions")
 
-
     # IActions
     def get_actions(self):
         return {
@@ -30,7 +36,6 @@ def get_actions(self):
             "package_version_list": action.package_version_list,
             "package_version_delete": action.package_version_delete,
             "package_version_exists": action.package_version_exists,
-
         }
 
     # IAuthFunctions
@@ -48,16 +53,41 @@ def get_validators(self):
         return {
             "package_version": validators.package_version,
         }
-    
+
     # ITemplateHelpers
     def get_helpers(self):
         return {
             "get_package_version_list": helpers.get_package_version_list,
-        }   
-
+        }
 
     # IBlueprints
     def get_blueprint(self):
         return [views.dataset_version]
-    
 
+    # IUploader
+    def get_resource_uploader(self, data_dict):
+        ## check if s3filestore is installed then use S3ResourceUpload
+        ## otherwise fallback to LocalResourceUpload
+        if "s3filestore" in toolkit.config.get("ckan.plugins", ""):
+            return S3ResourceUpload(data_dict)
+        else:
+            return LocalResourceUpload(data_dict)
+
+    # IResourceController
+    def before_resource_show(self, resource, **kwargs):
+        # This method is called before the resource
+        url = resource["url"]
+        if resource.get("url_type") == "upload":
+            # update the resource URL to include the last modified date
+            # in the format YYYY-MM-DD-HH-MM-SS
+            url_parts = url.rsplit("/")
+            if "download" in url_parts:
+                index = url_parts.index("download")
+                last_modified = resource.get("last_modified")
+                last_modified_str = _get_stringified_date(last_modified)
+                if index + 1 >= len(url_parts) or not any(
+                    last_modified_str in part for part in url_parts[index + 1 :]
+                ):
+                    url_parts.insert(index + 1, last_modified_str)
+                    resource["url"] = "/".join(url_parts)
+        return resource

ckanext/versions/uploader.py

@@ -0,0 +1,61 @@
+import os
+import datetime
+import ckan.plugins.toolkit as toolkit
+from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload
+
+
+def _get_stringified_date(lastmodified):
+    if isinstance(lastmodified, datetime.datetime):
+        last_modified_str = lastmodified.strftime("%Y-%m-%d-%H-%M-%S")
+    else:
+        last_modified_str = (
+            str(lastmodified)
+            .replace(":", "-")
+            .replace("T", "-")
+            .replace(" ", "-")
+            .split(".")[0]
+        )
+    return last_modified_str
+
+
+try:
+    from ckanext.s3filestore.uploader import S3ResourceUploader
+
+    class S3ResourceUpload(S3ResourceUploader):
+        def __init__(self, resource):
+            super().__init__(resource)
+            self.resource = resource
+
+        def get_path(self, id, filename):
+            last_modified_str = _get_stringified_date(
+                self.resource.get("last_modified")
+            )
+            request_timestamp = toolkit.request.view_args.get("timestamp")
+            last_modified_str = request_timestamp or last_modified_str
+            base_directory = self.get_directory(id, self.storage_path)
+            directory = os.path.join(base_directory, last_modified_str)
+            return os.path.join(directory, filename)
+
+except ImportError:
+    S3ResourceUpload = None
+
+
+class LocalResourceUpload(DefaultResourceUpload):
+    """A local resource uploader that takes revisions into account"""
+
+    def __init__(self, data_dict):
+        super(LocalResourceUpload, self).__init__(data_dict)
+        self.resource = data_dict
+
+    def get_path(self, id, filename=None):
+        filepath = super(LocalResourceUpload, self).get_path(id)
+        if self.resource and self.resource.get("last_modified", None):
+            request_timestamp = toolkit.request.view_args.get("timestamp")
+            last_modified_str = _get_stringified_date(self.resource["last_modified"])
+            last_modified_str = request_timestamp or last_modified_str
+            filepath = "-".join([filepath, last_modified_str])
+        return filepath
+
+    def upload(self, *args, **kwargs):
+        self.clear = False
+        return super(LocalResourceUpload, self).upload(*args, **kwargs)

ckanext/versions/views.py

@@ -4,6 +4,12 @@
 
 import ckan.plugins.toolkit as tk
 from ckan.views.dataset import _setup_template_variables
+from ckan.views.resource import download as downloader
+
+try:
+    from ckanext.s3filestore.views.resource import resource_download as s3_downloader
+except ImportError:
+    from ckan.views.resource import download as downloader
 
 log = logging.getLogger(__name__)
 dataset_version = Blueprint("dataset_version", __name__)
@@ -151,3 +157,34 @@ def view_resource(id, resource_id, version_id):
     view_func=view_resource,
     methods=["GET"],
 )
+
+
+def resource_version_download(package_type, id, resource_id, timestamp, filename):
+    """
+    Download a specific version of a resource by timestamp.
+    """
+    # This  use same resource_download function, but includes a timestamp argument
+    # to find the correct version of the resource in get_path method.
+    if "s3filestore" in tk.config.get("ckan.plugins", ""):
+
+        return s3_downloader(
+            package_type=package_type,
+            id=id,
+            resource_id=resource_id,
+            filename=filename,
+        )
+    else:
+        return downloader(
+            package_type=package_type,
+            id=id,
+            resource_id=resource_id,
+            filename=filename,
+        )
+
+
+dataset_version.add_url_rule(
+    "/dataset/<id>/resource/<resource_id>/download/<timestamp>/<filename>",
+    view_func=resource_version_download,
+    defaults={"package_type": "dataset"},
+    methods=["GET"],
+)