Skip to content

Commit 09b9a47

Browse files
committed
Feat (upload/download): Implemented file versioning
1 parent 0159b2c commit 09b9a47

File tree

3 files changed

+135
-7
lines changed

3 files changed

+135
-7
lines changed

ckanext/versions/plugin.py

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
1-
import os
21
import ckan.plugins as plugins
32
import ckan.plugins.toolkit as toolkit
3+
44
from ckanext.versions.logic import auth, action, validators
55
from ckanext.versions import helpers, views
6+
from ckanext.versions.uploader import (
7+
_get_stringified_date,
8+
LocalResourceUpload,
9+
S3ResourceUpload,
10+
)
611

712

813
class VersionsPlugin(plugins.SingletonPlugin):
@@ -12,14 +17,15 @@ class VersionsPlugin(plugins.SingletonPlugin):
1217
plugins.implements(plugins.IValidators, inherit=True)
1318
plugins.implements(plugins.ITemplateHelpers)
1419
plugins.implements(plugins.IBlueprint)
20+
plugins.implements(plugins.IUploader, inherit=True)
21+
plugins.implements(plugins.IResourceController, inherit=True)
1522

1623
# IConfigurer
1724
def update_config(self, config_):
1825
toolkit.add_template_directory(config_, "templates")
1926
toolkit.add_public_directory(config_, "public")
2027
toolkit.add_resource("assets", "versions")
2128

22-
2329
# IActions
2430
def get_actions(self):
2531
return {
@@ -30,7 +36,6 @@ def get_actions(self):
3036
"package_version_list": action.package_version_list,
3137
"package_version_delete": action.package_version_delete,
3238
"package_version_exists": action.package_version_exists,
33-
3439
}
3540

3641
# IAuthFunctions
@@ -48,16 +53,41 @@ def get_validators(self):
4853
return {
4954
"package_version": validators.package_version,
5055
}
51-
56+
5257
# ITemplateHelpers
5358
def get_helpers(self):
5459
return {
5560
"get_package_version_list": helpers.get_package_version_list,
56-
}
57-
61+
}
5862

5963
# IBlueprints
6064
def get_blueprint(self):
6165
return [views.dataset_version]
62-
6366

67+
# IUploader
68+
def get_resource_uploader(self, data_dict):
69+
## check if s3filestore is installed then use S3ResourceUpload
70+
## otherwise fallback to LocalResourceUpload
71+
if "s3filestore" in toolkit.config.get("ckan.plugins", ""):
72+
return S3ResourceUpload(data_dict)
73+
else:
74+
return LocalResourceUpload(data_dict)
75+
76+
# IResourceController
77+
def before_resource_show(self, resource, **kwargs):
78+
# This method is called before the resource
79+
url = resource["url"]
80+
if resource.get("url_type") == "upload":
81+
# update the resource URL to include the last modified date
82+
# in the format YYYY-MM-DD-HH-MM-SS
83+
url_parts = url.rsplit("/")
84+
if "download" in url_parts:
85+
index = url_parts.index("download")
86+
last_modified = resource.get("last_modified")
87+
last_modified_str = _get_stringified_date(last_modified)
88+
if index + 1 >= len(url_parts) or not any(
89+
last_modified_str in part for part in url_parts[index + 1 :]
90+
):
91+
url_parts.insert(index + 1, last_modified_str)
92+
resource["url"] = "/".join(url_parts)
93+
return resource

ckanext/versions/uploader.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import os
2+
import datetime
3+
import ckan.plugins.toolkit as toolkit
4+
from ckan.lib.uploader import ResourceUpload as DefaultResourceUpload
5+
6+
7+
def _get_stringified_date(lastmodified):
8+
if isinstance(lastmodified, datetime.datetime):
9+
last_modified_str = lastmodified.strftime("%Y-%m-%d-%H-%M-%S")
10+
else:
11+
last_modified_str = (
12+
str(lastmodified)
13+
.replace(":", "-")
14+
.replace("T", "-")
15+
.replace(" ", "-")
16+
.split(".")[0]
17+
)
18+
return last_modified_str
19+
20+
21+
try:
22+
from ckanext.s3filestore.uploader import S3ResourceUploader
23+
24+
class S3ResourceUpload(S3ResourceUploader):
25+
def __init__(self, resource):
26+
super().__init__(resource)
27+
self.resource = resource
28+
29+
def get_path(self, id, filename):
30+
last_modified_str = _get_stringified_date(
31+
self.resource.get("last_modified")
32+
)
33+
request_timestamp = toolkit.request.view_args.get("timestamp")
34+
last_modified_str = request_timestamp or last_modified_str
35+
base_directory = self.get_directory(id, self.storage_path)
36+
directory = os.path.join(base_directory, last_modified_str)
37+
return os.path.join(directory, filename)
38+
39+
except ImportError:
40+
S3ResourceUpload = None
41+
42+
43+
class LocalResourceUpload(DefaultResourceUpload):
44+
"""A local resource uploader that takes revisions into account"""
45+
46+
def __init__(self, data_dict):
47+
super(LocalResourceUpload, self).__init__(data_dict)
48+
self.resource = data_dict
49+
50+
def get_path(self, id, filename=None):
51+
filepath = super(LocalResourceUpload, self).get_path(id)
52+
if self.resource and self.resource.get("last_modified", None):
53+
request_timestamp = toolkit.request.view_args.get("timestamp")
54+
last_modified_str = _get_stringified_date(self.resource["last_modified"])
55+
last_modified_str = request_timestamp or last_modified_str
56+
filepath = "-".join([filepath, last_modified_str])
57+
return filepath
58+
59+
def upload(self, *args, **kwargs):
60+
self.clear = False
61+
return super(LocalResourceUpload, self).upload(*args, **kwargs)

ckanext/versions/views.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44

55
import ckan.plugins.toolkit as tk
66
from ckan.views.dataset import _setup_template_variables
7+
from ckan.views.resource import download as downloader
8+
9+
try:
10+
from ckanext.s3filestore.views.resource import resource_download as s3_downloader
11+
except ImportError:
12+
from ckan.views.resource import download as downloader
713

814
log = logging.getLogger(__name__)
915
dataset_version = Blueprint("dataset_version", __name__)
@@ -151,3 +157,34 @@ def view_resource(id, resource_id, version_id):
151157
view_func=view_resource,
152158
methods=["GET"],
153159
)
160+
161+
162+
def resource_version_download(package_type, id, resource_id, timestamp, filename):
163+
"""
164+
Download a specific version of a resource by timestamp.
165+
"""
166+
# This use same resource_download function, but includes a timestamp argument
167+
# to find the correct version of the resource in get_path method.
168+
if "s3filestore" in tk.config.get("ckan.plugins", ""):
169+
170+
return s3_downloader(
171+
package_type=package_type,
172+
id=id,
173+
resource_id=resource_id,
174+
filename=filename,
175+
)
176+
else:
177+
return downloader(
178+
package_type=package_type,
179+
id=id,
180+
resource_id=resource_id,
181+
filename=filename,
182+
)
183+
184+
185+
dataset_version.add_url_rule(
186+
"/dataset/<id>/resource/<resource_id>/download/<timestamp>/<filename>",
187+
view_func=resource_version_download,
188+
defaults={"package_type": "dataset"},
189+
methods=["GET"],
190+
)

0 commit comments

Comments
 (0)