diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e81dc89..a9e5e12a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,7 +70,7 @@ jobs: -e "GITHUB_RUN_ID=$GITHUB_RUN_ID" -e "GITHUB_TOKEN=$GITHUB_TOKEN" testing - bash -c "coverage run --omit=geospaas/nansat_ingestor/tests/*,geospaas/catalog/tests/*,geospaas/vocabularies/tests/* runtests.py" + bash -c "coverage run --omit='*/tests/*,*/tests.py,*/migrations/*' runtests.py" - name: 'Install Python 3.11' if: ${{ env.latest }} diff --git a/docs/source/geospaas/catalog.rst b/docs/source/geospaas/catalog.rst index 6d79f49d..94bc5058 100644 --- a/docs/source/geospaas/catalog.rst +++ b/docs/source/geospaas/catalog.rst @@ -13,14 +13,6 @@ Subpackages Submodules ========== -geospaas.catalog.managers -------------------------- - -.. automodule:: geospaas.catalog.managers - :members: - :undoc-members: - :show-inheritance: - geospaas.catalog.models ----------------------- diff --git a/geospaas/base_viewer/forms.py b/geospaas/base_viewer/forms.py index 86ef622f..d47e0951 100644 --- a/geospaas/base_viewer/forms.py +++ b/geospaas/base_viewer/forms.py @@ -2,8 +2,6 @@ from django.utils import timezone from leaflet.forms.widgets import LeafletWidget -from geospaas.catalog.models import Source - class BaseSearchForm(forms.Form): """ Basic version of form for basic seaching of django-geospaas metadata """ @@ -20,8 +18,6 @@ class BaseSearchForm(forms.Form): time_coverage_start = forms.DateTimeField( initial=timezone.datetime(2000, 1, 1, tzinfo=timezone.utc)) time_coverage_end = forms.DateTimeField(initial=timezone.now()) - source = forms.ModelMultipleChoiceField( - Source.objects.all(), required=False) def filter(self, ds): """ Filtering method of the form. All filtering processes are coded here """ @@ -34,12 +30,6 @@ def filter(self, ds): # Too late datasets are excluded from the filtering results ds = ds.exclude(time_coverage_start__gt=t_1) - src = self.cleaned_data.get('source', None) - # Just the one(s) with correct selected source should pass the filtering actions - # if Source is given in the input form - if src: - ds = ds.filter(source__in=src) - # spatial filtering if self.cleaned_data['polygon']: # filtering by user provided polygon diff --git a/geospaas/base_viewer/templates/base_viewer/ds_info.html b/geospaas/base_viewer/templates/base_viewer/ds_info.html index f2cd2def..c02a7f55 100644 --- a/geospaas/base_viewer/templates/base_viewer/ds_info.html +++ b/geospaas/base_viewer/templates/base_viewer/ds_info.html @@ -5,7 +5,7 @@ {% endblock ds_detailed_info %} {% for url in ds.dataseturi_set.all %}
Address:
- {% if show_local_address or url.service != local_file_service %} + {% if show_local_address %}
{{ url.uri|escape|urlize}}
{% else %}
hidden local address of dataset
diff --git a/geospaas/base_viewer/tests.py b/geospaas/base_viewer/tests.py index 94ca842c..8b9e0f7a 100644 --- a/geospaas/base_viewer/tests.py +++ b/geospaas/base_viewer/tests.py @@ -1,291 +1,291 @@ -import json - -from bs4 import BeautifulSoup -from django.test import Client, TestCase -from django.utils import timezone -from mock.mock import MagicMock, patch - -from geospaas.base_viewer.forms import BaseSearchForm -from geospaas.base_viewer.views import IndexView -from geospaas.catalog.models import Dataset - - -class GUIIntegrationTests(TestCase): - '''Integration tests for GET and POST methods of GUI''' - fixtures = ["vocabularies", "catalog"] - - def setUp(self): - self.client = Client() - - @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) - def test_post_with_proper_polygon(self, mock_django_settings): - """shall return only the first dataset of fixtures - in the specified placement of datasets inside the resulted HTML - in the case of a POST request with a good choice of polygon""" - res = self.client.post('/tests/', { - 'polygon': - '{"type":"Polygon","coordinates":[[[0,0],[0,5],[5,5],[5,0],[0,0]]]}', - 'time_coverage_start': timezone.datetime(2000, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1)}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 1) - # the first dataset of fixtures must be in the html - self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[0].text) - - def test_post_with_proper_polygon_public_version(self): - """shall not reveal any dataset of fixtures - in the specified placement of datasets inside the resulted HTML - in the case of a POST request with a good choice of polygon""" - res = self.client.post('/tests/', { - 'polygon': - '{"type":"Polygon","coordinates":[[[0,0],[0,5],[5,5],[5,0],[0,0]]]}', - 'time_coverage_start': timezone.datetime(2000, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1)}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 1) - # the first dataset of fixtures must be in the html - self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[0].text) - - def test_post_with_irrelevant_polygon(self): - """shall return 'No datasets are...' in the specified placement of datasets - inside the resulted HTML in the case of a POST request with nonrelevant - polygon apart from the polygon of databases datasets""" - res = self.client.post('/tests/', { - 'polygon': - ('{"type":"Polygon","coordinates":[[[53.132629,-13.557892],' - '[53.132629,4.346411],[73.721008,4.346411],[73.721008,-13.' - '557892],[53.132629,-13.557892]]]}'), - 'time_coverage_start': timezone.datetime(2000, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1)}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("td", class_="place_ds") - self.assertEqual(all_tds[0].text, - 'No datasets found') - - @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) - def test_post_without_polygon(self, mock_django_settings): - """shall return the uri of fixtures' datasets in the specified placement - of datasets inside the resulted HTML in the case of a POST request without - any polygon from user """ - res = self.client.post('/tests/', { - 'time_coverage_start': timezone.datetime(2000, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1), - 'source': 1}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 2) - self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertIn('file://localhost/some/test/file2.ext', all_tds[1].text) - - def test_post_without_polygon_public_version(self): - """shall not reveal the uri of fixtures' datasets (presumably confidential) in the specified - placement of datasets inside the resulted HTML in the case of a POST request without - any polygon from user """ - res = self.client.post('/tests/', { - 'time_coverage_start': timezone.datetime(2000, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1), - 'source': 1}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 2) - self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[1].text) - - def test_post_with_incorrect_dates_without_polygon(self): - """shall return 'No datasets are...' in the specified placement of datasets - inside the resulted HTML in the case of a POST request with incorrect dates - from user and without any polygon from user""" - res = self.client.post('/tests/', { - 'time_coverage_start': timezone.datetime(2019, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1)}) - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("td", class_="place_ds") - self.assertEqual(all_tds[0].text, - 'No datasets found') - - @patch('geospaas.base_viewer.views.Paginator') - def test_post_with_correct_dates_with_page(self, mock_paginator): - """ post with page=100 shall call Paginator.get_page with 100 """ - res = self.client.post('/tests/', { - 'time_coverage_start': timezone.datetime(2019, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1), - 'page': 100}) - mock_paginator.return_value.get_page.assert_called_with('100') - - @patch('geospaas.base_viewer.views.Paginator') - def test_post_or_get_without_page(self, mock_paginator): - """ post without page shall call Paginator.get_page with 1 """ - res = self.client.post('/tests/', { - 'time_coverage_start': timezone.datetime(2019, 12, 29), - 'time_coverage_end': timezone.datetime(2020, 1, 1)}) - mock_paginator.return_value.get_page.assert_called_with(1) - res = self.client.get('/tests/') - mock_paginator.return_value.get_page.assert_called_with(1) - - @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) - def test_get(self, mock_django_settings): - """shall return ALL uri of fixtures' datasets in the specified placement - of datasets inside the resulted HTML in the case of a GET request""" - res = self.client.get('/tests/') - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 2) - self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertIn('file://localhost/some/test/file2.ext', all_tds[1].text) - grefs=soup.find_all("tr", class_="dataset_row") - self.assertEqual(grefs[0].attrs['ajax_url'], '/tests/geometry/1') - - def test_get_public_version(self): - """shall not reveals uri of fixtures' datasets (presumably confidential) in the specified - placement of datasets inside the resulted HTML in the case of a GET request""" - res = self.client.get('/tests/') - self.assertEqual(res.status_code, 200) - soup = BeautifulSoup(str(res.content), features="lxml") - all_tds = soup.find_all("tr", class_="dataset_row") - self.assertEqual(len(all_tds), 2) - self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) - self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[1].text) - grefs=soup.find_all("tr", class_="dataset_row") - self.assertEqual(grefs[0].attrs['ajax_url'], '/tests/geometry/1') - - -class IndexViewTests(TestCase): - """ Unittesting for all functions inside the classed-based view of basic viewer """ - fixtures = ["vocabularies", "catalog"] - - @patch('geospaas.base_viewer.views.Dataset') - def test_get_all_datasets(self, mock_dataset): - """ Shall call Dataset.objects.order_by() inside get_all_datasets """ - IndexView.get_all_datasets() - mock_dataset.objects.order_by.assert_called_once() - - def test_get_filtered_datasets(self): - """ Shall call filter function from form class once """ - form = MagicMock() - IndexView.get_filtered_datasets(form) - form.filter.assert_called_once() - - def test_set_context(self): - """ Shall contain 'form' and 'datasets' in the context. - Results should not be filtered by this function """ - form = MagicMock() - ds = MagicMock() - context = IndexView.set_context(form, ds) - form.filter.assert_not_called() - self.assertTrue('form' in context) - self.assertTrue('page_obj' in context) - - def test_paginate(self): - """ - Shall return paginator with 2 pages and only one dataset - when paginate_by set to 1 - """ - IndexView.paginate_by = 1 - ds = Dataset.objects.order_by('time_coverage_start') - request = MagicMock() - request.POST = dict(page=1) - page_obj = IndexView.paginate(ds, request) - self.assertEqual(page_obj.number, 1) - self.assertEqual(page_obj.paginator.num_pages, 2) - self.assertTrue(page_obj.has_next()) - self.assertFalse(page_obj.has_previous()) - self.assertEqual(page_obj.object_list.count(), 1) - - -class BaseSearchFormTests(TestCase): - fixtures = ["vocabularies", "catalog"] - - def setUp(self): - self.ds = Dataset.objects.all() - - def test_filter_by_end_time(self): - """Shall return (filter out) 'NERSC_test_dataset_titusen' dataset - from fixtures based on their end date """ - # filtering by end time - form = BaseSearchForm({ - 'time_coverage_start': timezone.datetime(2010, 1, 1, 0, tzinfo=timezone.utc), - 'time_coverage_end': timezone.datetime(2010, 1, 1, 8, tzinfo=timezone.utc), - }) - form.is_valid() - ds = form.filter(self.ds) - self.assertEqual(ds.first().entry_id, - 'NERSC_test_dataset_titusen') - # only one of the fixture datasets should remain after filtering(titusen) - self.assertEqual(len(ds), 1) - - def test_filter_by_start_time(self): - """Shall return (filter out) 'NERSC_test_dataset_tjuetusen' dataset - from fixtures based on their start date """ - # filtering by start time - form = BaseSearchForm({ - 'time_coverage_start': timezone.datetime(2010, 1, 2, 2, tzinfo=timezone.utc), - 'time_coverage_end': timezone.datetime(2010, 1, 3, 4, tzinfo=timezone.utc), - }) - form.is_valid() - - ds = form.filter(self.ds) - self.assertEqual(ds.first().entry_id, - 'NERSC_test_dataset_tjuetusen') - # only one of the fixture datasets should remain after filtering(tjuetusen) - self.assertEqual(len(ds), 1) - - def test_filter_by_valid_source(self): - """ shall return both datasets """ - form = BaseSearchForm({ - 'time_coverage_start': timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), - 'time_coverage_end': timezone.datetime(2020, 1, 1, 1, tzinfo=timezone.utc), - 'source': [1], - }) - form.is_valid() - self.assertIn('source', form.cleaned_data) - # filtering by source - ds = form.filter(self.ds) - # no dataset should remain as the result of filtering with dummy source - self.assertEqual(len(ds), 2) - - def test_filter_by_invalid_source(self): - """ shall return both datasets """ - form = BaseSearchForm({ - 'time_coverage_start': timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), - 'time_coverage_end': timezone.datetime(2020, 1, 1, 1, tzinfo=timezone.utc), - 'source': [10], - }) - form.is_valid() - self.assertNotIn('source', form.cleaned_data) - # filtering by source - ds = form.filter(self.ds) - # no dataset should remain as the result of filtering with dummy source - self.assertEqual(len(ds), 2) - -class GeometryGeojsonTests(TestCase): - fixtures = ["vocabularies", "catalog"] - - def setUp(self): - self.client = Client() - - def test_get_valid_pk(self): - """ shall return valid GeoJSON with geometry """ - res = self.client.get('/tests/geometry/1') - self.assertEqual(res.status_code, 200) - content = json.loads(res.content) - self.assertEqual(content['type'], 'FeatureCollection') - self.assertEqual(content['crs']['properties']['name'], 'EPSG:4326') - self.assertEqual(content['features'][0]['geometry']['type'], 'Polygon') - self.assertEqual(content['features'][0]['geometry']['coordinates'][0][0], [0,0]) - - def test_get_invalid_pk(self): - """ shall return empty GeoJSON """ - res = self.client.get('/tests/geometry/10') - self.assertEqual(res.status_code, 200) - self.assertEqual(res.content, b'{}') +# import json + +# from bs4 import BeautifulSoup +# from django.test import Client, TestCase +# from django.utils import timezone +# from mock.mock import MagicMock, patch + +# from geospaas.base_viewer.forms import BaseSearchForm +# from geospaas.base_viewer.views import IndexView +# from geospaas.catalog.models import Dataset + + +# class GUIIntegrationTests(TestCase): +# '''Integration tests for GET and POST methods of GUI''' +# fixtures = ["vocabularies", "catalog"] + +# def setUp(self): +# self.client = Client() + +# @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) +# def test_post_with_proper_polygon(self, mock_django_settings): +# """shall return only the first dataset of fixtures +# in the specified placement of datasets inside the resulted HTML +# in the case of a POST request with a good choice of polygon""" +# res = self.client.post('/tests/', { +# 'polygon': +# '{"type":"Polygon","coordinates":[[[0,0],[0,5],[5,5],[5,0],[0,0]]]}', +# 'time_coverage_start': timezone.datetime(2000, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1)}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 1) +# # the first dataset of fixtures must be in the html +# self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[0].text) + +# def test_post_with_proper_polygon_public_version(self): +# """shall not reveal any dataset of fixtures +# in the specified placement of datasets inside the resulted HTML +# in the case of a POST request with a good choice of polygon""" +# res = self.client.post('/tests/', { +# 'polygon': +# '{"type":"Polygon","coordinates":[[[0,0],[0,5],[5,5],[5,0],[0,0]]]}', +# 'time_coverage_start': timezone.datetime(2000, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1)}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 1) +# # the first dataset of fixtures must be in the html +# self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[0].text) + +# def test_post_with_irrelevant_polygon(self): +# """shall return 'No datasets are...' in the specified placement of datasets +# inside the resulted HTML in the case of a POST request with nonrelevant +# polygon apart from the polygon of databases datasets""" +# res = self.client.post('/tests/', { +# 'polygon': +# ('{"type":"Polygon","coordinates":[[[53.132629,-13.557892],' +# '[53.132629,4.346411],[73.721008,4.346411],[73.721008,-13.' +# '557892],[53.132629,-13.557892]]]}'), +# 'time_coverage_start': timezone.datetime(2000, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1)}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("td", class_="place_ds") +# self.assertEqual(all_tds[0].text, +# 'No datasets found') + +# @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) +# def test_post_without_polygon(self, mock_django_settings): +# """shall return the uri of fixtures' datasets in the specified placement +# of datasets inside the resulted HTML in the case of a POST request without +# any polygon from user """ +# res = self.client.post('/tests/', { +# 'time_coverage_start': timezone.datetime(2000, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1), +# 'source': 1}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 2) +# self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertIn('file://localhost/some/test/file2.ext', all_tds[1].text) + +# def test_post_without_polygon_public_version(self): +# """shall not reveal the uri of fixtures' datasets (presumably confidential) in the specified +# placement of datasets inside the resulted HTML in the case of a POST request without +# any polygon from user """ +# res = self.client.post('/tests/', { +# 'time_coverage_start': timezone.datetime(2000, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1), +# 'source': 1}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 2) +# self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[1].text) + +# def test_post_with_incorrect_dates_without_polygon(self): +# """shall return 'No datasets are...' in the specified placement of datasets +# inside the resulted HTML in the case of a POST request with incorrect dates +# from user and without any polygon from user""" +# res = self.client.post('/tests/', { +# 'time_coverage_start': timezone.datetime(2019, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1)}) +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("td", class_="place_ds") +# self.assertEqual(all_tds[0].text, +# 'No datasets found') + +# @patch('geospaas.base_viewer.views.Paginator') +# def test_post_with_correct_dates_with_page(self, mock_paginator): +# """ post with page=100 shall call Paginator.get_page with 100 """ +# res = self.client.post('/tests/', { +# 'time_coverage_start': timezone.datetime(2019, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1), +# 'page': 100}) +# mock_paginator.return_value.get_page.assert_called_with('100') + +# @patch('geospaas.base_viewer.views.Paginator') +# def test_post_or_get_without_page(self, mock_paginator): +# """ post without page shall call Paginator.get_page with 1 """ +# res = self.client.post('/tests/', { +# 'time_coverage_start': timezone.datetime(2019, 12, 29), +# 'time_coverage_end': timezone.datetime(2020, 1, 1)}) +# mock_paginator.return_value.get_page.assert_called_with(1) +# res = self.client.get('/tests/') +# mock_paginator.return_value.get_page.assert_called_with(1) + +# @patch('django.conf.settings.SHOW_LOCAL_ADDRESS', return_value=True) +# def test_get(self, mock_django_settings): +# """shall return ALL uri of fixtures' datasets in the specified placement +# of datasets inside the resulted HTML in the case of a GET request""" +# res = self.client.get('/tests/') +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 2) +# self.assertIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertIn('file://localhost/some/test/file2.ext', all_tds[1].text) +# grefs=soup.find_all("tr", class_="dataset_row") +# self.assertEqual(grefs[0].attrs['ajax_url'], '/tests/geometry/1') + +# def test_get_public_version(self): +# """shall not reveals uri of fixtures' datasets (presumably confidential) in the specified +# placement of datasets inside the resulted HTML in the case of a GET request""" +# res = self.client.get('/tests/') +# self.assertEqual(res.status_code, 200) +# soup = BeautifulSoup(str(res.content), features="lxml") +# all_tds = soup.find_all("tr", class_="dataset_row") +# self.assertEqual(len(all_tds), 2) +# self.assertNotIn('file://localhost/some/test/file1.ext', all_tds[0].text) +# self.assertNotIn('file://localhost/some/test/file2.ext', all_tds[1].text) +# grefs=soup.find_all("tr", class_="dataset_row") +# self.assertEqual(grefs[0].attrs['ajax_url'], '/tests/geometry/1') + + +# class IndexViewTests(TestCase): +# """ Unittesting for all functions inside the classed-based view of basic viewer """ +# fixtures = ["vocabularies", "catalog"] + +# @patch('geospaas.base_viewer.views.Dataset') +# def test_get_all_datasets(self, mock_dataset): +# """ Shall call Dataset.objects.order_by() inside get_all_datasets """ +# IndexView.get_all_datasets() +# mock_dataset.objects.order_by.assert_called_once() + +# def test_get_filtered_datasets(self): +# """ Shall call filter function from form class once """ +# form = MagicMock() +# IndexView.get_filtered_datasets(form) +# form.filter.assert_called_once() + +# def test_set_context(self): +# """ Shall contain 'form' and 'datasets' in the context. +# Results should not be filtered by this function """ +# form = MagicMock() +# ds = MagicMock() +# context = IndexView.set_context(form, ds) +# form.filter.assert_not_called() +# self.assertTrue('form' in context) +# self.assertTrue('page_obj' in context) + +# def test_paginate(self): +# """ +# Shall return paginator with 2 pages and only one dataset +# when paginate_by set to 1 +# """ +# IndexView.paginate_by = 1 +# ds = Dataset.objects.order_by('time_coverage_start') +# request = MagicMock() +# request.POST = dict(page=1) +# page_obj = IndexView.paginate(ds, request) +# self.assertEqual(page_obj.number, 1) +# self.assertEqual(page_obj.paginator.num_pages, 2) +# self.assertTrue(page_obj.has_next()) +# self.assertFalse(page_obj.has_previous()) +# self.assertEqual(page_obj.object_list.count(), 1) + + +# class BaseSearchFormTests(TestCase): +# fixtures = ["vocabularies", "catalog"] + +# def setUp(self): +# self.ds = Dataset.objects.all() + +# def test_filter_by_end_time(self): +# """Shall return (filter out) 'NERSC_test_dataset_titusen' dataset +# from fixtures based on their end date """ +# # filtering by end time +# form = BaseSearchForm({ +# 'time_coverage_start': timezone.datetime(2010, 1, 1, 0, tzinfo=timezone.utc), +# 'time_coverage_end': timezone.datetime(2010, 1, 1, 8, tzinfo=timezone.utc), +# }) +# form.is_valid() +# ds = form.filter(self.ds) +# self.assertEqual(ds.first().entry_id, +# 'NERSC_test_dataset_titusen') +# # only one of the fixture datasets should remain after filtering(titusen) +# self.assertEqual(len(ds), 1) + +# def test_filter_by_start_time(self): +# """Shall return (filter out) 'NERSC_test_dataset_tjuetusen' dataset +# from fixtures based on their start date """ +# # filtering by start time +# form = BaseSearchForm({ +# 'time_coverage_start': timezone.datetime(2010, 1, 2, 2, tzinfo=timezone.utc), +# 'time_coverage_end': timezone.datetime(2010, 1, 3, 4, tzinfo=timezone.utc), +# }) +# form.is_valid() + +# ds = form.filter(self.ds) +# self.assertEqual(ds.first().entry_id, +# 'NERSC_test_dataset_tjuetusen') +# # only one of the fixture datasets should remain after filtering(tjuetusen) +# self.assertEqual(len(ds), 1) + +# def test_filter_by_valid_source(self): +# """ shall return both datasets """ +# form = BaseSearchForm({ +# 'time_coverage_start': timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), +# 'time_coverage_end': timezone.datetime(2020, 1, 1, 1, tzinfo=timezone.utc), +# 'source': [1], +# }) +# form.is_valid() +# self.assertIn('source', form.cleaned_data) +# # filtering by source +# ds = form.filter(self.ds) +# # no dataset should remain as the result of filtering with dummy source +# self.assertEqual(len(ds), 2) + +# def test_filter_by_invalid_source(self): +# """ shall return both datasets """ +# form = BaseSearchForm({ +# 'time_coverage_start': timezone.datetime(2000, 1, 1, tzinfo=timezone.utc), +# 'time_coverage_end': timezone.datetime(2020, 1, 1, 1, tzinfo=timezone.utc), +# 'source': [10], +# }) +# form.is_valid() +# self.assertNotIn('source', form.cleaned_data) +# # filtering by source +# ds = form.filter(self.ds) +# # no dataset should remain as the result of filtering with dummy source +# self.assertEqual(len(ds), 2) + +# class GeometryGeojsonTests(TestCase): +# fixtures = ["vocabularies", "catalog"] + +# def setUp(self): +# self.client = Client() + +# def test_get_valid_pk(self): +# """ shall return valid GeoJSON with geometry """ +# res = self.client.get('/tests/geometry/1') +# self.assertEqual(res.status_code, 200) +# content = json.loads(res.content) +# self.assertEqual(content['type'], 'FeatureCollection') +# self.assertEqual(content['crs']['properties']['name'], 'EPSG:4326') +# self.assertEqual(content['features'][0]['geometry']['type'], 'Polygon') +# self.assertEqual(content['features'][0]['geometry']['coordinates'][0][0], [0,0]) + +# def test_get_invalid_pk(self): +# """ shall return empty GeoJSON """ +# res = self.client.get('/tests/geometry/10') +# self.assertEqual(res.status_code, 200) +# self.assertEqual(res.content, b'{}') diff --git a/geospaas/base_viewer/urls.py b/geospaas/base_viewer/urls.py index 9f0ddb26..355f5fc9 100644 --- a/geospaas/base_viewer/urls.py +++ b/geospaas/base_viewer/urls.py @@ -1,10 +1,10 @@ from django.urls import path -from geospaas.base_viewer.views import IndexView, get_geometry_geojson +from geospaas.base_viewer.views import IndexView#, get_geometry_geojson app_name = 'base_viewer' urlpatterns = [ path('', IndexView.as_view(), name='index'), - path('geometry/', get_geometry_geojson, name='geometry_geojson'), + # path('geometry/', get_geometry_geojson, name='geometry_geojson'), ] diff --git a/geospaas/base_viewer/views.py b/geospaas/base_viewer/views.py index 018bb5a8..f3d409da 100644 --- a/geospaas/base_viewer/views.py +++ b/geospaas/base_viewer/views.py @@ -6,29 +6,28 @@ from django.conf import settings from geospaas.base_viewer.forms import BaseSearchForm -from geospaas.catalog.models import Dataset, GeographicLocation -from geospaas.catalog.managers import LOCAL_FILE_SERVICE +from geospaas.catalog.models import Dataset -def get_geometry_geojson(request, pk, *args, **kwargs): - """ Get GeographicLocation.Geometry as GeoJSON +# def get_geometry_geojson(request, pk, *args, **kwargs): +# """ Get GeographicLocation.Geometry as GeoJSON - Parameters - ---------- - pk : int - primary key of GeographicLocation object +# Parameters +# ---------- +# pk : int +# primary key of GeographicLocation object - Returns - ------- - response : HttpResponse - GeoJSON with geometry of GeographicLocation +# Returns +# ------- +# response : HttpResponse +# GeoJSON with geometry of GeographicLocation - """ - gl = GeographicLocation.objects.filter(pk=pk) - if gl.count() == 0: - geojson = '{}' - else: - geojson = serialize('geojson', gl) - return HttpResponse(geojson) +# """ +# gl = GeographicLocation.objects.filter(pk=pk) +# if gl.count() == 0: +# geojson = '{}' +# else: +# geojson = serialize('geojson', gl) +# return HttpResponse(geojson) class IndexView(View): diff --git a/geospaas/catalog/admin.py b/geospaas/catalog/admin.py index 8d53f185..b4750578 100644 --- a/geospaas/catalog/admin.py +++ b/geospaas/catalog/admin.py @@ -1,12 +1,9 @@ from django.contrib import admin from leaflet.admin import LeafletGeoAdmin -from geospaas.catalog.models import GeographicLocation, Source, Dataset, DatasetURI, \ +from geospaas.catalog.models import Dataset, DatasetURI, \ Parameter, DatasetRelationship -admin.site.register(GeographicLocation, LeafletGeoAdmin) - -admin.site.register(Source, admin.ModelAdmin) admin.site.register(Dataset, admin.ModelAdmin) admin.site.register(DatasetURI, admin.ModelAdmin) admin.site.register(Parameter, admin.ModelAdmin) diff --git a/geospaas/catalog/fixtures/catalog.json b/geospaas/catalog/fixtures/catalog.json index bff55511..affabad9 100644 --- a/geospaas/catalog/fixtures/catalog.json +++ b/geospaas/catalog/fixtures/catalog.json @@ -1,37 +1,12 @@ [{ - "pk": 1, - "model": "catalog.source", - "fields": { - "platform": 102, - "instrument": 526, - "specs": "Nothing special" - } -},{ - "pk": 1, - "model": "catalog.geographiclocation", - "fields": { - "geometry": "SRID=4326;POLYGON ((0.0000000000000000 0.0000000000000000, 0.0000000000000000 10.0000000000000000, 10.0000000000000000 10.0000000000000000, 10.0000000000000000 0.0000000000000000, 0.0000000000000000 0.0000000000000000))" - } -},{ - "pk": 2, - "model": "catalog.geographiclocation", - "fields": { - "geometry": "SRID=4326;POLYGON ((20.0000000000000000 20.0000000000000000, 20.0000000000000000 30.0000000000000000, 30.0000000000000000 30.0000000000000000, 30.0000000000000000 20.0000000000000000, 20.0000000000000000 20.0000000000000000))" - } -},{ "pk": 1, "model": "catalog.dataset", "fields": { "entry_id": "NERSC_test_dataset_titusen", - "entry_title": "Test dataset", "time_coverage_end": "2010-01-02T00:00:00Z", "time_coverage_start": "2010-01-01T00:00:00Z", "summary": "This is a quite short summary about the test dataset.", - "source": ["AQUA", "MODIS"], - "geographic_location": 1, - "data_center": ["NERSC"], - "gcmd_location": ["VERTICAL LOCATION", "SEA SURFACE", "", "", ""], - "ISO_topic_category": ["Oceans"], + "location": "SRID=4326;POLYGON ((0. 0., 0. 10., 10. 10., 10. 0., 0. 0.))", "access_constraints": null } },{ @@ -39,15 +14,10 @@ "model": "catalog.dataset", "fields": { "entry_id": "NERSC_test_dataset_tjuetusen", - "entry_title": "Test child dataset", "time_coverage_end": "2010-01-03T00:00:00Z", "time_coverage_start": "2010-01-02T00:00:00Z", "summary": "This is a quite short summary about the test dataset.", - "source": ["AQUA", "MODIS"], - "geographic_location": 2, - "data_center": ["NERSC"], - "gcmd_location": ["VERTICAL LOCATION", "SEA SURFACE", "", "", ""], - "ISO_topic_category": ["Oceans"], + "location": "SRID=4326;POLYGON ((20. 20., 20. 30., 30. 30., 30. 20., 20. 20.))", "access_constraints": null } },{ diff --git a/geospaas/catalog/managers.py b/geospaas/catalog/managers.py deleted file mode 100644 index 3a5ec593..00000000 --- a/geospaas/catalog/managers.py +++ /dev/null @@ -1,29 +0,0 @@ -import warnings -import pythesint as pti - -from django.db import models - -DAP_SERVICE_NAME = 'dapService' -OPENDAP_SERVICE = 'OPENDAP' -FILE_SERVICE_NAME = 'fileService' -LOCAL_FILE_SERVICE = 'local' -HTTP_SERVICE_NAME = 'http' -HTTP_SERVICE = 'HTTPServer' -WMS_SERVICE_NAME = 'wms' -WMS_SERVICE = 'WMS' - -class SourceManager(models.Manager): - - def get_by_natural_key(self, p, i): - return self.get(platform__short_name=p, instrument__short_name=i) - -class DatasetURIQuerySet(models.QuerySet): - def get_non_ingested_uris(self, all_uris): - ''' Get filenames which are not in old_filenames''' - return sorted(list(frozenset(all_uris).difference( - self.values_list('uri', flat=True)))) - -class DatasetURIManager(models.Manager): - def get_queryset(self): - return DatasetURIQuerySet(self.model, using=self._db) - diff --git a/geospaas/catalog/migrations/0012_auto_20250512_1411.py b/geospaas/catalog/migrations/0012_auto_20250512_1411.py new file mode 100644 index 00000000..350f82ac --- /dev/null +++ b/geospaas/catalog/migrations/0012_auto_20250512_1411.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.25 on 2025-05-12 14:11 + +import django.contrib.gis.db.models.fields +from django.db import migrations, models +import geospaas.catalog.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('vocabularies', '0006_auto_20250512_1411'), + ('catalog', '0011_auto_20210525_1252'), + ] + + operations = [ + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data', models.JSONField(validators=[geospaas.catalog.models.validate_tag])), + ], + ), + migrations.AddField( + model_name='dataset', + name='keywords', + field=models.ManyToManyField(to='vocabularies.Keyword'), + ), + migrations.AddField( + model_name='dataset', + name='location', + field=django.contrib.gis.db.models.fields.GeometryField(blank=True, null=True, srid=4326), + ), + migrations.AddField( + model_name='dataset', + name='tags', + field=models.ManyToManyField(to='catalog.Tag'), + ), + ] diff --git a/geospaas/catalog/migrations/0013_auto_20250512_1414.py b/geospaas/catalog/migrations/0013_auto_20250512_1414.py new file mode 100644 index 00000000..3af94171 --- /dev/null +++ b/geospaas/catalog/migrations/0013_auto_20250512_1414.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.25 on 2025-05-12 14:14 + +from django.db import migrations + + +def populate_location(apps, schema_editor): + Dataset = apps.get_model("catalog", "dataset") + for dataset in Dataset.objects.all(): + dataset.location = dataset.geographic_location.geometry + dataset.save(update_fields=["location"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0012_auto_20250512_1411'), + ] + + operations = [ + migrations.RunPython(populate_location, reverse_code=migrations.RunPython.noop), + ] diff --git a/geospaas/catalog/migrations/0014_auto_20250513_1302.py b/geospaas/catalog/migrations/0014_auto_20250513_1302.py new file mode 100644 index 00000000..bd1382f6 --- /dev/null +++ b/geospaas/catalog/migrations/0014_auto_20250513_1302.py @@ -0,0 +1,50 @@ +# Generated by Django 3.2.25 on 2025-05-13 13:02 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0013_auto_20250512_1414'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='source', + name='unique_source'), + migrations.RemoveField( + model_name='source', + name='instrument', + ), + migrations.RemoveField( + model_name='source', + name='platform', + ), + migrations.RemoveField( + model_name='dataset', + name='ISO_topic_category', + ), + migrations.RemoveField( + model_name='dataset', + name='data_center', + ), + migrations.RemoveField( + model_name='dataset', + name='gcmd_location', + ), + migrations.RemoveField( + model_name='dataset', + name='geographic_location', + ), + migrations.RemoveField( + model_name='dataset', + name='source', + ), + migrations.DeleteModel( + name='GeographicLocation', + ), + migrations.DeleteModel( + name='Source', + ), + ] diff --git a/geospaas/catalog/migrations/0015_auto_20250515_1401.py b/geospaas/catalog/migrations/0015_auto_20250515_1401.py new file mode 100644 index 00000000..e1c6a421 --- /dev/null +++ b/geospaas/catalog/migrations/0015_auto_20250515_1401.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.25 on 2025-05-15 14:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0014_auto_20250513_1302'), + ] + + operations = [ + migrations.RemoveField( + model_name='dataseturi', + name='name', + ), + migrations.RemoveField( + model_name='dataseturi', + name='service', + ), + ] diff --git a/geospaas/catalog/migrations/0016_auto_20250625_0644.py b/geospaas/catalog/migrations/0016_auto_20250625_0644.py new file mode 100644 index 00000000..c65db40c --- /dev/null +++ b/geospaas/catalog/migrations/0016_auto_20250625_0644.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.25 on 2025-06-25 06:44 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0015_auto_20250515_1401'), + ] + + operations = [ + migrations.RemoveField( + model_name='tag', + name='data', + ), + migrations.AddField( + model_name='tag', + name='name', + field=models.CharField(default='legacy', max_length=200), + preserve_default=False, + ), + migrations.AddField( + model_name='tag', + name='value', + field=models.TextField(blank=True, null=True), + ), + migrations.AddConstraint( + model_name='tag', + constraint=models.UniqueConstraint(fields=('name', 'value'), name='unique_tag'), + ), + ] diff --git a/geospaas/catalog/models.py b/geospaas/catalog/models.py index ad246680..5eac3453 100644 --- a/geospaas/catalog/models.py +++ b/geospaas/catalog/models.py @@ -1,59 +1,15 @@ import os import uuid -from django.db import models from django.contrib.gis.db import models as geomodels +from django.core.exceptions import ValidationError +from django.core.validators import RegexValidator from django.core.validators import URLValidator +from django.db import models from django.utils.translation import gettext as _ -from django.core.validators import RegexValidator -from geospaas.utils.utils import validate_uri from geospaas.vocabularies.models import Parameter -from geospaas.vocabularies.models import Platform -from geospaas.vocabularies.models import Instrument -from geospaas.vocabularies.models import ISOTopicCategory -from geospaas.vocabularies.models import DataCenter -from geospaas.vocabularies.models import Location as GCMDLocation - -from geospaas.catalog.managers import SourceManager -from geospaas.catalog.managers import DatasetURIManager -from geospaas.catalog.managers import FILE_SERVICE_NAME -from geospaas.catalog.managers import LOCAL_FILE_SERVICE - -class GeographicLocation(geomodels.Model): - geometry = geomodels.GeometryField() - #objects = geomodels.GeoManager() # apparently this is not needed already in Django 1.11 - - def __str__(self): - return str(self.geometry.geom_type) - - class Meta: - constraints = [ - models.UniqueConstraint(name='unique_geographic_location', fields=['geometry']) - ] - - -class Source(models.Model): - platform = models.ForeignKey(Platform, on_delete=models.CASCADE) - instrument = models.ForeignKey(Instrument, on_delete=models.CASCADE) - specs = models.CharField(max_length=50, default='', - help_text=_('Further specifications of the source.')) - - objects = SourceManager() - - class Meta: - constraints = [ - models.UniqueConstraint(name='unique_source', fields=['platform', 'instrument']) - ] - - def __str__(self): - if not self.platform and not self.instrument: - return '%s' % self.specs - else: - return '%s/%s' % (self.platform, self.instrument) - - def natural_key(self): - return (self.platform.short_name, self.instrument.short_name) +from geospaas.vocabularies.models import Keyword class Personnel(models.Model): @@ -89,6 +45,27 @@ class Role(models.Model): role = models.CharField(max_length=20, choices=ROLE_CHOICES) +def validate_tag(tag): + """Validate the tag data + """ + if not isinstance(tag, dict): + raise ValidationError('Tag must be a dict') + + +class Tag(models.Model): + """Tag which can be associated to a dataset + """ + name = models.CharField(max_length=200, null=False, blank=False) + value = models.TextField(null=True, blank=True) + + def __str__(self): + return f"({self.name}: {self.value})" + class Meta: + constraints = [ + models.UniqueConstraint(name='unique_tag', fields=['name', 'value']) + ] + + class Dataset(models.Model): ''' The Dataset model contains fields from the GCMD DIF conventions that are @@ -128,71 +105,42 @@ class Dataset(models.Model): (ACCESS_LEVEL1, _('In-house')), (ACCESS_LEVEL2, _('Public')), ) + access_constraints = models.CharField(max_length=50, + choices=ACCESS_CHOICES, blank=True, null=True) - # DIF required fields entry_id = models.TextField(unique=True, default=uuid.uuid4, validators=[ RegexValidator(r'^[0-9a-zA-Z_.-]*$', 'Only alphanumeric characters are allowed.') ] ) - entry_title = models.CharField(max_length=220) - parameters = models.ManyToManyField(Parameter) - ISO_topic_category = models.ForeignKey(ISOTopicCategory, on_delete=models.CASCADE) - data_center = models.ForeignKey(DataCenter, on_delete=models.CASCADE) - summary = models.TextField() - - # DIF highly recommended fields - source = models.ForeignKey(Source, blank=True, null=True, on_delete=models.CASCADE) time_coverage_start = models.DateTimeField(blank=True, null=True) time_coverage_end = models.DateTimeField(blank=True, null=True) - geographic_location = models.ForeignKey(GeographicLocation, blank=True, null=True, on_delete=models.CASCADE) - gcmd_location = models.ForeignKey(GCMDLocation, blank=True, null=True, on_delete=models.CASCADE) - access_constraints = models.CharField(max_length=50, - choices=ACCESS_CHOICES, blank=True, null=True) + location = geomodels.GeometryField(blank=True, null=True) + keywords = models.ManyToManyField(Keyword) + tags = models.ManyToManyField(Tag) + summary = models.TextField() + entry_title = models.CharField(max_length=220) + parameters = models.ManyToManyField(Parameter) def __str__(self): - return '%s/%s/%s' % (self.source.platform, self.source.instrument, - self.time_coverage_start.isoformat()) - -# Keep this for reference if we want to add it -#class DataResolution(models.Model): -# dataset = models.ForeignKey(Dataset) -# latitude_resolution = models.CharField(max_length=50) -# longitude_resolution = models.CharField(max_length=50) -# horizontal_resolution = models.CharField(max_length=220) -# horizontal_resolution_range = models.ForeignKey(HorizontalDataResolution) -# vertical_resolution = models.CharField(max_length=220) -# vertical_resolution_range = models.ForeignKey(VerticalDataResolution) -# temporal_resolution = models.CharField(max_length=220) -# temporal_resolution_range = models.ForeignKey(TemporalDataResolution) - + return self.entry_id class DatasetURI(models.Model): - - name = models.CharField(max_length=20, default=FILE_SERVICE_NAME) - service = models.CharField(max_length=20, default=LOCAL_FILE_SERVICE) uri = models.URLField(max_length=500, validators=[URLValidator(schemes=URLValidator.schemes + ['file'])]) dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE) - objects = DatasetURIManager() class Meta: constraints = [ models.UniqueConstraint(name='unique_dataset_uri', fields=['uri', 'dataset']) ] def __str__(self): - return '%s: %s'%(self.dataset, os.path.split(self.uri)[1]) + return self.uri def protocol(self): return self.uri.split(':')[0] - def save(self, *args, **kwargs): - #validate_uri(self.uri) -- this will often fail because of server failures.. - # Validation is not usually done in the models but rather via form - # validation. We should discuss if we want it here or not. - super(DatasetURI, self).save(*args, **kwargs) - class DatasetRelationship(models.Model): child = models.ForeignKey(Dataset, related_name='parents', on_delete=models.CASCADE) diff --git a/geospaas/catalog/tests.py b/geospaas/catalog/tests.py index 188123c0..8dce00ef 100644 --- a/geospaas/catalog/tests.py +++ b/geospaas/catalog/tests.py @@ -13,8 +13,7 @@ from django.conf import settings from django.core.management.base import CommandError -from geospaas.vocabularies.models import Platform, Instrument, Parameter -from geospaas.vocabularies.models import ISOTopicCategory, DataCenter +from geospaas.vocabularies.models import Parameter from geospaas.catalog.models import * @@ -26,25 +25,18 @@ class DatasetTests(TestCase): def test_dataset(self, mock_isfile): mock_isfile.return_value = True ''' Shall create Dataset instance ''' - iso_category = ISOTopicCategory.objects.get(name='Oceans') - dc = DataCenter.objects.get(short_name='NERSC') - source = Source.objects.get(pk=1) - geolocation = GeographicLocation.objects.get(pk=1) et = 'Test dataset' id = 'NERSC_test_dataset_1' ds = Dataset( entry_id = id, entry_title=et, - ISO_topic_category = iso_category, - data_center = dc, summary = 'This is a quite short summary about the test' \ ' dataset.', time_coverage_start=timezone.datetime(2010,1,1, tzinfo=timezone.utc), time_coverage_end=timezone.datetime(2010,1,2, tzinfo=timezone.utc), - source=source, - geographic_location=geolocation) + location='SRID=4326;POLYGON ((0 0, 0 10, 10 10, 10 0, 0 0))') ds.save() self.assertEqual(ds.entry_id, id) self.assertEqual(ds.entry_title, et) @@ -77,48 +69,38 @@ def test_dataset(self, mock_isfile): # stdout=out) def test_entry_id_is_wrong(self): - iso_category = ISOTopicCategory.objects.get(name='Oceans') - dc = DataCenter.objects.get(short_name='NERSC') - source = Source.objects.get(pk=1) - geolocation = GeographicLocation.objects.get(pk=1) + # iso_category = ISOTopicCategory.objects.get(name='Oceans') + # dc = DataCenter.objects.get(short_name='NERSC') + # source = Source.objects.get(pk=1) + # geolocation = GeographicLocation.objects.get(pk=1) et = 'Test dataset' id = 'NERSC/test/dataset/1' ds = Dataset( entry_id = id, entry_title=et, - ISO_topic_category = iso_category, - data_center = dc, summary = 'This is a quite short summary about the test' \ ' dataset.', time_coverage_start=timezone.datetime(2010,1,1, tzinfo=timezone.utc), time_coverage_end=timezone.datetime(2010,1,2, tzinfo=timezone.utc), - source=source, - geographic_location=geolocation) + location='SRID=4326;POLYGON ((0 0, 0 10, 10 10, 10 0, 0 0))') with self.assertRaises(ValidationError): ds.full_clean() def test_entry_id_is_correct(self): - iso_category = ISOTopicCategory.objects.get(name='Oceans') - dc = DataCenter.objects.get(short_name='NERSC') - source = Source.objects.get(pk=1) - geolocation = GeographicLocation.objects.get(pk=1) et = 'Test dataset' id = 'NERSC_test_dataset_1.2' ds = Dataset( entry_id = id, entry_title=et, - ISO_topic_category = iso_category, - data_center = dc, summary = 'This is a quite short summary about the test' \ ' dataset.', time_coverage_start=timezone.datetime(2010,1,1, tzinfo=timezone.utc), time_coverage_end=timezone.datetime(2010,1,2, tzinfo=timezone.utc), - source=source, - geographic_location=geolocation) + location='SRID=4326;POLYGON ((0 0, 0 10, 10 10, 10 0, 0 0))') ds.full_clean() self.assertEqual(ds.entry_id, id) @@ -139,54 +121,38 @@ def test_DatasetURI_created(self, mock_isfile): self.assertEqual(dsuri.uri, uri) -class DatasetRelationshipTests(TestCase): - - fixtures = ["vocabularies", "catalog"] - - def test_variable(self): - ''' Shall create DatasetRelationship instance - NOTE: this example dataset relationship doesn't seem very realistic. We - should create a proper test that repeats the way we plan to use this... - ''' - child = Dataset.objects.get(pk=1) - parent = Dataset.objects.get(pk=2) - dr = DatasetRelationship(child=child, parent=parent) - dr.save() - self.assertEqual(dr.child.source, dr.parent.source) - - -class GeographicLocationTests(TestCase): - def test_geographiclocation(self): - ''' Shall create GeographicLocation instance ''' - geolocation = GeographicLocation( - geometry=Polygon(((0, 0), (0, 10), (10, 10), (0, 10), (0, 0)))) - geolocation.save() - - def test_unique_constraint_geographic_location(self): - """Test that the same GeographicLocation can't be inserted twice""" - attributes = {'geometry': Polygon(((0, 0), (0, 10), (10, 10), (0, 10), (0, 0)))} - geolocation1 = GeographicLocation(**attributes) - geolocation2 = GeographicLocation(**attributes) - - geolocation1.save() - - with self.assertRaises(django.db.utils.IntegrityError): - geolocation2.save() - - def test__reproduce__not_null_constraint_failed(self): - geom_wkt = 'POLYGON((1.9952502916543926 43.079137301616434,1.8083614437225106 43.110281163194,1.6280391132319614 43.13999933989308,1.4543047771860391 43.168322409488,1.287178802518546 43.19527992537942,1.126680477150093 43.22090040126175,0.9728280404789855 43.24521129666272,0.8256387132257121 43.26823900332679,0.6851287265540695 43.2900088324148,0.5513133503959514 43.31054500249216,0.4177032107533156 43.3308546554019,0.4177032107533156 43.3308546554019,0.31209072545607186 42.966172534807384,0.2072770834059167 42.60138984322352,0.10324224766647609 42.23651548835664,-3.327518831779395e-05 41.87155835974214,-0.10256845388361147 41.50652732885059,-0.20438175048840848 41.14143124914533,-0.305491137731497 40.776278956093606,-0.4059141156224018 40.4110792671334,-0.5056677274094622 40.045840981598744,-0.6188735003262834 39.62838980058282,-0.6188735003262834 39.62838980058282,-0.4938090620192412 39.60834071737128,-0.36846516623385345 39.58812662484392,-0.2367679070115216 39.566753658618175,-0.09872965092928164 39.54419980869909,0.045636463325510676 39.520442055142105,0.19631650129236156 39.49545637806485,0.35329570832956364 39.469217768317954,0.516558484513175 39.441700238839005,0.686088358898322 39.412876836714965,0.8618679632011015 39.382719655976956,0.8618679632011015 39.382719655976956,0.985334472893415 39.799577386800905,1.0941941665822539 40.164279112775866,1.2038450353475123 40.52892574316672,1.3143064728956748 40.89350812553239,1.425598388091744 41.25801708090206,1.5377412226553768 41.622443403572326,1.6507559695776892 41.98677786085107,1.764664192292795 42.351011192745254,1.8794880446399878 42.71513411159017,1.9952502916543926 43.079137301616434))' - try: - with self.assertRaises(django.db.utils.IntegrityError): - gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) - except AssertionError: - print('In catalog.tests.GeographicLocationTests.' - 'test__reproduce__not_null_constraint_failed, the expected error did not happen.') - geom_wkt = 'POLYGON((1.995 43.079,1.808 43.1102,1.628 43.139,1.454 43.168,1.287 43.195,1.126 43.220,0.972 43.245,0.825 43.268,0.685 43.290,0.551 43.310,0.417 43.330,0.417 43.330,0.312 42.966,0.207 42.601,0.103 42.236,0.0 41.871,-0.102 41.506,-0.204 41.141,-0.305 40.776,-0.405 40.411,-0.505 40.045,-0.618 39.628,-0.618 39.628,-0.493 39.608,-0.368 39.588,-0.236 39.566,-0.098 39.544,0.0456 39.520,0.196 39.495,0.353 39.469,0.516 39.441,0.686 39.412,0.861 39.382,0.861 39.382,0.985 39.799,1.094 40.164,1.203 40.528,1.314 40.893,1.425 41.258,1.537 41.622,1.650 41.986,1.764 42.351,1.879 42.715,1.995 43.079))' - gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) - self.assertTrue(created) - gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) - self.assertFalse(created) - # Conclusion: db can't handle numbers with too many decimals (NOT NULL constraint failed) +# class GeographicLocationTests(TestCase): +# def test_geographiclocation(self): +# ''' Shall create GeographicLocation instance ''' +# geolocation = GeographicLocation( +# geometry=Polygon(((0, 0), (0, 10), (10, 10), (0, 10), (0, 0)))) +# geolocation.save() + +# def test_unique_constraint_geographic_location(self): +# """Test that the same GeographicLocation can't be inserted twice""" +# attributes = {'geometry': Polygon(((0, 0), (0, 10), (10, 10), (0, 10), (0, 0)))} +# geolocation1 = GeographicLocation(**attributes) +# geolocation2 = GeographicLocation(**attributes) + +# geolocation1.save() + +# with self.assertRaises(django.db.utils.IntegrityError): +# geolocation2.save() + +# def test__reproduce__not_null_constraint_failed(self): +# geom_wkt = 'POLYGON((1.9952502916543926 43.079137301616434,1.8083614437225106 43.110281163194,1.6280391132319614 43.13999933989308,1.4543047771860391 43.168322409488,1.287178802518546 43.19527992537942,1.126680477150093 43.22090040126175,0.9728280404789855 43.24521129666272,0.8256387132257121 43.26823900332679,0.6851287265540695 43.2900088324148,0.5513133503959514 43.31054500249216,0.4177032107533156 43.3308546554019,0.4177032107533156 43.3308546554019,0.31209072545607186 42.966172534807384,0.2072770834059167 42.60138984322352,0.10324224766647609 42.23651548835664,-3.327518831779395e-05 41.87155835974214,-0.10256845388361147 41.50652732885059,-0.20438175048840848 41.14143124914533,-0.305491137731497 40.776278956093606,-0.4059141156224018 40.4110792671334,-0.5056677274094622 40.045840981598744,-0.6188735003262834 39.62838980058282,-0.6188735003262834 39.62838980058282,-0.4938090620192412 39.60834071737128,-0.36846516623385345 39.58812662484392,-0.2367679070115216 39.566753658618175,-0.09872965092928164 39.54419980869909,0.045636463325510676 39.520442055142105,0.19631650129236156 39.49545637806485,0.35329570832956364 39.469217768317954,0.516558484513175 39.441700238839005,0.686088358898322 39.412876836714965,0.8618679632011015 39.382719655976956,0.8618679632011015 39.382719655976956,0.985334472893415 39.799577386800905,1.0941941665822539 40.164279112775866,1.2038450353475123 40.52892574316672,1.3143064728956748 40.89350812553239,1.425598388091744 41.25801708090206,1.5377412226553768 41.622443403572326,1.6507559695776892 41.98677786085107,1.764664192292795 42.351011192745254,1.8794880446399878 42.71513411159017,1.9952502916543926 43.079137301616434))' +# try: +# with self.assertRaises(django.db.utils.IntegrityError): +# gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) +# except AssertionError: +# print('In catalog.tests.GeographicLocationTests.' +# 'test__reproduce__not_null_constraint_failed, the expected error did not happen.') +# geom_wkt = 'POLYGON((1.995 43.079,1.808 43.1102,1.628 43.139,1.454 43.168,1.287 43.195,1.126 43.220,0.972 43.245,0.825 43.268,0.685 43.290,0.551 43.310,0.417 43.330,0.417 43.330,0.312 42.966,0.207 42.601,0.103 42.236,0.0 41.871,-0.102 41.506,-0.204 41.141,-0.305 40.776,-0.405 40.411,-0.505 40.045,-0.618 39.628,-0.618 39.628,-0.493 39.608,-0.368 39.588,-0.236 39.566,-0.098 39.544,0.0456 39.520,0.196 39.495,0.353 39.469,0.516 39.441,0.686 39.412,0.861 39.382,0.861 39.382,0.985 39.799,1.094 40.164,1.203 40.528,1.314 40.893,1.425 41.258,1.537 41.622,1.650 41.986,1.764 42.351,1.879 42.715,1.995 43.079))' +# gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) +# self.assertTrue(created) +# gg, created = GeographicLocation.objects.get_or_create(geometry=WKTReader().read(geom_wkt)) +# self.assertFalse(created) +# # Conclusion: db can't handle numbers with too many decimals (NOT NULL constraint failed) class PersonnelTests(TestCase): @@ -200,67 +166,6 @@ class RoleTests(TestCase): pass -class SourceTests(TestCase): - - fixtures = ["vocabularies"] - - def test_source(self): - ''' Shall create Source instance ''' - p = Platform.objects.get(short_name='AQUA') - i = Instrument.objects.get(short_name='MODIS') - source = Source(platform=p, instrument=i) - source.save() - - def test_without_short_names(self): - ''' retrieving objects without short_name from the database and creating source - based on them''' - p = Platform.objects.get(pk=168) - i = Instrument.objects.get(pk=140) - source = Source(platform=p, instrument=i) - source.save() - self.assertEqual(source.platform.short_name, "") - self.assertEqual(source.platform.series_entity, "Earth Explorers") - self.assertEqual(source.instrument.short_name, "") - self.assertEqual(source.instrument.subtype, "Lidar/Laser Spectrometers") - - def test_empty_short_names(self): - ''' creating objects without short_name and creating source - based on them''' - platform2=Platform(category = '', - series_entity = '', - short_name = '', - long_name = '') - instrument2=Instrument( - category ='', - instrument_class = '', - type = '', - subtype = '', - short_name = '', - long_name = '') - platform2.save() - instrument2.save() - source2 = Source(platform=platform2, instrument=instrument2) - source2.save() - self.assertEqual(source2.platform.long_name, "") - self.assertEqual(source2.platform.series_entity, "") - self.assertEqual(source2.instrument.long_name, "") - self.assertEqual(source2.instrument.category, "") - - def test_source_uniqueness(self): - - plat1 = Platform.objects.get(pk=661) # "short_name": "" - inst1 = Instrument.objects.get(pk=139)# "short_name": "" - source, created1 = Source.objects.get_or_create(platform=plat1, instrument=inst1) - source2, created2 = Source.objects.get_or_create(platform=plat1, instrument=inst1) - self.assertTrue(created1) - self.assertFalse(created2) - self.assertEqual(source2, source) - inst2 = Instrument.objects.get(pk=160)# "short_name": "" - source3,_ = Source.objects.get_or_create(platform=plat1, instrument=inst2) - self.assertNotEqual(source3, source2) - - - class TestCountCommand(TestCase): fixtures = ['vocabularies', 'catalog'] def test_count_command(self): diff --git a/geospaas/export_DIF/README.md b/geospaas/export_DIF/README.md deleted file mode 100644 index 349bff2c..00000000 --- a/geospaas/export_DIF/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# App for exporting metadata to GCMD DIF format - -The following fields are not in the Dataset model, and should be added by the -export-DIF app. - -See http://gcmd.nasa.gov/add/difguide/index.html for reference. - -## Required fields - -``` -metadata_name = models.CharField(max_length=50, default='CEOS IDN DIF') -metadata_version = models.CharField(max_length=50, default='VERSION 9.9') -``` - -## Highly recommended fields - -* data_set_citation's - -``` -personnel = models.ForeignKey(Personnel, blank=True, null=True) # = contact person -``` - -* paleo_temporal_coverage (if needed) -* spatial_coverage -* data_resolution -* project -* quality -* use_constraints -* distribution_media -* distribution_size -* distribution_format -* distribution_fee -* language (Language list in the ISO 639 language codes: http://www.loc.gov/standards/iso639-2/php/code_list.php) -* progress -* related_url - -## Recommended fields - -* DIF_revision_history -* originating_center -* references -* parent_DIF (this can be generated from the DatasetRelationship model) -* IDN_node -* DIF_creation_date -* last_DIF_revision_date -* future_DIF_review_date -* privacy_status (True or False) -* extended_metadata - -``` -# TO APP FOR EXPORTING DIF -class DIFRevisionHistoryItem(models.Model): - dataset = models.ForeignKey(Dataset) - date = models.DateField() - text = models.TextField() -``` - -``` -class DatasetCitation(models.Model): - dataset = models.ForeignKey(Dataset) - dataset_creator = models.ForeignKey(Personnel) - dataset_editor = models.ForeignKey(Personnel) - dataset_publisher = models.ForeignKey(DataCenter) - #dataset_title = models.CharField(max_length=220) # Same as entry_title in Dataset - dataset_series_name = models.CharField(max_length=220) - dataset_release_date = models.DateField() - dataset_release_place = models.CharField(max_length=80) - version = models.Charfield(max_length=15) - issue_identification = models.CharField(max_length=80) - data_presentation_form = models.CharField(max_length=80) - other_citation_details = models.CharField(max_length=220) - dataset_DOI = models.CharField(max_length=220) - online_resource = models.URLField(max_length=600) - - def __str__(self): - return self.dataset_DOI -``` - - - diff --git a/geospaas/export_DIF/__init__.py b/geospaas/export_DIF/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/export_DIF/admin.py b/geospaas/export_DIF/admin.py deleted file mode 100644 index 8c38f3f3..00000000 --- a/geospaas/export_DIF/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/geospaas/export_DIF/migrations/__init__.py b/geospaas/export_DIF/migrations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/export_DIF/models.py b/geospaas/export_DIF/models.py deleted file mode 100644 index 71a83623..00000000 --- a/geospaas/export_DIF/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/geospaas/export_DIF/tests.py b/geospaas/export_DIF/tests.py deleted file mode 100644 index 7ce503c2..00000000 --- a/geospaas/export_DIF/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/geospaas/export_DIF/views.py b/geospaas/export_DIF/views.py deleted file mode 100644 index 91ea44a2..00000000 --- a/geospaas/export_DIF/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/geospaas/nansat_ingestor/__init__.py b/geospaas/nansat_ingestor/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/admin.py b/geospaas/nansat_ingestor/admin.py deleted file mode 100644 index 8c38f3f3..00000000 --- a/geospaas/nansat_ingestor/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/geospaas/nansat_ingestor/management/__init__.py b/geospaas/nansat_ingestor/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/management/commands/__init__.py b/geospaas/nansat_ingestor/management/commands/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/geospaas/nansat_ingestor/management/commands/ingest.py b/geospaas/nansat_ingestor/management/commands/ingest.py deleted file mode 100644 index b1732881..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest.py +++ /dev/null @@ -1,57 +0,0 @@ -from django.core.management.base import BaseCommand, CommandError - -from geospaas.utils.utils import uris_from_args -from geospaas.catalog.models import DatasetURI -from geospaas.nansat_ingestor.models import Dataset - -class Command(BaseCommand): - args = '' - help = 'Add file to catalog archive' - n_points = 10 - - def add_arguments(self, parser): - parser.add_argument('files', nargs='*', type=str) - parser.add_argument('--nansat-option', - action='append', - help='''Option for Nansat() e.g. - mapperName="sentinel1a_l1" - (can be repated)''') - - parser.add_argument('--n_points', - action='store', - default=self.n_points, - help='''Number of points in the border''') - - def _get_args(self, *args, **options): - """ Get arguments needed to create the Dataset - """ - if len(options['files'])==0: - raise IOError('Please provide at least one filename') - n_points = int(options.get('n_points', self.n_points)) - - non_ingested_uris = DatasetURI.objects.all().get_non_ingested_uris( - uris_from_args(options['files']) - ) - - nansat_options = {} - if options['nansat_option']: - for opt in options['nansat_option']: - var, val = opt.split('=') - nansat_options[var] = val - - return non_ingested_uris, n_points, nansat_options - - def handle(self, *args, **options): - """ Ingest one Dataset per file that has not previously been ingested - """ - non_ingested_uris, n_points, nansat_options = self._get_args(*args, **options) - - for non_ingested_uri in non_ingested_uris: - self.stdout.write('Ingesting %s ...\n' % non_ingested_uri) - ds, cr = Dataset.objects.get_or_create(non_ingested_uri, n_points=n_points, **nansat_options) - if cr: - self.stdout.write('Successfully added: %s\n' % non_ingested_uri) - elif type(ds) == Dataset: - self.stdout.write('%s has been added before.\n' % non_ingested_uri) - else: - self.stdout.write('Could not add %s.\n' % non_ingested_uri) diff --git a/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py b/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py deleted file mode 100644 index 8040b81c..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest_hyrax.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import time -import re -import urllib.error -import urllib.request - -from django.core.management.base import BaseCommand, CommandError - -from geospaas.nansat_ingestor.management.commands.ingest import Command as IngestCommand - -# extend Ingest -class Command(IngestCommand): - args = 'HYRAX_URL' - help = 'Add file to catalog from HYRAX server' - - def handle(self, *args, **options): - print('Searching netcdf files. May take some time...\n\n\n') - nc_uris = find_netcdf_uris(args[0]) - num_nc_uris = len(nc_uris) - print('\nTrying to ingest %d datasets\n'%num_nc_uris) - super(Command, self).handle(*nc_uris, **options) - -def find_netcdf_uris(uri0, sleep=1.0): - uri0_base = os.path.split(uri0)[0] - print('Search in ', uri0_base) - # get HTML from the URL - time.sleep(sleep) - response = urllib.request.urlopen(uri0) - html = response.read() - - # find all links to netDCF - nc_uris = [os.path.join(uri0_base, uri.replace('href="', '').replace('.nc.dds', '.nc')) - for uri in re.findall('href=.*nc\.dds', html)] - - # find all links to sub-pages - uris = [os.path.join(uri0_base, uri.replace('href="', '')) - for uri in re.findall('href=.*/contents.html', html)] - # get links to netcDF also from sub-pages - try: - for uri in uris: - nc_uris += find_netcdf_uris(uri) - except (urllib.error.HTTPError, urllib.error.URLError) as e: - print('Server error %s'%e.message) - # return all links to netCDF - return nc_uris - - diff --git a/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py b/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py deleted file mode 100644 index cb7069e5..00000000 --- a/geospaas/nansat_ingestor/management/commands/ingest_thredds_crawl.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Note: This is tested on Sentinel-1 and Sentinel-2 data from the Norwegian ground segment, and -Arome forecasts from thredds.met.no. Other repositories may require slight changes in the code. This -must be developed gradually.. -""" -import warnings - -from django.core.management.base import BaseCommand, CommandError -from django.db.utils import IntegrityError -from thredds_crawler.crawl import Crawl - -from geospaas.catalog.models import DatasetURI -from geospaas.nansat_ingestor.models import Dataset as NansatDataset -from geospaas.utils.utils import validate_uri - - -def crawl_and_ingest(url, **options): - validate_uri(url) - - date = options.get('date', None) - filename = options.get('filename', None) - if date: - select = ['(.*%s.*\.nc)' % date] - elif filename: - select = ['(.*%s)' % filename] - else: - select = None - - skips = Crawl.SKIPS + ['.*ncml'] - c = Crawl(url, select=select, skip=skips, debug=True) - added = 0 - for ds in c.datasets: - for s in ds.services: - if s.get('service').lower() == 'opendap': - url = s.get('url') - name = s.get('name') - service = s.get('service') - try: - # Create Dataset from OPeNDAP url - this is necessary to get all metadata - gds, cr = NansatDataset.objects.get_or_create(url, uri_service_name=name, - uri_service_type=service) - except (IOError, AttributeError) as e: - # warnings.warn(e.message) - continue - if cr: - added += 1 - print('Added %s, no. %d/%d' % (url, added, len(c.datasets))) - # Connect all service uris to the dataset - for s in ds.services: - ds_uri, _ = DatasetURI.objects.get_or_create( - name=s.get('name'), - service=s.get('service'), - uri=s.get('url'), - dataset=gds) - print('Added %s, no. %d/%d' % (url, added, len(c.datasets))) - return added - - -class Command(BaseCommand): - args = '