Merge pull request #105 from nansencenter/issue101-non-unique-parameter

fix #101: use Parameter filter instead of get in nansat_ingestor.Data…

Author: opsdep

geospaas/catalog/tests.py

@@ -159,7 +159,8 @@ class DatasetParameterTests(TestCase):
     def test_add_sar_sigma0(self):
         ds = Dataset.objects.get(pk=1)
         p = Parameter.objects.get(
-                standard_name='surface_backwards_scattering_coefficient_of_radar_wave')
+                standard_name='surface_backwards_scattering_coefficient_of_radar_wave',
+                short_name='sigma0')
         dp = DatasetParameter(dataset=ds, parameter=p)
         dp.save()
         self.assertEqual(dp.parameter.short_name, 'sigma0')

geospaas/nansat_ingestor/managers.py

@@ -135,19 +135,26 @@ def get_or_create(self,
         all_band_meta = n.bands()
         for band_id in range(1, len(all_band_meta)+1):
             band_meta = all_band_meta[band_id]
-            if ('standard_name' in band_meta.keys()):
-                # There is no need to introduce latitude and longitude as a paramter for
-                # ingested dataset. Thus, these two are excluded from this loop
-                if ((band_meta['standard_name'] != 'latitude')
-                        & (band_meta['standard_name'] != 'longitude')):
-                    pp = Parameter.objects.get(
-                        standard_name=band_meta['standard_name'])
-                    dsp, dsp_created = DatasetParameter.objects.get_or_create(
-                        dataset=ds, parameter=pp)
-                    ds.parameters.add(pp)
+            standard_name = band_meta.get('standard_name', None)
+            short_name = band_meta.get('short_name', None)
+            units = band_meta.get('units', None)
+            if standard_name in ['latitude', 'longitude', None]:
+                continue
+            params = Parameter.objects.filter(standard_name=standard_name)
+            if params.count() > 1 and short_name is not None:
+                params = params.filter(short_name=short_name)
+            if params.count() > 1 and units is not None:
+                params = params.filter(units=units)
+            if params.count() >= 1:
+                dsp, dsp_created = DatasetParameter.objects.get_or_create(
+                        dataset=ds, parameter=params[0])
+                ds.parameters.add(params[0])
 
         # create dataset URI
-        ds_uri, _ = DatasetURI.objects.get_or_create(name=uri_service_name, service=uri_service_type, uri=uri,
-                                                     dataset=ds)
+        ds_uri, _ = DatasetURI.objects.get_or_create(
+            name=uri_service_name,
+            service=uri_service_type,
+            uri=uri,
+            dataset=ds)
 
         return ds, created

geospaas/nansat_ingestor/tests/tests.py renamed to geospaas/nansat_ingestor/tests.py

@@ -87,7 +87,20 @@ class BasetForTests(TestCase):
             'suffix': 'testing',
             'units': 'testing',
             'wkv': 'testing'},
-
+        5: {'colormap': 'gray',
+            'dataType': '6',
+            'long_name': 'Normalized Radar Cross Section',
+            'minmax': '0 0.1',
+            'name': 'gamma0_HH',
+            'PixelFunctionType': 'Sentinel1Calibration',
+            'polarization': 'HH',
+            'short_name': 'gamma0',
+            'SourceBand': '1',
+            'SourceFilename': '/vsimem/0BSD1QSPFL.vrt',
+            'standard_name': 'surface_backwards_scattering_coefficient_of_radar_wave',
+            'suffix': 'HH',
+            'units': 'm/m',
+            'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave'},
     }
 
     def setUp(self):
@@ -167,25 +180,31 @@ def test_fail_invalid_uri(self):
             ds, created = Dataset.objects.get_or_create(uri)
 
     @patch('os.path.isfile')
-    def test_exception_of_parameter_handling_for_longitude_and_latitude(self, mock_isfile):
-        '''shall return standard all specified parameter
-        without the parameter for latitude and longitude'''
+    def test_dont_add_longitude_latitude(self, mock_isfile):
+        """ shall not add latitude and longitude into DatasetParameter table """
         mock_isfile.return_value = True
         uri = 'file://localhost/some/folder/filename.ext'
         ds0, cr0 = Dataset.objects.get_or_create(uri)
+        ds_params_standard_names = ds0.parameters.values_list('standard_name', flat=True)
         # longitude should not be one of the parameters
-        self.assertNotIn(
-            self.predefined_band_metadata_dict[3]['standard_name'],
-            list(ds0.parameters.all().values('standard_name'))[0].values())
+        self.assertNotIn('longitude', ds_params_standard_names)
         # latitude should not be one of the parameters
-        self.assertNotIn(
-            self.predefined_band_metadata_dict[4]['standard_name'],
-            list(ds0.parameters.all().values('standard_name'))[0].values())
-        # other parameters must be in the parameters
-        self.assertIn(
-            self.predefined_band_metadata_dict[2]['standard_name'],
-            list(ds0.parameters.all().values('standard_name'))[0].values())
+        self.assertNotIn('latidtude', ds_params_standard_names)
 
+    @patch('os.path.isfile')
+    def test_add_sigma0_gamma0(self, mock_isfile):
+        """ shall add both sigma0 and gamma0 with same standard name  into DatasetParameter table """
+        mock_isfile.return_value = True
+        uri = 'file://localhost/some/folder/filename.ext'
+        ds0, cr0 = Dataset.objects.get_or_create(uri)
+        ds_params_standard_names = ds0.parameters.values_list('standard_name', flat=True)
+        ds_params_short_names = ds0.parameters.values_list('short_name', flat=True)
+        self.assertEqual(len(ds_params_standard_names), 2)
+        self.assertEqual(len(ds_params_short_names), 2)
+        self.assertIn('surface_backwards_scattering_coefficient_of_radar_wave',
+            ds_params_standard_names)
+        self.assertIn('sigma0', ds_params_short_names)
+        self.assertIn('gamma0', ds_params_short_names)
 
 class TestDatasetURI(BasetForTests):
     @patch('os.path.isfile')

geospaas/nansat_ingestor/tests/__init__.py

@@ -129639,5 +129639,15 @@
     },
     "model": "vocabularies.parameter",
     "pk": 58
+},
+{
+    "fields": {
+        "units": "m/m",
+        "standard_name": "surface_backwards_scattering_coefficient_of_radar_wave",
+        "gcmd_science_keyword": null,
+        "short_name": "gamma0"
+    },
+    "model": "vocabularies.parameter",
+    "pk": 59
 }
 ]