Merge pull request #77 from nansencenter/gcmd_api_changes

GCMD api changes

Author: aperrin66

.github/workflows/ci.yml

@@ -12,7 +12,6 @@ jobs:
     strategy:
       matrix:
         python_version:
-          - '3.7'
           - '3.8'
           - '3.9'
           - '3.10'

pythesint/gcmd_vocabulary.py

@@ -1,8 +1,9 @@
 from __future__ import absolute_import
 
-from collections import OrderedDict
+import csv
 import requests
 import warnings
+from collections import OrderedDict
 
 from pythesint.json_vocabulary import JSONVocabulary
 
@@ -31,57 +32,34 @@ def _fetch_online_data(self, version=None):
         except requests.RequestException:
             print("Could not get the vocabulary file at '{}'".format(self.url))
             raise
-        rlines = [line for line in r.text.splitlines()]
-        gcmd_list = []
-
-        _read_revision(rlines[0], gcmd_list)
-
-        categories = _get_categories(rlines)
-        self._check_categories(categories)
-
-        for line in rlines[2:]:
-            _read_line(line, gcmd_list, categories)
-
-        return gcmd_list
-
 
-def _read_revision(line, gcmd_list):
-    ''' Reads the line, extracts the Revision into a new dictionary and appends
-    it to gcmd_list
-    '''
-    # TODO: Cast exception if not found?
-    if 'Keyword Version' and 'Revision' in line:
-        meta = line.split('","')
-        gcmd_list.append({
-            'Revision': meta[1][10:],
-            'Keyword Version': meta[0].split(': ')[1]
-        })
-
-
-def _get_categories(lines):
-    '''Get the categories from the lines read from the source'''
-    return lines[1].split(',')[:-1]
-
-
-def _read_line(line, gcmd_list, categories):
-    ''' Converts line into dictionary values for elements in the categories
-    appends the dictionary to gcmd_list
-    '''
-    gcmd_keywords = line.split('","')
-    gcmd_keywords[0] = gcmd_keywords[0].strip('"')
-    if gcmd_keywords[0] == 'NOT APPLICABLE':
-        return
-    # Remove last item (the ID is not needed)
-    gcmd_keywords.pop(-1)
-    # skip record if it is longer than the definition of categories
-    if len(gcmd_keywords) > len(categories):
-        return
-    line_kw = OrderedDict()
-    for i, key in enumerate(categories):
-        if i < len(gcmd_keywords):
-            # if the record is equal to definition of categories
-            line_kw[key] = gcmd_keywords[i]
-        else:
-            # if the record is shorter than definition of categories: add empty string
-            line_kw[key] = ""
-    gcmd_list.append(line_kw)
+        lines = r.text.splitlines()
+        keywords = []
+        # Add version+revision information
+        self._read_revision(lines[0], keywords)
+        # parse actual CSV contents
+        reader = csv.DictReader(lines[1:], dialect='unix', restval='')
+        self._check_categories(reader.fieldnames)
+        keywords.extend(list(reader))
+        # remove UUID and extra fields
+        for kw in keywords[1:]:
+            for key in ('UUID', None):
+                try:
+                    del kw[key]
+                except KeyError:
+                    pass
+
+        return keywords
+
+    @staticmethod
+    def _read_revision(line, gcmd_list):
+        ''' Reads the line, extracts the Revision into a new dictionary and appends
+        it to gcmd_list
+        '''
+        # TODO: Cast exception if not found?
+        if 'Keyword Version' and 'Revision' in line:
+            meta = line.split('","')
+            gcmd_list.append({
+                'Revision': meta[1][10:],
+                'Keyword Version': meta[0].split(': ')[1]
+            })

pythesint/pythesintrc.yaml

@@ -28,7 +28,6 @@
       - Subtype
       - Short_Name
       - Long_Name
-    version: 13.6
 
 - name: gcmd_science_keyword
   module: gcmd_vocabulary
@@ -43,7 +42,6 @@
       - Variable_Level_2
       - Variable_Level_3
       - Detailed_Variable
-    version: 13.6
 
 - name: gcmd_provider
   module: gcmd_vocabulary
@@ -58,7 +56,6 @@
       - Short_Name
       - Long_Name
       - Data_Center_URL
-    version: 13.6
 
 - name: gcmd_platform
   module: gcmd_vocabulary
@@ -71,7 +68,6 @@
       - Sub_Category
       - Short_Name
       - Long_Name
-    version: 13.6
 
 - name: gcmd_location
   module: gcmd_vocabulary
@@ -85,7 +81,6 @@
       - Location_Subregion2
       - Location_Subregion3
       - Location_Subregion4
-    version: 13.6
 
 - name: gcmd_horizontalresolutionrange
   module: gcmd_vocabulary
@@ -94,7 +89,6 @@
     url: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/horizontalresolutionrange/?format=csv
     categories:
       - Horizontal_Resolution_Range
-    version: 13.6
 
 - name: gcmd_verticalresolutionrange
   module: gcmd_vocabulary
@@ -103,7 +97,6 @@
     url: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/verticalresolutionrange/?format=csv
     categories:
       - Vertical_Resolution_Range
-    version: 13.6
 
 - name: gcmd_temporalresolutionrange
   module: gcmd_vocabulary
@@ -112,7 +105,6 @@
     url: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/temporalresolutionrange/?format=csv
     categories:
       - Temporal_Resolution_Range
-    version: 13.6
 
 - name: gcmd_project
   module: gcmd_vocabulary
@@ -123,7 +115,6 @@
       - Bucket
       - Short_Name
       - Long_Name
-    version: 13.6
 
 - name: gcmd_rucontenttype
   module: gcmd_vocabulary
@@ -134,7 +125,6 @@
       - URLContentType
       - Type
       - Subtype
-    version: 13.6
 
 - name: cf_standard_name
   module: cf_vocabulary

pythesint/tests/test_gcmd_vocabulary.py

@@ -24,78 +24,12 @@ def test_read_revision(self):
                 ' representations can be found here: '
                 'http://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme'
                 '/instruments/?format=xml"')
-        pti.gcmd_vocabulary._read_revision(line, gcmd_list)
+        pti.gcmd_vocabulary.GCMDVocabulary._read_revision(line, gcmd_list)
         self.assertEqual(len(gcmd_list), 1)
         self.assertDictEqual(
             gcmd_list[0],
             {'Keyword Version': '8.1', 'Revision': '2016-01-08 13:40:40'})
 
-    def test_check_categories(self):
-        lines = [
-            'Revision',
-            'Category,Class,Type,Subtype,Short_Name,Long_Name,UUID'
-        ]
-        categories = ['Category', 'Class', 'Type', 'Subtype', 'Short_Name',
-                      'Long_Name']
-        self.assertListEqual(pti.gcmd_vocabulary._get_categories(lines), categories)
-
-    def test_read_line_simple(self):
-        gcmd_list = [{'Revision': '2016-01-08 13:40:40'}]
-        line = ('"Earth Remote Sensing Instruments","","","","","",'
-                '"6015ef7b-f3bd-49e1-9193-cc23db566b69"')
-        categories = ['Category', 'Class', 'Type', 'Subtype', 'Short_Name',
-                      'Long_Name']
-        pti.gcmd_vocabulary._read_line(line, gcmd_list, categories)
-        self.assertEqual(len(gcmd_list), 2)
-        self.assertEqual(gcmd_list[1], {'Category':
-                                        'Earth Remote Sensing Instruments',
-                                        'Class': '',
-                                        'Type': '',
-                                        'Subtype': '',
-                                        'Short_Name': '',
-                                        'Long_Name': ''})
-
-    def test_read_line_full(self):
-        gcmd_list = [{'Revision': '2016-01-08 13:40:40'}]
-        line = ('"Earth Remote Sensing Instruments","Active Remote Sensing",'
-                '"Altimeters","Lidar/Laser Altimeters","GLAS","Geoscience '
-                'Laser Altimeter System",'
-                '"57463f12-2a21-49f9-9477-718030d34291"')
-        categories = ['Category', 'Class', 'Type', 'Subtype', 'Short_Name',
-                      'Long_Name']
-        pti.gcmd_vocabulary._read_line(line, gcmd_list, categories)
-        self.assertEqual(len(gcmd_list), 2)
-        self.assertEqual(gcmd_list[1], {'Category':
-                                        'Earth Remote Sensing Instruments',
-                                        'Class': 'Active Remote Sensing',
-                                        'Type': 'Altimeters',
-                                        'Subtype': 'Lidar/Laser Altimeters',
-                                        'Short_Name': 'GLAS',
-                                        'Long_Name':
-                                        'Geoscience Laser Altimeter System'})
-
-    def test_read_line_not_applicable(self):
-        gcmd_list = [{'Revision': '2016-01-08 13:40:40'}]
-        line = ('"NOT APPLICABLE","","","","","",'
-                '"8129a4b9-b5f9-4585-87e6-4576c3a53682"')
-        categories = ['Category', 'Class', 'Type', 'Subtype', 'Short_Name',
-                      'Long_Name']
-        pti.gcmd_vocabulary._read_line(line, gcmd_list, categories)
-        self.assertEqual(len(gcmd_list), 1)
-        self.assertEqual(gcmd_list[0], {'Revision': '2016-01-08 13:40:40'})
-
-    def test_read_line_wrong_length(self):
-        gcmd_list = [{'Revision': '2016-01-08 13:40:40'}]
-        line = ('"Earth Remote Sensing Instruments","Active Remote Sensing",'
-                '"Altimeters","Lidar/Laser Altimeters","GLAS","Geoscience '
-                'Laser Altimeter System","Sneaky Extra Element",'
-                '"57463f12-2a21-49f9-9477-718030d34291"')
-        categories = ['Category', 'Class', 'Type', 'Subtype', 'Short_Name',
-                      'Long_Name']
-        pti.gcmd_vocabulary._read_line(line, gcmd_list, categories)
-        self.assertEqual(len(gcmd_list), 1)
-        self.assertEqual(gcmd_list[0], {'Revision': '2016-01-08 13:40:40'})
-
     def test_get_location_by_type(self):
         type = 'africa'
         a = pti.get_gcmd_location(type)