Merge pull request #18 from soda480/paging

soda480 · web-flow · commit dd094a54b7d3 · 2021-12-13T15:32:40.000-07:00
Paging
diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
 # github3api #
 [![GitHub Workflow Status](https://github.com/soda480/github3api/workflows/build/badge.svg)](https://github.com/soda480/github3api/actions)
 [![Code Coverage](https://codecov.io/gh/soda480/github3api/branch/master/graph/badge.svg)](https://codecov.io/gh/soda480/github3api)
-[![Code Grade](https://www.code-inspector.com/project/13337/status/svg)](https://frontend.code-inspector.com/project/13337/dashboard)
+[![Code Grade](https://api.codiga.io/project/13337/status/svg)](https://frontend.code-inspector.com/project/13337/dashboard)
 [![vulnerabilities](https://img.shields.io/badge/vulnerabilities-None-brightgreen)](https://pypi.org/project/bandit/)
-[![PyPI version](https://badge.fury.io/py/github3api.svg)](https://badge.fury.io/py/github3api)
+[![PyPI version](https://badge.fury.io/py/github3api.svg)](https://app.codiga.io/public/project/13337/github3api/dashboard)
 [![python](https://img.shields.io/badge/python-3.9-teal)](https://www.python.org/downloads/)
 
 An advanced REST client for the GitHub API. It is a subclass of [rest3client](https://pypi.org/project/rest3client/) tailored for the GitHub API with special optional directives for GET requests that can return all pages from an endpoint or return a generator that can be iterated over (for paged requests). By default all requests will be retried if ratelimit request limit is reached.
@@ -64,7 +64,7 @@ pip install github3api
 
 `GET all` directive - Get all pages from an endpoint and return list containing only matching attributes
 ```python
-for repo in client.get('/user/repos', _get='all', _attributes=['full_name']):
+for repo in client.get('/orgs/edgexfoundry/repos', _get='all', _attributes=['full_name']):
     print(repo['full_name'])
 ```
 
@@ -78,7 +78,6 @@ for page in client.get('/user/repos', _get='page'):
 `total` - Get total number of resources at given endpoint
 ```python
 print(client.total('/user/repos'))
-6218
 ```
 
 `graphql` - execute graphql query
diff --git a/build.py b/build.py
@@ -30,7 +30,7 @@
 authors = [Author('Emilio Reyes', 'emilio.reyes@intel.com')]
 summary = 'An advanced REST client for the GitHub API'
 url = 'https://github.com/soda480/github3api'
-version = '0.2.0'
+version = '0.3.0'
 default_task = [
     'clean',
     'analyze',
@@ -51,7 +51,7 @@ def set_properties(project):
     project.set_property('flake8_include_scripts', True)
     project.set_property('flake8_include_test_sources', True)
     project.set_property('flake8_ignore', 'E501, W503, F401, E722, W605')
-    project.build_depends_on_requirements('requirements-build.txt')
+    project.build_depends_on('mock')
     project.depends_on_requirements('requirements.txt')
     project.set_property('distutils_readme_description', True)
     project.set_property('distutils_description_overwrite', True)
diff --git a/requirements-build.txt b/requirements-build.txt
diff --git a/src/main/python/github3api/githubapi.py b/src/main/python/github3api/githubapi.py
@@ -70,63 +70,6 @@ def get_headers(self, **kwargs):
         headers['Accept'] = f'application/vnd.github.{self.version}+json'
         return headers
 
-    def _get_next_endpoint(self, url):
-        """ return next endpoint
-        """
-        if not url:
-            logger.debug('link header is empty')
-            return
-        endpoint = self.get_endpoint_from_url(url)
-        logger.debug(f'next endpoint is: {endpoint}')
-        return endpoint
-
-    def _get_all(self, endpoint, **kwargs):
-        """ return all pages from endpoint
-        """
-        logger.debug(f'get items from: {endpoint}')
-        items = []
-        while True:
-            url = None
-            response = super(GitHubAPI, self).get(endpoint, raw_response=True, **kwargs)
-            if response:
-                data = response.json()
-                if isinstance(data, list):
-                    items.extend(response.json())
-                else:
-                    items.append(data)
-                url = response.links.get('next', {}).get('url')
-
-            endpoint = self._get_next_endpoint(url)
-            if not endpoint:
-                logger.debug('no more pages to retrieve')
-                break
-
-        return items
-
-    def _get_page(self, endpoint, **kwargs):
-        """ return generator that yields pages from endpoint
-        """
-        while True:
-            response = super(GitHubAPI, self).get(endpoint, raw_response=True, **kwargs)
-            yield response.json()
-            endpoint = self._get_next_endpoint(response.links.get('next', {}).get('url'))
-            if not endpoint:
-                logger.debug('no more pages')
-                break
-
-    def get(self, endpoint, **kwargs):
-        """ ovverride get to provide paging support
-        """
-        directive = kwargs.pop('_get', None)
-        attributes = kwargs.pop('_attributes', None)
-        if directive == 'all':
-            items = self._get_all(endpoint, **kwargs)
-            return GitHubAPI.match_keys(items, attributes)
-        elif directive == 'page':
-            return self._get_page(endpoint, **kwargs)
-        else:
-            return super(GitHubAPI, self).get(endpoint, **kwargs)
-
     def total(self, endpoint):
         """ return total number of resources
         """
@@ -140,7 +83,7 @@ def total(self, endpoint):
         response = self.get(endpoint, raw_response=True)
         if response.links:
             last_url = response.links['last']['url']
-            endpoint = self.get_endpoint_from_url(last_url)
+            endpoint = self._get_endpoint_from_url(last_url)
             items = self.get(endpoint)
             per_page = GitHubAPI.get_per_page_from_url(last_url)
             last_page = GitHubAPI.get_page_from_url(last_url)
@@ -150,11 +93,6 @@ def total(self, endpoint):
             total = len(items)
         return total
 
-    def get_endpoint_from_url(self, url):
-        """ return endpoint from url
-        """
-        return url.replace(f'https://{self.hostname}', '')
-
     @staticmethod
     def get_page_from_url(url):
         """ get page query parameter form url
@@ -206,19 +144,6 @@ def log_ratelimit(ratelimit):
         """
         logger.debug(f"{ratelimit['remaining']}/{ratelimit['limit']} resets in {ratelimit['minutes']} min")
 
-    @staticmethod
-    def match_keys(items, attributes):
-        """ return list of items with matching keys from list of attributes
-        """
-        if not attributes:
-            return items
-        matched_items = []
-        for item in items:
-            matched_items.append({
-                key: item[key] for key in attributes if key in item
-            })
-        return matched_items
-
     @staticmethod
     def retry_ratelimit_error(exception):
         """ return True if exception is 403 HTTPError, False otherwise
diff --git a/src/unittest/python/test_githubapi.py b/src/unittest/python/test_githubapi.py
@@ -73,29 +73,6 @@ def tearDown(self):
 
         pass
 
-    def test__match_keys_Should_Return_Items_When_NoAttributes(self, *patches):
-        result = GitHubAPI.match_keys(self.items, None)
-        self.assertEqual(result, self.items)
-
-    def test__match_keys_Should_ReturnExpected_When_Called(self, *patches):
-        result = GitHubAPI.match_keys(self.items, ['name', 'key1'])
-        expected_result = [
-            {
-                'name': 'name1-mid-last1',
-                'key1': 'value1'
-            }, {
-                'name': 'name2-mid-last2',
-                'key1': 'value1'
-            }, {
-                'name': 'name3-med-last3',
-                'key1': 'value1'
-            }, {
-                'name': 'name4-mid-last4',
-                'key1': 'value1'
-            }
-        ]
-        self.assertEqual(result, expected_result)
-
     def test__get_ratelimit_Should_ReturnExpected_When_NoHeader(self, *patches):
         result = GitHubAPI.get_ratelimit({})
         expected_result = {}
@@ -184,116 +161,6 @@ def test__get_next_endpoint_Should_ReturnExpected_When_CalledWithNextEndpoint(se
         expected_result = '/organizations/27781926/repos?page=4'
         self.assertEqual(result, expected_result)
 
-    @patch('github3api.GitHubAPI._get_next_endpoint')
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_all_Should_ReturnExpected_When_GetReturnsList(self, get_patch, get_next_endpoint_patch, *patches):
-        response_mock1 = Mock()
-        response_mock1.json.return_value = ['item1', 'item2']
-        response_mock2 = Mock()
-        response_mock2.json.return_value = ['item3', 'item4']
-        get_patch.side_effect = [
-            response_mock1,
-            response_mock2
-        ]
-        get_next_endpoint_patch.side_effect = [
-            {'Link': 'link-header-value'},
-            {}
-        ]
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client._get_all('/repos/edgexfoundry/cd-management/milestones')
-        expected_result = ['item1', 'item2', 'item3', 'item4']
-        self.assertEqual(result, expected_result)
-
-    @patch('github3api.GitHubAPI._get_next_endpoint')
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_all_Should_ReturnExpected_When_GetReturnsDict(self, get_patch, get_next_endpoint_patch, *patches):
-        response_mock1 = Mock()
-        response_mock1.json.return_value = {'key1': 'value1'}
-        response_mock2 = Mock()
-        response_mock2.json.return_value = {'key2': 'value2'}
-        get_patch.side_effect = [
-            response_mock1,
-            response_mock2
-        ]
-        get_next_endpoint_patch.side_effect = [
-            {'Link': 'link-header-value'},
-            {}
-        ]
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client._get_all('/repos/edgexfoundry/cd-management/milestones')
-        expected_result = [{'key1': 'value1'}, {'key2': 'value2'}]
-        self.assertEqual(result, expected_result)
-
-    @patch('github3api.GitHubAPI._get_next_endpoint')
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_all_Should_ReturnEmptyList_When_NoResponse(self, get_patch, get_next_endpoint_patch, *patches):
-        get_patch.side_effect = [
-            None
-        ]
-        get_next_endpoint_patch.side_effect = [
-            None
-        ]
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client._get_all('/repos/edgexfoundry/cd-management/milestones')
-        expected_result = []
-        self.assertEqual(result, expected_result)
-
-    @patch('github3api.GitHubAPI._get_next_endpoint')
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_page_Should_ReturnExpected_When_Called(self, get_patch, get_next_endpoint_patch, *patches):
-        response_mock1 = Mock()
-        response_mock1.json.return_value = ['page1', 'page2']
-        response_mock2 = Mock()
-        response_mock2.json.return_value = ['page3', 'page4']
-        get_patch.side_effect = [response_mock1, response_mock2]
-        get_next_endpoint_patch.return_value = ['next-endpoint', 'next-endpoint', None]
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client._get_page('endpoint')
-        self.assertEqual(next(result), ['page1', 'page2'])
-        self.assertEqual(next(result), ['page3', 'page4'])
-        # with self.assertRaises(StopIteration):
-        #     next(result)
-
-    @patch('github3api.GitHubAPI._get_next_endpoint')
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_page_Should_ReturnExpected_When_NoEndpoint(self, get_patch, get_next_endpoint_patch, *patches):
-        response_mock1 = Mock()
-        response_mock1.json.return_value = ['page1', 'page2']
-        get_patch.side_effect = [response_mock1]
-        get_next_endpoint_patch.side_effect = [None]
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client._get_page('endpoint')
-        self.assertEqual(next(result), ['page1', 'page2'])
-        with self.assertRaises(StopIteration):
-            next(result)
-
-    @patch('github3api.GitHubAPI.match_keys')
-    @patch('github3api.GitHubAPI._get_all')
-    def test__get_Should_CallExpected_When_AllDirective(self, get_all_patch, match_keys_patch, *patches):
-        client = GitHubAPI(bearer_token='bearer-token')
-        endpoint = '/repos/edgexfoundry/cd-management/milestones'
-        attributes = ['key1', 'key2']
-        result = client.get(endpoint, _get='all', _attributes=attributes)
-        get_all_patch.assert_called_once_with(endpoint)
-        match_keys_patch.assert_called_once_with(get_all_patch.return_value, attributes)
-        self.assertEqual(result, match_keys_patch.return_value)
-
-    @patch('github3api.GitHubAPI._get_page')
-    def test__get_Should_CallExpected_When_GenDirective(self, get_page_patch, *patches):
-        client = GitHubAPI(bearer_token='bearer-token')
-        endpoint = '/repos/edgexfoundry/cd-management/milestones'
-        result = client.get(endpoint, _get='page')
-        get_page_patch.assert_called_once_with(endpoint)
-        self.assertEqual(result, get_page_patch.return_value)
-
-    @patch('github3api.githubapi.RESTclient.get')
-    def test__get_Should_CallExpected_When_NoDirective(self, get_patch, *patches):
-        client = GitHubAPI(bearer_token='bearer-token')
-        endpoint = '/repos/edgexfoundry/cd-management/milestones'
-        result = client.get(endpoint, k1='v1', k2='v2')
-        get_patch.assert_called_once_with(endpoint, k1='v1', k2='v2')
-        self.assertEqual(result, get_patch.return_value)
-
     @patch('github3api.githubapi.getenv', return_value='token')
     @patch('github3api.githubapi.GitHubAPI')
     def test__get_client_Should_CallAndReturnExpected_When_Called(self, githubapi_patch, getenv_patch, *patches):
@@ -322,12 +189,6 @@ def test__get_retries_Should_ReturnExpected_When_Called(self, *patches):
         ]
         self.assertEqual(client.retries, expected_retries)
 
-    def test__get_endpoint_from_url_Should_ReturnExpected_When_Called(self, *patches):
-        client = GitHubAPI(bearer_token='bearer-token')
-        result = client.get_endpoint_from_url('https://api.github.com/user/repos?page=2')
-        expected_result = '/user/repos?page=2'
-        self.assertEqual(result, expected_result)
-
     def test__get_page_from_url_Should_ReturnExpected_When_Match(self, *patches):
         result = GitHubAPI.get_page_from_url('https://api.github.com/user/repos?page=213')
         expected_result = 213
