Merge pull request #309 from scholarly-python-package/develop

Version 1.2.2 to fix issues with the ScraperAPI

Author: ipeirotis

scholarly/_navigator.py

@@ -101,12 +101,7 @@ def _get_page(self, pagerequest: str) -> str:
                 w = random.uniform(1,2)
                 time.sleep(w)
 
-                params = {}
-                if self.pm._ScraperAPI_KEY:
-                    params = { 'api_key': self.pm._ScraperAPI_KEY, 'url': pagerequest }
-                    pagerequest = 'http://api.scraperapi.com'
-                
-                resp = self._session.get(pagerequest, params=params, timeout=timeout)
+                resp = self._session.get(pagerequest, timeout=timeout)
                 self.logger.info("Session proxy config is {}".format(self._session.proxies))
 
                 has_captcha = self._requests_has_captcha(resp.text)

scholarly/_proxy_generator.py

@@ -409,7 +409,7 @@ def ScraperAPI(self, API_KEY):
         assert API_KEY is not None
         self._ScraperAPI_KEY = API_KEY
 
-        self._use_proxy(http='http://api.scraperapi.com/?api_key='+API_KEY)
+        self._use_proxy(http=f'http://scraperapi:{API_KEY}@proxy-server.scraperapi.com:8001')
 
     def has_proxy(self)-> bool:
         return self._proxy_gen or self._can_refresh_tor

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.2.1',
+    version='1.2.2',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca',
     description='Simple access to Google Scholar authors and citations',