Merge pull request #30 from opsdisk/add-language-support-for-search-results

Add search result language support

Author: opsdisk

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="yagooglesearch",
-    version="1.7.0",
+    version="1.8.0",
     author="Brennon Thomas",
     author_email="info@opsdisk.com",
     description="A Python library for executing intelligent, realistic-looking, and tunable Google searches.",

yagooglesearch/__init__.py

@@ -12,7 +12,7 @@
 
 # Custom Python libraries.
 
-__version__ = "1.7.0"
+__version__ = "1.8.0"
 
 # Logging
 ROOT_LOGGER = logging.getLogger("yagooglesearch")
@@ -38,13 +38,25 @@
 
 try:
     user_agents_file = os.path.join(install_folder, "user_agents.txt")
-    with open(user_agents_file) as fh:
+    with open(user_agents_file, "r") as fh:
         user_agents_list = [_.strip() for _ in fh.readlines()]
 
 except Exception:
     user_agents_list = [USER_AGENT]
 
 
+# Load the list of result languages.  Compiled by viewing the source code at https://www.google.com/advanced_search for
+# the supported languages.
+try:
+    result_languages_file = os.path.join(install_folder, "result_languages.txt")
+    with open(result_languages_file, "r") as fh:
+        result_languages_list = [_.strip().split("=")[0] for _ in fh.readlines()]
+
+except Exception as e:
+    print(f"There was an issue loading the result languages file.  Exception: {e}")
+    result_languages_list = []
+
+
 def get_tbs(from_date, to_date):
     """Helper function to format the tbs parameter dates.  Note that verbatim mode also uses the &tbs= parameter, but
     this function is just for customized search periods.
@@ -69,7 +81,8 @@ def __init__(
         self,
         query,
         tld="com",
-        lang="en",
+        lang_html_ui="en",
+        lang_result="lang_en",
         tbs="0",
         safe="off",
         start=0,
@@ -92,7 +105,8 @@ def __init__(
         SearchClient
         :param str query: Query string.  Must NOT be url-encoded.
         :param str tld: Top level domain.
-        :param str lang: Language.
+        :param str lang_html_ui: HTML User Interface language.
+        :param str lang_result: Search result language.
         :param str tbs: Verbatim search or time limits (e.g., "qdr:h" => last hour, "qdr:d" => last 24 hours, "qdr:m"
             => last month).
         :param str safe: Safe search.
@@ -127,7 +141,8 @@ def __init__(
 
         self.query = urllib.parse.quote_plus(query)
         self.tld = tld
-        self.lang = lang
+        self.lang_html_ui = lang_html_ui
+        self.lang_result = lang_result.lower()
         self.tbs = tbs
         self.safe = safe
         self.start = start
@@ -150,6 +165,13 @@ def __init__(
         ROOT_LOGGER.setLevel((6 - self.verbosity) * 10)
 
         # Argument checks.
+        if self.lang_result not in result_languages_list:
+            ROOT_LOGGER.error(
+                f"{self.lang_result} is not a valid language result.  See {result_languages_file} for the list of valid "
+                'languages.  Setting lang_result to "lang_en".'
+            )
+            self.lang_result = "lang_en"
+
         if self.num > 100:
             ROOT_LOGGER.warning("The largest value allowed by Google for num is 100.  Setting num to 100.")
             self.num = 100
@@ -171,6 +193,7 @@ def __init__(
             "safe",
             "start",
             "tbs",
+            "lr",
         )
 
         # Default user agent, unless instructed by the user to change it.
@@ -215,28 +238,28 @@ def update_urls(self):
 
         # First search requesting the default 10 search results.
         self.url_search = (
-            f"https://www.google.{self.tld}/search?hl={self.lang}&"
+            f"https://www.google.{self.tld}/search?hl={self.lang_html_ui}&lr={self.lang_result}&"
             f"q={self.query}&btnG=Google+Search&tbs={self.tbs}&safe={self.safe}&"
             f"cr={self.country}&filter=0"
         )
 
         # Subsequent searches starting at &start= and retrieving 10 search results at a time.
         self.url_next_page = (
-            f"https://www.google.{self.tld}/search?hl={self.lang}&"
+            f"https://www.google.{self.tld}/search?hl={self.lang_html_ui}&lr={self.lang_result}&"
             f"q={self.query}&start={self.start}&tbs={self.tbs}&safe={self.safe}&"
             f"cr={self.country}&filter=0"
         )
 
         # First search requesting more than the default 10 search results.
         self.url_search_num = (
-            f"https://www.google.{self.tld}/search?hl={self.lang}&"
+            f"https://www.google.{self.tld}/search?hl={self.lang_html_ui}&lr={self.lang_result}&"
             f"q={self.query}&num={self.num}&btnG=Google+Search&tbs={self.tbs}&"
             f"safe={self.safe}&cr={self.country}&filter=0"
         )
 
         # Subsequent searches starting at &start= and retrieving &num= search results at a time.
         self.url_next_page_num = (
-            f"https://www.google.{self.tld}/search?hl={self.lang}&"
+            f"https://www.google.{self.tld}/search?hl={self.lang_html_ui}&lr={self.lang_result}&"
             f"q={self.query}&start={self.start}&num={self.num}&tbs={self.tbs}&"
             f"safe={self.safe}&cr={self.country}&filter=0"
         )
@@ -458,10 +481,8 @@ def search(self):
                     url = self.url_search_num
 
             # Append extra GET parameters to the URL.  This is done on every iteration because we're rebuilding the
-            # entire URL at the end of this loop.
+            # entire URL at the end of this loop.  The keys and values are not URL encoded.
             for key, value in self.extra_params.items():
-                key = urllib.parse.quote_plus(key)
-                value = urllib.parse.quote_plus(value)
                 url += f"&{key}={value}"
 
             # Request Google search results.

yagooglesearch/result_languages.txt

@@ -0,0 +1,46 @@
+lang_af=Afrikaans
+lang_ar=Arabic
+lang_hy=Armenian
+lang_be=Belarusian
+lang_bg=Bulgarian
+lang_ca=Catalan
+lang_zh-CN=Chinese (Simplified)
+lang_zh-TW=Chinese (Traditional)
+lang_hr=Croatian
+lang_cs=Czech
+lang_da=Danish
+lang_nl=Dutch
+lang_en=English
+lang_eo=Esperanto
+lang_et=Estonian
+lang_tl=Filipino
+lang_fi=Finnish
+lang_fr=French
+lang_de=German
+lang_el=Greek
+lang_iw=Hebrew
+lang_hi=Hindi
+lang_hu=Hungarian
+lang_is=Icelandic
+lang_id=Indonesian
+lang_it=Italian
+lang_ja=Japanese
+lang_ko=Korean
+lang_lv=Latvian
+lang_lt=Lithuanian
+lang_no=Norwegian
+lang_fa=Persian
+lang_pl=Polish
+lang_pt=Portuguese
+lang_ro=Romanian
+lang_ru=Russian
+lang_sr=Serbian
+lang_sk=Slovak
+lang_sl=Slovenian
+lang_es=Spanish
+lang_sw=Swahili
+lang_sv=Swedish
+lang_th=Thai
+lang_tr=Turkish
+lang_uk=Ukrainian
+lang_vi=Vietnamese