Merge pull request #64 from my-dev-app/refactor_lib

Refactor lib

Author: 0x78f1935

README.md

@@ -18,7 +18,7 @@ By undeƒined
 
 # AProxyRelay: An Async Request Library with Proxy Rotation
 
-AProxyRelay is an asynchronous request library designed for easy data retrieval using various proxy servers. It seamlessly handles proxy rotation, preserves data that fails to be requested, and simplifies API scraping. The library is written in `Python 3.12.1` but is compatible with projects utilizing `Python 3.11.2`.
+AProxyRelay is an asynchronous request library designed for easy data retrieval using various proxy servers. It seamlessly handles proxy rotation, preserves data that fails to be requested, and simplifies API scraping. The library is written in `Python 3.12.2`.
 
 In addition, tested proxies will be shared with other people using this library. The more this library is utilized, the bigger the pool of available proxies.
 
@@ -33,37 +33,60 @@ AProxyRelay streamlines the process of making asynchronous requests with proxy s
 
 ### Example
 ```py
+# -*- mode: python ; coding: utf-8 -*-
 from aproxyrelay import AProxyRelay
 
+# Note: Duplicates will be removed by the library
 targets = [
-    'https://some-website.com/api/app?id=1551360',
-    'https://some-website.com/api/app?id=2072450',
-    'https://some-website.com/api/app?id=1924360',
-    'https://some-website.com/api/app?id=1707870',
-    'https://some-website.com/api/app?id=1839880',
+    'https://gg.my-dev.app/api/v1/proxies/available?zone=US&anonimity=all&protocol=all&page=1&size=100&type=example',
+    'https://gg.my-dev.app/api/v1/proxies/available?zone=DE&anonimity=all&protocol=all&page=1&size=100&type=example',
+    'https://gg.my-dev.app/api/v1/proxies/available?zone=NL&anonimity=all&protocol=all&page=1&size=100&type=example',
+    'https://gg.my-dev.app/api/v1/proxies/available?zone=CA&anonimity=all&protocol=all&page=1&size=100&type=example',
+    'https://gg.my-dev.app/api/v1/proxies/available?zone=AU&anonimity=all&protocol=all&page=1&size=100&type=example',
 ]
 
 # Initialize proxy relay
 proxy_relay = AProxyRelay(
     targets=targets,
     timeout=5,
-    test_proxy=True,
-    test_timeout=10,
-    zone='us',
+    scrape=True,
+    filter=True,
+    zones=['us'],
+    unpack=lambda data, target: data['results'],
+    debug=False,
 )
 
 # Fetch data
 data = proxy_relay.start()
 
 # Result Queue
 print(data.qsize())
+
+while not data.empty():
+    content = data.get()
+    print(content)
+
 ```
 
 ## A Proxy Relay: Installation
 Simply run
 
     pip install aproxyrelay
 
+### Parameters
+
+| Parameters  | Type          | Function                                       | Description                                                  |
+|-------------|---------------|------------------------------------------------|--------------------------------------------------------------|
+| targets     | list[str]      | Target endpoints provided in an array           | Each endpoint will be requested with an available proxy. If a proxy is unavailable and the request fails, we store it in a queue and try it out with another proxy until we have data. |
+| timeout     | int           | Allowed proxy timeout. **Defaults to 5**        | A proxy has to respond within the provided timeout to be considered valid. Otherwise, it will be discarded.                |
+| scrape      | bool          | Indicator to utilize the proxy scraper. **Defaults to True** | The decision to scrape for proxies is determined by the value of this parameter. When set to True (default), the proxy scraper is used, which is slower but provides a broader range of proxies. When set to False, proxies are fetched from a single source, offering a faster but more limited selection. |
+| filter      | bool          | Indicator for filtering bad proxies. **Defaults to True** | If set to True (default), the tool will test proxy connections before using them. This process might take a bit longer, but it ensures that the proxies are valid before utilization. |
+| zones       | list[str]      | An array of proxy zones. **Defaults to ['US']**  | Sometimes it matters where the proxy is located. Each item in this list ensures the proxy is located in that specific zone, and requests made from the proxy are coming from the location provided. It acts like a whitelist for allowed proxy locations. |
+| unpack      | lambda        | Anonymous function for unpacking data. **Defaults to `lambda data, target: data`** | When a request has been made to a target through a proxy and data has been fetched, this lambda method formats the result data before putting it into the result queue. **data** -> output from the target, **target** -> target URL. |
+| debug       | bool          | Indicator which enables debug mode. **Defaults to False** | When true, additional logging will be printed to the terminal, enabling debug mode. |
+
+
+
 ## A Proxy Relay: Local Development
 To install all library dependencies for local development, excluding the core code available locally, use the following command within a virtual environment:
 
@@ -89,10 +112,12 @@ from .core import ScraperCore
 class MainScraper(ScraperCore):
     def __init__(self) -> None:
         ScraperCore.__init__(self)
+        self.zone = None
 
     @classmethod
     async def format_url(cls, url, *args, **kwargs) -> str:
         """Formats URL before scraping, let us adjust query parameters for each parser"""
+        cls.zone = kwargs.get("zone", "us")
         new_url = f'{url}'
         return new_url
 

aproxyrelay/__init__.py

@@ -12,13 +12,12 @@
 Automatically rotates bad proxy servers, preserves data which failed to request.
 Makes scraping API's easy and fun.
 """
-import asyncio
+from asyncio import get_event_loop, gather
 from datetime import datetime, UTC
+from logging import basicConfig, INFO, DEBUG, getLogger
+from typing import Callable
 from queue import Queue
 
-import logging
-import sys
-
 from .core import AProxyRelayCore
 
 
@@ -27,40 +26,57 @@ def __init__(
         self,
         targets: list[str],
         timeout: int = 5,
-        test_proxy: bool = True,
-        test_timeout: int = 20,
-        zone: str = 'us',
+        scrape: bool = True,
+        filter: bool = True,
+        zones: list[str] = ['US'],  # noqa: B006
+        unpack: Callable = lambda data, target: data,
         debug: bool = False,
-        steam: bool = False
     ) -> None:
         """
         Initialize an instance of AProxyRelay.
 
         Args:
-            targets (list[str]): Target URL's to obtain data from.
-            timeout (int): Amount of time in seconds before a connection is cancelled if not succeeded.
-            test_proxy (bool): When True, test proxy connections before utilizing them.
-            test_timeout (int): Timeout for testing proxy connections in seconds.
-            zone (str): Zone identifier, e.g., 'us', 'nl', 'de', 'uk', etc etc.
-            debug (bool): Enable debug mode if True.
-            steam (bool): Enable Steam mode if True.
+            targets: list[str]: Target URL's to obtain data from.
+            timeout: int: Amount of time in seconds before a connection is cancelled if not succeeded.
+            scrape: bool: When True, scrape for proxies (Slow). Otherwise fetch them from one source (Fast).
+            filter: bool: When True, test proxy connections before utilizing them.
+            zone: list[str]: List of whitelisted proxy zones. Only use proxies located in the provided array.
+            unpack: Callable: Filter extracted data through an anonymous method.
+            debug: bool: When True, ouput debug logs to terminal.
+
+        Example:
+            ```py
+                proxy_relay = AProxyRelay(
+                    targets=targets,
+                    timeout=5,
+                    scrape=True,
+                    filter=True,
+                    zones=['US', 'DE'],
+                    unpack=lambda data, target: data[target.split('appids=')[1]]['success'],
+                    debug=True,
+                )
+            ```
         """
         # Configure the logger
-        logging.basicConfig(level=logging.INFO if not debug else logging.DEBUG)
-        self.logger = logging.getLogger(__name__)
-
-        # TODO raise exceptions
-        self.timeout = timeout
-        self.test_timeout = test_timeout
-        self.test_proxy = test_proxy
-        self.zone = zone.upper()
-        self.debug = debug
-        self._steam = steam
+        basicConfig(level=INFO if not debug else DEBUG)
+        self.logger = getLogger(__name__)
 
+        # Initialize Core
         AProxyRelayCore.__init__(self)
+
+        # TODO raise exceptions for class arguments
+        self._queue_target_process = Queue(maxsize=len(targets))
         for item in list(set(targets)):
             self._queue_target_process.put(item)
 
+        self.timeout = timeout
+        self.scrape = scrape
+        self.filter = filter
+        self.zones = [z.upper() for z in zones]
+        self.unpack = unpack
+        self.debug = debug
+        self.started = None
+
     async def _main(self) -> Queue:
         """
         Start the scrape task asynchronously. Once finished, you will end up with the data from the API in a Queue.
@@ -82,25 +98,14 @@ def start(self) -> Queue:
         Returns:
             Queue: A queue containing the scraped data from the API.
         """
-        started = datetime.now(UTC)
-        self.logger.info(f'Started proxy relay at {started} ... Please wait ...!')
+        self.started = datetime.now(UTC)
+        self.logger.info(f'Started proxy relay at {self.started} ... Please wait ...!')
 
-        if sys.platform == "win32":
-            loop = asyncio.ProactorEventLoop()
-        else:
-            loop = asyncio.SelectorEventLoop()
+        loop = get_event_loop()
         loop.set_debug(self.debug)
+        results = loop.run_until_complete(gather(self._main()))
+        result = results.pop()
 
-        try:
-            # Create a task and set its name
-            task = loop.create_task(self._main())
-            task.set_name("AProxyRelay")
-
-            loop.run_until_complete(task)
-            self.logger.info(f'Data scraped! Took {datetime.now(UTC) - started}, enjoy!')
-
-            result = task.result()
-        finally:
-            loop.close()
+        self.logger.info(f'Data scraped! Took {datetime.now(UTC) - self.started}, enjoy!')
 
         return result