Proxy Rotation

config/proxies.template_json

@@ -1,12 +1,6 @@
 {
     "proxies": [
-        {
-            "http": "http://user:password@ipaddress:port",
-            "https": "http://user:password@ipaddress:port"
-        },
-        {
-            "http": "http://user:password@ipaddress:port",
-            "https": "http://user:password@ipaddress:port"
-        }
+        "socks5://username:password@ip:port",
+        "socks5://ip:port"
     ]
 }

stores/amazon_monitoring.py

@@ -28,6 +28,8 @@
 from fake_useragent import UserAgent
 from amazoncaptcha import AmazonCaptcha
 
+from itertools import cycle
+
 from urllib.parse import urlparse
 
 import re
@@ -66,22 +68,6 @@
     policy = asyncio.WindowsSelectorEventLoopPolicy()
     asyncio.set_event_loop_policy(policy)
 
-# PDP_URL = "https://smile.amazon.com/gp/product/"
-# AMAZON_DOMAIN = "www.amazon.com.au"
-# AMAZON_DOMAIN = "www.amazon.com.br"
-# AMAZON_DOMAIN = "www.amazon.ca"
-# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.cn"
-# AMAZON_DOMAIN = "www.amazon.fr"
-# AMAZON_DOMAIN = "www.amazon.de"
-# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.in"
-# AMAZON_DOMAIN = "www.amazon.it"
-# AMAZON_DOMAIN = "www.amazon.co.jp"
-# AMAZON_DOMAIN = "www.amazon.com.mx"
-# AMAZON_DOMAIN = "www.amazon.nl"
-# AMAZON_DOMAIN = "www.amazon.es"
-# AMAZON_DOMAIN = "www.amazon.co.uk"
-# AMAZON_DOMAIN = "www.amazon.com"
-# AMAZON_DOMAIN = "www.amazon.se"
 
 AMAZON_URLS = {
     "BASE_URL": "https://{domain}/",
@@ -95,6 +81,7 @@
 
 PDP_PATH = "/dp/"
 REALTIME_INVENTORY_PATH = "gp/aod/ajax?asin="
+BAD_PROXIES_PATH = "config/bad_proxies.json"
 
 CONFIG_FILE_PATH = "config/amazon_requests_config.json"
 PROXY_FILE_PATH = "config/proxies.json"
@@ -111,6 +98,61 @@
 amazon_config = {}
 
 
+class ItemsHandler:
+    @classmethod
+    def create_items_pool(cls, item_list):
+        cls.items = cycle(item_list)
+
+    @classmethod
+    def pop(cls):
+        return next(cls.items)
+
+class BadProxyCollector:
+    @classmethod
+    def load(cls):
+        while True:
+            cls.last_check = time.time()
+            if os.path.exists(BAD_PROXIES_PATH):
+                try:
+                    with open(BAD_PROXIES_PATH) as f:
+                        cls.collection = json.load(f)
+                        return None
+                except:
+                    log.debug(f"{BAD_PROXIES_PATH} can't be decoded and will be deleted.")
+                    os.remove(BAD_PROXIES_PATH)
+            else:
+                cls.collection = dict()
+                return None
+
+    @classmethod
+    def record(cls, status, connector):
+        url = str(connector.proxy_url)
+
+        if status == 503 and url not in cls.collection:
+            cls.collection.update({url : {"banned" : True}})
+        if status == 200 and url in cls.collection:
+            cls.collection[url]["banned"] = False
+
+            try:
+                unbanned_time = cls.collection[url]["unban_time"]
+                if time.time() - unbanned_time >= 300:
+                    del cls.collection[url]
+            except KeyError:
+                cls.collection[url].update({"unban_time" : time.time()})
+
+    @classmethod
+    def save(cls):
+        if cls.timer() and cls.collection:
+            with open(BAD_PROXIES_PATH, "w") as f:
+                json.dump(cls.collection, f, indent=4)
+
+    @classmethod
+    def timer(cls):
+        if time.time() - cls.last_check >= 60:
+            return True
+        return False
+
+
 class AmazonMonitoringHandler(BaseStoreHandler):
     http_client = False
     http_20_client = False
@@ -123,8 +165,7 @@ def __init__(
         delay: float,
         amazon_config,
         tasks=1,
-        checkshipping=False,
-    ) -> None:
+        checkshipping=False,) -> None:
         log.debug("Initializing AmazonMonitoringHandler")
         super().__init__()
 
@@ -133,62 +174,53 @@ def __init__(
         self.notification_handler = notification_handler
         self.check_shipping = checkshipping
         self.item_list: typing.List[FGItem] = item_list
-        self.stock_checks = 0
         self.start_time = int(time.time())
         self.amazon_config = amazon_config
         ua = UserAgent()
 
         self.proxies = get_proxies(path=PROXY_FILE_PATH)
+        ItemsHandler.create_items_pool(self.item_list)
 
         # Initialize the Session we'll use for stock checking
         log.debug("Initializing Monitoring Sessions")
         self.sessions_list: Optional[List[AmazonMonitor]] = []
-        for idx in range(len(item_list)):
-            connector = None
-            if self.proxies and idx < len(self.proxies):
-                connector = ProxyConnector.from_url(self.proxies[idx]["https"])
-            self.sessions_list.append(
-                AmazonMonitor(
-                    headers=HEADERS,
-                    item=item_list[idx],
-                    amazon_config=self.amazon_config,
-                    connector=connector,
-                    delay=delay,
-                )
-            )
-            self.sessions_list[idx].headers.update({"user-agent": ua.random})
 
-
-# class Offers(NamedTuple):
-#     asin: str
-#     offerlistingid: str
-#     merchantid: str
-#     price: float
-#     timestamp: float
-#     __slots__ = ()
-#
-#     def __str__(self):
-#         return f"ASIN: {self.asin}; offerListingId: {self.offerlistingid}; merchantId: {self.merchantid}; price: {self.price}"
+        if self.proxies:
+            for idx in range(len(self.proxies)):
+                connector = ProxyConnector.from_url(self.proxies[idx])
+                self.sessions_list.append(
+                    AmazonMonitor(
+                        headers=HEADERS,
+                        amazon_config=self.amazon_config,
+                        connector=connector,
+                        delay=delay,
+                        issaver=False,
+                    )
+                )
+                self.sessions_list[idx].headers.update({"user-agent": ua.random})
+            self.sessions_list[-1].issaver = True
 
 
 class AmazonMonitor(aiohttp.ClientSession):
     def __init__(
         self,
-        item: FGItem,
         amazon_config: Dict,
         delay: float,
+        issaver: bool,
         *args,
         **kwargs,
     ):
         super(self.__class__, self).__init__(*args, **kwargs)
-        self.item = item
+        self.item = ItemsHandler.pop()
+        self.check_count = 1
         self.amazon_config = amazon_config
         self.domain = urlparse(self.item.furl.url).netloc
+        self.issaver = issaver
 
         self.delay = delay
-        if item.purchase_delay > 0:
+        if self.item.purchase_delay > 0:
             self.delay = 20
-        self.block_purchase_until = time.time() + item.purchase_delay
+        self.block_purchase_until = time.time() + self.item.purchase_delay
         log.debug("Initializing Monitoring Task")
 
     def assign_config(self, azn_config):
@@ -197,18 +229,18 @@ def assign_config(self, azn_config):
     def assign_delay(self, delay: float = 5):
         self.delay = delay
 
-    def assign_item(self, item: FGItem):
-        self.item = item
+    def next_item(self):
+        self.item = ItemsHandler.pop()
 
     def fail_recreate(self):
         # Something wrong, start a new task then kill this one
         log.debug("Max consecutive request fails reached. Restarting session")
         session = AmazonMonitor(
-            item=self.item,
             amazon_config=self.amazon_config,
             delay=self.delay,
             connector=self.connector,
             headers=HEADERS,
+            issaver=self.issaver,
         )
         session.headers.update({"user-agent": UserAgent().random})
         log.debug("Sesssion Created")
@@ -218,14 +250,18 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
         # Do first response outside of while loop, so we can continue on captcha checks
         # and return to start of while loop with that response. Requires the next response
         # to be grabbed at end of while loop
-        log.debug(f"Monitoring Task Started for {self.item.id}")
+
+        # log.debug(f"Monitoring Task Started for {self.item.id}")
+        if self.issaver:
+            BadProxyCollector.load()
 
         fail_counter = 0  # Count sequential get fails
         delay = self.delay
         end_time = time.time() + delay
         status, response_text = await self.aio_get(url=self.item.furl.url)
 
-        save_html_response("stock-check", status, response_text)
+        # save_html_response("stock-check", status, response_text)
+        BadProxyCollector.record(status, self.connector)
 
         # do this after each request
         fail_counter = check_fail(status=status, fail_counter=fail_counter)
@@ -234,10 +270,10 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
             future.set_result(session)
             return
 
-        check_count = 1
+
         # Loop will only exit if a qualified seller is returned.
         while True:
-            log.debug(f"{self.item.id} Stock Check Count: {check_count}")
+            log.debug(f"{self.item.id} : {self.connector.proxy_url} : Stock Check Count = {self.check_count}")
             tree = check_response(response_text)
             if tree is not None:
                 if captcha_element := has_captcha(tree):
@@ -287,15 +323,19 @@ async def stock_check(self, queue: asyncio.Queue, future: asyncio.Future):
             await wait_timer(end_time)
             end_time = time.time() + delay
             status, response_text = await self.aio_get(url=self.item.furl.url)
-            save_html_response("stock-check", status, response_text)
+            # save_html_response("stock-check", status, response_text)
+            BadProxyCollector.record(status, self.connector)
             # do this after each request
             fail_counter = check_fail(status=status, fail_counter=fail_counter)
             if fail_counter == -1:
                 session = self.fail_recreate()
                 future.set_result(session)
                 return
 
-            check_count += 1
+            self.check_count += 1
+            self.next_item()
+            if self.issaver:
+                BadProxyCollector.save()
 
     async def aio_get(self, url):
         text = None
@@ -526,6 +566,7 @@ def get_proxies(path=PROXY_FILE_PATH):
 
     return proxies
 
+
     # def verify(self):
     #     log.debug("Verifying item list...")
     #     items_to_purge = []
@@ -644,3 +685,34 @@ def get_proxies(path=PROXY_FILE_PATH):
     #     pickle.dump(item_cache, open(item_cache_file, "wb"))
     #
     #     return True
+
+
+# class Offers(NamedTuple):
+#     asin: str
+#     offerlistingid: str
+#     merchantid: str
+#     price: float
+#     timestamp: float
+#     __slots__ = ()
+#
+#     def __str__(self):
+#         return f"ASIN: {self.asin}; offerListingId: {self.offerlistingid}; merchantId: {self.merchantid}; price: {self.price}"
+
+
+# PDP_URL = "https://smile.amazon.com/gp/product/"
+# AMAZON_DOMAIN = "www.amazon.com.au"
+# AMAZON_DOMAIN = "www.amazon.com.br"
+# AMAZON_DOMAIN = "www.amazon.ca"
+# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.cn"
+# AMAZON_DOMAIN = "www.amazon.fr"
+# AMAZON_DOMAIN = "www.amazon.de"
+# NOT SUPPORTED AMAZON_DOMAIN = "www.amazon.in"
+# AMAZON_DOMAIN = "www.amazon.it"
+# AMAZON_DOMAIN = "www.amazon.co.jp"
+# AMAZON_DOMAIN = "www.amazon.com.mx"
+# AMAZON_DOMAIN = "www.amazon.nl"
+# AMAZON_DOMAIN = "www.amazon.es"
+# AMAZON_DOMAIN = "www.amazon.co.uk"
+# AMAZON_DOMAIN = "www.amazon.com"
+# AMAZON_DOMAIN = "www.amazon.se"
+