Fix download issue for games with large number of savefiles and improve performance

.github/workflows/python.yml

@@ -31,6 +31,7 @@ jobs:
         setuptools
         pyinstaller
         requests
+        requests_futures
         filelock
 
     - name: Optional dependencies (WebView)

legendary/api/egs.py

@@ -2,11 +2,14 @@
 # coding: utf-8
 
 import urllib.parse
+from urllib3.util import Retry
 
 import requests
 import requests.adapters
 import logging
 
+from requests_futures.sessions import FuturesSession
+
 from requests.auth import HTTPBasicAuth
 
 from legendary.models.exceptions import InvalidCredentialsError
@@ -43,9 +46,22 @@ def __init__(self, lc='en', cc='US', timeout=10.0):
         # increase maximum pool size for multithreaded metadata requests
         self.session.mount('https://', requests.adapters.HTTPAdapter(pool_maxsize=16))
 
+        retries = Retry(
+            total=3,
+            backoff_factor=0.1,
+            status_forcelist=[500, 501, 502, 503, 504],
+            allowed_methods={'GET'}
+        )
+
         self.unauth_session = requests.session()
         self.unauth_session.headers['User-Agent'] = self._user_agent
 
+        self.unauth_session.mount('https://', requests.adapters.HTTPAdapter(pool_connections=16, pool_maxsize=16, max_retries=retries))
+
+        self.unauth_future_session = FuturesSession(session=self.unauth_session,max_workers=16)
+        self.unauth_future_session.headers['User-Agent'] = self._user_agent
+
+
         self._oauth_basic = HTTPBasicAuth(self._user_basic, self._pw_basic)
 
         self.access_token = None
@@ -68,6 +84,7 @@ def update_egs_params(self, egs_params):
             self._store_user_agent = f'EpicGamesLauncher/{version}'
             self.session.headers['User-Agent'] = self._user_agent
             self.unauth_session.headers['User-Agent'] = self._user_agent
+            self.unauth_future_session.headers['User-Agent'] = self._user_agent
         # update label
         if label := egs_params['label']:
             self._label = label

legendary/core.py

@@ -1043,11 +1043,9 @@ def download_saves(self, app_name='', manifest_name='', save_dir='', clean_dir=F
             os.makedirs(save_path)
 
         _save_dir = save_dir
-        savegames = self.egs.get_user_cloud_saves(app_name=app_name)
-        files = savegames['files']
+        manifests = self.egs.get_user_cloud_saves(app_name=app_name,manifests=True)
+        files = manifests["files"]
         for fname, f in files.items():
-            if '.manifest' not in fname:
-                continue
             f_parts = fname.split('/')
 
             if manifest_name and f_parts[4] != manifest_name:
@@ -1088,17 +1086,26 @@ def download_saves(self, app_name='', manifest_name='', save_dir='', clean_dir=F
             m = self.load_manifest(r.content)
 
             # download chunks required for extraction
-            chunks = dict()
-            for chunk in m.chunk_data_list.elements:
-                cpath_p = fname.split('/', 3)[:3]
-                cpath_p.append(chunk.path)
-                cpath = '/'.join(cpath_p)
-                if cpath not in files:
-                    self.log.warning(f'Chunk {cpath} not in file list, save data may be incomplete!')
-                    continue
+            chunkPaths = list(chunk.path for chunk in m.chunk_data_list.elements)
+            chunkSaves = self.egs.get_user_cloud_saves(app_name=app_name, filenames=chunkPaths, manifests=False)
+            chunkFiles = chunkSaves["files"]
+
+            if len(chunkFiles) != len(chunkPaths):
+                self.log.warning(f'Expected {len(chunkPaths)} chunks, but only found {len(chunkFiles)}! Save may be incomplete.')
+                continue
 
-                self.log.debug(f'Downloading chunk "{cpath}"')
-                r = self.egs.unauth_session.get(files[cpath]['readLink'])
+            chunkLinks = [chunkFiles[c]['readLink'] for c in chunkFiles]
+            self.log.info(f'Downloading {len(chunkLinks)} chunks...')
+
+            def log_download(r, *args, **kwargs):
+                self.log.debug(f'Downloaded chunk {'/'.join(urlparse(r.url).path.split('/')[5:])} successfully.')
+
+            # map chunkLinks to self.egs.unauth_future_session.get
+            futures = [self.egs.unauth_future_session.get(link, hooks={'response': log_download}) for link in chunkLinks]
+
+            chunks = dict()
+            for future in futures:
+                r = future.result()
                 if r.status_code != 200:
                     self.log.error(f'Download failed, status code: {r.status_code}')
                     break

requirements.txt

@@ -1,2 +1,3 @@
 requests<3.0
+requests_futures
 filelock

setup.py

@@ -36,6 +36,7 @@
     ),
     install_requires=[
         'requests<3.0',
+        'requests_futures',
         'setuptools',
         'wheel',
         'filelock'