Merge branch 'r/3.2.0'

Author: bitdruid

.gitignore

@@ -1,4 +1,5 @@
 waybackup_snapshots/
+PKGBUILD
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -162,3 +163,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+.qodo

README.md

@@ -28,11 +28,12 @@ This tool allows you to download content from the Wayback Machine (archive.org).
    ```pip install .```
    - in a virtual env or use `--break-system-package`
 
-## Important notes
+## notes / issues / hints
 
-- Linux recommended: On Windows machines, the path length is limited. This can only be overcome by editing the registry. Files that exceed the path length will not be downloaded.
-- If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
+- Linux recommended: On Windows machines, the path length is limited. Files that exceed the path length will not be downloaded.
 - The tool uses a sqlite database to handle snapshots. The database will only persist while the download is running.
+- If you query an explicit file (e.g. a query-string `?query=this` or `login.html`), the `--explicit`-argument is recommended as a wildcard query may lead to an empty result.
+- Downloading directly into a network share is not recommended. The sqlite locking mechanism may cause issues. If you need to download into a network share, set the `--metadata` argument to a local path.
 
 <br>
 <br>
@@ -85,6 +86,9 @@ Limits the amount of snapshots to query from the CDX server. If an existing CDX
 - **`-o`**, **`--output`**:<br>
 Defaults to `waybackup_snapshots` in the current directory. The folder where downloaded files will be saved.
 
+- **`-m`**, **`--metadata`**<br>
+Change the folder where metadata will be saved (`cdx`/`db`/`csv`/`log`). Especially if you are downloading into a network share, you SHOULD set this to a local path because sqlite locking mechanism may cause issues with network shares.
+
 <!-- - **`--verbosity`** `<level>`:<br>
 Sets verbosity level. Options are `info`and `trace`. Default is `info`. -->
 
@@ -99,13 +103,27 @@ Sets the number of simultaneous download workers. Default is 1, safe range is ab
 
 - **`--no-redirect`**:<br>
 Disables following redirects of snapshots. Useful for preventing timestamp-folder mismatches caused by Archive.org redirects.
-  
+
 - **`--retry`** `<attempts>`:<br>
 Specifies number of retry attempts for failed downloads.
 
 - **`--delay`** `<seconds>`:<br>
 Specifies delay between download requests in seconds. Default is no delay (0).
 
+- **`--verbose`**:<br>
+Increase output verbosity.
+  - verbose:
+  ```
+  -----> Worker: 2 - Attempt: [1/1] Snapshot ID: [23/81]
+  SUCCESS   -> 200 OK
+            -> URL:  https://web.archive.org/web/20240225193302id_/https://example.com/assets/css/custom-styles.css
+            -> FILE: /home/manjaro/Stuff/python-wayback-machine-downloader/waybackup_snapshots/example.com/20240225193302id_/assets/css/custom-styles.css
+  ```
+  - non-verbose:
+  ```
+  55/81 - W:2 - SUCCESS - 20240225193302 - https://example.com/assets/css/custom-styles.css
+  ```
+
 <!-- - **`--convert-links`**:<br>
 If set, all links in the downloaded files will be converted to local links. This is useful for offline browsing. The links are converted to the local path structure. Show output with `--verbosity trace`. -->
 

dev/pip_build.sh

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "3.1.0"
+version = "3.2.0"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }
@@ -16,9 +16,10 @@ license = { file = "LICENSE" }
 readme = "README.md"
 requires-python = ">=3.8"
 dependencies = [
-    "pysqlite3-binary==0.5.4",
-    "requests==2.31.0",
-    "tqdm==4.66.2",
+    "pysqlite3-binary==0.5.4; sys_platform == 'linux'",
+    "pysqlite-binary; sys_platform == 'win32'",
+    "requests==2.32.3",
+    "tqdm==4.67.1",
     "python-magic==0.4.27; sys_platform == 'linux'",
     "python-magic-bin==0.4.14; sys_platform == 'win32'",
 ]

dev/venv_create.sh

@@ -6,15 +6,14 @@
 from importlib.metadata import version
 
 from pywaybackup.helper import url_split, sanitize_filename
-from pywaybackup.Exception import Exception as ex
 
 class Arguments:
-    
+
     def __init__(self):
-        
+
         parser = argparse.ArgumentParser(description='Download from wayback machine (archive.org)')
         parser.add_argument('-v', '--version', action='version', version='%(prog)s ' + version("pywaybackup") + ' by @bitdruid -> https://github.com/bitdruid')
-        
+
         required = parser.add_argument_group('required (one exclusive)')
         required.add_argument('-u', '--url', type=str, metavar="", help='url (with subdir/subdomain) to download')
         exclusive_required = required.add_mutually_exclusive_group(required=True)
@@ -32,16 +31,17 @@ def __init__(self):
         optional.add_argument('--limit', type=int, nargs='?', const=True, metavar='int', help='limit the number of snapshots to download')
 
         behavior = parser.add_argument_group('manipulate behavior')
-        behavior.add_argument('-o', '--output', type=str, metavar="", help='output folder - defaults to current directory')
+        behavior.add_argument('-o', '--output', type=str, metavar="", help='output for all files - defaults to current directory')
+        behavior.add_argument('-m', '--metadata', type=str, metavar="", help='change directory for db/cdx/csv/log files')
         behavior.add_argument('--log', action='store_true', help='save a log file into the output folder')
         behavior.add_argument('--progress', action='store_true', help='show a progress bar')
         behavior.add_argument('--no-redirect', action='store_true', help='do not follow redirects by archive.org')
-        #behavior.add_argument('--verbosity', type=str, default="info", metavar="", help='verbosity level (info, trace)')
         behavior.add_argument('--retry', type=int, default=0, metavar="", help='retry failed downloads (opt tries as int, else infinite)')
         behavior.add_argument('--workers', type=int, default=1, metavar="", help='number of workers (simultaneous downloads)')
         # behavior.add_argument('--convert-links', action='store_true', help='Convert all links in the files to local paths. Requires -c/--current')
         behavior.add_argument('--delay', type=int, default=0, metavar="", help='delay between each download in seconds')
-        
+        behavior.add_argument('--verbose', action='store_true', help='overwritten by progress - gives detailed output')
+
         special = parser.add_argument_group('special')
         special.add_argument('--reset', action='store_true', help='reset the job and ignore existing cdx/db/csv files')
         special.add_argument('--keep', action='store_true', help='keep all files after the job finished')
@@ -75,11 +75,11 @@ def init(cls):
 
         if cls.output is None:
             cls.output = os.path.join(os.getcwd(), "waybackup_snapshots")
+        if cls.metadata is None:
+            cls.metadata = cls.output
         os.makedirs(cls.output, exist_ok=True) if not cls.save else None
-        
-        if cls.log is True:
-            cls.log = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.log")
-        
+        os.makedirs(cls.metadata, exist_ok=True) if not cls.save else None
+                
         if cls.all:
             cls.mode = "all"
         if cls.last:
@@ -91,10 +91,13 @@ def init(cls):
 
         if cls.filetype:
             cls.filetype = [ft.lower().strip() for ft in cls.filetype.split(",")]
-        
-        cls.cdxfile = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.cdx")
-        cls.dbfile = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.db")
-        cls.csvfile = os.path.join(cls.output, f"waybackup_{sanitize_filename(cls.url)}.csv")
+
+        base_path = cls.metadata
+        base_name = f"waybackup_{sanitize_filename(cls.url)}"
+        cls.cdxfile = os.path.join(base_path, f"{base_name}.cdx")
+        cls.dbfile = os.path.join(base_path, f"{base_name}.db")
+        cls.csvfile = os.path.join(base_path, f"{base_name}.csv")
+        cls.log = os.path.join(base_path, f"{base_name}.log") if cls.log else None
 
         if cls.reset:
             os.remove(cls.cdxfile) if os.path.isfile(cls.cdxfile) else None

pyproject.toml

@@ -29,7 +29,7 @@ def define_root_steps(cls, filepath) -> str:
 
 
     @classmethod
-    def links(cls, filepath, status_message=None):
+    def links(cls, filepath, status_content=None):
         """
         Convert all links in a HTML / CSS / JS file to local paths.
         """
@@ -72,7 +72,7 @@ def local_url(original_url, domain, count) -> str:
             if original_url.startswith("//"):
                 external = True
             if external:
-                status_message.trace(status="", type=f"{count}/{len(links)}", message="External url")
+                status_message.trace(status="", info=f"{count}/{len(links)}", content="External url")
                 return original_url
 
             # convert the url to a relative path to the local root (download dir) if it's a valid path, else return the original url
@@ -87,7 +87,7 @@ def local_url(original_url, domain, count) -> str:
                 if original_url.startswith("../"): # if file is already ../ check if it's not too many steps up
                     original_url = f"{cls.define_root_steps(filepath)}{original_url.split('../')[-1].lstrip('/')}"
             else:
-                status_message.trace(status="", type="", message=f"{count}/{len(links)}: URL is not a valid path")
+                status_message.trace(status="", info="", content=f"{count}/{len(links)}: URL is not a valid path")
 
             return original_url
 
@@ -158,24 +158,24 @@ def validate_path(filepath: str) -> bool:
 
         if os.path.isfile(filepath):
             if magic.from_file(filepath, mime=True).split("/")[1] == "javascript":
-                status_message.trace(status="Error", type="", message="JS-file is not supported")
+                status_message.trace(status="Error", info="", content="JS-file is not supported")
                 return
             try:
-                with open(filepath, "r") as file:
+                with open(filepath, "r", encoding="utf-8") as file:
                     domain = config.domain
                     content = file.read()
                     links = extract_urls(content)
-                    status_message.store(message=f"\n-----> Convert: [{len(links)}] links in file")
+                    status_message.store(verbose=True, content=f"\n-----> Convert: [{len(links)}] links in file")
                     count = 1
                     for original_link in links:
-                        status_message.trace(status="ORIG", type=f"{count}/{len(links)}", message=original_link)
+                        status_message.trace(status="ORIG", info=f"{count}/{len(links)}", content=original_link)
                         new_link = local_url(original_link, domain, count)
                         if new_link != original_link:
-                            status_message.trace(status="CONV", type=f"{count}/{len(links)}", message=new_link)
+                            status_message.trace(status="CONV", info=f"{count}/{len(links)}", content=new_link)
                         content = content.replace(original_link, new_link)
                         count += 1
-                    file = open(filepath, "w")
+                    file = open(filepath, "w", encoding="utf-8")
                     file.write(content)
                     file.close()
             except UnicodeDecodeError:
-                status_message.trace(status="Error", type="", message="Could not decode file to convert links")
+                status_message.trace(status="Error", info="", content="Could not decode file to convert links")

pywaybackup/Arguments.py

@@ -15,19 +15,16 @@ class Exception:
 
     @classmethod
     def init(cls, output=None, command=None):
-        sys.excepthook = (
-            cls.exception_handler
-        )  # set custom exception handler (uncaught exceptions)
+        sys.excepthook = cls.exception_handler  # set custom exception handler (uncaught exceptions)
         cls.output = output
         cls.command = command
 
     @classmethod
     def exception(cls, message: str, e: Exception, tb=None):
         custom_tb = sys.exc_info()[-1] if tb is None else tb
-        original_tb = cls.relativate_path(
-            "".join(traceback.format_exception(type(e), e, e.__traceback__))
-        )
+        original_tb = cls.relativate_path("".join(traceback.format_exception(type(e), e, e.__traceback__)))
         exception_message = f"-------------------------\n!-- Exception: {message}\n"
+        local_vars = {}
         if custom_tb is not None:
             while custom_tb.tb_next:  # loop to last traceback frame
                 custom_tb = custom_tb.tb_next
@@ -53,14 +50,14 @@ def exception(cls, message: str, e: Exception, tb=None):
         # print(f"Full traceback:\n{original_tb}")
         if cls.new_debug:  # new run, overwrite file
             cls.new_debug = False
-            f = open(debug_file, "w")
+            f = open(debug_file, "w", encoding="utf-8")
             f.write("-------------------------\n")
             f.write(f"Version: {version('pywaybackup')}\n")
             f.write("-------------------------\n")
             f.write(f"Command: {cls.command}\n")
             f.write("-------------------------\n\n")
         else:  # current run, append to file
-            f = open(debug_file, "a")
+            f = open(debug_file, "a", encoding="utf-8")
         f.write(datetime.now().strftime("%Y-%m-%d %H:%M:%S") + "\n")
         f.write(exception_message + "\n")
         f.write("!-- Local Variables:\n")
@@ -75,16 +72,16 @@ def exception(cls, message: str, e: Exception, tb=None):
         f.close()
 
     @classmethod
-    def relativate_path(cls, input: str) -> str:
+    def relativate_path(cls, input_str: str) -> str:
         try:
             path_pattern = re.compile(r'File "([^"]+)"')
-            if os.path.isfile(input):  # case single path
-                return os.path.relpath(input, os.getcwd())
+            if os.path.isfile(input_str):  # case single path
+                return os.path.relpath(input_str, os.getcwd())
             input_modified = ""
-            input_lines = input.split("\n")
+            input_lines = input_str.split("\n")
             if len(input_lines) == 1:  # case single line
-                return input
-            for line in input.split("\n"):  # case multiple lines
+                return input_str
+            for line in input_str.split("\n"):  # case multiple lines
                 match = path_pattern.search(line)
                 if match:
                     original_path = match.group(1)
@@ -93,13 +90,11 @@ def relativate_path(cls, input: str) -> str:
                 input_modified += line + "\n"
             return input_modified
         except ValueError:
-            return input
+            return input_str
 
     @staticmethod
     def exception_handler(exception_type, exception, traceback):
         if issubclass(exception_type, KeyboardInterrupt):
             sys.__excepthook__(exception_type, exception, traceback)
             return
-        Exception.exception(
-            "UNCAUGHT EXCEPTION", exception, traceback
-        )  # uncaught exceptions also with custom scheme
+        Exception.exception("UNCAUGHT EXCEPTION", exception, traceback)  # uncaught exceptions also with custom scheme