Fix BadGzipFile #42

Author: bitdruid

pyproject.toml

@@ -7,7 +7,7 @@ packages = ["pywaybackup"]
 
 [project]
 name = "pywaybackup"
-version = "4.1.0"
+version = "4.1.1"
 description = "Query and download archive.org as simple as possible."
 authors = [
     { name = "bitdruid", email = "bitdruid@outlook.com" }

pywaybackup/archive_download.py

@@ -4,6 +4,7 @@
 import threading
 import time
 import urllib.parse
+from gzip import BadGzipFile
 from http import HTTPStatus
 from importlib.metadata import version
 from socket import timeout
@@ -174,16 +175,20 @@ def _download_loop(self, worker: Worker):
                                     download_attempt += 1  # try again 2x with same connection
                                     vb.write(
                                         verbose=True,
-                                        content=f"\n-----> Worker: {worker.id} \
-                                            - Attempt: [{worker.attempt}/{retry_max_attempt}] \
-                                            Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}] \
-                                            - {e.__class__.__name__} - requesting again in 50 seconds...",
+                                        content=(
+                                            f"\n-----> Worker: {worker.id}"
+                                            f" - Attempt: [{worker.attempt}/{retry_max_attempt}]"
+                                            f" Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}]"
+                                            f" - {e.__class__.__name__} - requesting again in 50 seconds..."
+                                        ),
                                     )
                                     vb.write(
                                         verbose=False,
-                                        content=f"Worker: {worker.id} \
-                                            - Snapshot {worker.snapshot.counter}/{self.sc._snapshot_total} \
-                                            - requesting again in 50 seconds...",
+                                        content=(
+                                            f"Worker: {worker.id}"
+                                            f" - Snapshot {worker.snapshot.counter}/{self.sc._snapshot_total}"
+                                            f" - requesting again in 50 seconds..."
+                                        ),
                                     )
                                     time.sleep(50)
                                     continue
@@ -193,26 +198,32 @@ def _download_loop(self, worker: Worker):
                                     download_attempt = download_max_attempt  # try again 1x with new connection
                                     vb.write(
                                         verbose=True,
-                                        content=f"\n-----> Worker: {worker.id} \
-                                            - Attempt: [{worker.attempt}/{retry_max_attempt}] \
-                                            Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}] \
-                                            - {e.__class__.__name__} - renewing connection in 15 seconds...",
+                                        content=(
+                                            f"\n-----> Worker: {worker.id}"
+                                            f" - Attempt: [{worker.attempt}/{retry_max_attempt}]"
+                                            f" Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}]"
+                                            f" - {e.__class__.__name__} - renewing connection in 15 seconds..."
+                                        ),
                                     )
                                     vb.write(
                                         verbose=False,
-                                        content=f"Worker: {worker.id} \
-                                            - Snapshot {worker.snapshot.counter}/{self.sc._snapshot_total} \
-                                            - renewing connection in 15 seconds...",
+                                        content=(
+                                            f"Worker: {worker.id}"
+                                            f" - Snapshot {worker.snapshot.counter}/{self.sc._snapshot_total}"
+                                            f" - renewing connection in 15 seconds..."
+                                        ),
                                     )
                                     time.sleep(15)
                                     worker.refresh_connection()
                                     continue
                             else:
                                 ex.exception(
-                                    message=f"\n-----> Worker: {worker.id} \
-                                        - Attempt: [{worker.attempt}/{retry_max_attempt}] \
-                                        Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}] \
-                                        - EXCEPTION - {e}",
+                                    message=(
+                                        f"\n-----> Worker: {worker.id}"
+                                        f" - Attempt: [{worker.attempt}/{retry_max_attempt}]"
+                                        f" Snapshot ID: [{worker.snapshot.counter}/{self.sc._snapshot_total}]"
+                                        f" - EXCEPTION - {e}"
+                                    ),
                                     e=e,
                                 )
                                 worker.attempt = retry_max_attempt
@@ -279,7 +290,11 @@ def _download(self, worker: Worker):
             if not os.path.isfile(context.output_file):
                 with open(context.output_file, "wb") as file:
                     if context.response.getheader("Content-Encoding") == "gzip":
-                        context.response_data = gzip.decompress(context.response_data)
+                        try:
+                            context.response_data = gzip.decompress(context.response_data)
+                        except BadGzipFile:
+                            vb.write(verbose=None, content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}")
+                            pass
                     file.write(context.response_data)
 
                 # check if file is downloaded

pywaybackup/db.py

@@ -17,6 +17,7 @@
 )
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
+from typing import Optional # python 3.8
 
 Base = declarative_base()
 
@@ -143,31 +144,31 @@ def write_progress(self, done: int, total: int):
         )
         self.session.commit()
 
-    def get_progress(self) -> str | None:
+    def get_progress(self) -> Optional[str]:
         """
         str or None: Progress string (e.g., '5 / 10') or None if not found.
         """
         return self.session.execute(
             select(waybackup_job.query_progress).where(waybackup_job.query_identifier == self.query_identifier)
         ).scalar_one_or_none()
 
-    def get_insert_complete(self) -> int | None:
+    def get_insert_complete(self) -> Optional[int]:
         """
         int or None: 1 if complete, 0 if not, or None if not found.
         """
         return self.session.execute(
             select(waybackup_job.insert_complete).where(waybackup_job.query_identifier == self.query_identifier)
         ).scalar_one_or_none()
 
-    def get_index_complete(self) -> int | None:
+    def get_index_complete(self) -> Optional[int]:
         """
         int or None: 1 if complete, 0 if not, or None if not found.
         """
         return self.session.execute(
             select(waybackup_job.index_complete).where(waybackup_job.query_identifier == self.query_identifier)
         ).scalar_one_or_none()
 
-    def get_filter_complete(self) -> int | None:
+    def get_filter_complete(self) -> Optional[int]:
         """
         int or None: 1 if complete, 0 if not, or None if not found.
         """