fix #43 #44 libmagic bugs + linelength

Author: bitdruid

pyproject.toml

@@ -19,8 +19,9 @@ dependencies = [
     "SQLAlchemy==2.0.43",
     "requests==2.32.3",
     "tqdm==4.67.1",
+    "pylibmagic==0.5.0",
     "python-magic==0.4.27; sys_platform == 'linux'",
-    "python-magic-bin==0.4.14; sys_platform == 'win32'",
+    "python-magic-bin==0.4.14; sys_platform == 'win32' or sys_platform == 'darwin'",
 ]
 
 [project.scripts]

pywaybackup/Exception.py

@@ -36,10 +36,7 @@ def exception(cls, message: str, e: Exception, tb=None):
             codeline = linecache.getline(filename, tb_line).strip()
             local_vars = tb_frame.f_locals
             exception_message += (
-                f"!-- File: {filename}\n"
-                f"!-- Function: {func_name}\n"
-                f"!-- Line: {tb_line}\n"
-                f"!-- Segment: {codeline}\n"
+                f"!-- File: {filename}\n!-- Function: {func_name}\n!-- Line: {tb_line}\n!-- Segment: {codeline}\n"
             )
         else:
             exception_message += "!-- Traceback is None\n"
@@ -96,4 +93,4 @@ def exception_handler(exception_type, exception, traceback):
         if issubclass(exception_type, KeyboardInterrupt):
             sys.__excepthook__(exception_type, exception, traceback)
             return
-        Exception.exception('UNCAUGHT EXCEPTION', exception, traceback)  # uncaught exceptions also with custom scheme
+        Exception.exception("UNCAUGHT EXCEPTION", exception, traceback)  # uncaught exceptions also with custom scheme

pywaybackup/PyWayBackup.py

@@ -407,7 +407,11 @@ def paths(self, rel: bool = False) -> dict:
             "log": self._log,
             "debug": self._debug,
         }
-        return {key: (os.path.relpath(path) if rel else path) for key, path in files.items() if path and os.path.exists(path)}
+        return {
+            key: (os.path.relpath(path) if rel else path)
+            for key, path in files.items()
+            if path and os.path.exists(path)
+        }
 
     def status(self) -> dict:
         """

pywaybackup/Snapshot.py

@@ -101,7 +101,9 @@ def modify(self, column, value):
             value: New value to set for the column.
         """
         column = getattr(waybackup_snapshots, column)
-        self._db.session.execute(update(waybackup_snapshots).where(waybackup_snapshots.scid == self.scid).values({column: value}))
+        self._db.session.execute(
+            update(waybackup_snapshots).where(waybackup_snapshots.scid == self.scid).values({column: value})
+        )
         self._db.session.commit()
 
     def create_output(self):

pywaybackup/SnapshotCollection.py

@@ -50,7 +50,9 @@ def _write_summary(self):
 
     def _reset_locked_snapshots(self):
         """Reset locked snapshots to unprocessed in the database."""
-        self.db.session.execute(update(waybackup_snapshots).where(waybackup_snapshots.response == "LOCK").values(response=None))
+        self.db.session.execute(
+            update(waybackup_snapshots).where(waybackup_snapshots.response == "LOCK").values(response=None)
+        )
         self.db.session.commit()
 
     def _finalize_db(self):
@@ -140,11 +142,19 @@ def _insert_batch_safe(line_batch):
                     waybackup_snapshots.url_origin,
                     waybackup_snapshots.url_archive,
                 )
-                .filter(tuple_(waybackup_snapshots.timestamp, waybackup_snapshots.url_origin, waybackup_snapshots.url_archive).in_(keys))
+                .filter(
+                    tuple_(
+                        waybackup_snapshots.timestamp, waybackup_snapshots.url_origin, waybackup_snapshots.url_archive
+                    ).in_(keys)
+                )
                 .all()
             )
             existing_rows = set(existing)
-            new_rows = [row for row in unique_batch if (row["timestamp"], row["url_origin"], row["url_archive"]) not in existing_rows]
+            new_rows = [
+                row
+                for row in unique_batch
+                if (row["timestamp"], row["url_origin"], row["url_archive"]) not in existing_rows
+            ]
             if new_rows:
                 self.db.session.bulk_insert_mappings(waybackup_snapshots, new_rows)
                 self.db.session.commit()
@@ -200,17 +210,25 @@ def _index_snapshots(self):
         # index for filtering last snapshots
         if self._mode_last:
             idx1 = Index(
-                "idx_waybackup_snapshots_url_origin_timestamp_desc", waybackup_snapshots.url_origin, waybackup_snapshots.timestamp.desc()
+                "idx_waybackup_snapshots_url_origin_timestamp_desc",
+                waybackup_snapshots.url_origin,
+                waybackup_snapshots.timestamp.desc(),
             )
             idx1.create(self.db.session.bind, checkfirst=True)
         # index for filtering first snapshots
         if self._mode_first:
             idx2 = Index(
-                "idx_waybackup_snapshots_url_origin_timestamp_asc", waybackup_snapshots.url_origin, waybackup_snapshots.timestamp.asc()
+                "idx_waybackup_snapshots_url_origin_timestamp_asc",
+                waybackup_snapshots.url_origin,
+                waybackup_snapshots.timestamp.asc(),
             )
             idx2.create(self.db.session.bind, checkfirst=True)
         # index for skippable snapshots
-        idx3 = Index("idx_waybackup_snapshots_timestamp_url_origin_response", waybackup_snapshots.timestamp, waybackup_snapshots.url_origin)
+        idx3 = Index(
+            "idx_waybackup_snapshots_timestamp_url_origin_response",
+            waybackup_snapshots.timestamp,
+            waybackup_snapshots.url_origin,
+        )
         idx3.create(self.db.session.bind, checkfirst=True)
 
     def _filter_snapshots(self):
@@ -224,7 +242,9 @@ def _filter_snapshots(self):
         def _filter_mode():
             self._filter_mode = 0
             if self._mode_last or self._mode_first:
-                ordering = waybackup_snapshots.timestamp.desc() if self._mode_last else waybackup_snapshots.timestamp.asc()
+                ordering = (
+                    waybackup_snapshots.timestamp.desc() if self._mode_last else waybackup_snapshots.timestamp.asc()
+                )
                 # assign row numbers per url_origin
                 rownum = (
                     func.row_number()
@@ -266,7 +286,9 @@ def _enumerate_counter():
 
         _filter_mode()
         _enumerate_counter()
-        self._filter_response = self.db.session.query(waybackup_snapshots).where(waybackup_snapshots.response.in_(["404", "301"])).count()
+        self._filter_response = (
+            self.db.session.query(waybackup_snapshots).where(waybackup_snapshots.response.in_(["404", "301"])).count()
+        )
         self.db.session.commit()
 
     def _skip_set(self):
@@ -280,7 +302,12 @@ def _skip_set(self):
             for row in f:
                 self.db.session.execute(
                     update(waybackup_snapshots)
-                    .where(and_(waybackup_snapshots.timestamp == row["timestamp"], waybackup_snapshots.url_origin == row["url_origin"]))
+                    .where(
+                        and_(
+                            waybackup_snapshots.timestamp == row["timestamp"],
+                            waybackup_snapshots.url_origin == row["url_origin"],
+                        )
+                    )
                     .values(
                         url_archive=row["url_archive"],
                         redirect_url=row["redirect_url"],

pywaybackup/Verbosity.py

@@ -68,9 +68,10 @@ def progress(cls, progress: int, maxval: int = None):
                     cls.pbar = Progressbar(
                         unit=" snapshot",
                         desc="download file".ljust(15),
-                        total=maxval, ascii="░▒█",
-                        bar_format="{l_bar}{bar:50}{r_bar}{bar:-10b}"
-                        )
+                        total=maxval,
+                        ascii="░▒█",
+                        bar_format="{l_bar}{bar:50}{r_bar}{bar:-10b}",
+                    )
                 if cls.pbar is not None and progress is not None and progress > 0:
                     cls.pbar.update(progress)
 
@@ -93,15 +94,30 @@ def filter_verbosity(cls, message: list):
 
 
 class Progressbar(Verbosity):
-    def __init__(self, unit: str, desc: str, unit_scale: bool = False, total: int = None, ascii: str = None, bar_format: str = None):
+    def __init__(
+        self,
+        unit: str,
+        desc: str,
+        unit_scale: bool = False,
+        total: int = None,
+        ascii: str = None,
+        bar_format: str = None,
+    ):
         if not super().silent:
             self.unit = unit
             self.desc = desc
             self.unit_scale = unit_scale
             self.total = total
             self.ascii = ascii
             self.bar_format = bar_format
-            self.pbar = tqdm(unit=self.unit, desc=self.desc, unit_scale=self.unit_scale, total=self.total, ascii=self.ascii, bar_format=self.bar_format)
+            self.pbar = tqdm(
+                unit=self.unit,
+                desc=self.desc,
+                unit_scale=self.unit_scale,
+                total=self.total,
+                ascii=self.ascii,
+                bar_format=self.bar_format,
+            )
 
     def update(self, progress: int):
         """

pywaybackup/archive_download.py

@@ -169,7 +169,13 @@ def _download_loop(self, worker: Worker):
                         try:
                             download_status = self._download(worker=worker)
 
-                        except (timeout, ConnectionRefusedError, ConnectionResetError, http.client.HTTPException, Exception) as e:
+                        except (
+                            timeout,
+                            ConnectionRefusedError,
+                            ConnectionResetError,
+                            http.client.HTTPException,
+                            Exception,
+                        ) as e:
                             if isinstance(e, (timeout, ConnectionRefusedError, ConnectionResetError)):
                                 if download_attempt < download_max_attempt:
                                     download_attempt += 1  # try again 2x with same connection
@@ -293,7 +299,10 @@ def _download(self, worker: Worker):
                         try:
                             context.response_data = gzip.decompress(context.response_data)
                         except BadGzipFile:
-                            vb.write(verbose=None, content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}")
+                            vb.write(
+                                verbose=None,
+                                content=f"Worker: {worker.id} - GZIP DECOMPRESS SKIPPED - {context.snapshot_url}",
+                            )
                             pass
                     file.write(context.response_data)
 
@@ -313,7 +322,9 @@ def __handle_redirect(self, context: DownloadContext, worker: Worker) -> None:
             context (DownloadContext): The download context.
             worker (Worker): The worker instance.
         """
-        worker.message.store(verbose=True, result="REDIRECT", content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=True, result="REDIRECT", content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=True, result="", info="FROM", content=context.snapshot_url)
         for _ in range(5):
             self.__download_response(context=context, worker=worker)
@@ -369,7 +380,9 @@ def __dl_result(self, context: DownloadContext, worker: Worker, result: str) ->
         Returns:
             bool: Always True (indicates result was processed).
         """
-        worker.message.store(verbose=True, result=result, content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=True, result=result, content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=False, result=result)
         worker.message.store(verbose=True, result="", info="URL", content=context.snapshot_url)
         worker.message.store(verbose=True, result="", info="FILE", content=context.output_file)
@@ -386,7 +399,9 @@ def __dl_fail(self, context: DownloadContext, worker: Worker) -> bool:
         Returns:
             bool: Always False (indicates failure was processed).
         """
-        worker.message.store(verbose=None, result="UNKNOWN", content=f"{context.response_status} {context.response_status_message}")
+        worker.message.store(
+            verbose=None, result="UNKNOWN", content=f"{context.response_status} {context.response_status_message}"
+        )
         worker.message.store(verbose=True, result="", info="URL", content=context.snapshot_url)
         return False
 

pywaybackup/archive_save.py

@@ -34,18 +34,24 @@ def save_page(url: str):
 
         if response_status == 302:
             location = response.getheader("Location")
-            snapshot_timestamp = datetime.strptime(url_get_timestamp(location), "%Y%m%d%H%M%S").strftime("%Y-%m-%d %H:%M:%S")
+            snapshot_timestamp = datetime.strptime(url_get_timestamp(location), "%Y%m%d%H%M%S").strftime(
+                "%Y-%m-%d %H:%M:%S"
+            )
             current_timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
             timestamp_difference = (
-                datetime.strptime(current_timestamp, "%Y-%m-%d %H:%M:%S") - datetime.strptime(snapshot_timestamp, "%Y-%m-%d %H:%M:%S")
+                datetime.strptime(current_timestamp, "%Y-%m-%d %H:%M:%S")
+                - datetime.strptime(snapshot_timestamp, "%Y-%m-%d %H:%M:%S")
             ).seconds / 60
             timestamp_difference = int(round(timestamp_difference, 0))
 
             if timestamp_difference < 1:
                 vb.write(verbose=None, content="\n-----> Response: 302 (new snapshot)")
                 vb.write(verbose=None, content=f"SNAPSHOT URL: {location}")
             elif timestamp_difference >= 1:
-                vb.write(verbose=None, content=f"\n-----> Response: 302 (existing snapshot - wait for {60 - timestamp_difference} minutes)")
+                vb.write(
+                    verbose=None,
+                    content=f"\n-----> Response: 302 (existing snapshot - wait for {60 - timestamp_difference} minutes)",
+                )
                 vb.write(verbose=None, content=f"SNAPSHOT URL: {location}")
                 vb.write(verbose=None, content=f"WAYBACK TIME: {snapshot_timestamp}")
                 vb.write(verbose=None, content=f"REQUEST TIME: {current_timestamp}")

pywaybackup/db.py

@@ -17,7 +17,7 @@
 )
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
-from typing import Optional # python 3.8
+from typing import Optional  # python 3.8
 
 Base = declarative_base()
 
@@ -113,7 +113,9 @@ def init(cls, dbfile, query_identifier):
         Base.metadata.create_all(engine)
 
         db = Database()
-        if db.session.execute(select(waybackup_job.query_identifier).where(query_identifier == query_identifier)).fetchone():
+        if db.session.execute(
+            select(waybackup_job.query_identifier).where(query_identifier == query_identifier)
+        ).fetchone():
             cls.query_exist = True
             cls.query_progress = db.get_progress()
         else:
@@ -140,7 +142,9 @@ def write_progress(self, done: int, total: int):
         """
         progress = f"{(done):,} / {(total):,}"
         self.session.execute(
-            update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(query_progress=progress)
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(query_progress=progress)
         )
         self.session.commit()
 
@@ -180,19 +184,31 @@ def set_insert_complete(self):
         """
         Mark the job's insertion phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(insert_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(insert_complete=1)
+        )
         self.session.commit()
 
     def set_index_complete(self):
         """
         Mark the job's indexing phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(index_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(index_complete=1)
+        )
         self.session.commit()
 
     def set_filter_complete(self):
         """
         Mark the job's filtering phase as complete in the database.
         """
-        self.session.execute(update(waybackup_job).where(waybackup_job.query_identifier == self.query_identifier).values(filter_complete=1))
+        self.session.execute(
+            update(waybackup_job)
+            .where(waybackup_job.query_identifier == self.query_identifier)
+            .values(filter_complete=1)
+        )
         self.session.commit()

pywaybackup/files.py

@@ -54,7 +54,9 @@ def _build_query(self):
 
         limit = f"&limit={self.limit}" if self.limit else ""
 
-        filter_statuscode = f"&filter=statuscode:({'|'.join(self.filter_statuscode)})$" if self.filter_statuscode else ""
+        filter_statuscode = (
+            f"&filter=statuscode:({'|'.join(self.filter_statuscode)})$" if self.filter_statuscode else ""
+        )
         filter_filetype = f"&filter=original:.*\\.({'|'.join(self.filter_filetype)})$" if self.filter_filetype else ""
 
         return f"https://web.archive.org/cdx/search/cdx?output=json&url={cdx_url}{period}&fl=timestamp,digest,mimetype,statuscode,original{limit}{filter_filetype}{filter_statuscode}"