Merge branch 'f/start-end-timestamp-range'

Author: bitdruid

README.md

@@ -34,7 +34,7 @@ This script allows you to download content from the Wayback Machine (archive.org
 ### Arguments
 
 - `-h`, `--help`: Show the help message and exit.
-- `-v`, `--version`: Show the script's version.
+- `-a`, `--about`: Show information about the script and exit.
 
 #### Required Arguments
 
@@ -50,9 +50,15 @@ This script allows you to download content from the Wayback Machine (archive.org
 
 - `-l`, `--list`: Only print the snapshots available within the specified range. Does not download the snapshots.
 - `-e`, `--explicit`: Only download the explicit given url. No wildcard subdomains or paths.
-- `-r RANGE`, `--range RANGE`: Specify the range in years for which to search and download snapshots.
 - `-o OUTPUT`, `--output OUTPUT`: The folder where downloaded files will be saved.
 
+- **Range Selection:**<br>
+Specify the range in years or a specific timestamp either start, end or both. If you specify the `range` argument, the `start` and `end` arguments will be ignored. Format for timestamps: YYYYMMDDhhmmss. You can only give a year or increase specificity by going through the timestamp starting on the left.<br>
+(year 2019, year+month 201901, year+month+day 20190101, year+month+day+hour 2019010112)
+   - `-r RANGE`, `--range RANGE`: Specify the range in years for which to search and download snapshots.
+   - `--start`: Timestamp to start searching.
+   - `--end`: Timestamp to end searching.
+
 #### Additional
 
 - `--verbosity [LEVEL]`: Set the verbosity: json (print json response), progress (show progress bar) or standard (default).

pywaybackup/SnapshotCollection.py

@@ -15,7 +15,7 @@ def __init__(self):
 
     def create_full(self, cdxResult):
         self.CDX_JSON = cdxResult.json()[1:]
-        self.CDX_LIST = [{"id": i, "timestamp": snapshot[0], "url": snapshot[1]} for i, snapshot in enumerate(self.CDX_JSON)]
+        self.CDX_LIST = [{"timestamp": snapshot[0], "url": snapshot[1]} for i, snapshot in enumerate(self.CDX_JSON)]
         self.CDX_LIST = sorted(self.CDX_LIST, key=lambda k: k['timestamp'], reverse=True)
 
     def create_current(self):
@@ -38,7 +38,7 @@ def create_entry(self, cdx_entry: dict, output: str) -> dict:
         else: download_dir = os.path.join(output, domain, timestamp, subdir)
         download_file = os.path.join(download_dir, filename)
         cdx_entry = {
-                "id": cdx_entry["id"],
+                "id": len(self.SNAPSHOT_COLLECTION),
                 "url": download_url, 
                 "file": download_file,
                 "timestamp": timestamp,

pywaybackup/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = "0.6.3"

pywaybackup/archive.py

@@ -83,14 +83,15 @@ def print_list(snapshots):
 
 
 # create filelist
-def query_list(snapshots: sc.SnapshotCollection, url: str, range: int, explicit: bool, mode: str):
+# timestamp format yyyyMMddhhmmss
+def query_list(snapshots: sc.SnapshotCollection, url: str, range: int, start: int, end: int, explicit: bool, mode: str):
     try:
         v.write("\nQuerying snapshots...")
-        if range:
-            range = datetime.now().year - range
-            range = "&from=" + str(range)
-        else:
-            range = ""
+        range = ""
+        if not range:
+            if start: range = range + f"&from={start}"
+            if end: range = range + f"&to={end}"
+        else: range = "&from=" + str(datetime.now().year - range)
         cdx_url = f"*.{url}/*" if not explicit else f"{url}"
         cdxQuery = f"https://web.archive.org/cdx/search/xd?output=json&url={cdx_url}{range}&fl=timestamp,original&filter=!statuscode:200"
         cdxResult = requests.get(cdxQuery)

pywaybackup/main.py

@@ -19,10 +19,12 @@ def main():
     optional = parser.add_argument_group('optional')
     optional.add_argument('-l', '--list', action='store_true', help='Only print snapshots (opt range in y)')
     optional.add_argument('-e', '--explicit', action='store_true', help='Search only for the explicit given url')
-    optional.add_argument('-r', '--range', type=int, help='Range in years to search')
     optional.add_argument('-o', '--output', type=str, help='Output folder')
-    optional.add_argument('-v', '--verbosity', type=str, default="standard", choices=["standard", "progress", "json"], help='Verbosity level')
+    optional.add_argument('-r', '--range', type=int, help='Range in years to search')
+    optional.add_argument('--start', type=int, help='Start timestamp format: YYYYMMDDhhmmss')
+    optional.add_argument('--end', type=int, help='End timestamp format: YYYYMMDDhhmmss')
     special = parser.add_argument_group('special')
+    special.add_argument('--verbosity', type=str, default="standard", choices=["standard", "progress", "json"], help='Verbosity level')
     special.add_argument('--retry', type=int, default=0, metavar="X-TIMES", help='Retry failed downloads (opt tries as int, else infinite)')
     special.add_argument('--worker', type=int, default=1, metavar="AMOUNT", help='Number of worker (simultaneous downloads)')
 
@@ -40,7 +42,7 @@ def main():
     else:
         if args.output is None:
             args.output = os.path.join(os.getcwd(), "waybackup_snapshots")
-        archive.query_list(snapshots, args.url, args.range, args.explicit, mode)
+        archive.query_list(snapshots, args.url, args.range, args.start, args.end, args.explicit, mode)
         if args.list:
             archive.print_list(snapshots)
         else: