Harmonise the cache directory

Author: tomncooper

.github/workflows/publish.yaml

@@ -40,7 +40,7 @@ jobs:
           cp templates/index.html site_files
 
       - name: Build the Kafka site
-        run: python ipper/main.py kafka output standalone dev/kip_mentions.csv site_files/kafka.html
+        run: python ipper/main.py kafka output standalone cache/mailbox_files/kip_mentions.csv site_files/kafka.html
 
       - name: Build the Flink site
         run: python ipper/main.py flink output cache/flip_wiki_cache.json site_files/flink.html site_files/flips 

ipper/kafka/mailing_list.py

@@ -30,7 +30,7 @@
     "from",
     "vote",
 ]
-CACHE_DIR = "cache"
+CACHE_DIR = "kafka_processed_mailbox_cache"
 CACHE_SUFFIX = ".cache.csv"
 
 

ipper/kafka/main.py

@@ -261,9 +261,10 @@ def run_init_cmd(args: Namespace) -> None:
     setup_wiki_download(args)
     print("Downloading Developer Mailing List Archives")
     args.mailing_list = "dev"
+    args.output_dir = "cache/mailbox_files"
     setup_mail_download(args)
     args.overwrite_cache = True
-    args.directory = "dev"
+    args.directory = "cache/mailbox_files"
     process_mail_archives(args)
 
 
@@ -280,11 +281,11 @@ def run_update_cmd(args: Namespace) -> None:
     args.mailing_list = "dev"
     updated_files: List[Path] = setup_mail_download(args)
     # Reprocess just the newly downloaded mail file
-    cache_dir: Path = Path("dev").joinpath(CACHE_DIR)
+    cache_dir: Path = Path("cache").joinpath(CACHE_DIR)
     process_mbox_files(updated_files, cache_dir, overwrite_cache=True)
     # Overwrite the kip mentions cache by process all the old mbox cache files
     # and the newly overwritten one(s)
-    args.directory = "dev"
+    args.directory = "cache"
     args.overwrite_cache = False
     process_mail_archives(args)
 

ipper/kafka/wiki.py

@@ -151,7 +151,7 @@ def get_kip_information(
     chunk: int = 100,
     update: bool = False,
     overwrite_cache: bool = False,
-    cache_filepath: str = "kip_wiki_cache.json",
+    cache_filepath: str = "cache/kip_wiki_cache.json",
     timeout: int = 30,
 ) -> dict[int, dict[str, Union[int, str]]]:
     """Gets the details of all child pages of the KIP main page that relate
@@ -170,6 +170,7 @@ def get_kip_information(
         if not update:
             return output
     else:
+        cache_file_path.parent.mkdir(parents=True, exist_ok=True)
         output = {}
 
     if cache_file_path.exists() and update:

ipper/main.py

@@ -13,7 +13,7 @@ def setup_top_level_parser() -> ArgumentParser:
         description="Ipper - The Improvement Proposal Enrichment program",
     )
     top_level_parser.set_defaults(func=lambda _: print(top_level_parser.format_help()))
-    
+
     top_level_subparsers = top_level_parser.add_subparsers()
 
     setup_kafka_parser(top_level_subparsers)