Port automation changes to prod

.github/workflows/staging-check.yml

@@ -27,8 +27,8 @@ jobs:
       - name: Compare output against staging
         run: |
           pip install requests
-          python tools/compare_out_files.py -b https://autoconfig-stage.thunderbird.net/v1.1/ tmp/
+          python tools/compare_out_files.py -l https://autoconfig-stage.thunderbird.net/generated_files.json -b https://autoconfig-stage.thunderbird.net/v1.1 tmp/
 
       - name: Calculate generated_files.json diff with prod
         run: |
-          python tools/calculate_generated_files_diff.py -b https://autoconfig.thunderbird.net/v1.1 -t ${{ secrets.GITHUB_TOKEN }} -r ${{ github.repository }} -n ${{ github.event.pull_request.number }} tmp/
+          python tools/calculate_generated_files_diff.py -b https://autoconfig.thunderbird.net -t ${{ secrets.GITHUB_TOKEN }} -r ${{ github.repository }} -n ${{ github.event.pull_request.number }} tmp/

.github/workflows/trigger-website-deploy.yml

@@ -0,0 +1,29 @@
+name: Trigger thunderbird-website deploy
+
+on:
+  push:
+    branches:
+      - master
+      - prod
+
+jobs:
+  trigger:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Generate GitHub App token
+        id: app-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ secrets.WEBSITE_AUTOMATION_APP_ID }}
+          private-key: ${{ secrets.WEBSITE_AUTOMATION_PRIVATE_KEY }}
+          owner: thunderbird
+          repositories: thunderbird-website
+
+      - name: Trigger thunderbird-website deployment
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          EVENT_TYPE=${{ github.ref_name == 'prod' && 'prod' || 'stage' }}
+          gh api repos/thunderbird/thunderbird-website/dispatches \
+            -f "event_type=$EVENT_TYPE" \
+            -f "client_payload[message]=triggered by autoconfig ${{ github.ref_name }}@${{ github.sha }}"

tools/compare_out_files.py

@@ -13,9 +13,15 @@
 GENERATED_FILES_NAME = "generated_files.json"
 
 
-def get_and_compare(file_name: str, base_url: str, local_folder: str) -> str:
-    """Reads a local file and compare it with its remote copy before returning
-    its content.
+def compare_file_content(file_name: str, local_folder: str, remote_content: str) -> str:
+    """Compares the local copy of a file with its remote content.
+
+    Args:
+        file_name: the name of the file to compare.
+        local_folder: the local folder in which to find the local copy of the
+            file.
+        remote_content: the content of the file as fetched from the remote
+            server.
 
     Returns:
         The file's content as served by the remote server, decoded as UTF-8
@@ -24,49 +30,92 @@ def get_and_compare(file_name: str, base_url: str, local_folder: str) -> str:
     Raises:
         RuntimeError if the local file's content doesn't match the remote copy.
     """
-    resp = requests.get(f"{base_url}/{file_name}")
-
-    # The response might not include an content-type header, and there are some
-    # non-ASCII characters in our XML files (e.g. in display names), so we need
-    # to explicitly tell `resp` what its encoding is.
-    resp.encoding = "utf-8"
-
     with open(os.path.join(local_folder, file_name), "r") as fp:
         local_list = fp.readlines()
 
         deltas = list(
             difflib.unified_diff(
                 local_list,
-                resp.text.splitlines(keepends=True),
+                remote_content.splitlines(keepends=True),
                 fromfile="local",
                 tofile="remote",
             )
         )
 
         if len(deltas) > 0:
             print(f"Diff deltas:\n\n{"".join(deltas)}", file=sys.stderr)
-            raise RuntimeError("local file list does not match staging copy")
+            raise RuntimeError("local file does not match staging copy")
+
+        return remote_content
+
+
+def get_and_compare_config(file_name: str, base_url: str, local_folder: str) -> str:
+    """Reads a local file and compare it with its remote copy before returning
+    its content.
+
+    Args:
+        file_name: the name of the file to fetch and compare.
+        base_url: URL from which to build the URL to fetch the remote file with.
+        local_folder: the local folder in which to find the local copy of the
+            file.
+
+    Returns:
+        The file's content as served by the remote server, decoded as UTF-8
+        text.
+
+    Raises:
+        RuntimeError if the local file's content doesn't match the remote copy.
+    """
+    resp = requests.get(f"{base_url}/{file_name}")
 
-        return resp.text
+    # The response might not include an content-type header, and there are some
+    # non-ASCII characters in our XML files (e.g. in display names), so we need
+    # to explicitly tell `resp` what its encoding is.
+    resp.encoding = "utf-8"
+
+    return compare_file_content(file_name, local_folder, resp.text)
 
 
-def get_file_list(base_url: str, local_folder: str) -> List[str]:
+def get_file_list(list_url: str, local_folder: str) -> List[str]:
     """Gets the list of files to compare.
 
+    Also checks that the local and remote copies of the list match.
+
+    Args:
+        list_url: the URL to the remote `generated_files.json` to fetch and
+            compare.
+        local_folder: the local folder in which to look for the local copy of
+            the file.
+
     Returns:
         The list of file names as per the `generated_files.json` file.
 
     Raises:
         RuntimeError if the local `generated_files.json` file does not match the
         remote copy.
     """
-    file_list = get_and_compare(GENERATED_FILES_NAME, base_url, local_folder)
+    resp = requests.get(list_url)
+
+    # The response might not include an content-type header, and there are some
+    # non-ASCII characters in our XML files (e.g. in display names), so we need
+    # to explicitly tell `resp` what its encoding is.
+    resp.encoding = "utf-8"
+
+    # Check if the response matches the local file, and parse it as JSON if so.
+    file_list = compare_file_content(GENERATED_FILES_NAME, local_folder, resp.text)
     return json.loads(file_list)
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-b", metavar="base_url", help="base URL serving ISPDB files")
+    parser.add_argument(
+        "-l", metavar="list_url", help="the URL for the generated_files.json file"
+    )
+    parser.add_argument(
+        "-b",
+        metavar="base_url",
+        help="base URL serving ISPDB config files (can be different from the URL provided with -l)",
+    )
     parser.add_argument(
         "folder", help="the folder containing the local ISPDB files to compare"
     )
@@ -79,14 +128,14 @@ def main():
 
     print("Fetching and comparing file list")
 
-    listed_files = get_file_list(base_url, args.folder)
+    listed_files = get_file_list(args.l, args.folder)
 
     failed_files: Dict[str, Exception] = {}
     for file in listed_files:
         print(f"Fetching and comparing {file}")
 
         try:
-            get_and_compare(file, base_url, args.folder)
+            get_and_compare_config(file, base_url, args.folder)
         except Exception as e:
             print(f"Comparison failed for file {file}: {e}", file=sys.stderr)
             failed_files[file] = e

tools/run_local_server.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+"""
+This script serves as a simple test server for the ISPDB.
+
+NOTE: This should *not* be used as a production server. It lacks any security
+considerations and could compromise the hosting system if hosted publicly. This
+script is intended for deploying ISPDB for local testing only.
+
+The benefit of using this script over serving the directory with the built-in
+`http.server` module is that this script sets the response `Content-Type` header
+appropriately.
+
+To run this script, execute the following commands:
+
+```
+$ cd /path/to/autoconfig # assumed to be the location where this repository is checked out.
+$ mkdir local_mirror
+$ python tools/convert.py -a -d local_mirror ispdb/* # Process the input files to the directory to serve.
+$ cd local_mirror
+$ python ../tools/run_local_server.py
+```
+The local ISPDB instance will be available on `http://localhost:8000/`
+"""
+
+from http.server import BaseHTTPRequestHandler, HTTPServer
+import os
+
+class XMLServerHandler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        # Get the requested path and strip the leading '/'
+        path = self.path[1:]
+
+        if os.path.isfile(path):
+            # Set the Content-Type header to send with the file.
+            self.send_response(200)
+            self.send_header('Content-Type', "application/xml")
+            self.end_headers()
+            with open(path, 'rb') as file:
+                self.wfile.write(file.read())
+        else:
+            self.send_response(404)
+            self.end_headers()
+            self.wfile.write(b"File not found")
+
+def run_server(server_class=HTTPServer, handler_class=XMLServerHandler):
+    server_address = ('', 8000)
+    httpd = server_class(server_address, handler_class)
+    print('Server running on port 8000...')
+    httpd.serve_forever()
+
+if __name__ == "__main__":
+    run_server()
+