diff --git a/.github/workflows/staging-check.yml b/.github/workflows/staging-check.yml index 15b7049..bd9c94b 100644 --- a/.github/workflows/staging-check.yml +++ b/.github/workflows/staging-check.yml @@ -27,8 +27,8 @@ jobs: - name: Compare output against staging run: | pip install requests - python tools/compare_out_files.py -b https://autoconfig-stage.thunderbird.net/v1.1/ tmp/ + python tools/compare_out_files.py -l https://autoconfig-stage.thunderbird.net/generated_files.json -b https://autoconfig-stage.thunderbird.net/v1.1 tmp/ - name: Calculate generated_files.json diff with prod run: | - python tools/calculate_generated_files_diff.py -b https://autoconfig.thunderbird.net/v1.1 -t ${{ secrets.GITHUB_TOKEN }} -r ${{ github.repository }} -n ${{ github.event.pull_request.number }} tmp/ + python tools/calculate_generated_files_diff.py -b https://autoconfig.thunderbird.net -t ${{ secrets.GITHUB_TOKEN }} -r ${{ github.repository }} -n ${{ github.event.pull_request.number }} tmp/ diff --git a/.github/workflows/trigger-website-deploy.yml b/.github/workflows/trigger-website-deploy.yml new file mode 100644 index 0000000..7ab44b3 --- /dev/null +++ b/.github/workflows/trigger-website-deploy.yml @@ -0,0 +1,29 @@ +name: Trigger thunderbird-website deploy + +on: + push: + branches: + - master + - prod + +jobs: + trigger: + runs-on: ubuntu-latest + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.WEBSITE_AUTOMATION_APP_ID }} + private-key: ${{ secrets.WEBSITE_AUTOMATION_PRIVATE_KEY }} + owner: thunderbird + repositories: thunderbird-website + + - name: Trigger thunderbird-website deployment + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + EVENT_TYPE=${{ github.ref_name == 'prod' && 'prod' || 'stage' }} + gh api repos/thunderbird/thunderbird-website/dispatches \ + -f "event_type=$EVENT_TYPE" \ + -f "client_payload[message]=triggered by autoconfig ${{ github.ref_name }}@${{ github.sha }}" diff --git a/tools/compare_out_files.py b/tools/compare_out_files.py index 4118c5e..a8b520b 100644 --- a/tools/compare_out_files.py +++ b/tools/compare_out_files.py @@ -13,9 +13,15 @@ GENERATED_FILES_NAME = "generated_files.json" -def get_and_compare(file_name: str, base_url: str, local_folder: str) -> str: - """Reads a local file and compare it with its remote copy before returning - its content. +def compare_file_content(file_name: str, local_folder: str, remote_content: str) -> str: + """Compares the local copy of a file with its remote content. + + Args: + file_name: the name of the file to compare. + local_folder: the local folder in which to find the local copy of the + file. + remote_content: the content of the file as fetched from the remote + server. Returns: The file's content as served by the remote server, decoded as UTF-8 @@ -24,20 +30,13 @@ def get_and_compare(file_name: str, base_url: str, local_folder: str) -> str: Raises: RuntimeError if the local file's content doesn't match the remote copy. """ - resp = requests.get(f"{base_url}/{file_name}") - - # The response might not include an content-type header, and there are some - # non-ASCII characters in our XML files (e.g. in display names), so we need - # to explicitly tell `resp` what its encoding is. - resp.encoding = "utf-8" - with open(os.path.join(local_folder, file_name), "r") as fp: local_list = fp.readlines() deltas = list( difflib.unified_diff( local_list, - resp.text.splitlines(keepends=True), + remote_content.splitlines(keepends=True), fromfile="local", tofile="remote", ) @@ -45,14 +44,49 @@ def get_and_compare(file_name: str, base_url: str, local_folder: str) -> str: if len(deltas) > 0: print(f"Diff deltas:\n\n{"".join(deltas)}", file=sys.stderr) - raise RuntimeError("local file list does not match staging copy") + raise RuntimeError("local file does not match staging copy") + + return remote_content + + +def get_and_compare_config(file_name: str, base_url: str, local_folder: str) -> str: + """Reads a local file and compare it with its remote copy before returning + its content. + + Args: + file_name: the name of the file to fetch and compare. + base_url: URL from which to build the URL to fetch the remote file with. + local_folder: the local folder in which to find the local copy of the + file. + + Returns: + The file's content as served by the remote server, decoded as UTF-8 + text. + + Raises: + RuntimeError if the local file's content doesn't match the remote copy. + """ + resp = requests.get(f"{base_url}/{file_name}") - return resp.text + # The response might not include an content-type header, and there are some + # non-ASCII characters in our XML files (e.g. in display names), so we need + # to explicitly tell `resp` what its encoding is. + resp.encoding = "utf-8" + + return compare_file_content(file_name, local_folder, resp.text) -def get_file_list(base_url: str, local_folder: str) -> List[str]: +def get_file_list(list_url: str, local_folder: str) -> List[str]: """Gets the list of files to compare. + Also checks that the local and remote copies of the list match. + + Args: + list_url: the URL to the remote `generated_files.json` to fetch and + compare. + local_folder: the local folder in which to look for the local copy of + the file. + Returns: The list of file names as per the `generated_files.json` file. @@ -60,13 +94,28 @@ def get_file_list(base_url: str, local_folder: str) -> List[str]: RuntimeError if the local `generated_files.json` file does not match the remote copy. """ - file_list = get_and_compare(GENERATED_FILES_NAME, base_url, local_folder) + resp = requests.get(list_url) + + # The response might not include an content-type header, and there are some + # non-ASCII characters in our XML files (e.g. in display names), so we need + # to explicitly tell `resp` what its encoding is. + resp.encoding = "utf-8" + + # Check if the response matches the local file, and parse it as JSON if so. + file_list = compare_file_content(GENERATED_FILES_NAME, local_folder, resp.text) return json.loads(file_list) def main(): parser = argparse.ArgumentParser() - parser.add_argument("-b", metavar="base_url", help="base URL serving ISPDB files") + parser.add_argument( + "-l", metavar="list_url", help="the URL for the generated_files.json file" + ) + parser.add_argument( + "-b", + metavar="base_url", + help="base URL serving ISPDB config files (can be different from the URL provided with -l)", + ) parser.add_argument( "folder", help="the folder containing the local ISPDB files to compare" ) @@ -79,14 +128,14 @@ def main(): print("Fetching and comparing file list") - listed_files = get_file_list(base_url, args.folder) + listed_files = get_file_list(args.l, args.folder) failed_files: Dict[str, Exception] = {} for file in listed_files: print(f"Fetching and comparing {file}") try: - get_and_compare(file, base_url, args.folder) + get_and_compare_config(file, base_url, args.folder) except Exception as e: print(f"Comparison failed for file {file}: {e}", file=sys.stderr) failed_files[file] = e diff --git a/tools/run_local_server.py b/tools/run_local_server.py new file mode 100755 index 0000000..b6061b4 --- /dev/null +++ b/tools/run_local_server.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +""" +This script serves as a simple test server for the ISPDB. + +NOTE: This should *not* be used as a production server. It lacks any security +considerations and could compromise the hosting system if hosted publicly. This +script is intended for deploying ISPDB for local testing only. + +The benefit of using this script over serving the directory with the built-in +`http.server` module is that this script sets the response `Content-Type` header +appropriately. + +To run this script, execute the following commands: + +``` +$ cd /path/to/autoconfig # assumed to be the location where this repository is checked out. +$ mkdir local_mirror +$ python tools/convert.py -a -d local_mirror ispdb/* # Process the input files to the directory to serve. +$ cd local_mirror +$ python ../tools/run_local_server.py +``` +The local ISPDB instance will be available on `http://localhost:8000/` +""" + +from http.server import BaseHTTPRequestHandler, HTTPServer +import os + +class XMLServerHandler(BaseHTTPRequestHandler): + def do_GET(self): + # Get the requested path and strip the leading '/' + path = self.path[1:] + + if os.path.isfile(path): + # Set the Content-Type header to send with the file. + self.send_response(200) + self.send_header('Content-Type', "application/xml") + self.end_headers() + with open(path, 'rb') as file: + self.wfile.write(file.read()) + else: + self.send_response(404) + self.end_headers() + self.wfile.write(b"File not found") + +def run_server(server_class=HTTPServer, handler_class=XMLServerHandler): + server_address = ('', 8000) + httpd = server_class(server_address, handler_class) + print('Server running on port 8000...') + httpd.serve_forever() + +if __name__ == "__main__": + run_server() +