Use config file instead of editing source code

.dockerignore

@@ -7,4 +7,5 @@
 !static/
 !css/
 !metadata.json
+!config-production.toml
 !tailwind.config.js

.github/workflows/binary-nightly.yaml

@@ -2,6 +2,11 @@ name: Binaries (nightly)
 
 on:
   workflow_dispatch:
+    inputs:
+      test:
+        description: "Deploy to test location"
+        type: boolean
+        default: true
   schedule:
     - cron: "0 1 * * *" # Triggers the build at 1:00 UTC time
 
@@ -205,25 +210,22 @@ jobs:
           mv "./$ARCHIVE_DIR/dune" "$HOME/.local/bin"
           rm -rf "./$ARCHIVE_DIR"
 
-      - name: Update config on test
-        if: ${{ github.ref == 'refs/heads/staging' }}
-        run: |
-          sed -i 's#let bucket_dir = .*#let bucket_dir = "/dune/test"#g' ./bin/config.ml
-          sed -i 's#let url = .*#let url = "https://get.dune.build/test"#g' ./bin/config.ml
-          git add --ignore-errors ./bin/config.ml
-          cat ./bin/config.ml
-
       - name: Install Sandworm deps && build
         run: dune build
 
       - name: Move artifacts to scope
         run: mv "/home/runner/artifacts" "."
 
-      - name: Export executables and generate html
-        shell: bash
+      - name: Export executables and generate HTML (test)
+        if: ${{ inputs.test }}
+        run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.binary.outputs.git-commit }}" --config config-test.toml
+
+      - name: Export executables and generate HTML (production)
+        if: ${{ !inputs.test }}
         run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.binary.outputs.git-commit }}"
 
       - name: Commit changes to branch
+        if: ${{ !inputs.test }}
         run: |
           git config --global user.name 'Sandworm'
           git config --global user.email 'ocaml-dune@users.noreply.github.com'

.github/workflows/binary-stable.yaml

@@ -192,13 +192,6 @@ jobs:
         with:
           path: /home/runner/artifacts
 
-      - name: Update config to point to test environment
-        if: ${{ github.ref == 'refs/heads/staging' || inputs.test }}
-        run: |
-          sed -i 's#let bucket_dir = .*#let bucket_dir = "/dune/test"#g' ./bin/config.ml
-          sed -i 's#let url = .*#let url = "https://get.dune.build/test"#g' ./bin/config.ml
-          cat ./bin/config.ml
-
       - name: "Install Sandworm deps & build"
         run: dune build
 
@@ -219,8 +212,12 @@ jobs:
           SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
           SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }}
 
-      - name: Export executables and generate html
-        shell: bash
+      - name: Export executables and generate HTML (test)
+        if: ${{ inputs.test }}
+        run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.check_build.outputs.commit }}" --tag "${{ needs.check_build.outputs.release }}" --config config-test.toml
+
+      - name: Export executables and generate HTML (production)
+        if: ${{ !inputs.test }}
         run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.check_build.outputs.commit }}" --tag "${{ needs.check_build.outputs.release }}"
 
       - name: Commit changes to branch

Dockerfile

@@ -15,6 +15,7 @@ RUN apk update && apk add --update libev gmp git
 WORKDIR /app
 COPY --from=build /home/sandworm/static static
 COPY --from=build /home/sandworm/metadata.json ./metadata.json
+COPY --from=build /home/sandworm/config-production.toml ./config-production.toml
 COPY --from=build /home/sandworm/_build/install/default/bin/sandworm /bin/sandworm
 EXPOSE 80
 CMD ["sandworm", "serve", "--port", "80"]

README.md

@@ -34,11 +34,22 @@ build`.
 
 ### Configure
 
-The configuration is in [bin/config.ml](./bin/config.ml)
-file. When running in the pipeline, the _sandworm_ binary is generated before
-its execution. As a result, the path taken is the root of this repository. If
-you want to run it locally, make sure the _files artifacts_ and `rclone.conf`
-are available in the directory where _sandworm_ binary is executed.
+The configuration for serving the redirects to the artifacts and the web site
+is stored in a TOML configuration file:
+
+  * `[config-production.toml](./config-production.toml)` is the configuration
+    for production runs of the binary distribution.
+  * `[config-test.toml](./config-test.toml)` specifies the configuration for
+    the test environment.
+
+When launching the _sandworm_ binary, it supports the `--config` option to
+override which configuration to use. It defaults to the production config file
+but the file can be edited or new files can be specified according to the
+needs.
+
+The upload of created binaries uses RClone, its configuration is set in the
+`rclone.conf` in the repository, where it specifies which host, username and
+authentication method is used.
 
 The export relies on an SSH key to the server. If you want to run your own
 tests, you need to have a server available by _SSH_ with an _SSH key_. Then,
@@ -53,14 +64,15 @@ key_file = </path/to/your/ssh/private/key>
 ```
 
 If you don't have a `/dune` directory on your server, you might want to change
-the `s3_bucket_ref` variable. It could be:
+the `bucket_dir` variable in the TOML config. For example:
 
-```ocaml
-let s3_bucket_ref = "dune-binary-distribution:/path/to/your/server/dir"
+```toml
+[server]
+bucket_dir = "/home/runner/dune/"
 ```
 
 > [!TIP]
-> For our use case, the _RClone_ configuration works with SFTP but, it is
+> For our use case, the _RClone_ configuration is set up to use SFTP but it is
 > compatible with any _RClone_ provider.
 
 ## Running
@@ -69,8 +81,6 @@ Now your setup is ready, you can execute this list of commands to generate or
 update the files:
 
 ```sh
-$ ls
-artifacts rclone.conf
 $ dune exec -- sandworm sync --commit [commit hash]
 ```
 

bin/config.ml

@@ -1,12 +1,53 @@
-module Server = struct
-  let bucket_dir = "/dune/"
-  let rclone_bucket_ref = Format.sprintf "dune-binary-distribution:%s" bucket_dir
-  let url = "https://get.dune.build/"
+module type FILE = sig
+  val file_name : string
 end
 
-module Path = struct
-  let artifacts_dir = "./artifacts"
-  let metadata = "./metadata.json"
-  let rclone = "./rclone.conf"
-  let install = "./static/install"
+module Production : FILE = struct
+  let file_name = "config-production.toml"
+end
+
+module type Configuration = sig
+  module Server : sig
+    val rclone_bucket_ref : string
+    val url : string
+  end
+
+  module Path : sig
+    val artifacts_dir : string
+    val metadata : string
+    val rclone : string
+    val install : string
+  end
+end
+
+module Make (F : FILE) : Configuration = struct
+  let toml_data =
+    match Toml.Parser.from_filename F.file_name with
+    | `Ok v -> v
+    | `Error (s, _location) -> failwith s
+  ;;
+
+  let get_or_fail section' key' =
+    match
+      Toml.Lenses.(get toml_data (key section' |-- table |-- key key' |-- string))
+    with
+    | Some v -> v
+    | None ->
+      failwith (Printf.sprintf "Config file is missing a value for %s.%s" section' key')
+  ;;
+
+  module Server = struct
+    let section = get_or_fail "server"
+    let bucket_dir = section "bucket_dir"
+    let rclone_bucket_ref = Printf.sprintf "dune-binary-distribution:%s" bucket_dir
+    let url = section "url"
+  end
+
+  module Path = struct
+    let section = get_or_fail "path"
+    let artifacts_dir = section "artifacts_dir"
+    let metadata = section "metadata"
+    let rclone = section "rclone"
+    let install = section "install"
+  end
 end

bin/config.mli

@@ -1,11 +1,21 @@
-module Server : sig
-  val rclone_bucket_ref : string
-  val url : string
+module type FILE = sig
+  val file_name : string
 end
 
-module Path : sig
-  val artifacts_dir : string
-  val metadata : string
-  val rclone : string
-  val install : string
+module Production : FILE
+
+module type Configuration = sig
+  module Server : sig
+    val rclone_bucket_ref : string
+    val url : string
+  end
+
+  module Path : sig
+    val artifacts_dir : string
+    val metadata : string
+    val rclone : string
+    val install : string
+  end
 end
+
+module Make (_ : FILE) : Configuration

bin/dune

@@ -1,4 +1,4 @@
 (executable
  (public_name sandworm)
  (name main)
- (libraries sandworm cmdliner dream dream-encoding css))
+ (libraries sandworm cmdliner dream dream-encoding css toml))

bin/main.ml

@@ -2,9 +2,9 @@ open Sandworm
 open Cmdliner
 
 module Common_args = struct
-  let metadata_file =
-    let doc = "The JSON file containing the metadata." in
-    Arg.(value & opt string Config.Path.metadata & info ~doc [ "metadata" ])
+  let config_file =
+    let doc = "The configuration file." in
+    Arg.(value & opt (some string) None & info ~doc [ "config" ])
   ;;
 
   let commit =
@@ -34,9 +34,9 @@ module Common_args = struct
 end
 
 module Sync = struct
-  let synchronise metadata_file ~commit ~tag dry_run =
+  let synchronise (module Config : Config.Configuration) ~commit ~tag dry_run =
     Format.printf "--> Start synchronisation\n";
-    let bundles = Metadata.import_from_json metadata_file in
+    let bundles = Metadata.import_from_json Config.Path.metadata in
     let daily_bundle =
       Metadata.Bundle.create_daily ~commit ~tag Metadata.Target.defaults
     in
@@ -82,19 +82,27 @@ module Sync = struct
     let bundles = Metadata.insert_unique daily_bundle bundles in
     let () =
       if dry_run
-      then Format.printf "- Export metadata to %s\n" metadata_file
-      else Metadata.export_to_json metadata_file bundles
+      then Format.printf "- Export metadata to %s\n" Config.Path.metadata
+      else Metadata.export_to_json Config.Path.metadata bundles
     in
     Format.printf "--> Completed ✓\n"
   ;;
 
   let term =
     let open Term.Syntax in
-    let+ metadata_file = Common_args.metadata_file
+    let+ config_file = Common_args.config_file
     and+ commit = Common_args.commit
     and+ tag = Common_args.tag
     and+ dry_run = Common_args.dry_run in
-    synchronise metadata_file ~commit ~tag dry_run
+    let config =
+      match config_file with
+      | None -> (module Config.Make (Config.Production) : Config.Configuration)
+      | Some file_name ->
+        (module Config.Make (struct
+             let file_name = file_name
+           end))
+    in
+    synchronise config ~commit ~tag dry_run
   ;;
 
   let info =
@@ -128,10 +136,10 @@ let find_latest_stable bundles =
 ;;
 
 module Http = struct
-  let serve dev metadata_file port =
+  let serve dev (module Config : Config.Configuration) port =
     let title = "Dune Nightly" in
     let base_url = Config.Server.url in
-    let bundles = Metadata.import_from_json metadata_file in
+    let bundles = Metadata.import_from_json Config.Path.metadata in
     let latest_release =
       match find_latest_stable bundles with
       | None -> "<RELEASE>"
@@ -151,9 +159,17 @@ module Http = struct
   let term =
     let open Term.Syntax in
     let+ dev = Common_args.dev
-    and+ metadata_file = Common_args.metadata_file
+    and+ config_file = Common_args.config_file
     and+ port = Common_args.port in
-    serve dev metadata_file port
+    let config =
+      match config_file with
+      | None -> (module Config.Make (Config.Production) : Config.Configuration)
+      | Some file_name ->
+        (module Config.Make (struct
+             let file_name = file_name
+           end))
+    in
+    serve dev config port
   ;;
 
   let info =

config-production.toml

@@ -0,0 +1,9 @@
+[server]
+bucket_dir = "/dune/"
+url = "https://get.dune.build/"
+
+[path]
+artifacts_dir = "./artifacts"
+metadata = "./metadata.json"
+rclone = "./rclone.conf"
+install = "./static/install"

config-test.toml

@@ -0,0 +1,9 @@
+[server]
+bucket_dir = "/dune/test"
+url = "https://get.dune.build/test"
+
+[path]
+artifacts_dir = "./artifacts"
+metadata = "./metadata.json"
+rclone = "./rclone.conf"
+install = "./static/install"

dune-project

@@ -43,6 +43,7 @@
   dream
   dream-encoding
   tailwindcss
+  (toml (>= 7))
   (ocaml-lsp-server :with-dev-setup)
   (ocamlformat
    (and

dune.lock/ISO8601.0.2.6.pkg

@@ -0,0 +1,13 @@
+(version 0.2.6)
+
+(build
+ (all_platforms ((action (run dune build -p %{pkg-self:name} -j %{jobs})))))
+
+(depends
+ (all_platforms
+  (dune base-unix ocaml)))
+
+(source
+ (fetch
+  (url https://github.com/ocaml-community/ISO8601.ml/archive/0.2.6.tar.gz)
+  (checksum md5=a460f01d409d51b7d537429881bfa276)))

dune.lock/lock.dune

@@ -1,6 +1,6 @@
 (lang package 0.1)
 
-(dependency_hash 8288efb50ad1673b812c3efd6f90c201)
+(dependency_hash ecf9ab5eb82405971ce3bf9b90aa5c9d)
 
 (ocaml ocaml-base-compiler)