diff --git a/.dockerignore b/.dockerignore index 1cf39e1..2401088 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,4 +7,5 @@ !static/ !css/ !metadata.json +!config-production.toml !tailwind.config.js diff --git a/.github/workflows/binary-nightly.yaml b/.github/workflows/binary-nightly.yaml index 1ece6d6..b90b5d0 100644 --- a/.github/workflows/binary-nightly.yaml +++ b/.github/workflows/binary-nightly.yaml @@ -2,6 +2,11 @@ name: Binaries (nightly) on: workflow_dispatch: + inputs: + test: + description: "Deploy to test location" + type: boolean + default: true schedule: - cron: "0 1 * * *" # Triggers the build at 1:00 UTC time @@ -205,25 +210,22 @@ jobs: mv "./$ARCHIVE_DIR/dune" "$HOME/.local/bin" rm -rf "./$ARCHIVE_DIR" - - name: Update config on test - if: ${{ github.ref == 'refs/heads/staging' }} - run: | - sed -i 's#let bucket_dir = .*#let bucket_dir = "/dune/test"#g' ./bin/config.ml - sed -i 's#let url = .*#let url = "https://get.dune.build/test"#g' ./bin/config.ml - git add --ignore-errors ./bin/config.ml - cat ./bin/config.ml - - name: Install Sandworm deps && build run: dune build - name: Move artifacts to scope run: mv "/home/runner/artifacts" "." - - name: Export executables and generate html - shell: bash + - name: Export executables and generate HTML (test) + if: ${{ inputs.test }} + run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.binary.outputs.git-commit }}" --config config-test.toml + + - name: Export executables and generate HTML (production) + if: ${{ !inputs.test }} run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.binary.outputs.git-commit }}" - name: Commit changes to branch + if: ${{ !inputs.test }} run: | git config --global user.name 'Sandworm' git config --global user.email 'ocaml-dune@users.noreply.github.com' diff --git a/.github/workflows/binary-stable.yaml b/.github/workflows/binary-stable.yaml index 591c46a..892a2df 100644 --- a/.github/workflows/binary-stable.yaml +++ b/.github/workflows/binary-stable.yaml @@ -192,13 +192,6 @@ jobs: with: path: /home/runner/artifacts - - name: Update config to point to test environment - if: ${{ github.ref == 'refs/heads/staging' || inputs.test }} - run: | - sed -i 's#let bucket_dir = .*#let bucket_dir = "/dune/test"#g' ./bin/config.ml - sed -i 's#let url = .*#let url = "https://get.dune.build/test"#g' ./bin/config.ml - cat ./bin/config.ml - - name: "Install Sandworm deps & build" run: dune build @@ -219,8 +212,12 @@ jobs: SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} SSH_PUBLIC_KEY: ${{ secrets.SSH_PUBLIC_KEY }} - - name: Export executables and generate html - shell: bash + - name: Export executables and generate HTML (test) + if: ${{ inputs.test }} + run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.check_build.outputs.commit }}" --tag "${{ needs.check_build.outputs.release }}" --config config-test.toml + + - name: Export executables and generate HTML (production) + if: ${{ !inputs.test }} run: ./_build/install/default/bin/sandworm sync --commit "${{ needs.check_build.outputs.commit }}" --tag "${{ needs.check_build.outputs.release }}" - name: Commit changes to branch diff --git a/Dockerfile b/Dockerfile index 6684727..e3486f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apk update && apk add --update libev gmp git WORKDIR /app COPY --from=build /home/sandworm/static static COPY --from=build /home/sandworm/metadata.json ./metadata.json +COPY --from=build /home/sandworm/config-production.toml ./config-production.toml COPY --from=build /home/sandworm/_build/install/default/bin/sandworm /bin/sandworm EXPOSE 80 CMD ["sandworm", "serve", "--port", "80"] diff --git a/README.md b/README.md index e00b114..02dd0bf 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,22 @@ build`. ### Configure -The configuration is in [bin/config.ml](./bin/config.ml) -file. When running in the pipeline, the _sandworm_ binary is generated before -its execution. As a result, the path taken is the root of this repository. If -you want to run it locally, make sure the _files artifacts_ and `rclone.conf` -are available in the directory where _sandworm_ binary is executed. +The configuration for serving the redirects to the artifacts and the web site +is stored in a TOML configuration file: + + * `[config-production.toml](./config-production.toml)` is the configuration + for production runs of the binary distribution. + * `[config-test.toml](./config-test.toml)` specifies the configuration for + the test environment. + +When launching the _sandworm_ binary, it supports the `--config` option to +override which configuration to use. It defaults to the production config file +but the file can be edited or new files can be specified according to the +needs. + +The upload of created binaries uses RClone, its configuration is set in the +`rclone.conf` in the repository, where it specifies which host, username and +authentication method is used. The export relies on an SSH key to the server. If you want to run your own tests, you need to have a server available by _SSH_ with an _SSH key_. Then, @@ -53,14 +64,15 @@ key_file = ``` If you don't have a `/dune` directory on your server, you might want to change -the `s3_bucket_ref` variable. It could be: +the `bucket_dir` variable in the TOML config. For example: -```ocaml -let s3_bucket_ref = "dune-binary-distribution:/path/to/your/server/dir" +```toml +[server] +bucket_dir = "/home/runner/dune/" ``` > [!TIP] -> For our use case, the _RClone_ configuration works with SFTP but, it is +> For our use case, the _RClone_ configuration is set up to use SFTP but it is > compatible with any _RClone_ provider. ## Running @@ -69,8 +81,6 @@ Now your setup is ready, you can execute this list of commands to generate or update the files: ```sh -$ ls -artifacts rclone.conf $ dune exec -- sandworm sync --commit [commit hash] ``` diff --git a/bin/config.ml b/bin/config.ml index e3e5e92..d184409 100644 --- a/bin/config.ml +++ b/bin/config.ml @@ -1,12 +1,53 @@ -module Server = struct - let bucket_dir = "/dune/" - let rclone_bucket_ref = Format.sprintf "dune-binary-distribution:%s" bucket_dir - let url = "https://get.dune.build/" +module type FILE = sig + val file_name : string end -module Path = struct - let artifacts_dir = "./artifacts" - let metadata = "./metadata.json" - let rclone = "./rclone.conf" - let install = "./static/install" +module Production : FILE = struct + let file_name = "config-production.toml" +end + +module type Configuration = sig + module Server : sig + val rclone_bucket_ref : string + val url : string + end + + module Path : sig + val artifacts_dir : string + val metadata : string + val rclone : string + val install : string + end +end + +module Make (F : FILE) : Configuration = struct + let toml_data = + match Toml.Parser.from_filename F.file_name with + | `Ok v -> v + | `Error (s, _location) -> failwith s + ;; + + let get_or_fail section' key' = + match + Toml.Lenses.(get toml_data (key section' |-- table |-- key key' |-- string)) + with + | Some v -> v + | None -> + failwith (Printf.sprintf "Config file is missing a value for %s.%s" section' key') + ;; + + module Server = struct + let section = get_or_fail "server" + let bucket_dir = section "bucket_dir" + let rclone_bucket_ref = Printf.sprintf "dune-binary-distribution:%s" bucket_dir + let url = section "url" + end + + module Path = struct + let section = get_or_fail "path" + let artifacts_dir = section "artifacts_dir" + let metadata = section "metadata" + let rclone = section "rclone" + let install = section "install" + end end diff --git a/bin/config.mli b/bin/config.mli index f86b0ec..269206a 100644 --- a/bin/config.mli +++ b/bin/config.mli @@ -1,11 +1,21 @@ -module Server : sig - val rclone_bucket_ref : string - val url : string +module type FILE = sig + val file_name : string end -module Path : sig - val artifacts_dir : string - val metadata : string - val rclone : string - val install : string +module Production : FILE + +module type Configuration = sig + module Server : sig + val rclone_bucket_ref : string + val url : string + end + + module Path : sig + val artifacts_dir : string + val metadata : string + val rclone : string + val install : string + end end + +module Make (_ : FILE) : Configuration diff --git a/bin/dune b/bin/dune index 0846f8f..85731b3 100644 --- a/bin/dune +++ b/bin/dune @@ -1,4 +1,4 @@ (executable (public_name sandworm) (name main) - (libraries sandworm cmdliner dream dream-encoding css)) + (libraries sandworm cmdliner dream dream-encoding css toml)) diff --git a/bin/main.ml b/bin/main.ml index 6a60406..76ec22e 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -2,9 +2,9 @@ open Sandworm open Cmdliner module Common_args = struct - let metadata_file = - let doc = "The JSON file containing the metadata." in - Arg.(value & opt string Config.Path.metadata & info ~doc [ "metadata" ]) + let config_file = + let doc = "The configuration file." in + Arg.(value & opt (some string) None & info ~doc [ "config" ]) ;; let commit = @@ -34,9 +34,9 @@ module Common_args = struct end module Sync = struct - let synchronise metadata_file ~commit ~tag dry_run = + let synchronise (module Config : Config.Configuration) ~commit ~tag dry_run = Format.printf "--> Start synchronisation\n"; - let bundles = Metadata.import_from_json metadata_file in + let bundles = Metadata.import_from_json Config.Path.metadata in let daily_bundle = Metadata.Bundle.create_daily ~commit ~tag Metadata.Target.defaults in @@ -82,19 +82,27 @@ module Sync = struct let bundles = Metadata.insert_unique daily_bundle bundles in let () = if dry_run - then Format.printf "- Export metadata to %s\n" metadata_file - else Metadata.export_to_json metadata_file bundles + then Format.printf "- Export metadata to %s\n" Config.Path.metadata + else Metadata.export_to_json Config.Path.metadata bundles in Format.printf "--> Completed ✓\n" ;; let term = let open Term.Syntax in - let+ metadata_file = Common_args.metadata_file + let+ config_file = Common_args.config_file and+ commit = Common_args.commit and+ tag = Common_args.tag and+ dry_run = Common_args.dry_run in - synchronise metadata_file ~commit ~tag dry_run + let config = + match config_file with + | None -> (module Config.Make (Config.Production) : Config.Configuration) + | Some file_name -> + (module Config.Make (struct + let file_name = file_name + end)) + in + synchronise config ~commit ~tag dry_run ;; let info = @@ -128,10 +136,10 @@ let find_latest_stable bundles = ;; module Http = struct - let serve dev metadata_file port = + let serve dev (module Config : Config.Configuration) port = let title = "Dune Nightly" in let base_url = Config.Server.url in - let bundles = Metadata.import_from_json metadata_file in + let bundles = Metadata.import_from_json Config.Path.metadata in let latest_release = match find_latest_stable bundles with | None -> "" @@ -151,9 +159,17 @@ module Http = struct let term = let open Term.Syntax in let+ dev = Common_args.dev - and+ metadata_file = Common_args.metadata_file + and+ config_file = Common_args.config_file and+ port = Common_args.port in - serve dev metadata_file port + let config = + match config_file with + | None -> (module Config.Make (Config.Production) : Config.Configuration) + | Some file_name -> + (module Config.Make (struct + let file_name = file_name + end)) + in + serve dev config port ;; let info = diff --git a/config-production.toml b/config-production.toml new file mode 100644 index 0000000..a1d5782 --- /dev/null +++ b/config-production.toml @@ -0,0 +1,9 @@ +[server] +bucket_dir = "/dune/" +url = "https://get.dune.build/" + +[path] +artifacts_dir = "./artifacts" +metadata = "./metadata.json" +rclone = "./rclone.conf" +install = "./static/install" diff --git a/config-test.toml b/config-test.toml new file mode 100644 index 0000000..1b576e1 --- /dev/null +++ b/config-test.toml @@ -0,0 +1,9 @@ +[server] +bucket_dir = "/dune/test" +url = "https://get.dune.build/test" + +[path] +artifacts_dir = "./artifacts" +metadata = "./metadata.json" +rclone = "./rclone.conf" +install = "./static/install" diff --git a/dune-project b/dune-project index 525e236..40b5d84 100644 --- a/dune-project +++ b/dune-project @@ -43,6 +43,7 @@ dream dream-encoding tailwindcss + (toml (>= 7)) (ocaml-lsp-server :with-dev-setup) (ocamlformat (and diff --git a/dune.lock/ISO8601.0.2.6.pkg b/dune.lock/ISO8601.0.2.6.pkg new file mode 100644 index 0000000..3cbbd6e --- /dev/null +++ b/dune.lock/ISO8601.0.2.6.pkg @@ -0,0 +1,13 @@ +(version 0.2.6) + +(build + (all_platforms ((action (run dune build -p %{pkg-self:name} -j %{jobs}))))) + +(depends + (all_platforms + (dune base-unix ocaml))) + +(source + (fetch + (url https://github.com/ocaml-community/ISO8601.ml/archive/0.2.6.tar.gz) + (checksum md5=a460f01d409d51b7d537429881bfa276))) diff --git a/dune.lock/lock.dune b/dune.lock/lock.dune index db068d7..2b2612a 100644 --- a/dune.lock/lock.dune +++ b/dune.lock/lock.dune @@ -1,6 +1,6 @@ (lang package 0.1) -(dependency_hash 8288efb50ad1673b812c3efd6f90c201) +(dependency_hash ecf9ab5eb82405971ce3bf9b90aa5c9d) (ocaml ocaml-base-compiler) diff --git a/dune.lock/toml.7.1.0.pkg b/dune.lock/toml.7.1.0.pkg new file mode 100644 index 0000000..a46f7c9 --- /dev/null +++ b/dune.lock/toml.7.1.0.pkg @@ -0,0 +1,20 @@ +(version 7.1.0) + +(build + (all_platforms + ((action + (withenv + ((= TZ "")) + (progn + (when %{pkg-self:dev} (run dune subst)) + (run dune build -p %{pkg-self:name} -j %{jobs} @install))))))) + +(depends + (all_platforms + (dune ocaml menhir ISO8601))) + +(source + (fetch + (url https://github.com/ocaml-toml/to.ml/archive/7.1.0.tar.gz) + (checksum + sha256=1d4e9c16ed9e24d46dd757ce94adc7fc8b2068eb5ff7cd2a70fce08135a752ef))) diff --git a/sandworm.opam b/sandworm.opam index 3ee9099..d5b1e36 100644 --- a/sandworm.opam +++ b/sandworm.opam @@ -35,6 +35,7 @@ depends: [ "dream" "dream-encoding" "tailwindcss" + "toml" {>= "7"} "ocaml-lsp-server" {with-dev-setup} "ocamlformat" {= "0.27.0" & with-dev-setup} "odoc" {with-doc}