feat: add --output_json to s3_sync() (#105)

This flag is not meant to be set in a `BUILD.bazel` file, but
when `bazel run`ning the target to write out a JSON file containing
which files where uploaded where and had which sha256 sum.

This makes in easy for CI jobs uploading artifacts to access that
information without having to dig into the bazel output tree.

Author: lukasoyen

MODULE.bazel

@@ -20,6 +20,16 @@ bazel_dep(name = "aspect_rules_py", version = "0.7.3", dev_dependency = True)
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 bazel_dep(name = "container_structure_test", version = "1.16.0", dev_dependency = True)
 
+bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains")
+bazel_lib_toolchains.jq()
+bazel_lib_toolchains.coreutils()
+use_repo(bazel_lib_toolchains, "coreutils_toolchains", "jq_toolchains")
+
+register_toolchains(
+    "@jq_toolchains//:all",
+    "@coreutils_toolchains//:all",
+)
+
 aws = use_extension("//aws:extensions.bzl", "aws")
 aws.toolchain(aws_cli_version = "2.13.0")
 use_repo(aws, "aws", "aws_darwin", "aws_linux-aarch64", "aws_linux-x86_64", "aws_toolchains")

aws/private/s3_sync.bzl

@@ -38,8 +38,11 @@ _ATTRS = {
 }
 
 def _s3_sync_impl(ctx):
+    coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"]
+    jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"]
+
     executable = ctx.actions.declare_file("{}/s3_sync.sh".format(ctx.label.name))
-    runfiles = [executable] + ctx.files.srcs
+    runfiles = [executable, coreutils.coreutils_info.bin, jq.jqinfo.bin] + ctx.files.srcs
     vars = []
     if int(bool(ctx.attr.bucket)) + int(bool(ctx.attr.bucket_file)) + int(bool(ctx.attr.destination_uri_file)) != 1:
         fail("Exactly one of 'bucket', 'bucket_file', or 'destination_uri_file' must be set")
@@ -61,6 +64,8 @@ def _s3_sync_impl(ctx):
         is_executable = True,
         substitutions = {
             "$aws": ctx.attr.aws[DefaultInfo].default_runfiles.files.to_list()[0].short_path,
+            "$coreutils": coreutils.coreutils_info.bin.short_path,
+            "$jq": jq.jqinfo.bin.short_path,
             "artifacts=()": "artifacts=({})".format(" ".join([s.short_path for s in ctx.files.srcs])),
             "# Collect Args": "\n".join(vars),
         },
@@ -76,4 +81,8 @@ s3_sync = rule(
     executable = True,
     attrs = _ATTRS,
     doc = _DOC,
+    toolchains = [
+        "@aspect_bazel_lib//lib:coreutils_toolchain_type",
+        "@aspect_bazel_lib//lib:jq_toolchain_type",
+    ],
 )

aws/private/s3_sync.sh

@@ -74,6 +74,7 @@ Options:
   --bucket_file <file>    The path to a file that contains the name of the S3 bucket.
   --[no]dry_run           Toggles whether the utility will run in dry-run mode.
                           Default: false
+  --output_json <file>    Collect SHA256 sums/S3 destination for every file and write as JSON to the specified file.
 
 Arguments:
   <artifact>              The path to a file or directory which will be copied to the S3 bucket.
@@ -91,6 +92,11 @@ s3_cp() {
     else
         warn "[DRY RUN] Would copy ${src} to ${dst}"
     fi
+
+    if [[ -n "${output_json_file}" ]]; then
+        local sha256_sum=$("$coreutils" sha256sum "${src}" | cut -d' ' -f1)
+        sha256_results+=('{"file":"'"${src}"'","sha256":"'"${sha256_sum}"'","s3_path":"'"${dst}"'"}')
+    fi
 }
 
 cp_artifact() {
@@ -107,10 +113,18 @@ cp_artifact() {
     fi
 }
 
+output_json_results() {
+    local output_file="${1}"
+    printf '%s\n' "${sha256_results[@]}" | "$jq" -s . > "${output_file}"
+    msg "JSON output written to ${output_file}"
+}
+
 # Collect Args
 
 dry_run=false
+output_json_file=""
 artifacts=()
+sha256_results=()
 
 while (("$#")); do
     case "${1}" in
@@ -137,6 +151,10 @@ while (("$#")); do
         dry_run="false"
         shift 1
         ;;
+    "--output_json")
+        output_json_file="${2}"
+        shift 2
+        ;;
     "--role")
         role="${2}"
         shift 2
@@ -213,6 +231,10 @@ else
     done
 fi
 
+if [[ -n "${output_json_file}" ]]; then
+    output_json_results "${output_json_file}"
+fi
+
 # shellcheck disable=SC2236
 if [[ ! -z "${role:-}" && "${dry_run}" == "false" ]]; then
     unset AWS_ACCESS_KEY_ID

examples/release_to_s3/BUILD.bazel

@@ -18,6 +18,8 @@ expand_template(
 #  bazel run //examples/release_to_s3 -- --dry_run
 # Use a different profile:
 #  bazel run //examples/release_to_s3 -- --profile=prod
+# Output a metadata JSON file:
+#  bazel run //examples/release_to_s3 -- --output_json $PWD/out.json
 s3_sync(
     name = "release_to_s3",
     srcs = ["my_file.txt"],