From 353c5f2f9e4f46871d1ee58d29f1fb341805d107 Mon Sep 17 00:00:00 2001 From: lummax Date: Mon, 21 Jul 2025 21:47:48 +0200 Subject: [PATCH] feat: add `--output_json` to `s3_sync()` This flag is not meant to be set in a `BUILD.bazel` file, but when `bazel run`ning the target to write out a JSON file containing which files where uploaded where and had which sha256 sum. This makes in easy for CI jobs uploading artifacts to access that information without having to dig into the bazel output tree. --- MODULE.bazel | 10 ++++++++++ aws/private/s3_sync.bzl | 11 ++++++++++- aws/private/s3_sync.sh | 22 ++++++++++++++++++++++ examples/release_to_s3/BUILD.bazel | 2 ++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1b434a2..6d02083 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,6 +20,16 @@ bazel_dep(name = "aspect_rules_py", version = "0.7.3", dev_dependency = True) bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) bazel_dep(name = "container_structure_test", version = "1.16.0", dev_dependency = True) +bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") +bazel_lib_toolchains.jq() +bazel_lib_toolchains.coreutils() +use_repo(bazel_lib_toolchains, "coreutils_toolchains", "jq_toolchains") + +register_toolchains( + "@jq_toolchains//:all", + "@coreutils_toolchains//:all", +) + aws = use_extension("//aws:extensions.bzl", "aws") aws.toolchain(aws_cli_version = "2.13.0") use_repo(aws, "aws", "aws_darwin", "aws_linux-aarch64", "aws_linux-x86_64", "aws_toolchains") diff --git a/aws/private/s3_sync.bzl b/aws/private/s3_sync.bzl index f8be8d1..70b14de 100644 --- a/aws/private/s3_sync.bzl +++ b/aws/private/s3_sync.bzl @@ -38,8 +38,11 @@ _ATTRS = { } def _s3_sync_impl(ctx): + coreutils = ctx.toolchains["@aspect_bazel_lib//lib:coreutils_toolchain_type"] + jq = ctx.toolchains["@aspect_bazel_lib//lib:jq_toolchain_type"] + executable = ctx.actions.declare_file("{}/s3_sync.sh".format(ctx.label.name)) - runfiles = [executable] + ctx.files.srcs + runfiles = [executable, coreutils.coreutils_info.bin, jq.jqinfo.bin] + ctx.files.srcs vars = [] if int(bool(ctx.attr.bucket)) + int(bool(ctx.attr.bucket_file)) + int(bool(ctx.attr.destination_uri_file)) != 1: fail("Exactly one of 'bucket', 'bucket_file', or 'destination_uri_file' must be set") @@ -61,6 +64,8 @@ def _s3_sync_impl(ctx): is_executable = True, substitutions = { "$aws": ctx.attr.aws[DefaultInfo].default_runfiles.files.to_list()[0].short_path, + "$coreutils": coreutils.coreutils_info.bin.short_path, + "$jq": jq.jqinfo.bin.short_path, "artifacts=()": "artifacts=({})".format(" ".join([s.short_path for s in ctx.files.srcs])), "# Collect Args": "\n".join(vars), }, @@ -76,4 +81,8 @@ s3_sync = rule( executable = True, attrs = _ATTRS, doc = _DOC, + toolchains = [ + "@aspect_bazel_lib//lib:coreutils_toolchain_type", + "@aspect_bazel_lib//lib:jq_toolchain_type", + ], ) diff --git a/aws/private/s3_sync.sh b/aws/private/s3_sync.sh index 681654d..9d8f174 100644 --- a/aws/private/s3_sync.sh +++ b/aws/private/s3_sync.sh @@ -74,6 +74,7 @@ Options: --bucket_file The path to a file that contains the name of the S3 bucket. --[no]dry_run Toggles whether the utility will run in dry-run mode. Default: false + --output_json Collect SHA256 sums/S3 destination for every file and write as JSON to the specified file. Arguments: The path to a file or directory which will be copied to the S3 bucket. @@ -91,6 +92,11 @@ s3_cp() { else warn "[DRY RUN] Would copy ${src} to ${dst}" fi + + if [[ -n "${output_json_file}" ]]; then + local sha256_sum=$("$coreutils" sha256sum "${src}" | cut -d' ' -f1) + sha256_results+=('{"file":"'"${src}"'","sha256":"'"${sha256_sum}"'","s3_path":"'"${dst}"'"}') + fi } cp_artifact() { @@ -107,10 +113,18 @@ cp_artifact() { fi } +output_json_results() { + local output_file="${1}" + printf '%s\n' "${sha256_results[@]}" | "$jq" -s . > "${output_file}" + msg "JSON output written to ${output_file}" +} + # Collect Args dry_run=false +output_json_file="" artifacts=() +sha256_results=() while (("$#")); do case "${1}" in @@ -137,6 +151,10 @@ while (("$#")); do dry_run="false" shift 1 ;; + "--output_json") + output_json_file="${2}" + shift 2 + ;; "--role") role="${2}" shift 2 @@ -213,6 +231,10 @@ else done fi +if [[ -n "${output_json_file}" ]]; then + output_json_results "${output_json_file}" +fi + # shellcheck disable=SC2236 if [[ ! -z "${role:-}" && "${dry_run}" == "false" ]]; then unset AWS_ACCESS_KEY_ID diff --git a/examples/release_to_s3/BUILD.bazel b/examples/release_to_s3/BUILD.bazel index 7670acb..7d686db 100644 --- a/examples/release_to_s3/BUILD.bazel +++ b/examples/release_to_s3/BUILD.bazel @@ -18,6 +18,8 @@ expand_template( # bazel run //examples/release_to_s3 -- --dry_run # Use a different profile: # bazel run //examples/release_to_s3 -- --profile=prod +# Output a metadata JSON file: +# bazel run //examples/release_to_s3 -- --output_json $PWD/out.json s3_sync( name = "release_to_s3", srcs = ["my_file.txt"],