feat: add reproducible central buildspec generation (#1115)

This Pull Request adds a new command called gen-build-spec. This command generates a buildspec, which contains the build related information for a PURL that Macaron has analyzed. The output file will be stored within output/<purl_based_path>/macaron.buildspec, where <purl_based_path> being the directory structure according to the input PackageURL.

Signed-off-by: Trong Nhan Mai <trong.nhan.mai@oracle.com>
Signed-off-by: Ben Selwyn-Smith <benselwynsmith@googlemail.com>
Signed-off-by: behnazh-w <behnaz.hassanshahi@oracle.com>
Co-authored-by: Ben Selwyn-Smith <benselwynsmith@googlemail.com>
Co-authored-by: behnazh-w <behnaz.hassanshahi@oracle.com>

Author: 3 people

scripts/release_scripts/run_macaron.sh

@@ -283,7 +283,7 @@ while [[ $# -gt 0 ]]; do
             entrypoint+=("macaron")
             ;;
         # Parsing commands for macaron entrypoint.
-        analyze|dump-defaults|verify-policy)
+        analyze|dump-defaults|verify-policy|gen-build-spec)
             command=$1
             shift
             break
@@ -359,6 +359,19 @@ elif [[ $command == "verify-policy" ]]; then
         esac
         shift
     done
+elif [[ $command == "gen-build-spec" ]]; then
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            -d|--database)
+                gen_build_spec_arg_database="$2"
+                shift
+                ;;
+            *)
+                rest_command+=("$1")
+                ;;
+        esac
+        shift
+    done
 elif [[ $command == "dump-defaults" ]]; then
     while [[ $# -gt 0 ]]; do
         case $1 in
@@ -512,6 +525,28 @@ if [[ -n "${arg_datalog_policy_file:-}" ]]; then
     mount_file "-f/--file" "$datalog_policy_file" "$datalog_policy_file_in_container" "ro,Z"
 fi
 
+# MACARON entrypoint - gen-build-spec command argvs
+# This is for macaron gen-build-spec command.
+# Determine the database path to be mounted into ${MACARON_WORKSPACE}/database/<database_file_name>.
+if [[ -n "${gen_build_spec_arg_database:-}" ]]; then
+    gen_build_spec_database_path="${gen_build_spec_arg_database}"
+    file_name="$(basename "${gen_build_spec_database_path}")"
+    gen_build_spec_database_path_in_container="${MACARON_WORKSPACE}/database/${file_name}"
+
+    argv_command+=("--database" "$gen_build_spec_database_path_in_container")
+    mount_file "-d/--database" "$gen_build_spec_database_path" "$gen_build_spec_database_path_in_container" "rw,Z"
+fi
+
+# Determine that ~/.gradle/gradle.properties exists to be mounted into ${MACARON_WORKSPACE}/gradle.properties
+if [[ -f "$HOME/.gradle/gradle.properties" ]]; then
+    mounts+=("-v" "$HOME/.gradle/gradle.properties":"${MACARON_WORKSPACE}/gradle.properties:ro,Z")
+fi
+
+# Determine that ~/.m2/settings.xml exists to be mounted into ${MACARON_WORKSPACE}/settings.xml
+if [[ -f "$HOME/.m2/settings.xml" ]]; then
+    mounts+=("-v" "$HOME/.m2/settings.xml":"${MACARON_WORKSPACE}/settings.xml:ro,Z")
+fi
+
 # Set up proxy.
 # We respect the host machine's proxy environment variables.
 proxy_var_names=(

src/macaron/__main__.py

@@ -14,6 +14,10 @@
 from packageurl import PackageURL
 
 import macaron
+from macaron.build_spec_generator.build_spec_generator import (
+    BuildSpecFormat,
+    gen_build_spec_for_purl,
+)
 from macaron.config.defaults import create_defaults, load_defaults
 from macaron.config.global_config import global_config
 from macaron.errors import ConfigurationError
@@ -235,6 +239,47 @@ def verify_policy(verify_policy_args: argparse.Namespace) -> int:
     return os.EX_USAGE
 
 
+def gen_build_spec(gen_build_spec_args: argparse.Namespace) -> int:
+    """Generate a build spec containing the build information discovered by Macaron.
+
+    Returns
+    -------
+    int
+        Returns os.EX_OK if successful or the corresponding error code on failure.
+    """
+    if not os.path.isfile(gen_build_spec_args.database):
+        logger.critical("The database file does not exist.")
+        return os.EX_OSFILE
+
+    output_format = gen_build_spec_args.output_format
+
+    try:
+        build_spec_format = BuildSpecFormat(output_format)
+    except ValueError:
+        logger.error("The output format %s is not supported.", output_format)
+        return os.EX_USAGE
+
+    try:
+        purl = PackageURL.from_string(gen_build_spec_args.package_url)
+    except ValueError as error:
+        logger.error("Cannot parse purl %s. Error %s", gen_build_spec_args.package_url, error)
+        return os.EX_USAGE
+
+    logger.info(
+        "Generating %s buildspec for PURL %s from %s.",
+        output_format,
+        purl,
+        gen_build_spec_args.database,
+    )
+
+    return gen_build_spec_for_purl(
+        purl=purl,
+        database_path=gen_build_spec_args.database,
+        build_spec_format=build_spec_format,
+        output_path=global_config.output_path,
+    )
+
+
 def find_source(find_args: argparse.Namespace) -> int:
     """Perform repo and commit finding for a passed PURL, or commit finding for a passed PURL and repo."""
     if repo_finder.find_source(find_args.package_url, find_args.repo_path or None):
@@ -283,6 +328,9 @@ def perform_action(action_args: argparse.Namespace) -> None:
 
             find_source(action_args)
 
+        case "gen-build-spec":
+            sys.exit(gen_build_spec(action_args))
+
         case _:
             logger.error("Macaron does not support command option %s.", action_args.action)
             sys.exit(os.EX_USAGE)
@@ -515,6 +563,30 @@ def main(argv: list[str] | None = None) -> None:
         ),
     )
 
+    # Generate a build spec containing rebuild information for a software component.
+    gen_build_spec_parser = sub_parser.add_parser(name="gen-build-spec")
+
+    gen_build_spec_parser.add_argument(
+        "-purl",
+        "--package-url",
+        required=True,
+        type=str,
+        help=("The PURL string of the software component to generate build spec for."),
+    )
+
+    gen_build_spec_parser.add_argument(
+        "--database",
+        help="Path to the database.",
+        required=True,
+    )
+
+    gen_build_spec_parser.add_argument(
+        "--output-format",
+        type=str,
+        help=('The output format. Can be rc-buildspec (Reproducible-central build spec) (default "rc-buildspec")'),
+        default="rc-buildspec",
+    )
+
     args = main_parser.parse_args(argv)
 
     if not args.action:

src/macaron/build_spec_generator/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.

src/macaron/build_spec_generator/build_command_patcher.py

@@ -0,0 +1,137 @@
+# Copyright (c) 2025 - 2025, Oracle and/or its affiliates. All rights reserved.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/.
+
+"""This module contains the implementation of the build command patching."""
+
+import logging
+from collections.abc import Mapping, Sequence
+
+from macaron.build_spec_generator.cli_command_parser import CLICommand, CLICommandParser, PatchCommandBuildTool
+from macaron.build_spec_generator.cli_command_parser.gradle_cli_parser import (
+    GradleCLICommandParser,
+    GradleOptionPatchValueType,
+)
+from macaron.build_spec_generator.cli_command_parser.maven_cli_parser import (
+    CommandLineParseError,
+    MavenCLICommandParser,
+    MavenOptionPatchValueType,
+    PatchBuildCommandError,
+)
+from macaron.build_spec_generator.cli_command_parser.unparsed_cli_command import UnparsedCLICommand
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+MVN_CLI_PARSER = MavenCLICommandParser()
+GRADLE_CLI_PARSER = GradleCLICommandParser()
+
+PatchValueType = GradleOptionPatchValueType | MavenOptionPatchValueType
+
+
+def _patch_commands(
+    cmds_sequence: Sequence[list[str]],
+    cli_parsers: Sequence[CLICommandParser],
+    patches: Mapping[
+        PatchCommandBuildTool,
+        Mapping[str, PatchValueType | None],
+    ],
+) -> list[CLICommand] | None:
+    """Patch the sequence of build commands, using the provided CLICommandParser instances.
+
+    For each command in `cmds_sequence`, it will be checked against all CLICommandParser instances until there is
+    one that can parse it, then a patch from ``patches`` is applied for this command if provided.
+
+    If a command doesn't have any corresponding ``CLICommandParser`` instance it will be parsed as UnparsedCLICommand,
+    which just holds the original command as a list of string, without any changes.
+    """
+    result: list[CLICommand] = []
+    for cmds in cmds_sequence:
+        effective_cli_parser = None
+        for cli_parser in cli_parsers:
+            if cli_parser.is_build_tool(cmds[0]):
+                effective_cli_parser = cli_parser
+                break
+
+        if not effective_cli_parser:
+            result.append(UnparsedCLICommand(original_cmds=cmds))
+            continue
+
+        try:
+            cli_command = effective_cli_parser.parse(cmds)
+        except CommandLineParseError as error:
+            logger.error(
+                "Failed to patch the cli command %s. Error %s.",
+                " ".join(cmds),
+                error,
+            )
+            return None
+
+        patch = patches.get(effective_cli_parser.build_tool, None)
+        if not patch:
+            result.append(cli_command)
+            continue
+
+        try:
+            new_cli_command = effective_cli_parser.apply_patch(
+                cli_command=cli_command,
+                patch_options=patch,
+            )
+        except PatchBuildCommandError as error:
+            logger.error(
+                "Failed to patch the build command %s. Error %s.",
+                " ".join(cmds),
+                error,
+            )
+            return None
+
+        result.append(new_cli_command)
+
+    return result
+
+
+def patch_commands(
+    cmds_sequence: Sequence[list[str]],
+    patches: Mapping[
+        PatchCommandBuildTool,
+        Mapping[str, PatchValueType | None],
+    ],
+) -> list[list[str]] | None:
+    """Patch a sequence of CLI commands.
+
+    For each command in this command sequence:
+
+    - If the command is not a build command, or it's a tool we do not support, it will be left intact.
+
+    - If the command is a build command we support, it will be patched, if a patch value is provided in ``patches``.
+      If no patch value is provided for a build command, it will be left intact.
+
+    `patches` is a mapping with:
+
+    - **Key**: an instance of the ``BuildTool`` enum
+
+    - **Value**: the patch value provided to ``CLICommandParser.apply_patch``. For more information on the patch value
+      see the concrete implementations of the ``CLICommandParser.apply_patch`` method.
+      For example: :class:`macaron.cli_command_parser.maven_cli_parser.MavenCLICommandParser.apply_patch`,
+      :class:`macaron.cli_command_parser.gradle_cli_parser.GradleCLICommandParser.apply_patch`.
+
+    This means that all commands that match a BuildTool will be applied by the same patch value.
+
+    Returns
+    -------
+    list[list[str]] | None
+        The patched command sequence or None if there is an error. The errors that can happen if any command
+        which we support is invalid in ``cmds_sequence``, or the patch value is valid.
+    """
+    result = []
+    patch_cli_commands = _patch_commands(
+        cmds_sequence=cmds_sequence,
+        cli_parsers=[MVN_CLI_PARSER, GRADLE_CLI_PARSER],
+        patches=patches,
+    )
+
+    if patch_cli_commands is None:
+        return None
+
+    for patch_cmd in patch_cli_commands:
+        result.append(patch_cmd.to_cmds())
+
+    return result