chore: improve the pypi spec for default case

Signed-off-by: behnazh-w <behnaz.hassanshahi@oracle.com>

Author: behnazh-w

src/macaron/build_spec_generator/common_spec/base_spec.py

@@ -73,10 +73,13 @@ class BaseBuildSpecDict(TypedDict, total=False):
     #: Entry point script, class, or binary for running the project.
     entry_point: NotRequired[str | None]
 
+    #: The build_requires is the required packages that need to be available in the build environment.
+    build_requires: NotRequired[dict[str, str]]
+
     #: A "back end" is tool that a "front end" (such as pip/build) would call to
     #: package the source distribution into the wheel format. build_backends would
     #: be a list of these that were used in building the wheel alongside their version.
-    build_backends: NotRequired[dict[str, str]]
+    build_backends: NotRequired[list[str]]
 
 
 class BaseBuildSpec(ABC):

src/macaron/build_spec_generator/common_spec/pypi_spec.py

@@ -10,6 +10,7 @@
 import tomli
 from packageurl import PackageURL
 from packaging.requirements import InvalidRequirement, Requirement
+from packaging.specifiers import InvalidSpecifier
 from packaging.utils import InvalidWheelFilename, parse_wheel_filename
 
 from macaron.build_spec_generator.build_command_patcher import CLI_COMMAND_PATCHES, patch_commands
@@ -110,14 +111,15 @@ def resolve_fields(self, purl: PackageURL) -> None:
 
         pypi_package_json = pypi_registry.find_or_create_pypi_asset(purl.name, purl.version, registry_info)
         patched_build_commands: list[list[str]] = []
+        build_requires_set: set[str] = set()
+        build_backends_set: set[str] = set()
+        parsed_build_requires: dict[str, str] = {}
+        python_version_set: set[str] = set()
+        wheel_name_python_version_list: list[str] = []
+        wheel_name_platforms: set[str] = set()
 
         if pypi_package_json is not None:
             if pypi_package_json.package_json or pypi_package_json.download(dest=""):
-                requires_array: list[str] = []
-                build_backends: dict[str, str] = {}
-                python_version_set: set[str] = set()
-                wheel_name_python_version_list: list[str] = []
-                wheel_name_platforms: set[str] = set()
 
                 # Get the Python constraints from the PyPI JSON response.
                 json_releases = pypi_package_json.get_releases()
@@ -135,59 +137,62 @@ def resolve_fields(self, purl: PackageURL) -> None:
                         wheel_contents, metadata_contents = self.read_directory(pypi_package_json.wheel_path, purl)
                         generator, version = self.read_generator_line(wheel_contents)
                         if generator != "":
-                            build_backends[generator] = "==" + version
-                        if generator != "setuptools":
-                            # Apply METADATA heuristics to determine setuptools version.
-                            if "License-File" in metadata_contents:
-                                build_backends["setuptools"] = "==" + defaults.get(
-                                    "heuristic.pypi", "setuptools_version_emitting_license"
-                                )
-                            elif "Platform: UNKNOWN" in metadata_contents:
-                                build_backends["setuptools"] = "==" + defaults.get(
-                                    "heuristic.pypi", "setuptools_version_emitting_platform_unknown"
-                                )
-                            else:
-                                build_backends["setuptools"] = "==" + defaults.get(
-                                    "heuristic.pypi", "default_setuptools"
-                                )
+                            parsed_build_requires[generator] = "==" + version.replace(" ", "")
+                        # Apply METADATA heuristics to determine setuptools version.
+                        elif "License-File" in metadata_contents:
+                            parsed_build_requires["setuptools"] = "==" + defaults.get(
+                                "heuristic.pypi", "setuptools_version_emitting_license"
+                            )
+                        elif "Platform: UNKNOWN" in metadata_contents:
+                            parsed_build_requires["setuptools"] = "==" + defaults.get(
+                                "heuristic.pypi", "setuptools_version_emitting_platform_unknown"
+                            )
                 except SourceCodeError:
                     logger.debug("Could not find pure wheel matching this PURL")
 
                 logger.debug("From .dist_info:")
-                logger.debug(build_backends)
+                logger.debug(parsed_build_requires)
 
                 try:
                     with pypi_package_json.sourcecode():
                         try:
                             pyproject_content = pypi_package_json.get_sourcecode_file_contents("pyproject.toml")
                             content = tomli.loads(pyproject_content.decode("utf-8"))
-                            build_system: dict[str, list[str]] = content.get("build-system", {})
-                            requires_array = build_system.get("requires", [])
+                            requires = json_extract(content, ["build-system", "requires"], list)
+                            if requires:
+                                build_requires_set.update(elem.replace(" ", "") for elem in requires)
+                            backend = json_extract(content, ["build-system", "build-backend"], str)
+                            if backend:
+                                build_backends_set.add(backend.replace(" ", ""))
 
                             python_version_constraint = json_extract(content, ["project", "requires-python"], str)
                             if python_version_constraint:
                                 python_version_set.add(python_version_constraint.replace(" ", ""))
-                            logger.debug("From pyproject.toml:")
-                            logger.debug(requires_array)
-                        except SourceCodeError:
-                            logger.debug("No pyproject.toml found")
-                except SourceCodeError:
-                    logger.debug("No source distribution found")
-
-                # Merge in pyproject.toml information only when the wheel dist_info does not contain the same
+                            logger.debug(
+                                "After analyzing pyproject.toml from the sdist: build-requires: %s, build_backend: %s",
+                                build_requires_set,
+                                build_backends_set,
+                            )
+                        except TypeError as error:
+                            logger.debug(
+                                "Found a type error while reading the pyproject.toml file from the sdist: %s", error
+                            )
+                        except tomli.TOMLDecodeError as error:
+                            logger.debug("Failed to read the pyproject.toml file from the sdist: %s", error)
+                        except SourceCodeError as error:
+                            logger.debug("No pyproject.toml found: %s", error)
+                except SourceCodeError as error:
+                    logger.debug("No source distribution found: %s", error)
+
+                # Merge in pyproject.toml information only when the wheel dist_info does not contain the same.
                 # Hatch is an interesting example of this merge being required.
-                for requirement in requires_array:
+                for requirement in build_requires_set:
                     try:
                         parsed_requirement = Requirement(requirement)
-                        if parsed_requirement.name not in build_backends:
-                            build_backends[parsed_requirement.name] = str(parsed_requirement.specifier)
-                    except InvalidRequirement:
-                        logger.debug("Malformed requirement encountered:")
-                        logger.debug(requirement)
-
-                logger.debug("Combined:")
-                logger.debug(build_backends)
-                self.data["build_backends"] = build_backends
+                        if parsed_requirement.name not in parsed_build_requires:
+                            parsed_build_requires[parsed_requirement.name] = str(parsed_requirement.specifier)
+                    except (InvalidRequirement, InvalidSpecifier) as error:
+                        logger.debug("Malformed requirement encountered %s : %s", requirement, error)
 
                 try:
                     # Get information from the wheel file name.
@@ -206,23 +211,33 @@ def resolve_fields(self, purl: PackageURL) -> None:
                 if "any" in wheel_name_platforms:
                     patched_build_commands = self.get_default_build_commands(self.data["build_tools"])
 
-            if not patched_build_commands:
-                # Resolve and patch build commands.
-                selected_build_commands = self.data["build_commands"] or self.get_default_build_commands(
-                    self.data["build_tools"]
-                )
+        # If we were not able to find any build  and backends, use the default setuptools.
+        if not parsed_build_requires:
+            parsed_build_requires["setuptools"] = "==" + defaults.get("heuristic.pypi", "default_setuptools")
+        if not build_backends_set:
+            build_backends_set.add("setuptools.build_meta")
 
-                patched_build_commands = (
-                    patch_commands(
-                        cmds_sequence=selected_build_commands,
-                        patches=CLI_COMMAND_PATCHES,
-                    )
-                    or []
+        logger.debug("Combined build-requires: %s", parsed_build_requires)
+        self.data["build_requires"] = parsed_build_requires
+        self.data["build_backends"] = list(build_backends_set)
+
+        if not patched_build_commands:
+            # Resolve and patch build commands.
+            selected_build_commands = self.data["build_commands"] or self.get_default_build_commands(
+                self.data["build_tools"]
+            )
+
+            patched_build_commands = (
+                patch_commands(
+                    cmds_sequence=selected_build_commands,
+                    patches=CLI_COMMAND_PATCHES,
                 )
-                if not patched_build_commands:
-                    raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.")
+                or []
+            )
+            if not patched_build_commands:
+                raise GenerateBuildSpecError(f"Failed to patch command sequences {selected_build_commands}.")
 
-            self.data["build_commands"] = patched_build_commands
+        self.data["build_commands"] = patched_build_commands
 
     def read_directory(self, wheel_path: str, purl: PackageURL) -> tuple[str, str]:
         """
@@ -286,5 +301,6 @@ def read_generator_line(self, wheel_contents: str) -> tuple[str, str]:
         for line in wheel_contents.splitlines():
             if line.startswith("Generator:"):
                 split_line = line.split(" ")
-                return split_line[1], split_line[2]
+                if len(split_line) > 2:
+                    return split_line[1], split_line[2]
         return "", ""

src/macaron/slsa_analyzer/build_tool/pyproject.py

@@ -6,10 +6,37 @@
 import logging
 import tomllib
 from pathlib import Path
+from typing import Any
+
+from tomli import TOMLDecodeError
+
+from macaron.json_tools import json_extract
 
 logger: logging.Logger = logging.getLogger(__name__)
 
 
+def get_content(pyproject_path: Path) -> dict[str, Any] | None:
+    """
+    Return the pyproject.toml content.
+
+    Parameters
+    ----------
+    pyproject_path : Path
+        The file path to the pyproject.toml file.
+
+    Returns
+    -------
+    dict[str, Any] | None
+        The [build-system] section as a dict, or None otherwise.
+    """
+    try:
+        with open(pyproject_path, "rb") as toml_file:
+            return tomllib.load(toml_file)
+    except (FileNotFoundError, TypeError, TOMLDecodeError) as error:
+        logger.debug("Failed to read the %s file: %s", pyproject_path, error)
+        return None
+
+
 def contains_build_tool(tool_name: str, pyproject_path: Path) -> bool:
     """
     Check if a given build tool is present in the [tool] section of a pyproject.toml file.
@@ -26,22 +53,16 @@ def contains_build_tool(tool_name: str, pyproject_path: Path) -> bool:
     bool
         True if the build tool is found in the [tool] section, False otherwise.
     """
-    try:
-        # Parse the pyproject.toml file.
-        with open(pyproject_path, "rb") as toml_file:
-            try:
-                data = tomllib.load(toml_file)
-                # Check for the existence of a [tool.<tool_name>] section.
-                if ("tool" in data) and (tool_name in data["tool"]):
-                    return True
-            except tomllib.TOMLDecodeError:
-                logger.debug("Failed to read the %s file: invalid toml file.", pyproject_path)
-                return False
-        return False
-    except FileNotFoundError:
-        logger.debug("Failed to read the %s file.", pyproject_path)
+    content = get_content(pyproject_path)
+    if not content:
         return False
 
+    # Check for the existence of a [tool.<tool_name>] section.
+    tools = json_extract(content, ["tool"], dict)
+    if tools and tool_name in tools:
+        return True
+    return False
+
 
 def build_system_contains_tool(tool_name: str, pyproject_path: Path) -> bool:
     """
@@ -59,27 +80,21 @@ def build_system_contains_tool(tool_name: str, pyproject_path: Path) -> bool:
     bool
         True if the tool is found in either the 'build-backend' or 'requires' of the [build-system] section, False otherwise.
     """
-    try:
-        with open(pyproject_path, "rb") as toml_file:
-            try:
-                data = tomllib.load(toml_file)
-                build_system = data.get("build-system", {})
-                backend = build_system.get("build-backend", "")
-                requires = build_system.get("requires", [])
-                # Check in 'build-backend'.
-                if tool_name in backend:
-                    return True
-                # Check in 'requires' list.
-                if any(tool_name in req for req in requires):
-                    return True
-            except tomllib.TOMLDecodeError:
-                logger.debug("Failed to read the %s file: invalid toml file.", pyproject_path)
-                return False
-        return False
-    except FileNotFoundError:
-        logger.debug("Failed to read the %s file.", pyproject_path)
+    content = get_content(pyproject_path)
+    if not content:
         return False
 
+    # Check in 'build-backend'.
+    backend = json_extract(content, ["build-system", "build-backend"], str)
+    if backend and tool_name in backend:
+        return True
+    # Check in 'requires' list.
+    requires = json_extract(content, ["build-system", "requires"], list)
+    if requires and any(tool_name in req for req in requires):
+        return True
+
+    return False
+
 
 def get_build_system(pyproject_path: Path) -> dict[str, str] | None:
     """
@@ -95,14 +110,8 @@ def get_build_system(pyproject_path: Path) -> dict[str, str] | None:
     dict[str, str] | None
         The [build-system] section as a dict, or None otherwise.
     """
-    try:
-        with open(pyproject_path, "rb") as toml_file:
-            try:
-                data = tomllib.load(toml_file)
-                return data.get("build-system", {}) or None
-            except tomllib.TOMLDecodeError:
-                logger.debug("Failed to read the %s file: invalid toml file.", pyproject_path)
-                return None
-    except FileNotFoundError:
-        logger.debug("Failed to read the %s file.", pyproject_path)
+    content = get_content(pyproject_path)
+    if not content:
         return None
+
+    return json_extract(content, ["build-system"], dict)

src/macaron/slsa_analyzer/package_registry/pypi_registry.py

@@ -555,13 +555,13 @@ class PyPIPackageJsonAsset:
     #: The asset content.
     package_json: dict
 
-    #: the source code temporary location name
+    #: The source code temporary location name.
     package_sourcecode_path: str
 
-    #: the wheel temporary location name
+    #: The wheel temporary location name.
     wheel_path: str
 
-    #: name of the wheel file
+    #: Name of the wheel file.
     wheel_filename: str
 
     #: The size of the asset (in bytes). This attribute is added to match the AssetLocator

tests/integration/cases/pypi_cachetools/expected_default.buildspec

@@ -22,8 +22,11 @@
             "build"
         ]
     ],
-    "build_backends": {
+    "build_requires": {
         "setuptools": "==(80.9.0)",
         "wheel": ""
-    }
+    },
+    "build_backends": [
+        "setuptools.build_meta"
+    ]
 }

tests/integration/cases/pypi_markdown-it-py/expected_default.buildspec

@@ -21,9 +21,11 @@
             "build"
         ]
     ],
-    "build_backends": {
+    "build_requires": {
         "flit": "==3.12.0",
-        "setuptools": "==56.2.0",
         "flit_core": "<4,>=3.4"
-    }
+    },
+    "build_backends": [
+        "flit_core.buildapi"
+    ]
 }

tests/integration/cases/pypi_toga/expected_default.buildspec

@@ -22,9 +22,12 @@
             "build"
         ]
     ],
-    "build_backends": {
+    "build_requires": {
         "setuptools": "==(80.3.1)",
-        "setuptools_scm": "==8.3.1",
-        "setuptools_dynamic_dependencies": "==1.0.0"
-    }
+        "setuptools_dynamic_dependencies": "==1.0.0",
+        "setuptools_scm": "==8.3.1"
+    },
+    "build_backends": [
+        "setuptools.build_meta"
+    ]
 }