Add commit tracking capabilities.

Refactor code to be compatible with the new pipeline.

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

minecode_pipeline/README.rst

@@ -0,0 +1,42 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+from pathlib import Path
+
+from minecode_pipelines.pipes.cargo import store_cargo_packages
+from minecode_pipelines.utils import get_changed_files
+
+
+def process_cargo_packages(cargo_repo, fed_repo):
+    base_path = Path(cargo_repo.working_tree_dir)
+    valid_files = get_changed_files(cargo_repo)  # start from empty tree hash
+
+    json_files = []
+    for file_path in base_path.glob("**/*"):
+        if not file_path.is_file() or file_path not in valid_files:
+            continue
+
+        if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
+            continue
+        json_files.append(file_path)
+
+    for idx, file_path in enumerate(json_files, start=1):
+        try:
+            with open(file_path, encoding="utf-8") as f:
+                packages = []
+                for line in f:
+                    if line.strip():
+                        packages.append(json.loads(line))
+
+        except (json.JSONDecodeError, UnicodeDecodeError):
+            continue
+
+        if packages:
+            push_commit = idx == len(json_files)  # only True on last
+            store_cargo_packages(packages, fed_repo, push_commit)

minecode_pipeline/__init__.py

@@ -19,27 +19,25 @@
 #
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
-import json
-from pathlib import Path
-
-from minecode_pipeline.pipes import cargo
 
+from git.repo.base import Repo
+from scanpipe.pipes.federatedcode import delete_local_clone
+from minecode.utils import get_temp_file
 from scanpipe.pipelines import Pipeline
-from fetchcode.vcs import fetch_via_vcs
 from scanpipe.pipes import federatedcode
+from minecode_pipelines.miners import cargo
 
 
-class MineCargo(Pipeline):
+class MineandPublishCargoPURLs(Pipeline):
     """Pipeline to mine Cargo (crates.io) packages and publish them to FederatedCode."""
 
-    repo_url = "git+https://github.com/rust-lang/crates.io-index"
-
     @classmethod
     def steps(cls):
         return (
             cls.check_federatedcode_eligibility,
             cls.clone_cargo_repo,
             cls.collect_packages_from_cargo,
+            cls.clean_cargo_repo,
         )
 
     def check_federatedcode_eligibility(self):
@@ -49,41 +47,25 @@ def check_federatedcode_eligibility(self):
         """
         federatedcode.check_federatedcode_eligibility(project=self.project)
 
-    def clone_cargo_repo(self, repo_url):
+    def clone_cargo_repo(self):
         """
         Clone the repo at repo_url and return the VCSResponse object
         """
-        self.vcs_response = fetch_via_vcs(repo_url)
-
-    def collect_packages_from_cargo(self):
-        base_path = Path(self.vcs_response.dest_dir)
+        conan_repo_url = "git+https://github.com/rust-lang/crates.io-index"
+        fed_repo_url = "git+https://github.com/ziadhany/cargo-test"
 
-        json_files = []
-        for file_path in base_path.glob("**/*"):
-            if not file_path.is_file():
-                continue
-            if file_path.name in {"config.json", "README.md", "update-dl-url.yml"}:
-                continue
-            json_files.append(file_path)
+        self.fed_repo = federatedcode.clone_repository(fed_repo_url)
+        self.cargo_repo = Repo.clone_from(conan_repo_url, get_temp_file())
 
-        for idx, file_path in enumerate(json_files, start=1):
-            try:
-                with open(file_path, encoding="utf-8") as f:
-                    packages = []
-                    for line in f:
-                        if line.strip():
-                            packages.append(json.loads(line))
-
-            except (json.JSONDecodeError, UnicodeDecodeError):
-                continue
-
-            if packages:
-                push_commit = idx == len(json_files)  # only True on last
-                cargo.collect_packages_from_cargo(packages, self.vcs_response, push_commit)
+    def collect_packages_from_cargo(self):
+        cargo.process_cargo_packages(self.cargo_repo, self.fed_repo)
 
     def clean_cargo_repo(self):
         """
-        Delete the VCS response repository if it exists.
+        Delete the federatedcode repository if it exists, and also delete the Cargo repository if it exists.
         """
-        if self.vcs_response:
-            self.vcs_response.delete()
+        if self.cargo_repo:
+            delete_local_clone(self.cargo_repo)
+
+        if self.fed_repo:
+            delete_local_clone(self.fed_repo)

minecode_pipeline/pipelines/__init__.py

@@ -7,12 +7,18 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
+from aboutcode.hashid import PURLS_FILENAME
 import os
-import saneyaml
-
+import textwrap
 from pathlib import Path
+import saneyaml
+from aboutcode import hashid
 
-from aboutcode.hashid import PURLS_FILENAME
+VERSION = os.environ.get("VERSION", "")
+PURLDB_ALLOWED_HOST = os.environ.get("FEDERATEDCODE_GIT_ALLOWED_HOST", "")
+author_name = os.environ.get("FEDERATEDCODE_GIT_SERVICE_NAME", "")
+author_email = os.environ.get("FEDERATEDCODE_GIT_SERVICE_EMAIL", "")
+remote_name = os.environ.get("FEDERATEDCODE_GIT_REMOTE_NAME", "origin")
 
 
 def write_packageurls_to_file(repo, base_dir, packageurls):
@@ -26,3 +32,41 @@ def write_data_to_file(path, data):
     path.parent.mkdir(parents=True, exist_ok=True)
     with open(path, encoding="utf-8", mode="w") as f:
         f.write(saneyaml.dump(data))
+
+
+def write_purls_to_repo(repo, package, updated_purls, push_commits=False):
+    """Write or update package purls in the repo and optionally commit/push changes."""
+    ppath = hashid.get_package_purls_yml_file_path(package)
+    git_stage_purls(updated_purls, repo, ppath)
+    if push_commits:
+        commit_and_push_changes(repo)
+
+
+def git_stage_purls(purls, repo, purls_file):
+    """Write package URLs to a file and stage it in the local Git repository."""
+    relative_purl_file_path = Path(purls_file)
+
+    write_to = Path(repo.working_dir) / relative_purl_file_path
+
+    write_data_to_file(path=write_to, data=purls)
+
+    repo.index.add([relative_purl_file_path])
+    return relative_purl_file_path
+
+
+def commit_and_push_changes(repo):
+    """
+    Commit staged changes to the local repository and push them
+    to the remote on the current active branch.
+    """
+
+    commit_message = f"""\
+    Add/Update list of available package versions
+    Tool: pkg:github/aboutcode-org/purldb@v{VERSION}
+    Reference: https://{PURLDB_ALLOWED_HOST}/
+    Signed-off-by: {author_name} <{author_email}>
+    """
+
+    default_branch = repo.active_branch.name
+    repo.index.commit(textwrap.dedent(commit_message))
+    repo.git.push(remote_name, default_branch, "--no-verify")

minecode_pipeline/pipes/__init__.py

@@ -1,9 +1,9 @@
 from packageurl import PackageURL
 from aboutcode.hashid import get_core_purl
-from minecode_pipeline.pipes import write_purls_to_repo
+from minecode_pipelines.pipes import write_purls_to_repo
 
 
-def collect_packages_from_cargo(packages, repo, push_commit=False):
+def store_cargo_packages(packages, fed_repo, push_commit=False):
     """Collect Cargo package versions into purls and write them to the repo."""
 
     if not packages:
@@ -22,4 +22,4 @@ def collect_packages_from_cargo(packages, repo, push_commit=False):
         purl = PackageURL(type="cargo", name=name, version=version).to_string()
         updated_purls.append(purl)
 
-    write_purls_to_repo(repo, base_purl, updated_purls, push_commit)
+    write_purls_to_repo(fed_repo, base_purl, updated_purls, push_commit)

minecode_pipeline/tests/__init__.py

@@ -40,6 +40,7 @@
 
 
 from scanpipe.pipes.federatedcode import clone_repository
+
 from scanpipe.pipes.federatedcode import commit_changes
 from scanpipe.pipes.federatedcode import push_changes
 from scanpipe.pipes.federatedcode import commit_and_push_changes