Add support to mine conan Package-URLs

minecode_pipelines/__init__.py

@@ -7,4 +7,4 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-VERSION = "0.0.1b10"
+VERSION = "0.0.1b11"

minecode_pipelines/miners/conan.py

@@ -0,0 +1,68 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+import saneyaml
+
+from scanpipe.pipes.federatedcode import commit_changes
+from scanpipe.pipes.federatedcode import push_changes
+from minecode_pipelines import VERSION
+from minecode_pipelines.pipes.conan import store_conan_packages
+
+PACKAGE_BATCH_SIZE = 1000
+
+
+def mine_and_publish_conan_packageurls(conan_index_repo, cloned_data_repo, logger):
+    base_path = Path(conan_index_repo.working_dir)
+
+    yml_files = []
+    for file_path in base_path.glob("recipes/**/*"):
+        if not file_path.name == "config.yml":
+            continue
+        yml_files.append(file_path)
+
+    file_counter = 0
+    purl_files = []
+    purls = []
+
+    total_files = len(yml_files)
+    logger(f"Processing total files: {total_files}")
+    for idx, file_path in enumerate(yml_files, start=1):
+        # Example: file_path = Path("repo_path/recipes/7zip/config.yml")
+        # - file_path.parts = ("repo_path", "recipes", "7zip", "config.yml")
+        # - file_path.parts[-2] = "7zip"  (the package name)
+        package = file_path.parts[-2]
+        with open(file_path, encoding="utf-8") as f:
+            versions = saneyaml.load(f)
+
+        if not versions:
+            continue
+
+        file_counter += 1
+        push_commit = file_counter >= PACKAGE_BATCH_SIZE or idx == total_files
+
+        result_store = store_conan_packages(package, versions, cloned_data_repo)
+        if result_store:
+            purl_file, base_purl = result_store
+            logger(f"writing packageURLs for package: {base_purl} at: {purl_file}")
+
+            purl_files.append(purl_file)
+            purls.append(str(base_purl))
+
+        if push_commit:
+            commit_changes(
+                repo=cloned_data_repo,
+                files_to_commit=purl_files,
+                purls=purls,
+                mine_type="packageURL",
+                tool_name="pkg:pypi/minecode-pipelines",
+                tool_version=VERSION,
+            )
+            push_changes(repo=cloned_data_repo)
+            file_counter = 0

minecode_pipelines/pipelines/mine_conan.py

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import os
+from scanpipe.pipelines import Pipeline
+from minecode_pipelines import pipes
+from minecode_pipelines.miners import conan
+from scanpipe.pipes import federatedcode
+
+MINECODE_CONAN_INDEX_REPO = "https://github.com/conan-io/conan-center-index"
+
+MINECODE_DATA_CONAN_REPO = os.environ.get(
+    "MINECODE_DATA_CONAN_REPO", "https://github.com/aboutcode-data/minecode-data-conan-test"
+)
+
+
+class MineConan(Pipeline):
+    """Pipeline to mine Conan packages and publish them to FederatedCode repo."""
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.clone_conan_repos,
+            cls.mine_and_publish_conan_package_urls,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_configured_and_available(logger=self.log)
+
+    def clone_conan_repos(self):
+        """
+        Clone the Conan-related repositories (index, data, and pipelines config)
+        and store their Repo objects in the corresponding instance variables.
+        """
+        self.conan_index_repo = federatedcode.clone_repository(MINECODE_CONAN_INDEX_REPO)
+        self.cloned_data_repo = federatedcode.clone_repository(MINECODE_DATA_CONAN_REPO)
+
+        if self.log:
+            self.log(
+                f"{MINECODE_CONAN_INDEX_REPO} repo cloned at: {self.conan_index_repo.working_dir}"
+            )
+            self.log(
+                f"{MINECODE_DATA_CONAN_REPO} repo cloned at: {self.cloned_data_repo.working_dir}"
+            )
+
+    def mine_and_publish_conan_package_urls(self):
+        conan.mine_and_publish_conan_packageurls(
+            self.conan_index_repo, self.cloned_data_repo, self.log
+        )
+
+    def delete_cloned_repos(self):
+        pipes.delete_cloned_repos(
+            repos=[self.conan_index_repo, self.cloned_data_repo],
+            logger=self.log,
+        )

minecode_pipelines/pipes/conan.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+from packageurl import PackageURL
+from pathlib import Path
+from aboutcode import hashid
+from minecode_pipelines.pipes import write_data_to_yaml_file
+
+
+def store_conan_packages(pacakge_name, versions_data, fed_repo):
+    """Collect Conan package versions into purls and write them to the repo."""
+
+    base_purl = PackageURL(type="conan", name=pacakge_name)
+
+    updated_purls = []
+    versions = list(versions_data["versions"].keys())
+    for version in versions:
+        purl = PackageURL(type="conan", name=pacakge_name, version=version).to_string()
+        updated_purls.append(purl)
+
+    ppath = hashid.get_package_purls_yml_file_path(base_purl)
+    purl_file_full_path = Path(fed_repo.working_dir) / ppath
+    write_data_to_yaml_file(path=purl_file_full_path, data=updated_purls)
+    return purl_file_full_path, base_purl

minecode_pipelines/tests/pipes/test_conan.py

@@ -0,0 +1,84 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import tempfile
+from pathlib import Path
+from unittest import mock
+from unittest.mock import Mock, patch
+import saneyaml
+import yaml
+
+from django.test import TestCase
+
+from minecode_pipelines.pipes import write_data_to_yaml_file
+from minecode_pipelines.pipes.conan import store_conan_packages
+
+DATA_DIR = Path(__file__).parent.parent / "test_data" / "conan"
+
+
+class ConanPipelineTests(TestCase):
+    @patch("minecode_pipelines.pipes.conan.write_data_to_yaml_file")
+    def test_collect_packages_from_cargo_calls_write(self, mock_write):
+        packages_file = DATA_DIR / "cairo-config.yml"
+        expected_file = DATA_DIR / "expected-cairo-purls.yml"
+
+        with open(packages_file, encoding="utf-8") as f:
+            versions_data = yaml.safe_load(f)
+
+        with open(expected_file, encoding="utf-8") as f:
+            expected = saneyaml.load(f)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            repo = Mock()
+            repo.working_dir = tmpdir
+
+            store_conan_packages("cairo", versions_data, repo)
+
+            mock_write.assert_called_once()
+            args, kwargs = mock_write.call_args
+            base_purl, written_packages = kwargs["path"], kwargs["data"]
+
+            expected_base_purl = (
+                Path(tmpdir) / "aboutcode-packages-conan-0" / "conan" / "cairo" / "purls.yml"
+            )
+
+            self.assertEqual(str(base_purl), str(expected_base_purl))
+            self.assertEqual(written_packages, expected)
+
+    def _assert_purls_written(self, purls):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            repo_dir = Path(tmpdir)
+
+            mock_repo = mock.MagicMock()
+            mock_repo.working_dir = str(repo_dir)
+            mock_repo.index.add = mock.MagicMock()
+
+            purls_file = repo_dir / "purls.yaml"
+
+            write_data_to_yaml_file(purls_file, purls)
+
+            self.assertTrue(purls_file.exists())
+
+            with open(purls_file, encoding="utf-8") as f:
+                content = saneyaml.load(f)
+
+            self.assertEqual(content, purls)
+
+    def test_add_purl_result_with_mock_repo(self):
+        purls = [
+            {"purl": "pkg:conan/cairo@1.18.0"},
+            {"purl": "pkg:conan/cairo@1.17.8"},
+            {"purl": "pkg:conan/cairo@1.17.6"},
+            {"purl": "pkg:conan/cairo@1.17.4"},
+        ]
+
+        self._assert_purls_written(purls)
+
+    def test_add_empty_purl_result_with_mock_repo(self):
+        self._assert_purls_written([])

minecode_pipelines/tests/test_data/conan/cairo-config.yml

@@ -0,0 +1,9 @@
+versions:
+  "1.18.0":
+    folder: all
+  "1.17.8":
+    folder: all
+  "1.17.6":
+    folder: all
+  "1.17.4":
+    folder: all

minecode_pipelines/tests/test_data/conan/expected-cairo-purls.yml

@@ -0,0 +1,4 @@
+- pkg:conan/cairo@1.18.0
+- pkg:conan/cairo@1.17.8
+- pkg:conan/cairo@1.17.6
+- pkg:conan/cairo@1.17.4

pyproject-minecode_pipelines.toml

@@ -4,7 +4,7 @@ build-backend = "flot.buildapi"
 
 [project]
 name = "minecode_pipelines"
-version = "0.0.1b10"
+version = "0.0.1b11"
 description = "A library for mining packageURLs and package metadata from ecosystem repositories."
 readme = "minecode_pipelines/README.rst"
 license = { text = "Apache-2.0" }
@@ -53,9 +53,10 @@ mine_maven = "minecode_pipelines.pipelines.mine_maven:MineMaven"
 mine_cargo = "minecode_pipelines.pipelines.mine_cargo:MineCargo"
 mine_debian = "minecode_pipelines.pipelines.mine_debian:MineDebian"
 mine_alpine = "minecode_pipelines.pipelines.mine_alpine:MineAlpine"
+mine_conan = "minecode_pipelines.pipelines.mine_conan:MineConan"
 
 [tool.bumpversion]
-current_version = "0.0.1b10"
+current_version = "0.0.1b11"
 allow_dirty = true
 
 files = [