[FEAT] Create a Cloud Service Type for GitHub Repository with Focus on Issues & Pull Requests

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,6 @@
+## Describe your changes
+
+## Checklist before requesting a review
+- [ ] I have performed a self-review of my code
+- [ ] If it is a core feature, I have added thorough tests.
+

README.md

@@ -1 +1,65 @@
-# plugin-github-inven-collector
+# plugin-github-inven-collector
+
+## Introduction
+**plugin-spaceone-inven-collector** is an Inventory Collector Plugin for SpaceONE, designed to collect resources from Github.
+This plugin has been developed with a new plugin framework.
+
+## Setup and Run
+Since the current spaceone-inventory package is in a pre-release state, follow the steps below to configure the package.
+
+### 1. Virtual Environment Setup
+Set up a virtual environment using the venv library.
+```bash
+python3 -m venv venv
+```
+```bash
+source venv/bin/activate
+```
+
+### 2. Package Installation
+Install the package with the following commands in the created virtual environment.
+
+```bash
+pip3 install -r pkg/pip_requirements.txt
+pip3 install --pre spaceone-inventory
+```
+
+### 3. Interpreter Configuration
+If you are using PyCharm, configure the virtual environment as the interpreter.
+![img.png](docs/interpreter_settings.png)
+![img.png](docs/settings_source_directory.png)
+![img.png](docs/run_settings.png)
+
+After following the above steps, run the Plugin Server.
+![img.png](docs/run_plugin_server.png)
+
+## Local Environment Testing
+After running the Plugin Server, **perform tests for each method** with the following commands.
+
+### Check available API methods:
+
+```bash
+spacectl api-resources
+```
+
+#### Collector.init
+```
+spacectl exec init inventory.Collector -f test/init.yml
+```
+
+
+#### Collector.verify
+
+```bash
+spacectl exec verify inventory.Collector -f test/verify.yml
+```
+
+#### Collector.collect
+
+```bash
+spacectl exec collect inventory.Collector -f test/collect.yml
+```
+
+#### Note
+Metadata will be defined as a dictionary and will be converted to YAML.
+The spaceone-inventory package is in a pre-release state, so the `--pre` option must be added when using pip install.

src/VERSION

@@ -0,0 +1 @@
+v1.0.0

src/plugin/connector/__init__.py

@@ -0,0 +1,74 @@
+import logging
+from spaceone.core.connector import BaseConnector
+import requests
+
+__all__ = ["GitHubConnector"]
+
+_LOGGER = logging.getLogger(__name__)
+
+class GitHubConnector(BaseConnector):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    @staticmethod
+    def send_request(url, headers, params=None, method="GET", page=None, per_page=30):
+        try:
+            if page:
+                for response in GitHubConnector._pagination(url, headers, params, per_page, page):
+                    yield response
+            else:
+                response = GitHubConnector._make_request(url, headers, params, method)
+                yield response
+        except Exception as e:
+            _LOGGER.error(f"Request Error: {e}")
+            raise e
+
+    @staticmethod
+    def _make_request(url, headers, params=None, method="GET"):
+        response = None
+        try:
+            if method == "GET":
+                response = requests.get(url, headers=headers, params=params)
+
+            response.raise_for_status()
+            response_json = response.json()
+
+            return response_json
+        except requests.exceptions.HTTPError as errh:
+            _LOGGER.error(f"HTTP Error: {errh}")
+        except requests.exceptions.ConnectionError as errc:
+            _LOGGER.error(f"Error Connecting: {errc}")
+        except requests.exceptions.Timeout as errt:
+            _LOGGER.error(f"Timeout Error: {errt}")
+        except requests.exceptions.RequestException as err:
+            _LOGGER.error(f"Request Error: {err}")
+
+        if response and not response.content:
+            _LOGGER.warning(f"Non-JSON response received: {response.content}")
+
+        return None
+
+    @staticmethod
+    def _pagination(url, headers, params, per_page, page):
+        responses = []
+        while True:
+            paginated_url = f"{url}{'&' if '&' in url else '?'}per_page={per_page}&page={page}"
+            response_json = GitHubConnector._make_request(paginated_url, headers, params)
+
+            if not response_json:
+                break
+
+            page += 1
+            responses.extend(response_json)
+
+        return responses
+
+    @staticmethod
+    def make_header(secret_data):
+        github_token = secret_data.get("github_token")
+        return {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {github_token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }

src/plugin/connector/repo_connector.py

@@ -0,0 +1,20 @@
+from . import GitHubConnector
+
+
+class RepoConnector(GitHubConnector):
+
+    def get_repositories(self, secret_data):
+        org_name = secret_data.get("org_name")
+        url = f"https://api.github.com/orgs/{org_name}/repos"
+        headers = self.make_header(secret_data)
+        return self.send_request(url, headers, page=1000)
+
+    def get_repo_issues(self, secret_data, repo_owner, repo_name):
+        url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues"
+        headers = self.make_header(secret_data)
+        return self.send_request(url, headers, page=1000)
+
+    def get_repo_pulls(self, secret_data, repo_owner, repo_name):
+        url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/pulls"
+        headers = self.make_header(secret_data)
+        return self.send_request(url, headers, page=1000)

src/plugin/main.py

@@ -1,66 +1,39 @@
 from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer
 
+from plugin.manager.repo_manager import RepoManager
 
 app = CollectorPluginServer()
 
 
 @app.route('Collector.init')
 def collector_init(params: dict) -> dict:
-    """ init plugin by options
-
-    Args:
-        params (CollectorInitRequest): {
-            'options': 'dict',    # Required
-            'domain_id': 'str'
-        }
-
-    Returns:
-        PluginResponse: {
-            'metadata': 'dict'
-        }
-    """
-    pass
-
-
-@app.route('Collector.verify')
-def collector_verify(params: dict) -> None:
-    """ Verifying collector plugin
-
-    Args:
-        params (CollectorVerifyRequest): {
-            'options': 'dict',      # Required
-            'secret_data': 'dict',  # Required
-            'schema': 'str',
-            'domain_id': 'str'
-        }
-
-    Returns:
-        None
-    """
-    pass
+    return {'metadata': {'options_schema': _create_options_schema()}}
 
 
 @app.route('Collector.collect')
 def collector_collect(params: dict) -> dict:
-    """ Collect external data
-
-    Args:
-        params (CollectorCollectRequest): {
-            'options': 'dict',      # Required
-            'secret_data': 'dict',  # Required
-            'schema': 'str',
-            'domain_id': 'str'
-        }
-
-    Returns:
-        Generator[ResourceResponse, None, None]
-        {
-            'state': 'SUCCESS | FAILURE',
-            'resource_type': 'inventory.CloudService | inventory.CloudServiceType | inventory.Region',
-            'resource_data': 'dict',
-            'match_keys': 'list',
-            'error_message': 'str'
-            'metadata': 'dict'
+    options = params['options']
+    secret_data = params['secret_data']
+    schema = params.get('schema')
+
+    repository_manager = RepoManager()
+    return repository_manager.collect_resources(options, secret_data, schema)
+
+
+def _create_options_schema():
+    return {
+        'required': ['items'],
+        'order': ['items'],
+        'type': 'object',
+        'properties': {
+            'items': {
+                'title': 'Item filter',
+                'type': 'array',
+                'items': {
+                    'enum': [
+                        'fields'
+                    ]
+                }
+            }
         }
-    """
-    pass
+    }

src/plugin/manager/repo_manager.py

@@ -0,0 +1,105 @@
+import logging
+
+from spaceone.inventory.plugin.collector.lib import *
+from ..connector.repo_connector import RepoConnector
+
+
+_LOGGER = logging.getLogger("cloudforet")
+
+
+class RepoManager:
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.cloud_service_group = "Repository"
+        self.cloud_service_type = "Issue"
+        self.provider = "Github"
+        self.metadata_path = "plugin/metadata/repository/issues.yaml"
+
+    def collect_resources(self, options, secret_data, schema):
+        try:
+            yield from self.collect_cloud_service_type(options, secret_data, schema)
+        except Exception as e:
+            yield make_error_response(
+                error=e,
+                provider=self.provider,
+                cloud_service_group=self.cloud_service_group,
+                cloud_service_type=self.cloud_service_type,
+                resource_type="inventory.CloudServiceType",
+            )
+
+        try:
+            yield from self.collect_cloud_service(options, secret_data, schema)
+        except Exception as e:
+            yield make_error_response(
+                error=e,
+                provider=self.provider,
+                cloud_service_group=self.cloud_service_group,
+                cloud_service_type=self.cloud_service_type,
+            )
+
+    def collect_cloud_service_type(self, options, secret_data, schema):
+        cloud_service_type = make_cloud_service_type(
+            name=self.cloud_service_type,
+            group=self.cloud_service_group,
+            provider=self.provider,
+            metadata_path=self.metadata_path,
+        )
+
+        yield make_response(
+            cloud_service_type=cloud_service_type,
+            match_keys=[["name", "reference.resource_id", "account", "provider"]],
+            resource_type="inventory.CloudServiceType",
+        )
+
+    def collect_cloud_service(self, options, secret_data, schema):
+        repo_connector = RepoConnector()
+        repos = repo_connector.get_repositories(secret_data)
+
+        all_issues = []
+
+        _LOGGER.debug(f"Repos: {repos}")
+
+        for repo in repos:
+            _LOGGER.debug(f"Processing repo: {repo}")
+
+            owner = repo.get("owner", {})
+            repo_name = repo.get("name")
+
+            if repo_name and owner:
+                repo_owner = owner.get("login")
+
+                _LOGGER.debug(f"Processing issues for repo: {repo_name}")
+
+                issues = repo_connector.get_repo_issues(secret_data, repo_owner, repo_name)
+                pulls = repo_connector.get_repo_pulls(secret_data, repo_owner, repo_name)
+
+                for issue in issues:
+                    issue["repo_name"] = repo_name
+                    issue["pulls"] = pulls
+
+                all_issues.extend(issues)
+            else:
+                _LOGGER.warning("Missing required attributes in repo: {repo}")
+
+        all_issues.sort(key=lambda x: x.get("created_at", ""), reverse=True)
+
+        for issue in all_issues:
+            labels = issue.get("labels", [])
+            issue_labels = {label.get("name"): label for label in labels}
+            issue["labels"] = issue_labels
+
+            cloud_service = make_cloud_service(
+                name=self.cloud_service_type,
+                cloud_service_type=self.cloud_service_type,
+                cloud_service_group=self.cloud_service_group,
+                provider=self.provider,
+                data=issue,
+            )
+
+            yield make_response(
+                cloud_service=cloud_service,
+                match_keys=[["name", "reference.resource_id", "account", "provider"]],
+            )
+