From f7d780932946e6a8d23d61b2b581f3f3a589ee64 Mon Sep 17 00:00:00 2001 From: Wanjin Noh Date: Mon, 13 Nov 2023 20:54:49 +0900 Subject: [PATCH 1/2] feat: add organization repository with table fields --- src/plugin/connector/organization/__init__.py | 0 .../organization/repository_connector.py | 13 ++++ src/plugin/main.py | 39 +++++------- src/plugin/manager/__init__.py | 0 src/plugin/manager/organization/__init__.py | 0 .../organization/repository_manager.py | 60 +++++++++++++++++++ src/plugin/metadata/__init__.py | 0 src/plugin/metadata/organization/__init__.py | 0 .../metadata/organization/repository.yaml | 22 +++++++ test/collect.yml | 5 ++ test/init.yml | 2 + 11 files changed, 118 insertions(+), 23 deletions(-) create mode 100644 src/plugin/connector/organization/__init__.py create mode 100644 src/plugin/connector/organization/repository_connector.py create mode 100644 src/plugin/manager/__init__.py create mode 100644 src/plugin/manager/organization/__init__.py create mode 100644 src/plugin/manager/organization/repository_manager.py create mode 100644 src/plugin/metadata/__init__.py create mode 100644 src/plugin/metadata/organization/__init__.py create mode 100644 src/plugin/metadata/organization/repository.yaml create mode 100644 test/collect.yml create mode 100644 test/init.yml diff --git a/src/plugin/connector/organization/__init__.py b/src/plugin/connector/organization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/connector/organization/repository_connector.py b/src/plugin/connector/organization/repository_connector.py new file mode 100644 index 0000000..939724a --- /dev/null +++ b/src/plugin/connector/organization/repository_connector.py @@ -0,0 +1,13 @@ +from .. import RequestConnector +import logging + + +class OrgRepositoryConnector(RequestConnector): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def list_repositories(self, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/orgs/{secret_data.get('org_name')}/repos" + response = self.send_request(url, headers, page=1) + return response diff --git a/src/plugin/main.py b/src/plugin/main.py index 47466e8..e561b11 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,12 +1,16 @@ +import logging + from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer +from plugin.manager.organization.repository_manager import OrgRepositoryManager +_LOGGER = logging.getLogger("cloudforet") app = CollectorPluginServer() -@app.route('Collector.init') +@app.route("Collector.init") def collector_init(params: dict) -> dict: - """ init plugin by options + """init plugin by options Args: params (CollectorInitRequest): { @@ -19,30 +23,13 @@ def collector_init(params: dict) -> dict: 'metadata': 'dict' } """ - pass - - -@app.route('Collector.verify') -def collector_verify(params: dict) -> None: - """ Verifying collector plugin - - Args: - params (CollectorVerifyRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - Returns: - None - """ - pass + return {"metadata": {}} -@app.route('Collector.collect') +@app.route("Collector.collect") def collector_collect(params: dict) -> dict: - """ Collect external data + """Collect external data Args: params (CollectorCollectRequest): { @@ -63,4 +50,10 @@ def collector_collect(params: dict) -> dict: 'metadata': 'dict' } """ - pass + + options = params["options"] + secret_data = params["secret_data"] + schema = params.get("schema") + + org_repository_manager = OrgRepositoryManager() + return org_repository_manager.collect_resources(options, secret_data, schema) diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/manager/organization/__init__.py b/src/plugin/manager/organization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/manager/organization/repository_manager.py b/src/plugin/manager/organization/repository_manager.py new file mode 100644 index 0000000..35a4d2f --- /dev/null +++ b/src/plugin/manager/organization/repository_manager.py @@ -0,0 +1,60 @@ +import logging +from spaceone.inventory.plugin.collector.lib import * + +from ...connector.organization.repository_connector import OrgRepositoryConnector + +_LOGGER = logging.getLogger("cloudforet") + + +class OrgRepositoryManager: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Organization" + self.cloud_service_type = "repository" + self.provider = "github" + self.metadata_path = "metadata/organization/repository.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + is_primary=True, + is_major=True, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + org_repository_connector = OrgRepositoryConnector() + repo_items = org_repository_connector.list_repositories(secret_data) + for item in repo_items: + cloud_service = make_cloud_service( + name=self.cloud_service_type, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=item, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) diff --git a/src/plugin/metadata/__init__.py b/src/plugin/metadata/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/metadata/organization/__init__.py b/src/plugin/metadata/organization/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/plugin/metadata/organization/repository.yaml b/src/plugin/metadata/organization/repository.yaml new file mode 100644 index 0000000..05f2056 --- /dev/null +++ b/src/plugin/metadata/organization/repository.yaml @@ -0,0 +1,22 @@ +--- +search: + fields: + - Description: data.description + - Url: data.html_url + - Topics: data.topics + - Language: data.language + + +table: + sort: + key: data.created_at + desc: true + fields: + - Description: data.description + - Url: data.html_url + - Topics: data.topics + - Language: data.language + - Open Issues Count: data.open_issues_count + - Forks Count: data.forks_count + - Stargazers Count: data.stargazers_count + - Watchers Count: data.watchers_count diff --git a/test/collect.yml b/test/collect.yml new file mode 100644 index 0000000..1882467 --- /dev/null +++ b/test/collect.yml @@ -0,0 +1,5 @@ +--- +options: {} +secret_data: + github_token: 12345 + org_name: 'forest-extension' \ No newline at end of file diff --git a/test/init.yml b/test/init.yml new file mode 100644 index 0000000..81988e1 --- /dev/null +++ b/test/init.yml @@ -0,0 +1,2 @@ +--- +options: {} \ No newline at end of file From d6c4f6bc7803280e32083d9c978b392e08b893b1 Mon Sep 17 00:00:00 2001 From: Wanjin Noh Date: Wed, 6 Dec 2023 00:36:47 +0900 Subject: [PATCH 2/2] feat: add detail info - issues, branches, languages, contributors, forks --- src/VERSION | 1 + .../organization/repository_connector.py | 40 +++++- src/plugin/main.py | 1 + .../organization/repository_manager.py | 134 ++++++++++++++---- .../metadata/organization/repository.yaml | 100 +++++++++++++ src/setup.py | 36 +++++ 6 files changed, 285 insertions(+), 27 deletions(-) create mode 100755 src/VERSION create mode 100644 src/setup.py diff --git a/src/VERSION b/src/VERSION new file mode 100755 index 0000000..0ec25f7 --- /dev/null +++ b/src/VERSION @@ -0,0 +1 @@ +v1.0.0 diff --git a/src/plugin/connector/organization/repository_connector.py b/src/plugin/connector/organization/repository_connector.py index 939724a..78fd099 100644 --- a/src/plugin/connector/organization/repository_connector.py +++ b/src/plugin/connector/organization/repository_connector.py @@ -8,6 +8,44 @@ def __init__(self, *args, **kwargs): def list_repositories(self, secret_data): headers = self.make_header(secret_data) - url = f"https://api.github.com/orgs/{secret_data.get('org_name')}/repos" + url = ( + f"https://api.github.com/orgs/{secret_data.get('organization_name')}/repos" + ) response = self.send_request(url, headers, page=1) return response + + def list_repository_issues(self, repo, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo}/issues" + response = self.send_request(url, headers, page=1) + return response + + def get_repository_languages(self, repo, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo}/languages" + response = self.send_request(url, headers) + return response + + def get_repository_contributors(self, repo, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo}/contributors" + response = self.send_request(url, headers) + return response + + def list_repository_branches(self, repo, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo}/branches" + response = self.send_request(url, headers, page=1) + return response + + def list_repository_forks(self, repo, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/repos/{secret_data.get('organization_name')}/{repo}/forks" + response = self.send_request(url, headers, page=1) + return response + + def get_user(self, user, secret_data): + headers = self.make_header(secret_data) + url = f"https://api.github.com/users/{user}" + response = self.send_request(url, headers) + return response diff --git a/src/plugin/main.py b/src/plugin/main.py index e561b11..cb3ca23 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -56,4 +56,5 @@ def collector_collect(params: dict) -> dict: schema = params.get("schema") org_repository_manager = OrgRepositoryManager() + # return org_repository_manager.collect_cloud_service_type(options, secret_data, schema) return org_repository_manager.collect_resources(options, secret_data, schema) diff --git a/src/plugin/manager/organization/repository_manager.py b/src/plugin/manager/organization/repository_manager.py index 35a4d2f..1fb033a 100644 --- a/src/plugin/manager/organization/repository_manager.py +++ b/src/plugin/manager/organization/repository_manager.py @@ -11,8 +11,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.cloud_service_group = "Organization" - self.cloud_service_type = "repository" - self.provider = "github" + self.cloud_service_type = "Repository" + self.provider = "github_wanjin" self.metadata_path = "metadata/organization/repository.yaml" def collect_resources(self, options, secret_data, schema): @@ -28,33 +28,115 @@ def collect_resources(self, options, secret_data, schema): ) def collect_cloud_service_type(self, options, secret_data, schema): - cloud_service_type = make_cloud_service_type( - name=self.cloud_service_type, - group=self.cloud_service_group, - provider=self.provider, - metadata_path=self.metadata_path, - is_primary=True, - is_major=True, - ) - - yield make_response( - cloud_service_type=cloud_service_type, - match_keys=[["name", "reference.resource_id", "account", "provider"]], - resource_type="inventory.CloudServiceType", - ) - - def collect_cloud_service(self, options, secret_data, schema): - org_repository_connector = OrgRepositoryConnector() - repo_items = org_repository_connector.list_repositories(secret_data) - for item in repo_items: - cloud_service = make_cloud_service( + try: + cloud_service_type = make_cloud_service_type( name=self.cloud_service_type, - cloud_service_type=self.cloud_service_type, - cloud_service_group=self.cloud_service_group, + group=self.cloud_service_group, provider=self.provider, - data=item, + metadata_path=self.metadata_path, + is_primary=True, + is_major=True, + tags={ + "spaceone:icon": "https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png" + }, ) + yield make_response( - cloud_service=cloud_service, + cloud_service_type=cloud_service_type, match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service(self, options, secret_data, schema): + try: + org_repository_connector = OrgRepositoryConnector() + repo_items = org_repository_connector.list_repositories(secret_data) + for item in repo_items: + # get issues + issue_items = org_repository_connector.list_repository_issues( + item["name"], secret_data + ) + item["issues"] = list(issue_items) + + # get languages + (language_info,) = org_repository_connector.get_repository_languages( + item["name"], secret_data + ) + languages = [] + for key, value in language_info.items(): + languages.append({"language": key, "bytes": value}) + item["languages"] = languages + + # get contributors + ( + contributor_items, + ) = org_repository_connector.get_repository_contributors( + item["name"], secret_data + ) + contributors_info = [] + # get contributors info + for contributor_item in contributor_items: + (contributor_info,) = org_repository_connector.get_user( + contributor_item["login"], secret_data + ) + contributor_info["contributions"] = contributor_item["contributions"] + contributors_info.append(contributor_info) + item["contributors"] = contributors_info + + # get branches + branch_items = org_repository_connector.list_repository_branches( + item["name"], secret_data + ) + item["branches"] = list(branch_items) + + # get forks + fork_items = org_repository_connector.list_repository_forks( + item["name"], secret_data + ) + forks_info = [] + # get forked repo owners info + for fork_item in fork_items: + (owner_info,) = org_repository_connector.get_user( + fork_item["owner"]["login"], secret_data + ) + forks_info.append( + { + "name": fork_item["name"], + "description": fork_item["description"], + "html_url": fork_item["html_url"], + "open_issues_count": fork_item["open_issues_count"], + "forks_count": fork_item["forks_count"], + "owner_info": owner_info, + } + ) + item["forks_info"] = forks_info + + cloud_service = make_cloud_service( + name=item["name"], + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=item, + ) + + yield make_response( + cloud_service=cloud_service, + match_keys=[ + ["name", "reference.resource_id", "account", "provider"] + ], + ) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, ) diff --git a/src/plugin/metadata/organization/repository.yaml b/src/plugin/metadata/organization/repository.yaml index 05f2056..0d3fd71 100644 --- a/src/plugin/metadata/organization/repository.yaml +++ b/src/plugin/metadata/organization/repository.yaml @@ -12,11 +12,111 @@ table: key: data.created_at desc: true fields: + - Name: data.name + link: data.html_url - Description: data.description - Url: data.html_url - Topics: data.topics - Language: data.language + type: badge + outline_color: blue.500 + text_color: blue.500 - Open Issues Count: data.open_issues_count - Forks Count: data.forks_count - Stargazers Count: data.stargazers_count - Watchers Count: data.watchers_count + + +tabs.0: + name: Issues + type: list + items: + - name: Issues + type: table + root_path: data.issues + fields: + - Number: number + - Title: title + link: html_url + - State: state + type: enum + enums: + - open: green.500 + - closed: red.500 + - Labels: labels.name + - Assignees: assignees.login + - Comments Count: comments + +tabs.1: + name: Languages + type: list + items: + - name: Languages + type: simple-table + root_path: data.languages + fields: + - Language: language + type: badge + outline_color: blue.500 + text_color: blue.500 + - Bytes of Code: bytes + type: size + display_unit: MB + source_unit: BYTES + + +tabs.2: + name: Contributors + type: list + items: + - name: Contributors + type: simple-table + root_path: data.contributors + fields: + - Name: name + - Contributions: contributions + - Github Url: html_url + - Email: email + - Company: company + - Location: location + - Followers: followers + - Following: following + +tabs.3: + name: Branches + type: list + items: + - name: Branches + type: simple-table + root_path: data.branches + fields: + - Name: name + - Protected: protected + - Commit: commit.sha + link: commit.url + +tabs.4: + name: Forks + type: list + items: + - name: Forked Repos + type: simple-table + root_path: data.forks_info + fields: + - Name: name + - Github Url: html_url + - Owner: owner_info.login + - Open Issues Count: open_issues_count + - Forks Count: forks_count + - name: Forked Repo Owners + type: simple-table + root_path: data.forks_info.owner_info + fields: + - Name: name + - ID: login + - Email: email + - Github Url: html_url + - Company: company + - Location: location + - Followers: followers + - Following: following \ No newline at end of file diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..b5a79fa --- /dev/null +++ b/src/setup.py @@ -0,0 +1,36 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="Collector plugin for Github", + long_description="", + url="https://github.com/WANZARGEN", + author="Wanjin Noh", + author_email="wanzargen@gmail.com", + license="Apache License 2.0", + packages=find_packages(), + install_requires=["spaceone-core", "spaceone-api", "spaceone-inventory"], + package_data={"plugin": ["metadata/organization/*.yaml"]}, + zip_safe=False, +)