From fc770622ff5c292f082bfb3b1021dd249dc35ced Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Mon, 13 Nov 2023 20:15:37 +0900 Subject: [PATCH 1/2] faet: add example --- src/plugin/connector/org_connector.py | 20 ++++++ src/plugin/main.py | 38 ++++------ src/plugin/manager/__init__.py | 13 ++++ src/plugin/manager/org_manager.py | 72 +++++++++++++++++++ src/plugin/manager/repository_manager.py | 72 +++++++++++++++++++ .../metadata/organizations/organization.yaml | 59 +++++++++++++++ 6 files changed, 250 insertions(+), 24 deletions(-) create mode 100644 src/plugin/connector/org_connector.py create mode 100644 src/plugin/manager/__init__.py create mode 100644 src/plugin/manager/org_manager.py create mode 100644 src/plugin/manager/repository_manager.py create mode 100644 src/plugin/metadata/organizations/organization.yaml diff --git a/src/plugin/connector/org_connector.py b/src/plugin/connector/org_connector.py new file mode 100644 index 0000000..121cb6e --- /dev/null +++ b/src/plugin/connector/org_connector.py @@ -0,0 +1,20 @@ +import requests +from ..connector import RequestConnector + + +class OrgConnector(RequestConnector): + def __init__(self): + super().__init__() + + def get_org(self, secret_data): + org_name = secret_data.get("org_name") + url = f"https://api.github.com/orgs/{org_name}" + headers = self.make_header(secret_data) + + return self.send_request(url, headers) + + def get_org_repos(self, secret_data): + org_name = secret_data.get("org_name") + url = f"https://api.github.com/orgs/{org_name}/repos" + headers = self.make_header(secret_data) + return self.send_request(url, headers, page=1, per_page=1) diff --git a/src/plugin/main.py b/src/plugin/main.py index 47466e8..03e69df 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,12 +1,15 @@ from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer +from .manager.org_manager import OrgManager +from .manager.repository_manager import RepositoryManager +from .manager import GithubBaseManager app = CollectorPluginServer() -@app.route('Collector.init') +@app.route("Collector.init") def collector_init(params: dict) -> dict: - """ init plugin by options + """init plugin by options Args: params (CollectorInitRequest): { @@ -19,30 +22,12 @@ def collector_init(params: dict) -> dict: 'metadata': 'dict' } """ - pass + return {"metadata": {}} -@app.route('Collector.verify') -def collector_verify(params: dict) -> None: - """ Verifying collector plugin - - Args: - params (CollectorVerifyRequest): { - 'options': 'dict', # Required - 'secret_data': 'dict', # Required - 'schema': 'str', - 'domain_id': 'str' - } - - Returns: - None - """ - pass - - -@app.route('Collector.collect') +@app.route("Collector.collect") def collector_collect(params: dict) -> dict: - """ Collect external data + """Collect external data Args: params (CollectorCollectRequest): { @@ -63,4 +48,9 @@ def collector_collect(params: dict) -> dict: 'metadata': 'dict' } """ - pass + + secret_data = params["secret_data"] + options = params["options"] + repo_manager = RepositoryManager() + + return repo_manager.collect_resources(options, secret_data, schema="") diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..030be02 --- /dev/null +++ b/src/plugin/manager/__init__.py @@ -0,0 +1,13 @@ +from spaceone.core.manager import BaseManager +from abc import abstractmethod, ABC, ABCMeta + + +class GithubBaseManager(BaseManager): + def __init__(self, *args, **kwargs): + super().__init__(args, kwargs) + + @classmethod + def get_managers(cls): + sub_classes = cls.__subclasses__() + print(sub_classes) + return sub_classes diff --git a/src/plugin/manager/org_manager.py b/src/plugin/manager/org_manager.py new file mode 100644 index 0000000..f7f458c --- /dev/null +++ b/src/plugin/manager/org_manager.py @@ -0,0 +1,72 @@ +import logging + +from spaceone.inventory.plugin.collector.lib import * +from ..manager import GithubBaseManager +from ..connector.org_connector import OrgConnector + +from abc import abstractmethod, ABC, ABCMeta + +_LOGGER = logging.getLogger("cloudforet") + + +class OrgManager(GithubBaseManager, metaclass=ABCMeta): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Organizations" + self.cloud_service_type = "Organization" + self.provider = "Github" + self.metadata_path = "plugin/metadata/organizations/organization.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + resource_type="inventory.CloudServiceType", + ) + + try: + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + org_connector = OrgConnector() + + for repo in org_connector.get_org(secret_data): + print(repo) + cloud_service = make_cloud_service( + name=self.cloud_service_type, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=repo, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) diff --git a/src/plugin/manager/repository_manager.py b/src/plugin/manager/repository_manager.py new file mode 100644 index 0000000..ec0d323 --- /dev/null +++ b/src/plugin/manager/repository_manager.py @@ -0,0 +1,72 @@ +import logging + +from spaceone.inventory.plugin.collector.lib import * +from ..manager import GithubBaseManager +from ..connector.org_connector import OrgConnector + +from abc import abstractmethod, ABC, ABCMeta + +_LOGGER = logging.getLogger("cloudforet") + + +class RepositoryManager(GithubBaseManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Organizations" + self.cloud_service_type = "Repository" + self.provider = "Github" + self.metadata_path = "plugin/metadata/organizations/organization.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + resource_type="inventory.CloudServiceType", + ) + + try: + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + org_connector = OrgConnector() + + for repo in org_connector.get_org_repos(secret_data): + print(repo) + cloud_service = make_cloud_service( + name=self.cloud_service_type, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=repo, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) diff --git a/src/plugin/metadata/organizations/organization.yaml b/src/plugin/metadata/organizations/organization.yaml new file mode 100644 index 0000000..b3ab616 --- /dev/null +++ b/src/plugin/metadata/organizations/organization.yaml @@ -0,0 +1,59 @@ +--- +search: + fields: + - Field ID: data.field + - Text Field: data.text + - State Field: data.state + - Badge Field: data.badge + - Datetime Field: data.datetime + - Image Field: data.image + - Enum Field: data.enum +table: + sort: + key: data.field + desc: true + fields: + - Field ID: data.field + - Text Field: data.text + - State Field: data.state + - Badge Field: data.badge + - Datetime Field: data.datetime + - Image Field: data.image + - Enum Field: data.enum +tabs.0: + name: Tap 1 + type: item + fields: + - Field ID: data.field + - Text Field: data.text + - State Field: data.state + - Badge Field: data.badge + - Datetime Field: data.datetime + - Image Field: data.image + - Enum Field: data.enum +tabs.1: + name: Tap 2 + type: list + items: + - key: data.member_id + name: Dynamic View 1 + type: item + fields: + - Field ID: data.field + - Text Field: data.text + - State Field: data.state + - Badge Field: data.badge + - Datetime Field: data.datetime + - Image Field: data.image + - Enum Field: data.enum + - key: data.part + name: Dynamic View 2 + type: simple-table + fields: + - Field ID: data.field + - Text Field: data.text + - State Field: data.state + - Badge Field: data.badge + - Datetime Field: data.datetime + - Image Field: data.image + - Enum Field: data.enum \ No newline at end of file From 2ad09d2c87d44c42a1921a2519a54a417599fe8b Mon Sep 17 00:00:00 2001 From: ImMin5 Date: Fri, 8 Dec 2023 21:46:38 +0900 Subject: [PATCH 2/2] faet: add base manager --- pkg/pip_requirements.txt | 3 +- src/VERSION | 1 + src/__init__.py | 1 + src/plugin/main.py | 96 +++++++++++++++++-- src/plugin/manager/__init__.py | 15 +-- src/plugin/manager/base.py | 22 +++++ src/plugin/manager/org_manager.py | 41 ++++---- src/plugin/manager/repository_manager.py | 9 +- .../metadata/organizations/organization.yaml | 56 +---------- src/setup.py | 35 +++++++ 10 files changed, 177 insertions(+), 102 deletions(-) create mode 100644 src/VERSION create mode 100644 src/plugin/manager/base.py create mode 100644 src/setup.py diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index fc9497f..93d4f63 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -1,2 +1 @@ -spaceone-core -spaceone-api \ No newline at end of file +spaceone-api diff --git a/src/VERSION b/src/VERSION new file mode 100644 index 0000000..60453e6 --- /dev/null +++ b/src/VERSION @@ -0,0 +1 @@ +v1.0.0 \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py index e69de29..8b13789 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/plugin/main.py b/src/plugin/main.py index 03e69df..f761cc5 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,8 +1,5 @@ from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer - -from .manager.org_manager import OrgManager -from .manager.repository_manager import RepositoryManager -from .manager import GithubBaseManager +from plugin.manager.base import GithubBaseManager app = CollectorPluginServer() @@ -25,6 +22,24 @@ def collector_init(params: dict) -> dict: return {"metadata": {}} +@app.route("Collector.verify") +def collector_verify(params: dict) -> None: + """Verifying collector plugin + + Args: + params (CollectorVerifyRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'schema': 'str', + 'domain_id': 'str' + } + + Returns: + None + """ + pass + + @app.route("Collector.collect") def collector_collect(params: dict) -> dict: """Collect external data @@ -34,6 +49,7 @@ def collector_collect(params: dict) -> dict: 'options': 'dict', # Required 'secret_data': 'dict', # Required 'schema': 'str', + 'task_options': 'dict', 'domain_id': 'str' } @@ -42,15 +58,81 @@ def collector_collect(params: dict) -> dict: { 'state': 'SUCCESS | FAILURE', 'resource_type': 'inventory.CloudService | inventory.CloudServiceType | inventory.Region', - 'resource_data': 'dict', + 'cloud_service_type': CloudServiceType, + 'cloud_service': CloudService, + 'region': Region, 'match_keys': 'list', 'error_message': 'str' 'metadata': 'dict' } + + CloudServiceType + { + 'name': 'str', # Required + 'group': 'str', # Required + 'provider': 'str', # Required + 'is_primary': 'bool', + 'is_major': 'bool', + 'metadata': 'dict', # Required + 'service_code': 'str', + 'tags': 'dict' + 'labels': 'list' + } + + CloudService + { + 'name': 'str', + 'cloud_service_type': 'str', # Required + 'cloud_service_group': 'str', # Required + 'provider': 'str', # Required + 'ip_addresses' : 'list', + 'account' : 'str', + 'instance_type': 'str', + 'instance_size': 'float', + 'region_code': 'str', + 'data': 'dict' # Required + 'metadata': 'dict' # Required + 'reference': 'dict' + 'tags' : 'dict' + } + + Region + { + 'name': 'str', + 'region_code': 'str', # Required + 'provider': 'str', # Required + 'tags': 'dict' + } + + Only one of the cloud_service_type, cloud_service and region fields is required. """ secret_data = params["secret_data"] options = params["options"] - repo_manager = RepositoryManager() + # options = {"cloud_service_types": ["Repository", "Organization"]} + + for cloud_service_manager_cls in GithubBaseManager.get_all_managers(options): + cloud_service_manager = cloud_service_manager_cls() + yield from cloud_service_manager.collect_resources( + options=options, secret_data=secret_data, schema="" + ) + + +@app.route("Job.get_tasks") +def job_get_tasks(params: dict) -> dict: + """Get job tasks + + Args: + params (JobGetTaskRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'domain_id': 'str' + } - return repo_manager.collect_resources(options, secret_data, schema="") + Returns: + TasksResponse: { + 'tasks': 'list' + } + + """ + pass diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py index 030be02..cb411f5 100644 --- a/src/plugin/manager/__init__.py +++ b/src/plugin/manager/__init__.py @@ -1,13 +1,2 @@ -from spaceone.core.manager import BaseManager -from abc import abstractmethod, ABC, ABCMeta - - -class GithubBaseManager(BaseManager): - def __init__(self, *args, **kwargs): - super().__init__(args, kwargs) - - @classmethod - def get_managers(cls): - sub_classes = cls.__subclasses__() - print(sub_classes) - return sub_classes +from .org_manager import OrganizationManager +from .repository_manager import RepositoryManager diff --git a/src/plugin/manager/base.py b/src/plugin/manager/base.py new file mode 100644 index 0000000..0f86c93 --- /dev/null +++ b/src/plugin/manager/base.py @@ -0,0 +1,22 @@ +from spaceone.core.manager import BaseManager + + +class GithubBaseManager(BaseManager): + def __init__(self, *args, **kwargs): + super().__init__(args, kwargs) + + @classmethod + def get_all_managers(cls, options): + cloud_service_types_option = options.get("cloud_service_types") + if cloud_service_types_option: + subclasses = [] + for subclass in cls.__subclasses__(): + if ( + subclass.__name__.replace("Manager", "") + in cloud_service_types_option + ): + subclasses.append(subclass) + return subclasses + + else: + return cls.__subclasses__() diff --git a/src/plugin/manager/org_manager.py b/src/plugin/manager/org_manager.py index f7f458c..a4d91c8 100644 --- a/src/plugin/manager/org_manager.py +++ b/src/plugin/manager/org_manager.py @@ -1,36 +1,24 @@ import logging +from plugin.manager.base import GithubBaseManager from spaceone.inventory.plugin.collector.lib import * -from ..manager import GithubBaseManager from ..connector.org_connector import OrgConnector -from abc import abstractmethod, ABC, ABCMeta - _LOGGER = logging.getLogger("cloudforet") -class OrgManager(GithubBaseManager, metaclass=ABCMeta): +class OrganizationManager(GithubBaseManager): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.cloud_service_group = "Organizations" self.cloud_service_type = "Organization" - self.provider = "Github" - self.metadata_path = "plugin/metadata/organizations/organization.yaml" + self.provider = "github_minho" + self.metadata_path = "metadata/organizations/organization.yaml" def collect_resources(self, options, secret_data, schema): try: yield from self.collect_cloud_service_type(options, secret_data, schema) - except Exception as e: - yield make_error_response( - error=e, - provider=self.provider, - cloud_service_group=self.cloud_service_group, - cloud_service_type=self.cloud_service_type, - resource_type="inventory.CloudServiceType", - ) - - try: yield from self.collect_cloud_service(options, secret_data, schema) except Exception as e: yield make_error_response( @@ -46,27 +34,34 @@ def collect_cloud_service_type(self, options, secret_data, schema): group=self.cloud_service_group, provider=self.provider, metadata_path=self.metadata_path, + is_primary=True, + is_major=True, ) yield make_response( cloud_service_type=cloud_service_type, - match_keys=[["name", "reference.resource_id", "account", "provider"]], + match_keys=[["name", "reference.resource_id", "provider"]], resource_type="inventory.CloudServiceType", ) def collect_cloud_service(self, options, secret_data, schema): org_connector = OrgConnector() - - for repo in org_connector.get_org(secret_data): - print(repo) + for org in org_connector.get_org(secret_data): cloud_service = make_cloud_service( - name=self.cloud_service_type, + name=org["name"], cloud_service_type=self.cloud_service_type, cloud_service_group=self.cloud_service_group, provider=self.provider, - data=repo, + data=org, ) yield make_response( cloud_service=cloud_service, - match_keys=[["name", "reference.resource_id", "account", "provider"]], + match_keys=[ + [ + "name", + "reference.resource_id", + "data.organization_name", + "provider", + ] + ], ) diff --git a/src/plugin/manager/repository_manager.py b/src/plugin/manager/repository_manager.py index ec0d323..d30ad03 100644 --- a/src/plugin/manager/repository_manager.py +++ b/src/plugin/manager/repository_manager.py @@ -1,11 +1,9 @@ import logging from spaceone.inventory.plugin.collector.lib import * -from ..manager import GithubBaseManager +from plugin.manager.base import GithubBaseManager from ..connector.org_connector import OrgConnector -from abc import abstractmethod, ABC, ABCMeta - _LOGGER = logging.getLogger("cloudforet") @@ -15,8 +13,8 @@ def __init__(self, *args, **kwargs): self.cloud_service_group = "Organizations" self.cloud_service_type = "Repository" - self.provider = "Github" - self.metadata_path = "plugin/metadata/organizations/organization.yaml" + self.provider = "github_minho" + self.metadata_path = "metadata/organizations/organization.yaml" def collect_resources(self, options, secret_data, schema): try: @@ -58,7 +56,6 @@ def collect_cloud_service(self, options, secret_data, schema): org_connector = OrgConnector() for repo in org_connector.get_org_repos(secret_data): - print(repo) cloud_service = make_cloud_service( name=self.cloud_service_type, cloud_service_type=self.cloud_service_type, diff --git a/src/plugin/metadata/organizations/organization.yaml b/src/plugin/metadata/organizations/organization.yaml index b3ab616..b88b895 100644 --- a/src/plugin/metadata/organizations/organization.yaml +++ b/src/plugin/metadata/organizations/organization.yaml @@ -1,59 +1,13 @@ --- search: fields: - - Field ID: data.field - - Text Field: data.text - - State Field: data.state - - Badge Field: data.badge - - Datetime Field: data.datetime - - Image Field: data.image - - Enum Field: data.enum + - Type: data.type + - Total Private Repo: data.total_private_repos + table: sort: key: data.field desc: true fields: - - Field ID: data.field - - Text Field: data.text - - State Field: data.state - - Badge Field: data.badge - - Datetime Field: data.datetime - - Image Field: data.image - - Enum Field: data.enum -tabs.0: - name: Tap 1 - type: item - fields: - - Field ID: data.field - - Text Field: data.text - - State Field: data.state - - Badge Field: data.badge - - Datetime Field: data.datetime - - Image Field: data.image - - Enum Field: data.enum -tabs.1: - name: Tap 2 - type: list - items: - - key: data.member_id - name: Dynamic View 1 - type: item - fields: - - Field ID: data.field - - Text Field: data.text - - State Field: data.state - - Badge Field: data.badge - - Datetime Field: data.datetime - - Image Field: data.image - - Enum Field: data.enum - - key: data.part - name: Dynamic View 2 - type: simple-table - fields: - - Field ID: data.field - - Text Field: data.text - - State Field: data.state - - Badge Field: data.badge - - Datetime Field: data.datetime - - Image Field: data.image - - Enum Field: data.enum \ No newline at end of file + - Type: data.type + - Total Private Repo: data.total_private_repos \ No newline at end of file diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..88673c1 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,35 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="MS Azure cloud service inventory collector", + long_description="", + url="https://cloudforet.io/", + author="Cloudforet Admin", + author_email="admin@cloudforet.io", + license="Apache License 2.0", + packages=find_packages(), + package_data={"plugin": ["metadata/*/*.yaml"]}, + zip_safe=False, +)