diff --git a/pkg/pip_requirements.txt b/pkg/pip_requirements.txt index fc9497f..93d4f63 100644 --- a/pkg/pip_requirements.txt +++ b/pkg/pip_requirements.txt @@ -1,2 +1 @@ -spaceone-core -spaceone-api \ No newline at end of file +spaceone-api diff --git a/src/VERSION b/src/VERSION new file mode 100644 index 0000000..60453e6 --- /dev/null +++ b/src/VERSION @@ -0,0 +1 @@ +v1.0.0 \ No newline at end of file diff --git a/src/__init__.py b/src/__init__.py index e69de29..8b13789 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/plugin/connector/org_connector.py b/src/plugin/connector/org_connector.py new file mode 100644 index 0000000..121cb6e --- /dev/null +++ b/src/plugin/connector/org_connector.py @@ -0,0 +1,20 @@ +import requests +from ..connector import RequestConnector + + +class OrgConnector(RequestConnector): + def __init__(self): + super().__init__() + + def get_org(self, secret_data): + org_name = secret_data.get("org_name") + url = f"https://api.github.com/orgs/{org_name}" + headers = self.make_header(secret_data) + + return self.send_request(url, headers) + + def get_org_repos(self, secret_data): + org_name = secret_data.get("org_name") + url = f"https://api.github.com/orgs/{org_name}/repos" + headers = self.make_header(secret_data) + return self.send_request(url, headers, page=1, per_page=1) diff --git a/src/plugin/main.py b/src/plugin/main.py index 47466e8..f761cc5 100644 --- a/src/plugin/main.py +++ b/src/plugin/main.py @@ -1,12 +1,12 @@ from spaceone.inventory.plugin.collector.lib.server import CollectorPluginServer - +from plugin.manager.base import GithubBaseManager app = CollectorPluginServer() -@app.route('Collector.init') +@app.route("Collector.init") def collector_init(params: dict) -> dict: - """ init plugin by options + """init plugin by options Args: params (CollectorInitRequest): { @@ -19,12 +19,12 @@ def collector_init(params: dict) -> dict: 'metadata': 'dict' } """ - pass + return {"metadata": {}} -@app.route('Collector.verify') +@app.route("Collector.verify") def collector_verify(params: dict) -> None: - """ Verifying collector plugin + """Verifying collector plugin Args: params (CollectorVerifyRequest): { @@ -40,15 +40,16 @@ def collector_verify(params: dict) -> None: pass -@app.route('Collector.collect') +@app.route("Collector.collect") def collector_collect(params: dict) -> dict: - """ Collect external data + """Collect external data Args: params (CollectorCollectRequest): { 'options': 'dict', # Required 'secret_data': 'dict', # Required 'schema': 'str', + 'task_options': 'dict', 'domain_id': 'str' } @@ -57,10 +58,81 @@ def collector_collect(params: dict) -> dict: { 'state': 'SUCCESS | FAILURE', 'resource_type': 'inventory.CloudService | inventory.CloudServiceType | inventory.Region', - 'resource_data': 'dict', + 'cloud_service_type': CloudServiceType, + 'cloud_service': CloudService, + 'region': Region, 'match_keys': 'list', 'error_message': 'str' 'metadata': 'dict' } + + CloudServiceType + { + 'name': 'str', # Required + 'group': 'str', # Required + 'provider': 'str', # Required + 'is_primary': 'bool', + 'is_major': 'bool', + 'metadata': 'dict', # Required + 'service_code': 'str', + 'tags': 'dict' + 'labels': 'list' + } + + CloudService + { + 'name': 'str', + 'cloud_service_type': 'str', # Required + 'cloud_service_group': 'str', # Required + 'provider': 'str', # Required + 'ip_addresses' : 'list', + 'account' : 'str', + 'instance_type': 'str', + 'instance_size': 'float', + 'region_code': 'str', + 'data': 'dict' # Required + 'metadata': 'dict' # Required + 'reference': 'dict' + 'tags' : 'dict' + } + + Region + { + 'name': 'str', + 'region_code': 'str', # Required + 'provider': 'str', # Required + 'tags': 'dict' + } + + Only one of the cloud_service_type, cloud_service and region fields is required. + """ + + secret_data = params["secret_data"] + options = params["options"] + # options = {"cloud_service_types": ["Repository", "Organization"]} + + for cloud_service_manager_cls in GithubBaseManager.get_all_managers(options): + cloud_service_manager = cloud_service_manager_cls() + yield from cloud_service_manager.collect_resources( + options=options, secret_data=secret_data, schema="" + ) + + +@app.route("Job.get_tasks") +def job_get_tasks(params: dict) -> dict: + """Get job tasks + + Args: + params (JobGetTaskRequest): { + 'options': 'dict', # Required + 'secret_data': 'dict', # Required + 'domain_id': 'str' + } + + Returns: + TasksResponse: { + 'tasks': 'list' + } + """ pass diff --git a/src/plugin/manager/__init__.py b/src/plugin/manager/__init__.py new file mode 100644 index 0000000..cb411f5 --- /dev/null +++ b/src/plugin/manager/__init__.py @@ -0,0 +1,2 @@ +from .org_manager import OrganizationManager +from .repository_manager import RepositoryManager diff --git a/src/plugin/manager/base.py b/src/plugin/manager/base.py new file mode 100644 index 0000000..0f86c93 --- /dev/null +++ b/src/plugin/manager/base.py @@ -0,0 +1,22 @@ +from spaceone.core.manager import BaseManager + + +class GithubBaseManager(BaseManager): + def __init__(self, *args, **kwargs): + super().__init__(args, kwargs) + + @classmethod + def get_all_managers(cls, options): + cloud_service_types_option = options.get("cloud_service_types") + if cloud_service_types_option: + subclasses = [] + for subclass in cls.__subclasses__(): + if ( + subclass.__name__.replace("Manager", "") + in cloud_service_types_option + ): + subclasses.append(subclass) + return subclasses + + else: + return cls.__subclasses__() diff --git a/src/plugin/manager/org_manager.py b/src/plugin/manager/org_manager.py new file mode 100644 index 0000000..a4d91c8 --- /dev/null +++ b/src/plugin/manager/org_manager.py @@ -0,0 +1,67 @@ +import logging + +from plugin.manager.base import GithubBaseManager +from spaceone.inventory.plugin.collector.lib import * +from ..connector.org_connector import OrgConnector + +_LOGGER = logging.getLogger("cloudforet") + + +class OrganizationManager(GithubBaseManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Organizations" + self.cloud_service_type = "Organization" + self.provider = "github_minho" + self.metadata_path = "metadata/organizations/organization.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + is_primary=True, + is_major=True, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + org_connector = OrgConnector() + for org in org_connector.get_org(secret_data): + cloud_service = make_cloud_service( + name=org["name"], + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=org, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[ + [ + "name", + "reference.resource_id", + "data.organization_name", + "provider", + ] + ], + ) diff --git a/src/plugin/manager/repository_manager.py b/src/plugin/manager/repository_manager.py new file mode 100644 index 0000000..d30ad03 --- /dev/null +++ b/src/plugin/manager/repository_manager.py @@ -0,0 +1,69 @@ +import logging + +from spaceone.inventory.plugin.collector.lib import * +from plugin.manager.base import GithubBaseManager +from ..connector.org_connector import OrgConnector + +_LOGGER = logging.getLogger("cloudforet") + + +class RepositoryManager(GithubBaseManager): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.cloud_service_group = "Organizations" + self.cloud_service_type = "Repository" + self.provider = "github_minho" + self.metadata_path = "metadata/organizations/organization.yaml" + + def collect_resources(self, options, secret_data, schema): + try: + yield from self.collect_cloud_service_type(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + resource_type="inventory.CloudServiceType", + ) + + try: + yield from self.collect_cloud_service(options, secret_data, schema) + except Exception as e: + yield make_error_response( + error=e, + provider=self.provider, + cloud_service_group=self.cloud_service_group, + cloud_service_type=self.cloud_service_type, + ) + + def collect_cloud_service_type(self, options, secret_data, schema): + cloud_service_type = make_cloud_service_type( + name=self.cloud_service_type, + group=self.cloud_service_group, + provider=self.provider, + metadata_path=self.metadata_path, + ) + + yield make_response( + cloud_service_type=cloud_service_type, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + resource_type="inventory.CloudServiceType", + ) + + def collect_cloud_service(self, options, secret_data, schema): + org_connector = OrgConnector() + + for repo in org_connector.get_org_repos(secret_data): + cloud_service = make_cloud_service( + name=self.cloud_service_type, + cloud_service_type=self.cloud_service_type, + cloud_service_group=self.cloud_service_group, + provider=self.provider, + data=repo, + ) + yield make_response( + cloud_service=cloud_service, + match_keys=[["name", "reference.resource_id", "account", "provider"]], + ) diff --git a/src/plugin/metadata/organizations/organization.yaml b/src/plugin/metadata/organizations/organization.yaml new file mode 100644 index 0000000..b88b895 --- /dev/null +++ b/src/plugin/metadata/organizations/organization.yaml @@ -0,0 +1,13 @@ +--- +search: + fields: + - Type: data.type + - Total Private Repo: data.total_private_repos + +table: + sort: + key: data.field + desc: true + fields: + - Type: data.type + - Total Private Repo: data.total_private_repos \ No newline at end of file diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..88673c1 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,35 @@ +# +# Copyright 2020 The SpaceONE Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages + +with open("VERSION", "r") as f: + VERSION = f.read().strip() + f.close() + +setup( + name="plugin-github-inven-collector", + version=VERSION, + description="MS Azure cloud service inventory collector", + long_description="", + url="https://cloudforet.io/", + author="Cloudforet Admin", + author_email="admin@cloudforet.io", + license="Apache License 2.0", + packages=find_packages(), + package_data={"plugin": ["metadata/*/*.yaml"]}, + zip_safe=False, +)