From 822ed8c45c075ac6655bef1d99f6cc00c42301bb Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 18 Aug 2023 14:46:58 +0000 Subject: [PATCH 01/15] modification for reaxflow-workflow-service --- marketplace/app/v0/object_storage.py | 34 ++++++++++++++ marketplace/app/v0/transformation.py | 46 +++++++++++++++++++ .../README.md | 0 .../__init__.py | 0 .../cli.py | 0 .../session.py | 0 .../utils.py | 0 7 files changed, 80 insertions(+) rename marketplace/{datasink_client => data_sink_client}/README.md (100%) rename marketplace/{datasink_client => data_sink_client}/__init__.py (100%) rename marketplace/{datasink_client => data_sink_client}/cli.py (100%) rename marketplace/{datasink_client => data_sink_client}/session.py (100%) rename marketplace/{datasink_client => data_sink_client}/utils.py (100%) diff --git a/marketplace/app/v0/object_storage.py b/marketplace/app/v0/object_storage.py index a8bdc06..d35c6cd 100644 --- a/marketplace/app/v0/object_storage.py +++ b/marketplace/app/v0/object_storage.py @@ -1,5 +1,6 @@ import json from typing import Dict, Union +from uuid import UUID import marketplace_standard_app_api.models.object_storage as object_storage from fastapi import UploadFile @@ -281,3 +282,36 @@ def query(self, query: str, meta_data: bool = False) -> Union[Dict, str]: self._proxy_path("query"), json={"query": query, "meta_data": meta_data} ).text return response + + @check_capability_availability + def upload_data_cache( + self, + file: UploadFile, + uuid: UUID = None, + metadata: dict = None, + ) -> object_storage.UploadDataResponse: + data = {} + if uuid: + data.update({"uuid": "aa"}) + response = self._client.post( + self._proxy_path("uploadDataCache"), + data=data, + files=file, + headers=_encode_metadata(metadata) if metadata else {}, + ) + try: + return object_storage.UploadDataResponse.parse_obj(response.json()).dict() + except Exception: + return "Error: Server returned {} while creating dataset {}: {}".format( + response.status_code, uuid, response.text + ) + + @check_capability_availability + def download_data_cache( + self, + uuid: UUID, + ): + return self._client.get( + self._proxy_path("downloadDataCache"), + params={"uuid": uuid}, + ) diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index d02c1f2..24af800 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -1,4 +1,6 @@ import marketplace_standard_app_api.models.transformation as transformation +from fastapi.responses import JSONResponse, Response +from pydantic.schema import schema from ..utils import check_capability_availability from .base import _MarketPlaceAppBase @@ -102,3 +104,47 @@ def get_transformation_state( params={"transformation_id": transformation_id}, ).json() ).state + + @check_capability_availability + def get_transformation_log( + self, transformation_id: transformation.TransformationId + ) -> Response: + return self._client.get( + self._proxy_path("getTransformationLog"), + params={"transformation_id": transformation_id}, + ).content + + @check_capability_availability + def new_model( + self, modelname: transformation.ModelName, data: dict + ) -> transformation.ModelCreateResponse: + return transformation.ModelCreateResponse.parse_obj( + self._client.post( + self._proxy_path("newModel"), params={"modelname": modelname}, json=data + ).json() + ) + + @check_capability_availability + def get_schema(self, modelname: transformation.ModelName) -> schema: + return self._client.get( + self._proxy_path("getSchema"), + params={"modelname": modelname}, + ).json() + + @check_capability_availability + def get_example(self, modelname: transformation.ModelName) -> JSONResponse: + return self._client.get( + self._proxy_path("getExample"), + params={"modelname": modelname}, + ).json() + + @check_capability_availability + def get_models( + self, limit: int = 100, offset: int = 0 + ) -> transformation.RegisteredModels: + return transformation.RegisteredModels.parse_obj( + self._client.get( + self._proxy_path("getModels"), + params={"limit": limit, "offset": offset}, + ).json() + ) diff --git a/marketplace/datasink_client/README.md b/marketplace/data_sink_client/README.md similarity index 100% rename from marketplace/datasink_client/README.md rename to marketplace/data_sink_client/README.md diff --git a/marketplace/datasink_client/__init__.py b/marketplace/data_sink_client/__init__.py similarity index 100% rename from marketplace/datasink_client/__init__.py rename to marketplace/data_sink_client/__init__.py diff --git a/marketplace/datasink_client/cli.py b/marketplace/data_sink_client/cli.py similarity index 100% rename from marketplace/datasink_client/cli.py rename to marketplace/data_sink_client/cli.py diff --git a/marketplace/datasink_client/session.py b/marketplace/data_sink_client/session.py similarity index 100% rename from marketplace/datasink_client/session.py rename to marketplace/data_sink_client/session.py diff --git a/marketplace/datasink_client/utils.py b/marketplace/data_sink_client/utils.py similarity index 100% rename from marketplace/datasink_client/utils.py rename to marketplace/data_sink_client/utils.py From 0f01a5210abde327dd609e897e65fd656e953ead Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Tue, 22 Aug 2023 16:13:49 +0000 Subject: [PATCH 02/15] remove unused functions --- marketplace/app/v0/object_storage.py | 34 ---------------------------- 1 file changed, 34 deletions(-) diff --git a/marketplace/app/v0/object_storage.py b/marketplace/app/v0/object_storage.py index d35c6cd..a8bdc06 100644 --- a/marketplace/app/v0/object_storage.py +++ b/marketplace/app/v0/object_storage.py @@ -1,6 +1,5 @@ import json from typing import Dict, Union -from uuid import UUID import marketplace_standard_app_api.models.object_storage as object_storage from fastapi import UploadFile @@ -282,36 +281,3 @@ def query(self, query: str, meta_data: bool = False) -> Union[Dict, str]: self._proxy_path("query"), json={"query": query, "meta_data": meta_data} ).text return response - - @check_capability_availability - def upload_data_cache( - self, - file: UploadFile, - uuid: UUID = None, - metadata: dict = None, - ) -> object_storage.UploadDataResponse: - data = {} - if uuid: - data.update({"uuid": "aa"}) - response = self._client.post( - self._proxy_path("uploadDataCache"), - data=data, - files=file, - headers=_encode_metadata(metadata) if metadata else {}, - ) - try: - return object_storage.UploadDataResponse.parse_obj(response.json()).dict() - except Exception: - return "Error: Server returned {} while creating dataset {}: {}".format( - response.status_code, uuid, response.text - ) - - @check_capability_availability - def download_data_cache( - self, - uuid: UUID, - ): - return self._client.get( - self._proxy_path("downloadDataCache"), - params={"uuid": uuid}, - ) From 0581c4e9d2728892df1b274025e50e332e80ed9a Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Wed, 23 Aug 2023 16:08:31 +0000 Subject: [PATCH 03/15] review changes --- marketplace/app/v0/object_storage.py | 12 +++++++----- marketplace/app/v0/transformation.py | 22 ++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/marketplace/app/v0/object_storage.py b/marketplace/app/v0/object_storage.py index a8bdc06..5545e01 100644 --- a/marketplace/app/v0/object_storage.py +++ b/marketplace/app/v0/object_storage.py @@ -8,6 +8,8 @@ from .base import _MarketPlaceAppBase from .utils import _decode_metadata, _encode_metadata +DEFAULT_COLLECTION_NAME = "DEFAULT_COLLECTION" + class MarketPlaceObjectStorageApp(_MarketPlaceAppBase): @check_capability_availability @@ -106,7 +108,7 @@ def create_collection( @check_capability_availability def create_dataset( self, - collection_name: object_storage.CollectionName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, dataset_name: object_storage.DatasetName = None, metadata: dict = None, file: UploadFile = None, @@ -153,8 +155,8 @@ def create_dataset_metadata( @check_capability_availability def get_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: return self._client.get( self._proxy_path("getDataset"), @@ -196,8 +198,8 @@ def create_or_replace_dataset_metadata( @check_capability_availability def delete_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ): return self._client.delete( self._proxy_path("deleteDataset"), @@ -252,8 +254,8 @@ def get_collection_metadata_dcat( @check_capability_availability def get_dataset_metadata_dcat( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: response: dict = self._client.get( self._proxy_path("getDatasetMetadataDcat"), @@ -264,9 +266,9 @@ def get_dataset_metadata_dcat( @check_capability_availability def query_dataset( self, - collection_name: object_storage.CollectionName, dataset_name: object_storage.DatasetName, query: str, + collection_name: object_storage.CollectionName = DEFAULT_COLLECTION_NAME, ) -> Union[Dict, str]: response: dict = self._client.post( self._proxy_path("queryDataset"), diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index 24af800..edf3e29 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -20,11 +20,19 @@ def get_transformation_list( @check_capability_availability def new_transformation( - self, new_transformation: transformation.NewTransformationModel + self, + new_transformation: transformation.NewTransformationModel, + config: dict = None, ) -> transformation.TransformationCreateResponse: + params = {} + # send additional key value as query parameters if some app needs it + if config is not None: + params.update(config) return transformation.TransformationCreateResponse.parse_obj( self._client.post( - self._proxy_path("newTransformation"), json=new_transformation + self._proxy_path("newTransformation"), + json=new_transformation, + params=params, ).json() ) @@ -114,16 +122,6 @@ def get_transformation_log( params={"transformation_id": transformation_id}, ).content - @check_capability_availability - def new_model( - self, modelname: transformation.ModelName, data: dict - ) -> transformation.ModelCreateResponse: - return transformation.ModelCreateResponse.parse_obj( - self._client.post( - self._proxy_path("newModel"), params={"modelname": modelname}, json=data - ).json() - ) - @check_capability_availability def get_schema(self, modelname: transformation.ModelName) -> schema: return self._client.get( From d79d05216b034bd9ec45af99b7d01ceacc9d5c53 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Thu, 24 Aug 2023 08:50:21 +0000 Subject: [PATCH 04/15] few more review changes --- marketplace/app/v0/transformation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index edf3e29..98f6a3d 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -123,17 +123,17 @@ def get_transformation_log( ).content @check_capability_availability - def get_schema(self, modelname: transformation.ModelName) -> schema: + def get_schema(self, model_name: transformation.ModelName) -> schema: return self._client.get( self._proxy_path("getSchema"), - params={"modelname": modelname}, + params={"model_name": model_name}, ).json() @check_capability_availability - def get_example(self, modelname: transformation.ModelName) -> JSONResponse: + def get_example(self, model_name: transformation.ModelName) -> JSONResponse: return self._client.get( self._proxy_path("getExample"), - params={"modelname": modelname}, + params={"model_name": model_name}, ).json() @check_capability_availability From 4be4c9f5090388a2406e275d593999856ba4a4c5 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Wed, 30 Aug 2023 13:32:59 +0000 Subject: [PATCH 05/15] reaxpro workflow changes --- .../collection_dcat.py | 0 .../dataset_dcat.py | 0 .../delete_collection.py | 0 .../delete_dataset.py | 0 .../download_file.py | 0 .../download_folder.py | 0 .../list_collections.py | 0 .../list_datasets.py | 0 .../query.py | 0 .../query_dataset.py | 0 .../upload_files_from_path.py | 0 .../upload_folder.py | 0 marketplace/app/v0/system.py | 21 +++++++++++ marketplace/app/v0/transformation.py | 36 ------------------- marketplace/data_sink_client/session.py | 11 +++--- 15 files changed, 25 insertions(+), 43 deletions(-) rename examples/{datasink_client => data_sink_client}/collection_dcat.py (100%) rename examples/{datasink_client => data_sink_client}/dataset_dcat.py (100%) rename examples/{datasink_client => data_sink_client}/delete_collection.py (100%) rename examples/{datasink_client => data_sink_client}/delete_dataset.py (100%) rename examples/{datasink_client => data_sink_client}/download_file.py (100%) rename examples/{datasink_client => data_sink_client}/download_folder.py (100%) rename examples/{datasink_client => data_sink_client}/list_collections.py (100%) rename examples/{datasink_client => data_sink_client}/list_datasets.py (100%) rename examples/{datasink_client => data_sink_client}/query.py (100%) rename examples/{datasink_client => data_sink_client}/query_dataset.py (100%) rename examples/{datasink_client => data_sink_client}/upload_files_from_path.py (100%) rename examples/{datasink_client => data_sink_client}/upload_folder.py (100%) create mode 100644 marketplace/app/v0/system.py diff --git a/examples/datasink_client/collection_dcat.py b/examples/data_sink_client/collection_dcat.py similarity index 100% rename from examples/datasink_client/collection_dcat.py rename to examples/data_sink_client/collection_dcat.py diff --git a/examples/datasink_client/dataset_dcat.py b/examples/data_sink_client/dataset_dcat.py similarity index 100% rename from examples/datasink_client/dataset_dcat.py rename to examples/data_sink_client/dataset_dcat.py diff --git a/examples/datasink_client/delete_collection.py b/examples/data_sink_client/delete_collection.py similarity index 100% rename from examples/datasink_client/delete_collection.py rename to examples/data_sink_client/delete_collection.py diff --git a/examples/datasink_client/delete_dataset.py b/examples/data_sink_client/delete_dataset.py similarity index 100% rename from examples/datasink_client/delete_dataset.py rename to examples/data_sink_client/delete_dataset.py diff --git a/examples/datasink_client/download_file.py b/examples/data_sink_client/download_file.py similarity index 100% rename from examples/datasink_client/download_file.py rename to examples/data_sink_client/download_file.py diff --git a/examples/datasink_client/download_folder.py b/examples/data_sink_client/download_folder.py similarity index 100% rename from examples/datasink_client/download_folder.py rename to examples/data_sink_client/download_folder.py diff --git a/examples/datasink_client/list_collections.py b/examples/data_sink_client/list_collections.py similarity index 100% rename from examples/datasink_client/list_collections.py rename to examples/data_sink_client/list_collections.py diff --git a/examples/datasink_client/list_datasets.py b/examples/data_sink_client/list_datasets.py similarity index 100% rename from examples/datasink_client/list_datasets.py rename to examples/data_sink_client/list_datasets.py diff --git a/examples/datasink_client/query.py b/examples/data_sink_client/query.py similarity index 100% rename from examples/datasink_client/query.py rename to examples/data_sink_client/query.py diff --git a/examples/datasink_client/query_dataset.py b/examples/data_sink_client/query_dataset.py similarity index 100% rename from examples/datasink_client/query_dataset.py rename to examples/data_sink_client/query_dataset.py diff --git a/examples/datasink_client/upload_files_from_path.py b/examples/data_sink_client/upload_files_from_path.py similarity index 100% rename from examples/datasink_client/upload_files_from_path.py rename to examples/data_sink_client/upload_files_from_path.py diff --git a/examples/datasink_client/upload_folder.py b/examples/data_sink_client/upload_folder.py similarity index 100% rename from examples/datasink_client/upload_folder.py rename to examples/data_sink_client/upload_folder.py diff --git a/marketplace/app/v0/system.py b/marketplace/app/v0/system.py new file mode 100644 index 0000000..b18c65d --- /dev/null +++ b/marketplace/app/v0/system.py @@ -0,0 +1,21 @@ +from typing import Optional + +from fastapi.responses import JSONResponse, Response + +from ..utils import check_capability_availability + + +@check_capability_availability +def get_logs(self, id: Optional[str], limit: int = 100, offset: int = 0) -> Response: + return self._client.get( + self._proxy_path("getLogs"), + params={"id": id, "limit": limit, "offset": offset}, + ).content + + +@check_capability_availability +def get_info(self, config: dict = None) -> JSONResponse: + params = {} + if config is not None: + params.update(config) + return self._client.get(self._proxy_path("getInfo"), params=params).json() diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index 98f6a3d..ee6e259 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -1,6 +1,4 @@ import marketplace_standard_app_api.models.transformation as transformation -from fastapi.responses import JSONResponse, Response -from pydantic.schema import schema from ..utils import check_capability_availability from .base import _MarketPlaceAppBase @@ -112,37 +110,3 @@ def get_transformation_state( params={"transformation_id": transformation_id}, ).json() ).state - - @check_capability_availability - def get_transformation_log( - self, transformation_id: transformation.TransformationId - ) -> Response: - return self._client.get( - self._proxy_path("getTransformationLog"), - params={"transformation_id": transformation_id}, - ).content - - @check_capability_availability - def get_schema(self, model_name: transformation.ModelName) -> schema: - return self._client.get( - self._proxy_path("getSchema"), - params={"model_name": model_name}, - ).json() - - @check_capability_availability - def get_example(self, model_name: transformation.ModelName) -> JSONResponse: - return self._client.get( - self._proxy_path("getExample"), - params={"model_name": model_name}, - ).json() - - @check_capability_availability - def get_models( - self, limit: int = 100, offset: int = 0 - ) -> transformation.RegisteredModels: - return transformation.RegisteredModels.parse_obj( - self._client.get( - self._proxy_path("getModels"), - params={"limit": limit, "offset": offset}, - ).json() - ) diff --git a/marketplace/data_sink_client/session.py b/marketplace/data_sink_client/session.py index 8b1f331..9af48b9 100644 --- a/marketplace/data_sink_client/session.py +++ b/marketplace/data_sink_client/session.py @@ -138,10 +138,9 @@ def create_collection(self, collection_name, sub_collection_id): collection_name=collection_name, config=config ) if "collection_id" not in response: - print(response) return None else: - return response["collection_id"] + return response except Exception as e: print( @@ -294,13 +293,11 @@ def create_datasets_from_paths(self, paths, collection_name, dataset_names): response_list = [] if collection_name is not None: - collection_id = self.create_collection( + response = self.create_collection( collection_name=collection_name, sub_collection_id=None ) - if collection_id is not None: - response_list.append((collection_name, collection_id)) - else: - return + if response is not None: + response_list.append((collection_name, response["collection_id"])) else: raise Exception("collection title cannot be empty.") From a5f11e4d0f231b9018f5081e6868e9f0f43df7e4 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Mon, 4 Sep 2023 14:50:19 +0000 Subject: [PATCH 06/15] fix for query bug --- marketplace/data_sink_client/session.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/marketplace/data_sink_client/session.py b/marketplace/data_sink_client/session.py index 9af48b9..a117a9d 100644 --- a/marketplace/data_sink_client/session.py +++ b/marketplace/data_sink_client/session.py @@ -244,7 +244,9 @@ def query_dataset(self, collection_name, dataset_name, query): :returns: List of data """ - response = self.marketPlace.query_dataset(collection_name, dataset_name, query) + response = self.marketPlace.query_dataset( + collection_name=collection_name, dataset_name=dataset_name, query=query + ) return response @reconfigure_if_expired From 8dd7267ebda1a214ce9b600de8b921ce918eb63a Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Thu, 7 Sep 2023 15:36:51 +0000 Subject: [PATCH 07/15] inherit missing system class in marketplace app --- marketplace/app/v0/__init__.py | 5 ++++- marketplace/app/v0/system.py | 29 ++++++++++++++++------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/marketplace/app/v0/__init__.py b/marketplace/app/v0/__init__.py index 9fa95ef..9178003 100644 --- a/marketplace/app/v0/__init__.py +++ b/marketplace/app/v0/__init__.py @@ -1,6 +1,9 @@ from .object_storage import MarketPlaceObjectStorageApp +from .system import MarketPlaceSystemApp from .transformation import MarketPlaceTransformationApp -class MarketPlaceApp(MarketPlaceObjectStorageApp, MarketPlaceTransformationApp): +class MarketPlaceApp( + MarketPlaceObjectStorageApp, MarketPlaceTransformationApp, MarketPlaceSystemApp +): pass diff --git a/marketplace/app/v0/system.py b/marketplace/app/v0/system.py index b18c65d..66346ec 100644 --- a/marketplace/app/v0/system.py +++ b/marketplace/app/v0/system.py @@ -3,19 +3,22 @@ from fastapi.responses import JSONResponse, Response from ..utils import check_capability_availability +from .base import _MarketPlaceAppBase -@check_capability_availability -def get_logs(self, id: Optional[str], limit: int = 100, offset: int = 0) -> Response: - return self._client.get( - self._proxy_path("getLogs"), - params={"id": id, "limit": limit, "offset": offset}, - ).content +class MarketPlaceSystemApp(_MarketPlaceAppBase): + @check_capability_availability + def get_logs( + self, id: Optional[str], limit: int = 100, offset: int = 0 + ) -> Response: + return self._client.get( + self._proxy_path("getLogs"), + params={"id": id, "limit": limit, "offset": offset}, + ).content - -@check_capability_availability -def get_info(self, config: dict = None) -> JSONResponse: - params = {} - if config is not None: - params.update(config) - return self._client.get(self._proxy_path("getInfo"), params=params).json() + @check_capability_availability + def get_info(self, config: dict = None) -> JSONResponse: + params = {} + if config is not None: + params.update(config) + return self._client.get(self._proxy_path("getInfo"), params=params).json() From c2907eb50e523e2683b9866ff956614d9737c936 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 8 Sep 2023 09:07:38 +0000 Subject: [PATCH 08/15] move system file content to base --- marketplace/app/v0/base.py | 18 +++++++++++++++++- marketplace/app/v0/system.py | 24 ------------------------ 2 files changed, 17 insertions(+), 25 deletions(-) delete mode 100644 marketplace/app/v0/system.py diff --git a/marketplace/app/v0/base.py b/marketplace/app/v0/base.py index 253ac07..a2ed6b8 100644 --- a/marketplace/app/v0/base.py +++ b/marketplace/app/v0/base.py @@ -1,7 +1,7 @@ from typing import Optional from urllib.parse import urljoin -from fastapi.responses import HTMLResponse +from fastapi.responses import HTMLResponse, JSONResponse, Response from marketplace_standard_app_api.models.system import GlobalSearchResponse from marketplace.client import MarketPlaceClient @@ -41,3 +41,19 @@ def global_search( params={"q": q, "limit": limit, "offset": offset}, ).json() ) + + @check_capability_availability + def get_logs( + self, id: Optional[str], limit: int = 100, offset: int = 0 + ) -> Response: + return self._client.get( + self._proxy_path("getLogs"), + params={"id": id, "limit": limit, "offset": offset}, + ).content + + @check_capability_availability + def get_info(self, config: dict = None) -> JSONResponse: + params = {} + if config is not None: + params.update(config) + return self._client.get(self._proxy_path("getInfo"), params=params).json() diff --git a/marketplace/app/v0/system.py b/marketplace/app/v0/system.py deleted file mode 100644 index 66346ec..0000000 --- a/marketplace/app/v0/system.py +++ /dev/null @@ -1,24 +0,0 @@ -from typing import Optional - -from fastapi.responses import JSONResponse, Response - -from ..utils import check_capability_availability -from .base import _MarketPlaceAppBase - - -class MarketPlaceSystemApp(_MarketPlaceAppBase): - @check_capability_availability - def get_logs( - self, id: Optional[str], limit: int = 100, offset: int = 0 - ) -> Response: - return self._client.get( - self._proxy_path("getLogs"), - params={"id": id, "limit": limit, "offset": offset}, - ).content - - @check_capability_availability - def get_info(self, config: dict = None) -> JSONResponse: - params = {} - if config is not None: - params.update(config) - return self._client.get(self._proxy_path("getInfo"), params=params).json() From f1d6efea88e372d59466a9b05a42418ba9f097a7 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 8 Sep 2023 09:12:31 +0000 Subject: [PATCH 09/15] remove the system import --- marketplace/app/v0/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/marketplace/app/v0/__init__.py b/marketplace/app/v0/__init__.py index 9178003..9fa95ef 100644 --- a/marketplace/app/v0/__init__.py +++ b/marketplace/app/v0/__init__.py @@ -1,9 +1,6 @@ from .object_storage import MarketPlaceObjectStorageApp -from .system import MarketPlaceSystemApp from .transformation import MarketPlaceTransformationApp -class MarketPlaceApp( - MarketPlaceObjectStorageApp, MarketPlaceTransformationApp, MarketPlaceSystemApp -): +class MarketPlaceApp(MarketPlaceObjectStorageApp, MarketPlaceTransformationApp): pass From ee346794ed7740d2703946fa2bf97900519c3150 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 8 Sep 2023 10:02:09 +0000 Subject: [PATCH 10/15] update standard app version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index d2da37b..a160cbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ classifiers = packages = find: install_requires = fastapi>0.75,<=0.95.1 - marketplace-standard-app-api~=0.4 + marketplace-standard-app-api~=0.6.0 packaging>=21.3,<=23.0 pika~=1.2 python-keycloak==2.12.0 From 832aeaf8a42520cc39c6fa4fdf8ad8cba5dd6a16 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 8 Sep 2023 10:04:31 +0000 Subject: [PATCH 11/15] update standard app version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index a160cbf..3d40a49 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ classifiers = packages = find: install_requires = fastapi>0.75,<=0.95.1 - marketplace-standard-app-api~=0.6.0 + marketplace-standard-app-api~=0.6 packaging>=21.3,<=23.0 pika~=1.2 python-keycloak==2.12.0 From b579abe8f64bcc99503056424337611942d602fc Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Mon, 18 Sep 2023 14:48:33 +0000 Subject: [PATCH 12/15] keycloak authentication and datasink rename --- .env_template | 6 +- examples/data_sink_client/query.py | 6 -- examples/data_sink_client/query_dataset.py | 6 -- .../collection_dcat.py | 2 +- .../dataset_dcat.py | 2 +- .../delete_collection.py | 2 +- .../delete_dataset.py | 2 +- .../download_file.py | 2 +- .../download_folder.py | 2 +- .../list_collections.py | 2 +- .../list_datasets.py | 2 +- examples/datasink_client/query.py | 14 ++++ examples/datasink_client/query_dataset.py | 16 +++++ .../upload_files_from_path.py | 2 +- .../upload_folder.py | 2 +- marketplace/client.py | 53 +++++++++++++- .../README.md | 0 .../__init__.py | 0 .../cli.py | 2 +- .../session.py | 69 +------------------ .../utils.py | 0 setup.cfg | 24 +++---- 22 files changed, 110 insertions(+), 106 deletions(-) delete mode 100644 examples/data_sink_client/query.py delete mode 100644 examples/data_sink_client/query_dataset.py rename examples/{data_sink_client => datasink_client}/collection_dcat.py (64%) rename examples/{data_sink_client => datasink_client}/dataset_dcat.py (67%) rename examples/{data_sink_client => datasink_client}/delete_collection.py (64%) rename examples/{data_sink_client => datasink_client}/delete_dataset.py (67%) rename examples/{data_sink_client => datasink_client}/download_file.py (81%) rename examples/{data_sink_client => datasink_client}/download_folder.py (80%) rename examples/{data_sink_client => datasink_client}/list_collections.py (58%) rename examples/{data_sink_client => datasink_client}/list_datasets.py (63%) create mode 100644 examples/datasink_client/query.py create mode 100644 examples/datasink_client/query_dataset.py rename examples/{data_sink_client => datasink_client}/upload_files_from_path.py (77%) rename examples/{data_sink_client => datasink_client}/upload_folder.py (78%) rename marketplace/{data_sink_client => datasink_client}/README.md (100%) rename marketplace/{data_sink_client => datasink_client}/__init__.py (100%) rename marketplace/{data_sink_client => datasink_client}/cli.py (95%) rename marketplace/{data_sink_client => datasink_client}/session.py (88%) rename marketplace/{data_sink_client => datasink_client}/utils.py (100%) diff --git a/.env_template b/.env_template index 8658dd3..0662ef2 100644 --- a/.env_template +++ b/.env_template @@ -1,5 +1,5 @@ # Marketplace URL -MP_HOST="https://dataspace.reaxpro.eu" +MP_HOST="https://materials-marketplace.eu/" # Access token for markeplace connection. Need to change time to time when expired MP_ACCESS_TOKEN=.... @@ -8,7 +8,9 @@ MP_ACCESS_TOKEN=.... CLIENT_ID="2c791805-ea52-4446-af97-80c0355a73b4" # optional: if incase if we want to get access_token directly from key_cloak -KEYCLOAK_SERVER_URL="https://dataspace.reaxpro.eu/auth/" +# keycloak configuration details can be obtained from marketplace admin +# you can ignore setting MP_ACCESS_TOKEN if you configure keycloak authentication +KEYCLOAK_SERVER_URL="https://materials-marketplace.eu/" KEYCLOAK_CLIENT_ID=.... KEYCLOAK_REALM_NAME=.... KEYCLOAK_CLIENT_SECRET_KEY=.... diff --git a/examples/data_sink_client/query.py b/examples/data_sink_client/query.py deleted file mode 100644 index 70bc9fc..0000000 --- a/examples/data_sink_client/query.py +++ /dev/null @@ -1,6 +0,0 @@ -from marketplace.data_sink_client.session import MPSession - -query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" -with MPSession() as test: - objects = test.query(query=query) - print(objects) diff --git a/examples/data_sink_client/query_dataset.py b/examples/data_sink_client/query_dataset.py deleted file mode 100644 index 271a72e..0000000 --- a/examples/data_sink_client/query_dataset.py +++ /dev/null @@ -1,6 +0,0 @@ -from marketplace.data_sink_client.session import MPSession - -query = "SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5" -with MPSession() as test: - objects = test.query_dataset(collection_name="c1", dataset_name="d1", query=query) - print(objects) diff --git a/examples/data_sink_client/collection_dcat.py b/examples/datasink_client/collection_dcat.py similarity index 64% rename from examples/data_sink_client/collection_dcat.py rename to examples/datasink_client/collection_dcat.py index 1786e71..a46d788 100644 --- a/examples/data_sink_client/collection_dcat.py +++ b/examples/datasink_client/collection_dcat.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.get_collection_dcat(collection_name="c1") diff --git a/examples/data_sink_client/dataset_dcat.py b/examples/datasink_client/dataset_dcat.py similarity index 67% rename from examples/data_sink_client/dataset_dcat.py rename to examples/datasink_client/dataset_dcat.py index 130fade..0d3bc11 100644 --- a/examples/data_sink_client/dataset_dcat.py +++ b/examples/datasink_client/dataset_dcat.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.get_dataset_dcat(collection_name="c1", dataset_name="d1") diff --git a/examples/data_sink_client/delete_collection.py b/examples/datasink_client/delete_collection.py similarity index 64% rename from examples/data_sink_client/delete_collection.py rename to examples/datasink_client/delete_collection.py index 4916ebc..e3543a0 100644 --- a/examples/data_sink_client/delete_collection.py +++ b/examples/datasink_client/delete_collection.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.delete_collection(collection_name="c1") diff --git a/examples/data_sink_client/delete_dataset.py b/examples/datasink_client/delete_dataset.py similarity index 67% rename from examples/data_sink_client/delete_dataset.py rename to examples/datasink_client/delete_dataset.py index e40eec5..a344623 100644 --- a/examples/data_sink_client/delete_dataset.py +++ b/examples/datasink_client/delete_dataset.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.delete_dataset(collection_name="c1", dataset_name="d1") diff --git a/examples/data_sink_client/download_file.py b/examples/datasink_client/download_file.py similarity index 81% rename from examples/data_sink_client/download_file.py rename to examples/datasink_client/download_file.py index 47980ab..ce37062 100644 --- a/examples/data_sink_client/download_file.py +++ b/examples/datasink_client/download_file.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.download_dataset( diff --git a/examples/data_sink_client/download_folder.py b/examples/datasink_client/download_folder.py similarity index 80% rename from examples/data_sink_client/download_folder.py rename to examples/datasink_client/download_folder.py index bb97e4f..78b43ae 100644 --- a/examples/data_sink_client/download_folder.py +++ b/examples/datasink_client/download_folder.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.download_datasets_from_collection( diff --git a/examples/data_sink_client/list_collections.py b/examples/datasink_client/list_collections.py similarity index 58% rename from examples/data_sink_client/list_collections.py rename to examples/datasink_client/list_collections.py index edee211..bf31a11 100644 --- a/examples/data_sink_client/list_collections.py +++ b/examples/datasink_client/list_collections.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.list_collections() diff --git a/examples/data_sink_client/list_datasets.py b/examples/datasink_client/list_datasets.py similarity index 63% rename from examples/data_sink_client/list_datasets.py rename to examples/datasink_client/list_datasets.py index 4cc21a0..ee7bcf9 100644 --- a/examples/data_sink_client/list_datasets.py +++ b/examples/datasink_client/list_datasets.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.list_datasets(collection_name="c1") diff --git a/examples/datasink_client/query.py b/examples/datasink_client/query.py new file mode 100644 index 0000000..d1d5572 --- /dev/null +++ b/examples/datasink_client/query.py @@ -0,0 +1,14 @@ +from marketplace.datasink_client.session import MPSession + +marketplace_url = "https://materials-marketplace.eu/" +access_token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJHUzMzZGxWQ2hXd2tGVExUS2xrU2VzcGxjUHRCdDlHV3VxM0tHWnduZG5rIn0.eyJleHAiOjE2OTM1ODkzNzYsImlhdCI6MTY5MzU2MDU3NiwiYXV0aF90aW1lIjoxNjkzNTYwNTc1LCJqdGkiOiI4ZjdkZjAxNy04YWQ1LTQ5MzAtOWFhNS1mOTI2YmUzYmNiMWUiLCJpc3MiOiJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldS9hdXRoL3JlYWxtcy9tYXJrZXRwbGFjZSIsImF1ZCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCIsImFjY2Vzcy1jb250cm9sIl0sInN1YiI6ImZkODllMzdlLWE3NzMtNDBhNC05YTU3LTgyMDQxZGU2YTExZCIsInR5cCI6IkJlYXJlciIsImF6cCI6ImZyb250ZW5kIiwibm9uY2UiOiIzYmMyN2FhZi02NDQwLTQ5YjAtODRkZS0xMjM0MzU3MGM2OGIiLCJzZXNzaW9uX3N0YXRlIjoiN2U2MWQzOTYtOGE5NC00OWRmLWJkMTMtNTlhYThiMmRhOTExIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldSJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZGVmYXVsdC1yb2xlcy1tYXJrZXRwbGFjZSIsIm9mZmxpbmVfYWNjZXNzIiwiYXBwLWFkbWluIiwidW1hX2F1dGhvcml6YXRpb24iLCJhcHAtdXNlciJdfSwicmVzb3VyY2VfYWNjZXNzIjp7InJlYWxtLW1hbmFnZW1lbnQiOnsicm9sZXMiOlsibWFuYWdlLWlkZW50aXR5LXByb3ZpZGVycyIsIm1hbmFnZS11c2VycyIsIm1hbmFnZS1jbGllbnRzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX0sImZyb250ZW5kIjp7InJvbGVzIjpbImFkbWluIiwidXNlciJdfSwiYWNjZXNzLWNvbnRyb2wiOnsicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl19fSwic2NvcGUiOiJvcGVuaWQgY2xpZW50X3JvbGVzX2Zyb250ZW5kIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3ZTYxZDM5Ni04YTk0LTQ5ZGYtYmQxMy01OWFhOGIyZGE5MTEiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl0sIm5hbWUiOiJLaXJhbiBLdW1hcmFzd2FteSIsInByZWZlcnJlZF91c2VybmFtZSI6ImtpcmFuLmt1bWFyYXN3YW15QGl3bS5mcmF1bmhvZmVyLmRlIiwiZ2l2ZW5fbmFtZSI6IktpcmFuIiwiZmFtaWx5X25hbWUiOiJLdW1hcmFzd2FteSIsImVtYWlsIjoia2lyYW4ua3VtYXJhc3dhbXlAaXdtLmZyYXVuaG9mZXIuZGUifQ.fATcyTImfCaaslKfhPaI3i5_NcTDDr_iyqK6praBaacTIARhp0CzrHlw5w9IkGv2iu3I_ByOevl-tZzEuOPCvg4W69rLsVct8AKvB7OoPXrNpIEOi8RFVk6YM3GUij4OwBsKJ0Juof79LTYbXbiinf_pw_tC9ENFLU1jGaTzLFR07qRTgSFmY0EwjEFdFkOQKCdrVrW_5UrSNTh1EITnIAIudlB6tFheA-tN-DpI4rPKlRobO30tTpX7SLdUC4TDt1_BtjAkS2twHSuOcKWvXnLcnYo_IpDYNuRSuQMfAbuPuvyh2LmuJbdCENBWhl1P_5DMbiqBWIA_-oP0SV3Yow" + + +client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" + +query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" +with MPSession( + marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id +) as test: + objects = test.query(query=query) + print(objects) diff --git a/examples/datasink_client/query_dataset.py b/examples/datasink_client/query_dataset.py new file mode 100644 index 0000000..0f28cac --- /dev/null +++ b/examples/datasink_client/query_dataset.py @@ -0,0 +1,16 @@ +from marketplace.datasink_client.session import MPSession + +marketplace_url = "https://materials-marketplace.eu/" +access_token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJHUzMzZGxWQ2hXd2tGVExUS2xrU2VzcGxjUHRCdDlHV3VxM0tHWnduZG5rIn0.eyJleHAiOjE2OTM1ODkzNzYsImlhdCI6MTY5MzU2MDU3NiwiYXV0aF90aW1lIjoxNjkzNTYwNTc1LCJqdGkiOiI4ZjdkZjAxNy04YWQ1LTQ5MzAtOWFhNS1mOTI2YmUzYmNiMWUiLCJpc3MiOiJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldS9hdXRoL3JlYWxtcy9tYXJrZXRwbGFjZSIsImF1ZCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCIsImFjY2Vzcy1jb250cm9sIl0sInN1YiI6ImZkODllMzdlLWE3NzMtNDBhNC05YTU3LTgyMDQxZGU2YTExZCIsInR5cCI6IkJlYXJlciIsImF6cCI6ImZyb250ZW5kIiwibm9uY2UiOiIzYmMyN2FhZi02NDQwLTQ5YjAtODRkZS0xMjM0MzU3MGM2OGIiLCJzZXNzaW9uX3N0YXRlIjoiN2U2MWQzOTYtOGE5NC00OWRmLWJkMTMtNTlhYThiMmRhOTExIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldSJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZGVmYXVsdC1yb2xlcy1tYXJrZXRwbGFjZSIsIm9mZmxpbmVfYWNjZXNzIiwiYXBwLWFkbWluIiwidW1hX2F1dGhvcml6YXRpb24iLCJhcHAtdXNlciJdfSwicmVzb3VyY2VfYWNjZXNzIjp7InJlYWxtLW1hbmFnZW1lbnQiOnsicm9sZXMiOlsibWFuYWdlLWlkZW50aXR5LXByb3ZpZGVycyIsIm1hbmFnZS11c2VycyIsIm1hbmFnZS1jbGllbnRzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX0sImZyb250ZW5kIjp7InJvbGVzIjpbImFkbWluIiwidXNlciJdfSwiYWNjZXNzLWNvbnRyb2wiOnsicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl19fSwic2NvcGUiOiJvcGVuaWQgY2xpZW50X3JvbGVzX2Zyb250ZW5kIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3ZTYxZDM5Ni04YTk0LTQ5ZGYtYmQxMy01OWFhOGIyZGE5MTEiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl0sIm5hbWUiOiJLaXJhbiBLdW1hcmFzd2FteSIsInByZWZlcnJlZF91c2VybmFtZSI6ImtpcmFuLmt1bWFyYXN3YW15QGl3bS5mcmF1bmhvZmVyLmRlIiwiZ2l2ZW5fbmFtZSI6IktpcmFuIiwiZmFtaWx5X25hbWUiOiJLdW1hcmFzd2FteSIsImVtYWlsIjoia2lyYW4ua3VtYXJhc3dhbXlAaXdtLmZyYXVuaG9mZXIuZGUifQ.fATcyTImfCaaslKfhPaI3i5_NcTDDr_iyqK6praBaacTIARhp0CzrHlw5w9IkGv2iu3I_ByOevl-tZzEuOPCvg4W69rLsVct8AKvB7OoPXrNpIEOi8RFVk6YM3GUij4OwBsKJ0Juof79LTYbXbiinf_pw_tC9ENFLU1jGaTzLFR07qRTgSFmY0EwjEFdFkOQKCdrVrW_5UrSNTh1EITnIAIudlB6tFheA-tN-DpI4rPKlRobO30tTpX7SLdUC4TDt1_BtjAkS2twHSuOcKWvXnLcnYo_IpDYNuRSuQMfAbuPuvyh2LmuJbdCENBWhl1P_5DMbiqBWIA_-oP0SV3Yow" + + +client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" + +query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" +with MPSession( + marketplace_host_url=marketplace_url, access_token=access_token, client_id=client_id +) as test: + objects = test.query_dataset( + collection_name="c1", dataset_name="data_test", query=query + ) + print(objects) diff --git a/examples/data_sink_client/upload_files_from_path.py b/examples/datasink_client/upload_files_from_path.py similarity index 77% rename from examples/data_sink_client/upload_files_from_path.py rename to examples/datasink_client/upload_files_from_path.py index 3bf19b4..1121a8a 100644 --- a/examples/data_sink_client/upload_files_from_path.py +++ b/examples/datasink_client/upload_files_from_path.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.create_dataset_from_path( diff --git a/examples/data_sink_client/upload_folder.py b/examples/datasink_client/upload_folder.py similarity index 78% rename from examples/data_sink_client/upload_folder.py rename to examples/datasink_client/upload_folder.py index b2e62e0..8641c1d 100644 --- a/examples/data_sink_client/upload_folder.py +++ b/examples/datasink_client/upload_folder.py @@ -1,4 +1,4 @@ -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession with MPSession() as test: objects = test.create_datasets_from_sourcedir( diff --git a/marketplace/client.py b/marketplace/client.py index 5a4a25a..3fef0d7 100644 --- a/marketplace/client.py +++ b/marketplace/client.py @@ -6,9 +6,11 @@ """ import os +from functools import wraps from urllib.parse import urljoin import requests +from keycloak import KeycloakOpenID from requests import Response from .version import __version__ @@ -16,6 +18,49 @@ MP_DEFAULT_HOST = "https://materials-marketplace.eu/" +def configure_token(func): + @wraps(func) + def func_(self, *arg, **kwargs): + r = func(self, *arg, **kwargs) + if r.status_code == 401: + token = configure() + if token is None: + raise ( + "Authentication failure. Please provide valid MP_ACCESS_TOKEN or keycloak credentials." + ) + os.environ["MP_ACCESS_TOKEN"] = token + self.access_token = token + + r = func(self, *arg, **kwargs) + if r.status_code > 400: + raise Exception("Server returned with an Exception. Details: ", r.text) + return r + + return func_ + + +def configure(): + # Configure client + server_url = os.environ.get("KEYCLOAK_SERVER_URL") + client_id = os.environ.get("KEYCLOAK_CLIENT_ID") + realm_name = os.environ.get("KEYCLOAK_REALM_NAME") + client_key = os.environ.get("KEYCLOAK_CLIENT_SECRET_KEY") + user = os.environ.get("MARKETPLACE_USERNAME") + passwd = os.environ.get("MARKETPLACE_PASSWORD") + keycloak_openid = KeycloakOpenID( + server_url=server_url, + client_id=client_id, + realm_name=realm_name, + client_secret_key=client_key, + ) + try: + token = keycloak_openid.token(user, passwd) + token = token["access_token"] + return token + except Exception: + return None + + class MarketPlaceClient: """Interact with the MarketPlace platform.""" @@ -24,7 +69,7 @@ def __init__(self, marketplace_host_url=None, access_token=None): "MP_HOST", MP_DEFAULT_HOST, ) - access_token = access_token or os.environ["MP_ACCESS_TOKEN"] + access_token = access_token or os.environ.get("MP_ACCESS_TOKEN") self.marketplace_host_url = marketplace_host_url self.access_token = access_token @@ -55,20 +100,26 @@ def _request(self, op, path, **kwargs) -> Response: full_url = urljoin(self.marketplace_host_url, path) return op(url=full_url, **kwargs) + @configure_token def get(self, path: str, **kwargs): return self._request(requests.get, path, **kwargs) + @configure_token def post(self, path: str, **kwargs): return self._request(requests.post, path, **kwargs) + @configure_token def put(self, path: str, **kwargs): return self._request(requests.put, path, **kwargs) + @configure_token def delete(self, path: str, **kwargs): return self._request(requests.delete, path, **kwargs) + @configure_token def head(self, path: str, **kwargs): return self._request(requests.head, path, **kwargs) + @configure_token def patch(self, path: str, **kwargs): return self._request(requests.patch, path, **kwargs) diff --git a/marketplace/data_sink_client/README.md b/marketplace/datasink_client/README.md similarity index 100% rename from marketplace/data_sink_client/README.md rename to marketplace/datasink_client/README.md diff --git a/marketplace/data_sink_client/__init__.py b/marketplace/datasink_client/__init__.py similarity index 100% rename from marketplace/data_sink_client/__init__.py rename to marketplace/datasink_client/__init__.py diff --git a/marketplace/data_sink_client/cli.py b/marketplace/datasink_client/cli.py similarity index 95% rename from marketplace/data_sink_client/cli.py rename to marketplace/datasink_client/cli.py index 50466b1..a34bd79 100644 --- a/marketplace/data_sink_client/cli.py +++ b/marketplace/datasink_client/cli.py @@ -6,7 +6,7 @@ import click -from marketplace.data_sink_client.session import MPSession +from marketplace.datasink_client.session import MPSession class CommaSeparatedListofPythonLiteralValues(click.Option): diff --git a/marketplace/data_sink_client/session.py b/marketplace/datasink_client/session.py similarity index 88% rename from marketplace/data_sink_client/session.py rename to marketplace/datasink_client/session.py index a117a9d..75bc857 100644 --- a/marketplace/data_sink_client/session.py +++ b/marketplace/datasink_client/session.py @@ -1,61 +1,13 @@ import os import os.path -from functools import wraps - -# from dotenv import find_dotenv, load_dotenv -from keycloak import KeycloakOpenID from marketplace.app import MarketPlaceClient, get_app -from marketplace.data_sink_client.utils import ( +from marketplace.datasink_client.utils import ( get_collections_from_catalog, parse_objects_from_collection, ) -def reconfigure_if_expired(func): - @wraps(func) - def func_(self, *arg, **kwargs): - try: - r = func(self, *arg, **kwargs) - return r - except Exception as e: - print( - "API encountered exception. Please check if all your environment variables configured properly once. Error details: ", - str(e), - ) - # temporary work around to catch Un Authorized error - """error = str(e) - if len(error.split("401 Unauthorized")) > 0: - print("Token expired. Reconfiguring again.") - token = configure() - os.environ["MP_ACCESS_TOKEN"] = token - r = func(self, *arg, **kwargs) - return r - else: - raise RuntimeError(e)""" - - return func_ - - -def configure(): - # Configure client - server_url = os.environ.get("KEYCLOAK_SERVER_URL") - client_id = os.environ.get("KEYCLOAK_CLIENT_ID") - realm_name = os.environ.get("KEYCLOAK_REALM_NAME") - client_key = os.environ.get("KEYCLOAK_CLIENT_SECRET_KEY") - user = os.environ.get("MARKETPLACE_USERNAME") - passwd = os.environ.get("MARKETPLACE_PASSWORD") - keycloak_openid = KeycloakOpenID( - server_url=server_url, - client_id=client_id, - realm_name=realm_name, - client_secret_key=client_key, - ) - token = keycloak_openid.token(user, passwd) - token = token["access_token"] - return token - - class MPSession: """ReaxPro-MarketPlace Session API Wrapper. @@ -66,7 +18,6 @@ class MPSession: """ - @reconfigure_if_expired def __init__( self, marketplace_host_url=None, access_token=None, client_id=None, **kwargs ): @@ -77,7 +28,6 @@ def __init__( ) self.marketPlace = get_app(app_id=CLIENT_ID, client=mp_client) - @reconfigure_if_expired def create_dataset( self, collection_name, dataset_name, sub_collection_id, abs_path ): @@ -120,7 +70,6 @@ def create_dataset( ) return None - @reconfigure_if_expired def create_collection(self, collection_name, sub_collection_id): """create collection/catalog to the MarketPlace DataSink. @@ -149,7 +98,6 @@ def create_collection(self, collection_name, sub_collection_id): ) return None - @reconfigure_if_expired def get_dataset(self, collection_name=None, dataset_name=None): """Get binary data from a get request @@ -164,7 +112,6 @@ def get_dataset(self, collection_name=None, dataset_name=None): return response - @reconfigure_if_expired def get_collection_dcat(self, collection_name=None): """Get a collection/catalog object from a get request @@ -177,7 +124,6 @@ def get_collection_dcat(self, collection_name=None): ) return response - @reconfigure_if_expired def get_dataset_dcat(self, collection_name=None, dataset_name=None): """Get a dataset dcat object from a get request @@ -191,7 +137,6 @@ def get_dataset_dcat(self, collection_name=None, dataset_name=None): ) return response - @reconfigure_if_expired def list_collections(self): """Returns list of Collections. @@ -200,7 +145,6 @@ def list_collections(self): response = self.marketPlace.list_collections() return response - @reconfigure_if_expired def list_datasets(self, collection_name): """Returns list of datasets for a specific collection. @@ -211,7 +155,6 @@ def list_datasets(self, collection_name): response = self.marketPlace.list_datasets(collection_name) return response - @reconfigure_if_expired def delete_collection(self, collection_name): """Delete a collection from datasink. @@ -222,7 +165,6 @@ def delete_collection(self, collection_name): response = self.marketPlace.delete_collection(collection_name) return response - @reconfigure_if_expired def delete_dataset(self, collection_name, dataset_name): """Delete a dataset from datasink. @@ -234,7 +176,6 @@ def delete_dataset(self, collection_name, dataset_name): response = self.marketPlace.delete_dataset(collection_name, dataset_name) return response - @reconfigure_if_expired def query_dataset(self, collection_name, dataset_name, query): """Execute a aparql query on a dataset stored in datasink. @@ -249,7 +190,6 @@ def query_dataset(self, collection_name, dataset_name, query): ) return response - @reconfigure_if_expired def query(self, query, meta_data=False): """Execute a aparql query on a dataset stored in datasink. @@ -261,7 +201,6 @@ def query(self, query, meta_data=False): response = self.marketPlace.query(query, meta_data=meta_data) return response - @reconfigure_if_expired def create_dataset_from_path(self, path, collection_name=None, dataset_name=None): if not os.path.exists(path): raise Exception("File " + path + " does not exist.") @@ -281,7 +220,6 @@ def create_dataset_from_path(self, path, collection_name=None, dataset_name=None ) return response - @reconfigure_if_expired def create_datasets_from_paths(self, paths, collection_name, dataset_names): """Inject a list of datasets. A single InformationPackage will be created. @@ -314,7 +252,6 @@ def create_datasets_from_paths(self, paths, collection_name, dataset_names): return response_list - @reconfigure_if_expired def create_datasets_from_sourcedir( self, sourcedir: str, collection_name: str = None ): @@ -348,7 +285,6 @@ def create_datasets_from_sourcedir( return response_list - @reconfigure_if_expired def create_objects_from_sourcedir( self, collection_name, sourcedir, collection_id, response_list=[] ): @@ -377,7 +313,6 @@ def create_objects_from_sourcedir( # print("added file: ", (os.path.join(sourcedir, file), dataset_id)) return response_list - @reconfigure_if_expired def download_dataset( self, collection_name, @@ -400,7 +335,6 @@ def download_dataset( result.append({"download_path": file_path}) return result - @reconfigure_if_expired def download_datasets_from_search_query( self, collection_search_query, @@ -440,7 +374,6 @@ def download_datasets_from_search_query( return result - @reconfigure_if_expired def download_datasets_from_collection( self, collection_name, diff --git a/marketplace/data_sink_client/utils.py b/marketplace/datasink_client/utils.py similarity index 100% rename from marketplace/data_sink_client/utils.py rename to marketplace/datasink_client/utils.py diff --git a/setup.cfg b/setup.cfg index 3d40a49..e855911 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,18 +31,18 @@ python_requires = >=3.8 [options.entry_points] console_scripts = - list_collections = marketplace.data_sink_client.cli:list_collections - list_datasets = marketplace.data_sink_client.cli:list_datasets - get_collection_dcat = marketplace.data_sink_client.cli:get_collection_dcat - get_dataset_dcat = marketplace.data_sink_client.cli:get_dataset_dcat - delete_dataset = marketplace.data_sink_client.cli:delete_dataset - delete_collection = marketplace.data_sink_client.cli:delete_collection - upload_file_from_path = marketplace.data_sink_client.cli:upload_file_from_path - upload_folder = marketplace.data_sink_client.cli:upload_files_from_folder - download_folder = marketplace.data_sink_client.cli:download_folder - download_file = marketplace.data_sink_client.cli:download_file - query = marketplace.data_sink_client.cli:query - query_dataset = marketplace.data_sink_client.cli:query_dataset + list_collections = marketplace.datasink_client.cli:list_collections + list_datasets = marketplace.datasink_client.cli:list_datasets + get_collection_dcat = marketplace.datasink_client.cli:get_collection_dcat + get_dataset_dcat = marketplace.datasink_client.cli:get_dataset_dcat + delete_dataset = marketplace.datasink_client.cli:delete_dataset + delete_collection = marketplace.datasink_client.cli:delete_collection + upload_file_from_path = marketplace.datasink_client.cli:upload_file_from_path + upload_folder = marketplace.datasink_client.cli:upload_files_from_folder + download_folder = marketplace.datasink_client.cli:download_folder + download_file = marketplace.datasink_client.cli:download_file + query = marketplace.datasink_client.cli:query + query_dataset = marketplace.datasink_client.cli:query_dataset [options.extras_require] dev = From 4a57eed52cb41bc25aa78eb86fee674489fbbb36 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Mon, 18 Sep 2023 14:51:34 +0000 Subject: [PATCH 13/15] keycloak authentication and datasink rename --- examples/datasink_client/query.py | 3 +-- examples/datasink_client/query_dataset.py | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/examples/datasink_client/query.py b/examples/datasink_client/query.py index d1d5572..24dd7de 100644 --- a/examples/datasink_client/query.py +++ b/examples/datasink_client/query.py @@ -1,8 +1,7 @@ from marketplace.datasink_client.session import MPSession marketplace_url = "https://materials-marketplace.eu/" -access_token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJHUzMzZGxWQ2hXd2tGVExUS2xrU2VzcGxjUHRCdDlHV3VxM0tHWnduZG5rIn0.eyJleHAiOjE2OTM1ODkzNzYsImlhdCI6MTY5MzU2MDU3NiwiYXV0aF90aW1lIjoxNjkzNTYwNTc1LCJqdGkiOiI4ZjdkZjAxNy04YWQ1LTQ5MzAtOWFhNS1mOTI2YmUzYmNiMWUiLCJpc3MiOiJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldS9hdXRoL3JlYWxtcy9tYXJrZXRwbGFjZSIsImF1ZCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCIsImFjY2Vzcy1jb250cm9sIl0sInN1YiI6ImZkODllMzdlLWE3NzMtNDBhNC05YTU3LTgyMDQxZGU2YTExZCIsInR5cCI6IkJlYXJlciIsImF6cCI6ImZyb250ZW5kIiwibm9uY2UiOiIzYmMyN2FhZi02NDQwLTQ5YjAtODRkZS0xMjM0MzU3MGM2OGIiLCJzZXNzaW9uX3N0YXRlIjoiN2U2MWQzOTYtOGE5NC00OWRmLWJkMTMtNTlhYThiMmRhOTExIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldSJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZGVmYXVsdC1yb2xlcy1tYXJrZXRwbGFjZSIsIm9mZmxpbmVfYWNjZXNzIiwiYXBwLWFkbWluIiwidW1hX2F1dGhvcml6YXRpb24iLCJhcHAtdXNlciJdfSwicmVzb3VyY2VfYWNjZXNzIjp7InJlYWxtLW1hbmFnZW1lbnQiOnsicm9sZXMiOlsibWFuYWdlLWlkZW50aXR5LXByb3ZpZGVycyIsIm1hbmFnZS11c2VycyIsIm1hbmFnZS1jbGllbnRzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX0sImZyb250ZW5kIjp7InJvbGVzIjpbImFkbWluIiwidXNlciJdfSwiYWNjZXNzLWNvbnRyb2wiOnsicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl19fSwic2NvcGUiOiJvcGVuaWQgY2xpZW50X3JvbGVzX2Zyb250ZW5kIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3ZTYxZDM5Ni04YTk0LTQ5ZGYtYmQxMy01OWFhOGIyZGE5MTEiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl0sIm5hbWUiOiJLaXJhbiBLdW1hcmFzd2FteSIsInByZWZlcnJlZF91c2VybmFtZSI6ImtpcmFuLmt1bWFyYXN3YW15QGl3bS5mcmF1bmhvZmVyLmRlIiwiZ2l2ZW5fbmFtZSI6IktpcmFuIiwiZmFtaWx5X25hbWUiOiJLdW1hcmFzd2FteSIsImVtYWlsIjoia2lyYW4ua3VtYXJhc3dhbXlAaXdtLmZyYXVuaG9mZXIuZGUifQ.fATcyTImfCaaslKfhPaI3i5_NcTDDr_iyqK6praBaacTIARhp0CzrHlw5w9IkGv2iu3I_ByOevl-tZzEuOPCvg4W69rLsVct8AKvB7OoPXrNpIEOi8RFVk6YM3GUij4OwBsKJ0Juof79LTYbXbiinf_pw_tC9ENFLU1jGaTzLFR07qRTgSFmY0EwjEFdFkOQKCdrVrW_5UrSNTh1EITnIAIudlB6tFheA-tN-DpI4rPKlRobO30tTpX7SLdUC4TDt1_BtjAkS2twHSuOcKWvXnLcnYo_IpDYNuRSuQMfAbuPuvyh2LmuJbdCENBWhl1P_5DMbiqBWIA_-oP0SV3Yow" - +access_token = "PASTE_TOKEN_HERE" client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" diff --git a/examples/datasink_client/query_dataset.py b/examples/datasink_client/query_dataset.py index 0f28cac..a3c069b 100644 --- a/examples/datasink_client/query_dataset.py +++ b/examples/datasink_client/query_dataset.py @@ -1,9 +1,7 @@ from marketplace.datasink_client.session import MPSession marketplace_url = "https://materials-marketplace.eu/" -access_token = "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJHUzMzZGxWQ2hXd2tGVExUS2xrU2VzcGxjUHRCdDlHV3VxM0tHWnduZG5rIn0.eyJleHAiOjE2OTM1ODkzNzYsImlhdCI6MTY5MzU2MDU3NiwiYXV0aF90aW1lIjoxNjkzNTYwNTc1LCJqdGkiOiI4ZjdkZjAxNy04YWQ1LTQ5MzAtOWFhNS1mOTI2YmUzYmNiMWUiLCJpc3MiOiJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldS9hdXRoL3JlYWxtcy9tYXJrZXRwbGFjZSIsImF1ZCI6WyJyZWFsbS1tYW5hZ2VtZW50IiwiYWNjb3VudCIsImFjY2Vzcy1jb250cm9sIl0sInN1YiI6ImZkODllMzdlLWE3NzMtNDBhNC05YTU3LTgyMDQxZGU2YTExZCIsInR5cCI6IkJlYXJlciIsImF6cCI6ImZyb250ZW5kIiwibm9uY2UiOiIzYmMyN2FhZi02NDQwLTQ5YjAtODRkZS0xMjM0MzU3MGM2OGIiLCJzZXNzaW9uX3N0YXRlIjoiN2U2MWQzOTYtOGE5NC00OWRmLWJkMTMtNTlhYThiMmRhOTExIiwiYWNyIjoiMSIsImFsbG93ZWQtb3JpZ2lucyI6WyJodHRwczovL21hdGVyaWFscy1tYXJrZXRwbGFjZS5ldSJdLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOlsiZGVmYXVsdC1yb2xlcy1tYXJrZXRwbGFjZSIsIm9mZmxpbmVfYWNjZXNzIiwiYXBwLWFkbWluIiwidW1hX2F1dGhvcml6YXRpb24iLCJhcHAtdXNlciJdfSwicmVzb3VyY2VfYWNjZXNzIjp7InJlYWxtLW1hbmFnZW1lbnQiOnsicm9sZXMiOlsibWFuYWdlLWlkZW50aXR5LXByb3ZpZGVycyIsIm1hbmFnZS11c2VycyIsIm1hbmFnZS1jbGllbnRzIl19LCJhY2NvdW50Ijp7InJvbGVzIjpbIm1hbmFnZS1hY2NvdW50IiwibWFuYWdlLWFjY291bnQtbGlua3MiLCJ2aWV3LXByb2ZpbGUiXX0sImZyb250ZW5kIjp7InJvbGVzIjpbImFkbWluIiwidXNlciJdfSwiYWNjZXNzLWNvbnRyb2wiOnsicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl19fSwic2NvcGUiOiJvcGVuaWQgY2xpZW50X3JvbGVzX2Zyb250ZW5kIHByb2ZpbGUgZW1haWwiLCJzaWQiOiI3ZTYxZDM5Ni04YTk0LTQ5ZGYtYmQxMy01OWFhOGIyZGE5MTEiLCJlbWFpbF92ZXJpZmllZCI6dHJ1ZSwicm9sZXMiOlsiYWRtaW4iLCJ1c2VyIl0sIm5hbWUiOiJLaXJhbiBLdW1hcmFzd2FteSIsInByZWZlcnJlZF91c2VybmFtZSI6ImtpcmFuLmt1bWFyYXN3YW15QGl3bS5mcmF1bmhvZmVyLmRlIiwiZ2l2ZW5fbmFtZSI6IktpcmFuIiwiZmFtaWx5X25hbWUiOiJLdW1hcmFzd2FteSIsImVtYWlsIjoia2lyYW4ua3VtYXJhc3dhbXlAaXdtLmZyYXVuaG9mZXIuZGUifQ.fATcyTImfCaaslKfhPaI3i5_NcTDDr_iyqK6praBaacTIARhp0CzrHlw5w9IkGv2iu3I_ByOevl-tZzEuOPCvg4W69rLsVct8AKvB7OoPXrNpIEOi8RFVk6YM3GUij4OwBsKJ0Juof79LTYbXbiinf_pw_tC9ENFLU1jGaTzLFR07qRTgSFmY0EwjEFdFkOQKCdrVrW_5UrSNTh1EITnIAIudlB6tFheA-tN-DpI4rPKlRobO30tTpX7SLdUC4TDt1_BtjAkS2twHSuOcKWvXnLcnYo_IpDYNuRSuQMfAbuPuvyh2LmuJbdCENBWhl1P_5DMbiqBWIA_-oP0SV3Yow" - - +access_token = "PASTE_TOKEN_HERE" client_id = "edb56699-9377-4f41-b1c7-ef2f46dac707" query = """SELECT ?subject ?predicate ?object WHERE {{ ?subject ?predicate ?object . }} LIMIT 5""" From a83c96ce54b17e5e814f1eba700428c088f1a86c Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 29 Sep 2023 15:49:37 +0000 Subject: [PATCH 14/15] review changes --- marketplace/app/v0/transformation.py | 14 ++++++++++++ marketplace/client.py | 32 ++++++++++++++++------------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/marketplace/app/v0/transformation.py b/marketplace/app/v0/transformation.py index ee6e259..459b7f4 100644 --- a/marketplace/app/v0/transformation.py +++ b/marketplace/app/v0/transformation.py @@ -22,6 +22,20 @@ def new_transformation( new_transformation: transformation.NewTransformationModel, config: dict = None, ) -> transformation.TransformationCreateResponse: + """ + Creates a new transformation. + + Args: + - new_transformation (NewTransformationModel): A dictionary representing + parameters to create a new transformation. + - config (dict): A dictionary representing query parameters. + Any key-value passed inside this dictionary are sent as query parameters + to the application. + + Retruns: + TransformationCreateResponse: A dictionary containing id of the + created transformation. + """ params = {} # send additional key value as query parameters if some app needs it if config is not None: diff --git a/marketplace/client.py b/marketplace/client.py index 3fef0d7..868b044 100644 --- a/marketplace/client.py +++ b/marketplace/client.py @@ -23,23 +23,32 @@ def configure_token(func): def func_(self, *arg, **kwargs): r = func(self, *arg, **kwargs) if r.status_code == 401: - token = configure() - if token is None: - raise ( - "Authentication failure. Please provide valid MP_ACCESS_TOKEN or keycloak credentials." + response = configure() + if not response["status"] == "success": + raise Exception( + "User authentication failure. Reason:" + response["message"] ) + token = response["token"] os.environ["MP_ACCESS_TOKEN"] = token self.access_token = token r = func(self, *arg, **kwargs) if r.status_code > 400: - raise Exception("Server returned with an Exception. Details: ", r.text) + raise Exception("Server returned with an Exception. Details: " + r.text) return r return func_ def configure(): + """ + Authenticates a user with marketplace username and password using keycloak + authentication module. If the authentication is succesfull then this method returns + user token within a dictionary. Otherwise, an exception is caught and the + resaon for failure is returned as dict. In order to use keycloak authentication + with username and password we have to configure all the necessary keycloak + environment variables. Configurations can be obtained from market place admin. + """ # Configure client server_url = os.environ.get("KEYCLOAK_SERVER_URL") client_id = os.environ.get("KEYCLOAK_CLIENT_ID") @@ -56,9 +65,9 @@ def configure(): try: token = keycloak_openid.token(user, passwd) token = token["access_token"] - return token - except Exception: - return None + return {"status": "success", "token": token, "message": ""} + except Exception as e: + return {"status": "failure", "token": None, "message": str(e)} class MarketPlaceClient: @@ -95,31 +104,26 @@ def userinfo(self): userinfo.raise_for_status() return userinfo.json() + @configure_token def _request(self, op, path, **kwargs) -> Response: kwargs.setdefault("headers", {}).update(self.default_headers) full_url = urljoin(self.marketplace_host_url, path) return op(url=full_url, **kwargs) - @configure_token def get(self, path: str, **kwargs): return self._request(requests.get, path, **kwargs) - @configure_token def post(self, path: str, **kwargs): return self._request(requests.post, path, **kwargs) - @configure_token def put(self, path: str, **kwargs): return self._request(requests.put, path, **kwargs) - @configure_token def delete(self, path: str, **kwargs): return self._request(requests.delete, path, **kwargs) - @configure_token def head(self, path: str, **kwargs): return self._request(requests.head, path, **kwargs) - @configure_token def patch(self, path: str, **kwargs): return self._request(requests.patch, path, **kwargs) From 293ed920073adb8207fbefbe87bfb43579cebce5 Mon Sep 17 00:00:00 2001 From: Kiran Kumaraswamy Date: Fri, 29 Sep 2023 15:50:58 +0000 Subject: [PATCH 15/15] remove default client_id --- .env_template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env_template b/.env_template index 0662ef2..ff63470 100644 --- a/.env_template +++ b/.env_template @@ -5,7 +5,7 @@ MP_ACCESS_TOKEN=.... # Env variables needed for data-sink connection otherwise optional # Client-ID of the datasink application -CLIENT_ID="2c791805-ea52-4446-af97-80c0355a73b4" +CLIENT_ID=.... # optional: if incase if we want to get access_token directly from key_cloak # keycloak configuration details can be obtained from marketplace admin