diff --git a/grisera/__init__.py b/grisera/__init__.py index d4403fb..04b0f09 100644 --- a/grisera/__init__.py +++ b/grisera/__init__.py @@ -6,6 +6,10 @@ from .activity_execution.activity_execution_router import ActivityExecutionRouter, router as activity_execution_router from .activity_execution.activity_execution_service import ActivityExecutionService +from .additional_parameter.additional_parameter_model import * +from .additional_parameter.additional_parameter_router import AdditionalParameterRouter, router as additional_parameter_router +from .additional_parameter.additional_parameter_service import AdditionalParameterService + from .appearance.appearance_model import * from .appearance.appearance_router import AppearanceRouter, router as appearance_router from .appearance.appearance_service import AppearanceService @@ -26,6 +30,10 @@ from .experiment.experiment_router import ExperimentRouter, router as experiment_router from .experiment.experiment_service import ExperimentService +from .file.file_model import * +from .file.file_router import FileRouter, router as file_router +from .file.file_service import FileService + from .helpers.hateoas import prepare_links, get_links from .helpers.helpers import create_stub_from_response @@ -46,6 +54,7 @@ from .modality.modality_service import ModalityService from .models.base_model_out import BaseModelOut +from .models.importable_model import ImportableModel from .models.not_found_model import NotFoundByIdModel from .models.relation_information_model import RelationInformation diff --git a/grisera/activity/activity_model.py b/grisera/activity/activity_model.py index 4841795..b917a5c 100644 --- a/grisera/activity/activity_model.py +++ b/grisera/activity/activity_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn @@ -23,7 +24,7 @@ class Activity(str, Enum): group = "group" -class ActivityPropertyIn(BaseModel): +class ActivityPropertyIn(ImportableModel): """ Model of activity execution to acquire from client diff --git a/grisera/activity_execution/activity_execution_model.py b/grisera/activity_execution/activity_execution_model.py index 561fbe4..6b8ca8a 100644 --- a/grisera/activity_execution/activity_execution_model.py +++ b/grisera/activity_execution/activity_execution_model.py @@ -3,10 +3,11 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn -class ActivityExecutionPropertyIn(BaseModel): +class ActivityExecutionPropertyIn(ImportableModel): """ Model of activity execution to acquire from client diff --git a/grisera/additional_parameter/__init__.py b/grisera/additional_parameter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/grisera/additional_parameter/additional_parameter_model.py b/grisera/additional_parameter/additional_parameter_model.py new file mode 100644 index 0000000..f6a07ad --- /dev/null +++ b/grisera/additional_parameter/additional_parameter_model.py @@ -0,0 +1,71 @@ +from enum import Enum +from typing import Optional, Union, List + +from pydantic import BaseModel + +from grisera.models.base_model_out import BaseModelOut + + +class ParameterType(str, Enum): + """ + Types of additional parameters + + Attributes: + participant (str): Parameter for participants + activity (str): Parameter for activities + participant_state (str): Parameter for participant states + activity_execution (str): Parameter for activity executions + """ + + participant = "participant" + activity = "activity" + participant_state = "participantState" + activity_execution = "activityExecution" + + +class AdditionalParameterIn(BaseModel): + """ + Model of additional parameter to acquire from client + + Attributes: + name (str): Name of the parameter + key (Optional[str]): Key identifier for the parameter + type (ParameterType): Type of parameter (participant, activity, etc.) + options (Optional[List[str]]): Available options for the parameter + """ + + name: str + key: Optional[str] + type: ParameterType + options: Optional[List[str]] = [] + + +class BasicAdditionalParameterOut(AdditionalParameterIn): + """ + Basic model of additional parameter to send to client as a result of request + + Attributes: + id (Optional[int | str]): Id of parameter returned from api + dataset_id (Optional[int | str]): Id of dataset this parameter belongs to + """ + + id: Optional[Union[int, str]] + dataset_id: Optional[Union[int, str]] + + +class AdditionalParameterOut(BasicAdditionalParameterOut, BaseModelOut): + """ + Model of additional parameter with relationships to send to client as a result of request + """ + pass + + +class AdditionalParametersOut(BaseModelOut): + """ + Model of additional parameters to send to client as a result of request + + Attributes: + parameters (List[BasicAdditionalParameterOut]): Parameters from database + """ + + parameters: List[BasicAdditionalParameterOut] = [] \ No newline at end of file diff --git a/grisera/additional_parameter/additional_parameter_router.py b/grisera/additional_parameter/additional_parameter_router.py new file mode 100644 index 0000000..db5688b --- /dev/null +++ b/grisera/additional_parameter/additional_parameter_router.py @@ -0,0 +1,126 @@ +from typing import Union + +from fastapi import Response, Depends +from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter + +from grisera.additional_parameter.additional_parameter_model import ( + AdditionalParameterIn, + AdditionalParameterOut, + AdditionalParametersOut, +) +from grisera.helpers.hateoas import get_links +from grisera.helpers.helpers import check_dataset_permission +from grisera.models.not_found_model import NotFoundByIdModel +from grisera.services.service import service +from grisera.services.service_factory import ServiceFactory + +router = InferringRouter(dependencies=[Depends(check_dataset_permission)]) + + +@cbv(router) +class AdditionalParameterRouter: + """ + Class for routing additional parameter based requests + + Attributes: + additional_parameter_service (AdditionalParameterService): Service instance for additional parameters + """ + + def __init__(self, service_factory: ServiceFactory = Depends(service.get_service_factory)): + self.additional_parameter_service = service_factory.get_additional_parameter_service() + + @router.post("/parameters", tags=["parameters"], response_model=AdditionalParameterOut) + async def create_parameter(self, parameter: AdditionalParameterIn, response: Response, dataset_id: Union[int, str]): + """ + Create additional parameter in database + """ + + create_response = self.additional_parameter_service.save_additional_parameter(parameter, dataset_id) + if create_response.errors is not None: + response.status_code = 422 + + # add links from hateoas + create_response.links = get_links(router) + + return create_response + + @router.get("/parameters", tags=["parameters"], response_model=AdditionalParametersOut) + async def get_parameters(self, response: Response, dataset_id: Union[int, str]): + """ + Get additional parameters from database + """ + + get_response = self.additional_parameter_service.get_additional_parameters(dataset_id) + + # add links from hateoas + get_response.links = get_links(router) + + return get_response + + @router.get( + "/parameters/{parameter_id}", + tags=["parameters"], + response_model=Union[AdditionalParameterOut, NotFoundByIdModel], + ) + async def get_parameter( + self, parameter_id: Union[str, int], response: Response, dataset_id: Union[int, str] + ): + """ + Get additional parameter from database + """ + + get_response = self.additional_parameter_service.get_additional_parameter(parameter_id, dataset_id) + if get_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + get_response.links = get_links(router) + + return get_response + + @router.delete( + "/parameters/{parameter_id}", + tags=["parameters"], + response_model=Union[AdditionalParameterOut, NotFoundByIdModel], + ) + async def delete_parameter( + self, parameter_id: Union[int, str], response: Response, dataset_id: Union[int, str] + ): + """ + Delete additional parameter from database + """ + delete_response = self.additional_parameter_service.delete_additional_parameter(parameter_id, dataset_id) + if delete_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + delete_response.links = get_links(router) + + return delete_response + + @router.put( + "/parameters/{parameter_id}", + tags=["parameters"], + response_model=Union[AdditionalParameterOut, NotFoundByIdModel], + ) + async def update_parameter( + self, + parameter_id: Union[int, str], + parameter: AdditionalParameterIn, + response: Response, + dataset_id: Union[int, str] + ): + """ + Update additional parameter model in database + """ + update_response = self.additional_parameter_service.update_additional_parameter( + parameter_id, parameter, dataset_id + ) + if update_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + update_response.links = get_links(router) + + return update_response \ No newline at end of file diff --git a/grisera/additional_parameter/additional_parameter_service.py b/grisera/additional_parameter/additional_parameter_service.py new file mode 100644 index 0000000..1860ad0 --- /dev/null +++ b/grisera/additional_parameter/additional_parameter_service.py @@ -0,0 +1,87 @@ +from typing import Union + +from grisera.additional_parameter.additional_parameter_model import AdditionalParameterIn + + +class AdditionalParameterService: + """ + Abstract class to handle logic of additional parameters requests + + """ + + def save_additional_parameter(self, parameter: AdditionalParameterIn, dataset_id: Union[int, str]): + """ + Send request to database to create new additional parameter + + Args: + parameter (AdditionalParameterIn): Additional parameter to be added + dataset_id (int | str): name of dataset + + Returns: + Result of request as additional parameter object + """ + raise Exception("save_additional_parameter not implemented yet") + + def get_additional_parameters(self, dataset_id: Union[int, str]): + """ + Send request to database to get additional parameters + + Args: + dataset_id (int | str): name of dataset + + Returns: + Result of request as list of additional parameters objects + """ + raise Exception("get_additional_parameters not implemented yet") + + def get_additional_parameter(self, parameter_id: Union[int, str], dataset_id: Union[int, str]): + """ + Send request to database to get given additional parameter + + Args: + parameter_id (int | str): identity of additional parameter + dataset_id (int | str): name of dataset + + Returns: + Result of request as additional parameter object + """ + raise Exception("get_additional_parameter not implemented yet") + + def delete_additional_parameter(self, parameter_id: Union[int, str], dataset_id: Union[int, str]): + """ + Send request to database to delete given additional parameter + + Args: + parameter_id (int | str): identity of additional parameter + dataset_id (int | str): name of dataset + + Returns: + Result of request as additional parameter object + """ + raise Exception("delete_additional_parameter not implemented yet") + + def update_additional_parameter(self, parameter_id: Union[int, str], parameter: AdditionalParameterIn, dataset_id: Union[int, str]): + """ + Send request to database to update given additional parameter + + Args: + parameter_id (int | str): identity of additional parameter + parameter (AdditionalParameterIn): Properties to update + dataset_id (int | str): name of dataset + + Returns: + Result of request as additional parameter object + """ + raise Exception("update_additional_parameter not implemented yet") + + def get_additional_parameters_by_dataset(self, dataset_id: Union[int, str]): + """ + Send request to database to get additional parameters for given dataset + + Args: + dataset_id (int | str): name of dataset + + Returns: + Result of request as list of additional parameters objects + """ + raise Exception("get_additional_parameters_by_dataset not implemented yet") \ No newline at end of file diff --git a/grisera/appearance/appearance_model.py b/grisera/appearance/appearance_model.py index e51d87d..e2b1599 100644 --- a/grisera/appearance/appearance_model.py +++ b/grisera/appearance/appearance_model.py @@ -20,11 +20,13 @@ class AppearanceOcclusionIn(BaseModel): glasses (bool): Does appearance contain glasses beard (FacialHair): Length of beard moustache (FacialHair): Length of moustache + external_id (Optional[str]): External ID from source system """ beard: FacialHair moustache: FacialHair glasses: bool + external_id: Optional[str] class BasicAppearanceOcclusionOut(AppearanceOcclusionIn): @@ -58,12 +60,14 @@ class AppearanceSomatotypeIn(BaseModel): ectomorph (float): Range of ectomorph appearance measure endomorph (float): Range of endomorph appearance measure mesomorph (float): Range of mesomorph appearance measure + external_id (Optional[str]): External ID from source system """ ectomorph: float endomorph: float mesomorph: float + external_id: Optional[str] class BasicAppearanceSomatotypeOut(AppearanceSomatotypeIn): diff --git a/grisera/arrangement/arrangement_model.py b/grisera/arrangement/arrangement_model.py index dfa089f..934369d 100644 --- a/grisera/arrangement/arrangement_model.py +++ b/grisera/arrangement/arrangement_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class Arrangement(tuple, Enum): @@ -25,18 +26,20 @@ class Arrangement(tuple, Enum): personal_group = ("personal group", None) -class ArrangementIn(BaseModel): +class ArrangementIn(ImportableModel): """ Model of arrangement Attributes: arrangement (str): Type of arrangement + """ arrangement_type: str arrangement_distance: Optional[str] + class BasicArrangementOut(ArrangementIn): """ Model of arrangement in dataset diff --git a/grisera/channel/channel_model.py b/grisera/channel/channel_model.py index b7f5423..b790b40 100644 --- a/grisera/channel/channel_model.py +++ b/grisera/channel/channel_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class Types(tuple, Enum): @@ -35,7 +36,7 @@ class Types(tuple, Enum): temperature = ("Temperature", "Temperature channel") -class ChannelIn(BaseModel): +class ChannelIn(ImportableModel): """ Model of channel diff --git a/grisera/channel/channel_router.py b/grisera/channel/channel_router.py index 5cf3f97..ed85412 100644 --- a/grisera/channel/channel_router.py +++ b/grisera/channel/channel_router.py @@ -44,6 +44,31 @@ async def create_channel(self, channel: ChannelIn, response: Response, dataset_i return create_response + @router.put( + "/channels/{channel_id}", + tags=["channels"], + response_model=Union[ChannelOut, NotFoundByIdModel], + ) + async def update_channel( + self, + channel_id: Union[int, str], + channel: ChannelIn, + response: Response, + dataset_id: Union[int, str] + ): + """ + Update channel in database + """ + update_response = self.channel_service.update_channel(channel_id, channel, dataset_id) + + if update_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + update_response.links = get_links(router) + + return update_response + @router.get( "/channels/{channel_id}", tags=["channels"], @@ -78,4 +103,4 @@ async def get_channels(self, response: Response, dataset_id: Union[int, str]): # add links from hateoas get_response.links = get_links(router) - return get_response + return get_response \ No newline at end of file diff --git a/grisera/channel/channel_service.py b/grisera/channel/channel_service.py index 827c2e7..8ab99a5 100644 --- a/grisera/channel/channel_service.py +++ b/grisera/channel/channel_service.py @@ -6,7 +6,6 @@ class ChannelService: """ Abstract class to handle logic of channel requests - """ def save_channel(self, channel: ChannelIn, dataset_id: Union[int, str]): @@ -45,3 +44,17 @@ def get_channel(self, channel_id: Union[int, str], dataset_id: Union[int, str], Result of request as channel object """ raise Exception("get_channel not implemented yet") + + def update_channel(self, channel_id: Union[int, str], channel: ChannelIn, dataset_id: Union[int, str]): + """ + Send request to graph api to update given channel + + Args: + channel_id (int | str): identity of channel + channel (ChannelIn): Properties to update + dataset_id (int | str): name of dataset + + Returns: + Result of request as channel object + """ + raise Exception("update_channel not implemented yet") \ No newline at end of file diff --git a/grisera/clients/__init__.py b/grisera/clients/__init__.py new file mode 100644 index 0000000..2cc5f34 --- /dev/null +++ b/grisera/clients/__init__.py @@ -0,0 +1,3 @@ +""" +Shared clients module for external services like MinIO, databases, etc. +""" \ No newline at end of file diff --git a/grisera/clients/minio_client.py b/grisera/clients/minio_client.py new file mode 100644 index 0000000..6695e69 --- /dev/null +++ b/grisera/clients/minio_client.py @@ -0,0 +1,204 @@ +import os +from datetime import timedelta +from typing import Optional, Dict, Any + +from minio import Minio +from minio.error import S3Error +from fastapi import HTTPException + + +class MinIOClient: + """ + MinIO client wrapper for file storage operations + """ + + def __init__(self, bucket_name: str): + self.access_key = os.getenv("AWS_ACCESS_KEY_ID") + self.secret_key = os.getenv("AWS_SECRET_ACCESS_KEY") + self.region = os.getenv("AWS_REGION", "us-east-1") + self.endpoint = os.getenv("MINIO_ENDPOINT", "s3:9000") + self.public_endpoint_url = os.getenv("MINIO_PUBLIC_ENDPOINT", "http://localhost:9000") + self.bucket_name = bucket_name + + # Parse public endpoint URL + from urllib.parse import urlparse + parsed = urlparse(self.public_endpoint_url) + self.public_endpoint = parsed.netloc # This includes port if specified + + self._client = None + self._public_client = None + + def _is_secure_endpoint(self, url: str) -> bool: + """Check if endpoint should use secure connection""" + return url.startswith('https://') + + @property + def client(self) -> Minio: + """Get MinIO client instance""" + if self._client is None: + self._client = Minio( + self.endpoint, + access_key=self.access_key, + secret_key=self.secret_key, + secure=False, # Internal endpoint is typically http + region=self.region, + ) + return self._client + + @property + def public_client(self) -> Minio: + """Get public MinIO client instance for URL generation""" + if self._public_client is None: + secure = self._is_secure_endpoint(self.public_endpoint_url) + self._public_client = Minio( + self.public_endpoint, + access_key=self.access_key, + secret_key=self.secret_key, + secure=secure, + region=self.region, + ) + return self._public_client + + def ensure_bucket_exists(self) -> None: + """Ensure the bucket exists, create if it doesn't""" + try: + if not self.client.bucket_exists(self.bucket_name): + self.client.make_bucket(self.bucket_name) + # Set bucket to private by default (no public policy) + self._ensure_bucket_is_private() + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to create bucket: {str(e)}") + + def _ensure_bucket_is_private(self) -> None: + """Ensure bucket has no public read policy""" + try: + # Remove any existing bucket policy to make it private + self.client.delete_bucket_policy(self.bucket_name) + except S3Error: + # Ignore errors - bucket might not have any policy set + pass + + def make_bucket_private(self) -> None: + """ + Explicitly make bucket private (can be called manually for existing buckets) + """ + try: + self._ensure_bucket_is_private() + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to make bucket private: {str(e)}") + + def upload_file(self, object_name: str, file_data: bytes, content_type: str) -> None: + """ + Upload file to MinIO storage + + Args: + object_name (str): Object name in storage + file_data (bytes): File content + content_type (str): MIME type + """ + try: + import io + + self.ensure_bucket_exists() + + file_stream = io.BytesIO(file_data) + self.client.put_object( + self.bucket_name, + object_name, + file_stream, + length=len(file_data), + content_type=content_type, + ) + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to upload file: {str(e)}") + + def get_file(self, object_name: str): + """ + Get file from MinIO storage + + Args: + object_name (str): Object name in storage + + Returns: + MinIO response object + """ + try: + return self.client.get_object(self.bucket_name, object_name) + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to get file: {str(e)}") + + def delete_file(self, object_name: str) -> None: + """ + Delete file from MinIO storage + + Args: + object_name (str): Object name in storage + """ + try: + self.client.remove_object(self.bucket_name, object_name) + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to delete file: {str(e)}") + + def generate_download_url(self, object_name: str, original_filename: str, + expires_hours: int = 1) -> Dict[str, Any]: + """ + Generate pre-signed download URL + + Args: + object_name (str): Object name in storage + original_filename (str): Original filename for download + expires_hours (int): URL expiration time in hours + + Returns: + Dict with download URL and metadata + """ + try: + download_url = self.public_client.presigned_get_object( + self.bucket_name, + object_name, + expires=timedelta(hours=expires_hours), + response_headers={ + 'response-content-disposition': f'attachment; filename="{original_filename}"' + } + ) + + return { + "download_url": download_url, + "filename": original_filename, + "expires_in": expires_hours * 3600 + } + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to generate download URL: {str(e)}") + + def generate_preview_url(self, object_name: str, content_type: str = None, + expires_hours: int = 1) -> Dict[str, Any]: + """ + Generate pre-signed preview URL for viewing file in browser + + Args: + object_name (str): Object name in storage + content_type (str): MIME type for proper browser rendering + expires_hours (int): URL expiration time in hours + + Returns: + Dict with preview URL and metadata + """ + try: + # Response headers for inline viewing (no download) + response_headers = {} + if content_type: + response_headers['response-content-type'] = content_type + + preview_url = self.public_client.presigned_get_object( + self.bucket_name, + object_name, + expires=timedelta(hours=expires_hours), + response_headers=response_headers if response_headers else None + ) + + return { + "preview_url": preview_url, + "expires_in": expires_hours * 3600 + } + except S3Error as e: + raise HTTPException(status_code=500, detail=f"Failed to generate preview URL: {str(e)}") \ No newline at end of file diff --git a/grisera/dataset/dataset_model.py b/grisera/dataset/dataset_model.py index f2893b5..3f4edef 100644 --- a/grisera/dataset/dataset_model.py +++ b/grisera/dataset/dataset_model.py @@ -3,11 +3,13 @@ from pydantic import BaseModel +from grisera.models.importable_model import ImportableModel + from grisera.property.property_model import PropertyIn from grisera.models.base_model_out import BaseModelOut -class DatasetIn(BaseModel): +class DatasetIn(ImportableModel): """ Model of dataset to acquire from client @@ -17,6 +19,7 @@ class DatasetIn(BaseModel): rights (Optional[str]): Rights set to the dataset given by user date (Optional[date]): Date of the dataset given by user description (Optional[date]): Description of the dataset given by user + additional_properties (Optional[List[PropertyIn]]): Additional properties for dataset """ @@ -25,6 +28,7 @@ class DatasetIn(BaseModel): rights: Optional[str] date: Optional[date] description: Optional[str] + additional_properties: Optional[List[PropertyIn]] @@ -43,9 +47,12 @@ class DatasetOut(BasicDatasetOut, BaseModelOut): Model of dataset to send to client as a result of request Attributes: + parameters (Optional[List]): Additional parameters for dataset errors (Optional[Any]): Optional errors appeared during query executions links (Optional[list): Hateoas implementation """ + + parameters: Optional[List] = None class DatasetsOut(BaseModelOut): diff --git a/grisera/dataset/dataset_router.py b/grisera/dataset/dataset_router.py index aed82ec..55b4a29 100644 --- a/grisera/dataset/dataset_router.py +++ b/grisera/dataset/dataset_router.py @@ -36,6 +36,7 @@ class DatasetRouter: def __init__(self, service_factory: ServiceFactory = Depends(service.get_service_factory)): self.dataset_service = service_factory.get_dataset_service() + self.additional_parameter_service = service_factory.get_additional_parameter_service() self.channel_service = service_factory.get_channel_service() self.modality_service = service_factory.get_modality_service() self.life_activity_service = service_factory.get_life_activity_service() @@ -91,6 +92,11 @@ async def get_dataset(self, response: Response, dataset_id: Union[int, str]): get_response = self.dataset_service.get_dataset(dataset_id) if get_response.errors is not None: response.status_code = 404 + else: + # Get additional parameters for the dataset + parameters_response = self.additional_parameter_service.get_additional_parameters_by_dataset(dataset_id) + if parameters_response.errors is None: + get_response.parameters = parameters_response.parameters # add links from hateoas get_response.links = get_links(router) diff --git a/grisera/experiment/experiment_model.py b/grisera/experiment/experiment_model.py index 633b08f..11ed5f5 100644 --- a/grisera/experiment/experiment_model.py +++ b/grisera/experiment/experiment_model.py @@ -3,10 +3,11 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn -class ExperimentIn(BaseModel): +class ExperimentIn(ImportableModel): """ Model of experiment to acquire from client diff --git a/grisera/file/__init__.py b/grisera/file/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/grisera/file/archive_extractor.py b/grisera/file/archive_extractor.py new file mode 100644 index 0000000..ec17737 --- /dev/null +++ b/grisera/file/archive_extractor.py @@ -0,0 +1,205 @@ +import io +import gzip +import zipfile +import tarfile +from typing import List, Tuple, Generator +from uuid import uuid4 + +from fastapi import HTTPException + +from grisera.file.file_validation import FileValidator + + +class ArchiveExtractor: + """ + Service for extracting various archive formats with recursive support + """ + + def __init__(self, max_depth: int = 5): + self.max_depth = max_depth + self.validator = FileValidator() + + def extract_archive(self, content: bytes, filename: str, content_type: str = None, + base_path: str = "", base_uuid: str = None, + current_depth: int = 0) -> List[Tuple[bytes, str, str]]: + """ + Extract archive with recursive support + + Args: + content (bytes): Archive content + filename (str): Archive filename + content_type (str): MIME type + base_path (str): Base path for extracted files + base_uuid (str): UUID for organization + current_depth (int): Current recursion depth + + Returns: + List of tuples: (file_content, original_filename, display_path) + """ + if current_depth >= self.max_depth: + raise HTTPException(status_code=400, detail="Maximum archive nesting depth exceeded") + + if base_uuid is None: + base_uuid = str(uuid4()) + + is_archive, archive_type = self.validator.is_archive_file(filename, content_type) + + if not is_archive: + raise HTTPException(status_code=400, detail=f"File is not a supported archive: {filename}") + + if archive_type == 'zip': + return self._extract_zip(content, base_path, base_uuid, current_depth) + elif archive_type in ['tar', 'tar.gz']: + return self._extract_tar(content, base_path, base_uuid, current_depth) + elif archive_type == 'gz': + return self._extract_gz(content, base_path, base_uuid, current_depth) + else: + raise HTTPException(status_code=400, detail=f"Unsupported archive type: {archive_type}") + + def _extract_zip(self, zip_content: bytes, base_path: str, base_uuid: str, + current_depth: int) -> List[Tuple[bytes, str, str]]: + """Extract ZIP archive recursively""" + extracted_files = [] + + try: + with zipfile.ZipFile(io.BytesIO(zip_content), 'r') as zip_ref: + # Security check for dangerous paths + for member in zip_ref.namelist(): + self.validator.validate_archive_path(member) + + for member in zip_ref.namelist(): + # Skip directories + if member.endswith('/'): + continue + + try: + member_content = zip_ref.read(member) + full_path = f"{base_path}/{member}" if base_path else member + + # Check if this file is also an archive + is_nested_archive, archive_type = self.validator.is_archive_file(member) + + if is_nested_archive and member_content: + try: + nested_files = self.extract_archive( + member_content, + member, + base_path=self._get_base_path_for_archive(full_path, archive_type), + base_uuid=base_uuid, + current_depth=current_depth + 1 + ) + extracted_files.extend(nested_files) + except (tarfile.TarError, zipfile.BadZipFile, OSError, gzip.BadGzipFile): + # If it's not actually a valid archive, treat as regular file + extracted_files.append((member_content, member, full_path)) + else: + # Regular file + extracted_files.append((member_content, member, full_path)) + + except Exception as e: + # Skip corrupted files but continue with others + continue + + except zipfile.BadZipFile: + raise HTTPException(status_code=400, detail="Invalid ZIP file") + + return extracted_files + + def _extract_tar(self, tar_content: bytes, base_path: str, base_uuid: str, + current_depth: int) -> List[Tuple[bytes, str, str]]: + """Extract TAR archive recursively""" + extracted_files = [] + + try: + with tarfile.open(fileobj=io.BytesIO(tar_content), mode='r:*') as tar_ref: + # Security check for dangerous paths + for member in tar_ref.getnames(): + self.validator.validate_archive_path(member) + + for member in tar_ref.getmembers(): + # Skip directories + if member.isdir(): + continue + + try: + file_obj = tar_ref.extractfile(member) + if file_obj is None: + continue + + member_content = file_obj.read() + full_path = f"{base_path}/{member.name}" if base_path else member.name + + # Check if this file is also an archive + is_nested_archive, archive_type = self.validator.is_archive_file(member.name) + + if is_nested_archive and member_content: + try: + nested_files = self.extract_archive( + member_content, + member.name, + base_path=self._get_base_path_for_archive(full_path, archive_type), + base_uuid=base_uuid, + current_depth=current_depth + 1 + ) + extracted_files.extend(nested_files) + except (tarfile.TarError, zipfile.BadZipFile, OSError, gzip.BadGzipFile): + # If it's not actually a valid archive, treat as regular file + extracted_files.append((member_content, member.name, full_path)) + else: + # Regular file + extracted_files.append((member_content, member.name, full_path)) + + except Exception as e: + # Skip corrupted files but continue with others + continue + + except tarfile.TarError: + raise HTTPException(status_code=400, detail="Invalid TAR file") + + return extracted_files + + def _extract_gz(self, gz_content: bytes, base_path: str, base_uuid: str, + current_depth: int) -> List[Tuple[bytes, str, str]]: + """Extract GZ file (single file compression)""" + try: + # Decompress the GZ file + decompressed_content = gzip.decompress(gz_content) + + # Determine decompressed filename + if base_path.endswith('.gz'): + decompressed_filename = base_path[:-3] # Remove .gz extension + else: + decompressed_filename = base_path + '_decompressed' + + # Check if decompressed content is an archive + is_nested_archive, archive_type = self.validator.is_archive_file(decompressed_filename) + + if is_nested_archive and decompressed_content: + try: + return self.extract_archive( + decompressed_content, + decompressed_filename, + base_path=self._get_base_path_for_archive(decompressed_filename, archive_type), + base_uuid=base_uuid, + current_depth=current_depth + 1 + ) + except (tarfile.TarError, zipfile.BadZipFile, OSError, gzip.BadGzipFile): + # If it's not actually a valid archive, treat as regular file + pass + + # Return as regular file + import os + original_filename = os.path.basename(decompressed_filename) + return [(decompressed_content, original_filename, decompressed_filename)] + + except (OSError, gzip.BadGzipFile): + raise HTTPException(status_code=400, detail="Invalid GZ file") + + def _get_base_path_for_archive(self, full_path: str, archive_type: str) -> str: + """Get base path for nested archive extraction""" + if archive_type == 'tar.gz': + # Remove .tar.gz extension + return full_path.rsplit('.', 2)[0] if full_path.count('.') >= 2 else full_path.rsplit('.', 1)[0] + else: + # Remove single extension + return full_path.rsplit('.', 1)[0] \ No newline at end of file diff --git a/grisera/file/file_model.py b/grisera/file/file_model.py new file mode 100644 index 0000000..b76490c --- /dev/null +++ b/grisera/file/file_model.py @@ -0,0 +1,61 @@ +from datetime import datetime +from typing import Optional, List, Union + +from pydantic import BaseModel + +from grisera.models.base_model_out import BaseModelOut + + +class FileIn(BaseModel): + """ + Model of file to acquire from client + + Attributes: + filename (Optional[str]): Generated filename in storage + original_filename (Optional[str]): Original filename uploaded by user + name (Optional[str]): Custom name given by user + size (Optional[int]): File size in bytes + content_type (Optional[str]): MIME type of the file + dataset_id (Union[int, str]): ID of associated dataset (required) + """ + + filename: Optional[str] + original_filename: Optional[str] + name: Optional[str] + size: Optional[int] + content_type: Optional[str] + dataset_id: Union[int, str] + + +class BasicFileOut(FileIn): + """ + Model of file + + Attributes: + id (Optional[int | str]): Id of file returned from api + uploaded_at (Optional[datetime]): Upload timestamp + """ + id: Optional[Union[int, str]] + uploaded_at: Optional[datetime] + + +class FileOut(BasicFileOut, BaseModelOut): + """ + Model of file to send to client as a result of request + + Attributes: + errors (Optional[Any]): Optional errors appeared during query executions + links (Optional[list): Hateoas implementation + """ + + +class FilesOut(BaseModelOut): + """ + Model of list of files + + Attributes: + files (Optional[List[BasicFileOut]]): List of files to send + errors (Optional[Any]): Optional errors appeared during query executions + links (Optional[list): Hateoas implementation + """ + files: Optional[List[BasicFileOut]] \ No newline at end of file diff --git a/grisera/file/file_router.py b/grisera/file/file_router.py new file mode 100644 index 0000000..90113e2 --- /dev/null +++ b/grisera/file/file_router.py @@ -0,0 +1,226 @@ +from typing import Union, List + +from fastapi import Response, Depends, UploadFile, File, Form, HTTPException +from fastapi.responses import StreamingResponse +from fastapi_utils.cbv import cbv +from fastapi_utils.inferring_router import InferringRouter + +from grisera.clients.minio_client import MinIOClient +from grisera.file.file_model import FileOut, FilesOut +from grisera.file.file_service import FileService +from grisera.file.file_validation import FileValidator +from grisera.file.upload_handler import UploadHandler +from grisera.helpers.hateoas import get_links +from grisera.helpers.helpers import check_dataset_permission +from grisera.models.not_found_model import NotFoundByIdModel +from grisera.services.service import service +from grisera.services.service_factory import ServiceFactory + +router = InferringRouter(dependencies=[Depends(check_dataset_permission)]) + + +@cbv(router) +class FileRouter: + """ + Class for routing file based requests + + Attributes: + file_service (FileService): Service instance for files + """ + + def __init__(self, service_factory: ServiceFactory = Depends(service.get_service_factory)): + self.file_service = service_factory.get_file_service() + self.minio_client = MinIOClient("files") + self.validator = FileValidator() + self.upload_handler = UploadHandler(self.file_service, router) + + @router.get("/files", tags=["files"], response_model=FilesOut) + def get_files(self, response: Response, dataset_id: Union[int, str]) -> FilesOut: + """ + Get all files metadata for a specific dataset + + Args: + dataset_id (Union[int, str]): Dataset ID to filter files + + Returns: + FilesOut: List of files with metadata filtered by dataset + """ + files = self.file_service.get_files_by_dataset(dataset_id) + response.status_code = 200 + + return FilesOut( + files=files, + links=get_links(router) + ) + + @router.get("/files/{file_id}", tags=["files"], response_model=Union[FileOut, NotFoundByIdModel]) + def get_file(self, file_id: Union[int, str], response: Response) -> Union[FileOut, NotFoundByIdModel]: + """ + Get file metadata by ID + + Args: + file_id (Union[int, str]): File ID + + Returns: + Union[FileOut, NotFoundByIdModel]: File metadata or not found response + """ + file_data = self.file_service.get_file_by_id(file_id) + + if file_data is None: + response.status_code = 404 + return NotFoundByIdModel(id=file_id, errors="File not found") + + response.status_code = 200 + return FileOut( + **file_data.dict(), + links=get_links(router) + ) + + @router.post("/files/upload", tags=["files"], response_model=Union[FileOut, List[FileOut]]) + async def upload_file(self, response: Response, file: UploadFile = File(...), + name: str = Form(...), dataset_id: str = Form(...)) -> Union[FileOut, List[FileOut]]: + """ + Upload a file and store it in MinIO S3 storage. + If the file is a ZIP archive, it will be extracted and each file uploaded separately. + + Args: + file (UploadFile): File to upload + name (str): Custom name given by user (form field, required) + dataset_id (str): Associated dataset ID (form field, required) + + Returns: + Union[FileOut, List[FileOut]]: File metadata or list of extracted files metadata + """ + try: + result = await self.upload_handler.handle_upload(file, name, dataset_id) + response.status_code = 201 + return result + except HTTPException: + raise + except Exception as e: + response.status_code = 500 + raise HTTPException(status_code=500, detail=f"Unexpected error during file upload: {str(e)}") + + @router.get("/files/{file_id}/download", tags=["files"]) + def download_file(self, file_id: Union[int, str], response: Response): + """ + Generate pre-signed URL for direct download from MinIO storage + + Args: + file_id (Union[int, str]): File ID + + Returns: + Dict: Pre-signed download URL + """ + file_data = self.file_service.get_file_by_id(file_id) + + if file_data is None: + raise HTTPException(status_code=404, detail="File not found") + + try: + download_info = self.minio_client.generate_download_url( + file_data.filename, + file_data.original_filename + ) + + response.status_code = 200 + return download_info + except HTTPException: + raise + + @router.get("/files/{file_id}/preview", tags=["files"]) + def preview_file(self, file_id: Union[int, str]): + """ + Preview file content (for text and image files) + + Args: + file_id (Union[int, str]): File ID + + Returns: + File content for preview + """ + file_data = self.file_service.get_file_by_id(file_id) + + if file_data is None: + raise HTTPException(status_code=404, detail="File not found") + + if not self.validator.is_previewable(file_data.content_type): + raise HTTPException(status_code=400, detail="Preview not available for this file type") + + try: + file_response = self.minio_client.get_file(file_data.filename) + + # Headers for optimized streaming of large files + headers = { + "Content-Length": str(file_data.size), + "Accept-Ranges": "bytes", + "Cache-Control": "public, max-age=3600", # Allow caching for preview + } + + return StreamingResponse( + file_response, + media_type=file_data.content_type, + headers=headers + ) + except HTTPException: + raise + + @router.get("/files/{file_id}/preview-url", tags=["files"]) + def get_preview_url(self, file_id: Union[int, str], response: Response): + """ + Generate pre-signed URL for file preview in browser + + Args: + file_id (Union[int, str]): File ID + + Returns: + Dict: Pre-signed preview URL + """ + file_data = self.file_service.get_file_by_id(file_id) + + if file_data is None: + raise HTTPException(status_code=404, detail="File not found") + + if not self.validator.is_previewable(file_data.content_type): + raise HTTPException(status_code=400, detail="Preview not available for this file type") + + try: + preview_info = self.minio_client.generate_preview_url( + file_data.filename, + file_data.content_type + ) + + response.status_code = 200 + return preview_info + except HTTPException: + raise + + @router.delete("/files/{file_id}", tags=["files"]) + def delete_file(self, file_id: Union[int, str], response: Response): + """ + Delete file and its metadata + + Args: + file_id (Union[int, str]): File ID + + Returns: + Dict with success message + """ + file_data = self.file_service.get_file_by_id(file_id) + + if file_data is None: + response.status_code = 404 + return NotFoundByIdModel(id=file_id, errors="File not found") + + try: + # Delete from MinIO + self.minio_client.delete_file(file_data.filename) + + # Delete metadata + self.file_service.delete_file(file_id) + + response.status_code = 200 + return {"message": "File deleted successfully", "links": get_links(router)} + + except HTTPException: + raise \ No newline at end of file diff --git a/grisera/file/file_service.py b/grisera/file/file_service.py new file mode 100644 index 0000000..60f0617 --- /dev/null +++ b/grisera/file/file_service.py @@ -0,0 +1,95 @@ +from datetime import datetime +from typing import Union, Optional, Dict, Any + +from grisera.file.file_model import BasicFileOut + + +class FileService: + """ + Object to handle logic of files requests + + Attributes: + files_storage (dict): In-memory storage for files metadata + """ + + def __init__(self): + self.files_storage = {} + + def save_file_metadata(self, filename: str, original_filename: str, name: str, size: int, + content_type: str, dataset_id: Union[int, str]) -> BasicFileOut: + """ + Save file metadata + + Args: + filename (str): Generated filename in storage + original_filename (str): Original filename + name (str): Custom name given by user + size (int): File size + content_type (str): MIME type + dataset_id (Union[int, str]): Associated dataset ID + + Returns: + BasicFileOut: Created file metadata + """ + file_id = str(len(self.files_storage) + 1) + file_data = BasicFileOut( + id=file_id, + filename=filename, + original_filename=original_filename, + name=name, + size=size, + content_type=content_type, + dataset_id=dataset_id, + uploaded_at=datetime.utcnow() + ) + self.files_storage[file_id] = file_data + return file_data + + def get_files(self) -> list: + """ + Get all files + + Returns: + list: List of all files + """ + return list(self.files_storage.values()) + + def get_files_by_dataset(self, dataset_id: Union[int, str]) -> list: + """ + Get files filtered by dataset ID + + Args: + dataset_id (Union[int, str]): Dataset ID + + Returns: + list: List of files for the dataset + """ + return [file for file in self.files_storage.values() if str(file.dataset_id) == str(dataset_id)] + + def get_file_by_id(self, file_id: Union[int, str]) -> Optional[BasicFileOut]: + """ + Get file by ID + + Args: + file_id (Union[int, str]): File ID + + Returns: + Optional[BasicFileOut]: File metadata if found + """ + return self.files_storage.get(str(file_id)) + + def delete_file(self, file_id: Union[int, str]) -> bool: + """ + Delete file metadata + + Args: + file_id (Union[int, str]): File ID + + Returns: + bool: True if deleted, False if not found + """ + file_key = str(file_id) + if file_key in self.files_storage: + del self.files_storage[file_key] + return True + return False \ No newline at end of file diff --git a/grisera/file/file_validation.py b/grisera/file/file_validation.py new file mode 100644 index 0000000..42645d9 --- /dev/null +++ b/grisera/file/file_validation.py @@ -0,0 +1,198 @@ +import mimetypes +from typing import Tuple, List + +from fastapi import HTTPException + + +class FileValidator: + """ + Service for file validation and security checks + """ + + def __init__(self): + self.max_file_size = 1024 * 1024 * 1024 # 1GB + self.max_archive_size = 1024 * 1024 * 1024 # 1GB + + self.allowed_extensions = { + # Documents + '.txt', '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + '.odt', '.ods', '.odp', '.rtf', + # Images + '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.svg', '.webp', + # Audio + '.mp3', '.wav', '.flac', '.aac', '.ogg', '.wma', '.m4a', + # Video + '.mp4', '.avi', '.mov', '.wmv', '.webm', '.mkv', '.flv', '.m4v', + '.3gp', '.mpg', '.mpeg', '.ogv', '.ts', '.mts', '.vob', + # Archives + '.zip', '.tar', '.gz', '.tgz', '.tar.gz', + # Data files + '.csv', '.json', '.xml', '.yaml', '.yml', + # Code files + '.py', '.js', '.html', '.css', '.sql', '.md', + } + + self.previewable_types = [ + 'text/', # Text files + 'image/', # Images + 'application/pdf', # PDF files + 'application/json',# JSON files + 'application/xml', # XML files + 'video/', # Video files + 'audio/', # Audio files + ] + + def validate_file_upload(self, filename: str, content: bytes, content_type: str = None) -> None: + """ + Validate file for upload + + Args: + filename (str): Original filename + content (bytes): File content + content_type (str): MIME type + + Raises: + HTTPException: If validation fails + """ + # Check filename + if not filename or filename.strip() == '': + raise HTTPException(status_code=400, detail="Filename cannot be empty") + + # Check file size + file_size = len(content) + if file_size == 0: + raise HTTPException(status_code=400, detail="File cannot be empty") + + is_archive, _ = self.is_archive_file(filename, content_type) + max_size = self.max_archive_size if is_archive else self.max_file_size + + if file_size > max_size: + size_mb = max_size / (1024 * 1024) + file_type = "archive" if is_archive else "file" + raise HTTPException( + status_code=400, + detail=f"File too large. Maximum {file_type} size is {size_mb}MB" + ) + + # Check file extension + self.validate_file_extension(filename) + + # Security checks + self.validate_filename_security(filename) + + def validate_file_extension(self, filename: str) -> None: + """ + Validate file extension + + Args: + filename (str): Filename to validate + + Raises: + HTTPException: If extension is not allowed + """ + filename_lower = filename.lower() + + # Check for allowed extensions + is_allowed = any(filename_lower.endswith(ext) for ext in self.allowed_extensions) + + if not is_allowed: + raise HTTPException( + status_code=400, + detail=f"File type not allowed. Allowed extensions: {', '.join(sorted(self.allowed_extensions))}" + ) + + def validate_filename_security(self, filename: str) -> None: + """ + Validate filename for security issues + + Args: + filename (str): Filename to validate + + Raises: + HTTPException: If security validation fails + """ + # Check for dangerous characters + dangerous_chars = ['<', '>', '"', '|', '?', '*', ':', '\\'] + if any(char in filename for char in dangerous_chars): + raise HTTPException(status_code=400, detail="Filename contains illegal characters") + + # Check for path traversal + if '..' in filename or filename.startswith('/') or filename.startswith('\\'): + raise HTTPException(status_code=400, detail="Invalid filename path") + + # Check filename length + if len(filename) > 255: + raise HTTPException(status_code=400, detail="Filename too long (max 255 characters)") + + def validate_archive_path(self, path: str) -> None: + """ + Validate path from archive extraction + + Args: + path (str): Path to validate + + Raises: + HTTPException: If path is dangerous + """ + if path.startswith('/') or '..' in path: + raise HTTPException(status_code=400, detail="Invalid file path in archive") + + def is_archive_file(self, filename: str, content_type: str = None) -> Tuple[bool, str]: + """ + Check if file is an archive and return archive type + + Args: + filename (str): Filename to check + content_type (str): MIME type + + Returns: + Tuple[bool, str]: (is_archive, archive_type) + """ + filename_lower = filename.lower() + + # ZIP files + if filename_lower.endswith('.zip') or content_type == 'application/zip': + return True, 'zip' + + # TAR files + if (filename_lower.endswith('.tar') or + content_type in ['application/x-tar', 'application/tar']): + return True, 'tar' + + # TAR.GZ files + if (filename_lower.endswith('.tar.gz') or filename_lower.endswith('.tgz') or + content_type in ['application/gzip', 'application/x-gzip', 'application/x-tar-gz']): + return True, 'tar.gz' + + # GZ files (single file compression) + if (filename_lower.endswith('.gz') and not filename_lower.endswith('.tar.gz') or + content_type in ['application/gzip', 'application/x-gzip']): + return True, 'gz' + + return False, None + + def is_previewable(self, content_type: str) -> bool: + """ + Check if file type is previewable + + Args: + content_type (str): MIME type + + Returns: + bool: True if previewable + """ + return any(content_type.startswith(ptype) or content_type == ptype + for ptype in self.previewable_types) + + def get_content_type(self, filename: str) -> str: + """ + Get content type for filename + + Args: + filename (str): Filename + + Returns: + str: MIME type + """ + content_type, _ = mimetypes.guess_type(filename) + return content_type if content_type else 'application/octet-stream' \ No newline at end of file diff --git a/grisera/file/upload_handler.py b/grisera/file/upload_handler.py new file mode 100644 index 0000000..f9fddda --- /dev/null +++ b/grisera/file/upload_handler.py @@ -0,0 +1,165 @@ +from typing import List, Union +from uuid import uuid4 + +from fastapi import UploadFile, HTTPException + +from grisera.clients.minio_client import MinIOClient +from grisera.file.archive_extractor import ArchiveExtractor +from grisera.file.file_validation import FileValidator +from grisera.file.file_model import FileOut +from grisera.file.file_service import FileService +from grisera.helpers.hateoas import get_links + + +class UploadHandler: + """ + Main handler for file upload operations + """ + + def __init__(self, file_service: FileService, router=None): + self.file_service = file_service + self.router = router + self.validator = FileValidator() + self.minio_client = MinIOClient("files") + self.archive_extractor = ArchiveExtractor() + + async def handle_upload(self, file: UploadFile, name: str, + dataset_id: str) -> Union[FileOut, List[FileOut]]: + """ + Handle file upload with support for archives + + Args: + file (UploadFile): Uploaded file + name (str): Custom name for the file + dataset_id (str): Associated dataset ID + + Returns: + Union[FileOut, List[FileOut]]: Single file or list of extracted files + """ + # Validate input parameters + self._validate_upload_params(file, name, dataset_id) + + # Read file content + file_content = await file.read() + + # Validate file + self.validator.validate_file_upload(file.filename, file_content, file.content_type) + + # Check if file is an archive + is_archive, archive_type = self.validator.is_archive_file(file.filename, file.content_type) + + if is_archive: + return await self._handle_archive_upload(file_content, file.filename, + file.content_type, name, dataset_id) + else: + return await self._handle_single_file_upload(file_content, file.filename, + file.content_type, name, dataset_id) + + def _validate_upload_params(self, file: UploadFile, name: str, dataset_id: str) -> None: + """Validate upload parameters""" + if not name or name.strip() == '': + raise HTTPException(status_code=400, detail="File name is required") + + if not dataset_id or dataset_id.strip() == '': + raise HTTPException(status_code=400, detail="Dataset ID is required") + + if not file.filename: + raise HTTPException(status_code=400, detail="No file provided") + + async def _handle_single_file_upload(self, file_content: bytes, filename: str, + content_type: str, name: str, + dataset_id: str) -> FileOut: + """Handle upload of a single file""" + base_uuid = str(uuid4()) + + # Use custom name for display path if provided, otherwise use original filename + display_path = name if name else filename + + return self._upload_single_file( + file_content=file_content, + original_filename=filename, + display_path=display_path, + base_uuid=base_uuid, + custom_name=name, + dataset_id=dataset_id, + content_type=content_type + ) + + async def _handle_archive_upload(self, file_content: bytes, filename: str, + content_type: str, name: str, + dataset_id: str) -> List[FileOut]: + """Handle upload and extraction of archive files""" + # Extract archive + extracted_files = self.archive_extractor.extract_archive( + content=file_content, + filename=filename, + content_type=content_type + ) + + if not extracted_files: + raise HTTPException(status_code=400, detail="No valid files found in archive") + + # Upload all extracted files + uploaded_files = [] + base_uuid = str(uuid4()) + + for file_data, original_filename, display_path in extracted_files: + # Get content type for extracted file + extracted_content_type = self.validator.get_content_type(original_filename) + + uploaded_file = self._upload_single_file( + file_content=file_data, + original_filename=original_filename, + display_path=display_path, + base_uuid=base_uuid, + custom_name=f"{name}/{display_path}", + dataset_id=dataset_id, + content_type=extracted_content_type + ) + uploaded_files.append(uploaded_file) + + return uploaded_files + + def _upload_single_file(self, file_content: bytes, original_filename: str, + display_path: str, base_uuid: str, custom_name: str, + dataset_id: str, content_type: str = None) -> FileOut: + """ + Upload a single file to storage and save metadata + + Args: + file_content (bytes): File content + original_filename (str): Original filename + display_path (str): Display path for the file + base_uuid (str): Base UUID for storage organization + custom_name (str): Custom name for display + dataset_id (str): Dataset ID + content_type (str): MIME type + + Returns: + FileOut: File metadata with links + """ + # Generate object name for storage + object_name = f"{base_uuid}/{display_path}" + + # Determine content type if not provided + if not content_type: + content_type = self.validator.get_content_type(original_filename) + + # Upload to MinIO + self.minio_client.upload_file(object_name, file_content, content_type) + + # Save file metadata + file_metadata = self.file_service.save_file_metadata( + filename=object_name, + original_filename=original_filename, + name=custom_name, + size=len(file_content), + content_type=content_type, + dataset_id=dataset_id + ) + + # Return with HATEOAS links + return FileOut( + **file_metadata.dict(), + links=get_links(self.router) if self.router else [] + ) \ No newline at end of file diff --git a/grisera/life_activity/life_activity_model.py b/grisera/life_activity/life_activity_model.py index 6fe0a68..43199c8 100644 --- a/grisera/life_activity/life_activity_model.py +++ b/grisera/life_activity/life_activity_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class LifeActivity(str, Enum): @@ -31,7 +32,7 @@ class LifeActivity(str, Enum): brain_activity = "brain activity" -class LifeActivityIn(BaseModel): +class LifeActivityIn(ImportableModel): """ Model of actions of a human body observed during experiment diff --git a/grisera/life_activity/life_activity_router.py b/grisera/life_activity/life_activity_router.py index 849b5b2..df51c05 100644 --- a/grisera/life_activity/life_activity_router.py +++ b/grisera/life_activity/life_activity_router.py @@ -50,6 +50,31 @@ async def create_life_activity( return create_response + @router.put( + "/life_activities/{life_activity_id}", + tags=["life activities"], + response_model=Union[LifeActivityOut, NotFoundByIdModel], + ) + async def update_life_activity( + self, + life_activity_id: Union[int, str], + life_activity: LifeActivityIn, + response: Response, + dataset_id: Union[int, str] + ): + """ + Update life activity in database + """ + update_response = self.life_activity_service.update_life_activity(life_activity_id, life_activity, dataset_id) + + if update_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + update_response.links = get_links(router) + + return update_response + @router.get( "/life_activities/{life_activity_id}", tags=["life activities"], @@ -90,4 +115,4 @@ async def get_life_activities(self, response: Response, dataset_id: Union[int, s # add links from hateoas get_response.links = get_links(router) - return get_response + return get_response \ No newline at end of file diff --git a/grisera/life_activity/life_activity_service.py b/grisera/life_activity/life_activity_service.py index b7e4fea..bd5c152 100644 --- a/grisera/life_activity/life_activity_service.py +++ b/grisera/life_activity/life_activity_service.py @@ -6,7 +6,6 @@ class LifeActivityService: """ Abstract class to handle logic of life activity requests - """ def save_life_activity(self, life_activity: LifeActivityIn, dataset_id: Union[int, str]): @@ -47,3 +46,17 @@ def get_life_activity(self, life_activity_id: Union[int, str], dataset_id: Union Result of request as life activity object """ raise Exception("get_life_activity not implemented yet") + + def update_life_activity(self, life_activity_id: Union[int, str], life_activity: LifeActivityIn, dataset_id: Union[int, str]): + """ + Send request to graph api to update given life activity + + Args: + life_activity_id (int | str): identity of life activity + life_activity (LifeActivityIn): Life activity to update + dataset_id (int | str): name of dataset + + Returns: + Result of request as life activity object + """ + raise Exception("update_life_activity not implemented yet") \ No newline at end of file diff --git a/grisera/measure/measure_model.py b/grisera/measure/measure_model.py index d2f932c..93e15db 100644 --- a/grisera/measure/measure_model.py +++ b/grisera/measure/measure_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class Measure(tuple, Enum): """ @@ -66,7 +67,7 @@ class MeasureRelationIn(BaseModel): measure_name_id: Optional[Union[int, str]] -class MeasureIn(MeasurePropertyIn, MeasureRelationIn): +class MeasureIn(MeasurePropertyIn, MeasureRelationIn, ImportableModel): """ Full model of measure to acquire from client """ diff --git a/grisera/measure_name/measure_name_model.py b/grisera/measure_name/measure_name_model.py index 9f4c2ef..61b45c8 100644 --- a/grisera/measure_name/measure_name_model.py +++ b/grisera/measure_name/measure_name_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class MeasureName(tuple, Enum): @@ -39,7 +40,7 @@ class MeasureName(tuple, Enum): valence = ("Valence", "PAD") -class MeasureNameIn(BaseModel): +class MeasureNameIn(ImportableModel): """ Model of measure name diff --git a/grisera/modality/modality_model.py b/grisera/modality/modality_model.py index 999eda4..1fb7bdb 100644 --- a/grisera/modality/modality_model.py +++ b/grisera/modality/modality_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel class Modality(str, Enum): @@ -45,7 +46,7 @@ class Modality(str, Enum): neural_activity = "neural activity" -class ModalityIn(BaseModel): +class ModalityIn(ImportableModel): """ Model of modality observed during experiment diff --git a/grisera/modality/modality_router.py b/grisera/modality/modality_router.py index 2f6e22a..a86e953 100644 --- a/grisera/modality/modality_router.py +++ b/grisera/modality/modality_router.py @@ -45,6 +45,31 @@ async def create_modality(self, modality: ModalityIn, response: Response, datase return create_response + @router.put( + "/modalities/{modality_id}", + tags=["modalities"], + response_model=Union[ModalityOut, NotFoundByIdModel], + ) + async def update_modality( + self, + modality_id: Union[int, str], + modality: ModalityIn, + response: Response, + dataset_id: Union[int, str] + ): + """ + Update modality in database + """ + update_response = self.modality_service.update_modality(modality_id, modality, dataset_id) + + if update_response.errors is not None: + response.status_code = 404 + + # add links from hateoas + update_response.links = get_links(router) + + return update_response + @router.get( "/modalities/{modality_id}", tags=["modalities"], @@ -78,4 +103,4 @@ async def get_modalities(self, response: Response, dataset_id: Union[int, str]): # add links from hateoas get_response.links = get_links(router) - return get_response + return get_response \ No newline at end of file diff --git a/grisera/modality/modality_service.py b/grisera/modality/modality_service.py index 8a75f1d..ee81ad6 100644 --- a/grisera/modality/modality_service.py +++ b/grisera/modality/modality_service.py @@ -6,7 +6,6 @@ class ModalityService: """ Abstract class to handle logic of modality requests - """ def save_modality(self, modality: ModalityIn, dataset_id: Union[int, str]): @@ -47,3 +46,17 @@ def get_modality(self, modality_id: Union[int, str], dataset_id: Union[int, str] Result of request as modality object """ raise Exception("get_modality not implemented yet") + + def update_modality(self, modality_id: Union[int, str], modality: ModalityIn, dataset_id: Union[int, str]): + """ + Send request to graph api to update given modality + + Args: + modality_id (int | str): identity of modality + modality (ModalityIn): Modality to update + dataset_id (int | str): name of dataset + + Returns: + Result of request as modality object + """ + raise Exception("update_modality not implemented yet") \ No newline at end of file diff --git a/grisera/models/__init__.py b/grisera/models/__init__.py index e69de29..b28b04f 100644 --- a/grisera/models/__init__.py +++ b/grisera/models/__init__.py @@ -0,0 +1,3 @@ + + + diff --git a/grisera/models/importable_model.py b/grisera/models/importable_model.py new file mode 100644 index 0000000..7964f78 --- /dev/null +++ b/grisera/models/importable_model.py @@ -0,0 +1,16 @@ +from datetime import datetime +from typing import Optional +from pydantic import BaseModel + + +class ImportableModel(BaseModel): + """ + Base model for entities that can be imported from external sources + + Attributes: + external_id (Optional[str]): External ID from import source (e.g. @id from JSON) + """ + external_id: Optional[str] = None + import_job_id: Optional[str] = None + import_timestamp: Optional[datetime] = None + diff --git a/grisera/observable_information/observable_information_model.py b/grisera/observable_information/observable_information_model.py index 7c452c9..df7011a 100644 --- a/grisera/observable_information/observable_information_model.py +++ b/grisera/observable_information/observable_information_model.py @@ -3,9 +3,10 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel -class ObservableInformationIn(BaseModel): +class ObservableInformationIn(ImportableModel): """ Model of information observed during experiment @@ -13,6 +14,7 @@ class ObservableInformationIn(BaseModel): modality_id (Optional[Union[int, str]]): Id od modality life_activity_id (Optional[Union[int, str]]): Id of life activity recording_id (Optional[Union[int, str]]): Id of recording + """ modality_id: Optional[Union[int, str]] @@ -20,6 +22,7 @@ class ObservableInformationIn(BaseModel): recording_id: Optional[Union[int, str]] + class BasicObservableInformationOut(ObservableInformationIn): """ Model of information observed during experiment in database diff --git a/grisera/participant/participant_model.py b/grisera/participant/participant_model.py index 65d678b..94aad31 100644 --- a/grisera/participant/participant_model.py +++ b/grisera/participant/participant_model.py @@ -5,6 +5,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn @@ -23,7 +24,7 @@ class Sex(str, Enum): not_given = "not given" -class ParticipantIn(BaseModel): +class ParticipantIn(ImportableModel): """ Model of participant to acquire from client @@ -32,14 +33,12 @@ class ParticipantIn(BaseModel): date_of_birth (Optional[date]): Date of birth of participant sex (Optional[str]): Sex of participant disorder (Optional[str]): Type of disorder - additional_properties (Optional[List[PropertyIn]]): Additional properties for participant """ name: Optional[str] date_of_birth: Optional[date] sex: Optional[str] disorder: Optional[str] - additional_properties: Optional[List[PropertyIn]] class BasicParticipantOut(ParticipantIn): @@ -78,4 +77,4 @@ class ParticipantsOut(BaseModelOut): # Circular import exception prevention from grisera.participant_state.participant_state_model import ParticipantStateOut -ParticipantOut.update_forward_refs() +ParticipantOut.update_forward_refs() \ No newline at end of file diff --git a/grisera/participant_state/participant_state_model.py b/grisera/participant_state/participant_state_model.py index 1fdf028..387d9a9 100644 --- a/grisera/participant_state/participant_state_model.py +++ b/grisera/participant_state/participant_state_model.py @@ -4,10 +4,11 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn -class ParticipantStatePropertyIn(BaseModel): +class ParticipantStatePropertyIn(ImportableModel): """ Model of participant state to acquire from client diff --git a/grisera/participation/participation_model.py b/grisera/participation/participation_model.py index 69f82e8..6c0da5a 100644 --- a/grisera/participation/participation_model.py +++ b/grisera/participation/participation_model.py @@ -3,9 +3,10 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel -class ParticipationIn(BaseModel): +class ParticipationIn(ImportableModel): """ Participation model in database diff --git a/grisera/personality/personality_model.py b/grisera/personality/personality_model.py index 479842e..588d49e 100644 --- a/grisera/personality/personality_model.py +++ b/grisera/personality/personality_model.py @@ -15,6 +15,7 @@ class PersonalityBigFiveIn(BaseModel): extroversion (float): Scale of being outgoing, talkative, energetic neuroticism (float): Scale of lack of self-control, poor ability to manage psychological stress openess (float): Scale of openness (Intellect) reflects imagination, creativity + external_id (Optional[str]): External ID from source system """ @@ -23,6 +24,7 @@ class PersonalityBigFiveIn(BaseModel): extroversion: float neuroticism: float openess: float + external_id: Optional[str] class BasicPersonalityBigFiveOut(PersonalityBigFiveIn): @@ -55,10 +57,12 @@ class PersonalityPanasIn(BaseModel): Attributes: negative_affect (float): Scale of negative affect to community positive_affect (float): Scale of positive affect to community + external_id (Optional[str]): External ID from source system """ negative_affect: float positive_affect: float + external_id: Optional[str] class BasicPersonalityPanasOut(PersonalityPanasIn): diff --git a/grisera/recording/recording_model.py b/grisera/recording/recording_model.py index e3fe96a..3536329 100644 --- a/grisera/recording/recording_model.py +++ b/grisera/recording/recording_model.py @@ -3,6 +3,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn @@ -30,10 +31,9 @@ class RecordingRelationIn(BaseModel): registered_channel_id: Optional[Union[int, str]] -class RecordingIn(RecordingPropertyIn, RecordingRelationIn): +class RecordingIn(RecordingPropertyIn, RecordingRelationIn, ImportableModel): """ Full model of recording to acquire from client - """ diff --git a/grisera/registered_channel/registered_channel_model.py b/grisera/registered_channel/registered_channel_model.py index 59bd516..ba8fa2d 100644 --- a/grisera/registered_channel/registered_channel_model.py +++ b/grisera/registered_channel/registered_channel_model.py @@ -3,9 +3,11 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel +from grisera.property.property_model import PropertyIn -class RegisteredChannelIn(BaseModel): +class RegisteredChannelIn(ImportableModel): """ Model of registered channel to acquire from client @@ -16,6 +18,7 @@ class RegisteredChannelIn(BaseModel): channel_id: Optional[Union[int, str]] registered_data_id: Optional[Union[int, str]] + additional_properties: Optional[List[PropertyIn]] class BasicRegisteredChannelOut(RegisteredChannelIn): diff --git a/grisera/registered_data/registered_data_model.py b/grisera/registered_data/registered_data_model.py index 871f420..46b764f 100644 --- a/grisera/registered_data/registered_data_model.py +++ b/grisera/registered_data/registered_data_model.py @@ -3,10 +3,11 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn -class RegisteredDataIn(BaseModel): +class RegisteredDataIn(ImportableModel): """ Model of registered data to acquire from client diff --git a/grisera/registered_data/registered_data_router.py b/grisera/registered_data/registered_data_router.py index c38c172..8fbbd28 100644 --- a/grisera/registered_data/registered_data_router.py +++ b/grisera/registered_data/registered_data_router.py @@ -1,9 +1,10 @@ from typing import Union +from uuid import uuid4 -from fastapi import Response, Depends +from fastapi import Response, Depends, UploadFile, File from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter - +from grisera.clients.minio_client import MinIOClient from grisera.helpers.hateoas import get_links from grisera.helpers.helpers import check_dataset_permission from grisera.models.not_found_model import NotFoundByIdModel @@ -29,6 +30,73 @@ class RegisteredDataRouter: def __init__(self, service_factory: ServiceFactory = Depends(service.get_service_factory)): self.registered_data_service = service_factory.get_registered_data_service() + self.minio_client = MinIOClient("recordings") + + @router.post("/registered-data/upload-file", tags=["upload"]) + async def upload_file(self, response: Response, file: UploadFile = File(...)): + """ + Upload a file associated with a recording and store it in MinIO S3 storage + """ + try: + # Read file content + file_content = await file.read() + + # Generate unique object name + uuid = uuid4() + object_name = f"{uuid}/{file.filename}" + + # Upload using our MinIO client (configured for recordings bucket) + self.minio_client.upload_file(object_name, file_content, file.content_type) + + # Add HATEOAS links + links = get_links(router) + + response.status_code = 200 + return { + "message": "File uploaded successfully", + "object_name": object_name, # Store object name instead of public URL + "uuid": str(uuid), + "filename": file.filename, + "links": links, + } + except Exception as e: + response.status_code = 500 + return {"error": f"Failed to upload file: {str(e)}"} + + @router.get("/registered-data/preview/{object_name:path}", tags=["upload"]) + def get_preview_url(self, object_name: str, response: Response): + """ + Generate pre-signed URL for previewing recording file + + Args: + object_name (str): Object name in MinIO (uuid/filename format) + + Returns: + Dict: Pre-signed preview URL and metadata + """ + try: + # Extract filename for content type detection + filename = object_name.split('/')[-1] if '/' in object_name else object_name + + # Import here to avoid circular imports + from grisera.file.file_validation import FileValidator + validator = FileValidator() + content_type = validator.get_content_type(filename) + + if not validator.is_previewable(content_type): + response.status_code = 400 + return {"error": "Preview not available for this file type"} + + preview_info = self.minio_client.generate_preview_url( + object_name, + content_type + ) + + response.status_code = 200 + return preview_info + except Exception as e: + response.status_code = 500 + return {"error": f"Failed to generate preview URL: {str(e)}"} @router.post( "/registered_data", tags=["registered data"], response_model=RegisteredDataOut diff --git a/grisera/scenario/scenario_model.py b/grisera/scenario/scenario_model.py index bf257c2..cbcc869 100644 --- a/grisera/scenario/scenario_model.py +++ b/grisera/scenario/scenario_model.py @@ -5,20 +5,23 @@ from grisera.property.property_model import PropertyIn from grisera.activity_execution.activity_execution_model import ActivityExecutionIn from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel -class ScenarioIn(BaseModel): +class ScenarioIn(ImportableModel): """ Model of scenario to acquire from client Attributes: experiment_id (int | str): id of experiment, the scenario belongs to activity_executions (List[ActivityExecutionIn]): list of activity executions in scenario + additional_properties (Optional[List[PropertyIn]]): Additional properties for activity """ experiment_id: Optional[Union[int, str]] activity_executions: Optional[List[ActivityExecutionIn]] + additional_properties: Optional[List[PropertyIn]] diff --git a/grisera/services/not_implemented_service_factory.py b/grisera/services/not_implemented_service_factory.py index 50139e7..37db449 100644 --- a/grisera/services/not_implemented_service_factory.py +++ b/grisera/services/not_implemented_service_factory.py @@ -20,6 +20,7 @@ from grisera.scenario.scenario_service import ScenarioService from grisera.services.service_factory import ServiceFactory from grisera.time_series.time_series_service import TimeSeriesService +from grisera.file.file_service import FileService class NotImplementedServiceFactory(ServiceFactory): @@ -85,3 +86,6 @@ def get_scenario_service(self) -> ScenarioService: def get_time_series_service(self) -> TimeSeriesService: pass + + def get_file_service(self) -> FileService: + pass diff --git a/grisera/services/service_factory.py b/grisera/services/service_factory.py index 7270b3b..5332337 100644 --- a/grisera/services/service_factory.py +++ b/grisera/services/service_factory.py @@ -2,6 +2,7 @@ from grisera.activity.activity_service import ActivityService from grisera.activity_execution.activity_execution_service import ActivityExecutionService +from grisera.additional_parameter.additional_parameter_service import AdditionalParameterService from grisera.appearance.appearance_service import AppearanceService from grisera.arrangement.arrangement_service import ArrangementService from grisera.channel.channel_service import ChannelService @@ -21,6 +22,7 @@ from grisera.registered_data.registered_data_service import RegisteredDataService from grisera.scenario.scenario_service import ScenarioService from grisera.time_series.time_series_service import TimeSeriesService +from grisera.file.file_service import FileService class ServiceFactory(): @@ -108,3 +110,10 @@ def get_scenario_service(self) -> ScenarioService: @abstractmethod def get_time_series_service(self) -> TimeSeriesService: pass + + @abstractmethod + def get_file_service(self) -> FileService: + pass + + def get_additional_parameter_service(self) -> AdditionalParameterService: + pass diff --git a/grisera/time_series/time_series_model.py b/grisera/time_series/time_series_model.py index b600033..469f69e 100644 --- a/grisera/time_series/time_series_model.py +++ b/grisera/time_series/time_series_model.py @@ -4,6 +4,7 @@ from pydantic import BaseModel from grisera.models.base_model_out import BaseModelOut +from grisera.models.importable_model import ImportableModel from grisera.property.property_model import PropertyIn @@ -132,7 +133,7 @@ class TimeSeriesRelationIn(BaseModel): measure_id: Optional[Union[int, str]] -class TimeSeriesIn(TimeSeriesPropertyIn, TimeSeriesRelationIn): +class TimeSeriesIn(TimeSeriesPropertyIn, TimeSeriesRelationIn, ImportableModel): """ Full model of time series to acquire from client """ @@ -189,6 +190,17 @@ class TimeSeriesNodesOut(BaseModelOut): time_series_nodes: List[BasicTimeSeriesOut] = [] +class DetailedTimeSeriesNodesOut(BaseModelOut): + """ + Model of detailed time series nodes to send to client as a result of request + + Attributes: + time_series_nodes (List[TimeSeriesOut]): Detailed time series nodes with full relations from database + """ + + time_series_nodes: List[TimeSeriesOut] = [] + + # Circular import exception prevention from grisera.measure.measure_model import MeasureOut from grisera.observable_information.observable_information_model import ObservableInformationOut diff --git a/grisera/time_series/time_series_router.py b/grisera/time_series/time_series_router.py index 62a40bc..28f1dd8 100644 --- a/grisera/time_series/time_series_router.py +++ b/grisera/time_series/time_series_router.py @@ -1,10 +1,12 @@ -from typing import Union, Optional +from typing import Union, Optional, List +from uuid import uuid4 -from fastapi import Response, Depends +from fastapi import Response, Depends, UploadFile, File from fastapi_utils.cbv import cbv from fastapi_utils.inferring_router import InferringRouter from starlette.requests import Request +from grisera.clients.minio_client import MinIOClient from grisera.helpers.hateoas import get_links from grisera.helpers.helpers import check_dataset_permission from grisera.models.not_found_model import NotFoundByIdModel @@ -13,6 +15,7 @@ from grisera.time_series.time_series_model import ( TimeSeriesIn, TimeSeriesNodesOut, + DetailedTimeSeriesNodesOut, TimeSeriesOut, TimeSeriesPropertyIn, TimeSeriesRelationIn, @@ -34,6 +37,73 @@ class TimeSeriesRouter: def __init__(self, service_factory: ServiceFactory = Depends(service.get_service_factory)): self.time_series_service = service_factory.get_time_series_service() + self.minio_client = MinIOClient("recordings") # Same bucket as registered_data + + @router.post("/time-series/upload-file", tags=["upload"]) + async def upload_file(self, response: Response, file: UploadFile = File(...)): + """ + Upload a file associated with time series and store it in MinIO S3 storage + """ + try: + # Read file content + file_content = await file.read() + + # Generate unique object name + uuid = uuid4() + object_name = f"{uuid}/{file.filename}" + + # Upload using our MinIO client + self.minio_client.upload_file(object_name, file_content, file.content_type) + + # Add HATEOAS links + links = get_links(router) + + response.status_code = 200 + return { + "message": "File uploaded successfully", + "object_name": object_name, # Store object name instead of public URL + "uuid": str(uuid), + "filename": file.filename, + "links": links, + } + except Exception as e: + response.status_code = 500 + return {"error": f"Failed to upload file: {str(e)}"} + + @router.get("/time-series/preview/{object_name:path}", tags=["upload"]) + def get_preview_url(self, object_name: str, response: Response): + """ + Generate pre-signed URL for previewing time series file + + Args: + object_name (str): Object name in MinIO (uuid/filename format) + + Returns: + Dict: Pre-signed preview URL and metadata + """ + try: + # Extract filename for content type detection + filename = object_name.split('/')[-1] if '/' in object_name else object_name + + # Import here to avoid circular imports + from grisera.file.file_validation import FileValidator + validator = FileValidator() + content_type = validator.get_content_type(filename) + + if not validator.is_previewable(content_type): + response.status_code = 400 + return {"error": "Preview not available for this file type"} + + preview_info = self.minio_client.generate_preview_url( + object_name, + content_type + ) + + response.status_code = 200 + return preview_info + except Exception as e: + response.status_code = 500 + return {"error": f"Failed to generate preview URL: {str(e)}"} @router.post("/time_series", tags=["time series"], response_model=TimeSeriesOut) async def create_time_series(self, time_series: TimeSeriesIn, response: Response, dataset_id: Union[int, str]): @@ -124,6 +194,31 @@ async def get_time_series_nodes(self, response: Response, dataset_id: Union[int, return get_response + @router.get("/time_series/detailed", tags=["time series"], response_model=DetailedTimeSeriesNodesOut) + async def get_time_series_detailed(self, response: Response, dataset_id: Union[int, str], + activity_execution_id: str, + participant_id: str): + """ + Get time series with full details (observable informations, measures, etc.) from database. + + This endpoint is optimized for frontend use - returns detailed time series data + with all related entities included, filtered by activity execution and participant. + + Args: + dataset_id: Name of dataset + activity_execution_id: Filter by activity execution id (required) + participant_id: Filter by participant id (required) + """ + + get_response = self.time_series_service.get_time_series_detailed( + dataset_id, activity_execution_id, participant_id + ) + + # add links from hateoas + get_response.links = get_links(router) + + return get_response + @router.get( "/time_series/{time_series_id}", tags=["time series"], diff --git a/grisera/time_series/time_series_service.py b/grisera/time_series/time_series_service.py index d92f577..d2a9930 100644 --- a/grisera/time_series/time_series_service.py +++ b/grisera/time_series/time_series_service.py @@ -123,3 +123,22 @@ def update_time_series_relationships(self, time_series_id: Union[int, str], Result of request as time series object """ raise Exception("update_time_series_relationships not implemented yet") + + def get_time_series_detailed(self, dataset_id: Union[int, str], + activity_execution_id: str, + participant_id: str): + """ + Get time series with full details (observable informations, measures, etc.) from database. + + This method is optimized for frontend use - returns detailed time series data + with all related entities included, filtered by activity execution and participant. + + Args: + dataset_id (int | str): name of dataset + activity_execution_id (str): Filter by activity execution id (required) + participant_id (str): Filter by participant id (required) + + Returns: + DetailedTimeSeriesNodesOut: Detailed time series objects with full relations + """ + raise Exception("get_time_series_detailed not implemented yet") diff --git a/setup.py b/setup.py index 3bb25e2..130ffe6 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ from setuptools import setup, find_packages -VERSION = '0.0.38.30' +# VERSION = '0.0.38.30' +VERSION = 'dev' DESCRIPTION = 'Grisera-api package' LONG_DESCRIPTION = 'Graph Representation Integrating Signals for Emotion Recognition and Analysis (GRISERA) framework provides a persistent model for storing integrated signals and methods for its creation.' @@ -21,7 +22,8 @@ 'fastapi-utils', 'pydantic~=1.10.6', 'starlette~=0.26.1', - 'pyjwt' + 'pyjwt', + 'minio~=7.1.0', ], classifiers=[ "Development Status :: 1 - Planning", @@ -30,4 +32,4 @@ "Operating System :: Unix", "Operating System :: Microsoft :: Windows", ] -) \ No newline at end of file +)