From 65fbfc2815cff4e81d39c302b225f717fc9efac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 11:50:30 +0200 Subject: [PATCH 01/16] refactor service module --- csfunctions/service/__init__.py | 10 ++++ csfunctions/service/base.py | 38 +++++++++++++++ csfunctions/{service.py => service/numgen.py} | 48 +------------------ poetry.lock | 16 ++++++- pyproject.toml | 3 +- 5 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 csfunctions/service/__init__.py create mode 100644 csfunctions/service/base.py rename csfunctions/{service.py => service/numgen.py} (50%) diff --git a/csfunctions/service/__init__.py b/csfunctions/service/__init__.py new file mode 100644 index 0000000..114adf7 --- /dev/null +++ b/csfunctions/service/__init__.py @@ -0,0 +1,10 @@ +from csfunctions.service.numgen import NumberGeneratorService + + +class Service: + """ + Provides access to services on the elements instance, e.g. generating numbers. + """ + + def __init__(self, service_url: str | None, service_token: str | None): + self.generator = NumberGeneratorService(service_url, service_token) diff --git a/csfunctions/service/base.py b/csfunctions/service/base.py new file mode 100644 index 0000000..ae0d568 --- /dev/null +++ b/csfunctions/service/base.py @@ -0,0 +1,38 @@ +from typing import Optional + +import requests + + +class Unauthorized(Exception): + pass + + +class BaseService: + """ + Base class for services. + """ + + def __init__(self, service_url: str | None, service_token: str | None): + self.service_url = service_url + self.service_token = service_token + + def request(self, endpoint: str, method: str = "GET", params: Optional[dict] = None) -> dict | list: + """ + Make a request to the access service. + """ + if self.service_url is None: + raise ValueError("No service url given.") + if self.service_token is None: + raise ValueError("No service token given.") + + headers = {"Authorization": f"Bearer {self.service_token}"} + params = params or {} + url = self.service_url.rstrip("/") + "/" + endpoint.lstrip("/") + response = requests.request(method, url=url, params=params, headers=headers, timeout=10) + + if response.status_code == 401: + raise Unauthorized + if response.status_code == 200: + return response.json() + else: + raise ValueError(f"Access service responded with status code {response.status_code}.") diff --git a/csfunctions/service.py b/csfunctions/service/numgen.py similarity index 50% rename from csfunctions/service.py rename to csfunctions/service/numgen.py index 2d9638a..69f605a 100644 --- a/csfunctions/service.py +++ b/csfunctions/service/numgen.py @@ -1,50 +1,4 @@ -from typing import Optional - -import requests - - -class Service: - """ - Provides access to services on the elements instance, e.g. generating numbers. - """ - - def __init__(self, service_url: str | None, service_token: str | None): - self.generator = NumberGeneratorService(service_url, service_token) - - -class Unauthorized(Exception): - pass - - -class BaseService: - """ - Base class for services. - """ - - def __init__(self, service_url: str | None, service_token: str | None): - self.service_url = service_url - self.service_token = service_token - - def request(self, endpoint: str, method: str = "GET", params: Optional[dict] = None) -> dict | list: - """ - Make a request to the access service. - """ - if self.service_url is None: - raise ValueError("No service url given.") - if self.service_token is None: - raise ValueError("No service token given.") - - headers = {"Authorization": f"Bearer {self.service_token}"} - params = params or {} - url = self.service_url.rstrip("/") + "/" + endpoint.lstrip("/") - response = requests.request(method, url=url, params=params, headers=headers, timeout=10) - - if response.status_code == 401: - raise Unauthorized - if response.status_code == 200: - return response.json() - else: - raise ValueError(f"Access service responded with status code {response.status_code}.") +from csfunctions.service.base import BaseService class NumberGeneratorService(BaseService): diff --git a/poetry.lock b/poetry.lock index cb93489..52da4a4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -884,6 +884,20 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "types-requests" +version = "2.32.4.20250809" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.9" +files = [ + {file = "types_requests-2.32.4.20250809-py3-none-any.whl", hash = "sha256:f73d1832fb519ece02c85b1f09d5f0dd3108938e7d47e7f94bbfa18a6782b163"}, + {file = "types_requests-2.32.4.20250809.tar.gz", hash = "sha256:d8060de1c8ee599311f56ff58010fb4902f462a1470802cf9f6ed27bc46c4df3"}, +] + +[package.dependencies] +urllib3 = ">=2" + [[package]] name = "typing-extensions" version = "4.13.0" @@ -988,4 +1002,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "ea4efaa8f5c19e40b84dbe5b646b7b9ed44a45a0e1ec1e0c0e20b675f1bdb154" +content-hash = "740404c62a8150ad07efae769ef5c2cce37eff365b002b66caa4e2b96f4ec43f" diff --git a/pyproject.toml b/pyproject.toml index c29badf..d962b9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contactsoftware-functions" -version = "0.14.0" +version = "0.15.0.dev0" readme = "README.md" license = "MIT" @@ -33,6 +33,7 @@ requests-mock = "^1.12.1" mkdocs = "^1.6.1" mkdocs-material = "^9.6.14" mkdocs-link-marker = "^0.1.3" +types-requests = "^2.32.4.20250809" [tool.ruff] line-length = 120 From 9fddab50c97ae0f8c386945d45555cea95868410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:23:38 +0200 Subject: [PATCH 02/16] file upload service --- csfunctions/handler.py | 4 +- csfunctions/service/__init__.py | 7 +- csfunctions/service/base.py | 40 +++-- csfunctions/service/file_upload.py | 161 +++++++++++++++++++++ csfunctions/service/file_upload_schemas.py | 73 ++++++++++ poetry.lock | 13 +- pyproject.toml | 1 + 7 files changed, 284 insertions(+), 15 deletions(-) create mode 100644 csfunctions/service/file_upload.py create mode 100644 csfunctions/service/file_upload_schemas.py diff --git a/csfunctions/handler.py b/csfunctions/handler.py index 44e85c0..9faff8c 100644 --- a/csfunctions/handler.py +++ b/csfunctions/handler.py @@ -96,9 +96,7 @@ def execute(function_name: str, request_body: str, function_dir: str = "src") -> link_objects(request.event) function_callback = get_function_callable(function_name, function_dir) - service = Service( - str(request.metadata.service_url) if request.metadata.service_url else None, request.metadata.service_token - ) + service = Service(metadata=request.metadata) response = function_callback(request.metadata, request.event, service) diff --git a/csfunctions/service/__init__.py b/csfunctions/service/__init__.py index 114adf7..f04bfd3 100644 --- a/csfunctions/service/__init__.py +++ b/csfunctions/service/__init__.py @@ -1,3 +1,5 @@ +from csfunctions.metadata import MetaData +from csfunctions.service.file_upload import FileUploadService from csfunctions.service.numgen import NumberGeneratorService @@ -6,5 +8,6 @@ class Service: Provides access to services on the elements instance, e.g. generating numbers. """ - def __init__(self, service_url: str | None, service_token: str | None): - self.generator = NumberGeneratorService(service_url, service_token) + def __init__(self, metadata: MetaData): + self.generator = NumberGeneratorService(metadata=metadata) + self.file_upload = FileUploadService(metadata=metadata) diff --git a/csfunctions/service/base.py b/csfunctions/service/base.py index ae0d568..301e5d2 100644 --- a/csfunctions/service/base.py +++ b/csfunctions/service/base.py @@ -2,36 +2,58 @@ import requests +from csfunctions.metadata import MetaData + class Unauthorized(Exception): pass +class Conflict(Exception): + pass + + +class NotFound(Exception): + pass + + +class UnprocessableEntity(Exception): + pass + + class BaseService: """ Base class for services. """ - def __init__(self, service_url: str | None, service_token: str | None): - self.service_url = service_url - self.service_token = service_token + def __init__(self, metadata: MetaData): + # Store full metadata for services that need additional fields (e.g. app_user) + self.metadata = metadata - def request(self, endpoint: str, method: str = "GET", params: Optional[dict] = None) -> dict | list: + def request( + self, endpoint: str, method: str = "GET", params: Optional[dict] = None, json: Optional[dict] = None + ) -> dict | list: """ Make a request to the access service. """ - if self.service_url is None: + if self.metadata.service_url is None: raise ValueError("No service url given.") - if self.service_token is None: + if self.metadata.service_token is None: raise ValueError("No service token given.") - headers = {"Authorization": f"Bearer {self.service_token}"} + headers = {"Authorization": f"Bearer {self.metadata.service_token}"} params = params or {} - url = self.service_url.rstrip("/") + "/" + endpoint.lstrip("/") - response = requests.request(method, url=url, params=params, headers=headers, timeout=10) + url = str(self.metadata.service_url).rstrip("/") + "/" + endpoint.lstrip("/") + response = requests.request(method, url=url, params=params, headers=headers, timeout=10, json=json) if response.status_code == 401: raise Unauthorized + elif response.status_code == 409: + raise Conflict + elif response.status_code == 404: + raise NotFound + elif response.status_code == 422: + raise UnprocessableEntity(response.text) if response.status_code == 200: return response.json() else: diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py new file mode 100644 index 0000000..7605508 --- /dev/null +++ b/csfunctions/service/file_upload.py @@ -0,0 +1,161 @@ +import hashlib +from copy import deepcopy +from random import choice +from string import ascii_letters +from typing import BinaryIO + +import requests + +from csfunctions.service.base import BaseService +from csfunctions.service.file_upload_schemas import ( + CompleteFileUploadRequest, + CreateNewFileRequest, + CreateNewFileResponse, + GeneratePresignedUrlRequest, + PresignedWriteUrls, +) + + +def _generate_lock_id(): + return "".join(choice(ascii_letters) for i in range(12)) # nosec + + +class FileUploadService(BaseService): + def _create_new_file(self, filename: str, parent_object_id: str, persno: str, check_access: bool = True) -> str: + """Creates a new (empty) file attached to the parent object and returns the cdb_object_id.""" + response_json = self.request( + endpoint="/file_upload/create", + method="POST", + json=CreateNewFileRequest( + parent_object_id=parent_object_id, filename=filename, persno=persno, check_access=check_access + ).model_dump(), + ) + data = CreateNewFileResponse.model_validate(response_json) + return data.file_object_id + + def _get_presigned_write_urls( + self, file_object_id: str, filesize: int, lock_id: str, persno: str, check_access: bool = True + ) -> PresignedWriteUrls: + response_json = self.request( + endpoint=f"/file_upload/{file_object_id}/generate_presigned_url", + method="POST", + json=GeneratePresignedUrlRequest( + check_access=check_access, persno=persno, filesize=filesize, lock_id=lock_id + ).model_dump(), + ) + + return PresignedWriteUrls.model_validate(response_json) + + def _upload_file_content( + self, presigned_urls: PresignedWriteUrls, stream: BinaryIO + ) -> tuple[PresignedWriteUrls, str]: + """Upload file stream in chunks using presigned URLs and return updated context + sha256 hash.""" + etags: list[str] = [] + sha256 = hashlib.sha256() + for url in presigned_urls.urls: + data: bytes = stream.read(presigned_urls.chunksize) + sha256.update(data) + resp = requests.put(url, data=data, headers=presigned_urls.headers, timeout=20) + # 20 second timeout to stay below 30s max execution time of the Function + # otherwise we won't get a proper error message in the logs + resp.raise_for_status() + etag = resp.headers.get("ETag") + if etag: + etags.append(etag) + updated = deepcopy(presigned_urls) + if etags: + updated.etags = etags + return updated, sha256.hexdigest() + + @staticmethod + def _get_stream_size(stream: BinaryIO) -> int: + if not stream.seekable(): + raise ValueError("Stream is not seekable; size cannot be determined.") + current_pos = stream.tell() + stream.seek(0, 2) + size = stream.tell() + stream.seek(current_pos) + return size + + def _complete_upload( + self, + file_object_id: str, + filesize: int, + lock_id: str, + presigned_urls: PresignedWriteUrls, + persno: str, + check_access: bool = True, + sha256: str | None = None, + delete_derived_files: bool = True, + ) -> None: + self.request( + endpoint=f"/file_upload/{file_object_id}/complete", + method="POST", + json=CompleteFileUploadRequest( + filesize=filesize, + check_access=check_access, + persno=persno, + presigned_write_urls=presigned_urls, + lock_id=lock_id, + sha256=sha256, + delete_derived_files=delete_derived_files, + ).model_dump(), + ) + + def upload_file_content( + self, + file_object_id: str, + stream: BinaryIO, + persno: str | None = None, + check_access: bool = True, + filesize: int | None = None, + delete_derived_files: bool = True, + ) -> None: + persno = persno or self.metadata.app_user + if filesize is None: + filesize = self._get_stream_size(stream) + lock_id = _generate_lock_id() + presigned = self._get_presigned_write_urls( + file_object_id=file_object_id, + filesize=filesize, + lock_id=lock_id, + persno=persno, + check_access=check_access, + ) + presigned_with_etags, sha256 = self._upload_file_content(presigned_urls=presigned, stream=stream) + self._complete_upload( + file_object_id=file_object_id, + filesize=filesize, + lock_id=lock_id, + presigned_urls=presigned_with_etags, + persno=persno, + check_access=check_access, + sha256=sha256, + delete_derived_files=delete_derived_files, + ) + + def upload_new_file( + self, + parent_object_id: str, + filename: str, + stream: BinaryIO, + persno: str | None = None, + check_access: bool = True, + filesize: int | None = None, + ) -> str: + persno = persno or self.metadata.app_user + file_object_id = self._create_new_file( + filename=filename, + parent_object_id=parent_object_id, + persno=persno, + check_access=check_access, + ) + self.upload_file_content( + file_object_id=file_object_id, + stream=stream, + persno=persno, + check_access=check_access, + filesize=filesize, + delete_derived_files=False, + ) + return file_object_id diff --git a/csfunctions/service/file_upload_schemas.py b/csfunctions/service/file_upload_schemas.py new file mode 100644 index 0000000..fc73ac9 --- /dev/null +++ b/csfunctions/service/file_upload_schemas.py @@ -0,0 +1,73 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class PresignedWriteUrls(BaseModel): + """ + Context object for managing upload information. + """ + + blob_id: str + urls: list[str] + chunksize: int + upload_id: Optional[str] = None + etags: Optional[list[str]] = None + block_ids: Optional[list[str]] = None + headers: Optional[dict[str, str]] = None + metadata: Optional[dict[str, str]] = None + signature: Optional[str] = None + + +class GeneratePresignedUrlRequest(BaseModel): + """ + Response model for generating presigned URLs. + """ + + filesize: int = Field(..., description="The size of the file you want to upload in bytes.", ge=0) + check_access: bool = Field(..., description="Whether to check access permissions.") + persno: str = Field(..., description="The persno of the user who is uploading the file.") + lock_id: str = Field(..., description="Provide some random string to lock the file for upload.") + + +class CompleteFileUploadRequest(BaseModel): + """ + Request model for completing a file upload. + """ + + filesize: int = Field(..., description="The size of the file you want to upload in bytes.", ge=0) + check_access: bool = Field(..., description="Whether to check access permissions.") + persno: str = Field(..., description="The persno of the user who is uploading the file.") + presigned_write_urls: PresignedWriteUrls = Field(..., description="The presigned write URLs for the file upload.") + sha256: Optional[str] = Field(None, description="The SHA256 hash of the file content.") + lock_id: str = Field(..., description="The lock ID the file was locked with") + delete_derived_files: bool = Field(True, description="Whether to delete derived files (e.g. converted pdfs).") + + +class AbortFileUploadRequest(BaseModel): + """ + Request model for aborting a file upload. + """ + + presigned_write_urls: PresignedWriteUrls = Field(..., description="The presigned write URLs for the file upload.") + lock_id: str = Field(..., description="The lock ID the file was locked with") + persno: str = Field(..., description="The persno of the user who is uploading the file.") + + +class CreateNewFileRequest(BaseModel): + """ + Request model for creating a new file. + """ + + parent_object_id: str = Field(..., description="cdb_object_id of the object the file should be attached to.") + filename: str = Field(..., description="The name of the file to create.") + persno: str = Field(..., description="The persno of the user creating the file.") + check_access: bool = Field(..., description="Whether to check access permissions.") + + +class CreateNewFileResponse(BaseModel): + """ + Response model for creating a new file. + """ + + file_object_id: str = Field(..., description="The cdb_object_id of the newly created file.") diff --git a/poetry.lock b/poetry.lock index 52da4a4..ca936e7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -884,6 +884,17 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20250822" +description = "Typing stubs for PyYAML" +optional = false +python-versions = ">=3.9" +files = [ + {file = "types_pyyaml-6.0.12.20250822-py3-none-any.whl", hash = "sha256:1fe1a5e146aa315483592d292b72a172b65b946a6d98aa6ddd8e4aa838ab7098"}, + {file = "types_pyyaml-6.0.12.20250822.tar.gz", hash = "sha256:259f1d93079d335730a9db7cff2bcaf65d7e04b4a56b5927d49a612199b59413"}, +] + [[package]] name = "types-requests" version = "2.32.4.20250809" @@ -1002,4 +1013,4 @@ watchdog = ["watchdog (>=2.3)"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "740404c62a8150ad07efae769ef5c2cce37eff365b002b66caa4e2b96f4ec43f" +content-hash = "b7712a2e8234c9518f559ed079c9cfce76ec255931c7ffd5cc4e44925884b805" diff --git a/pyproject.toml b/pyproject.toml index d962b9b..b7e23e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ mkdocs = "^1.6.1" mkdocs-material = "^9.6.14" mkdocs-link-marker = "^0.1.3" types-requests = "^2.32.4.20250809" +types-pyyaml = "^6.0.12.20250822" [tool.ruff] line-length = 120 From cb5e28dd6e99476a5d1cf43b53a488ed93b38cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:30:38 +0200 Subject: [PATCH 03/16] fix service test --- tests/test_service.py | 46 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index b98c537..d8f975d 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,9 +1,10 @@ +from datetime import datetime from unittest import TestCase import requests_mock -from csfunctions import Service -from csfunctions.service import BaseService, Unauthorized +from csfunctions import MetaData, Service +from csfunctions.service.base import BaseService, Unauthorized class TestNumberGeneratorService(TestCase): @@ -14,15 +15,30 @@ class TestNumberGeneratorService(TestCase): @classmethod def setUpClass(cls) -> None: - cls.service = Service(cls.service_url, cls.service_token) + cls.endpoint = "numgen" + cls.service_url = "https://some_service_url" + cls.service_token = "some_service_token" # nosec + metadata = MetaData.model_validate( + { + "request_id": "req-1", + "app_lang": "en", + "app_user": "tester", + "request_datetime": datetime(2000, 1, 1), + "transaction_id": "txn-1", + "instance_url": "https://instance.contact-cloud.com", + "service_url": cls.service_url, + "service_token": cls.service_token, + "db_service_url": None, + } + ) + cls.service = Service(metadata=metadata) @requests_mock.Mocker() def test_get_number(self, mock_request: requests_mock.Mocker): mock_request.get(f"{self.service_url}/{self.endpoint}?name=test&count=1", text='{"numbers": [1,2,3]}') - number = self.service.generator.get_number("test") last_request = mock_request.last_request - + self.assertIsNotNone(last_request) self.assertEqual("GET", last_request.method) self.assertEqual(f"Bearer {self.service_token}", last_request.headers["Authorization"]) self.assertEqual(1, number) @@ -30,10 +46,9 @@ def test_get_number(self, mock_request: requests_mock.Mocker): @requests_mock.Mocker() def test_get_numbers(self, mock_request: requests_mock.Mocker): mock_request.get(f"{self.service_url}/{self.endpoint}?name=test&count=3", text='{"numbers": [1,2,3]}') - numbers = self.service.generator.get_numbers("test", 3) last_request = mock_request.last_request - + self.assertIsNotNone(last_request) self.assertEqual("GET", last_request.method) self.assertEqual(f"Bearer {self.service_token}", last_request.headers["Authorization"]) self.assertEqual([1, 2, 3], numbers) @@ -48,10 +63,24 @@ def test_request(self, mock_request: requests_mock.Mocker): params = {"param1": 1, "param2": 2} mock_request.get(f"{service_url}/{endpoint}?param1=1¶m2=2", text="{}") - service = BaseService(service_url, service_token) + metadata = MetaData.model_validate( + { + "request_id": "req-1", + "app_lang": "en", + "app_user": "tester", + "request_datetime": datetime(2000, 1, 1), + "transaction_id": "txn-1", + "instance_url": "https://instance.contact-cloud.com", + "service_url": service_url, + "service_token": service_token, + "db_service_url": None, + } + ) + service = BaseService(metadata=metadata) response = service.request(endpoint, "GET", params) last_request = mock_request.last_request + self.assertIsNotNone(last_request) self.assertEqual("GET", last_request.method) self.assertEqual(f"Bearer {service_token}", last_request.headers["Authorization"]) self.assertEqual({}, response) @@ -70,6 +99,7 @@ def test_request(self, mock_request: requests_mock.Mocker): mock_request.get(f"{service_url}/{endpoint}", text="{}") response = service.request(endpoint, "GET") last_request = mock_request.last_request + self.assertIsNotNone(last_request) self.assertEqual("GET", last_request.method) self.assertEqual(f"Bearer {service_token}", last_request.headers["Authorization"]) self.assertEqual({}, response) From cd36ebd5378c3111dddb5dd206d19610e33031ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:45:56 +0200 Subject: [PATCH 04/16] add tests --- csfunctions/service/file_upload.py | 4 +- tests/test_service.py | 192 +++++++++++++++++++++++++++++ 2 files changed, 194 insertions(+), 2 deletions(-) diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index 7605508..4382d17 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -46,7 +46,7 @@ def _get_presigned_write_urls( return PresignedWriteUrls.model_validate(response_json) - def _upload_file_content( + def _upload_from_stream( self, presigned_urls: PresignedWriteUrls, stream: BinaryIO ) -> tuple[PresignedWriteUrls, str]: """Upload file stream in chunks using presigned URLs and return updated context + sha256 hash.""" @@ -122,7 +122,7 @@ def upload_file_content( persno=persno, check_access=check_access, ) - presigned_with_etags, sha256 = self._upload_file_content(presigned_urls=presigned, stream=stream) + presigned_with_etags, sha256 = self._upload_from_stream(presigned_urls=presigned, stream=stream) self._complete_upload( file_object_id=file_object_id, filesize=filesize, diff --git a/tests/test_service.py b/tests/test_service.py index d8f975d..4533009 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,10 +1,14 @@ +import io from datetime import datetime from unittest import TestCase +from unittest.mock import MagicMock, patch import requests_mock from csfunctions import MetaData, Service from csfunctions.service.base import BaseService, Unauthorized +from csfunctions.service.file_upload import FileUploadService +from csfunctions.service.file_upload_schemas import PresignedWriteUrls class TestNumberGeneratorService(TestCase): @@ -103,3 +107,191 @@ def test_request(self, mock_request: requests_mock.Mocker): self.assertEqual("GET", last_request.method) self.assertEqual(f"Bearer {service_token}", last_request.headers["Authorization"]) self.assertEqual({}, response) + + +class TestFileUploadService(TestCase): + def setUp(self): + self.metadata = MetaData.model_validate( + { + "request_id": "req-1", + "app_lang": "en", + "app_user": "tester", + "request_datetime": datetime(2000, 1, 1), + "transaction_id": "txn-1", + "instance_url": "https://instance.contact-cloud.com", + "service_url": "https://some_service_url", + "service_token": "some_service_token", + "db_service_url": None, + } + ) + self.service = FileUploadService(metadata=self.metadata) + + def test_create_new_file(self): + # Patch self.service.request to return a valid response + with patch.object(self.service, "request", return_value={"file_object_id": "file123"}) as mock_request: + file_id = self.service._create_new_file("test.txt", "parent1", "tester") + self.assertEqual(file_id, "file123") + mock_request.assert_called_once() + args, kwargs = mock_request.call_args + self.assertIn("endpoint", kwargs) + self.assertEqual(kwargs["endpoint"], "/file_upload/create") + self.assertEqual(kwargs["method"], "POST") + self.assertIn("json", kwargs) + self.assertEqual(kwargs["json"]["filename"], "test.txt") + self.assertEqual(kwargs["json"]["parent_object_id"], "parent1") + + def test_get_presigned_write_urls(self): + mock_response = { + "blob_id": "blob123", + "urls": ["https://upload.url/1", "https://upload.url/2"], + "chunksize": 1024, + "headers": {"Authorization": "Bearer token"}, + } + with patch.object(self.service, "request", return_value=mock_response) as mock_request: + result = self.service._get_presigned_write_urls("file123", 2048, "lockid", "tester") + self.assertEqual(result.blob_id, "blob123") + self.assertEqual(result.chunksize, 1024) + self.assertEqual(result.urls, ["https://upload.url/1", "https://upload.url/2"]) + mock_request.assert_called_once() + args, kwargs = mock_request.call_args + self.assertIn("endpoint", kwargs) + self.assertTrue("generate_presigned_url" in kwargs["endpoint"]) + self.assertEqual(kwargs["method"], "POST") + self.assertEqual(kwargs["json"]["filesize"], 2048) + self.assertEqual(kwargs["json"]["lock_id"], "lockid") + + def test_upload_from_stream(self): + # Test with multiple URLs and ETags + presigned = PresignedWriteUrls( + blob_id="blob123", + urls=["https://upload.url/1", "https://upload.url/2"], + chunksize=2, + headers={"Authorization": "Bearer token"}, + ) + # Simulate a file-like object with 4 bytes, so 2 chunks + stream = io.BytesIO(b"abcd") + # Each call to requests.put returns a different ETag + mock_response1 = MagicMock() + mock_response1.headers = {"ETag": "etag1"} + mock_response1.raise_for_status = MagicMock() + mock_response2 = MagicMock() + mock_response2.headers = {"ETag": "etag2"} + mock_response2.raise_for_status = MagicMock() + with patch("requests.put", side_effect=[mock_response1, mock_response2]) as mock_put: + updated, sha256 = self.service._upload_from_stream(presigned, stream) + self.assertEqual(updated.etags, ["etag1", "etag2"]) + self.assertEqual(len(sha256), 64) # sha256 hex length + import hashlib + + expected_hash = hashlib.sha256(b"abcd").hexdigest() + self.assertEqual(sha256, expected_hash) + self.assertEqual(mock_put.call_count, 2) + # Check each call + call_args_list = mock_put.call_args_list + self.assertEqual(call_args_list[0][0][0], "https://upload.url/1") + self.assertEqual(call_args_list[0][1]["data"], b"ab") + self.assertEqual(call_args_list[1][0][0], "https://upload.url/2") + self.assertEqual(call_args_list[1][1]["data"], b"cd") + for call_args in call_args_list: + self.assertEqual(call_args[1]["headers"], {"Authorization": "Bearer token"}) + + def test_get_stream_size(self): + stream = io.BytesIO(b"abcde") + size = self.service._get_stream_size(stream) + self.assertEqual(size, 5) + # Check that stream position is unchanged + self.assertEqual(stream.tell(), 0) + + def test_complete_upload(self): + presigned = PresignedWriteUrls( + blob_id="blob123", urls=["https://upload.url/1"], chunksize=4, headers={"Authorization": "Bearer token"} + ) + with patch.object(self.service, "request", return_value=None) as mock_request: + self.service._complete_upload( + file_object_id="file123", + filesize=4, + lock_id="lockid", + presigned_urls=presigned, + persno="tester", + sha256="deadbeef", + ) + mock_request.assert_called_once() + args, kwargs = mock_request.call_args + self.assertIn("endpoint", kwargs) + self.assertTrue("complete" in kwargs["endpoint"]) + self.assertEqual(kwargs["method"], "POST") + self.assertEqual(kwargs["json"].get("file_object_id", "file123"), "file123") + self.assertEqual(kwargs["json"]["sha256"], "deadbeef") + + def test_upload_new_file(self): + # Patch _create_new_file and upload_file_content + with ( + patch.object(self.service, "_create_new_file", return_value="file123") as mock_create, + patch.object(self.service, "upload_file_content", return_value=None) as mock_upload, + ): + stream = io.BytesIO(b"abc") + file_id = self.service.upload_new_file("parent1", "test.txt", stream) + self.assertEqual(file_id, "file123") + mock_create.assert_called_once_with( + filename="test.txt", + parent_object_id="parent1", + persno="tester", + check_access=True, + ) + mock_upload.assert_called_once() + args, kwargs = mock_upload.call_args + self.assertEqual(kwargs["file_object_id"], "file123") + self.assertEqual(kwargs["stream"].getvalue(), b"abc") + + def test_upload_file_content(self): + # Patch internal methods to isolate upload_file_content logic + with ( + patch.object(self.service, "_get_stream_size", return_value=4) as mock_size, + patch.object(self.service, "_get_presigned_write_urls") as mock_presigned, + patch.object(self.service, "_upload_from_stream") as mock_upload, + patch.object(self.service, "_complete_upload") as mock_complete, + ): + # Setup mocks + mock_presigned.return_value = PresignedWriteUrls( + blob_id="blob123", + urls=["https://upload.url/1", "https://upload.url/2"], + chunksize=2, + headers={"Authorization": "Bearer token"}, + ) + mock_upload.return_value = ( + PresignedWriteUrls( + blob_id="blob123", + urls=["https://upload.url/1", "https://upload.url/2"], + chunksize=2, + headers={"Authorization": "Bearer token"}, + etags=["etag1", "etag2"], + ), + "deadbeef", + ) + stream = io.BytesIO(b"abcd") + # Call method + self.service.upload_file_content( + file_object_id="file123", + stream=stream, + persno="tester", + check_access=True, + filesize=None, + delete_derived_files=False, + ) + mock_size.assert_called_once_with(stream) + mock_presigned.assert_called_once_with( + file_object_id="file123", + filesize=4, + lock_id=mock_presigned.call_args[1]["lock_id"], + persno="tester", + check_access=True, + ) + mock_upload.assert_called_once_with(presigned_urls=mock_presigned.return_value, stream=stream) + mock_complete.assert_called_once() + args, kwargs = mock_complete.call_args + self.assertEqual(kwargs["file_object_id"], "file123") + self.assertEqual(kwargs["filesize"], 4) + self.assertEqual(kwargs["presigned_urls"].etags, ["etag1", "etag2"]) + self.assertEqual(kwargs["persno"], "tester") + self.assertEqual(kwargs["sha256"], "deadbeef") + self.assertEqual(kwargs["delete_derived_files"], False) From 1a1f2585bf9b50993be63f01ef2ff7a8f58a144f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:52:14 +0200 Subject: [PATCH 05/16] abort on error --- csfunctions/service/file_upload.py | 46 +++++++++++++++++++------ tests/test_service.py | 54 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index 4382d17..0abc05d 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -8,6 +8,7 @@ from csfunctions.service.base import BaseService from csfunctions.service.file_upload_schemas import ( + AbortFileUploadRequest, CompleteFileUploadRequest, CreateNewFileRequest, CreateNewFileResponse, @@ -102,6 +103,19 @@ def _complete_upload( ).model_dump(), ) + def _abort_upload( + self, file_object_id: str, lock_id: str, persno: str, presigned_write_urls: PresignedWriteUrls + ) -> None: + self.request( + endpoint=f"/file_upload/{file_object_id}/abort", + method="POST", + json=AbortFileUploadRequest( + lock_id=lock_id, + persno=persno, + presigned_write_urls=presigned_write_urls, + ).model_dump(), + ) + def upload_file_content( self, file_object_id: str, @@ -122,17 +136,27 @@ def upload_file_content( persno=persno, check_access=check_access, ) - presigned_with_etags, sha256 = self._upload_from_stream(presigned_urls=presigned, stream=stream) - self._complete_upload( - file_object_id=file_object_id, - filesize=filesize, - lock_id=lock_id, - presigned_urls=presigned_with_etags, - persno=persno, - check_access=check_access, - sha256=sha256, - delete_derived_files=delete_derived_files, - ) + try: + presigned_with_etags, sha256 = self._upload_from_stream(presigned_urls=presigned, stream=stream) + self._complete_upload( + file_object_id=file_object_id, + filesize=filesize, + lock_id=lock_id, + presigned_urls=presigned_with_etags, + persno=persno, + check_access=check_access, + sha256=sha256, + delete_derived_files=delete_derived_files, + ) + except Exception as e: + # if something goes wrong during upload we try to abort + self._abort_upload( + file_object_id=file_object_id, + lock_id=lock_id, + persno=persno, + presigned_write_urls=presigned, + ) + raise e def upload_new_file( self, diff --git a/tests/test_service.py b/tests/test_service.py index 4533009..4434eec 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -295,3 +295,57 @@ def test_upload_file_content(self): self.assertEqual(kwargs["persno"], "tester") self.assertEqual(kwargs["sha256"], "deadbeef") self.assertEqual(kwargs["delete_derived_files"], False) + + def test_abort_upload(self): + presigned = PresignedWriteUrls( + blob_id="blob123", + urls=["https://upload.url/1"], + chunksize=4, + headers={"Authorization": "Bearer token"}, + ) + with patch.object(self.service, "request", return_value=None) as mock_request: + self.service._abort_upload( + file_object_id="file123", + lock_id="lockid", + persno="tester", + presigned_write_urls=presigned, + ) + mock_request.assert_called_once() + args, kwargs = mock_request.call_args + self.assertIn("endpoint", kwargs) + self.assertTrue("abort" in kwargs["endpoint"]) + self.assertEqual(kwargs["method"], "POST") + self.assertEqual(kwargs["json"]["lock_id"], "lockid") + self.assertEqual(kwargs["json"]["persno"], "tester") + self.assertEqual(kwargs["json"]["presigned_write_urls"], presigned.model_dump()) + + def test_upload_file_content_aborts_on_error(self): + # Patch internal methods to simulate error and check abort + with ( + patch.object(self.service, "_get_stream_size", return_value=4), + patch.object(self.service, "_get_presigned_write_urls") as mock_presigned, + patch.object(self.service, "_upload_from_stream", side_effect=Exception("upload error")), + patch.object(self.service, "_abort_upload") as mock_abort, + ): + mock_presigned.return_value = PresignedWriteUrls( + blob_id="blob123", + urls=["https://upload.url/1", "https://upload.url/2"], + chunksize=2, + headers={"Authorization": "Bearer token"}, + ) + stream = io.BytesIO(b"abcd") + with self.assertRaises(Exception) as cm: + self.service.upload_file_content( + file_object_id="file123", + stream=stream, + persno="tester", + check_access=True, + filesize=None, + delete_derived_files=False, + ) + self.assertEqual(str(cm.exception), "upload error") + mock_abort.assert_called_once() + args, kwargs = mock_abort.call_args + self.assertEqual(kwargs["file_object_id"], "file123") + self.assertEqual(kwargs["persno"], "tester") + self.assertEqual(kwargs["presigned_write_urls"], mock_presigned.return_value) From 3d6730ebb8e05459ee08251b714511cc046936ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:55:49 +0200 Subject: [PATCH 06/16] add docstrings --- csfunctions/service/file_upload.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index 0abc05d..df8b75c 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -125,6 +125,18 @@ def upload_file_content( filesize: int | None = None, delete_derived_files: bool = True, ) -> None: + """ + Uploads content to an existing file object in chunks using presigned URLs. + Handles aborting the upload if an error occurs. + + Args: + file_object_id: The ID of the file object to upload to. + stream: A binary stream containing the file data. + persno: The user/person number who is uploading the file (default is user that triggered the Function). + check_access: Whether to check access permissions. + filesize: Size of the file in bytes (required only if the stream is not seekable). + delete_derived_files: Whether to delete derived files after upload. + """ persno = persno or self.metadata.app_user if filesize is None: filesize = self._get_stream_size(stream) @@ -167,6 +179,20 @@ def upload_new_file( check_access: bool = True, filesize: int | None = None, ) -> str: + """ + Creates a new file attached to the parent object and uploads content from the provided stream. + + Args: + parent_object_id: The ID of the parent object to attach the file to. + filename: The name of the new file. + stream: A binary stream containing the file data. + persno: The user/person number who is uploading the file (default is user that triggered the Function). + check_access: Whether to check access permissions. + filesize: Size of the file in bytes (required only if the stream is not seekable). + + Returns: + The ID of the newly created file object. + """ persno = persno or self.metadata.app_user file_object_id = self._create_new_file( filename=filename, From b5cda044aa0dec8a4f2c40555f10ea665cba0da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 12:57:23 +0200 Subject: [PATCH 07/16] more docstrings --- csfunctions/service/file_upload.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index df8b75c..322df1e 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -18,12 +18,13 @@ def _generate_lock_id(): + """Generate a random 12-character lock ID.""" return "".join(choice(ascii_letters) for i in range(12)) # nosec class FileUploadService(BaseService): def _create_new_file(self, filename: str, parent_object_id: str, persno: str, check_access: bool = True) -> str: - """Creates a new (empty) file attached to the parent object and returns the cdb_object_id.""" + """Create a new empty file attached to the parent object.""" response_json = self.request( endpoint="/file_upload/create", method="POST", @@ -37,6 +38,7 @@ def _create_new_file(self, filename: str, parent_object_id: str, persno: str, ch def _get_presigned_write_urls( self, file_object_id: str, filesize: int, lock_id: str, persno: str, check_access: bool = True ) -> PresignedWriteUrls: + """Request presigned URLs for uploading file chunks.""" response_json = self.request( endpoint=f"/file_upload/{file_object_id}/generate_presigned_url", method="POST", @@ -50,7 +52,7 @@ def _get_presigned_write_urls( def _upload_from_stream( self, presigned_urls: PresignedWriteUrls, stream: BinaryIO ) -> tuple[PresignedWriteUrls, str]: - """Upload file stream in chunks using presigned URLs and return updated context + sha256 hash.""" + """Upload file stream in chunks and return updated presigned URLs and sha256 hash.""" etags: list[str] = [] sha256 = hashlib.sha256() for url in presigned_urls.urls: @@ -70,6 +72,7 @@ def _upload_from_stream( @staticmethod def _get_stream_size(stream: BinaryIO) -> int: + """Get the size of a seekable stream.""" if not stream.seekable(): raise ValueError("Stream is not seekable; size cannot be determined.") current_pos = stream.tell() @@ -89,6 +92,7 @@ def _complete_upload( sha256: str | None = None, delete_derived_files: bool = True, ) -> None: + """Mark the upload as complete and finalize the file.""" self.request( endpoint=f"/file_upload/{file_object_id}/complete", method="POST", @@ -106,6 +110,7 @@ def _complete_upload( def _abort_upload( self, file_object_id: str, lock_id: str, persno: str, presigned_write_urls: PresignedWriteUrls ) -> None: + """Abort an ongoing file upload.""" self.request( endpoint=f"/file_upload/{file_object_id}/abort", method="POST", From e7ffaf35054ff1ac00be657c1746e3c17ce25619 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 13:01:15 +0200 Subject: [PATCH 08/16] improve docstrings --- csfunctions/service/__init__.py | 11 +++++++++++ csfunctions/service/file_upload.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/csfunctions/service/__init__.py b/csfunctions/service/__init__.py index f04bfd3..7730438 100644 --- a/csfunctions/service/__init__.py +++ b/csfunctions/service/__init__.py @@ -1,7 +1,18 @@ from csfunctions.metadata import MetaData +from csfunctions.service.base import Conflict, NotFound, Unauthorized, UnprocessableEntity from csfunctions.service.file_upload import FileUploadService from csfunctions.service.numgen import NumberGeneratorService +__all__ = [ + "Service", + "FileUploadService", + "NumberGeneratorService", + "Conflict", + "NotFound", + "Unauthorized", + "UnprocessableEntity", +] + class Service: """ diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index 322df1e..f15d214 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -141,6 +141,11 @@ def upload_file_content( check_access: Whether to check access permissions. filesize: Size of the file in bytes (required only if the stream is not seekable). delete_derived_files: Whether to delete derived files after upload. + + Raises: + csfunctions.service.Unauthorized: If access check fails. + csfunctions.service.Conflict: If the file is already locked. + csfunctions.service.NotFound: If the file object does not exist. """ persno = persno or self.metadata.app_user if filesize is None: @@ -197,6 +202,10 @@ def upload_new_file( Returns: The ID of the newly created file object. + + Raises: + csfunctions.service.Unauthorized: If access check fails. + csfunctions.service.NotFound: If the parent object does not exist. """ persno = persno or self.metadata.app_user file_object_id = self._create_new_file( From f98126d1ab46919cd7dddb0ee9c5d27be42c5f09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 13:38:36 +0200 Subject: [PATCH 09/16] catch rate limit exceeded --- csfunctions/service/base.py | 6 ++++++ csfunctions/service/file_upload.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/csfunctions/service/base.py b/csfunctions/service/base.py index 301e5d2..4a0a0ae 100644 --- a/csfunctions/service/base.py +++ b/csfunctions/service/base.py @@ -21,6 +21,10 @@ class UnprocessableEntity(Exception): pass +class RateLimitExceeded(Exception): + pass + + class BaseService: """ Base class for services. @@ -54,6 +58,8 @@ def request( raise NotFound elif response.status_code == 422: raise UnprocessableEntity(response.text) + elif response.status_code == 429: + raise RateLimitExceeded(response.text) if response.status_code == 200: return response.json() else: diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index f15d214..2f11605 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -146,6 +146,7 @@ def upload_file_content( csfunctions.service.Unauthorized: If access check fails. csfunctions.service.Conflict: If the file is already locked. csfunctions.service.NotFound: If the file object does not exist. + csfunctions.service.RateLimitExceeded: If the services rate limit is exceeded. """ persno = persno or self.metadata.app_user if filesize is None: @@ -206,6 +207,7 @@ def upload_new_file( Raises: csfunctions.service.Unauthorized: If access check fails. csfunctions.service.NotFound: If the parent object does not exist. + csfunctions.service.RateLimitExceeded: If the services rate limit is exceeded. """ persno = persno or self.metadata.app_user file_object_id = self._create_new_file( From 66688990c83b8dd9a2cd4a8e18c25ff5e8b7a3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 13:38:57 +0200 Subject: [PATCH 10/16] add docs --- docs/reference/service.md | 139 ++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 2 files changed, 140 insertions(+) create mode 100644 docs/reference/service.md diff --git a/docs/reference/service.md b/docs/reference/service.md new file mode 100644 index 0000000..03b7912 --- /dev/null +++ b/docs/reference/service.md @@ -0,0 +1,139 @@ +# Functions Access Service + +The Functions Access Service provides access to backend services like number generators and file uplads. These services are available through the `service` parameter that is passed to all Functions. + +The Access Service is ratelimited to `100 req/min` per token. Functions recieve a fresh token for every call. + +```python +from csfunctions import Service + +def my_function(metadata, event, service: Service): + ... + +``` + +## Number Generation + + +You can use the `service.generator` to generate unique numbers, for example for part numbers or document IDs. + +### Methods + +- `get_number(name: str) -> int` + Retrieve one number from the given generator. +- `get_numbers(name: str, count: int) -> list[int]` + Retrieve multiple numbers from the given generator in one request. + Maximum for `count` is 100. + +**Example:** + +```python +new_number = service.generator.get_number("external_part_number") +# Returns an integer, e.g. 123 +``` + +To generate multiple numbers at once: + +```python +numbers = service.generator.get_numbers("external_part_number", count=5) +# Returns a list of integers +``` + +## File Uploads + + + +The `service.file_upload` object allows you to upload new files to the CIM Database Cloud or overwrite existing ones. + + + +### Upload a new file + +```python +service.upload_new_file( + self, + parent_object_id: str, + filename: str, + stream: BinaryIO, + persno: str | None = None, + check_access: bool = True, + filesize: int | None = None, + ) -> str: +``` +Creates a new file attached to the parent object and uploads content from the provided stream. Returns the new file object ID. + +| Parameter | Type | Description | +| ------------------ | ------------- | --------------------------------------------------------------------------------------------- | +| `parent_object_id` | `str` | The ID of the parent object to which the new file will be attached. | +| `filename` | `str` | The name of the new file to be uploaded. | +| `stream` | `BinaryIO` | A binary stream containing the file data to upload. | +| `persno` | `str \| None` | The user/person number uploading the file (defaults to the user that triggered the Function). | +| `check_access` | `bool` | Whether to check access permissions before uploading. Defaults to `True`. | +| `filesize` | `int \| None` | Size of the file in bytes (required only if the stream is not seekable). | + +**Exceptions:** + +- `csfunctions.service.Unauthorized`: If access check fails. +- `csfunctions.service.NotFound`: If the parent object does not exist. + + +!!! info + Uploading new files performs 3 requests to the Functions Access Service, which count towards the ratelimit of `100 req/min` per token. + +**Example:** + +```python +with open("myfile.pdf", "rb") as f: + file_object_id = service.file_upload.upload_new_file( + parent_object_id="123456", + filename="myfile.pdf", + stream=f + ) +``` + +### Overwrite an existing file + +```python +service.file_upload.upload_file_content( + file_object_id: str, + stream: BinaryIO, + persno: str | None = None, + check_access: bool = True, + filesize: int | None = None, + delete_derived_files: bool = True, + ) -> None: +``` +Uploads new content to an existing file object, overwriting its previous contents. + +| Parameter | Type | Description | +| ---------------------- | ------------- | -------------------------------------------------------------------------------------------------------------------- | +| `file_object_id` | `str` | The ID of the file object to upload to (must already exist). | +| `stream` | `BinaryIO` | A binary stream containing the file data to upload. | +| `persno` | `str \| None` | The user/person number uploading the file (defaults to the user that triggered the Function). | +| `check_access` | `bool` | Whether to check access permissions before uploading. Defaults to `True`. | +| `filesize` | `int \| None` | Size of the file in bytes (required only if the stream is not seekable). | +| `delete_derived_files` | `bool` | Whether to delete derived files (e.g. converted pdfs) after upload and trigger a new conversion. Defaults to `True`. | + +!!! warning + Overwriting files is only possible if the file is not locked! + +**Exceptions:** + +- `csfunctions.service.Unauthorized`: If access check fails. +- `csfunctions.service.Conflict`: If the file is already locked. +- `csfunctions.service.NotFound`: If the file object does not exist. +- `csfunctions.service.RateLimitExceeded`: If the services rate limit is exceeded. + +!!! info + Uploading new files performs 2 requests to the Functions Access Service, which count towards the ratelimit of `100 req/min` per token. + +**Example:** + +```python +file = doc.files[0] +with open("updated_file.pdf", "rb") as f: + service.file_upload.upload_file_content( + file_object_id=file.cdb_object_id, + stream=f + ) +``` diff --git a/mkdocs.yml b/mkdocs.yml index 182a258..a48dd2e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ nav: - reference/events.md - reference/objects.md - reference/actions.md + - reference/service.md - development_server.md - Python runtime: reference/runtime.md - Releases: release_notes.md From 0a2dbe3fb906e63637e8fc985f0bd72e006d152d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 14:58:58 +0200 Subject: [PATCH 11/16] Update docs/reference/service.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/reference/service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/service.md b/docs/reference/service.md index 03b7912..8269676 100644 --- a/docs/reference/service.md +++ b/docs/reference/service.md @@ -1,6 +1,6 @@ # Functions Access Service -The Functions Access Service provides access to backend services like number generators and file uplads. These services are available through the `service` parameter that is passed to all Functions. +The Functions Access Service provides access to backend services like number generators and file uploads. These services are available through the `service` parameter that is passed to all Functions. The Access Service is ratelimited to `100 req/min` per token. Functions recieve a fresh token for every call. From 540177dbfa82c0684daaf22fdd69ca1efbea93ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Fri, 22 Aug 2025 14:59:14 +0200 Subject: [PATCH 12/16] Update docs/reference/service.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docs/reference/service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/service.md b/docs/reference/service.md index 8269676..c25bfa8 100644 --- a/docs/reference/service.md +++ b/docs/reference/service.md @@ -2,7 +2,7 @@ The Functions Access Service provides access to backend services like number generators and file uploads. These services are available through the `service` parameter that is passed to all Functions. -The Access Service is ratelimited to `100 req/min` per token. Functions recieve a fresh token for every call. +The Access Service is ratelimited to `100 req/min` per token. Functions receive a fresh token for every call. ```python from csfunctions import Service From 8feb29f76425c18989001b2363420d3780943305 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Tue, 26 Aug 2025 11:35:41 +0200 Subject: [PATCH 13/16] update dev1 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b7e23e6..53689a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contactsoftware-functions" -version = "0.15.0.dev0" +version = "0.15.0.dev1" readme = "README.md" license = "MIT" From 4090d981db299bb1eedd43aadbb9cf22c60c930a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Tue, 26 Aug 2025 12:38:58 +0200 Subject: [PATCH 14/16] Add Forbidden exception handling and update documentation for access checks --- csfunctions/service/base.py | 6 ++++++ csfunctions/service/file_upload.py | 6 ++++-- docs/reference/service.md | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/csfunctions/service/base.py b/csfunctions/service/base.py index 4a0a0ae..f1ca8a6 100644 --- a/csfunctions/service/base.py +++ b/csfunctions/service/base.py @@ -9,6 +9,10 @@ class Unauthorized(Exception): pass +class Forbidden(Exception): + pass + + class Conflict(Exception): pass @@ -52,6 +56,8 @@ def request( if response.status_code == 401: raise Unauthorized + if response.status_code == 403: + raise Forbidden elif response.status_code == 409: raise Conflict elif response.status_code == 404: diff --git a/csfunctions/service/file_upload.py b/csfunctions/service/file_upload.py index 2f11605..2c3eddd 100644 --- a/csfunctions/service/file_upload.py +++ b/csfunctions/service/file_upload.py @@ -143,7 +143,8 @@ def upload_file_content( delete_derived_files: Whether to delete derived files after upload. Raises: - csfunctions.service.Unauthorized: If access check fails. + csfunctions.service.Forbidden: If access check fails. + csfunctions.service.Unauthorized: If the service token is invalid. csfunctions.service.Conflict: If the file is already locked. csfunctions.service.NotFound: If the file object does not exist. csfunctions.service.RateLimitExceeded: If the services rate limit is exceeded. @@ -205,7 +206,8 @@ def upload_new_file( The ID of the newly created file object. Raises: - csfunctions.service.Unauthorized: If access check fails. + csfunctions.service.Forbidden: If access check fails. + csfunctions.service.Unauthorized: If the service token is invalid. csfunctions.service.NotFound: If the parent object does not exist. csfunctions.service.RateLimitExceeded: If the services rate limit is exceeded. """ diff --git a/docs/reference/service.md b/docs/reference/service.md index c25bfa8..6a519d8 100644 --- a/docs/reference/service.md +++ b/docs/reference/service.md @@ -73,7 +73,8 @@ Creates a new file attached to the parent object and uploads content from the pr **Exceptions:** -- `csfunctions.service.Unauthorized`: If access check fails. +- `csfunctions.service.Unauthorized`: If th service token is invalid. +- `csfunctions.service.Forbidden`: If access check fails. - `csfunctions.service.NotFound`: If the parent object does not exist. @@ -119,7 +120,8 @@ Uploads new content to an existing file object, overwriting its previous content **Exceptions:** -- `csfunctions.service.Unauthorized`: If access check fails. +- `csfunctions.service.Unauthorized`: If th service token is invalid. +- `csfunctions.service.Forbidden`: If access check fails. - `csfunctions.service.Conflict`: If the file is already locked. - `csfunctions.service.NotFound`: If the file object does not exist. - `csfunctions.service.RateLimitExceeded`: If the services rate limit is exceeded. From a75fbc32fa8036cd27ffcf60423ec61310bbd23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Wed, 27 Aug 2025 09:29:33 +0200 Subject: [PATCH 15/16] Update docs/reference/service.md Co-authored-by: Julian Alberts --- docs/reference/service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/service.md b/docs/reference/service.md index 6a519d8..31c5d42 100644 --- a/docs/reference/service.md +++ b/docs/reference/service.md @@ -50,7 +50,7 @@ The `service.file_upload` object allows you to upload new files to the CIM Datab ### Upload a new file ```python -service.upload_new_file( +service.file_upload.upload_new_file( self, parent_object_id: str, filename: str, From 0461def8c7910821fe05d634dfa65057fcebf565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jens=20K=C3=BCrten?= Date: Wed, 27 Aug 2025 09:29:49 +0200 Subject: [PATCH 16/16] Update docs/reference/service.md Co-authored-by: Julian Alberts --- docs/reference/service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/service.md b/docs/reference/service.md index 31c5d42..31f2e63 100644 --- a/docs/reference/service.md +++ b/docs/reference/service.md @@ -102,7 +102,7 @@ service.file_upload.upload_file_content( check_access: bool = True, filesize: int | None = None, delete_derived_files: bool = True, - ) -> None: + ) -> None ``` Uploads new content to an existing file object, overwriting its previous contents.