diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d368e26..8d82212a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ Changes are grouped as follows - `Security` in case of vulnerabilities. +## 7.10.1 + +### Fixed + +* Removed manual setting of `is_uploaded` property on `CogniteExtractorFileApply` objects during file upload to allow SDK to manage upload status. + + ## 7.10.0 ### Added diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index ff5383df..1b4e8fc5 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,7 +16,7 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "7.10.0" +__version__ = "7.10.1" from .base import Extractor __all__ = ["Extractor"] diff --git a/cognite/extractorutils/uploader/files.py b/cognite/extractorutils/uploader/files.py index 1bfaea9a..93911127 100644 --- a/cognite/extractorutils/uploader/files.py +++ b/cognite/extractorutils/uploader/files.py @@ -520,9 +520,6 @@ def upload_file( else: self._upload_bytes(size, file, file_meta) - if isinstance(file_meta, CogniteExtractorFileApply): - file_meta.is_uploaded = True - if self.post_upload_function: try: self.post_upload_function([file_meta]) diff --git a/pyproject.toml b/pyproject.toml index 2bc57839..2390d1cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cognite-extractor-utils" -version = "7.10.0" +version = "7.10.1" description = "Utilities for easier development of extractors for CDF" authors = [ {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"} @@ -15,7 +15,7 @@ classifiers = [ ] dependencies = [ - "cognite-sdk>=7.75.2", + "cognite-sdk>=7.90.0", "prometheus-client>=0.7.0,<=1.0.0", "arrow>=1.0.0", "pyyaml>=5.3.0,<7", diff --git a/tests/tests_unit/test_cdf_upload_queues.py b/tests/tests_unit/test_cdf_upload_queues.py index 04d94687..7c53107c 100644 --- a/tests/tests_unit/test_cdf_upload_queues.py +++ b/tests/tests_unit/test_cdf_upload_queues.py @@ -16,12 +16,16 @@ import time from datetime import datetime, timezone from io import BytesIO -from typing import Any +from typing import Any, BinaryIO from unittest.mock import Mock, patch from httpx import URL, Request -from cognite.client.data_classes import Event, Row +from cognite.client.data_classes import Event, FileMetadata, Row +from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import ( + CogniteExtractorFileApply, +) +from cognite.client.testing import CogniteClientMock from cognite.extractorutils.uploader import ( EventUploadQueue, IOFileUploadQueue, @@ -300,3 +304,40 @@ def test_mock_private_link_upload(MockCogniteClient: Mock) -> None: response: Request = queue._get_file_upload_request(mock_download_url, mock_stream, len(bytes_)) assert response.url.netloc == base_url.netloc + + +@patch("cognite.client.CogniteClient") +def test_manually_set_is_uploaded(mock_client: CogniteClientMock) -> None: + mock_client.config.max_workers = 4 + + queue = IOFileUploadQueue( + cdf_client=mock_client, + max_queue_size=10, + ) + + file_apply = CogniteExtractorFileApply( + external_id="test-file", + name="test-file.txt", + space="test-space", + ) + + # Record the initial state of is_uploaded + initial_is_uploaded = file_apply.is_uploaded + + def read_file() -> BinaryIO: + return BytesIO(b"test content") + + # Mock the upload methods + with patch.object(queue, "_upload_bytes"), patch.object(queue, "_upload_only_metadata") as mock_upload_metadata: + mock_file_metadata = Mock(spec=FileMetadata) + mock_file_metadata.mime_type = "text/plain" + mock_upload_metadata.return_value = (mock_file_metadata, "https://upload.url") + + with queue: + queue.add_io_to_upload_queue(file_apply, read_file) + queue.upload() + + # Verify that is_uploaded was NOT manually changed by the uploader + assert file_apply.is_uploaded == initial_is_uploaded, ( + "is_uploaded should not be manually changed by the uploader, it should be managed by the SDK" + )