From fd4b8ef26a7743b0a9c9d07dbad8bd03e39574ec Mon Sep 17 00:00:00 2001 From: Hemant Mishra Date: Wed, 26 Nov 2025 20:11:50 +0530 Subject: [PATCH 1/3] Removed is_uploaded manual updation --- CHANGELOG.md | 7 +++++++ cognite/extractorutils/__init__.py | 2 +- cognite/extractorutils/uploader/files.py | 3 --- pyproject.toml | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d368e26..5bd08948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,13 @@ Changes are grouped as follows - `Security` in case of vulnerabilities. +## 7.11.0 + +### Fixed + +* Removed manual setting of `is_uploaded` property on `CogniteExtractorFileApply` objects during file upload to allow SDK to manage upload status. + + ## 7.10.0 ### Added diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index ff5383df..ef9f8213 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,7 +16,7 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "7.10.0" +__version__ = "7.11.0" from .base import Extractor __all__ = ["Extractor"] diff --git a/cognite/extractorutils/uploader/files.py b/cognite/extractorutils/uploader/files.py index 1bfaea9a..93911127 100644 --- a/cognite/extractorutils/uploader/files.py +++ b/cognite/extractorutils/uploader/files.py @@ -520,9 +520,6 @@ def upload_file( else: self._upload_bytes(size, file, file_meta) - if isinstance(file_meta, CogniteExtractorFileApply): - file_meta.is_uploaded = True - if self.post_upload_function: try: self.post_upload_function([file_meta]) diff --git a/pyproject.toml b/pyproject.toml index 2bc57839..4cd54130 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cognite-extractor-utils" -version = "7.10.0" +version = "7.11.0" description = "Utilities for easier development of extractors for CDF" authors = [ {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"} From cbcd2eaabc864390580a3a287072d3c0e6429244 Mon Sep 17 00:00:00 2001 From: Hemant Mishra Date: Thu, 27 Nov 2025 14:36:05 +0530 Subject: [PATCH 2/3] Add test cases and update sdk --- pyproject.toml | 2 +- tests/tests_unit/test_cdf_upload_queues.py | 45 +++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4cd54130..fa581fa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ ] dependencies = [ - "cognite-sdk>=7.75.2", + "cognite-sdk>=7.90.0", "prometheus-client>=0.7.0,<=1.0.0", "arrow>=1.0.0", "pyyaml>=5.3.0,<7", diff --git a/tests/tests_unit/test_cdf_upload_queues.py b/tests/tests_unit/test_cdf_upload_queues.py index 04d94687..7c53107c 100644 --- a/tests/tests_unit/test_cdf_upload_queues.py +++ b/tests/tests_unit/test_cdf_upload_queues.py @@ -16,12 +16,16 @@ import time from datetime import datetime, timezone from io import BytesIO -from typing import Any +from typing import Any, BinaryIO from unittest.mock import Mock, patch from httpx import URL, Request -from cognite.client.data_classes import Event, Row +from cognite.client.data_classes import Event, FileMetadata, Row +from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import ( + CogniteExtractorFileApply, +) +from cognite.client.testing import CogniteClientMock from cognite.extractorutils.uploader import ( EventUploadQueue, IOFileUploadQueue, @@ -300,3 +304,40 @@ def test_mock_private_link_upload(MockCogniteClient: Mock) -> None: response: Request = queue._get_file_upload_request(mock_download_url, mock_stream, len(bytes_)) assert response.url.netloc == base_url.netloc + + +@patch("cognite.client.CogniteClient") +def test_manually_set_is_uploaded(mock_client: CogniteClientMock) -> None: + mock_client.config.max_workers = 4 + + queue = IOFileUploadQueue( + cdf_client=mock_client, + max_queue_size=10, + ) + + file_apply = CogniteExtractorFileApply( + external_id="test-file", + name="test-file.txt", + space="test-space", + ) + + # Record the initial state of is_uploaded + initial_is_uploaded = file_apply.is_uploaded + + def read_file() -> BinaryIO: + return BytesIO(b"test content") + + # Mock the upload methods + with patch.object(queue, "_upload_bytes"), patch.object(queue, "_upload_only_metadata") as mock_upload_metadata: + mock_file_metadata = Mock(spec=FileMetadata) + mock_file_metadata.mime_type = "text/plain" + mock_upload_metadata.return_value = (mock_file_metadata, "https://upload.url") + + with queue: + queue.add_io_to_upload_queue(file_apply, read_file) + queue.upload() + + # Verify that is_uploaded was NOT manually changed by the uploader + assert file_apply.is_uploaded == initial_is_uploaded, ( + "is_uploaded should not be manually changed by the uploader, it should be managed by the SDK" + ) From b8a2115a33b1f5116ed6f6a07e94dd8c0288bce7 Mon Sep 17 00:00:00 2001 From: Hemant Mishra Date: Thu, 27 Nov 2025 17:39:37 +0530 Subject: [PATCH 3/3] Change version --- CHANGELOG.md | 2 +- cognite/extractorutils/__init__.py | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bd08948..8d82212a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Changes are grouped as follows - `Security` in case of vulnerabilities. -## 7.11.0 +## 7.10.1 ### Fixed diff --git a/cognite/extractorutils/__init__.py b/cognite/extractorutils/__init__.py index ef9f8213..1b4e8fc5 100644 --- a/cognite/extractorutils/__init__.py +++ b/cognite/extractorutils/__init__.py @@ -16,7 +16,7 @@ Cognite extractor utils is a Python package that simplifies the development of new extractors. """ -__version__ = "7.11.0" +__version__ = "7.10.1" from .base import Extractor __all__ = ["Extractor"] diff --git a/pyproject.toml b/pyproject.toml index fa581fa8..2390d1cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cognite-extractor-utils" -version = "7.11.0" +version = "7.10.1" description = "Utilities for easier development of extractors for CDF" authors = [ {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"}