Update ruff and fix linting issues

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     -   id: ruff
         args: [--fix, --exit-non-zero-on-fix]
     -   id: ruff-format
-    rev: v0.11.12
+    rev: v0.15.6
 -   repo: local
     hooks:
     -   id: mypy
@@ -14,7 +14,7 @@ repos:
         entry: uv run mypy --non-interactive --install-types cognite
         pass_filenames: false
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-ast
       - id: debug-statements

cognite/extractorutils/base.py

@@ -28,11 +28,11 @@
 from types import TracebackType
 from typing import Any, Generic, Literal, TypeVar
 
-from dotenv import load_dotenv
-
 from cognite.client import CogniteClient
 from cognite.client.data_classes import ExtractionPipeline, ExtractionPipelineRun
 from cognite.client.exceptions import CogniteAPIError
+from dotenv import load_dotenv
+
 from cognite.extractorutils.configtools import (
     BaseConfig,
     ConfigResolver,

cognite/extractorutils/configtools/elements.py

@@ -28,16 +28,16 @@
 from urllib.parse import urljoin, urlparse
 
 import yaml
-from prometheus_client import REGISTRY, start_http_server
-from typing_extensions import Self
-
 from cognite.client import ClientConfig, CogniteClient
 from cognite.client.credentials import (
     CredentialProvider,
     OAuthClientCertificate,
     OAuthClientCredentials,
 )
 from cognite.client.data_classes import Asset, DataSet, ExtractionPipeline
+from prometheus_client import REGISTRY, start_http_server
+from typing_extensions import Self
+
 from cognite.extractorutils._inner_util import resolve_log_level_for_httpx
 from cognite.extractorutils.configtools._util import _load_certificate_data
 from cognite.extractorutils.exceptions import InvalidConfigError

cognite/extractorutils/configtools/loaders.py

@@ -34,9 +34,9 @@
 from azure.core.exceptions import HttpResponseError, ResourceNotFoundError, ServiceRequestError
 from azure.identity import ClientSecretCredential, DefaultAzureCredential
 from azure.keyvault.secrets import SecretClient
+from cognite.client import CogniteClient
 from yaml.scanner import ScannerError
 
-from cognite.client import CogniteClient
 from cognite.extractorutils.configtools._util import _to_snake_case
 from cognite.extractorutils.configtools.elements import (
     BaseConfig,

cognite/extractorutils/metrics.py

@@ -48,13 +48,13 @@ def __init__(self):
 
 import arrow
 import psutil
+from cognite.client import CogniteClient
+from cognite.client.data_classes import Asset, TimeSeries
+from cognite.client.exceptions import CogniteDuplicatedError
 from prometheus_client import Gauge, Info, Metric
 from prometheus_client.core import REGISTRY
 from prometheus_client.exposition import basic_auth_handler, delete_from_gateway, pushadd_to_gateway
 
-from cognite.client import CogniteClient
-from cognite.client.data_classes import Asset, TimeSeries
-from cognite.client.exceptions import CogniteDuplicatedError
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader.time_series import DataPointList, TimeSeriesUploadQueue
 from cognite.extractorutils.util import EitherId

cognite/extractorutils/statestore/hashing.py

@@ -15,10 +15,10 @@
 from typing import Any
 
 import orjson
-
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Row
 from cognite.client.exceptions import CogniteAPIError
+
 from cognite.extractorutils._inner_util import _DecimalDecoder, _DecimalEncoder
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.util import cognite_exceptions, retry

cognite/extractorutils/statestore/watermark.py

@@ -41,6 +41,7 @@
 
 from cognite.client import CogniteClient
 from cognite.client.exceptions import CogniteAPIError
+
 from cognite.extractorutils._inner_util import _DecimalDecoder, _DecimalEncoder
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader import DataPointList

cognite/extractorutils/unstable/configuration/loaders.py

@@ -8,10 +8,10 @@
 from pathlib import Path
 from typing import TextIO, TypeVar
 
-from pydantic import ValidationError
-
 from cognite.client import CogniteClient
 from cognite.client.exceptions import CogniteAPIError
+from pydantic import ValidationError
+
 from cognite.extractorutils.configtools.loaders import _load_yaml_dict_raw
 from cognite.extractorutils.exceptions import InvalidConfigError as OldInvalidConfigError
 from cognite.extractorutils.unstable.configuration.exceptions import InvalidConfigError

cognite/extractorutils/unstable/configuration/models.py

@@ -11,12 +11,6 @@
 from time import sleep
 from typing import Annotated, Any, Literal, TypeVar
 
-from humps import kebabize
-from prometheus_client import REGISTRY, start_http_server
-from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler
-from pydantic_core import CoreSchema, core_schema
-from typing_extensions import assert_never
-
 from cognite.client import CogniteClient
 from cognite.client.config import ClientConfig
 from cognite.client.credentials import (
@@ -25,6 +19,12 @@
     OAuthClientCredentials,
 )
 from cognite.client.data_classes import Asset, DataSet
+from humps import kebabize
+from prometheus_client import REGISTRY, start_http_server
+from pydantic import BaseModel, ConfigDict, Field, GetCoreSchemaHandler
+from pydantic_core import CoreSchema, core_schema
+from typing_extensions import assert_never
+
 from cognite.extractorutils.configtools._util import _load_certificate_data
 from cognite.extractorutils.exceptions import InvalidConfigError
 from cognite.extractorutils.metrics import AbstractMetricsPusher, CognitePusher, PrometheusPusher

cognite/extractorutils/unstable/core/checkin_worker.py

@@ -16,10 +16,10 @@
 from threading import RLock
 from time import sleep
 
-from requests import Response
-
 from cognite.client import CogniteClient
 from cognite.client.exceptions import CogniteAPIError, CogniteAuthError, CogniteConnectionError
+from requests import Response
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.unstable.configuration.models import ConfigRevision
 from cognite.extractorutils.unstable.core._dto import (

cognite/extractorutils/unstable/core/runtime.py

@@ -38,15 +38,15 @@ def main() -> None:
 from typing import Any, Generic, TypeVar
 from uuid import uuid4
 
-from requests.exceptions import ConnectionError as RequestsConnectionError
-from typing_extensions import assert_never
-
 from cognite.client import CogniteClient
 from cognite.client.exceptions import (
     CogniteAPIError,
     CogniteAuthError,
     CogniteConnectionError,
 )
+from requests.exceptions import ConnectionError as RequestsConnectionError
+from typing_extensions import assert_never
+
 from cognite.extractorutils.metrics import BaseMetrics, MetricsType
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.unstable.configuration.exceptions import InvalidArgumentError, InvalidConfigError

cognite/extractorutils/uploader/_base.py

@@ -20,8 +20,8 @@
 from typing import Any
 
 from arrow import Arrow
-
 from cognite.client import CogniteClient
+
 from cognite.extractorutils._inner_util import _resolve_log_level
 from cognite.extractorutils.threading import CancellationToken
 

cognite/extractorutils/uploader/assets.py

@@ -23,6 +23,7 @@
 from cognite.client import CogniteClient
 from cognite.client.data_classes.assets import Asset
 from cognite.client.exceptions import CogniteDuplicatedError
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,

cognite/extractorutils/uploader/data_modeling.py

@@ -8,6 +8,7 @@
 
 from cognite.client import CogniteClient
 from cognite.client.data_classes.data_modeling import EdgeApply, NodeApply
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,

cognite/extractorutils/uploader/events.py

@@ -22,6 +22,7 @@
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Event
 from cognite.client.exceptions import CogniteDuplicatedError
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,

cognite/extractorutils/uploader/files.py

@@ -29,17 +29,17 @@
 )
 from urllib.parse import ParseResult, urlparse
 
-from h11._util import LocalProtocolError
-from httpx import URL, Client, Headers, Request, StreamConsumed, SyncByteStream
-from requests.utils import super_len
-
 from cognite.client import CogniteClient
 from cognite.client.data_classes import FileMetadata, FileMetadataUpdate
 from cognite.client.data_classes.data_modeling import NodeId
 from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import (
     CogniteExtractorFileApply,
 )
 from cognite.client.utils._identifier import IdentifierSequence
+from h11._util import LocalProtocolError
+from httpx import URL, Client, Headers, Request, StreamConsumed, SyncByteStream
+from requests.utils import super_len
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,
@@ -383,7 +383,7 @@ def _upload_empty_file(
         self,
         file_meta: FileMetadataOrCogniteExtractorFile,
     ) -> None:
-        file_meta_response, url = self._upload_only_metadata(file_meta)
+        _file_meta_response, url = self._upload_only_metadata(file_meta)
 
         self._upload_only_file_reference(file_meta, url)
 

cognite/extractorutils/uploader/raw.py

@@ -22,9 +22,9 @@
 
 import arrow
 from arrow import Arrow
-
 from cognite.client import CogniteClient
 from cognite.client.data_classes import Row
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,

cognite/extractorutils/uploader/time_series.py

@@ -33,6 +33,7 @@
 from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import CogniteExtractorTimeSeriesApply
 from cognite.client.data_classes.data_modeling.instances import DirectRelationReference
 from cognite.client.exceptions import CogniteDuplicatedError, CogniteNotFoundError
+
 from cognite.extractorutils.threading import CancellationToken
 from cognite.extractorutils.uploader._base import (
     RETRIES,

cognite/extractorutils/uploader_extractor.py

@@ -23,9 +23,9 @@
 from types import TracebackType
 from typing import Any, TypeVar
 
+from cognite.client import CogniteClient
 from more_itertools import peekable
 
-from cognite.client import CogniteClient
 from cognite.extractorutils.base import Extractor
 from cognite.extractorutils.configtools import BaseConfig, TimeIntervalConfig
 from cognite.extractorutils.metrics import BaseMetrics
@@ -176,7 +176,7 @@ def __enter__(self) -> "UploaderExtractor":
         """
         super().__enter__()
 
-        queue_config = self.config.queues if self.config.queues else QueueConfigClass()
+        queue_config = self.config.queues or QueueConfigClass()
         self.event_queue = EventUploadQueue(
             self.cognite_client,
             max_queue_size=queue_config.event_size,

cognite/extractorutils/uploader_types.py

@@ -10,6 +10,7 @@
 from cognite.client.data_classes import Event as _Event
 from cognite.client.data_classes import Row as _Row
 from cognite.client.data_classes.data_modeling import NodeId
+
 from cognite.extractorutils.uploader.time_series import DataPoint
 
 

cognite/extractorutils/util.py

@@ -19,21 +19,21 @@
 import io
 import logging
 import random
-from collections.abc import Callable, Generator, Iterable
+from collections.abc import Callable, Generator, Iterable, Sequence
 from datetime import datetime, timezone
 from functools import partial, wraps
 from io import RawIOBase
 from threading import Thread
 from time import time
 from typing import Any, TypeVar
 
-from decorator import decorator
-
 from cognite.client import CogniteClient
 from cognite.client._api.assets import AssetsAPI
 from cognite.client._api.time_series import TimeSeriesAPI
 from cognite.client.data_classes import Asset, ExtractionPipelineRun, TimeSeries
 from cognite.client.exceptions import CogniteAPIError, CogniteException, CogniteFileUploadError, CogniteNotFoundError
+from decorator import decorator
+
 from cognite.extractorutils.threading import CancellationToken
 
 
@@ -400,8 +400,8 @@ def retry(
 
     @decorator
     def retry_decorator(f: Callable[..., _T2], *fargs: Any, **fkwargs: Any) -> _T2:  # noqa: ANN401
-        args = fargs if fargs else []
-        kwargs = fkwargs if fkwargs else {}
+        args: Sequence[Any] = fargs or []
+        kwargs: dict[str, Any] = fkwargs or {}
 
         return _retry_internal(
             partial(f, *args, **kwargs),

pyproject.toml

@@ -36,16 +36,10 @@ dependencies = [
     "simple-winservice>=0.1.0 ; sys_platform == 'win32'",
 ]
 
-[project.urls]
-repository = "https://github.com/cognitedata/python-extractor-utils"
-
-[project.optional-dependencies]
-experimental = ["cognite-sdk-experimental"]
-
-[tool.uv]
-dev-dependencies = [
-    "mypy>=1.14.1",
-    "ruff>=0.9.0",
+[dependency-groups]
+dev = [
+    "mypy>=1.19.1",
+    "ruff>=0.15.0",
     "pytest>=8.0.0",
     "pytest-cov>=6.0.0",
     "sphinx>=7.0.0",
@@ -65,6 +59,12 @@ dev-dependencies = [
     "requests-mock>=1.12.1",
 ]
 
+[project.urls]
+repository = "https://github.com/cognitedata/python-extractor-utils"
+
+[project.optional-dependencies]
+experimental = ["cognite-sdk-experimental"]
+
 [tool.ruff]
 exclude = [
     ".git",
@@ -123,8 +123,5 @@ only-include = ["cognite"]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 
-[dependency-groups]
-dev = []
-
 [project.scripts]
 simple-extractor = "cognite.examples.unstable.extractors.simple_extractor.main:main"

tests/conftest.py

@@ -5,13 +5,13 @@
 from enum import Enum
 
 import pytest
-from prometheus_client.core import REGISTRY
-
 from cognite.client import CogniteClient
 from cognite.client.config import ClientConfig
 from cognite.client.credentials import OAuthClientCredentials
 from cognite.client.data_classes.data_modeling import NodeId
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
+from prometheus_client.core import REGISTRY
+
 from cognite.extractorutils import metrics
 
 NUM_NODES = 5000

tests/test_unstable/conftest.py

@@ -9,11 +9,11 @@
 
 import pytest
 import requests_mock
-from prometheus_client.core import Counter
-
 from cognite.client import CogniteClient
 from cognite.client.config import ClientConfig
 from cognite.client.credentials import OAuthClientCredentials
+from prometheus_client.core import Counter
+
 from cognite.extractorutils.metrics import BaseMetrics
 from cognite.extractorutils.unstable.configuration.models import (
     ConnectionConfig,