diff --git a/pyproject.toml b/pyproject.toml index 176a9e42..6ce06c08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ dynamic = ["version"] description = "Async crawler and parsing service for data.gouv.fr" authors = [{ name = "Opendata Team", email = "opendatateam@data.gouv.fr" }] dependencies = [ + "boto3>=1.35.0", "aiohttp>=3.10.3", "asyncpg>=0.29.0", "coloredlogs>=15.0.1", @@ -16,7 +17,6 @@ dependencies = [ "humanfriendly>=10.0", "json-stream>=2.3.3", "marshmallow>=3.14.1", - "minio>=7.2.8", "owslib>=0.35.0", "progressist>=0.1.0", "pyarrow>=16.1.0", diff --git a/tests/test_analysis/test_analysis_csv.py b/tests/test_analysis/test_analysis_csv.py index ef2c34e3..f9b8a7b4 100644 --- a/tests/test_analysis/test_analysis_csv.py +++ b/tests/test_analysis/test_analysis_csv.py @@ -18,7 +18,7 @@ from udata_hydra.crawl.check_resources import check_resource from udata_hydra.db.check import Check from udata_hydra.db.resource import Resource -from udata_hydra.utils.minio import MinIOClient +from udata_hydra.utils.s3 import S3Client pytestmark = pytest.mark.asyncio @@ -652,30 +652,24 @@ async def test_csv_to_geojson_pmtiles(db, params, clean_db, mocker): assert res is None mock_func.assert_not_called() else: - minio_url = "my.minio.fr" + s3_endpoint = "s3.example.com" geojson_bucket = "geojson_bucket" - geojson_folder = "geojson_folder" pmtiles_bucket = "pmtiles_bucket" - pmtiles_folder = "pmtiles_folder" - mocker.patch("udata_hydra.config.MINIO_URL", minio_url) - mocked_minio = MagicMock() - mocked_minio.fput_object.return_value = None - mocked_minio.bucket_exists.return_value = True - with patch("udata_hydra.utils.minio.Minio", return_value=mocked_minio): - mocked_minio_client_geojson = MinIOClient( - bucket=geojson_bucket, folder=geojson_folder - ) - mocked_minio_client_pmtiles = MinIOClient( - bucket=pmtiles_bucket, folder=pmtiles_folder - ) + mocker.patch("udata_hydra.config.S3_ENDPOINT", s3_endpoint) + mocked_resource = MagicMock() + mocked_resource.meta.client.head_bucket.return_value = {} + mocked_resource.Bucket.return_value = MagicMock() + with patch("udata_hydra.utils.s3.boto3.resource", return_value=mocked_resource): + mocked_s3_client_geojson = S3Client(bucket=geojson_bucket) + mocked_s3_client_pmtiles = S3Client(bucket=pmtiles_bucket) with ( patch( - "udata_hydra.analysis.geojson.minio_client_geojson", - new=mocked_minio_client_geojson, + "udata_hydra.analysis.geojson.s3_client_geojson", + new=mocked_s3_client_geojson, ), patch( - "udata_hydra.analysis.geojson.minio_client_pmtiles", - new=mocked_minio_client_pmtiles, + "udata_hydra.analysis.geojson.s3_client_pmtiles", + new=mocked_s3_client_pmtiles, ), ): result = await csv_to_geojson_and_pmtiles(fp.name, inspection, RESOURCE_ID) @@ -714,20 +708,14 @@ async def test_csv_to_geojson_pmtiles(db, params, clean_db, mocker): ] ) assert not any(col in feat["properties"] for col in expected_formats) - assert ( - geojson_url - == f"https://{minio_url}/{geojson_bucket}/{geojson_folder}/{RESOURCE_ID}.geojson" - ) + assert geojson_url == f"https://{s3_endpoint}/{geojson_bucket}/{RESOURCE_ID}.geojson" assert isinstance(geojson_size, int) # checking PMTiles with open(f"{RESOURCE_ID}.pmtiles", "rb") as f: header = f.read(7) assert header == b"PMTiles" - assert ( - pmtiles_url - == f"https://{minio_url}/{pmtiles_bucket}/{pmtiles_folder}/{RESOURCE_ID}.pmtiles" - ) + assert pmtiles_url == f"https://{s3_endpoint}/{pmtiles_bucket}/{RESOURCE_ID}.pmtiles" assert isinstance(pmtiles_size, int) # Clean up files after tests diff --git a/tests/test_analysis/test_geojson.py b/tests/test_analysis/test_geojson.py index 1a368121..1dd60573 100644 --- a/tests/test_analysis/test_geojson.py +++ b/tests/test_analysis/test_geojson.py @@ -13,7 +13,7 @@ csv_to_geojson, geojson_to_pmtiles, ) -from udata_hydra.utils.minio import MinIOClient +from udata_hydra.utils.s3 import S3Client from udata_hydra.utils.timer import Timer log = logging.getLogger("udata-hydra") @@ -44,24 +44,21 @@ async def test_geojson_to_pmtiles_invalid_geometry(): @pytest.mark.asyncio async def test_geojson_to_pmtiles_valid_geometry(mocker): """Test handling of valid geometry""" - minio_url = "my.minio.fr" + s3_endpoint = "s3.example.com" bucket = "bucket" - folder = "folder" - mocker.patch("udata_hydra.config.MINIO_URL", minio_url) - mocked_minio = MagicMock() - mocked_minio.fput_object.return_value = None - mocked_minio.bucket_exists.return_value = True + mocker.patch("udata_hydra.config.S3_ENDPOINT", s3_endpoint) + mocked_resource = MagicMock() + mocked_resource.meta.client.head_bucket.return_value = {} + mocked_resource.Bucket.return_value = MagicMock() # Make sure that we don't crash even if output pmtiles already exists Path(f"{RESOURCE_ID}.pmtiles").touch() - with patch("udata_hydra.utils.minio.Minio", return_value=mocked_minio): - mocked_minio_client = MinIOClient(bucket=bucket, folder=folder) + with patch("udata_hydra.utils.s3.boto3.resource", return_value=mocked_resource): + mocked_s3_client = S3Client(bucket=bucket) with ( - patch("udata_hydra.analysis.geojson.minio_client_pmtiles", new=mocked_minio_client), + patch("udata_hydra.analysis.geojson.s3_client_pmtiles", new=mocked_s3_client), patch("udata_hydra.config.REMOVE_GENERATED_FILES", False), ): - mock_os = mocker.patch("udata_hydra.utils.minio.os") - mock_os.path = os.path - mock_os.remove.return_value = None + mocker.patch("udata_hydra.utils.s3.Path.unlink", MagicMock()) size, url = await geojson_to_pmtiles( Path("tests/data/valid.geojson"), Path(f"{RESOURCE_ID}.pmtiles") ) @@ -69,7 +66,7 @@ async def test_geojson_to_pmtiles_valid_geometry(mocker): with open(f"{RESOURCE_ID}.pmtiles", "rb") as f: header = f.read(7) assert header == b"PMTiles" - assert url == f"https://{minio_url}/{bucket}/{folder}/{RESOURCE_ID}.pmtiles" + assert url == f"https://{s3_endpoint}/{bucket}/{RESOURCE_ID}.pmtiles" # size slightly differs depending on the env assert 850 <= size <= 900 os.remove(f"{RESOURCE_ID}.pmtiles") @@ -112,22 +109,19 @@ async def test_csv_to_geojson_big_file( # Create timer for performance measurement timer = Timer("csv-to-geojson-performance-test") - # Mock MinIO for the test - minio_url = "my.minio.fr" + # Mock S3 for the test + s3_endpoint = "s3.example.com" bucket = "bucket" - folder = "folder" - mocker.patch("udata_hydra.config.MINIO_URL", minio_url) - mocked_minio = MagicMock() - mocked_minio.fput_object.return_value = None - mocked_minio.bucket_exists.return_value = True + mocker.patch("udata_hydra.config.S3_ENDPOINT", s3_endpoint) + mocked_resource = MagicMock() + mocked_resource.meta.client.head_bucket.return_value = {} + mocked_resource.Bucket.return_value = MagicMock() - with patch("udata_hydra.utils.minio.Minio", return_value=mocked_minio): - mocked_minio_client = MinIOClient(bucket=bucket, folder=folder) + with patch("udata_hydra.utils.s3.boto3.resource", return_value=mocked_resource): + mocked_s3_client = S3Client(bucket=bucket) - with patch("udata_hydra.analysis.geojson.minio_client_geojson", new=mocked_minio_client): - mock_os = mocker.patch("udata_hydra.utils.minio.os") - mock_os.path = os.path - mock_os.remove.return_value = None + with patch("udata_hydra.analysis.geojson.s3_client_geojson", new=mocked_s3_client): + mocker.patch("udata_hydra.utils.s3.Path.unlink", MagicMock()) # Analyze the CSV with csv_detective first inspection, df = csv_detective_routine( @@ -145,7 +139,7 @@ async def test_csv_to_geojson_big_file( file_path=str(csv_path), inspection=inspection, output_file_path=test_geojson_path, - upload_to_minio=False, + upload_to_s3=False, ) timer.mark("geojson-conversion") @@ -198,25 +192,22 @@ async def test_geojson_to_pmtiles_big_file(mocker, input_file: str | None): # Create timer for performance measurement timer = Timer("geojson-to-pmtiles-performance-test") - # Mock MinIO for the test - minio_url = "my.minio.fr" + # Mock S3 for the test + s3_endpoint = "s3.example.com" bucket = "bucket" - folder = "folder" - mocker.patch("udata_hydra.config.MINIO_URL", minio_url) - mocked_minio = MagicMock() - mocked_minio.fput_object.return_value = None - mocked_minio.bucket_exists.return_value = True + mocker.patch("udata_hydra.config.S3_ENDPOINT", s3_endpoint) + mocked_resource = MagicMock() + mocked_resource.meta.client.head_bucket.return_value = {} + mocked_resource.Bucket.return_value = MagicMock() - with patch("udata_hydra.utils.minio.Minio", return_value=mocked_minio): - mocked_minio_client = MinIOClient(bucket=bucket, folder=folder) + with patch("udata_hydra.utils.s3.boto3.resource", return_value=mocked_resource): + mocked_s3_client = S3Client(bucket=bucket) with ( - patch("udata_hydra.analysis.geojson.minio_client_pmtiles", new=mocked_minio_client), + patch("udata_hydra.analysis.geojson.s3_client_pmtiles", new=mocked_s3_client), patch("udata_hydra.config.REMOVE_GENERATED_FILES", False), ): - mock_os = mocker.patch("udata_hydra.utils.minio.os") - mock_os.path = os.path - mock_os.remove.return_value = None + mocker.patch("udata_hydra.utils.s3.Path.unlink", MagicMock()) # Test the performance of geojson_to_pmtiles with the real file result = await geojson_to_pmtiles(geojson_path, test_pmtiles_path) @@ -227,7 +218,7 @@ async def test_geojson_to_pmtiles_big_file(mocker, input_file: str | None): with test_pmtiles_path.open("rb") as f: header = f.read(7) assert header == b"PMTiles" - assert pmtiles_url == f"https://{minio_url}/{bucket}/{folder}/{geojson_path.stem}.pmtiles" + assert pmtiles_url == f"https://{s3_endpoint}/{bucket}/{geojson_path.stem}.pmtiles" # The size should be significantly larger than the small test file assert pmtiles_size > 5000 # Should be much larger than the 850-900 range of small file diff --git a/tests/test_analysis/test_parquet_export.py b/tests/test_analysis/test_parquet_export.py index 29f69e1a..2ea17cb5 100755 --- a/tests/test_analysis/test_parquet_export.py +++ b/tests/test_analysis/test_parquet_export.py @@ -12,8 +12,8 @@ csv_to_parquet, generate_records, ) -from udata_hydra.utils.minio import MinIOClient from udata_hydra.utils.parquet import save_as_parquet, save_as_parquet_from_db +from udata_hydra.utils.s3 import S3Client pytestmark = pytest.mark.asyncio @@ -109,18 +109,17 @@ async def execute_csv_to_parquet() -> tuple[str, int] | None: assert not await execute_csv_to_parquet() else: - minio_url = "my.minio.fr" + s3_endpoint = "s3.example.com" bucket = "bucket" - folder = "folder" - mocker.patch("udata_hydra.config.MINIO_URL", minio_url) - mocked_minio = MagicMock() - mocked_minio.fput_object.return_value = None - mocked_minio.bucket_exists.return_value = True - with patch("udata_hydra.utils.minio.Minio", return_value=mocked_minio): - mocked_minio_client = MinIOClient(bucket=bucket, folder=folder) - with patch("udata_hydra.analysis.csv.minio_client", new=mocked_minio_client): + mocker.patch("udata_hydra.config.S3_ENDPOINT", s3_endpoint) + mocked_resource = MagicMock() + mocked_resource.meta.client.head_bucket.return_value = {} + mocked_resource.Bucket.return_value = MagicMock() + with patch("udata_hydra.utils.s3.boto3.resource", return_value=mocked_resource): + mocked_s3_client = S3Client(bucket=bucket) + with patch("udata_hydra.analysis.csv.s3_client", new=mocked_s3_client): result = await execute_csv_to_parquet() assert result is not None parquet_url, parquet_size = result - assert parquet_url == f"https://{minio_url}/{bucket}/{folder}/{RESOURCE_ID}.parquet" + assert parquet_url == f"https://{s3_endpoint}/{bucket}/{RESOURCE_ID}.parquet" assert isinstance(parquet_size, int) diff --git a/tests/test_utils/test_s3_client.py b/tests/test_utils/test_s3_client.py new file mode 100644 index 00000000..e4115b9a --- /dev/null +++ b/tests/test_utils/test_s3_client.py @@ -0,0 +1,41 @@ +"""Unit tests for S3Client (object key prefix / public URL).""" + +from collections.abc import Iterator +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from pytest_mock import MockerFixture + +from udata_hydra.utils.s3 import S3Client + + +@pytest.fixture +def mock_s3(mocker: MockerFixture) -> Iterator[MagicMock]: + mocker.patch("udata_hydra.config.S3_ENDPOINT", "s3.example.com") + resource = MagicMock() + resource.meta.client.head_bucket.return_value = {} + bucket = MagicMock() + resource.Bucket.return_value = bucket + with patch("udata_hydra.utils.s3.boto3.resource", return_value=resource): + yield bucket + + +def test_s3_client_upload_at_bucket_root(mock_s3: MagicMock, tmp_path: Path) -> None: + f = tmp_path / "file.parquet" + f.write_bytes(b"x") + client = S3Client(bucket="my-bucket") + url = client.send_file(f, delete_source=False) + + mock_s3.upload_file.assert_called_once_with(str(f), "file.parquet") + assert url == "https://s3.example.com/my-bucket/file.parquet" + + +def test_s3_client_upload_with_prefix(mock_s3: MagicMock, tmp_path: Path) -> None: + f = tmp_path / "file.parquet" + f.write_bytes(b"x") + client = S3Client(bucket="my-bucket", prefix="exports") + url = client.send_file(f, delete_source=False) + + mock_s3.upload_file.assert_called_once_with(str(f), "exports/file.parquet") + assert url == "https://s3.example.com/my-bucket/exports/file.parquet" diff --git a/udata_hydra/__init__.py b/udata_hydra/__init__.py index 69fbe0e4..04d42d65 100644 --- a/udata_hydra/__init__.py +++ b/udata_hydra/__init__.py @@ -1,12 +1,9 @@ import importlib.metadata -import logging import os import re import tomllib from pathlib import Path -log = logging.getLogger("udata-hydra") - class Configurator: """Loads a dict of config from TOML file(s) and behaves like an object, ie config.VALUE""" diff --git a/udata_hydra/analysis/csv.py b/udata_hydra/analysis/csv.py index d04087d2..222c1e70 100644 --- a/udata_hydra/analysis/csv.py +++ b/udata_hydra/analysis/csv.py @@ -47,8 +47,8 @@ remove_remainders, ) from udata_hydra.utils.casting import generate_records -from udata_hydra.utils.minio import MinIOClient from udata_hydra.utils.parquet import save_as_parquet, save_as_parquet_from_db +from udata_hydra.utils.s3 import S3Client log = logging.getLogger("udata-hydra") @@ -75,7 +75,7 @@ } RESERVED_COLS = ("__id", "cmin", "cmax", "collation", "ctid", "tableoid", "xmin", "xmax") -minio_client = MinIOClient(bucket=config.MINIO_PARQUET_BUCKET, folder=config.MINIO_PARQUET_FOLDER) +s3_client = S3Client(bucket=config.S3_PARQUET_BUCKET, prefix=config.S3_PARQUET_PREFIX) async def analyse_csv( @@ -353,7 +353,7 @@ async def csv_to_parquet( output_filename=resource_id, ) parquet_size: int = os.path.getsize(parquet_file) - parquet_url: str = minio_client.send_file(parquet_file) + parquet_url: str = s3_client.send_file(parquet_file) await Check.update( check_id, diff --git a/udata_hydra/analysis/geojson.py b/udata_hydra/analysis/geojson.py index 63804fa4..5e1dc048 100644 --- a/udata_hydra/analysis/geojson.py +++ b/udata_hydra/analysis/geojson.py @@ -22,19 +22,15 @@ remove_remainders, ) from udata_hydra.utils.casting import generate_records -from udata_hydra.utils.minio import MinIOClient +from udata_hydra.utils.s3 import S3Client DEFAULT_GEOJSON_FILEPATH = Path("converted_from_csv.geojson") DEFAULT_PMTILES_FILEPATH = Path("converted_from_geojson.pmtiles") log = logging.getLogger("udata-hydra") -minio_client_pmtiles = MinIOClient( - bucket=config.MINIO_PMTILES_BUCKET, folder=config.MINIO_PMTILES_FOLDER -) -minio_client_geojson = MinIOClient( - bucket=config.MINIO_GEOJSON_BUCKET, folder=config.MINIO_GEOJSON_FOLDER -) +s3_client_pmtiles = S3Client(bucket=config.S3_PMTILES_BUCKET, prefix=config.S3_PMTILES_PREFIX) +s3_client_geojson = S3Client(bucket=config.S3_GEOJSON_BUCKET, prefix=config.S3_GEOJSON_PREFIX) async def analyse_geojson( @@ -114,10 +110,10 @@ async def csv_to_geojson( file_path: str, inspection: dict, output_file_path: Path, - upload_to_minio: bool = True, + upload_to_s3: bool = True, ) -> tuple[int, str | None] | None: """ - Convert a CSV file to GeoJSON format and optionally upload to MinIO. + Convert a CSV file to GeoJSON format and optionally upload to S3. Detects geographical columns (geometry, latlon, lonlat, or lat/lon) and converts CSV data to GeoJSON features. Rows with NaN values in geographical columns are skipped. @@ -126,11 +122,11 @@ async def csv_to_geojson( file_path: Target CSV file to convert. inspection: CSV detective analysis results with column format detection. output_file_path: Path where the GeoJSON file should be saved. - upload_to_minio: Whether to upload to MinIO (default: True). + upload_to_s3: Whether to upload to S3 (default: True). Returns: geojson_size: Size of the GeoJSON file in bytes. - geojson_url: URL of the GeoJSON file on MinIO. None if it was not uploaded to MinIO. + geojson_url: URL of the GeoJSON file on S3. None if it was not uploaded to S3. """ def cast_latlon(latlon: str) -> list[float]: @@ -238,9 +234,9 @@ def get_features( geojson_size: int = os.path.getsize(output_file_path) - if upload_to_minio: - log.debug(f"Sending GeoJSON file {output_file_path} to MinIO") - geojson_url = minio_client_geojson.send_file(str(output_file_path), delete_source=False) + if upload_to_s3: + log.debug(f"Sending GeoJSON file {output_file_path} to S3") + geojson_url = s3_client_geojson.send_file(output_file_path, delete_source=False) else: geojson_url = None @@ -250,19 +246,19 @@ def get_features( async def geojson_to_pmtiles( input_file_path: Path, output_file_path: Path, - upload_to_minio: bool = True, + upload_to_s3: bool = True, ) -> tuple[int, str | None]: """ - Convert a GeoJSON file to PMTiles file and optionally upload to MinIO. + Convert a GeoJSON file to PMTiles file and optionally upload to S3. Args: input_file_path: GeoJSON file path to convert. output_file_path: Path where the PMTiles file should be saved. - upload_to_minio: Whether to upload to MinIO (default: True). + upload_to_s3: Whether to upload to S3 (default: True). Returns: pmtiles_size: size of the PMTiles file. - pmtiles_url: URL of the PMTiles file on MinIO. None if it was not uploaded to MinIO. + pmtiles_url: URL of the PMTiles file on S3. None if it was not uploaded to S3. """ log.debug(f"Converting GeoJSON file '{input_file_path}' to PMTiles file '{output_file_path}'") @@ -283,10 +279,10 @@ async def geojson_to_pmtiles( pmtiles_size: int = os.path.getsize(output_file_path) - if upload_to_minio: - log.debug(f"Sending PMTiles file {output_file_path} to MinIO") - pmtiles_url = minio_client_pmtiles.send_file( - str(output_file_path), delete_source=config.REMOVE_GENERATED_FILES + if upload_to_s3: + log.debug(f"Sending PMTiles file {output_file_path} to S3") + pmtiles_url = s3_client_pmtiles.send_file( + output_file_path, delete_source=config.REMOVE_GENERATED_FILES ) else: pmtiles_url = None @@ -305,9 +301,7 @@ async def csv_to_geojson_and_pmtiles( log.debug("CSV_TO_GEOJSON turned off, skipping geojson/PMtiles export.") return None - log.debug( - f"Converting to geojson and PMtiles if relevant for {resource_id} and sending to MinIO." - ) + log.debug(f"Converting to geojson and PMtiles if relevant for {resource_id} and sending to S3.") if resource_id: geojson_filepath = Path(f"{resource_id}.geojson") @@ -319,7 +313,7 @@ async def csv_to_geojson_and_pmtiles( pmtiles_filepath = DEFAULT_PMTILES_FILEPATH # Convert CSV to GeoJSON - result = await csv_to_geojson(file_path, inspection, geojson_filepath, upload_to_minio=True) + result = await csv_to_geojson(file_path, inspection, geojson_filepath, upload_to_s3=True) if result is None: return None geojson_size, geojson_url = result diff --git a/udata_hydra/cli.py b/udata_hydra/cli.py index 8805f7c6..029d8282 100644 --- a/udata_hydra/cli.py +++ b/udata_hydra/cli.py @@ -408,8 +408,8 @@ async def _analyse_csv_cli( await Check.delete(check["id"]) await Resource.delete(resource_id=tmp_resource_id, hard_delete=True) - # Clean up MinIO files if any (parquet, etc.) - # Note: This would require additional MinIO cleanup logic + # Clean up S3 files if any (parquet, etc.) + # Note: This would require additional S3 cleanup logic log.info(f"Cleaned up temporary data for {url}") except Exception as e: @@ -503,12 +503,12 @@ async def _convert_csv_to_geojson_cli(csv_filepath: str): log.info("Converting to GeoJSON...") try: - # Convert to GeoJSON (no MinIO upload, no database updates) + # Convert to GeoJSON (no S3 upload, no database updates) result = await csv_to_geojson( file_path=str(csv_path), inspection=inspection, output_file_path=geojson_filepath, - upload_to_minio=False, + upload_to_s3=False, ) if result: @@ -563,9 +563,9 @@ async def _convert_geojson_to_pmtiles_cli(geojson_filepath: str): pmtiles_filepath = Path(f"{geojson_path.stem}.pmtiles") try: - # Convert to PMTiles (no MinIO upload, no database updates) + # Convert to PMTiles (no S3 upload, no database updates) pmtiles_size, pmtiles_url = await geojson_to_pmtiles( - input_file_path=geojson_path, output_file_path=pmtiles_filepath, upload_to_minio=False + input_file_path=geojson_path, output_file_path=pmtiles_filepath, upload_to_s3=False ) log.info("Conversion successful!") diff --git a/udata_hydra/config_default.toml b/udata_hydra/config_default.toml index 4bbe828f..2d4899c7 100644 --- a/udata_hydra/config_default.toml +++ b/udata_hydra/config_default.toml @@ -87,27 +87,28 @@ UDATA_URI_API_KEY = "" # -- File generation cleanup, only patched in the tests -- # REMOVE_GENERATED_FILES = true -# -- Minio / datalake settings -- # -MINIO_URL = "" # no scheme -MINIO_USER = "" -MINIO_PWD = "" +# -- S3-compatible object storage (e.g. OVH) -- # +# Hostname only (no scheme); used as https://{S3_ENDPOINT} for boto3 and public URLs. +S3_ENDPOINT = "" +S3_ACCESS_KEY_ID = "" +S3_SECRET_ACCESS_KEY = "" # -- Parquet conversion settings -- # CSV_TO_PARQUET = false DB_TO_PARQUET = false MIN_LINES_FOR_PARQUET = 200 -MINIO_PARQUET_BUCKET = "" -MINIO_PARQUET_FOLDER = "" # no trailing slash +S3_PARQUET_BUCKET = "" +S3_PARQUET_PREFIX = "" # optional object key prefix, no leading/trailing slash; empty = bucket root # -- PMTiles conversion settings -- # GEOJSON_TO_PMTILES = false -MINIO_PMTILES_BUCKET = "" -MINIO_PMTILES_FOLDER = "" # no trailing slash +S3_PMTILES_BUCKET = "" +S3_PMTILES_PREFIX = "" # optional object key prefix, no leading/trailing slash; empty = bucket root # -- Geojson conversion settings -- # CSV_TO_GEOJSON = false -MINIO_GEOJSON_BUCKET = "" -MINIO_GEOJSON_FOLDER = "" # no trailing slash +S3_GEOJSON_BUCKET = "" +S3_GEOJSON_PREFIX = "" # optional object key prefix, no leading/trailing slash; empty = bucket root # -- Parquet analysis settings -- # PARQUET_TO_DB = false diff --git a/udata_hydra/utils/minio.py b/udata_hydra/utils/minio.py deleted file mode 100644 index 663918f3..00000000 --- a/udata_hydra/utils/minio.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import os - -from minio import Minio - -from udata_hydra import config - -log = logging.getLogger("udata-hydra") - - -class MinIOClient: - def __init__(self, bucket: str, folder: str): - self.user = config.MINIO_USER - self.password = config.MINIO_PWD - self.bucket = bucket - self.folder = folder - self.client = Minio( - config.MINIO_URL or "test", - access_key=self.user or "test", - secret_key=self.password or "test", - secure=True, - ) - if self.bucket: - self.bucket_exists = self.client.bucket_exists(self.bucket) - if not self.bucket_exists: - raise ValueError(f"Bucket '{self.bucket}' does not exist.") - - def send_file( - self, - file_path: str, - delete_source: bool = True, - ) -> str: - if self.bucket is None: - raise AttributeError("A bucket has to be specified.") - if os.path.isfile(file_path): - file_name = os.path.basename(file_path) - self.client.fput_object( - self.bucket, - f"{self.folder}/{file_name}", - file_path, - ) - if delete_source: - os.remove(file_path) - return f"https://{config.MINIO_URL}/{self.bucket}/{self.folder}/{file_name}" - else: - raise Exception(f"file '{file_path}' does not exists") diff --git a/udata_hydra/utils/s3.py b/udata_hydra/utils/s3.py new file mode 100644 index 00000000..f0f68256 --- /dev/null +++ b/udata_hydra/utils/s3.py @@ -0,0 +1,66 @@ +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from udata_hydra import config + +# Match datagouvfr_data_pipelines S3 defaults for slow networks / large uploads. +_DEFAULT_BOTO_CONFIG = {"connect_timeout": 3600, "read_timeout": 3600} + + +def _normalize_prefix(prefix: str | None) -> str | None: + """Return an object key prefix without leading/trailing slashes, or None when unset.""" + if prefix is None: + return None + p = prefix.strip().strip("/") + return p if p else None + + +class S3Client: + def __init__(self, bucket: str, prefix: str | None = None): + self.user = config.S3_ACCESS_KEY_ID + self.password = config.S3_SECRET_ACCESS_KEY + self.bucket = bucket + self.prefix = _normalize_prefix(prefix) + host = config.S3_ENDPOINT or "test" + self._resource = boto3.resource( + "s3", + endpoint_url=f"https://{host}", + aws_access_key_id=self.user or "test", + aws_secret_access_key=self.password or "test", + config=Config(**_DEFAULT_BOTO_CONFIG), + ) + self._client = self._resource.meta.client + if self.bucket: + self._ensure_bucket_exists() + + def _ensure_bucket_exists(self) -> None: + try: + self._client.head_bucket(Bucket=self.bucket) + except ClientError as e: + code = e.response.get("Error", {}).get("Code", "") + if code in ("404", "NoSuchBucket", "NotFound"): + raise ValueError(f"Bucket '{self.bucket}' does not exist.") from e + raise + + def send_file( + self, + file_path: str | Path, + delete_source: bool = True, + ) -> str: + if self.bucket is None: + raise AttributeError("A bucket has to be specified.") + path = Path(file_path) + if path.is_file(): + file_name = path.name + key = f"{self.prefix}/{file_name}" if self.prefix else file_name + self._resource.Bucket(self.bucket).upload_file(str(path), key) + if delete_source: + path.unlink() + if self.prefix: + return f"https://{config.S3_ENDPOINT}/{self.bucket}/{self.prefix}/{file_name}" + return f"https://{config.S3_ENDPOINT}/{self.bucket}/{file_name}" + else: + raise Exception(f"file '{path}' does not exists") diff --git a/uv.lock b/uv.lock index e25f2173..aeafae98 100644 --- a/uv.lock +++ b/uv.lock @@ -143,39 +143,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] -[[package]] -name = "argon2-cffi" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi-bindings" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, -] - -[[package]] -name = "argon2-cffi-bindings" -version = "25.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/57/96b8b9f93166147826da5f90376e784a10582dd39a393c99bb62cfcf52f0/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aecba1723ae35330a008418a91ea6cfcedf6d31e5fbaa056a166462ff066d500", size = 54121, upload-time = "2025-07-30T10:01:50.815Z" }, - { url = "https://files.pythonhosted.org/packages/0a/08/a9bebdb2e0e602dde230bdde8021b29f71f7841bd54801bcfd514acb5dcf/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:2630b6240b495dfab90aebe159ff784d08ea999aa4b0d17efa734055a07d2f44", size = 29177, upload-time = "2025-07-30T10:01:51.681Z" }, - { url = "https://files.pythonhosted.org/packages/b6/02/d297943bcacf05e4f2a94ab6f462831dc20158614e5d067c35d4e63b9acb/argon2_cffi_bindings-25.1.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:7aef0c91e2c0fbca6fc68e7555aa60ef7008a739cbe045541e438373bc54d2b0", size = 31090, upload-time = "2025-07-30T10:01:53.184Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/44365f3d75053e53893ec6d733e4a5e3147502663554b4d864587c7828a7/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e021e87faa76ae0d413b619fe2b65ab9a037f24c60a1e6cc43457ae20de6dc6", size = 81246, upload-time = "2025-07-30T10:01:54.145Z" }, - { url = "https://files.pythonhosted.org/packages/09/52/94108adfdd6e2ddf58be64f959a0b9c7d4ef2fa71086c38356d22dc501ea/argon2_cffi_bindings-25.1.0-cp39-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3e924cfc503018a714f94a49a149fdc0b644eaead5d1f089330399134fa028a", size = 87126, upload-time = "2025-07-30T10:01:55.074Z" }, - { url = "https://files.pythonhosted.org/packages/72/70/7a2993a12b0ffa2a9271259b79cc616e2389ed1a4d93842fac5a1f923ffd/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:c87b72589133f0346a1cb8d5ecca4b933e3c9b64656c9d175270a000e73b288d", size = 80343, upload-time = "2025-07-30T10:01:56.007Z" }, - { url = "https://files.pythonhosted.org/packages/78/9a/4e5157d893ffc712b74dbd868c7f62365618266982b64accab26bab01edc/argon2_cffi_bindings-25.1.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1db89609c06afa1a214a69a462ea741cf735b29a57530478c06eb81dd403de99", size = 86777, upload-time = "2025-07-30T10:01:56.943Z" }, - { url = "https://files.pythonhosted.org/packages/74/cd/15777dfde1c29d96de7f18edf4cc94c385646852e7c7b0320aa91ccca583/argon2_cffi_bindings-25.1.0-cp39-abi3-win32.whl", hash = "sha256:473bcb5f82924b1becbb637b63303ec8d10e84c8d241119419897a26116515d2", size = 27180, upload-time = "2025-07-30T10:01:57.759Z" }, - { url = "https://files.pythonhosted.org/packages/e2/c6/a759ece8f1829d1f162261226fbfd2c6832b3ff7657384045286d2afa384/argon2_cffi_bindings-25.1.0-cp39-abi3-win_amd64.whl", hash = "sha256:a98cd7d17e9f7ce244c0803cad3c23a7d379c301ba618a5fa76a67d116618b98", size = 31715, upload-time = "2025-07-30T10:01:58.56Z" }, - { url = "https://files.pythonhosted.org/packages/42/b9/f8d6fa329ab25128b7e98fd83a3cb34d9db5b059a9847eddb840a0af45dd/argon2_cffi_bindings-25.1.0-cp39-abi3-win_arm64.whl", hash = "sha256:b0fdbcf513833809c882823f98dc2f931cf659d9a1429616ac3adebb49f5db94", size = 27149, upload-time = "2025-07-30T10:01:59.329Z" }, -] - [[package]] name = "asttokens" version = "2.4.1" @@ -239,60 +206,40 @@ wheels = [ ] [[package]] -name = "certifi" -version = "2025.8.3" +name = "boto3" +version = "1.42.89" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/0c/f7bccb22b245cabf392816baba20f9e95f78ace7dbc580fd40136e80e732/boto3-1.42.89.tar.gz", hash = "sha256:3e43aacc0801bba9bcd23a8c271c089af297a69565f783fcdd357ae0e330bf1e", size = 113165, upload-time = "2026-04-13T19:36:17.516Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, + { url = "https://files.pythonhosted.org/packages/b9/33/55103ba5ef9975ea54b8d39e69b76eb6e9fded3beae5f01065e26951a3a1/boto3-1.42.89-py3-none-any.whl", hash = "sha256:6204b189f4d0c655535f43d7eaa57ff4e8d965b8463c97e45952291211162932", size = 140556, upload-time = "2026-04-13T19:36:13.894Z" }, ] [[package]] -name = "cffi" -version = "2.0.0" +name = "botocore" +version = "1.42.89" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344, upload-time = "2025-09-08T23:22:26.456Z" }, - { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560, upload-time = "2025-09-08T23:22:28.197Z" }, - { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613, upload-time = "2025-09-08T23:22:29.475Z" }, - { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476, upload-time = "2025-09-08T23:22:31.063Z" }, - { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374, upload-time = "2025-09-08T23:22:32.507Z" }, - { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597, upload-time = "2025-09-08T23:22:34.132Z" }, - { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574, upload-time = "2025-09-08T23:22:35.443Z" }, - { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971, upload-time = "2025-09-08T23:22:36.805Z" }, - { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972, upload-time = "2025-09-08T23:22:38.436Z" }, - { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078, upload-time = "2025-09-08T23:22:39.776Z" }, - { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076, upload-time = "2025-09-08T23:22:40.95Z" }, - { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820, upload-time = "2025-09-08T23:22:42.463Z" }, - { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635, upload-time = "2025-09-08T23:22:43.623Z" }, - { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271, upload-time = "2025-09-08T23:22:44.795Z" }, - { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048, upload-time = "2025-09-08T23:22:45.938Z" }, - { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529, upload-time = "2025-09-08T23:22:47.349Z" }, - { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097, upload-time = "2025-09-08T23:22:48.677Z" }, - { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983, upload-time = "2025-09-08T23:22:50.06Z" }, - { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519, upload-time = "2025-09-08T23:22:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572, upload-time = "2025-09-08T23:22:52.902Z" }, - { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963, upload-time = "2025-09-08T23:22:54.518Z" }, - { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361, upload-time = "2025-09-08T23:22:55.867Z" }, - { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932, upload-time = "2025-09-08T23:22:57.188Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557, upload-time = "2025-09-08T23:22:58.351Z" }, - { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762, upload-time = "2025-09-08T23:22:59.668Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230, upload-time = "2025-09-08T23:23:00.879Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043, upload-time = "2025-09-08T23:23:02.231Z" }, - { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" }, - { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" }, - { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" }, - { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" }, - { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" }, - { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909, upload-time = "2025-09-08T23:23:14.32Z" }, - { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402, upload-time = "2025-09-08T23:23:15.535Z" }, - { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780, upload-time = "2025-09-08T23:23:16.761Z" }, + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/cc/e6be943efa9051bd15c2ee14077c2b10d6e27c9e9385fc43a03a5c4ed8b5/botocore-1.42.89.tar.gz", hash = "sha256:95ac52f472dad29942f3088b278ab493044516c16dbf9133c975af16527baa99", size = 15206290, upload-time = "2026-04-13T19:36:02.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/f1/90a7b8eda38b7c3a65ca7ee0075bdf310b6b471cb1b95fab6e8994323a50/botocore-1.42.89-py3-none-any.whl", hash = "sha256:d9b786c8d9db6473063b4cc5be0ba7e6a381082307bd6afb69d4216f9fa95f35", size = 14887287, upload-time = "2026-04-13T19:35:56.677Z" }, +] + +[[package]] +name = "certifi" +version = "2025.8.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/67/960ebe6bf230a96cda2e0abcf73af550ec4f090005363542f0765df162e0/certifi-2025.8.3.tar.gz", hash = "sha256:e564105f78ded564e3ae7c923924435e1daa7463faeab5bb932bc53ffae63407", size = 162386, upload-time = "2025-08-03T03:07:47.08Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/48/1549795ba7742c948d2ad169c1c8cdbae65bc450d6cd753d124b17c8cd32/certifi-2025.8.3-py3-none-any.whl", hash = "sha256:f6c12493cfb1b06ba2ff328595af9350c65d6644968e5d3a2ffd78699af217a5", size = 161216, upload-time = "2025-08-03T03:07:45.777Z" }, ] [[package]] @@ -654,7 +601,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/de/f28ced0a67749cac23fecb02b694f6473f47686dff6afaa211d186e2ef9c/greenlet-3.2.4-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:96378df1de302bc38e99c3a9aa311967b7dc80ced1dcc6f171e99842987882a2", size = 272305, upload-time = "2025-08-07T13:15:41.288Z" }, { url = "https://files.pythonhosted.org/packages/09/16/2c3792cba130000bf2a31c5272999113f4764fd9d874fb257ff588ac779a/greenlet-3.2.4-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1ee8fae0519a337f2329cb78bd7a8e128ec0f881073d43f023c7b8d4831d5246", size = 632472, upload-time = "2025-08-07T13:42:55.044Z" }, { url = "https://files.pythonhosted.org/packages/ae/8f/95d48d7e3d433e6dae5b1682e4292242a53f22df82e6d3dda81b1701a960/greenlet-3.2.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94abf90142c2a18151632371140b3dba4dee031633fe614cb592dbb6c9e17bc3", size = 644646, upload-time = "2025-08-07T13:45:26.523Z" }, - { url = "https://files.pythonhosted.org/packages/d5/5e/405965351aef8c76b8ef7ad370e5da58d57ef6068df197548b015464001a/greenlet-3.2.4-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:4d1378601b85e2e5171b99be8d2dc85f594c79967599328f95c1dc1a40f1c633", size = 640519, upload-time = "2025-08-07T13:53:13.928Z" }, { url = "https://files.pythonhosted.org/packages/25/5d/382753b52006ce0218297ec1b628e048c4e64b155379331f25a7316eb749/greenlet-3.2.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0db5594dce18db94f7d1650d7489909b57afde4c580806b8d9203b6e79cdc079", size = 639707, upload-time = "2025-08-07T13:18:27.146Z" }, { url = "https://files.pythonhosted.org/packages/1f/8e/abdd3f14d735b2929290a018ecf133c901be4874b858dd1c604b9319f064/greenlet-3.2.4-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2523e5246274f54fdadbce8494458a2ebdcdbc7b802318466ac5606d3cded1f8", size = 587684, upload-time = "2025-08-07T13:18:25.164Z" }, { url = "https://files.pythonhosted.org/packages/5d/65/deb2a69c3e5996439b0176f6651e0052542bb6c8f8ec2e3fba97c9768805/greenlet-3.2.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:1987de92fec508535687fb807a5cea1560f6196285a4cde35c100b8cd632cc52", size = 1116647, upload-time = "2025-08-07T13:42:38.655Z" }, @@ -665,7 +611,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/69/9b804adb5fd0671f367781560eb5eb586c4d495277c93bde4307b9e28068/greenlet-3.2.4-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:3b67ca49f54cede0186854a008109d6ee71f66bd57bb36abd6d0a0267b540cdd", size = 274079, upload-time = "2025-08-07T13:15:45.033Z" }, { url = "https://files.pythonhosted.org/packages/46/e9/d2a80c99f19a153eff70bc451ab78615583b8dac0754cfb942223d2c1a0d/greenlet-3.2.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ddf9164e7a5b08e9d22511526865780a576f19ddd00d62f8a665949327fde8bb", size = 640997, upload-time = "2025-08-07T13:42:56.234Z" }, { url = "https://files.pythonhosted.org/packages/3b/16/035dcfcc48715ccd345f3a93183267167cdd162ad123cd93067d86f27ce4/greenlet-3.2.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f28588772bb5fb869a8eb331374ec06f24a83a9c25bfa1f38b6993afe9c1e968", size = 655185, upload-time = "2025-08-07T13:45:27.624Z" }, - { url = "https://files.pythonhosted.org/packages/31/da/0386695eef69ffae1ad726881571dfe28b41970173947e7c558d9998de0f/greenlet-3.2.4-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:5c9320971821a7cb77cfab8d956fa8e39cd07ca44b6070db358ceb7f8797c8c9", size = 649926, upload-time = "2025-08-07T13:53:15.251Z" }, { url = "https://files.pythonhosted.org/packages/68/88/69bf19fd4dc19981928ceacbc5fd4bb6bc2215d53199e367832e98d1d8fe/greenlet-3.2.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c60a6d84229b271d44b70fb6e5fa23781abb5d742af7b808ae3f6efd7c9c60f6", size = 651839, upload-time = "2025-08-07T13:18:30.281Z" }, { url = "https://files.pythonhosted.org/packages/19/0d/6660d55f7373b2ff8152401a83e02084956da23ae58cddbfb0b330978fe9/greenlet-3.2.4-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b3812d8d0c9579967815af437d96623f45c0f2ae5f04e366de62a12d83a8fb0", size = 607586, upload-time = "2025-08-07T13:18:28.544Z" }, { url = "https://files.pythonhosted.org/packages/8e/1a/c953fdedd22d81ee4629afbb38d2f9d71e37d23caace44775a3a969147d4/greenlet-3.2.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:abbf57b5a870d30c4675928c37278493044d7c14378350b3aa5d484fa65575f0", size = 1123281, upload-time = "2025-08-07T13:42:39.858Z" }, @@ -676,7 +621,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/49/e8/58c7f85958bda41dafea50497cbd59738c5c43dbbea5ee83d651234398f4/greenlet-3.2.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1a921e542453fe531144e91e1feedf12e07351b1cf6c9e8a3325ea600a715a31", size = 272814, upload-time = "2025-08-07T13:15:50.011Z" }, { url = "https://files.pythonhosted.org/packages/62/dd/b9f59862e9e257a16e4e610480cfffd29e3fae018a68c2332090b53aac3d/greenlet-3.2.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd3c8e693bff0fff6ba55f140bf390fa92c994083f838fece0f63be121334945", size = 641073, upload-time = "2025-08-07T13:42:57.23Z" }, { url = "https://files.pythonhosted.org/packages/f7/0b/bc13f787394920b23073ca3b6c4a7a21396301ed75a655bcb47196b50e6e/greenlet-3.2.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:710638eb93b1fa52823aa91bf75326f9ecdfd5e0466f00789246a5280f4ba0fc", size = 655191, upload-time = "2025-08-07T13:45:29.752Z" }, - { url = "https://files.pythonhosted.org/packages/f2/d6/6adde57d1345a8d0f14d31e4ab9c23cfe8e2cd39c3baf7674b4b0338d266/greenlet-3.2.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c5111ccdc9c88f423426df3fd1811bfc40ed66264d35aa373420a34377efc98a", size = 649516, upload-time = "2025-08-07T13:53:16.314Z" }, { url = "https://files.pythonhosted.org/packages/7f/3b/3a3328a788d4a473889a2d403199932be55b1b0060f4ddd96ee7cdfcad10/greenlet-3.2.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76383238584e9711e20ebe14db6c88ddcedc1829a9ad31a584389463b5aa504", size = 652169, upload-time = "2025-08-07T13:18:32.861Z" }, { url = "https://files.pythonhosted.org/packages/ee/43/3cecdc0349359e1a527cbf2e3e28e5f8f06d3343aaf82ca13437a9aa290f/greenlet-3.2.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23768528f2911bcd7e475210822ffb5254ed10d71f4028387e5a99b4c6699671", size = 610497, upload-time = "2025-08-07T13:18:31.636Z" }, { url = "https://files.pythonhosted.org/packages/b8/19/06b6cf5d604e2c382a6f31cafafd6f33d5dea706f4db7bdab184bad2b21d/greenlet-3.2.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:00fadb3fedccc447f517ee0d3fd8fe49eae949e1cd0f6a611818f4f6fb7dc83b", size = 1121662, upload-time = "2025-08-07T13:42:41.117Z" }, @@ -728,6 +672,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "json-stream" version = "2.3.3" @@ -873,22 +826,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "minio" -version = "7.2.18" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "argon2-cffi" }, - { name = "certifi" }, - { name = "pycryptodome" }, - { name = "typing-extensions" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/71/99/1ad8733fa3f2fa82726e470f8c321e17f9321083b234ab45ad6b59d80d9f/minio-7.2.18.tar.gz", hash = "sha256:173402a5716099159c5659f9de75be204ebe248557b9f1cc9cf45aa70e9d3024", size = 135544, upload-time = "2025-09-29T17:00:28.238Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/ae/f32695da4f93de50dd7075100dab8cf689a9d96270f58ce6f940fd044a3e/minio-7.2.18-py3-none-any.whl", hash = "sha256:f23a6edbff8d0bc4b5c1a61b2628a01c5a3342aefc613ff9c276012e6321108f", size = 93120, upload-time = "2025-09-29T17:00:26.86Z" }, -] - [[package]] name = "more-itertools" version = "10.8.0" @@ -1313,45 +1250,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/4e/519c1bc1876625fe6b71e9a28287c43ec2f20f73c658b9ae1d485c0c206e/pyarrow-21.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:222c39e2c70113543982c6b34f3077962b44fca38c0bd9e68bb6781534425c10", size = 26371006, upload-time = "2025-07-18T00:56:56.379Z" }, ] -[[package]] -name = "pycparser" -version = "2.23" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/e3/59cd50310fc9b59512193629e1984c1f95e5c8ae6e5d8c69532ccc65a7fe/pycparser-2.23-py3-none-any.whl", hash = "sha256:e5c6e8d3fbad53479cab09ac03729e0a9faf2bee3db8208a550daf5af81a5934", size = 118140, upload-time = "2025-09-09T13:23:46.651Z" }, -] - -[[package]] -name = "pycryptodome" -version = "3.23.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, - { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, - { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, - { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, - { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, - { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, - { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, - { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, - { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, - { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, - { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, - { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, - { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, - { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, - { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, - { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, - { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, - { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, - { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, - { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, - { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, -] - [[package]] name = "pygments" version = "2.19.2" @@ -1666,6 +1564,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c3/12/28fa2f597a605884deb0f65c1b1ae05111051b2a7030f5d8a4ff7f4599ba/ruff-0.13.2-py3-none-win_arm64.whl", hash = "sha256:da711b14c530412c827219312b7d7fbb4877fb31150083add7e8c5336549cea7", size = 12484437, upload-time = "2025-09-25T14:54:08.022Z" }, ] +[[package]] +name = "s3transfer" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/04/74127fc843314818edfa81b5540e26dd537353b123a4edc563109d8f17dd/s3transfer-0.16.0.tar.gz", hash = "sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920", size = 153827, upload-time = "2025-12-01T02:30:59.114Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/51/727abb13f44c1fcf6d145979e1535a35794db0f6e450a0cb46aa24732fe2/s3transfer-0.16.0-py3-none-any.whl", hash = "sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe", size = 86830, upload-time = "2025-12-01T02:30:57.729Z" }, +] + [[package]] name = "sentry-sdk" version = "2.39.0" @@ -1928,13 +1838,13 @@ source = { editable = "." } dependencies = [ { name = "aiohttp" }, { name = "asyncpg" }, + { name = "boto3" }, { name = "coloredlogs" }, { name = "csv-detective" }, { name = "dateparser" }, { name = "humanfriendly" }, { name = "json-stream" }, { name = "marshmallow" }, - { name = "minio" }, { name = "owslib" }, { name = "progressist" }, { name = "pyarrow" }, @@ -1967,13 +1877,13 @@ dev = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.10.3" }, { name = "asyncpg", specifier = ">=0.29.0" }, + { name = "boto3", specifier = ">=1.35.0" }, { name = "coloredlogs", specifier = ">=15.0.1" }, { name = "csv-detective", specifier = "==0.11.2" }, { name = "dateparser", specifier = ">=1.1.7" }, { name = "humanfriendly", specifier = ">=10.0" }, { name = "json-stream", specifier = ">=2.3.3" }, { name = "marshmallow", specifier = ">=3.14.1" }, - { name = "minio", specifier = ">=7.2.8" }, { name = "owslib", specifier = ">=0.35.0" }, { name = "progressist", specifier = ">=0.1.0" }, { name = "pyarrow", specifier = ">=16.1.0" },