squash

Author: Grzegorz Pustulka

.github/workflows/cicd.yml

@@ -28,6 +28,7 @@ jobs:
           xpack.security.enabled: false
           xpack.security.transport.ssl.enabled: false
           ES_JAVA_OPTS: -Xms512m -Xmx1g
+          action.destructive_requires_name: false
         ports:
           - 9200:9200
 
@@ -44,6 +45,7 @@ jobs:
           xpack.security.enabled: false
           xpack.security.transport.ssl.enabled: false
           ES_JAVA_OPTS: -Xms512m -Xmx1g
+          action.destructive_requires_name: false
         ports:
           - 9400:9400
 
@@ -60,6 +62,7 @@ jobs:
           plugins.security.disabled: true
           plugins.security.ssl.http.enabled: true
           OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+          action.destructive_requires_name: false
         ports:
           - 9202:9202
 

CHANGELOG.md

@@ -8,6 +8,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added 
+
+- Added comprehensive index management system with dynamic selection and insertion strategies for improved performance and scalability [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added `ENABLE_DATETIME_INDEX_FILTERING` environment variable to enable datetime-based index selection using collection IDs. Requires indexes in format: `STAC_ITEMS_INDEX_PREFIX_collection-id_start_year-start_month-start_day-end_year-end_month-end_day`, e.g. `items_sentinel-2-l2a_2025-06-06-2025-09-22`. Default is `false`. [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added `DATETIME_INDEX_MAX_SIZE_GB` environment variable to set maximum size limit in GB for datetime-based indexes. When an index exceeds this size, a new time-partitioned index will be created. Default is `25` GB. Only applies when `ENABLE_DATETIME_INDEX_FILTERING` is enabled. [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added search engine adapter system with support for both Elasticsearch and OpenSearch [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `SearchEngineAdapter` base class with engine-specific implementations
+  - `ElasticsearchAdapter` and `OpenSearchAdapter` with tailored index creation methods
+  - Automatic engine type detection based on client class
+  - `SearchEngineAdapterFactory` for creating appropriate adapters
+- Added datetime-based index selection strategies with caching support [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `AsyncDatetimeBasedIndexSelector` and `SyncDatetimeBasedIndexSelector` for temporal filtering
+  - `IndexCacheManager` with configurable TTL-based cache expiration (default 1 hour)
+  - `AsyncIndexAliasLoader` and `SyncIndexAliasLoader` for alias management
+  - `UnfilteredIndexSelector` as fallback for returning all available indexes
+- Added index insertion strategies with automatic partitioning [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - Simple insertion strategy (`AsyncSimpleIndexInserter`, `SyncSimpleIndexInserter`) for traditional single-index-per-collection approach
+  - Datetime-based insertion strategy (`AsyncDatetimeIndexInserter`, `SyncDatetimeIndexInserter`) with time-based partitioning
+  - Automatic index size monitoring and splitting when limits exceeded
+  - Handling of chronologically early data and bulk operations
+- Added index management utilities [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `IndexSizeManager` for size monitoring and overflow handling
+  - `DatetimeIndexManager` for datetime-based index operations
+  - Factory patterns (`IndexInsertionFactory`, `IndexSelectorFactory`) for strategy creation based on configuration
+
+
 ## [v6.1.0] - 2025-07-24
 
 ### Added

Makefile

@@ -73,10 +73,10 @@ test-opensearch:
 
 .PHONY: test
 test:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest --cov=stac_fastapi --cov-report=term-missing'
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing'
 	docker compose down
 
-	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest --cov=stac_fastapi --cov-report=term-missing'
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing'
 	docker compose down
 
 .PHONY: run-database-es

README.md

@@ -72,6 +72,7 @@ services:
     hostname: elasticsearch
     environment:
       ES_JAVA_OPTS: -Xms512m -Xmx1g
+      action.destructive_requires_name: false
     volumes:
       - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
@@ -86,6 +87,7 @@ services:
       - discovery.type=single-node
       - plugins.security.disabled=true
       - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+      - action.destructive_requires_name=false
     volumes:
       - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
       - ./opensearch/snapshots:/usr/share/opensearch/snapshots

compose.yml

@@ -37,6 +37,7 @@
     BulkTransactionMethod,
     Items,
 )
+from stac_fastapi.sfeos_helpers.database import return_date
 from stac_fastapi.types import stac as stac_types
 from stac_fastapi.types.conformance import BASE_CONFORMANCE_CLASSES
 from stac_fastapi.types.core import AsyncBaseCoreClient
@@ -324,10 +325,16 @@ async def item_collection(
             search=search, collection_ids=[collection_id]
         )
 
-        if datetime:
+        try:
+            datetime_search = return_date(datetime)
             search = self.database.apply_datetime_filter(
-                search=search, interval=datetime
+                search=search, datetime_search=datetime_search
             )
+        except (ValueError, TypeError) as e:
+            # Handle invalid interval formats if return_date fails
+            msg = f"Invalid interval format: {datetime}, error: {e}"
+            logger.error(msg)
+            raise HTTPException(status_code=400, detail=msg)
 
         if bbox:
             bbox = [float(x) for x in bbox]
@@ -342,6 +349,7 @@ async def item_collection(
             sort=None,
             token=token,
             collection_ids=[collection_id],
+            datetime_search=datetime_search,
         )
 
         items = [
@@ -500,10 +508,16 @@ async def post_search(
                 search=search, collection_ids=search_request.collections
             )
 
-        if search_request.datetime:
+        try:
+            datetime_search = return_date(search_request.datetime)
             search = self.database.apply_datetime_filter(
-                search=search, interval=search_request.datetime
+                search=search, datetime_search=datetime_search
             )
+        except (ValueError, TypeError) as e:
+            # Handle invalid interval formats if return_date fails
+            msg = f"Invalid interval format: {search_request.datetime}, error: {e}"
+            logger.error(msg)
+            raise HTTPException(status_code=400, detail=msg)
 
         if search_request.bbox:
             bbox = search_request.bbox
@@ -560,6 +574,7 @@ async def post_search(
             token=search_request.token,
             sort=sort,
             collection_ids=search_request.collections,
+            datetime_search=datetime_search,
         )
 
         fields = (

stac_fastapi/core/stac_fastapi/core/core.py

@@ -1,4 +1,5 @@
 """Utility functions to handle datetime parsing."""
+
 from datetime import datetime, timezone
 
 from stac_fastapi.types.rfc3339 import rfc3339_str_to_datetime

stac_fastapi/core/stac_fastapi/core/datetime_utils.py

@@ -1,4 +1,5 @@
 """Serializers."""
+
 import abc
 from copy import deepcopy
 from typing import Any, List, Optional

stac_fastapi/core/stac_fastapi/core/serializers.py

@@ -1,4 +1,5 @@
 """database session management."""
+
 import logging
 
 import attr