From 102f792a803bacad784a84159906d9d87fe10fdd Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Mon, 22 Dec 2025 14:44:34 +0100 Subject: [PATCH 1/2] feat: Add hidden item filtering with HIDE_ITEM_PATH Adds ability for item hiding via HIDE_ITEM_PATH env var which contains the path to variable on item level specifying if item should be hidden or not. Hidden items are excluded from search results and get_one_item() returns NotFoundError for hidden items. - Updated execute_search() to filter hidden items from search/count queries - Modified get_one_item() to respect hidden status - Uses add_hidden_filter() utility for consistent filtering --- .../elasticsearch/database_logic.py | 26 ++++++++- .../stac_fastapi/opensearch/database_logic.py | 26 ++++++++- .../sfeos_helpers/database/utils.py | 34 ++++++++++- stac_fastapi/tests/resources/test_item.py | 56 +++++++++++++++++++ 4 files changed, 136 insertions(+), 6 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index c3eadc7ae..6e3788299 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -53,6 +53,7 @@ add_collections_to_body, ) from stac_fastapi.sfeos_helpers.database.utils import ( + add_hidden_filter, merge_to_operations, operations_to_script, ) @@ -406,12 +407,22 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: Notes: The Item is retrieved from the Elasticsearch database using the `client.get` method, with the index for the Collection as the target index and the combined `mk_item_id` as the document id. + Item is hidden if hide_item_path is configured via env var. """ try: + base_query = {"term": {"_id": mk_item_id(item_id, collection_id)}} + + HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None) + + if HIDE_ITEM_PATH: + query = add_hidden_filter(base_query, HIDE_ITEM_PATH) + else: + query = base_query + response = await self.client.search( index=index_alias_by_collection_id(collection_id), body={ - "query": {"term": {"_id": mk_item_id(item_id, collection_id)}}, + "query": query, "size": 1, }, ) @@ -854,6 +865,10 @@ async def execute_search( size_limit = min(limit + 1, max_result_window) + HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None) + if HIDE_ITEM_PATH and query: + query = add_hidden_filter(query, HIDE_ITEM_PATH) + search_task = asyncio.create_task( self.client.search( index=index_param, @@ -865,11 +880,18 @@ async def execute_search( ) ) + # Apply hidden filter to count query as well + count_query = search.to_dict(count=True) + if HIDE_ITEM_PATH and "query" in count_query: + count_query["query"] = add_hidden_filter( + count_query["query"], HIDE_ITEM_PATH + ) + count_task = asyncio.create_task( self.client.count( index=index_param, ignore_unavailable=ignore_unavailable, - body=search.to_dict(count=True), + body=count_query, ) ) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 78d13d9c9..c0c3f69bb 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -49,6 +49,7 @@ add_collections_to_body, ) from stac_fastapi.sfeos_helpers.database.utils import ( + add_hidden_filter, merge_to_operations, operations_to_script, ) @@ -406,12 +407,22 @@ async def get_one_item(self, collection_id: str, item_id: str) -> Dict: Notes: The Item is retrieved from the Opensearch database using the `client.get` method, with the index for the Collection as the target index and the combined `mk_item_id` as the document id. + Item is hidden if hide_item_path is configured via env var. """ try: + base_query = {"term": {"_id": mk_item_id(item_id, collection_id)}} + + HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None) + + if HIDE_ITEM_PATH: + query = add_hidden_filter(base_query, HIDE_ITEM_PATH) + else: + query = base_query + response = await self.client.search( index=index_alias_by_collection_id(collection_id), body={ - "query": {"term": {"_id": mk_item_id(item_id, collection_id)}}, + "query": query, "size": 1, }, ) @@ -846,7 +857,11 @@ async def execute_search( index_param = ITEM_INDICES query = add_collections_to_body(collection_ids, query) - if query: + HIDE_ITEM_PATH = os.getenv("HIDE_ITEM_PATH", None) + + if HIDE_ITEM_PATH: + search_body["query"] = add_hidden_filter(query, HIDE_ITEM_PATH) + elif query: search_body["query"] = query search_after = None @@ -871,11 +886,16 @@ async def execute_search( ) ) + # Ensure hidden item is not counted + count_query = search.to_dict(count=True) + if "query" in count_query: + count_query["query"] = add_hidden_filter(count_query["query"]) + count_task = asyncio.create_task( self.client.count( index=index_param, ignore_unavailable=ignore_unavailable, - body=search.to_dict(count=True), + body=count_query, ) ) diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py index eaa596fad..c149ee8eb 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/database/utils.py @@ -5,7 +5,7 @@ """ import logging -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union from stac_fastapi.core.utilities import bbox2polygon, get_bool_env from stac_fastapi.extensions.core.transaction.request import ( @@ -361,3 +361,35 @@ def operations_to_script(operations: List, create_nest: bool = False) -> Dict: "lang": "painless", "params": params, } + + +def add_hidden_filter( + query: Optional[Dict[str, Any]] = None, hide_item_path: Optional[str] = None +) -> Dict[str, Any]: + """Add hidden filter to a query to exclude hidden items. + + Args: + query: Optional Elasticsearch query to combine with hidden filter + hide_item_path: Path to the hidden field (e.g., "properties._private.hidden") + If None or empty, return original query (no filtering) + + Returns: + Query with hidden filter applied + """ + if not hide_item_path: + return query or {"match_all": {}} + + hidden_filter = { + "bool": { + "should": [ + {"term": {hide_item_path: False}}, + {"bool": {"must_not": {"exists": {"field": hide_item_path}}}}, + ], + "minimum_should_match": 1, + } + } + + if query: + return {"bool": {"must": [query, hidden_filter]}} + else: + return hidden_filter diff --git a/stac_fastapi/tests/resources/test_item.py b/stac_fastapi/tests/resources/test_item.py index 4231f1029..3ea7728cb 100644 --- a/stac_fastapi/tests/resources/test_item.py +++ b/stac_fastapi/tests/resources/test_item.py @@ -1163,3 +1163,59 @@ async def test_search_datetime_with_null_datetime( await txn_client.delete_collection(test_collection["id"]) except Exception as e: logger.warning(f"Failed to delete collection: {e}") + + +@pytest.mark.asyncio +async def test_hidden_item_true(app_client, txn_client, load_test_data): + """Test item with hidden=true is filtered out.""" + + os.environ["HIDE_ITEM_PATH"] = "properties._private.hidden" + + test_collection = load_test_data("test_collection.json") + test_collection["id"] = "test-collection-hidden-true" + await create_collection(txn_client, collection=test_collection) + test_item = load_test_data("test_item.json") + test_item["collection"] = test_collection["id"] + test_item["id"] = "hidden-item-true" + test_item["properties"]["_private"] = {"hidden": True} + + await create_item(txn_client, test_item) + + resp = await app_client.get("/search", params={"ids": test_item["id"]}) + assert resp.status_code == 200 + result = resp.json() + assert len(result["features"]) == 0 + + resp = await app_client.get( + f"/collections/{test_item['collection']}/items/{test_item['id']}" + ) + assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_hidden_item_false(app_client, txn_client, load_test_data, monkeypatch): + """Test that item with hidden=false is not filtered out.""" + + os.environ["HIDE_ITEM_PATH"] = "properties._private.hidden" + + test_collection = load_test_data("test_collection.json") + test_collection["id"] = "test-collection-hidden-false" + await create_collection(txn_client, collection=test_collection) + + test_item = load_test_data("test_item.json") + test_item["id"] = "hidden-item-false" + test_item["collection"] = test_collection["id"] + test_item["properties"]["_private"] = {"hidden": False} + + await create_item(txn_client, test_item) + await refresh_indices(txn_client) + + resp = await app_client.get("/search", params={"ids": test_item["id"]}) + assert resp.status_code == 200 + result = resp.json() + assert len(result["features"]) == 1 + + resp = await app_client.get( + f"/collections/{test_item['collection']}/items/{test_item['id']}" + ) + assert resp.status_code == 200 From f541dfd1250e1d77e7e71c959be4bbb1c26b6632 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Mon, 22 Dec 2025 14:54:44 +0100 Subject: [PATCH 2/2] docs: update changelog and readme --- CHANGELOG.md | 2 ++ README.md | 1 + 2 files changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e016fa980..db85321b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- Added configurable hidden item filtering via HIDE_ITEM_PATH environment variable. [#566](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/566) + ### Changed ### Fixed diff --git a/README.md b/README.md index b050691bd..fe7ddddfb 100644 --- a/README.md +++ b/README.md @@ -518,6 +518,7 @@ You can customize additional settings in your `.env` file: | `PROPERTIES_END_DATETIME_FIELD` | Specifies the field used for the upper value of a datetime range for the items in the backend database. | `properties.end_datetime` | Optional | | `COLLECTION_FIELD` | Specifies the field used for the collection an item belongs to in the backend database | `collection` | Optional | | `GEOMETRY_FIELD` | Specifies the field containing the geometry of the items in the backend database | `geometry` | Optional | +| `HIDE_ITEM_PATH` | Path to boolean field that marks items as hidden (excluded from search) or not. If null, the item is returned. | `None` | Optional | > [!NOTE]