check potential issue with change of collection body (#303)

* check potential issue with change of collection body

* adapt tests

* more tests

* update

* update docs

* remove space

* Update README.md

* exclude hydrate markers

* Refactor/pgstac version tests (#305)

* update pypgstac version and tests multiple versions

* use python 3.11

* revert

---------

Co-authored-by: Pete Gadomski <pete.gadomski@gmail.com>

Author: vincentsarago

.github/workflows/cicd.yaml

@@ -6,18 +6,12 @@ on:
     branches: [main]
   workflow_dispatch:
 
-
 jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - {python: '3.12', pypgstac: '0.9.*'}
-          - {python: '3.12', pypgstac: '0.8.*'}
-          - {python: '3.11', pypgstac: '0.8.*'}
-          - {python: '3.10', pypgstac: '0.8.*'}
-          - {python: '3.9', pypgstac: '0.8.*'}
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']
 
     timeout-minutes: 20
 
@@ -46,7 +40,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install .[dev,server,validation]
-          python -m pip install "pypgstac==${{ matrix.pypgstac }}"
 
       - name: Run test suite
         run:  python -m pytest --cov stac_fastapi.pgstac --cov-report xml --cov-report term-missing

README.md

@@ -43,7 +43,12 @@ For more than millions of records it is recommended to either set a low connecti
 
 ### Hydration
 
-To configure **stac-fastapi-pgstac** to [hydrate search result items in the API](https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations), set the `USE_API_HYDRATE` environment variable to `true` or explicitly set the option in the PGStac Settings object.
+To configure **stac-fastapi-pgstac** to [hydrate search result items at the API level](https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations), set the `USE_API_HYDRATE` environment variable to `true`. If `false` (default) the hydration will be done in the database.
+
+| use_api_hydrate (API) | nohydrate (PgSTAC) | Hydration |
+|                  --- |                --- |       --- |
+|                False |              False |    PgSTAC |
+|                 True |               True |       API |
 
 ### Migrations
 

setup.py

@@ -16,7 +16,8 @@
     "buildpg",
     "brotli_asgi",
     "cql2>=0.3.6",
-    "pypgstac>=0.8,<0.10",
+    "pypgstac>=0.9,<0.10",
+    "hydraters>=0.1.3",
     "typing_extensions>=4.9.0",
     "jsonpatch>=1.33.0",
     "json-merge-patch>=0.3.0",
@@ -25,7 +26,6 @@
 extra_reqs = {
     "dev": [
         "pystac[validation]",
-        "pypgstac[psycopg]==0.9.*",
         "pytest-postgresql",
         "pytest",
         "pytest-cov",
@@ -36,6 +36,8 @@
         "httpx",
         "twine",
         "wheel",
+        "psycopg[binary]==3.1.*",
+        "psycopg-pool==3.1.*",
     ],
     "docs": [
         "black>=23.10.1",
@@ -67,6 +69,8 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
         "License :: OSI Approved :: MIT License",
     ],
     keywords="STAC FastAPI COG",

stac_fastapi/pgstac/config.py

@@ -174,6 +174,23 @@ class Settings(ApiSettings):
 
     prefix_path: str = ""
     use_api_hydrate: bool = False
+    """
+    When USE_API_HYDRATE=TRUE, PgSTAC database will receive `NO_HYDRATE=TRUE`
+
+    | use_api_hydrate | nohydrate | Hydration |
+    |             --- |       --- |       --- |
+    |           False |     False |    PgSTAC |
+    |            True |      True |       API |
+
+    ref: https://stac-utils.github.io/pgstac/pgstac/#runtime-configurations
+    """
+    exclude_hydrate_markers: bool = True
+    """
+    In some case, PgSTAC can return `DO_NOT_MERGE_MARKER` markers (`𒍟※`).
+    If `EXCLUDE_HYDRATE_MARKERS=TRUE` and `USE_API_HYDRATE=TRUE`, stac-fastapi-pgstac
+    will exclude those values from the responses.
+    """
+
     invalid_id_chars: List[str] = DEFAULT_INVALID_ID_CHARS
     base_item_cache: Type[BaseItemCache] = DefaultBaseItemCache
 

stac_fastapi/pgstac/core.py

@@ -331,7 +331,11 @@ async def _get_base_item(collection_id: str) -> Dict[str, Any]:
                 # Exclude None values
                 base_item = {k: v for k, v in base_item.items() if v is not None}
 
-                feature = hydrate(base_item, feature)
+                feature = hydrate(
+                    base_item,
+                    feature,
+                    strip_unmatched_markers=settings.exclude_hydrate_markers,
+                )
 
                 # Grab ids needed for links that may be removed by the fields extension.
                 collection_id = feature.get("collection")

tests/conftest.py

@@ -9,7 +9,6 @@
 import pytest
 from fastapi import APIRouter
 from httpx import ASGITransport, AsyncClient
-from pypgstac import __version__ as pgstac_version
 from pypgstac.db import PgstacDB
 from pypgstac.migrate import Migrate
 from pytest_postgresql.janitor import DatabaseJanitor
@@ -54,12 +53,6 @@
 logger = logging.getLogger(__name__)
 
 
-requires_pgstac_0_9_2 = pytest.mark.skipif(
-    tuple(map(int, pgstac_version.split("."))) < (0, 9, 2),
-    reason="PgSTAC>=0.9.2 required",
-)
-
-
 @pytest.fixture(scope="session")
 def database(postgresql_proc):
     with DatabaseJanitor(
@@ -79,7 +72,13 @@ def database(postgresql_proc):
         yield jan
 
 
-@pytest.fixture(autouse=True)
+@pytest.fixture(
+    params=[
+        "0.8.6",
+        "0.9.8",
+    ],
+    autouse=True,
+)
 async def pgstac(database):
     connection = f"postgresql://{database.user}:{quote(database.password)}@{database.host}:{database.port}/{database.dbname}"
     yield
@@ -100,7 +99,7 @@ async def pgstac(database):
 # Run all the tests that use the api_client in both db hydrate and api hydrate mode
 @pytest.fixture(
     params=[
-        # hydratation, prefix, model_validation
+        # API hydratation, prefix, model_validation
         (False, "", False),
         (False, "/router_prefix", False),
         (True, "", False),

tests/data/test2_item.json

@@ -64,7 +64,7 @@
           "nodata": 0,
           "offset": 2.03976,
           "data_type": "uint8",
-          "spatial_resolution": 60
+          "spatial_resolution": 80
         }
       ]
     },
@@ -172,7 +172,6 @@
       "type": "image/tiff; application=geotiff; profile=cloud-optimized",
       "roles": ["cloud"],
       "title": "Pixel Quality Assessment Band (QA_PIXEL)",
-      "description": "Collection 2 Level-1 Pixel Quality Assessment Band",
       "raster:bands": [
         {
           "unit": "bit index",

tests/resources/test_collection.py

@@ -4,8 +4,6 @@
 import pytest
 from stac_pydantic import Collection
 
-from ..conftest import requires_pgstac_0_9_2
-
 
 async def test_create_collection(app_client, load_test_data: Callable):
     in_json = load_test_data("test_collection.json")
@@ -349,11 +347,15 @@ async def test_get_collections_search(
     assert len(resp.json()["collections"]) == 2
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_collection_search_freetext(
     app_client, load_test_collection, load_test2_collection
 ):
+    res = await app_client.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     # free-text
     resp = await app_client.get(
         "/collections",
@@ -388,11 +390,15 @@ async def test_collection_search_freetext(
     assert len(resp.json()["collections"]) == 0
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_collection_search_freetext_advanced(
     app_client_advanced_freetext, load_test_collection, load_test2_collection
 ):
+    res = await app_client_advanced_freetext.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     # free-text
     resp = await app_client_advanced_freetext.get(
         "/collections",
@@ -436,9 +442,13 @@ async def test_collection_search_freetext_advanced(
     assert len(resp.json()["collections"]) == 0
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_all_collections_with_pagination(app_client, load_test_data):
+    res = await app_client.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     data = load_test_data("test_collection.json")
     collection_id = data["id"]
     for ii in range(0, 12):
@@ -468,9 +478,13 @@ async def test_all_collections_with_pagination(app_client, load_test_data):
     assert {"root", "self"} == {link["rel"] for link in links}
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_all_collections_without_pagination(app_client_no_ext, load_test_data):
+    res = await app_client_no_ext.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     data = load_test_data("test_collection.json")
     collection_id = data["id"]
     for ii in range(0, 12):
@@ -491,11 +505,15 @@ async def test_all_collections_without_pagination(app_client_no_ext, load_test_d
     assert {"root", "self"} == {link["rel"] for link in links}
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_get_collections_search_pagination(
     app_client, load_test_collection, load_test2_collection
 ):
+    res = await app_client.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     resp = await app_client.get("/collections")
     assert resp.json()["numberReturned"] == 2
     assert resp.json()["numberMatched"] == 2
@@ -621,12 +639,16 @@ async def test_get_collections_search_pagination(
     assert {"root", "self"} == {link["rel"] for link in links}
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.xfail(strict=False)
 @pytest.mark.asyncio
 async def test_get_collections_search_offset_1(
     app_client, load_test_collection, load_test2_collection
 ):
+    res = await app_client.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     # BUG: pgstac doesn't return a `prev` link when limit is not set
     # offset=1, should have a `previous` link
     resp = await app_client.get(

tests/resources/test_item.py

@@ -18,8 +18,6 @@
 
 from stac_fastapi.pgstac.models.links import CollectionLinks
 
-from ..conftest import requires_pgstac_0_9_2
-
 
 async def test_create_collection(app_client, load_test_data: Callable):
     in_json = load_test_data("test_collection.json")
@@ -1693,9 +1691,13 @@ async def test_get_search_link_media(app_client):
     assert get_self_link["type"] == "application/geo+json"
 
 
-@requires_pgstac_0_9_2
 @pytest.mark.asyncio
 async def test_item_search_freetext(app_client, load_test_data, load_test_collection):
+    res = await app_client.get("/_mgmt/health")
+    pgstac_version = res.json()["pgstac"]["pgstac_version"]
+    if tuple(map(int, pgstac_version.split("."))) < (0, 9, 2):
+        pass
+
     test_item = load_test_data("test_item.json")
     resp = await app_client.post(
         f"/collections/{test_item['collection']}/items", json=test_item
@@ -1722,3 +1724,73 @@ async def test_item_search_freetext(app_client, load_test_data, load_test_collec
         params={"q": "yo"},
     )
     assert resp.json()["numberReturned"] == 0
+
+
+@pytest.mark.asyncio
+async def test_item_asset_change(app_client, load_test_data):
+    """Check that changing item_assets in collection does
+    not affect existing items if hydration should not occur.
+
+    """
+    # load collection
+    data = load_test_data("test2_collection.json")
+    collection_id = data["id"]
+
+    resp = await app_client.post("/collections", json=data)
+    assert "item_assets" in data
+    assert resp.status_code == 201
+    assert "item_assets" in resp.json()
+
+    # load items
+    test_item = load_test_data("test2_item.json")
+    resp = await app_client.post(f"/collections/{collection_id}/items", json=test_item)
+    assert resp.status_code == 201
+
+    # check list of items
+    resp = await app_client.get(
+        f"/collections/{collection_id}/items", params={"limit": 1}
+    )
+    assert len(resp.json()["features"]) == 1
+    assert resp.status_code == 200
+
+    # NOTE: API or PgSTAC Hydration we should get the same values as original Item
+    assert (
+        test_item["assets"]["red"]["raster:bands"]
+        == resp.json()["features"][0]["assets"]["red"]["raster:bands"]
+    )
+
+    # NOTE: `description` is not in the item body but in the collection's item-assets
+    # because it's not in the original item it won't be hydrated
+    assert not resp.json()["features"][0]["assets"]["qa_pixel"].get("description")
+
+    ###########################################################################
+    # Remove item_assets in collection
+    operations = [{"op": "remove", "path": "/item_assets"}]
+    resp = await app_client.patch(f"/collections/{collection_id}", json=operations)
+    assert resp.status_code == 200
+
+    # Make sure item_assets is not in collection response
+    resp = await app_client.get(f"/collections/{collection_id}")
+    assert resp.status_code == 200
+    assert "item_assets" not in resp.json()
+    ###########################################################################
+
+    resp = await app_client.get(
+        f"/collections/{collection_id}/items", params={"limit": 1}
+    )
+    assert len(resp.json()["features"]) == 1
+    assert resp.status_code == 200
+
+    # NOTE: here we should only get `scale`, `offset` and `spatial_resolution`
+    # because the other values were stripped on ingestion (dehydration is a default in PgSTAC)
+    # scale and offset are no in item-asset and spatial_resolution is different, so the value in the item body is kept
+    assert ["scale", "offset", "spatial_resolution"] == list(
+        resp.json()["features"][0]["assets"]["red"]["raster:bands"][0]
+    )
+
+    # Only run this test for PgSTAC hydratation because `exclude_hydrate_markers=True` by default
+    if not app_client._transport.app.state.settings.use_api_hydrate:
+        # NOTE: `description` is not in the original item but in the collection's item-assets
+        # We get "𒍟※" because PgSTAC set it when ingesting (`description`is item-assets)
+        # because we removed item-assets, pgstac cannot hydrate this field, and thus return "𒍟※"
+        assert resp.json()["features"][0]["assets"]["qa_pixel"]["description"] == "𒍟※"