diff --git a/openapi/index_openapi.json b/openapi/index_openapi.json index 90eca7b06..c0119fafe 100644 --- a/openapi/index_openapi.json +++ b/openapi/index_openapi.json @@ -477,6 +477,19 @@ "title": "BaseRelationshipResource", "description": "Minimum requirements to represent a relationship resource" }, + "EntryMetadata": { + "properties": { + "property_metadata": { + "type": "object", + "title": "Property Metadata", + "description": "An object containing per-entry and per-property metadata. The keys are the names of the fields in attributes for which metadata is available. The values belonging to these keys are dictionaries containing the relevant metadata fields. See also [Metadata properties](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#metadata-properties)" + } + }, + "additionalProperties": true, + "type": "object", + "title": "EntryMetadata", + "description": "Contains the metadata for the attributes of an entry" + }, "EntryRelationships": { "properties": { "references": { @@ -536,13 +549,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/EntryResourceAttributes", @@ -1324,13 +1337,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/LinksResourceAttributes", diff --git a/openapi/openapi.json b/openapi/openapi.json index 10d3c22e2..7a21bbfac 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -1838,6 +1838,19 @@ ], "title": "EntryInfoResponse" }, + "EntryMetadata": { + "properties": { + "property_metadata": { + "type": "object", + "title": "Property Metadata", + "description": "An object containing per-entry and per-property metadata. The keys are the names of the fields in attributes for which metadata is available. The values belonging to these keys are dictionaries containing the relevant metadata fields. See also [Metadata properties](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#metadata-properties)" + } + }, + "additionalProperties": true, + "type": "object", + "title": "EntryMetadata", + "description": "Contains the metadata for the attributes of an entry" + }, "EntryRelationships": { "properties": { "references": { @@ -1897,13 +1910,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/EntryResourceAttributes", @@ -2481,13 +2494,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/LinksResourceAttributes", @@ -2978,13 +2991,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/ReferenceResourceAttributes" @@ -4115,13 +4128,13 @@ "meta": { "anyOf": [ { - "$ref": "#/components/schemas/Meta" + "$ref": "#/components/schemas/EntryMetadata" }, { "type": "null" } ], - "description": "a meta object containing non-standard meta-information about a resource that can not be represented as an attribute or relationship." + "description": "A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata." }, "attributes": { "$ref": "#/components/schemas/StructureResourceAttributes" diff --git a/optimade/models/entries.py b/optimade/models/entries.py index e9a0075bb..b167aedf7 100644 --- a/optimade/models/entries.py +++ b/optimade/models/entries.py @@ -1,9 +1,9 @@ from datetime import datetime from typing import Annotated, Any, ClassVar, Literal -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, model_validator -from optimade.models.jsonapi import Attributes, Relationships, Resource +from optimade.models.jsonapi import Attributes, Meta, Relationships, Resource from optimade.models.optimade_json import ( BaseRelationshipResource, DataType, @@ -117,6 +117,36 @@ def cast_immutable_id_to_str(cls, value: Any) -> str: return value +class EntryMetadata(Meta): + """Contains the metadata for the attributes of an entry""" + + property_metadata: dict = StrictField( + None, + description="""An object containing per-entry and per-property metadata. The keys are the names of the fields in attributes for which metadata is available. The values belonging to these keys are dictionaries containing the relevant metadata fields. See also [Metadata properties](https://github.com/Materials-Consortia/OPTIMADE/blob/develop/optimade.rst#metadata-properties)""", + ) + + @field_validator("property_metadata", mode="before") + def check_property_metadata_subfields(cls, value: Any) -> dict: + """Loop through any per-property metadata field and check that + the subfields are prefixed correctly. + + """ + error_fields: list[str] = [] + + if value is not None and isinstance(value, dict): + for field in value: + if property_metadata := value.get(field): + for subfield in property_metadata: + if not subfield.startswith("_"): + error_fields.append(subfield) + + if error_fields: + raise ValueError( + f"The keys under the field `property_metadata` need to be prefixed. The field(s) {error_fields} are not prefixed." + ) + return value + + class EntryResource(Resource): """The base model for an entry resource.""" @@ -171,6 +201,14 @@ class EntryResource(Resource): ), ] + meta: Annotated[ + EntryMetadata | None, + StrictField( + None, + description="""A [JSON API meta object](https://jsonapi.org/format/1.1/#document-meta) that is used to communicate metadata.""", + ), + ] = None + relationships: Annotated[ EntryRelationships | None, StrictField( @@ -179,6 +217,47 @@ class EntryResource(Resource): ), ] = None + @model_validator(mode="before") + def check_meta(cls, data: Any) -> dict | None: + """Validator to check whether the per-entry `meta` field is valid, + including stripping out any per-property metadata for properties that + do not otherwise appear in the model. + + """ + + meta = data.get("meta") + if not meta: + return data + + if property_metadata := meta.pop("property_metadata", None): + # check that all the fields under property metadata are in attributes + attributes = data.get("attributes", {}) + property_error_fields: list[str] = [] + for subfield in property_metadata: + if subfield not in attributes: + property_error_fields.append(subfield) + + if property_error_fields: + raise ValueError( + f"The keys under the field `property_metadata` need to match with the field names in attributes. The field(s) {property_error_fields} are however not present in attributes {attributes}" + ) + + meta["property_metadata"] = property_metadata + + meta_error_fields: list[str] = [] + for field in meta: + if field not in EntryMetadata.model_fields: + if not field.startswith("_"): + meta_error_fields.append(field) + + if meta_error_fields: + raise ValueError( + f"The keys under the field `meta` need to be prefixed if not otherwise defined. The field(s) {meta_error_fields} are not defined for per-entry `meta`." + ) + + data["meta"] = meta + return data + class EntryInfoProperty(BaseModel): description: Annotated[ diff --git a/optimade/server/data/test_structures.json b/optimade/server/data/test_structures.json index 5a91e3b61..c189ad981 100644 --- a/optimade/server/data/test_structures.json +++ b/optimade/server/data/test_structures.json @@ -3,6 +3,13 @@ "_id": { "$oid": "5cfb441f053b174410700d02" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": "Pure Metals" + } + } + }, "assemblies": null, "chemsys": "Ac", "cartesian_site_positions": [ @@ -80,6 +87,13 @@ "_id": { "$oid": "5cfb441f053b174410700d03" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": "Actinides_Alloys" + } + } + }, "assemblies": null, "chemsys": "Ac-Ag-Ir", "cartesian_site_positions": [ @@ -197,6 +211,13 @@ "_id": { "$oid": "5cfb441f053b174410700d04" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": "Actinides_Alloys" + } + } + }, "assemblies": null, "chemsys": "Ac-Ag-Pb", "cartesian_site_positions": [ @@ -323,6 +344,13 @@ "_id": { "$oid": "5cfb441f053b174410700d18" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": "Actinides_Alloys" + } + } + }, "assemblies": null, "chemsys": "Ac-Mg", "cartesian_site_positions": [ @@ -413,6 +441,13 @@ "_id": { "$oid": "5cfb441f053b174410700d1f" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": null + } + } + }, "assemblies": null, "chemsys": "Ac-O", "cartesian_site_positions": [ @@ -515,6 +550,11 @@ "_id": { "$oid": "5cfb441f053b174410700d6f" }, + "meta": { + "property_metadata": { + "elements_ratios": {} + } + }, "assemblies": null, "chemsys": "Ac-Cu-F-O", "cartesian_site_positions": [ @@ -639,6 +679,13 @@ "_id": { "$oid": "5cfb441f053b174410700dc9" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project": "Pure Metals" + } + } + }, "assemblies": null, "chemsys": "Ag", "cartesian_site_positions": [ @@ -706,6 +753,11 @@ "_id": { "$oid": "5cfb441f053b174410700ddd" }, + "meta": { + "property_metadata": { + "elements_ratios": null + } + }, "assemblies": null, "chemsys": "Ag-Br-Cl-Te", "cartesian_site_positions": [ @@ -896,6 +948,9 @@ "_id": { "$oid": "5cfb441f053b174410700e04" }, + "meta": { + "property_metadata": {} + }, "assemblies": null, "chemsys": "Ag-C-Cl-N-O-S", "cartesian_site_positions": [ @@ -1072,6 +1127,9 @@ "_id": { "$oid": "5cfb441f053b174410700e11" }, + "meta": { + "property_metadata": null + }, "assemblies": null, "chemsys": "Ag-C-Cl-H-N", "cartesian_site_positions": [ diff --git a/optimade/server/mappers/entries.py b/optimade/server/mappers/entries.py index 8f499f9b5..d9f4aaaad 100644 --- a/optimade/server/mappers/entries.py +++ b/optimade/server/mappers/entries.py @@ -74,7 +74,13 @@ class BaseResourceMapper: PROVIDER_FIELDS: tuple[str, ...] = () ENTRY_RESOURCE_CLASS: type[EntryResource] = EntryResource RELATIONSHIP_ENTRY_TYPES: set[str] = {"references", "structures"} - TOP_LEVEL_NON_ATTRIBUTES_FIELDS: set[str] = {"id", "type", "relationships", "links"} + TOP_LEVEL_NON_ATTRIBUTES_FIELDS: set[str] = { + "id", + "type", + "relationships", + "links", + "meta", + } @classmethod @lru_cache(maxsize=NUM_ENTRY_TYPES) @@ -118,18 +124,10 @@ def all_aliases(cls) -> Iterable[tuple[str, str]]: @classproperty @lru_cache(maxsize=1) def SUPPORTED_PREFIXES(cls) -> set[str]: - """A set of prefixes handled by this entry type. - - !!! note - This implementation only includes the provider prefix, - but in the future this property may be extended to include other - namespaces (for serving fields from, e.g., other providers or - domain-specific terms). - - """ + """A set of prefixes handled by this entry type.""" from optimade.server.config import CONFIG - return {CONFIG.provider.prefix} + return set(CONFIG.supported_prefixes) @classproperty def ALL_ATTRIBUTES(cls) -> set[str]: diff --git a/optimade/server/routers/utils.py b/optimade/server/routers/utils.py index b2f161869..0407e43aa 100644 --- a/optimade/server/routers/utils.py +++ b/optimade/server/routers/utils.py @@ -122,6 +122,13 @@ def handle_response_fields( for field in exclude_fields: if field in new_entry["attributes"]: del new_entry["attributes"][field] + if new_entry.get("meta") and ( + property_meta_data_fields := new_entry.get("meta").get( # type: ignore[union-attr] + "property_metadata" + ) + ): + if field in property_meta_data_fields: + del new_entry["meta"]["property_metadata"][field] # Include missing fields that were requested in `response_fields` for field in include_fields: diff --git a/tests/models/test_data/test_good_structures.json b/tests/models/test_data/test_good_structures.json index 562ef9069..8ee8dd447 100644 --- a/tests/models/test_data/test_good_structures.json +++ b/tests/models/test_data/test_good_structures.json @@ -166,6 +166,13 @@ "last_modified": { "$date": "2019-06-08T05:13:37.331Z" }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_originates_from_project":"piezoelectic_perovskites" + } + } + }, "band_gap": 1.23456, "chemsys": "C-H-Cl-N-Na-O-Os-P", "elements": ["C", "Cl", "H", "N", "Na", "O", "Os", "P"], diff --git a/tests/models/test_entries.py b/tests/models/test_entries.py index a3ab7e318..7f383abee 100644 --- a/tests/models/test_entries.py +++ b/tests/models/test_entries.py @@ -1,7 +1,7 @@ import pytest from pydantic import ValidationError -from optimade.models.entries import EntryRelationships +from optimade.models.entries import EntryRelationships, EntryResource def test_simple_relationships(): @@ -48,3 +48,50 @@ def test_advanced_relationships(): } with pytest.raises(ValidationError): EntryRelationships(**relationship) + + +def test_meta(): + good_entry_resource = { + "id": "goodstruct123", + "type": "structure", + "attributes": { + "last_modified": "2023-07-21T05:13:37.331Z", + "elements": ["Ac"], + "_exmpl_database_specific_property": "value1", + "elements_ratios": [1.0], + }, + "meta": { + "property_metadata": { + "elements_ratios": { + "_exmpl_mearsurement_method": "ICP-OES", + }, + "_exmpl_database_specific_property": { + "_exmpl_metadata_property": "metadata_value" + }, + } + }, + } + + EntryResource(**good_entry_resource) + + # Test that other prefixed fields are allowed in meta + good_entry_resource["meta"]["_other_database_specific_property"] = { + "_exmpl_metadata_property": "entry 3" + } + + EntryResource(**good_entry_resource) + + bad_entry_resources = [good_entry_resource.copy() for _ in range(4)] + bad_entry_resources[0]["meta"]["property_metadata"][ + "_exmpl_database_specific_property" + ] = {"metadata_property": "entry 0"} + bad_entry_resources[1]["meta"]["property_metadata"][ + "database_specific_property" + ] = {"_exmpl_metadata_property": "entry 1"} + bad_entry_resources[2]["meta"]["database_specific_property"] = { + "_exmpl_metadata_property": "entry 2" + } + + for bad_entry in bad_entry_resources: + with pytest.raises(ValueError): + EntryResource(**bad_entry) diff --git a/tests/server/query_params/conftest.py b/tests/server/query_params/conftest.py index 0d8a2c2a0..b94daf9eb 100644 --- a/tests/server/query_params/conftest.py +++ b/tests/server/query_params/conftest.py @@ -77,11 +77,13 @@ def inner( response = get_good_response(request, server) expected_fields.add("attributes") - + expected_fields.discard("meta") response_fields = set() for entry in response["data"]: response_fields.update(set(entry.keys())) response_fields.update(set(entry["attributes"].keys())) + # As "meta" is an optional field the response may or may not have it, so we remove it here to prevent problems in the assert below. + response_fields.discard("meta") assert sorted(expected_fields) == sorted(response_fields) return inner diff --git a/tests/server/routers/test_structures.py b/tests/server/routers/test_structures.py index a0e089f2d..7f8bfa726 100644 --- a/tests/server/routers/test_structures.py +++ b/tests/server/routers/test_structures.py @@ -69,6 +69,12 @@ def test_structures_endpoint_data(self): assert self.json_response["data"]["type"] == "structures" assert "attributes" in self.json_response["data"] assert "_exmpl_chemsys" in self.json_response["data"]["attributes"] + assert ( + self.json_response["data"]["meta"]["property_metadata"]["elements_ratios"][ + "_exmpl_originates_from_project" + ] + == "Pure Metals" + ) def test_check_response_single_structure(check_response): diff --git a/tests/server/test_client.py b/tests/server/test_client.py index 796cb1a7e..69d9b7bbc 100644 --- a/tests/server/test_client.py +++ b/tests/server/test_client.py @@ -509,7 +509,7 @@ def test_list_properties( results = cli.list_properties("structures") for database in results: - assert len(results[database]) == 27, str(results[database]) + assert len(results[database]) == 28, str(results[database]) results = cli.search_property("structures", "site") for database in results: