diff --git a/commhandler/CHANGELOG.md b/commhandler/CHANGELOG.md index 34d3185..d8de98f 100644 --- a/commhandler/CHANGELOG.md +++ b/commhandler/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.0] - 2025-10-27 + +### Added + +- JSONata expressions as response query in REST / messaging data points and message filters +- XPath expressions as response query in REST / messaging data points and message filters + +### Changed + +- requires `sgr-specification` 2.2 +- JSON values are returned as Python types, no longer converted to string + + ## [0.4.1] - 2025-10-20 ### Fixed diff --git a/commhandler/pyproject.toml b/commhandler/pyproject.toml index 3a17cc8..adfe53e 100644 --- a/commhandler/pyproject.toml +++ b/commhandler/pyproject.toml @@ -19,6 +19,8 @@ maintainers = [ dependencies = [ "Jinja2>=3.1.0,<4.0.0", "jmespath>=1.0.0,<2.0.0", + "jsonata-python>=0.6.0,<1.0.0", + "parsel>=1.10.0,<1.11.0", "pymodbus>=3.7.0,<3.8.0", "gmqtt>=0.7.0,<1.0.0", "xsdata>=25.0.0,<26.0.0", @@ -26,7 +28,7 @@ dependencies = [ "aiohttp>=3.12.0,<4.0.0", "certifi", "cachetools>=6.2.0,<7.0.0", - "sgr-specification>=2.1.0,<3.0.0" + "sgr-specification>=2.2.0,<3.0.0" ] [project.urls] diff --git a/commhandler/requirements.txt b/commhandler/requirements.txt index 3e0cda5..f26bc79 100644 --- a/commhandler/requirements.txt +++ b/commhandler/requirements.txt @@ -1,5 +1,7 @@ Jinja2>=3.1.0,<4.0.0 jmespath>=1.0.0,<2.0.0 +jsonata-python>=0.6.0,<1.0.0 +parsel>=1.10.0,<1.11.0 pymodbus>=3.7.0,<3.8.0 gmqtt>=0.7.0,<1.0.0 xsdata>=25.0.0,<26.0.0 @@ -7,4 +9,4 @@ xmlschema>=4.1.0,<5.0.0 aiohttp>=3.12.0,<4.0.0 certifi cachetools>=6.2.0,<7.0.0 -sgr-specification>=2.1.0,<3.0.0 +sgr-specification>=2.2.0,<3.0.0 diff --git a/commhandler/setup.py b/commhandler/setup.py index ccbc4e1..33fedc5 100644 --- a/commhandler/setup.py +++ b/commhandler/setup.py @@ -1,5 +1,5 @@ from setuptools import setup, find_packages setup( - version="0.4.0", + version="0.5.0", ) diff --git a/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py b/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py index 519188e..1de5461 100644 --- a/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py +++ b/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py @@ -6,12 +6,16 @@ import re import json import jmespath +import jsonata +import parsel from typing import Any, Generic, Optional, TypeVar from sgr_specification.v0.generic.base_types import ( JmespathFilterType, PlaintextFilterType, - RegexFilterType + RegexFilterType, + XpathFilterType, + JsonataFilterType ) from sgr_specification.v0.product.messaging_types import MessageFilter @@ -24,6 +28,8 @@ class MessagingFilter(ABC, Generic[T]): The base class for message filter implementations. """ + _filter_spec: T + def __init__(self, filter_spec: T): self._filter_spec = filter_spec @@ -86,6 +92,44 @@ def is_filter_match(self, payload: Any) -> bool: return match is not None +class XPathMessagingFilter(MessagingFilter[XpathFilterType]): + """ + Implements an XPath filter for message payloads. + """ + + def __init__(self, filter_spec: XpathFilterType): + super(XPathMessagingFilter, self).__init__(filter_spec) + + def is_filter_match(self, payload: Any) -> bool: + ret_value = str(payload) + regex = self._filter_spec.matches_regex or '.' + if self._filter_spec.query: + selector = parsel.Selector(ret_value) + ret_value = selector.xpath(self._filter_spec.query).get() + + match = re.match(regex, ret_value) + return match is not None + + +class JSONataMessagingFilter(MessagingFilter[JsonataFilterType]): + """ + Implements a JSONata filter for message payloads. + """ + + def __init__(self, filter_spec: JsonataFilterType): + super(JSONataMessagingFilter, self).__init__(filter_spec) + + def is_filter_match(self, payload: Any) -> bool: + ret_value = str(payload) + regex = self._filter_spec.matches_regex or '.' + if self._filter_spec.query: + expression = jsonata.Jsonata(self._filter_spec.query) + ret_value = json.dumps(expression.evaluate(json.loads(payload))) + + match = re.match(regex, ret_value) + return match is not None + + def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]: """ Creates a messaging filter from specification. @@ -106,7 +150,9 @@ def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]: elif filter.plaintext_filter: return PlaintextMessagingFilter(filter.plaintext_filter) elif filter.regex_filter: - raise Exception('regexFilter not supported') + return RegexMessagingFilter(filter.regex_filter) elif filter.xpapath_filter: - raise Exception('xpapathFilter not supported') + return XPathMessagingFilter(filter.xpapath_filter) + elif filter.jsonata_filter: + return JSONataMessagingFilter(filter.jsonata_filter) return None diff --git a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py index d84428a..50f5466 100644 --- a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py @@ -3,9 +3,11 @@ import logging import re import jmespath +import parsel from collections.abc import Callable from typing import Any, NoReturn, Optional +import jsonata from sgr_specification.v0.generic import ResponseQueryType from sgr_specification.v0.product import ( DeviceFrame, @@ -212,6 +214,24 @@ def _on_message(self, payload: Any): # type: ignore query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' query_match = re.match(query_expression, str(payload)) ret_value = query_match.group() if query_match is not None else str(payload) + elif ( + self._in_cmd + and self._in_cmd.response_query + and self._in_cmd.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' + selector = parsel.Selector(str(payload)) + ret_value = selector.xpath(query_expression).get() + elif ( + self._in_cmd + and self._in_cmd.response_query + and self._in_cmd.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' + expression = jsonata.Jsonata(query_expression) + ret_value = expression.evaluate(json.loads(payload)) else: # plain response ret_value = str(payload) diff --git a/commhandler/src/sgr_commhandler/driver/rest/authentication.py b/commhandler/src/sgr_commhandler/driver/rest/authentication.py index ff3ce4b..71b0605 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/authentication.py +++ b/commhandler/src/sgr_commhandler/driver/rest/authentication.py @@ -10,8 +10,10 @@ import aiohttp import jmespath +import parsel from aiohttp.client import ClientSession from jmespath.exceptions import JMESPathError +import jsonata from multidict import CIMultiDict from sgr_specification.v0.generic.base_types import ResponseQueryType from sgr_specification.v0.product import RestApiInterface @@ -116,14 +118,14 @@ async def authenticate_with_bearer_token( ): # JMESPath expression query_expression = service_call.response_query.query if service_call.response_query.query else '' - token = jmespath.search(query_expression, json.loads(response.body)) + token = str(jmespath.search(query_expression, json.loads(response.body))) elif ( service_call.response_query and service_call.response_query.query_type == ResponseQueryType.JMESPATH_MAPPING ): # JMESPath mappings mappings = service_call.response_query.jmes_path_mappings.mapping if service_call.response_query.jmes_path_mappings else [] - token = jmespath_mapping.map_json_response(response.body, mappings) + token = str(jmespath_mapping.map_json_response(response.body, mappings)) elif ( service_call.response_query and service_call.response_query.query_type == ResponseQueryType.REGULAR_EXPRESSION @@ -132,6 +134,22 @@ async def authenticate_with_bearer_token( query_expression = service_call.response_query.query if service_call.response_query.query else '' query_match = re.match(query_expression, response.body) token = query_match.group() if query_match is not None else None + elif ( + service_call.response_query + and service_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = service_call.response_query.query if service_call.response_query.query else '' + selector = parsel.Selector(response.body) + token = str(selector.xpath(query_expression).get()) + elif ( + service_call.response_query + and service_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = service_call.response_query.query if service_call.response_query.query else '' + expression = jsonata.Jsonata(query_expression) + token = str(expression.evaluate(json.loads(response.body))) else: # plaintext token = response.body diff --git a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py index 9b88941..a96a069 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py @@ -11,8 +11,10 @@ import aiohttp import certifi import jmespath +import parsel from aiohttp import ClientConnectionError, ClientResponseError from cachetools import TTLCache +import jsonata from multidict import CIMultiDict from sgr_specification.v0.generic.base_types import ResponseQueryType from sgr_specification.v0.product import ( @@ -234,6 +236,28 @@ async def get_val(self, parameters: Optional[dict[str, str]] = None, skip_cache: query_match = re.match(query_expression, response.body) if query_match is not None: return query_match.group() + elif ( + self._read_call.response_query + and self._read_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = template.substitute( + self._read_call.response_query.query if self._read_call.response_query.query else '', + substitutions + ) + selector = parsel.Selector(response.body) + return selector.xpath(query_expression).get() + elif ( + self._read_call.response_query + and self._read_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = template.substitute( + self._read_call.response_query.query if self._read_call.response_query.query else '', + substitutions + ) + expression = jsonata.Jsonata(query_expression) + return expression.evaluate(json.loads(response.body)) # plain response ret_value = response.body diff --git a/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py b/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py index 644d168..e9254b1 100644 --- a/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py +++ b/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py @@ -52,7 +52,7 @@ def __repr__(self) -> str: return f'' -def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> str: +def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> Any: """ Converts the structure of a JSON string using JMESpath mappings. @@ -65,8 +65,8 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s Returns ------- - str - the mapped JSON string + Any + the mapped JSON structure """ if len(mappings) == 0: @@ -93,7 +93,7 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s enhanced_map = _enhance_with_namings(flat_representation, names) - return json.dumps(_build_json_node(map_to, list(enhanced_map.values()))) + return _build_json_node(map_to, list(enhanced_map.values())) def _map_to_flat_list(json_str: str, keyword_map: dict[str, str]) -> Optional[dict[RecordKey, dict[str, Any]]]: diff --git a/commhandler/tests/test_utils/test_jmespath.py b/commhandler/tests/test_utils/test_jmespath.py index 72530ef..bd5c1ae 100644 --- a/commhandler/tests/test_utils/test_jmespath.py +++ b/commhandler/tests/test_utils/test_jmespath.py @@ -60,7 +60,7 @@ def test_jmespath_mapping_1(): } ]) - json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings) + json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings)) assert json_mapped == json_expected @@ -138,5 +138,5 @@ def test_jmespath_mapping_2(): } ]) - json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings) + json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings)) assert json_mapped == json_expected diff --git a/specification/README.md b/specification/README.md index 9c728b2..44074cb 100644 --- a/specification/README.md +++ b/specification/README.md @@ -26,7 +26,7 @@ The actual XML schema files reside in the separate [SGrSpecifications](https://g Install _xsdata CLI_: ```bash -pip install xsdata[cli] +pip install -r requirements-dev.txt ``` Check out both the [SGrPython](https://github.com/SmartGridready/SGrPython) and diff --git a/specification/requirements-dev.txt b/specification/requirements-dev.txt new file mode 100644 index 0000000..2e3e6c0 --- /dev/null +++ b/specification/requirements-dev.txt @@ -0,0 +1 @@ +xsdata[cli]>=25.0.0,<26.0.0 diff --git a/specification/setup.py b/specification/setup.py index 76b4d35..5f8316a 100644 --- a/specification/setup.py +++ b/specification/setup.py @@ -1,5 +1,5 @@ from setuptools import find_packages, setup setup( - version="2.1.0", + version="2.2.0", )