From f6f8bb25201de993fd971d42134084c568a3d6ea Mon Sep 17 00:00:00 2001 From: Matthias Krebs Date: Thu, 23 Oct 2025 11:07:45 +0200 Subject: [PATCH 1/5] prepared commhandler 0.5 and specification 2.2. added jsonata dependency. --- commhandler/CHANGELOG.md | 12 ++++++++++++ commhandler/pyproject.toml | 3 ++- commhandler/requirements.txt | 3 ++- commhandler/setup.py | 2 +- specification/requirements-dev.txt | 1 + specification/setup.py | 2 +- 6 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 specification/requirements-dev.txt diff --git a/commhandler/CHANGELOG.md b/commhandler/CHANGELOG.md index 34d3185..1ffff2d 100644 --- a/commhandler/CHANGELOG.md +++ b/commhandler/CHANGELOG.md @@ -5,6 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] + +### Added + +- JSONata expressions as response query in REST / messaging data points and message filters +- response or template query can be used in write operations + +### Changed + +- requires `sgr-specification` 2.2 + + ## [0.4.1] - 2025-10-20 ### Fixed diff --git a/commhandler/pyproject.toml b/commhandler/pyproject.toml index 3a17cc8..213e8d4 100644 --- a/commhandler/pyproject.toml +++ b/commhandler/pyproject.toml @@ -19,6 +19,7 @@ maintainers = [ dependencies = [ "Jinja2>=3.1.0,<4.0.0", "jmespath>=1.0.0,<2.0.0", + "jsonata-python>=0.6.0,<1.0.0", "pymodbus>=3.7.0,<3.8.0", "gmqtt>=0.7.0,<1.0.0", "xsdata>=25.0.0,<26.0.0", @@ -26,7 +27,7 @@ dependencies = [ "aiohttp>=3.12.0,<4.0.0", "certifi", "cachetools>=6.2.0,<7.0.0", - "sgr-specification>=2.1.0,<3.0.0" + "sgr-specification>=2.2.0,<3.0.0" ] [project.urls] diff --git a/commhandler/requirements.txt b/commhandler/requirements.txt index 3e0cda5..b996dba 100644 --- a/commhandler/requirements.txt +++ b/commhandler/requirements.txt @@ -1,5 +1,6 @@ Jinja2>=3.1.0,<4.0.0 jmespath>=1.0.0,<2.0.0 +jsonata-python>=0.6.0,<1.0.0 pymodbus>=3.7.0,<3.8.0 gmqtt>=0.7.0,<1.0.0 xsdata>=25.0.0,<26.0.0 @@ -7,4 +8,4 @@ xmlschema>=4.1.0,<5.0.0 aiohttp>=3.12.0,<4.0.0 certifi cachetools>=6.2.0,<7.0.0 -sgr-specification>=2.1.0,<3.0.0 +sgr-specification>=2.2.0,<3.0.0 diff --git a/commhandler/setup.py b/commhandler/setup.py index ccbc4e1..33fedc5 100644 --- a/commhandler/setup.py +++ b/commhandler/setup.py @@ -1,5 +1,5 @@ from setuptools import setup, find_packages setup( - version="0.4.0", + version="0.5.0", ) diff --git a/specification/requirements-dev.txt b/specification/requirements-dev.txt new file mode 100644 index 0000000..2e3e6c0 --- /dev/null +++ b/specification/requirements-dev.txt @@ -0,0 +1 @@ +xsdata[cli]>=25.0.0,<26.0.0 diff --git a/specification/setup.py b/specification/setup.py index 76b4d35..5f8316a 100644 --- a/specification/setup.py +++ b/specification/setup.py @@ -1,5 +1,5 @@ from setuptools import find_packages, setup setup( - version="2.1.0", + version="2.2.0", ) From cf164d078a5c3f85339005d71784ac833c91d0c6 Mon Sep 17 00:00:00 2001 From: Matthias Krebs Date: Thu, 23 Oct 2025 11:36:55 +0200 Subject: [PATCH 2/5] implemented JSONata response query. --- .../driver/messaging/messaging_interface_async.py | 10 ++++++++++ .../sgr_commhandler/driver/rest/authentication.py | 13 +++++++++++-- .../driver/rest/restapi_interface_async.py | 12 ++++++++++++ .../src/sgr_commhandler/utils/jmespath_mapping.py | 8 ++++---- commhandler/tests/test_utils/test_jmespath.py | 4 ++-- 5 files changed, 39 insertions(+), 8 deletions(-) diff --git a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py index d84428a..c1f4064 100644 --- a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py @@ -6,6 +6,7 @@ from collections.abc import Callable from typing import Any, NoReturn, Optional +import jsonata from sgr_specification.v0.generic import ResponseQueryType from sgr_specification.v0.product import ( DeviceFrame, @@ -212,6 +213,15 @@ def _on_message(self, payload: Any): # type: ignore query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' query_match = re.match(query_expression, str(payload)) ret_value = query_match.group() if query_match is not None else str(payload) + elif ( + self._in_cmd + and self._in_cmd.response_query + and self._in_cmd.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' + expression = jsonata.Jsonata(query_expression) + ret_value = expression.evaluate(json.loads(payload)) else: # plain response ret_value = str(payload) diff --git a/commhandler/src/sgr_commhandler/driver/rest/authentication.py b/commhandler/src/sgr_commhandler/driver/rest/authentication.py index ff3ce4b..1b8586b 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/authentication.py +++ b/commhandler/src/sgr_commhandler/driver/rest/authentication.py @@ -12,6 +12,7 @@ import jmespath from aiohttp.client import ClientSession from jmespath.exceptions import JMESPathError +import jsonata from multidict import CIMultiDict from sgr_specification.v0.generic.base_types import ResponseQueryType from sgr_specification.v0.product import RestApiInterface @@ -116,14 +117,14 @@ async def authenticate_with_bearer_token( ): # JMESPath expression query_expression = service_call.response_query.query if service_call.response_query.query else '' - token = jmespath.search(query_expression, json.loads(response.body)) + token = str(jmespath.search(query_expression, json.loads(response.body))) elif ( service_call.response_query and service_call.response_query.query_type == ResponseQueryType.JMESPATH_MAPPING ): # JMESPath mappings mappings = service_call.response_query.jmes_path_mappings.mapping if service_call.response_query.jmes_path_mappings else [] - token = jmespath_mapping.map_json_response(response.body, mappings) + token = str(jmespath_mapping.map_json_response(response.body, mappings)) elif ( service_call.response_query and service_call.response_query.query_type == ResponseQueryType.REGULAR_EXPRESSION @@ -132,6 +133,14 @@ async def authenticate_with_bearer_token( query_expression = service_call.response_query.query if service_call.response_query.query else '' query_match = re.match(query_expression, response.body) token = query_match.group() if query_match is not None else None + elif ( + service_call.response_query + and service_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = service_call.response_query.query if service_call.response_query.query else '' + expression = jsonata.Jsonata(query_expression) + token = str(expression.evaluate(json.loads(response.body))) else: # plaintext token = response.body diff --git a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py index 9b88941..7b589e8 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py @@ -13,6 +13,7 @@ import jmespath from aiohttp import ClientConnectionError, ClientResponseError from cachetools import TTLCache +import jsonata from multidict import CIMultiDict from sgr_specification.v0.generic.base_types import ResponseQueryType from sgr_specification.v0.product import ( @@ -234,6 +235,17 @@ async def get_val(self, parameters: Optional[dict[str, str]] = None, skip_cache: query_match = re.match(query_expression, response.body) if query_match is not None: return query_match.group() + elif ( + self._read_call.response_query + and self._read_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION + ): + # JSONata expression + query_expression = template.substitute( + self._read_call.response_query.query if self._read_call.response_query.query else '', + substitutions + ) + expression = jsonata.Jsonata(query_expression) + return expression.evaluate(json.loads(response.body)) # plain response ret_value = response.body diff --git a/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py b/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py index 644d168..e9254b1 100644 --- a/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py +++ b/commhandler/src/sgr_commhandler/utils/jmespath_mapping.py @@ -52,7 +52,7 @@ def __repr__(self) -> str: return f'' -def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> str: +def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> Any: """ Converts the structure of a JSON string using JMESpath mappings. @@ -65,8 +65,8 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s Returns ------- - str - the mapped JSON string + Any + the mapped JSON structure """ if len(mappings) == 0: @@ -93,7 +93,7 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s enhanced_map = _enhance_with_namings(flat_representation, names) - return json.dumps(_build_json_node(map_to, list(enhanced_map.values()))) + return _build_json_node(map_to, list(enhanced_map.values())) def _map_to_flat_list(json_str: str, keyword_map: dict[str, str]) -> Optional[dict[RecordKey, dict[str, Any]]]: diff --git a/commhandler/tests/test_utils/test_jmespath.py b/commhandler/tests/test_utils/test_jmespath.py index 72530ef..bd5c1ae 100644 --- a/commhandler/tests/test_utils/test_jmespath.py +++ b/commhandler/tests/test_utils/test_jmespath.py @@ -60,7 +60,7 @@ def test_jmespath_mapping_1(): } ]) - json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings) + json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings)) assert json_mapped == json_expected @@ -138,5 +138,5 @@ def test_jmespath_mapping_2(): } ]) - json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings) + json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings)) assert json_mapped == json_expected From 0935fff330be30ce295cc4edf325d487f120e631 Mon Sep 17 00:00:00 2001 From: Matthias Krebs Date: Thu, 23 Oct 2025 12:33:29 +0200 Subject: [PATCH 3/5] added XPath response query and message filter. --- commhandler/CHANGELOG.md | 2 +- commhandler/pyproject.toml | 1 + commhandler/requirements.txt | 1 + .../driver/messaging/messaging_filter.py | 52 +++++++++++++++++-- .../messaging/messaging_interface_async.py | 10 ++++ .../driver/rest/authentication.py | 9 ++++ .../driver/rest/restapi_interface_async.py | 12 +++++ 7 files changed, 83 insertions(+), 4 deletions(-) diff --git a/commhandler/CHANGELOG.md b/commhandler/CHANGELOG.md index 1ffff2d..4974bc6 100644 --- a/commhandler/CHANGELOG.md +++ b/commhandler/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - JSONata expressions as response query in REST / messaging data points and message filters -- response or template query can be used in write operations +- XPath expressions as response query in REST / messaging data points and message filters ### Changed diff --git a/commhandler/pyproject.toml b/commhandler/pyproject.toml index 213e8d4..adfe53e 100644 --- a/commhandler/pyproject.toml +++ b/commhandler/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "Jinja2>=3.1.0,<4.0.0", "jmespath>=1.0.0,<2.0.0", "jsonata-python>=0.6.0,<1.0.0", + "parsel>=1.10.0,<1.11.0", "pymodbus>=3.7.0,<3.8.0", "gmqtt>=0.7.0,<1.0.0", "xsdata>=25.0.0,<26.0.0", diff --git a/commhandler/requirements.txt b/commhandler/requirements.txt index b996dba..f26bc79 100644 --- a/commhandler/requirements.txt +++ b/commhandler/requirements.txt @@ -1,6 +1,7 @@ Jinja2>=3.1.0,<4.0.0 jmespath>=1.0.0,<2.0.0 jsonata-python>=0.6.0,<1.0.0 +parsel>=1.10.0,<1.11.0 pymodbus>=3.7.0,<3.8.0 gmqtt>=0.7.0,<1.0.0 xsdata>=25.0.0,<26.0.0 diff --git a/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py b/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py index 519188e..1de5461 100644 --- a/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py +++ b/commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py @@ -6,12 +6,16 @@ import re import json import jmespath +import jsonata +import parsel from typing import Any, Generic, Optional, TypeVar from sgr_specification.v0.generic.base_types import ( JmespathFilterType, PlaintextFilterType, - RegexFilterType + RegexFilterType, + XpathFilterType, + JsonataFilterType ) from sgr_specification.v0.product.messaging_types import MessageFilter @@ -24,6 +28,8 @@ class MessagingFilter(ABC, Generic[T]): The base class for message filter implementations. """ + _filter_spec: T + def __init__(self, filter_spec: T): self._filter_spec = filter_spec @@ -86,6 +92,44 @@ def is_filter_match(self, payload: Any) -> bool: return match is not None +class XPathMessagingFilter(MessagingFilter[XpathFilterType]): + """ + Implements an XPath filter for message payloads. + """ + + def __init__(self, filter_spec: XpathFilterType): + super(XPathMessagingFilter, self).__init__(filter_spec) + + def is_filter_match(self, payload: Any) -> bool: + ret_value = str(payload) + regex = self._filter_spec.matches_regex or '.' + if self._filter_spec.query: + selector = parsel.Selector(ret_value) + ret_value = selector.xpath(self._filter_spec.query).get() + + match = re.match(regex, ret_value) + return match is not None + + +class JSONataMessagingFilter(MessagingFilter[JsonataFilterType]): + """ + Implements a JSONata filter for message payloads. + """ + + def __init__(self, filter_spec: JsonataFilterType): + super(JSONataMessagingFilter, self).__init__(filter_spec) + + def is_filter_match(self, payload: Any) -> bool: + ret_value = str(payload) + regex = self._filter_spec.matches_regex or '.' + if self._filter_spec.query: + expression = jsonata.Jsonata(self._filter_spec.query) + ret_value = json.dumps(expression.evaluate(json.loads(payload))) + + match = re.match(regex, ret_value) + return match is not None + + def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]: """ Creates a messaging filter from specification. @@ -106,7 +150,9 @@ def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]: elif filter.plaintext_filter: return PlaintextMessagingFilter(filter.plaintext_filter) elif filter.regex_filter: - raise Exception('regexFilter not supported') + return RegexMessagingFilter(filter.regex_filter) elif filter.xpapath_filter: - raise Exception('xpapathFilter not supported') + return XPathMessagingFilter(filter.xpapath_filter) + elif filter.jsonata_filter: + return JSONataMessagingFilter(filter.jsonata_filter) return None diff --git a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py index c1f4064..50f5466 100644 --- a/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py @@ -3,6 +3,7 @@ import logging import re import jmespath +import parsel from collections.abc import Callable from typing import Any, NoReturn, Optional @@ -213,6 +214,15 @@ def _on_message(self, payload: Any): # type: ignore query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' query_match = re.match(query_expression, str(payload)) ret_value = query_match.group() if query_match is not None else str(payload) + elif ( + self._in_cmd + and self._in_cmd.response_query + and self._in_cmd.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else '' + selector = parsel.Selector(str(payload)) + ret_value = selector.xpath(query_expression).get() elif ( self._in_cmd and self._in_cmd.response_query diff --git a/commhandler/src/sgr_commhandler/driver/rest/authentication.py b/commhandler/src/sgr_commhandler/driver/rest/authentication.py index 1b8586b..71b0605 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/authentication.py +++ b/commhandler/src/sgr_commhandler/driver/rest/authentication.py @@ -10,6 +10,7 @@ import aiohttp import jmespath +import parsel from aiohttp.client import ClientSession from jmespath.exceptions import JMESPathError import jsonata @@ -133,6 +134,14 @@ async def authenticate_with_bearer_token( query_expression = service_call.response_query.query if service_call.response_query.query else '' query_match = re.match(query_expression, response.body) token = query_match.group() if query_match is not None else None + elif ( + service_call.response_query + and service_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = service_call.response_query.query if service_call.response_query.query else '' + selector = parsel.Selector(response.body) + token = str(selector.xpath(query_expression).get()) elif ( service_call.response_query and service_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION diff --git a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py index 7b589e8..a96a069 100644 --- a/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py +++ b/commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py @@ -11,6 +11,7 @@ import aiohttp import certifi import jmespath +import parsel from aiohttp import ClientConnectionError, ClientResponseError from cachetools import TTLCache import jsonata @@ -235,6 +236,17 @@ async def get_val(self, parameters: Optional[dict[str, str]] = None, skip_cache: query_match = re.match(query_expression, response.body) if query_match is not None: return query_match.group() + elif ( + self._read_call.response_query + and self._read_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION + ): + # XPath expression + query_expression = template.substitute( + self._read_call.response_query.query if self._read_call.response_query.query else '', + substitutions + ) + selector = parsel.Selector(response.body) + return selector.xpath(query_expression).get() elif ( self._read_call.response_query and self._read_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION From 7658b6060805a295de04daea4adcd4b7e5546e94 Mon Sep 17 00:00:00 2001 From: Matthias Krebs Date: Thu, 23 Oct 2025 12:50:10 +0200 Subject: [PATCH 4/5] updated readme and changelog. --- commhandler/CHANGELOG.md | 1 + specification/README.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/commhandler/CHANGELOG.md b/commhandler/CHANGELOG.md index 4974bc6..d0d84da 100644 --- a/commhandler/CHANGELOG.md +++ b/commhandler/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - requires `sgr-specification` 2.2 +- JSON values are returned as Python types, no longer converted to string ## [0.4.1] - 2025-10-20 diff --git a/specification/README.md b/specification/README.md index 9c728b2..44074cb 100644 --- a/specification/README.md +++ b/specification/README.md @@ -26,7 +26,7 @@ The actual XML schema files reside in the separate [SGrSpecifications](https://g Install _xsdata CLI_: ```bash -pip install xsdata[cli] +pip install -r requirements-dev.txt ``` Check out both the [SGrPython](https://github.com/SmartGridready/SGrPython) and From 78b765dcb64a42dac010f0c7cc4e090a8a3bb44c Mon Sep 17 00:00:00 2001 From: Matthias Krebs Date: Mon, 27 Oct 2025 13:55:21 +0100 Subject: [PATCH 5/5] updated changelog. --- commhandler/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commhandler/CHANGELOG.md b/commhandler/CHANGELOG.md index d0d84da..d8de98f 100644 --- a/commhandler/CHANGELOG.md +++ b/commhandler/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [0.5.0] - 2025-10-27 ### Added