Jsonata

commhandler/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.0] - 2025-10-27
+
+### Added
+
+- JSONata expressions as response query in REST / messaging data points and message filters
+- XPath expressions as response query in REST / messaging data points and message filters
+
+### Changed
+
+- requires `sgr-specification` 2.2
+- JSON values are returned as Python types, no longer converted to string
+
+
 ## [0.4.1] - 2025-10-20
 
 ### Fixed

commhandler/pyproject.toml

@@ -19,14 +19,16 @@ maintainers = [
 dependencies = [
     "Jinja2>=3.1.0,<4.0.0",
     "jmespath>=1.0.0,<2.0.0",
+    "jsonata-python>=0.6.0,<1.0.0",
+    "parsel>=1.10.0,<1.11.0",
     "pymodbus>=3.7.0,<3.8.0",
     "gmqtt>=0.7.0,<1.0.0",
     "xsdata>=25.0.0,<26.0.0",
     "xmlschema>=4.1.0,<5.0.0",
     "aiohttp>=3.12.0,<4.0.0",
     "certifi",
     "cachetools>=6.2.0,<7.0.0",
-    "sgr-specification>=2.1.0,<3.0.0"
+    "sgr-specification>=2.2.0,<3.0.0"
 ]
 
 [project.urls]

commhandler/requirements.txt

@@ -1,10 +1,12 @@
 Jinja2>=3.1.0,<4.0.0
 jmespath>=1.0.0,<2.0.0
+jsonata-python>=0.6.0,<1.0.0
+parsel>=1.10.0,<1.11.0
 pymodbus>=3.7.0,<3.8.0
 gmqtt>=0.7.0,<1.0.0
 xsdata>=25.0.0,<26.0.0
 xmlschema>=4.1.0,<5.0.0
 aiohttp>=3.12.0,<4.0.0
 certifi
 cachetools>=6.2.0,<7.0.0
-sgr-specification>=2.1.0,<3.0.0
+sgr-specification>=2.2.0,<3.0.0

commhandler/setup.py

@@ -1,5 +1,5 @@
 from setuptools import setup, find_packages
 
 setup(
-    version="0.4.0",
+    version="0.5.0",
 )

commhandler/src/sgr_commhandler/driver/messaging/messaging_filter.py

@@ -6,12 +6,16 @@
 import re
 import json
 import jmespath
+import jsonata
+import parsel
 from typing import Any, Generic, Optional, TypeVar
 
 from sgr_specification.v0.generic.base_types import (
     JmespathFilterType,
     PlaintextFilterType,
-    RegexFilterType
+    RegexFilterType,
+    XpathFilterType,
+    JsonataFilterType
 )
 from sgr_specification.v0.product.messaging_types import MessageFilter
 
@@ -24,6 +28,8 @@ class MessagingFilter(ABC, Generic[T]):
     The base class for message filter implementations.
     """
 
+    _filter_spec: T
+
     def __init__(self, filter_spec: T):
         self._filter_spec = filter_spec
 
@@ -86,6 +92,44 @@ def is_filter_match(self, payload: Any) -> bool:
         return match is not None
 
 
+class XPathMessagingFilter(MessagingFilter[XpathFilterType]):
+    """
+    Implements an XPath filter for message payloads.
+    """
+
+    def __init__(self, filter_spec: XpathFilterType):
+        super(XPathMessagingFilter, self).__init__(filter_spec)
+
+    def is_filter_match(self, payload: Any) -> bool:
+        ret_value = str(payload)
+        regex = self._filter_spec.matches_regex or '.'
+        if self._filter_spec.query:
+            selector = parsel.Selector(ret_value)
+            ret_value = selector.xpath(self._filter_spec.query).get()
+
+        match = re.match(regex, ret_value)
+        return match is not None
+
+
+class JSONataMessagingFilter(MessagingFilter[JsonataFilterType]):
+    """
+    Implements a JSONata filter for message payloads.
+    """
+
+    def __init__(self, filter_spec: JsonataFilterType):
+        super(JSONataMessagingFilter, self).__init__(filter_spec)
+
+    def is_filter_match(self, payload: Any) -> bool:
+        ret_value = str(payload)
+        regex = self._filter_spec.matches_regex or '.'
+        if self._filter_spec.query:
+            expression = jsonata.Jsonata(self._filter_spec.query)
+            ret_value = json.dumps(expression.evaluate(json.loads(payload)))
+
+        match = re.match(regex, ret_value)
+        return match is not None
+
+
 def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]:
     """
     Creates a messaging filter from specification.
@@ -106,7 +150,9 @@ def get_messaging_filter(filter: MessageFilter) -> Optional[MessagingFilter]:
     elif filter.plaintext_filter:
         return PlaintextMessagingFilter(filter.plaintext_filter)
     elif filter.regex_filter:
-        raise Exception('regexFilter not supported')
+        return RegexMessagingFilter(filter.regex_filter)
     elif filter.xpapath_filter:
-        raise Exception('xpapathFilter not supported')
+        return XPathMessagingFilter(filter.xpapath_filter)
+    elif filter.jsonata_filter:
+        return JSONataMessagingFilter(filter.jsonata_filter)
     return None

commhandler/src/sgr_commhandler/driver/messaging/messaging_interface_async.py

@@ -3,9 +3,11 @@
 import logging
 import re
 import jmespath
+import parsel
 from collections.abc import Callable
 from typing import Any, NoReturn, Optional
 
+import jsonata
 from sgr_specification.v0.generic import ResponseQueryType
 from sgr_specification.v0.product import (
     DeviceFrame,
@@ -212,6 +214,24 @@ def _on_message(self, payload: Any):  # type: ignore
             query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else ''
             query_match = re.match(query_expression, str(payload))
             ret_value = query_match.group() if query_match is not None else str(payload)
+        elif (
+            self._in_cmd
+            and self._in_cmd.response_query
+            and self._in_cmd.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION
+        ):
+            # XPath expression
+            query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else ''
+            selector = parsel.Selector(str(payload))
+            ret_value = selector.xpath(query_expression).get()
+        elif (
+            self._in_cmd
+            and self._in_cmd.response_query
+            and self._in_cmd.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION
+        ):
+            # JSONata expression
+            query_expression = self._in_cmd.response_query.query if self._in_cmd.response_query.query else ''
+            expression = jsonata.Jsonata(query_expression)
+            ret_value = expression.evaluate(json.loads(payload))
         else:
             # plain response
             ret_value = str(payload)

commhandler/src/sgr_commhandler/driver/rest/authentication.py

@@ -10,8 +10,10 @@
 
 import aiohttp
 import jmespath
+import parsel
 from aiohttp.client import ClientSession
 from jmespath.exceptions import JMESPathError
+import jsonata
 from multidict import CIMultiDict
 from sgr_specification.v0.generic.base_types import ResponseQueryType
 from sgr_specification.v0.product import RestApiInterface
@@ -116,14 +118,14 @@ async def authenticate_with_bearer_token(
                         ):
                             # JMESPath expression
                             query_expression = service_call.response_query.query if service_call.response_query.query else ''
-                            token = jmespath.search(query_expression, json.loads(response.body))
+                            token = str(jmespath.search(query_expression, json.loads(response.body)))
                         elif (
                             service_call.response_query
                             and service_call.response_query.query_type == ResponseQueryType.JMESPATH_MAPPING
                         ):
                             # JMESPath mappings
                             mappings = service_call.response_query.jmes_path_mappings.mapping if service_call.response_query.jmes_path_mappings else []
-                            token = jmespath_mapping.map_json_response(response.body, mappings)
+                            token = str(jmespath_mapping.map_json_response(response.body, mappings))
                         elif (
                             service_call.response_query
                             and service_call.response_query.query_type == ResponseQueryType.REGULAR_EXPRESSION
@@ -132,6 +134,22 @@ async def authenticate_with_bearer_token(
                             query_expression = service_call.response_query.query if service_call.response_query.query else ''
                             query_match = re.match(query_expression, response.body)
                             token = query_match.group() if query_match is not None else None
+                        elif (
+                            service_call.response_query
+                            and service_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION
+                        ):
+                            # XPath expression
+                            query_expression = service_call.response_query.query if service_call.response_query.query else ''
+                            selector = parsel.Selector(response.body)
+                            token = str(selector.xpath(query_expression).get())
+                        elif (
+                            service_call.response_query
+                            and service_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION
+                        ):
+                            # JSONata expression
+                            query_expression = service_call.response_query.query if service_call.response_query.query else ''
+                            expression = jsonata.Jsonata(query_expression)
+                            token = str(expression.evaluate(json.loads(response.body)))
                         else:
                             # plaintext
                             token = response.body

commhandler/src/sgr_commhandler/driver/rest/restapi_interface_async.py

@@ -11,8 +11,10 @@
 import aiohttp
 import certifi
 import jmespath
+import parsel
 from aiohttp import ClientConnectionError, ClientResponseError
 from cachetools import TTLCache
+import jsonata
 from multidict import CIMultiDict
 from sgr_specification.v0.generic.base_types import ResponseQueryType
 from sgr_specification.v0.product import (
@@ -234,6 +236,28 @@ async def get_val(self, parameters: Optional[dict[str, str]] = None, skip_cache:
             query_match = re.match(query_expression, response.body)
             if query_match is not None:
                 return query_match.group()
+        elif (
+            self._read_call.response_query
+            and self._read_call.response_query.query_type == ResponseQueryType.XPATH_EXPRESSION
+        ):
+            # XPath expression
+            query_expression = template.substitute(
+                self._read_call.response_query.query if self._read_call.response_query.query else '',
+                substitutions
+            )
+            selector = parsel.Selector(response.body)
+            return selector.xpath(query_expression).get()
+        elif (
+            self._read_call.response_query
+            and self._read_call.response_query.query_type == ResponseQueryType.JSONATA_EXPRESSION
+        ):
+            # JSONata expression
+            query_expression = template.substitute(
+                self._read_call.response_query.query if self._read_call.response_query.query else '',
+                substitutions
+            )
+            expression = jsonata.Jsonata(query_expression)
+            return expression.evaluate(json.loads(response.body))
 
         # plain response
         ret_value = response.body

commhandler/src/sgr_commhandler/utils/jmespath_mapping.py

@@ -52,7 +52,7 @@ def __repr__(self) -> str:
         return f'<RecordKey={self.key()}>'
 
 
-def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> str:
+def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> Any:
     """
     Converts the structure of a JSON string using JMESpath mappings.
 
@@ -65,8 +65,8 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s
 
     Returns
     -------
-    str
-        the mapped JSON string
+    Any
+        the mapped JSON structure
     """
 
     if len(mappings) == 0:
@@ -93,7 +93,7 @@ def map_json_response(response: str, mappings: list[JmespathMappingRecord]) -> s
 
     enhanced_map = _enhance_with_namings(flat_representation, names)
 
-    return json.dumps(_build_json_node(map_to, list(enhanced_map.values())))
+    return _build_json_node(map_to, list(enhanced_map.values()))
 
 
 def _map_to_flat_list(json_str: str, keyword_map: dict[str, str]) -> Optional[dict[RecordKey, dict[str, Any]]]:

commhandler/tests/test_utils/test_jmespath.py

@@ -60,7 +60,7 @@ def test_jmespath_mapping_1():
         }
     ])
 
-    json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings)
+    json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings))
     assert json_mapped == json_expected
 
 
@@ -138,5 +138,5 @@ def test_jmespath_mapping_2():
         }
     ])
 
-    json_mapped = jmespath_mapping.map_json_response(json_input, jmes_mappings)
+    json_mapped = json.dumps(jmespath_mapping.map_json_response(json_input, jmes_mappings))
     assert json_mapped == json_expected

specification/README.md

@@ -26,7 +26,7 @@ The actual XML schema files reside in the separate [SGrSpecifications](https://g
 Install _xsdata CLI_:
 
 ```bash
-pip install xsdata[cli]
+pip install -r requirements-dev.txt
 ```
 
 Check out both the [SGrPython](https://github.com/SmartGridready/SGrPython) and

specification/requirements-dev.txt

@@ -0,0 +1 @@
+xsdata[cli]>=25.0.0,<26.0.0

specification/setup.py

@@ -1,5 +1,5 @@
 from setuptools import find_packages, setup
 
 setup(
-    version="2.1.0",
+    version="2.2.0",
 )