From d281492450d46aff7e836ae8d29132cd75aee8af Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 16 Sep 2024 14:44:00 -0700 Subject: [PATCH 01/81] First pass at a heartbeat monitor. Doesn't yet check monitored heartbeats --- dripline/implementations/__init__.py | 1 + dripline/implementations/heartbeat_monitor.py | 76 +++++++++++++++++++ .../integration/docker-compose-services.yaml | 12 +++ .../services/heartbeat-monitor.yaml | 11 +++ 4 files changed, 100 insertions(+) create mode 100644 dripline/implementations/heartbeat_monitor.py create mode 100644 tests/integration/services/heartbeat-monitor.yaml diff --git a/dripline/implementations/__init__.py b/dripline/implementations/__init__.py index 3d36b05d..32a467ff 100644 --- a/dripline/implementations/__init__.py +++ b/dripline/implementations/__init__.py @@ -3,6 +3,7 @@ from .config import * from .ethernet_scpi_service import * from .entity_endpoints import * +from .heartbeat_monitor import * from .key_value_store import * from .postgres_interface import * from .postgres_sensor_logger import * diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py new file mode 100644 index 00000000..79ab68fa --- /dev/null +++ b/dripline/implementations/heartbeat_monitor.py @@ -0,0 +1,76 @@ +''' +A service for monitoring service heartbeats +''' + +from __future__ import absolute_import + +# standard libs +import logging + +import time +from datetime import datetime + +# internal imports +from dripline.core import AlertConsumer, Endpoint + +__all__ = [] +logger = logging.getLogger(__name__) + +__all__.append('HeartbeatTracker') +class HeartbeatTracker(Endpoint): + ''' + ''' + def __init__(self, **kwargs): + ''' + ''' + super(HeartbeatTracker, self).__init__(**kwargs) + self.last_timestamp = time.time() + self.is_active = True + + def process_heartbeat(self, timestamp): + ''' + ''' + logger.debug(f'New timestamp for {self.name}: {timestamp}') + dt = datetime.fromisoformat(timestamp) + posix_time = dt.timestamp() + logger.debug(f'Time since epoch: {posix_time}') + self.last_timestamp = posix_time + + def check_delay(self): + ''' + ''' + diff = time.time() - self.last_timestamp + if self.is_active: + if diff > self.parent.critical_threshold_s: + # report critical + logger.critical(f'Missing heartbeat: {self.name}') + else: + if diff > self.parent.warning_threshold_s: + # report warning + logger.warning(f'Missing heartbeat: {self.name}') + else: + # report inactive heartbeat received + logger.debug(f'Inactive heartbeat: time difference: {diff}') + + +__all__.append('HeartbeatMonitor') +class HeartbeatMonitor(AlertConsumer): + ''' + An alert consumer which listens to heartbeat messages and keeps track of the time since the last was received + + ''' + def __init__(self, time_between_checks_s=60, warning_threshold_s=120, critical_threshold_s=300, **kwargs): + ''' + ''' + super(HeartbeatMonitor, self).__init__(**kwargs) + + self.time_between_checks_s = time_between_checks_s + self.warning_threshold_s = warning_threshold_s + self.critical_threshold_s = critical_threshold_s + + def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): + if not a_routing_key_data['service_name'] in self.sync_children: + logger.warning(f'received unexpected heartbeat;\npayload: {a_payload}\nrouting key data: {a_routing_key_data}\ntimestamp: {a_message_timestamp}') + return + + self.sync_children[a_routing_key_data['service_name']].process_heartbeat(a_message_timestamp) diff --git a/tests/integration/docker-compose-services.yaml b/tests/integration/docker-compose-services.yaml index c2a1bfff..9b06faf0 100644 --- a/tests/integration/docker-compose-services.yaml +++ b/tests/integration/docker-compose-services.yaml @@ -76,6 +76,18 @@ services: configs: - dl_pw.txt + heartbeat-monitor: + image: ghcr.io/driplineorg/dripline-python:${DLPY_IMG_TAG:-latest-dev} + depends_on: + rabbit-broker: + condition: service_healthy + volumes: + - ./services/heartbeat-monitor.yaml:/root/heartbeat-monitor.yaml + command: + bash -c "dl-serve -vv -b rabbit-broker -u dripline --password-file /dl_pw.txt -c /root/heartbeat-monitor.yaml heartbeat_interval_s=10" + configs: + - dl_pw.txt + configs: dl_pw.txt: file: ./password.txt diff --git a/tests/integration/services/heartbeat-monitor.yaml b/tests/integration/services/heartbeat-monitor.yaml new file mode 100644 index 00000000..57d6e260 --- /dev/null +++ b/tests/integration/services/heartbeat-monitor.yaml @@ -0,0 +1,11 @@ +name: heartbeat_monitor +module: HeartbeatMonitor + +# AlertConsumer Inits +alert_keys: + - "heartbeat.#" +alert_key_parser_re: 'heartbeat\.(?P\w+)' + +endpoints: + - name: heartbeat_monitor + module: HeartbeatTracker From 13ef3df34295fe2134fa06c8cb521ca6268bdaa8 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 13 Dec 2024 09:54:37 -0800 Subject: [PATCH 02/81] Switch rabbitmq to v4 --- tests/integration/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/docker-compose.yaml b/tests/integration/docker-compose.yaml index aa0c27ab..d22546cb 100644 --- a/tests/integration/docker-compose.yaml +++ b/tests/integration/docker-compose.yaml @@ -2,7 +2,7 @@ services: # The broker for the mesh rabbit-broker: - image: rabbitmq:3-management + image: rabbitmq:4-management ports: - "15672:15672" environment: From 2cd1feb149f84edd987efd765c867b73d3bb31e6 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 6 Jan 2025 16:17:15 -0800 Subject: [PATCH 03/81] Adapting Dockerfile to enable in-container dev builds --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f881e57a..c49d8a20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG img_repo=dripline-cpp #ARG img_tag=develop ARG img_tag=v2.10.0 -FROM ${img_user}/${img_repo}:${img_tag} +FROM ${img_user}/${img_repo}:${img_tag} AS deps ## would prefer not to do this, just run ldconfig after the build to get things ## into the ld.so.conf cache... use this only when developing and adding libs @@ -14,7 +14,10 @@ RUN apt-get update && \ apt-get --fix-missing -y install \ libpq-dev && \ rm -rf /var/lib/apt/lists/* && \ - pip install ipython pytest + pip install ipython pytest &&\ + pip install aiohttp sqlalchemy psycopg2 PyYAML uuid asteval colorlog + +FROM deps COPY . /usr/local/src_py/ From 68dac738b902ba3c1302d607e8c6219005cacb39 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 6 Jan 2025 16:17:39 -0800 Subject: [PATCH 04/81] Adding a compose file to use a dev build --- tests/integration/docker-compose-dev.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/integration/docker-compose-dev.yaml diff --git a/tests/integration/docker-compose-dev.yaml b/tests/integration/docker-compose-dev.yaml new file mode 100644 index 00000000..7a7bee46 --- /dev/null +++ b/tests/integration/docker-compose-dev.yaml @@ -0,0 +1,19 @@ +# Compose file that can be used for a development workflow, integrated with the rest of the integration environment +# The dripline-python source path is mounted into the container, assuming that this is being run from [dl-py top]/tests/integration +# Once the container is started, run `pip install -e /usr/local/src_dev` +# Then you should be able to run dripline applications and edit library source files on the host. + +services: + dev: + image: ghcr.io/driplineorg/dripline-python:${DLPY_IMG_TAG:-latest-dev} + depends_on: + rabbit-broker: + condition: service_healthy + volumes: + - ../..:/usr/local/src_dev + - ./dripline_mesh.yaml:/root/.dripline_mesh.yaml + environment: + - DRIPLINE_USER=dripline + - DRIPLINE_PASSWORD=dripline + command: > + bash From d4782d1ab151b8e925d7630aaa77afb7fdbb0b7f Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 6 Jan 2025 16:18:05 -0800 Subject: [PATCH 05/81] Basic working version of the heartbeat monitor --- dripline/implementations/heartbeat_monitor.py | 79 +++++++++++++++++-- 1 file changed, 72 insertions(+), 7 deletions(-) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index 79ab68fa..85ffc099 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -9,6 +9,8 @@ import time from datetime import datetime +from enum import Enum +import threading # internal imports from dripline.core import AlertConsumer, Endpoint @@ -23,9 +25,10 @@ class HeartbeatTracker(Endpoint): def __init__(self, **kwargs): ''' ''' - super(HeartbeatTracker, self).__init__(**kwargs) + Endpoint.__init__(self, **kwargs) self.last_timestamp = time.time() self.is_active = True + self.status = HeartbeatTracker.Status.OK def process_heartbeat(self, timestamp): ''' @@ -41,17 +44,26 @@ def check_delay(self): ''' diff = time.time() - self.last_timestamp if self.is_active: - if diff > self.parent.critical_threshold_s: + if diff > self.service.critical_threshold_s: # report critical logger.critical(f'Missing heartbeat: {self.name}') + self.status = HeartbeatTracker.Status.CRITICAL else: - if diff > self.parent.warning_threshold_s: + if diff > self.service.warning_threshold_s: # report warning logger.warning(f'Missing heartbeat: {self.name}') + self.status = HeartbeatTracker.Status.WARNING + else: + logger.debug(f'Heartbeat status ok: {self.name}') + self.status = HeartbeatTracker.Status.OK else: # report inactive heartbeat received logger.debug(f'Inactive heartbeat: time difference: {diff}') + class Status(Enum): + OK = 0 + WARNING = 1 + CRITICAL = 2 __all__.append('HeartbeatMonitor') class HeartbeatMonitor(AlertConsumer): @@ -59,18 +71,71 @@ class HeartbeatMonitor(AlertConsumer): An alert consumer which listens to heartbeat messages and keeps track of the time since the last was received ''' - def __init__(self, time_between_checks_s=60, warning_threshold_s=120, critical_threshold_s=300, **kwargs): + def __init__(self, time_between_checks_s=20, warning_threshold_s=120, critical_threshold_s=300, add_unknown_heartbeats=True, **kwargs): ''' + Args: + time_between_checks_s (int): number of seconds between heartbeat status checks + warning_threshold_s (int): warning threshold for missing heartbeats (in seconds) + critical_threshold_s (int): critical threshold for missing heartbeats (in seconds) + add_unknown_heartbeats (bool): whether or not to add a new endpoint if an unknown heartbeat is received + socket_timeout (int): number of seconds to wait for a reply from the device before timeout. ''' - super(HeartbeatMonitor, self).__init__(**kwargs) + AlertConsumer.__init__(self, **kwargs) + # Total sleep time is made of multiple smaller sleeps between checking whether the application is cancelled, + # assuming that time_between_checks_s is larger than the appproximately 5 seconds between checking whether the application is canclled + # Sleep time shouldn't be less than time_between_checks_s, so n_sleeps is forced to be 1 or more. self.time_between_checks_s = time_between_checks_s + self.n_sleeps = max(1, round(time_between_checks_s / 5)) + self.sleep_time_s = self.time_between_checks_s / self.n_sleeps + logger.warning(f'Time between checks: {self.time_between_checks_s}, n_sleeps: {self.n_sleeps}, sleep_time: {self.sleep_time_s}') + self.warning_threshold_s = warning_threshold_s self.critical_threshold_s = critical_threshold_s + self.add_unknown_heartbeats = add_unknown_heartbeats + + def run(self): + monitor_thread = threading.Thread(target=self.monitor_heartbeats) + + monitor_thread.start() + + # Run the standard service in the main thread + AlertConsumer.run(self) + + monitor_thread.join() + + def monitor_heartbeats(self): + while not self.is_canceled(): + try: + logger.debug('Checking endpoints') + self.run_checks() + + logger.debug(f'Sleeping for {self.time_between_checks_s} s') + for i in range(self.n_sleeps): + if self.is_canceled(): + return + time.sleep(self.sleep_time_s) + except Exception as err: + logger.error(f'Exception caught in monitor_heartbeats\' outer check: {err}') + self.cancel(1) + + def run_checks(self): + for an_endpoint in self.sync_children.values(): + try: + an_endpoint.check_delay() + except Exception as err: + logger.error(f'Unable to get status of endpoint {an_endpoint.name}: {err}') def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): - if not a_routing_key_data['service_name'] in self.sync_children: + service_name = a_routing_key_data['service_name'] + if not service_name in self.sync_children: logger.warning(f'received unexpected heartbeat;\npayload: {a_payload}\nrouting key data: {a_routing_key_data}\ntimestamp: {a_message_timestamp}') + if self.add_unknown_heartbeats: + self.add_child(HeartbeatTracker(name=service_name)) + logger.debug(f'Added endpoint for unknown heartbeat from {service_name}') return - self.sync_children[a_routing_key_data['service_name']].process_heartbeat(a_message_timestamp) + try: + self.sync_children[service_name].process_heartbeat(a_message_timestamp) + except Exception as err: + logger.error(f'Unable to handle payload for heartbeat from service {service_name}: {err}') From 28039c32f96b03195f247a81ee99cc71bb0ad621 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 6 Jan 2025 17:13:01 -0800 Subject: [PATCH 06/81] Add unknown status and collect monitoring data to be processed --- dripline/implementations/heartbeat_monitor.py | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index 85ffc099..0fc7018b 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -14,6 +14,7 @@ # internal imports from dripline.core import AlertConsumer, Endpoint +import scarab __all__ = [] logger = logging.getLogger(__name__) @@ -28,7 +29,7 @@ def __init__(self, **kwargs): Endpoint.__init__(self, **kwargs) self.last_timestamp = time.time() self.is_active = True - self.status = HeartbeatTracker.Status.OK + self.status = HeartbeatTracker.Status.UNKNOWN def process_heartbeat(self, timestamp): ''' @@ -59,11 +60,14 @@ def check_delay(self): else: # report inactive heartbeat received logger.debug(f'Inactive heartbeat: time difference: {diff}') + self.status = HeartbeatTracker.Status.UNKNOWN + return self.status class Status(Enum): OK = 0 WARNING = 1 CRITICAL = 2 + UNKNOWN = -1 __all__.append('HeartbeatMonitor') class HeartbeatMonitor(AlertConsumer): @@ -88,7 +92,7 @@ def __init__(self, time_between_checks_s=20, warning_threshold_s=120, critical_t self.time_between_checks_s = time_between_checks_s self.n_sleeps = max(1, round(time_between_checks_s / 5)) self.sleep_time_s = self.time_between_checks_s / self.n_sleeps - logger.warning(f'Time between checks: {self.time_between_checks_s}, n_sleeps: {self.n_sleeps}, sleep_time: {self.sleep_time_s}') + #logger.warning(f'Time between checks: {self.time_between_checks_s}, n_sleeps: {self.n_sleeps}, sleep_time: {self.sleep_time_s}') self.warning_threshold_s = warning_threshold_s self.critical_threshold_s = critical_threshold_s @@ -105,26 +109,54 @@ def run(self): monitor_thread.join() def monitor_heartbeats(self): + ''' + Performs heartbeat monitoring + ''' while not self.is_canceled(): try: logger.debug('Checking endpoints') - self.run_checks() + self.process_report(self.run_checks()) - logger.debug(f'Sleeping for {self.time_between_checks_s} s') + #logger.debug(f'Sleeping for {self.time_between_checks_s} s') for i in range(self.n_sleeps): if self.is_canceled(): return time.sleep(self.sleep_time_s) except Exception as err: logger.error(f'Exception caught in monitor_heartbeats\' outer check: {err}') - self.cancel(1) + scarab.SignalHandler.cancel_all(1) def run_checks(self): + ''' + Checks all endpoints and collects endpoint names by heartbeat tracker status. + ''' + report_data = { + HeartbeatTracker.Status.OK: [], + HeartbeatTracker.Status.WARNING: [], + HeartbeatTracker.Status.CRITICAL: [], + HeartbeatTracker.Status.UNKNOWN: [], + } for an_endpoint in self.sync_children.values(): try: - an_endpoint.check_delay() + report_data[an_endpoint.check_delay()].append(an_endpoint.name) except Exception as err: logger.error(f'Unable to get status of endpoint {an_endpoint.name}: {err}') + return report_data + + def process_report(self, report_data): + ''' + Print out the information from the monitoring report data. + + This function can be overridden to handle the monitoring report differently. + ''' + if report_data[HeartbeatTracker.Status.CRITICAL]: + logger.error(f'Services with CRITICAL status:\n{report_data[HeartbeatTracker.Status.CRITICAL]}') + if report_data[HeartbeatTracker.Status.WARNING]: + logger.warning(f'Services with WARNING status:\n{report_data[HeartbeatTracker.Status.WARNING]}') + if report_data[HeartbeatTracker.Status.OK]: + logger.info(f'Services with OK status:\n{report_data[HeartbeatTracker.Status.OK]}') + if report_data[HeartbeatTracker.Status.UNKNOWN]: + logger.info(f'Services with UNKNOWN status:\n{report_data[HeartbeatTracker.Status.UNKNOWN]}') def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): service_name = a_routing_key_data['service_name'] From f2cefde019813b317feab1c9b42f5af9cec8aedd Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 6 Jan 2025 17:16:05 -0800 Subject: [PATCH 07/81] Add a note about the location of the _dripline library in a pip -e build --- tests/integration/services/heartbeat-monitor.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/services/heartbeat-monitor.yaml b/tests/integration/services/heartbeat-monitor.yaml index 57d6e260..ec1d959d 100644 --- a/tests/integration/services/heartbeat-monitor.yaml +++ b/tests/integration/services/heartbeat-monitor.yaml @@ -1,6 +1,9 @@ name: heartbeat_monitor module: HeartbeatMonitor +heartbeat_interval_s: 30 +warning_threshold_s: 45 + # AlertConsumer Inits alert_keys: - "heartbeat.#" From 1fbfc0d91bdc5a3e5cf12db3cda40a9b163876d1 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 7 Jan 2025 14:38:35 -0800 Subject: [PATCH 08/81] Added note about library location in the dev build --- tests/integration/docker-compose-dev.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/docker-compose-dev.yaml b/tests/integration/docker-compose-dev.yaml index 7a7bee46..334dc249 100644 --- a/tests/integration/docker-compose-dev.yaml +++ b/tests/integration/docker-compose-dev.yaml @@ -2,6 +2,9 @@ # The dripline-python source path is mounted into the container, assuming that this is being run from [dl-py top]/tests/integration # Once the container is started, run `pip install -e /usr/local/src_dev` # Then you should be able to run dripline applications and edit library source files on the host. +# Note that the _dripline library does not get installed in the normal location with a `pip -e` installation, +# and it instead ends up in the source directory. +# You can preface any dl-serve command with `PYTHONPATH=/usr/local/src_dev` to set the python path correctly. services: dev: From 1ef7aaa48c3367fb8c2e5681c42761d4bfa49ed7 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 7 Jan 2025 14:39:07 -0800 Subject: [PATCH 09/81] Return information about the time-since-last-heartbeat as part of the status check --- dripline/implementations/heartbeat_monitor.py | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index 0fc7018b..fe54632a 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -8,7 +8,7 @@ import logging import time -from datetime import datetime +from datetime import datetime, timedelta from enum import Enum import threading @@ -61,7 +61,7 @@ def check_delay(self): # report inactive heartbeat received logger.debug(f'Inactive heartbeat: time difference: {diff}') self.status = HeartbeatTracker.Status.UNKNOWN - return self.status + return {'status': self.status, 'time_since_last_hb': diff} class Status(Enum): OK = 0 @@ -138,7 +138,13 @@ def run_checks(self): } for an_endpoint in self.sync_children.values(): try: - report_data[an_endpoint.check_delay()].append(an_endpoint.name) + endpoint_report = an_endpoint.check_delay() + report_data[endpoint_report['status']].append( + { + 'name': an_endpoint.name, + 'time_since_last_hb': endpoint_report['time_since_last_hb'], + } + ) except Exception as err: logger.error(f'Unable to get status of endpoint {an_endpoint.name}: {err}') return report_data @@ -150,13 +156,21 @@ def process_report(self, report_data): This function can be overridden to handle the monitoring report differently. ''' if report_data[HeartbeatTracker.Status.CRITICAL]: - logger.error(f'Services with CRITICAL status:\n{report_data[HeartbeatTracker.Status.CRITICAL]}') + logger.error('Services with CRITICAL status:') + for endpoint_data in report_data[HeartbeatTracker.Status.CRITICAL]: + logger.error(f'\t{endpoint_data['name']} -- TSLH: {timedelta(seconds=endpoint_data['time_since_last_hb'])}') if report_data[HeartbeatTracker.Status.WARNING]: - logger.warning(f'Services with WARNING status:\n{report_data[HeartbeatTracker.Status.WARNING]}') + logger.warning('Services with WARNING status:') + for endpoint_data in report_data[HeartbeatTracker.Status.WARNING]: + logger.warning(f'\t{endpoint_data['name']} -- TSLH: {timedelta(seconds=endpoint_data['time_since_last_hb'])}') if report_data[HeartbeatTracker.Status.OK]: - logger.info(f'Services with OK status:\n{report_data[HeartbeatTracker.Status.OK]}') + logger.info(f'Services with OK status:') + for endpoint_data in report_data[HeartbeatTracker.Status.OK]: + logger.info(f'\t{endpoint_data['name']} -- TSLH: {timedelta(seconds=endpoint_data['time_since_last_hb'])}') if report_data[HeartbeatTracker.Status.UNKNOWN]: - logger.info(f'Services with UNKNOWN status:\n{report_data[HeartbeatTracker.Status.UNKNOWN]}') + logger.info(f'Services with UNKNOWN status:') + for endpoint_data in report_data[HeartbeatTracker.Status.UNKNOWN]: + logger.info(f'\t{endpoint_data['name']} -- TSLH: {timedelta(seconds=endpoint_data['time_since_last_hb'])}') def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): service_name = a_routing_key_data['service_name'] @@ -171,3 +185,7 @@ def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): self.sync_children[service_name].process_heartbeat(a_message_timestamp) except Exception as err: logger.error(f'Unable to handle payload for heartbeat from service {service_name}: {err}') + + def do_get(self): + return self.run_checks() + \ No newline at end of file From 3c07fff996141c6252ce493be7461be65253b7d3 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 7 Jan 2025 14:39:41 -0800 Subject: [PATCH 10/81] Fix use of the signal handler --- bin/dl-serve | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/dl-serve b/bin/dl-serve index b3986cf6..79006815 100755 --- a/bin/dl-serve +++ b/bin/dl-serve @@ -74,11 +74,14 @@ class Serve(dripline.core.ObjectCreator): # Create the service and register it with the signal handler the_service = self.create_object( this_config, 'Service', this_auth ) sig_handler.add_cancelable(the_service) + #scarab.SignalHandler.add_cancelable(the_service) logger.info(f'Service {the_service.name} has been built') # Run the service the_service.run() + scarab.SignalHandler.remove_cancelable(the_service) + if __name__ == '__main__': # App object the_main = scarab.MainApp() From a1a9bc6c2482682cb05d7474e3c5416acbe36fec Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 15 Jan 2025 15:59:08 -0800 Subject: [PATCH 11/81] Adding a status-check message --- dripline/implementations/heartbeat_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index fe54632a..6690a6ab 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -155,6 +155,7 @@ def process_report(self, report_data): This function can be overridden to handle the monitoring report differently. ''' + logger.info('Heartbeat Monitor Status Check') if report_data[HeartbeatTracker.Status.CRITICAL]: logger.error('Services with CRITICAL status:') for endpoint_data in report_data[HeartbeatTracker.Status.CRITICAL]: From c9ac9c21901e137fab3f9a495b48272b795ec136 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 15 Jan 2025 16:00:17 -0800 Subject: [PATCH 12/81] Match C++ logging colors and fix verbosity setting --- bin/dl-serve | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/dl-serve b/bin/dl-serve index 79006815..19376b42 100755 --- a/bin/dl-serve +++ b/bin/dl-serve @@ -32,7 +32,16 @@ class Serve(dripline.core.ObjectCreator): try: import colorlog modified_format = base_format.format('%(log_color)s', '%(purple)s') - self._logging_format = colorlog.ColoredFormatter(modified_format, datefmt=time_format[:-4]) + self._logging_format = colorlog.ColoredFormatter(modified_format, + datefmt=time_format[:-4], + log_colors={ + 'DEBUG': 'cyan', + 'INFO': 'green', + 'WARNING': 'yellow', + 'ERROR': 'red', + 'CRITICAL': 'red,bg_white', + }, + ) except ImportError: modified_format = base_format.format(' ', '') self._logging_format = logging.Formatter(modified_format, time_format[:-4]) @@ -52,9 +61,10 @@ class Serve(dripline.core.ObjectCreator): this_config_param = the_app.primary_config this_auth = the_app.auth - verbosity = the_app.global_verbosity + verbosity = scarab.s2py_verbosity(the_app.global_verbosity) #print(f"verbosity is {verbosity}") self.handler.setLevel(verbosity) + logger.setLevel(verbosity) sig_handler = scarab.SignalHandler() From 59408ba78d196dfd48fe7f230b599aca3d86bce8 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 15 Jan 2025 16:02:26 -0800 Subject: [PATCH 13/81] Use dl-cpp v2.10.1 --- .github/workflows/publish.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 4af57065..1b3a1df9 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,7 +13,7 @@ env: BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.0' + BASE_IMAGE_TAG: 'v2.10.1' # BASE_IMAGE_TAG: 'develop' # DEV_SUFFIX: '' diff --git a/Dockerfile b/Dockerfile index c49d8a20..f2e8cd54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG img_user=ghcr.io/driplineorg ARG img_repo=dripline-cpp #ARG img_tag=develop -ARG img_tag=v2.10.0 +ARG img_tag=v2.10.1 FROM ${img_user}/${img_repo}:${img_tag} AS deps From 50e1e5b19b4830e4164ac2d81383497508e2ea47 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 24 Jan 2025 21:43:58 -0800 Subject: [PATCH 14/81] Add missing ThrowReply --- dripline/core/entity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index cb9ef3bd..de991806 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -7,7 +7,7 @@ import scarab from .endpoint import Endpoint -from dripline.core import MsgAlert +from dripline.core import MsgAlert, ThrowReply __all__ = [] import logging From 89b65c5a335b3991454c61a4895c077722e93459 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 30 Jan 2025 11:01:04 -0800 Subject: [PATCH 15/81] Trying to fix an import error --- dripline/core/entity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index de991806..23f771b7 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -1,13 +1,13 @@ import datetime import functools -import types import numbers import scarab +from _dripline import MsgAlert from .endpoint import Endpoint -from dripline.core import MsgAlert, ThrowReply +from .throw_reply import ThrowReply __all__ = [] import logging From 307faf78aafac01c86b4be221dbaeaa70282a798 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 31 Jan 2025 17:23:03 -0800 Subject: [PATCH 16/81] Hopefully fixing MsgAlert import --- dripline/core/entity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 23f771b7..7e074f34 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -5,7 +5,7 @@ import scarab -from _dripline import MsgAlert +from _dripline.core import MsgAlert from .endpoint import Endpoint from .throw_reply import ThrowReply __all__ = [] From ea99d8aa004526210bd923784150fd1753b29dd7 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 3 Feb 2025 16:08:44 -0800 Subject: [PATCH 17/81] Test ubuntu-22.04 GHA runners --- .github/workflows/publish.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 1b3a1df9..f10dc965 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -21,7 +21,7 @@ jobs: test_docker: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 env: TAG: gha-test @@ -94,7 +94,7 @@ jobs: build_and_push: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: | github.event_name == 'push' || From 2defb64c1939dd7f8f53ba869c400c93253a28ee Mon Sep 17 00:00:00 2001 From: "Walter C. Pettus" Date: Wed, 12 Feb 2025 20:41:49 -0800 Subject: [PATCH 18/81] Alternate log condition triggers, log_interval now does not automatically trigger logging --- dripline/core/entity.py | 54 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 7e074f34..6478cb87 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -31,6 +31,11 @@ def wrapper(*args, **kwargs): values.update({'value_raw': args[0]}) logger.debug('set done, now log') self.log_a_value(values) + try: + this_value = float(values[self._check_field]) + except (TypeError, ValueError): + this_value = False + self._last_log_value = this_value return result return wrapper @@ -58,16 +63,29 @@ class Entity(Endpoint): ''' #check_on_set -> allows for more complex logic to confirm successful value updates # (for example, the success condition may be measuring another endpoint) - def __init__(self, get_on_set=False, log_routing_key_prefix='sensor_value', log_interval=0, log_on_set=False, calibration=None, **kwargs): + def __init__(self, + get_on_set=False, + log_on_set=False, + log_routing_key_prefix='sensor_value', + log_interval=0, + max_interval=0, + max_fractional_change=0, + check_field='value_cal', + calibration=None, + **kwargs): ''' Args: get_on_set: if true, calls to on_set are immediately followed by an on_get, which is returned + log_on_set: if true, always call log_a_value() immediately after on_set + **Note:** requires get_on_set be true, overrides must be equivalent log_routing_key_prefix: first term in routing key used in alert messages which log values - log_interval: how often to log the Entity's value. If 0 then scheduled logging is disabled; + log_interval: how often to check the Entity's value. If 0 then scheduled logging is disabled; if a number, interpreted as number of seconds; if a dict, unpacked as arguments to the datetime.time_delta initializer; if a datetime.timedelta taken as the new value - log_on_set: if true, always call log_a_value() immediately after on_set - **Note:** requires get_on_set be true, overrides must be equivalent + max_interval: max allowed time interval between logging, allows usage of conditional logging. If 0, + then logging values occurs every log_interval. + max_fractional_change: max allowed fractional difference between subsequent values to trigger log condition. + check_field: result field to check, 'value_cal' or 'value_raw' calibration (string || dict) : if string, updated with raw on_get() result via str.format() in @calibrate decorator, used to populate raw and calibrated values fields of a result payload. If a dictionary, the raw result is used @@ -81,13 +99,16 @@ def __init__(self, get_on_set=False, log_routing_key_prefix='sensor_value', log_ # keep a reference to the on_set (possibly decorated in a subclass), needed for changing *_on_set configurations self.__initial_on_set = self.on_set - self._get_on_set = None self._log_on_set = None self.get_on_set = get_on_set self.log_on_set = log_on_set self.log_interval = log_interval + self._max_interval = max_interval + self._max_fractional_change = max_fractional_change + self._check_field = check_field self._log_action_id = None + self._last_log_time = None @property def get_on_set(self): @@ -136,10 +157,31 @@ def log_interval(self, new_interval): def scheduled_log(self): logger.debug("in a scheduled log event") result = self.on_get() + try: + this_value = float(result[self._check_field]) + except (TypeError, ValueError): + this_value = False + # Various checks for log condition + if self._last_log_time is None: + logger.debug("log because no last log") + elif (datetime.datetime.utcnow() - self._last_log_time).total_seconds() > self._max_interval: + logger.debug("log because too much time") + elif this_value is False: + logger.warning(f"cannot check value change for {self.name}") + return + elif ((self._last_log_value == 0 and this_value != 0) or + (self._last_log_value != 0 and\ + abs((self._last_log_value - this_value)/self._last_log_value)>self._max_fractional_change)): + logger.debug("log because change magnitude") + else: + logger.debug("no log condition met, not logging") + return + self._last_log_value = this_value self.log_a_value(result) def log_a_value(self, the_value): - logger.debug(f"value to log is:\n{the_value}") + logger.info(f"value to log for {self.name} is:\n{the_value}") + self._last_log_time = datetime.datetime.utcnow() the_alert = MsgAlert.create(payload=scarab.to_param(the_value), routing_key=f'{self.log_routing_key_prefix}.{self.name}') alert_sent = self.service.send(the_alert) From 72e69cbb6914effab4bdb2eb3998f818723a735c Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 14 Feb 2025 15:23:27 -0800 Subject: [PATCH 19/81] Update dl-cpp version used to 2.10.2 --- .github/workflows/publish.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index f10dc965..f8a93e68 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,7 +13,7 @@ env: BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.1' + BASE_IMAGE_TAG: 'v2.10.2' # BASE_IMAGE_TAG: 'develop' # DEV_SUFFIX: '' diff --git a/Dockerfile b/Dockerfile index f2e8cd54..f09c51f2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG img_user=ghcr.io/driplineorg ARG img_repo=dripline-cpp #ARG img_tag=develop -ARG img_tag=v2.10.1 +ARG img_tag=v2.10.2 FROM ${img_user}/${img_repo}:${img_tag} AS deps From 2121d766dfed8f1babea676d78585aa074e9615e Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 14 Feb 2025 15:33:51 -0800 Subject: [PATCH 20/81] Add Receiver to _Service inheritance in python binding --- module_bindings/dripline_core/_service_pybind.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/module_bindings/dripline_core/_service_pybind.hh b/module_bindings/dripline_core/_service_pybind.hh index 244876a1..873bc8b0 100644 --- a/module_bindings/dripline_core/_service_pybind.hh +++ b/module_bindings/dripline_core/_service_pybind.hh @@ -26,6 +26,7 @@ namespace dripline_pybind _service_trampoline, dripline::core, dripline::endpoint, + dripline::receiver, dripline::scheduler<>, scarab::cancelable >( mod, "_Service", "Service binding" ) From f9a86d9e4687c4ace7eed2886034193a3a1a4752 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Fri, 14 Feb 2025 17:49:52 -0800 Subject: [PATCH 21/81] Added RequestSender class that should handle sending dl messages. Modified Service to include this class. --- dripline/core/request_sender.py | 118 ++++++++++++++++++++++++++++++++ dripline/core/service.py | 5 +- 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 dripline/core/request_sender.py diff --git a/dripline/core/request_sender.py b/dripline/core/request_sender.py new file mode 100644 index 00000000..f9c55474 --- /dev/null +++ b/dripline/core/request_sender.py @@ -0,0 +1,118 @@ +__all__ = [] + +import scarab + +from _dripline.core import op_t, create_dripline_auth_spec, Core, DriplineConfig, Receiver, MsgRequest, MsgReply, DriplineError + +import logging +logger = logging.getLogger(__name__) + +__all__.append("RequestSender") +class RequestSender(): + ''' + A mixin class that provides convenient methods for dripline interactions in a dripline objects. + Intended for use as a dripline client in scripts or interactive sessions. + ''' + def __init__(self, sender: Core, timeout_s: int=10, confirm_retcodes: bool=True): + ''' + Configures an interface with the necessary parameters. + + Parameters + ---------- + sender : Core + Provide a Core object which will use this mixin. This object will be the one to actually send the messages as it implements the Core functions + timeout_s: int, optional + Time to wait for a reply, in seconds -- default is 10 s + confirm_retcodes: bool, optional + If True, and if a reply is received with retcode != 0, raises an exception -- default is True + ''' + self.sender = sender + self._confirm_retcode = confirm_retcodes + self.timeout_s = timeout_s + self._receiver = Receiver() + + + def _send_request(self, msgop, target, specifier=None, payload=None, timeout=None, lockout_key=None): + ''' + internal helper method to standardize sending request messages + ''' + a_specifier = specifier if specifier is not None else "" + a_request = MsgRequest.create(payload=scarab.to_param(payload), msg_op=msgop, routing_key=target, specifier=a_specifier) + a_request.lockout_key = lockout_key if lockout_key is not None else "" + receive_reply = self.sender.send(a_request) + if not receive_reply.successful_send: + raise DriplineError('unable to send request') + return receive_reply + + def _receive_reply(self, reply_pkg, timeout_s): + ''' + internal helper method to standardize receiving reply messages + ''' + sig_handler = scarab.SignalHandler() + sig_handler.add_cancelable(self._receiver) + result = self._receiver.wait_for_reply(reply_pkg, timeout_s * 1000) # receiver expects ms + sig_handler.remove_cancelable(self._receiver) + return result + + def get(self, endpoint: str, specifier: str=None, lockout_key=None, timeout_s: int=0) -> MsgReply: + ''' + Send a get request to an endpoint and return the reply message. + + Parameters + ---------- + endpoint: str + Routing key to which the request should be sent. + specifier: str, optional + Specifier to add to the request, if needed. + timeout_s: int | float, optional + Maximum time to wait for a reply in seconds (default is 0) + A timeout of 0 seconds means no timeout will be used. + ''' + reply_pkg = self._send_request( msgop=op_t.get, target=endpoint, specifier=specifier, lockout_key=lockout_key ) + result = self._receive_reply( reply_pkg, timeout_s ) + return result + + def set(self, endpoint: str, value: str | int | float | bool, specifier: str=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: + ''' + Send a set request to an endpoint and return the reply message. + + Parameters + ---------- + endpoint: str + Routing key to which the request should be sent. + value: str | int | float | bool + Value to assign in the set operation + specifier: str, optional + Specifier to add to the request, if needed. + timeout_s: int | float, optional + Maximum time to wait for a reply in seconds (default is 0) + A timeout of 0 seconds means no timeout will be used. + ''' + payload = {'values':[value]} + reply_pkg = self._send_request( msgop=op_t.set, target=endpoint, specifier=specifier, payload=payload, lockout_key=lockout_key ) + result = self._receive_reply( reply_pkg, timeout_s ) + return result + + def cmd(self, endpoint: str, specifier: str, ordered_args=None, keyed_args=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: + ''' + Send a cmd request to an endpoint and return the reply message. + + Parameters + ---------- + endpoint: str + Routing key to which the request should be sent. + ordered_args: array, optional + Array of values to assign under 'values' in the payload, if any + keyed_args: dict, optional + Keyword arguments to assign to the payload, if any + specifier: str + Specifier to add to the request. For a dripline-python endpoint, this will be the method executed. + timeout_s: int | float, optional + Maximum time to wait for a reply in seconds (default is 0) + A timeout of 0 seconds means no timeout will be used. + ''' + payload = {'values': [] if ordered_args is None else ordered_args} + payload.update({} if keyed_args is None else keyed_args) + reply_pkg = self._send_request( msgop=op_t.cmd, target=endpoint, specifier=specifier, lockout_key=lockout_key, payload=payload ) + result = self._receive_reply( reply_pkg, timeout_s ) + return result diff --git a/dripline/core/service.py b/dripline/core/service.py index 3a185220..ee6077ef 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -4,6 +4,7 @@ from _dripline.core import _Service, DriplineConfig, create_dripline_auth_spec from .throw_reply import ThrowReply from .object_creator import ObjectCreator +from .request_sender import RequestSender import datetime import logging @@ -11,7 +12,7 @@ logger = logging.getLogger(__name__) __all__.append('Service') -class Service(_Service, ObjectCreator): +class Service(_Service, ObjectCreator, RequestSender): ''' The primary unit of software that connects to a broker and typically provides an interface with an instrument or other software. @@ -130,6 +131,8 @@ def __init__(self, name, make_connection=True, endpoints=None, add_endpoints_now _Service.__init__(self, config=scarab.to_param(config), auth=auth, make_connection=make_connection) + RequestSender.__init__(self, sender=self) + # Endpoints self.endpoint_configs = endpoints if( add_endpoints_now ): From 63d0d554f2c284bd2c4ca6cfe8d5b50e084a65c2 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Tue, 18 Feb 2025 18:39:02 -0800 Subject: [PATCH 22/81] Added request_sender class. Tested with service. Not tested with Interface yet. --- dripline/core/interface.py | 89 ++------------------------------- dripline/core/request_sender.py | 18 +++++-- 2 files changed, 18 insertions(+), 89 deletions(-) diff --git a/dripline/core/interface.py b/dripline/core/interface.py index 05182b1d..bd78a146 100644 --- a/dripline/core/interface.py +++ b/dripline/core/interface.py @@ -3,12 +3,13 @@ import scarab from _dripline.core import op_t, create_dripline_auth_spec, Core, DriplineConfig, Receiver, MsgRequest, MsgReply, DriplineError +from .request_sender import RequestSender import logging logger = logging.getLogger(__name__) __all__.append("Interface") -class Interface(Core): +class Interface(Core, RequestSender): ''' A class that provides user-friendly methods for dripline interactions in a Python interpreter. Intended for use as a dripline client in scripts or interactive sessions. @@ -62,93 +63,9 @@ def __init__(self, username: str | dict=None, password: str | dict=None, driplin auth.process_spec() Core.__init__(self, config=scarab.to_param(dripline_config), auth=auth) + RequestSender.__init__(self, sender=self) self._confirm_retcode = confirm_retcodes self.timeout_s = timeout_s self._receiver = Receiver() - - def _send_request(self, msgop, target, specifier=None, payload=None, timeout=None, lockout_key=None): - ''' - internal helper method to standardize sending request messages - ''' - a_specifier = specifier if specifier is not None else "" - a_request = MsgRequest.create(payload=scarab.to_param(payload), msg_op=msgop, routing_key=target, specifier=a_specifier) - a_request.lockout_key = lockout_key if lockout_key is not None else "" - receive_reply = self.send(a_request) - if not receive_reply.successful_send: - raise DriplineError('unable to send request') - return receive_reply - - def _receive_reply(self, reply_pkg, timeout_s): - ''' - internal helper method to standardize receiving reply messages - ''' - sig_handler = scarab.SignalHandler() - sig_handler.add_cancelable(self._receiver) - result = self._receiver.wait_for_reply(reply_pkg, timeout_s * 1000) # receiver expects ms - sig_handler.remove_cancelable(self._receiver) - return result - - def get(self, endpoint: str, specifier: str=None, lockout_key=None, timeout_s: int=0) -> MsgReply: - ''' - Send a get request to an endpoint and return the reply message. - - Parameters - ---------- - endpoint: str - Routing key to which the request should be sent. - specifier: str, optional - Specifier to add to the request, if needed. - timeout_s: int | float, optional - Maximum time to wait for a reply in seconds (default is 0) - A timeout of 0 seconds means no timeout will be used. - ''' - reply_pkg = self._send_request( msgop=op_t.get, target=endpoint, specifier=specifier, lockout_key=lockout_key ) - result = self._receive_reply( reply_pkg, timeout_s ) - return result - - def set(self, endpoint: str, value: str | int | float | bool, specifier: str=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: - ''' - Send a set request to an endpoint and return the reply message. - - Parameters - ---------- - endpoint: str - Routing key to which the request should be sent. - value: str | int | float | bool - Value to assign in the set operation - specifier: str, optional - Specifier to add to the request, if needed. - timeout_s: int | float, optional - Maximum time to wait for a reply in seconds (default is 0) - A timeout of 0 seconds means no timeout will be used. - ''' - payload = {'values':[value]} - reply_pkg = self._send_request( msgop=op_t.set, target=endpoint, specifier=specifier, payload=payload, lockout_key=lockout_key ) - result = self._receive_reply( reply_pkg, timeout_s ) - return result - - def cmd(self, endpoint: str, specifier: str, ordered_args=None, keyed_args=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: - ''' - Send a cmd request to an endpoint and return the reply message. - - Parameters - ---------- - endpoint: str - Routing key to which the request should be sent. - ordered_args: array, optional - Array of values to assign under 'values' in the payload, if any - keyed_args: dict, optional - Keyword arguments to assign to the payload, if any - specifier: str - Specifier to add to the request. For a dripline-python endpoint, this will be the method executed. - timeout_s: int | float, optional - Maximum time to wait for a reply in seconds (default is 0) - A timeout of 0 seconds means no timeout will be used. - ''' - payload = {'values': [] if ordered_args is None else ordered_args} - payload.update({} if keyed_args is None else keyed_args) - reply_pkg = self._send_request( msgop=op_t.cmd, target=endpoint, specifier=specifier, lockout_key=lockout_key, payload=payload ) - result = self._receive_reply( reply_pkg, timeout_s ) - return result diff --git a/dripline/core/request_sender.py b/dripline/core/request_sender.py index f9c55474..53ae452d 100644 --- a/dripline/core/request_sender.py +++ b/dripline/core/request_sender.py @@ -2,8 +2,9 @@ import scarab -from _dripline.core import op_t, create_dripline_auth_spec, Core, DriplineConfig, Receiver, MsgRequest, MsgReply, DriplineError +from _dripline.core import op_t, Core, Receiver, MsgRequest, MsgReply, DriplineError +import uuid import logging logger = logging.getLogger(__name__) @@ -32,13 +33,24 @@ def __init__(self, sender: Core, timeout_s: int=10, confirm_retcodes: bool=True) self._receiver = Receiver() + def _check_lockout_key(self, lockout_key:str=None): + nilkey = uuid.UUID('00000000-0000-0000-0000-000000000000') + if lockout_key is None: + return nilkey + try: + return uuid.UUID(lockout_key) + except ValueError: + logger.warning("Lockout Key '{}' is not properly uuid formatted. Defaulting to nilkey. ".format(lockout_key)) + return nilkey + + def _send_request(self, msgop, target, specifier=None, payload=None, timeout=None, lockout_key=None): ''' internal helper method to standardize sending request messages ''' a_specifier = specifier if specifier is not None else "" a_request = MsgRequest.create(payload=scarab.to_param(payload), msg_op=msgop, routing_key=target, specifier=a_specifier) - a_request.lockout_key = lockout_key if lockout_key is not None else "" + a_request.lockout_key = self._check_lockout_key(lockout_key) receive_reply = self.sender.send(a_request) if not receive_reply.successful_send: raise DriplineError('unable to send request') @@ -52,7 +64,7 @@ def _receive_reply(self, reply_pkg, timeout_s): sig_handler.add_cancelable(self._receiver) result = self._receiver.wait_for_reply(reply_pkg, timeout_s * 1000) # receiver expects ms sig_handler.remove_cancelable(self._receiver) - return result + return result.payload.to_python() def get(self, endpoint: str, specifier: str=None, lockout_key=None, timeout_s: int=0) -> MsgReply: ''' From 79076038dc47ed9ff24da965401efe75af499829 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Mon, 3 Mar 2025 19:38:20 -0500 Subject: [PATCH 23/81] Removed weird encapsulation fo response in do_get_request. --- dripline/core/service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dripline/core/service.py b/dripline/core/service.py index ee6077ef..7ff8d239 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -173,10 +173,7 @@ def do_get_request(self, a_request_message): logger.debug(f"specifier is: {a_specifier}") an_attribute = getattr(self, a_specifier) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - the_node = scarab.ParamNode() - the_node["values"] = scarab.ParamArray() - the_node["values"].push_back(scarab.ParamValue(an_attribute)) - return a_request_message.reply(payload=the_node) + return a_request_message.reply(payload=scarab.ParamValue(an_attribute)) except AttributeError as this_error: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") From a3c2ca29332dddb93c172a3e2fa14b7533859780 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Mon, 7 Apr 2025 11:38:25 -0700 Subject: [PATCH 24/81] Changed do_get_request to handle returning dictionaries --- dripline/core/service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dripline/core/service.py b/dripline/core/service.py index 3a185220..794c92b8 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -170,10 +170,7 @@ def do_get_request(self, a_request_message): logger.debug(f"specifier is: {a_specifier}") an_attribute = getattr(self, a_specifier) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - the_node = scarab.ParamNode() - the_node["values"] = scarab.ParamArray() - the_node["values"].push_back(scarab.ParamValue(an_attribute)) - return a_request_message.reply(payload=the_node) + return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) except AttributeError as this_error: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") From e344704b6d060c41787cc37285b48cc9656740fe Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Mon, 7 Apr 2025 13:09:04 -0700 Subject: [PATCH 25/81] Changed result_to_scarab_payload to handle dictionaries and any. --- dripline/core/endpoint.py | 10 ++++------ dripline/core/service.py | 7 ++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index ab408ccc..80dc450b 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -34,8 +34,9 @@ def result_to_scarab_payload(self, result: str): try: return scarab.to_param(result) except Exception as e: - raise ThrowReply('service_error_bad_payload', - f"{self.name} unable to convert result to scarab payload: {result}") + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) def do_get_request(self, a_request_message): logger.info("in do_get_request") @@ -46,10 +47,7 @@ def do_get_request(self, a_request_message): logger.debug(f"specifier is: {a_specifier}") an_attribute = getattr(self, a_specifier) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - the_node = scarab.ParamNode() - the_node.add("values", scarab.ParamArray()) - the_node["values"].push_back(scarab.ParamValue(an_attribute)) - return a_request_message.reply(payload=the_node) + return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) except AttributeError as this_error: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") diff --git a/dripline/core/service.py b/dripline/core/service.py index 7ff8d239..aaa5127d 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -161,8 +161,9 @@ def result_to_scarab_payload(self, result: str): try: return scarab.to_param(result) except Exception as e: - raise ThrowReply('service_error_bad_payload', - f"{self.name} unable to convert result to scarab payload: {result}") + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) def do_get_request(self, a_request_message): logger.info("in get_request") @@ -173,7 +174,7 @@ def do_get_request(self, a_request_message): logger.debug(f"specifier is: {a_specifier}") an_attribute = getattr(self, a_specifier) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - return a_request_message.reply(payload=scarab.ParamValue(an_attribute)) + return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) except AttributeError as this_error: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") From e19cbb074afdf34d86f204ba1a0fa265bab528ff Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Mon, 7 Apr 2025 13:11:27 -0700 Subject: [PATCH 26/81] Adjusted do_get_request and result_to_scarab_payload to handle dicts and an. --- dripline/core/endpoint.py | 10 ++++------ dripline/core/service.py | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index ab408ccc..80dc450b 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -34,8 +34,9 @@ def result_to_scarab_payload(self, result: str): try: return scarab.to_param(result) except Exception as e: - raise ThrowReply('service_error_bad_payload', - f"{self.name} unable to convert result to scarab payload: {result}") + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) def do_get_request(self, a_request_message): logger.info("in do_get_request") @@ -46,10 +47,7 @@ def do_get_request(self, a_request_message): logger.debug(f"specifier is: {a_specifier}") an_attribute = getattr(self, a_specifier) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - the_node = scarab.ParamNode() - the_node.add("values", scarab.ParamArray()) - the_node["values"].push_back(scarab.ParamValue(an_attribute)) - return a_request_message.reply(payload=the_node) + return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) except AttributeError as this_error: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") diff --git a/dripline/core/service.py b/dripline/core/service.py index 794c92b8..26efb961 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -158,8 +158,9 @@ def result_to_scarab_payload(self, result: str): try: return scarab.to_param(result) except Exception as e: - raise ThrowReply('service_error_bad_payload', - f"{self.name} unable to convert result to scarab payload: {result}") + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) def do_get_request(self, a_request_message): logger.info("in get_request") From 4d595e428181a0cf4fc77255ea5bbe8995b95894 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Mon, 7 Apr 2025 20:18:45 -0700 Subject: [PATCH 27/81] Added RequestReceiver class as a mixin for endpoint and service. Get returns with 'values' field without a specifier. --- dripline/core/endpoint.py | 94 ++++---------------------- dripline/core/request_receiver.py | 106 ++++++++++++++++++++++++++++++ dripline/core/service.py | 101 ++++------------------------ 3 files changed, 133 insertions(+), 168 deletions(-) create mode 100644 dripline/core/request_receiver.py diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index 80dc450b..c7300f9a 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -3,6 +3,7 @@ import scarab from _dripline.core import _Endpoint from .throw_reply import ThrowReply +from .request_receiver import RequestReceiver import logging @@ -11,7 +12,7 @@ __all__.append('Endpoint') -class Endpoint(_Endpoint): +class Endpoint(_Endpoint, RequestReceiver): ''' Base class for all objects which can be sent dripline requests. Every object described in a runtime configuration passed to `dl-serve` should derive from this class (either directly or indirectly). @@ -24,86 +25,17 @@ def __init__(self, name): ''' _Endpoint.__init__(self, name) - def result_to_scarab_payload(self, result: str): - """ - Intercept result values and throw error if scarab is unable to convert to param - TODO: Handles global Exception case, could be more specific - Args: - result (str): request message passed - """ - try: - return scarab.to_param(result) - except Exception as e: - result_str = str(result) - logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") - return scarab.to_param(result_str) - - def do_get_request(self, a_request_message): - logger.info("in do_get_request") - a_specifier = a_request_message.specifier.to_string() - if (a_specifier): - logger.debug("has specifier") - try: - logger.debug(f"specifier is: {a_specifier}") - an_attribute = getattr(self, a_specifier) - logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) - except AttributeError as this_error: - raise ThrowReply('service_error_invalid_specifier', - f"endpoint {self.name} has no attribute {a_specifier}, unable to get") - else: - logger.debug('no specifier') - the_value = self.on_get() - return a_request_message.reply(payload=self.result_to_scarab_payload(the_value)) + def do_get_request(self, a_request_message): + return self._do_get_request(a_request_message) - def do_set_request(self, a_request_message): - a_specifier = a_request_message.specifier.to_string() - try: - new_value = a_request_message.payload["values"][0]() - except Exception as err: - raise ThrowReply('service_error_bad_payload', f'Set called with invalid values: {err}') - new_value = getattr(new_value, "as_"+new_value.type())() - logger.debug(f'new_value is [{new_value}]') - if ( a_specifier ): - if not hasattr(self, a_specifier): - raise ThrowReply('service_error_invalid_specifier', - "endpoint {} has no attribute {}, unable to set".format(self.name, a_specifier)) - setattr(self, a_specifier, new_value) - return a_request_message.reply() - else: - result = self.on_set(new_value) - return a_request_message.reply(payload=self.result_to_scarab_payload(result)) - - def do_cmd_request(self, a_request_message): - # Note: any command executed in this way must return a python data structure which is - # able to be converted to a Param object (to be returned in the reply message) - method_name = a_request_message.specifier.to_string() - try: - method_ref = getattr(self, method_name) - except AttributeError as e: - raise ThrowReply('service_error_invalid_method', - "error getting command's corresponding method: {}".format(str(e))) - the_kwargs = a_request_message.payload.to_python() - the_args = the_kwargs.pop('values', []) - try: - result = method_ref(*the_args, **the_kwargs) - except TypeError as e: - raise ThrowReply('service_error_invalid_value', - f'A TypeError occurred while calling the requested method for endpoint {self.name}: {method_name}. Values provided may be invalid.\nOriginal error: {str(e)}') - return a_request_message.reply(payload=self.result_to_scarab_payload(result)) + def do_set_request(self, a_request_message): + return self._do_set_request(a_request_message) + def do_cmd_request(self, a_request_message): + return self._do_cmd_request(a_request_message) + def on_get(self): - ''' - placeholder method for getting the value of an endpoint. - Implementations may override to enable OP_GET operations. - The implementation must return a value which is able to be passed to the ParamValue constructor. - ''' - raise ThrowReply('service_error_invalid_method', "{} does not implement on_get".format(self.__class__)) - - def on_set(self, _value): - ''' - placeholder method for setting the value of an endpoint. - Implementations may override to enable OP_SET operations. - Any returned object must already be a scarab::Param object - ''' - raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) + return self._on_get() + + def on_set(self, value): + return self._on_set(value) \ No newline at end of file diff --git a/dripline/core/request_receiver.py b/dripline/core/request_receiver.py new file mode 100644 index 00000000..06eb6d2e --- /dev/null +++ b/dripline/core/request_receiver.py @@ -0,0 +1,106 @@ +__all__ = [] + +import scarab + +from .throw_reply import ThrowReply + + +import logging +logger = logging.getLogger(__name__) + +__all__.append("RequestReceiver") +class RequestReceiver(): + ''' + A mixin class that provides methods for dripline responses in a dripline objects. + Intended for use as a centralized object containing response methods in dripline. + ''' + + def result_to_scarab_payload(self, result: str): + """ + Intercept result values and throw error if scarab is unable to convert to param + TODO: Handles global Exception case, could be more specific + Args: + result (str): request message passed + """ + try: + return scarab.to_param(result) + except Exception as e: + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) + + def _do_get_request(self, a_request_message): + logger.info("in get_request") + a_specifier = a_request_message.specifier.to_string() + if (a_specifier): + logger.debug("has specifier") + try: + logger.debug(f"specifier is: {a_specifier}") + an_attribute = getattr(self, a_specifier) + the_node = scarab.ParamNode() + the_node.add("values", scarab.ParamArray()) + the_node["values"].push_back(self.result_to_scarab_payload(an_attribute)) + logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") + return a_request_message.reply(payload=the_node) + except AttributeError as this_error: + raise ThrowReply('service_error_invalid_specifier', + f"endpoint {self.name} has no attribute {a_specifier}, unable to get") + else: + logger.debug('no specifier') + the_value = self._on_get() + return a_request_message.reply(payload=self.result_to_scarab_payload(the_value)) + + def _do_set_request(self, a_request_message): + + a_specifier = a_request_message.specifier.to_string() + if not "values" in a_request_message.payload: + raise ThrowReply('service_error_bad_payload', + 'setting called without values, but values are required for set') + new_value = a_request_message.payload["values"][0]() + new_value = getattr(new_value, "as_" + new_value.type())() + logger.debug(f'Attempting to set new_value to [{new_value}]') + + if (a_specifier): + if not hasattr(self, a_specifier): + raise ThrowReply('service_error_invalid_specifier', + "endpoint {} has no attribute {}, unable to set".format(self.name, a_specifier)) + setattr(self, a_specifier, new_value) + return a_request_message.reply() + else: + result = self.on_set(new_value) + return a_request_message.reply(payload=self.result_to_scarab_payload(result)) + + def _do_cmd_request(self, a_request_message): + # Note: any command executed in this way must return a python data structure which is + # able to be converted to a Param object (to be returned in the reply message) + method_name = a_request_message.specifier.to_string() + try: + method_ref = getattr(self, method_name) + except AttributeError as e: + raise ThrowReply('service_error_invalid_method', + "error getting command's corresponding method: {}".format(str(e))) + the_kwargs = a_request_message.payload.to_python() + the_args = the_kwargs.pop('values', []) + print(f'args: {the_args} -- kwargs: {the_kwargs}') + try: + result = method_ref(*the_args, **the_kwargs) + except TypeError as e: + raise ThrowReply('service_error_invalid_value', + f'A TypeError occurred while calling the requested method for endpoint {self.name}: {method_name}. Values provided may be invalid.\nOriginal error: {str(e)}') + return a_request_message.reply(payload=self.result_to_scarab_payload(result)) + + def _on_get(self): + ''' + placeholder method for getting the value of an endpoint. + Implementations may override to enable OP_GET operations. + The implementation must return a value which is able to be passed to the ParamValue constructor. + ''' + raise ThrowReply('service_error_invalid_method', "{} does not implement on_get".format(self.__class__)) + + def _on_set(self, _value): + ''' + placeholder method for setting the value of an endpoint. + Implementations may override to enable OP_SET operations. + Any returned object must already be a scarab::Param object + ''' + raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) \ No newline at end of file diff --git a/dripline/core/service.py b/dripline/core/service.py index aaa5127d..5eb23e30 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -5,6 +5,7 @@ from .throw_reply import ThrowReply from .object_creator import ObjectCreator from .request_sender import RequestSender +from .request_receiver import RequestReceiver import datetime import logging @@ -12,7 +13,7 @@ logger = logging.getLogger(__name__) __all__.append('Service') -class Service(_Service, ObjectCreator, RequestSender): +class Service(_Service, ObjectCreator, RequestSender, RequestReceiver): ''' The primary unit of software that connects to a broker and typically provides an interface with an instrument or other software. @@ -130,7 +131,6 @@ def __init__(self, name, make_connection=True, endpoints=None, add_endpoints_now auth.process_spec() _Service.__init__(self, config=scarab.to_param(config), auth=auth, make_connection=make_connection) - RequestSender.__init__(self, sender=self) # Endpoints @@ -149,91 +149,18 @@ def add_endpoints_from_config(self): if getattr(an_endpoint, 'log_interval', datetime.timedelta(seconds=0)) > datetime.timedelta(seconds=0): logger.debug("queue up start logging for '{}'".format(an_endpoint.name)) an_endpoint.start_logging() + + def do_get_request(self, a_request_message): + return self._do_get_request(a_request_message) + def do_set_request(self, a_request_message): + return self._do_set_request(a_request_message) - def result_to_scarab_payload(self, result: str): - """ - Intercept result values and throw error if scarab is unable to convert to param - TODO: Handles global Exception case, could be more specific - Args: - result (str): request message passed - """ - try: - return scarab.to_param(result) - except Exception as e: - result_str = str(result) - logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") - return scarab.to_param(result_str) - - def do_get_request(self, a_request_message): - logger.info("in get_request") - a_specifier = a_request_message.specifier.to_string() - if (a_specifier): - logger.debug("has specifier") - try: - logger.debug(f"specifier is: {a_specifier}") - an_attribute = getattr(self, a_specifier) - logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") - return a_request_message.reply(payload=self.result_to_scarab_payload(an_attribute)) - except AttributeError as this_error: - raise ThrowReply('service_error_invalid_specifier', - f"endpoint {self.name} has no attribute {a_specifier}, unable to get") - else: - logger.debug('no specifier') - the_value = self.on_get() - return a_request_message.reply(payload=self.result_to_scarab_payload(the_value)) - - def do_set_request(self, a_request_message): - - a_specifier = a_request_message.specifier.to_string() - if not "values" in a_request_message.payload: - raise ThrowReply('service_error_bad_payload', - 'setting called without values, but values are required for set') - new_value = a_request_message.payload["values"][0]() - new_value = getattr(new_value, "as_" + new_value.type())() - logger.debug(f'Attempting to set new_value to [{new_value}]') - - if (a_specifier): - if not hasattr(self, a_specifier): - raise ThrowReply('service_error_invalid_specifier', - "endpoint {} has no attribute {}, unable to set".format(self.name, a_specifier)) - setattr(self, a_specifier, new_value) - return a_request_message.reply() - else: - result = self.on_set(new_value) - return a_request_message.reply(payload=self.result_to_scarab_payload(result)) - - def do_cmd_request(self, a_request_message): - # Note: any command executed in this way must return a python data structure which is - # able to be converted to a Param object (to be returned in the reply message) - method_name = a_request_message.specifier.to_string() - try: - method_ref = getattr(self, method_name) - except AttributeError as e: - raise ThrowReply('service_error_invalid_method', - "error getting command's corresponding method: {}".format(str(e))) - the_kwargs = a_request_message.payload.to_python() - the_args = the_kwargs.pop('values', []) - print(f'args: {the_args} -- kwargs: {the_kwargs}') - try: - result = method_ref(*the_args, **the_kwargs) - except TypeError as e: - raise ThrowReply('service_error_invalid_value', - f'A TypeError occurred while calling the requested method for endpoint {self.name}: {method_name}. Values provided may be invalid.\nOriginal error: {str(e)}') - return a_request_message.reply(payload=self.result_to_scarab_payload(result)) - + def do_cmd_request(self, a_request_message): + return self._do_cmd_request(a_request_message) + def on_get(self): - ''' - placeholder method for getting the value of an endpoint. - Implementations may override to enable OP_GET operations. - The implementation must return a value which is able to be passed to the ParamValue constructor. - ''' - raise ThrowReply('service_error_invalid_method', "{} does not implement on_get".format(self.__class__)) - - def on_set(self, _value): - ''' - placeholder method for setting the value of an endpoint. - Implementations may override to enable OP_SET operations. - Any returned object must already be a scarab::Param object - ''' - raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) + return self._on_get() + + def on_set(self, value): + return self._on_set(value) From 271c21ae506886abe4e4ab8beb54ac5396d6b5c9 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 10 Apr 2025 11:09:37 -0700 Subject: [PATCH 28/81] Update dripline-cpp version to v2.10.3 --- .github/workflows/publish.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index f8a93e68..0b96a341 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,7 +13,7 @@ env: BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.2' + BASE_IMAGE_TAG: 'v2.10.3' # BASE_IMAGE_TAG: 'develop' # DEV_SUFFIX: '' diff --git a/Dockerfile b/Dockerfile index f09c51f2..b192ba4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG img_user=ghcr.io/driplineorg ARG img_repo=dripline-cpp #ARG img_tag=develop -ARG img_tag=v2.10.2 +ARG img_tag=v2.10.3 FROM ${img_user}/${img_repo}:${img_tag} AS deps From bb17798a761cdf40e2aa9cf69072284d568d6148 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Thu, 10 Apr 2025 11:50:39 -0700 Subject: [PATCH 29/81] updated versions. --- CMakeLists.txt | 2 +- chart/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0989b999..6589732d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.5) # <3.5 is deprecated by CMake -project( DriplinePy VERSION 5.0.0 ) +project( DriplinePy VERSION 5.0.1 ) cmake_policy( SET CMP0074 NEW ) diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 80e33a20..fb1a0682 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 ## the appVersion is used as the container image tag for the main container in the pod from the deployment ## it can be overridden by a values.yaml file in image.tag -appVersion: "v5.0.0" +appVersion: "v5.0.1" description: Deploy a dripline-python microservice name: dripline-python version: 1.1.2 From d54beca7812eaccf75b23cc706332172a8bb6f06 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Thu, 10 Apr 2025 11:51:18 -0700 Subject: [PATCH 30/81] undo --- CMakeLists.txt | 2 +- chart/Chart.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6589732d..0989b999 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.5) # <3.5 is deprecated by CMake -project( DriplinePy VERSION 5.0.1 ) +project( DriplinePy VERSION 5.0.0 ) cmake_policy( SET CMP0074 NEW ) diff --git a/chart/Chart.yaml b/chart/Chart.yaml index fb1a0682..80e33a20 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 ## the appVersion is used as the container image tag for the main container in the pod from the deployment ## it can be overridden by a values.yaml file in image.tag -appVersion: "v5.0.1" +appVersion: "v5.0.0" description: Deploy a dripline-python microservice name: dripline-python version: 1.1.2 From 3cfb40c3e4f438566bfba8796df3517b1e06ca76 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 10 Apr 2025 16:48:03 -0700 Subject: [PATCH 31/81] Add UUID setting from string test --- tests/test_messages.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_messages.py b/tests/test_messages.py index 1780532f..ea0ae337 100644 --- a/tests/test_messages.py +++ b/tests/test_messages.py @@ -50,6 +50,11 @@ def test_lockout_key_roundtrip(): lk_rt = a_request.lockout_key # get the id back assert(lk_rt == random_uuid) # test being able to set and then get the id + random_uuid_2 = uuid.uuid4() + a_request.lockout_key = str(random_uuid_2) # set the id from a string + lk_rt_2 = a_request.lockout_key # get the id back + assert(lk_rt_2 == random_uuid_2) # test being able to set and then get the id + def test_request_reply_default(): a_request = _dripline.core.MsgRequest.create(reply_to = "a_receiver") a_reply = a_request.reply() From 2ca25d2b81b66172c4215ce84c335c8c22029237 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Thu, 10 Apr 2025 16:48:45 -0700 Subject: [PATCH 32/81] _check_lockout_key handles uuid objs and strs --- dripline/core/request_sender.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dripline/core/request_sender.py b/dripline/core/request_sender.py index 53ae452d..21740253 100644 --- a/dripline/core/request_sender.py +++ b/dripline/core/request_sender.py @@ -33,10 +33,12 @@ def __init__(self, sender: Core, timeout_s: int=10, confirm_retcodes: bool=True) self._receiver = Receiver() - def _check_lockout_key(self, lockout_key:str=None): + def _check_lockout_key(self, lockout_key:str | uuid.UUID =None): nilkey = uuid.UUID('00000000-0000-0000-0000-000000000000') if lockout_key is None: return nilkey + if type(lockout_key) == uuid.UUID: + return lockout_key try: return uuid.UUID(lockout_key) except ValueError: From 5cdfe03ab72be8d2b57e4b60e6cd5ca8b7533e18 Mon Sep 17 00:00:00 2001 From: Paul Kolbeck Date: Thu, 10 Apr 2025 18:19:26 -0700 Subject: [PATCH 33/81] Implemented changes from pull request comments. --- dripline/core/endpoint.py | 7 +------ dripline/core/request_receiver.py | 6 +++--- dripline/core/request_sender.py | 30 ++++++++++-------------------- dripline/core/service.py | 7 +------ 4 files changed, 15 insertions(+), 35 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index c7300f9a..0b92d101 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -33,9 +33,4 @@ def do_set_request(self, a_request_message): def do_cmd_request(self, a_request_message): return self._do_cmd_request(a_request_message) - - def on_get(self): - return self._on_get() - - def on_set(self, value): - return self._on_set(value) \ No newline at end of file + \ No newline at end of file diff --git a/dripline/core/request_receiver.py b/dripline/core/request_receiver.py index 06eb6d2e..7f087cd3 100644 --- a/dripline/core/request_receiver.py +++ b/dripline/core/request_receiver.py @@ -47,7 +47,7 @@ def _do_get_request(self, a_request_message): f"endpoint {self.name} has no attribute {a_specifier}, unable to get") else: logger.debug('no specifier') - the_value = self._on_get() + the_value = self.on_get() return a_request_message.reply(payload=self.result_to_scarab_payload(the_value)) def _do_set_request(self, a_request_message): @@ -89,7 +89,7 @@ def _do_cmd_request(self, a_request_message): f'A TypeError occurred while calling the requested method for endpoint {self.name}: {method_name}. Values provided may be invalid.\nOriginal error: {str(e)}') return a_request_message.reply(payload=self.result_to_scarab_payload(result)) - def _on_get(self): + def on_get(self): ''' placeholder method for getting the value of an endpoint. Implementations may override to enable OP_GET operations. @@ -97,7 +97,7 @@ def _on_get(self): ''' raise ThrowReply('service_error_invalid_method', "{} does not implement on_get".format(self.__class__)) - def _on_set(self, _value): + def on_set(self, _value): ''' placeholder method for setting the value of an endpoint. Implementations may override to enable OP_SET operations. diff --git a/dripline/core/request_sender.py b/dripline/core/request_sender.py index 21740253..c6ad47f2 100644 --- a/dripline/core/request_sender.py +++ b/dripline/core/request_sender.py @@ -3,7 +3,6 @@ import scarab from _dripline.core import op_t, Core, Receiver, MsgRequest, MsgReply, DriplineError - import uuid import logging logger = logging.getLogger(__name__) @@ -32,27 +31,18 @@ def __init__(self, sender: Core, timeout_s: int=10, confirm_retcodes: bool=True) self.timeout_s = timeout_s self._receiver = Receiver() - - def _check_lockout_key(self, lockout_key:str | uuid.UUID =None): - nilkey = uuid.UUID('00000000-0000-0000-0000-000000000000') - if lockout_key is None: - return nilkey - if type(lockout_key) == uuid.UUID: - return lockout_key - try: - return uuid.UUID(lockout_key) - except ValueError: - logger.warning("Lockout Key '{}' is not properly uuid formatted. Defaulting to nilkey. ".format(lockout_key)) - return nilkey - - - def _send_request(self, msgop, target, specifier=None, payload=None, timeout=None, lockout_key=None): + def _send_request(self, msgop, target, specifier=None, payload=None, timeout=None, lockout_key:str|uuid.UUID=None): ''' internal helper method to standardize sending request messages ''' a_specifier = specifier if specifier is not None else "" a_request = MsgRequest.create(payload=scarab.to_param(payload), msg_op=msgop, routing_key=target, specifier=a_specifier) - a_request.lockout_key = self._check_lockout_key(lockout_key) + if lockout_key is not None: + try: + a_request.lockout_key = lockout_key + except RuntimeError as err: + err.add_note(f"Lockout key [{lockout_key}] is not uuid format compliant.") + raise receive_reply = self.sender.send(a_request) if not receive_reply.successful_send: raise DriplineError('unable to send request') @@ -68,7 +58,7 @@ def _receive_reply(self, reply_pkg, timeout_s): sig_handler.remove_cancelable(self._receiver) return result.payload.to_python() - def get(self, endpoint: str, specifier: str=None, lockout_key=None, timeout_s: int=0) -> MsgReply: + def get(self, endpoint: str, specifier: str=None, lockout_key: str | uuid.UUID=None, timeout_s: int=0) -> MsgReply: ''' Send a get request to an endpoint and return the reply message. @@ -86,7 +76,7 @@ def get(self, endpoint: str, specifier: str=None, lockout_key=None, timeout_s: i result = self._receive_reply( reply_pkg, timeout_s ) return result - def set(self, endpoint: str, value: str | int | float | bool, specifier: str=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: + def set(self, endpoint: str, value: str | int | float | bool, specifier: str=None, lockout_key: str | uuid.UUID=None, timeout_s: int | float=0) -> MsgReply: ''' Send a set request to an endpoint and return the reply message. @@ -107,7 +97,7 @@ def set(self, endpoint: str, value: str | int | float | bool, specifier: str=Non result = self._receive_reply( reply_pkg, timeout_s ) return result - def cmd(self, endpoint: str, specifier: str, ordered_args=None, keyed_args=None, lockout_key=None, timeout_s: int | float=0) -> MsgReply: + def cmd(self, endpoint: str, specifier: str, ordered_args=None, keyed_args=None, lockout_key: str | uuid.UUID=None, timeout_s: int | float=0) -> MsgReply: ''' Send a cmd request to an endpoint and return the reply message. diff --git a/dripline/core/service.py b/dripline/core/service.py index 5eb23e30..fa65ac94 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -158,9 +158,4 @@ def do_set_request(self, a_request_message): def do_cmd_request(self, a_request_message): return self._do_cmd_request(a_request_message) - - def on_get(self): - return self._on_get() - - def on_set(self, value): - return self._on_set(value) + \ No newline at end of file From c28bde10dd787bacc1c1442c89e30b2a58e09b2d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Apr 2025 14:22:59 -0700 Subject: [PATCH 34/81] Add bindings to service::send() and comments on how send bindings are setup. --- dripline/core/request_sender.py | 6 ++--- .../dripline_core/_service_pybind.hh | 22 +++++++++++++++++++ .../dripline_core/_service_trampoline.hh | 18 +++++++-------- module_bindings/dripline_core/core_pybind.hh | 5 +++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/dripline/core/request_sender.py b/dripline/core/request_sender.py index c6ad47f2..b4023743 100644 --- a/dripline/core/request_sender.py +++ b/dripline/core/request_sender.py @@ -13,14 +13,14 @@ class RequestSender(): A mixin class that provides convenient methods for dripline interactions in a dripline objects. Intended for use as a dripline client in scripts or interactive sessions. ''' - def __init__(self, sender: Core, timeout_s: int=10, confirm_retcodes: bool=True): + def __init__(self, sender, timeout_s: int=10, confirm_retcodes: bool=True): ''' Configures an interface with the necessary parameters. Parameters ---------- - sender : Core - Provide a Core object which will use this mixin. This object will be the one to actually send the messages as it implements the Core functions + sender : Core, Service, or other class that implements send(MsgRequest) + Provide an object that will actually send the messages as it implements the send() function timeout_s: int, optional Time to wait for a reply, in seconds -- default is 10 s confirm_retcodes: bool, optional diff --git a/module_bindings/dripline_core/_service_pybind.hh b/module_bindings/dripline_core/_service_pybind.hh index 244876a1..ae83ff3c 100644 --- a/module_bindings/dripline_core/_service_pybind.hh +++ b/module_bindings/dripline_core/_service_pybind.hh @@ -41,6 +41,28 @@ namespace dripline_pybind .def_property( "auth", (scarab::authentication& (dripline::service::*)()) &dripline::service::auth, [](_service& a_service, const scarab::authentication& a_auth){a_service.auth() = a_auth;}, pybind11::return_value_policy::reference_internal ) + + // Notes on send() bindings + // The Service.send() functions are useful because they set the sender service name in the message before sending. + // The bound functions use lambdas because the dripline::service functions include amqp_ptr_t arguments which aren't known to pybind11. + // Therefore when called from Python, the send process will use the default parameter, a new AMQP connection. + // The bindings to these functions are not included in the trampoline class because we're not directly overriding the C++ send() functions. + .def( "send", + [](dripline::service& a_service, dripline::request_ptr_t a_request){return a_service.send(a_request);}, + DL_BIND_CALL_GUARD_STREAMS_AND_GIL, + "send a request message" + ) + .def( "send", + [](dripline::service& a_service, dripline::reply_ptr_t a_reply){return a_service.send(a_reply);}, + DL_BIND_CALL_GUARD_STREAMS_AND_GIL, + "send a reply message" + ) + .def( "send", + [](dripline::service& a_service, dripline::alert_ptr_t an_alert){return a_service.send(an_alert);}, + DL_BIND_CALL_GUARD_STREAMS_AND_GIL, + "send an alert message" + ) + .def_property( "enable_scheduling", &dripline::service::get_enable_scheduling, &dripline::service::set_enable_scheduling ) .def_property_readonly( "alerts_exchange", (std::string& (dripline::service::*)()) &dripline::service::alerts_exchange ) .def_property_readonly( "requests_exchange", (std::string& (dripline::service::*)()) &dripline::service::requests_exchange ) diff --git a/module_bindings/dripline_core/_service_trampoline.hh b/module_bindings/dripline_core/_service_trampoline.hh index 5c7047e5..4d538adc 100644 --- a/module_bindings/dripline_core/_service_trampoline.hh +++ b/module_bindings/dripline_core/_service_trampoline.hh @@ -21,7 +21,7 @@ namespace dripline_pybind }; - class _service_trampoline : public _service + class _service_trampoline : public dripline::service { public: using _service::_service; //inherit constructors @@ -30,41 +30,41 @@ namespace dripline_pybind // Local overrides bool bind_keys() override { - PYBIND11_OVERRIDE( bool, _service, bind_keys, ); + PYBIND11_OVERRIDE( bool, dripline::service, bind_keys, ); } void run() override { - PYBIND11_OVERRIDE( void, _service, run, ); + PYBIND11_OVERRIDE( void, dripline::service, run, ); } //// Overrides from Endpoint // Overrides for virtual on_[messgate-type]_message() dripline::reply_ptr_t on_request_message( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, on_request_message, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, on_request_message, a_request ); } void on_reply_message( const dripline::reply_ptr_t a_reply ) override { - PYBIND11_OVERRIDE( void, _service, on_reply_message, a_reply ); + PYBIND11_OVERRIDE( void, dripline::service, on_reply_message, a_reply ); } void on_alert_message( const dripline::alert_ptr_t a_alert ) override { - PYBIND11_OVERRIDE( void, _service, on_alert_message, a_alert ); + PYBIND11_OVERRIDE( void, dripline::service, on_alert_message, a_alert ); } // Overrides for virtual do_[request-type]_request dripline::reply_ptr_t do_get_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_get_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_get_request, a_request ); } dripline::reply_ptr_t do_set_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_set_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_set_request, a_request ); } dripline::reply_ptr_t do_cmd_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_cmd_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_cmd_request, a_request ); } }; diff --git a/module_bindings/dripline_core/core_pybind.hh b/module_bindings/dripline_core/core_pybind.hh index 65338d16..5b2b0674 100644 --- a/module_bindings/dripline_core/core_pybind.hh +++ b/module_bindings/dripline_core/core_pybind.hh @@ -42,6 +42,11 @@ namespace dripline_pybind pybind11::arg( "make_connection" ) = true ) + // Notes on send() bindings + // The bound functions use lambdas because the dripline::core functions include amqp_ptr_t arguments which aren't known to pybind11. + // Therefore when called from Python, the send process will use the default parameter, a new AMQP connection. + // The bindings to these functions are not included in a trampoline class because we're not directly overriding the C++ send() functions. + // Therefore calls to send() from a base-class pointer will not redirect appropriately to the derived-class versions of send(). .def( "send", [](dripline::core& a_core, dripline::request_ptr_t a_request){return a_core.send(a_request);}, DL_BIND_CALL_GUARD_STREAMS_AND_GIL, From 705e32c82c5d01f14e2e02fdcadb418c1454dd83 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Apr 2025 14:56:08 -0700 Subject: [PATCH 35/81] Undoing the breaking changes I made in the previous commit to the _service trampoline class. --- .../dripline_core/_service_trampoline.hh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/module_bindings/dripline_core/_service_trampoline.hh b/module_bindings/dripline_core/_service_trampoline.hh index 4d538adc..f0400d07 100644 --- a/module_bindings/dripline_core/_service_trampoline.hh +++ b/module_bindings/dripline_core/_service_trampoline.hh @@ -21,50 +21,50 @@ namespace dripline_pybind }; - class _service_trampoline : public dripline::service + class _service_trampoline : public _service { public: using _service::_service; //inherit constructors - _service_trampoline(_service &&base) : _service(std::move(base)) {} + //_service_trampoline(_service &&base) : _service(std::move(base)) {} // Local overrides bool bind_keys() override { - PYBIND11_OVERRIDE( bool, dripline::service, bind_keys, ); + PYBIND11_OVERRIDE( bool, _service, bind_keys, ); } void run() override { - PYBIND11_OVERRIDE( void, dripline::service, run, ); + PYBIND11_OVERRIDE( void, _service, run, ); } //// Overrides from Endpoint // Overrides for virtual on_[messgate-type]_message() dripline::reply_ptr_t on_request_message( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, on_request_message, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, on_request_message, a_request ); } void on_reply_message( const dripline::reply_ptr_t a_reply ) override { - PYBIND11_OVERRIDE( void, dripline::service, on_reply_message, a_reply ); + PYBIND11_OVERRIDE( void, _service, on_reply_message, a_reply ); } void on_alert_message( const dripline::alert_ptr_t a_alert ) override { - PYBIND11_OVERRIDE( void, dripline::service, on_alert_message, a_alert ); + PYBIND11_OVERRIDE( void, _service, on_alert_message, a_alert ); } // Overrides for virtual do_[request-type]_request dripline::reply_ptr_t do_get_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_get_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_get_request, a_request ); } dripline::reply_ptr_t do_set_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_set_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_set_request, a_request ); } dripline::reply_ptr_t do_cmd_request( const dripline::request_ptr_t a_request ) override { - PYBIND11_OVERRIDE( dripline::reply_ptr_t, dripline::service, do_cmd_request, a_request ); + PYBIND11_OVERRIDE( dripline::reply_ptr_t, _service, do_cmd_request, a_request ); } }; From 1310ae45305109e5b236823b428d98839bbf941e Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Apr 2025 17:17:53 -0700 Subject: [PATCH 36/81] Simplified on_get/set/cmd_request() function calls and added some documentation --- dripline/core/endpoint.py | 11 ------- dripline/core/interface.py | 2 ++ dripline/core/request_receiver.py | 53 ++++++++++++++++++++----------- 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index 0b92d101..99e92eb1 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -1,8 +1,6 @@ __all__ = [] -import scarab from _dripline.core import _Endpoint -from .throw_reply import ThrowReply from .request_receiver import RequestReceiver import logging @@ -24,13 +22,4 @@ def __init__(self, name): name (str) : the name of the endpoint, specifies the binding key for request messages to which this object should reply ''' _Endpoint.__init__(self, name) - - def do_get_request(self, a_request_message): - return self._do_get_request(a_request_message) - - def do_set_request(self, a_request_message): - return self._do_set_request(a_request_message) - - def do_cmd_request(self, a_request_message): - return self._do_cmd_request(a_request_message) \ No newline at end of file diff --git a/dripline/core/interface.py b/dripline/core/interface.py index bd78a146..344c6a74 100644 --- a/dripline/core/interface.py +++ b/dripline/core/interface.py @@ -13,6 +13,8 @@ class Interface(Core, RequestSender): ''' A class that provides user-friendly methods for dripline interactions in a Python interpreter. Intended for use as a dripline client in scripts or interactive sessions. + + See :py:class:dripline.core.RequestSender for the message-sending interface. ''' def __init__(self, username: str | dict=None, password: str | dict=None, dripline_mesh: dict=None, timeout_s: int=10, confirm_retcodes: bool=True): ''' diff --git a/dripline/core/request_receiver.py b/dripline/core/request_receiver.py index 7f087cd3..a25a331d 100644 --- a/dripline/core/request_receiver.py +++ b/dripline/core/request_receiver.py @@ -13,23 +13,23 @@ class RequestReceiver(): ''' A mixin class that provides methods for dripline responses in a dripline objects. Intended for use as a centralized object containing response methods in dripline. - ''' - def result_to_scarab_payload(self, result: str): - """ - Intercept result values and throw error if scarab is unable to convert to param - TODO: Handles global Exception case, could be more specific - Args: - result (str): request message passed - """ - try: - return scarab.to_param(result) - except Exception as e: - result_str = str(result) - logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") - return scarab.to_param(result_str) + This class is used by Endpoint and Service to include the dripline-standard behavior + for handling and responding to requests. - def _do_get_request(self, a_request_message): + Any class using RequestReceiver will get the following features: + + * When handling a get/set request that does *not* include a specifier, + the function ``on_[get/set]()`` will be called. + * The default implementations of ``on_[get/set]()`` raise a `ThrowReply``; + If a response to get/set requests with no specifier is desired, + a derived class should override these functions. + * When handling a cmd request, the specifier is required. + The specifier provides the method name to be called. + If no method exists with the name given by the specifier, an error is returned. + ''' + + def do_get_request(self, a_request_message): logger.info("in get_request") a_specifier = a_request_message.specifier.to_string() if (a_specifier): @@ -42,7 +42,7 @@ def _do_get_request(self, a_request_message): the_node["values"].push_back(self.result_to_scarab_payload(an_attribute)) logger.debug(f"attribute '{a_specifier}' value is [{an_attribute}]") return a_request_message.reply(payload=the_node) - except AttributeError as this_error: + except AttributeError: raise ThrowReply('service_error_invalid_specifier', f"endpoint {self.name} has no attribute {a_specifier}, unable to get") else: @@ -50,7 +50,7 @@ def _do_get_request(self, a_request_message): the_value = self.on_get() return a_request_message.reply(payload=self.result_to_scarab_payload(the_value)) - def _do_set_request(self, a_request_message): + def do_set_request(self, a_request_message): a_specifier = a_request_message.specifier.to_string() if not "values" in a_request_message.payload: @@ -70,7 +70,7 @@ def _do_set_request(self, a_request_message): result = self.on_set(new_value) return a_request_message.reply(payload=self.result_to_scarab_payload(result)) - def _do_cmd_request(self, a_request_message): + def do_cmd_request(self, a_request_message): # Note: any command executed in this way must return a python data structure which is # able to be converted to a Param object (to be returned in the reply message) method_name = a_request_message.specifier.to_string() @@ -103,4 +103,19 @@ def on_set(self, _value): Implementations may override to enable OP_SET operations. Any returned object must already be a scarab::Param object ''' - raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) \ No newline at end of file + raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) + + def result_to_scarab_payload(self, result: str): + """ + Intercept result values and throw error if scarab is unable to convert to param + TODO: Handles global Exception case, could be more specific + Args: + result (str): request message passed + """ + try: + return scarab.to_param(result) + except Exception as e: + result_str = str(result) + logger.warning(f"Bad payload: [{result_str}] is not of type bool, int, float, str, or dict. Converting to str.") + return scarab.to_param(result_str) + From 1488b1b34fb229a2fd4174b97fab0f2965b464a5 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 16 Apr 2025 14:48:35 -0700 Subject: [PATCH 37/81] Fixing on_[op]_request() functions to correctly get trampolined calls from the C++ base class --- dripline/core/endpoint.py | 53 ++++++++++++++++++++++++++++++- dripline/core/request_receiver.py | 14 ++++---- 2 files changed, 60 insertions(+), 7 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index 99e92eb1..d3ea6892 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -22,4 +22,55 @@ def __init__(self, name): name (str) : the name of the endpoint, specifies the binding key for request messages to which this object should reply ''' _Endpoint.__init__(self, name) - \ No newline at end of file + + def do_get_request(self, a_request_message): + ''' + Default function for handling an OP_GET request message addressed to this endpoint. + + .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + dripline-python behavior for an endpoint receiving a get request, including using the specifier to access attributes, + and calling on_get() when there is no specifier. + As an extension author you might typically override RequestReciever.on_get(), but leave this function alone. + + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_get_request() in RequestReceiver. + + Args: + a_request_message (MsgRequest): the message receveived by this endpoint + ''' + + return RequestReceiver.do_get_request(self, a_request_message) + + def do_set_request(self, a_request_message): + ''' + Default function for handling an OP_SET request message addressed to this endpoint. + + .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + dripline-python behavior for an endpoint receiving a set request, including using the specifier to access attributes, + and calling on_set() when there is no specifier. + As an extension author you might typically override RequestReciever.on_set(), but leave this function alone. + + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_set_request() in RequestReceiver. + + Args: + a_request_message (MsgRequest): the message receveived by this endpoint + ''' + + return RequestReceiver.do_set_request(self, a_request_message) + + def do_cmd_request(self, a_request_message): + ''' + Default function for handling an OP_CMD request message addressed to this endpoint. + + .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + dripline-python behavior for an endpoint receiving a cmd request, namesly using the specifier to call endpoint methods. + + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_cmd_request() in RequestReceiver. + + Args: + a_request_message (MsgRequest): the message receveived by this endpoint + ''' + + return RequestReceiver.do_cmd_request(self, a_request_message) diff --git a/dripline/core/request_receiver.py b/dripline/core/request_receiver.py index a25a331d..9daf787c 100644 --- a/dripline/core/request_receiver.py +++ b/dripline/core/request_receiver.py @@ -74,11 +74,13 @@ def do_cmd_request(self, a_request_message): # Note: any command executed in this way must return a python data structure which is # able to be converted to a Param object (to be returned in the reply message) method_name = a_request_message.specifier.to_string() + if method_name == "": + raise ThrowReply('service_error_invalid_method', 'No specifier was provided for an OP_CMD request') try: method_ref = getattr(self, method_name) except AttributeError as e: raise ThrowReply('service_error_invalid_method', - "error getting command's corresponding method: {}".format(str(e))) + f'error getting command\'s corresponding method: {str(e)}') the_kwargs = a_request_message.payload.to_python() the_args = the_kwargs.pop('values', []) print(f'args: {the_args} -- kwargs: {the_kwargs}') @@ -91,16 +93,16 @@ def do_cmd_request(self, a_request_message): def on_get(self): ''' - placeholder method for getting the value of an endpoint. - Implementations may override to enable OP_GET operations. - The implementation must return a value which is able to be passed to the ParamValue constructor. + Placeholder method for getting the value of an endpoint. + Implementations should override to enable OP_GET operations. + The implementation must return a value which is convertible to a scarab param object. ''' raise ThrowReply('service_error_invalid_method', "{} does not implement on_get".format(self.__class__)) def on_set(self, _value): ''' - placeholder method for setting the value of an endpoint. - Implementations may override to enable OP_SET operations. + Placeholder method for setting the value of an endpoint. + Implementations should override to enable OP_SET operations. Any returned object must already be a scarab::Param object ''' raise ThrowReply('service_error_invalid_method', "{} does not implement on_set".format(self.__class__)) From e9326be8f4af93a91215063c710b89f7f853ef92 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 16 Apr 2025 15:42:59 -0700 Subject: [PATCH 38/81] Improving tests in test_endpoint.py that check for raising of exceptions --- tests/test_endpoint.py | 47 ++++++++---------------------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index 20291206..c2baa2cd 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -1,5 +1,7 @@ import scarab, dripline.core +import pytest + def test_endpoint_creation(): a_name = "an_endpoint" an_endpoint = dripline.core.Endpoint(a_name) @@ -28,43 +30,27 @@ def test_on_request_message(): def test_on_reply_message(): an_endpoint = dripline.core.Endpoint("hello") a_reply = dripline.core.MsgReply.create() - print(a_reply) - flag = False - try: + with pytest.raises(dripline.core.DriplineError) as excinfo: an_endpoint.on_reply_message(a_reply) - except Exception: # seems like this is not throwing an actual DriplineError, just a generic Exception. dripline.core.DriplineError - flag = True - assert(flag) def test_on_alert_message(): an_endpoint = dripline.core.Endpoint("hello") an_alert = dripline.core.MsgAlert.create() - flag = False - try: + with pytest.raises(dripline.core.DriplineError) as excinfo: an_endpoint.on_alert_message(an_alert) - except Exception: # seems like this is not throwing an actual DriplineError, just a generic Exception. dripline.core.DriplineError - flag =True - assert(flag) def test_do_get_request_no_specifier(): an_endpoint = dripline.core.Endpoint("hello") a_get_request = dripline.core.MsgRequest.create() flag = False - try: + with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_get_request(a_get_request) - except dripline.core.ThrowReply: - flag = True - assert(flag) def test_do_get_request_invalid_specifier(): an_endpoint = dripline.core.Endpoint("an_endpoint") a_get_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "namee", "a_receiver") - correct_error = False - try: + with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_get_request(a_get_request) - except dripline.core.ThrowReply as e: - correct_error = True - assert(correct_error) def test_do_get_request_valid_specifier(): a_name = "an_endpoint" @@ -83,12 +69,8 @@ def test_do_set_request_no_specifier(): the_node.add("values", scarab.ParamArray()) the_node["values"].push_back(scarab.ParamValue("a_better_endpoint")) a_set_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.set, "hey") - correct_error = False - try: + with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_set_request(a_set_request) - except dripline.core.ThrowReply as e: - correct_error = True - assert(correct_error) def test_do_set_request_invalid_specifier(): an_endpoint = dripline.core.Endpoint("an_endpoint") @@ -96,13 +78,8 @@ def test_do_set_request_invalid_specifier(): the_node.add("values", scarab.ParamArray()) the_node["values"].push_back(scarab.ParamValue("a_better_endpoint")) a_set_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.set, "hey", "namee", "a_receiver") - correct_error = False - try: + with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_set_request(a_set_request) - except dripline.core.ThrowReply as e: - correct_error = True - print('content of error:\n{}'.format(dir(e))) - assert(correct_error) def test_do_set_request_valid_specifier(): value1 = "an_endpoint" @@ -125,14 +102,8 @@ class EndpointWithMember(dripline.core.Endpoint): def test_do_cmd_request_invalid_specifier(): an_endpoint = dripline.core.Endpoint("an_endpoint") a_cmd_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.cmd, "hey", "on_gett", "a_receiver") - correct_error = False - try: + with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_cmd_request(a_cmd_request) - except dripline.core.ThrowReply as e: - correct_error = True - except Exception as e: - print("got a [{}]".format(type(e))) - assert(correct_error) def test_do_cmd_request_valid_specifier(): class AnotherEndpoint(dripline.core.Endpoint): From 8a29b91964d022cead2a0beab335d584841f059b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 17 Apr 2025 14:22:48 -0700 Subject: [PATCH 39/81] Change class name to avoid confusion with dl-cpp classes: RequestReceiver --> RequestHandler --- dripline/core/endpoint.py | 22 +++++++++---------- ...request_receiver.py => request_handler.py} | 6 ++--- dripline/core/service.py | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) rename dripline/core/{request_receiver.py => request_handler.py} (97%) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index d3ea6892..36843090 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -1,7 +1,7 @@ __all__ = [] from _dripline.core import _Endpoint -from .request_receiver import RequestReceiver +from .request_receiver import RequestHandler import logging @@ -10,7 +10,7 @@ __all__.append('Endpoint') -class Endpoint(_Endpoint, RequestReceiver): +class Endpoint(_Endpoint, RequestHandler): ''' Base class for all objects which can be sent dripline requests. Every object described in a runtime configuration passed to `dl-serve` should derive from this class (either directly or indirectly). @@ -27,50 +27,50 @@ def do_get_request(self, a_request_message): ''' Default function for handling an OP_GET request message addressed to this endpoint. - .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic dripline-python behavior for an endpoint receiving a get request, including using the specifier to access attributes, and calling on_get() when there is no specifier. As an extension author you might typically override RequestReciever.on_get(), but leave this function alone. .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from - the C++ base class. It intentionally just calls the version of do_get_request() in RequestReceiver. + the C++ base class. It intentionally just calls the version of do_get_request() in RequestHandler. Args: a_request_message (MsgRequest): the message receveived by this endpoint ''' - return RequestReceiver.do_get_request(self, a_request_message) + return RequestHandler.do_get_request(self, a_request_message) def do_set_request(self, a_request_message): ''' Default function for handling an OP_SET request message addressed to this endpoint. - .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic dripline-python behavior for an endpoint receiving a set request, including using the specifier to access attributes, and calling on_set() when there is no specifier. As an extension author you might typically override RequestReciever.on_set(), but leave this function alone. .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from - the C++ base class. It intentionally just calls the version of do_set_request() in RequestReceiver. + the C++ base class. It intentionally just calls the version of do_set_request() in RequestHandler. Args: a_request_message (MsgRequest): the message receveived by this endpoint ''' - return RequestReceiver.do_set_request(self, a_request_message) + return RequestHandler.do_set_request(self, a_request_message) def do_cmd_request(self, a_request_message): ''' Default function for handling an OP_CMD request message addressed to this endpoint. - .. note: For dripline extension developers -- This function, as defined in RequestReceiver, implements the characteristic + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic dripline-python behavior for an endpoint receiving a cmd request, namesly using the specifier to call endpoint methods. .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from - the C++ base class. It intentionally just calls the version of do_cmd_request() in RequestReceiver. + the C++ base class. It intentionally just calls the version of do_cmd_request() in RequestHandler. Args: a_request_message (MsgRequest): the message receveived by this endpoint ''' - return RequestReceiver.do_cmd_request(self, a_request_message) + return RequestHandler.do_cmd_request(self, a_request_message) diff --git a/dripline/core/request_receiver.py b/dripline/core/request_handler.py similarity index 97% rename from dripline/core/request_receiver.py rename to dripline/core/request_handler.py index 9daf787c..705145b5 100644 --- a/dripline/core/request_receiver.py +++ b/dripline/core/request_handler.py @@ -8,8 +8,8 @@ import logging logger = logging.getLogger(__name__) -__all__.append("RequestReceiver") -class RequestReceiver(): +__all__.append("RequestHandler") +class RequestHandler(): ''' A mixin class that provides methods for dripline responses in a dripline objects. Intended for use as a centralized object containing response methods in dripline. @@ -17,7 +17,7 @@ class RequestReceiver(): This class is used by Endpoint and Service to include the dripline-standard behavior for handling and responding to requests. - Any class using RequestReceiver will get the following features: + Any class using RequestHandler will get the following features: * When handling a get/set request that does *not* include a specifier, the function ``on_[get/set]()`` will be called. diff --git a/dripline/core/service.py b/dripline/core/service.py index fa65ac94..e01b039f 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -5,7 +5,7 @@ from .throw_reply import ThrowReply from .object_creator import ObjectCreator from .request_sender import RequestSender -from .request_receiver import RequestReceiver +from .request_receiver import RequestHandler import datetime import logging @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) __all__.append('Service') -class Service(_Service, ObjectCreator, RequestSender, RequestReceiver): +class Service(_Service, ObjectCreator, RequestSender, RequestHandler): ''' The primary unit of software that connects to a broker and typically provides an interface with an instrument or other software. From d6f1c7e54504b64ebd55aeab05eaee0b5d985739 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 17 Apr 2025 15:57:59 -0700 Subject: [PATCH 40/81] Fixed a couple more names --- dripline/core/endpoint.py | 2 +- dripline/core/service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dripline/core/endpoint.py b/dripline/core/endpoint.py index 36843090..e1250a80 100644 --- a/dripline/core/endpoint.py +++ b/dripline/core/endpoint.py @@ -1,7 +1,7 @@ __all__ = [] from _dripline.core import _Endpoint -from .request_receiver import RequestHandler +from .request_handler import RequestHandler import logging diff --git a/dripline/core/service.py b/dripline/core/service.py index e01b039f..5fa23267 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -5,7 +5,7 @@ from .throw_reply import ThrowReply from .object_creator import ObjectCreator from .request_sender import RequestSender -from .request_receiver import RequestHandler +from .request_handler import RequestHandler import datetime import logging From ff8e495c665c1265a4c856f073d914ed3a303730 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 17 Apr 2025 15:58:13 -0700 Subject: [PATCH 41/81] Added classes to __init__.py --- dripline/core/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dripline/core/__init__.py b/dripline/core/__init__.py index a98e28d0..3166a117 100644 --- a/dripline/core/__init__.py +++ b/dripline/core/__init__.py @@ -9,6 +9,8 @@ from .entity import * from .interface import * from .object_creator import * +from .request_handler import * +from .request_sender import * from .return_codes import * from .service import * from .throw_reply import * From f3d492d003ee6a3ee76d7a94f233c9111cf41e55 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:31:06 -0700 Subject: [PATCH 42/81] Fixing deprecated utc timezone use --- dripline/core/entity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 6478cb87..44333606 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -164,7 +164,7 @@ def scheduled_log(self): # Various checks for log condition if self._last_log_time is None: logger.debug("log because no last log") - elif (datetime.datetime.utcnow() - self._last_log_time).total_seconds() > self._max_interval: + elif (datetime.datetime.now(datetime.timezone.utc) - self._last_log_time).total_seconds() > self._max_interval: logger.debug("log because too much time") elif this_value is False: logger.warning(f"cannot check value change for {self.name}") @@ -181,7 +181,7 @@ def scheduled_log(self): def log_a_value(self, the_value): logger.info(f"value to log for {self.name} is:\n{the_value}") - self._last_log_time = datetime.datetime.utcnow() + self._last_log_time = datetime.datetime.now(datetime.timezone.utc) the_alert = MsgAlert.create(payload=scarab.to_param(the_value), routing_key=f'{self.log_routing_key_prefix}.{self.name}') alert_sent = self.service.send(the_alert) From a07dd2b7441a22ecd9bd954f33d6e50459aef832 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:31:26 -0700 Subject: [PATCH 43/81] Removing unnecessary imports --- dripline/core/interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dripline/core/interface.py b/dripline/core/interface.py index 344c6a74..61ee69ac 100644 --- a/dripline/core/interface.py +++ b/dripline/core/interface.py @@ -2,7 +2,7 @@ import scarab -from _dripline.core import op_t, create_dripline_auth_spec, Core, DriplineConfig, Receiver, MsgRequest, MsgReply, DriplineError +from _dripline.core import create_dripline_auth_spec, Core, DriplineConfig, Receiver from .request_sender import RequestSender import logging From 98d1b6f06a696a300fc99da615b4595ff4784a12 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:31:56 -0700 Subject: [PATCH 44/81] Fixing Service send-mesage API --- dripline/core/service.py | 56 +++++++++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/dripline/core/service.py b/dripline/core/service.py index 5fa23267..ad4574bd 100644 --- a/dripline/core/service.py +++ b/dripline/core/service.py @@ -150,12 +150,54 @@ def add_endpoints_from_config(self): logger.debug("queue up start logging for '{}'".format(an_endpoint.name)) an_endpoint.start_logging() - def do_get_request(self, a_request_message): - return self._do_get_request(a_request_message) + def do_get_request(self, a_request_message): + ''' + Default function for handling an OP_GET request message addressed to this service. + + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic + dripline-python behavior for an service receiving a get request, including using the specifier to access attributes, + and calling on_get() when there is no specifier. + As an extension author you might typically override RequestReciever.on_get(), but leave this function alone. + + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_get_request() in RequestHandler. + + Args: + a_request_message (MsgRequest): the message receveived by this service + ''' + + return RequestHandler.do_get_request(self, a_request_message) + + def do_set_request(self, a_request_message): + ''' + Default function for handling an OP_SET request message addressed to this service. + + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic + dripline-python behavior for an service receiving a set request, including using the specifier to access attributes, + and calling on_set() when there is no specifier. + As an extension author you might typically override RequestReciever.on_set(), but leave this function alone. - def do_set_request(self, a_request_message): - return self._do_set_request(a_request_message) + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_set_request() in RequestHandler. + + Args: + a_request_message (MsgRequest): the message receveived by this service + ''' + + return RequestHandler.do_set_request(self, a_request_message) + + def do_cmd_request(self, a_request_message): + ''' + Default function for handling an OP_CMD request message addressed to this service. + + .. note: For dripline extension developers -- This function, as defined in RequestHandler, implements the characteristic + dripline-python behavior for an service receiving a cmd request, namesly using the specifier to call service methods. + + .. note: For core dripline developers -- This function has to be here to correctly receive trampolined calls from + the C++ base class. It intentionally just calls the version of do_cmd_request() in RequestHandler. + + Args: + a_request_message (MsgRequest): the message receveived by this service + ''' - def do_cmd_request(self, a_request_message): - return self._do_cmd_request(a_request_message) - \ No newline at end of file + return RequestHandler.do_cmd_request(self, a_request_message) From a0315a7900ad43ea814f796eda9c174960d25001 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:32:31 -0700 Subject: [PATCH 45/81] Fixing service binding --- module_bindings/dripline_core/_service_pybind.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module_bindings/dripline_core/_service_pybind.hh b/module_bindings/dripline_core/_service_pybind.hh index 41bbd5c0..5b3f787d 100644 --- a/module_bindings/dripline_core/_service_pybind.hh +++ b/module_bindings/dripline_core/_service_pybind.hh @@ -49,17 +49,17 @@ namespace dripline_pybind // Therefore when called from Python, the send process will use the default parameter, a new AMQP connection. // The bindings to these functions are not included in the trampoline class because we're not directly overriding the C++ send() functions. .def( "send", - [](dripline::service& a_service, dripline::request_ptr_t a_request){return a_service.send(a_request);}, + [](_service& a_service, dripline::request_ptr_t a_request){return a_service.send(a_request);}, DL_BIND_CALL_GUARD_STREAMS_AND_GIL, "send a request message" ) .def( "send", - [](dripline::service& a_service, dripline::reply_ptr_t a_reply){return a_service.send(a_reply);}, + [](_service& a_service, dripline::reply_ptr_t a_reply){return a_service.send(a_reply);}, DL_BIND_CALL_GUARD_STREAMS_AND_GIL, "send a reply message" ) .def( "send", - [](dripline::service& a_service, dripline::alert_ptr_t an_alert){return a_service.send(an_alert);}, + [](_service& a_service, dripline::alert_ptr_t an_alert){return a_service.send(an_alert);}, DL_BIND_CALL_GUARD_STREAMS_AND_GIL, "send an alert message" ) From bf2d05a2f4bce332cee3283f40d66faf3fafc128 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:33:16 -0700 Subject: [PATCH 46/81] Add send_error_message binding --- module_bindings/dripline_core/core_pybind.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/module_bindings/dripline_core/core_pybind.hh b/module_bindings/dripline_core/core_pybind.hh index 5b2b0674..9b355cc5 100644 --- a/module_bindings/dripline_core/core_pybind.hh +++ b/module_bindings/dripline_core/core_pybind.hh @@ -23,6 +23,7 @@ namespace dripline_pybind std::shared_ptr< dripline::sent_msg_pkg > >( mod, "SentMessagePackage", "Data structure for sent messages" ) .def_property_readonly( "successful_send", [](const dripline::sent_msg_pkg& an_obj){ return an_obj.f_successful_send; } ) + .def_property_readonly( "send_error_message", [](const dripline::sent_msg_pkg& an_obj){ return an_obj.f_send_error_message; } ) ; all_items.push_back( "Core" ); From 93610f42219bcda97f0a3282a3f293e5b3b1cb2d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:33:40 -0700 Subject: [PATCH 47/81] Removing unnecessary line --- tests/test_endpoint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_endpoint.py b/tests/test_endpoint.py index c2baa2cd..6efe6c9e 100644 --- a/tests/test_endpoint.py +++ b/tests/test_endpoint.py @@ -42,7 +42,6 @@ def test_on_alert_message(): def test_do_get_request_no_specifier(): an_endpoint = dripline.core.Endpoint("hello") a_get_request = dripline.core.MsgRequest.create() - flag = False with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = an_endpoint.do_get_request(a_get_request) From 6484826acb31e4d963606cf90ce7be7ab9e4e4c8 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 25 Apr 2025 17:33:50 -0700 Subject: [PATCH 48/81] Adding test_service.py --- tests/test_service.py | 131 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/test_service.py diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 00000000..13c3c29c --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,131 @@ +import scarab, dripline.core + +import pytest + +def test_service_creation(): + a_name = "a_service" + a_service = dripline.core.Service(a_name) + assert(a_service.name == a_name) + +def test_submit_request_message(): + a_name = "a_service" + a_service = dripline.core.Service(a_name) + a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") + a_reply = a_service.submit_request_message(a_request) + assert(isinstance(a_reply, dripline.core.MsgReply)) + assert(a_reply.return_code == 0) + assert(a_reply.correlation_id == a_request.correlation_id) + a_reply.payload.to_python()['values'] == [a_name] + +def test_on_request_message(): + a_name = "a_service" + a_service = dripline.core.Service(a_name) + a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") + a_reply = a_service.on_request_message(a_request) + assert(isinstance(a_reply, dripline.core.MsgReply)) + assert(a_reply.return_code == 0) # 0 + assert(a_reply.correlation_id == a_request.correlation_id) + a_reply.payload.to_python()['values'] == [a_name] + +def test_on_reply_message(): + a_service = dripline.core.Service("hello") + a_reply = dripline.core.MsgReply.create() + with pytest.raises(dripline.core.DriplineError) as excinfo: + a_service.on_reply_message(a_reply) + +def test_on_alert_message(): + a_service = dripline.core.Service("hello") + an_alert = dripline.core.MsgAlert.create() + with pytest.raises(dripline.core.DriplineError) as excinfo: + a_service.on_alert_message(an_alert) + +def test_do_get_request_no_specifier(): + a_service = dripline.core.Service("hello") + a_get_request = dripline.core.MsgRequest.create() + with pytest.raises(dripline.core.ThrowReply) as excinfo: + a_reply = a_service.do_get_request(a_get_request) + +def test_do_get_request_invalid_specifier(): + a_service = dripline.core.Service("a_service") + a_get_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "namee", "a_receiver") + with pytest.raises(dripline.core.ThrowReply) as excinfo: + a_reply = a_service.do_get_request(a_get_request) + +def test_do_get_request_valid_specifier(): + a_name = "a_service" + a_service = dripline.core.Service(a_name) + a_get_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") + a_reply = a_service.do_get_request(a_get_request) + assert(isinstance(a_reply, dripline.core.MsgReply)) + assert(a_reply.return_code == 0) + assert(a_reply.correlation_id == a_get_request.correlation_id) + a_reply.payload.to_python()['values'] == [a_name] + +def test_do_set_request_no_specifier(): + print("start test") + a_service = dripline.core.Service("hello") + the_node = scarab.ParamNode() + the_node.add("values", scarab.ParamArray()) + the_node["values"].push_back(scarab.ParamValue("a_better_service")) + a_set_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.set, "hey") + with pytest.raises(dripline.core.ThrowReply) as excinfo: + a_reply = a_service.do_set_request(a_set_request) + +def test_do_set_request_invalid_specifier(): + a_service = dripline.core.Service("a_service") + the_node = scarab.ParamNode() + the_node.add("values", scarab.ParamArray()) + the_node["values"].push_back(scarab.ParamValue("a_better_service")) + a_set_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.set, "hey", "namee", "a_receiver") + with pytest.raises(dripline.core.ThrowReply) as excinfo: + a_reply = a_service.do_set_request(a_set_request) + +def test_do_set_request_valid_specifier(): + value1 = "a_service" + value2 = "a_better_service" + ## the service base class doesn't have any settable members, create one: + class ServiceWithMember(dripline.core.Service): + a_value = value1 + a_service = ServiceWithMember("a_service") + the_node = scarab.ParamNode() + the_node.add("values", scarab.ParamArray()) + the_node["values"].push_back(scarab.ParamValue(value2)) + a_set_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.set, "hey", "a_value", "a_receiver") + a_reply = a_service.do_set_request(a_set_request) + assert(isinstance(a_reply, dripline.core.MsgReply)) + assert(a_reply.return_code == 0) + assert(a_reply.correlation_id == a_set_request.correlation_id) + print(a_service.name) + assert(a_service.a_value == value2) + +def test_do_cmd_request_invalid_specifier(): + a_service = dripline.core.Service("a_service") + a_cmd_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.cmd, "hey", "on_gett", "a_receiver") + with pytest.raises(dripline.core.ThrowReply) as excinfo: + a_reply = a_service.do_cmd_request(a_cmd_request) + +def test_do_cmd_request_valid_specifier(): + class AnotherService(dripline.core.Service): + def __init__(self, name): + dripline.core.Service.__init__(self, name) + def a_method(self, n1, n2): + return n1 + n2 + a_service = AnotherService("a_service") + the_node = scarab.ParamNode() + the_node.add("values", scarab.ParamArray()) + n1, n2 = 10, 13 + the_node["values"].push_back(scarab.ParamValue(n1)) + the_node["values"].push_back(scarab.ParamValue(n2)) + a_cmd_request = dripline.core.MsgRequest.create(the_node, dripline.core.op_t.cmd, "hey", "a_method", "a_receiver") + a_reply = a_service.do_cmd_request(a_cmd_request) + assert(isinstance(a_reply, dripline.core.MsgReply)) + assert(a_reply.return_code == 0) + assert(a_reply.correlation_id == a_cmd_request.correlation_id) + assert(a_reply.payload.to_python() == n1 + n2) + +def test_send_request(): + a_service = dripline.core.Service("a_service") + a_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.get, "hey", "on_get", "a_receiver") + a_sent_msg = a_service.send(a_request) + assert not a_sent_msg.successful_send + assert 'Reply code: 312 NO_ROUTE' in a_sent_msg.send_error_message From 56368a8b59397abcff44630f27ea2af4d006479a Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 29 Apr 2025 11:02:21 -0700 Subject: [PATCH 49/81] Specify no connection made in test_service --- tests/test_service.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_service.py b/tests/test_service.py index 13c3c29c..def3c606 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -4,12 +4,12 @@ def test_service_creation(): a_name = "a_service" - a_service = dripline.core.Service(a_name) + a_service = dripline.core.Service(a_name, make_connection=False) assert(a_service.name == a_name) def test_submit_request_message(): a_name = "a_service" - a_service = dripline.core.Service(a_name) + a_service = dripline.core.Service(a_name, make_connection=False) a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") a_reply = a_service.submit_request_message(a_request) assert(isinstance(a_reply, dripline.core.MsgReply)) @@ -19,7 +19,7 @@ def test_submit_request_message(): def test_on_request_message(): a_name = "a_service" - a_service = dripline.core.Service(a_name) + a_service = dripline.core.Service(a_name, make_connection=False) a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") a_reply = a_service.on_request_message(a_request) assert(isinstance(a_reply, dripline.core.MsgReply)) @@ -28,32 +28,32 @@ def test_on_request_message(): a_reply.payload.to_python()['values'] == [a_name] def test_on_reply_message(): - a_service = dripline.core.Service("hello") + a_service = dripline.core.Service("hello", make_connection=False) a_reply = dripline.core.MsgReply.create() with pytest.raises(dripline.core.DriplineError) as excinfo: a_service.on_reply_message(a_reply) def test_on_alert_message(): - a_service = dripline.core.Service("hello") + a_service = dripline.core.Service("hello", make_connection=False) an_alert = dripline.core.MsgAlert.create() with pytest.raises(dripline.core.DriplineError) as excinfo: a_service.on_alert_message(an_alert) def test_do_get_request_no_specifier(): - a_service = dripline.core.Service("hello") + a_service = dripline.core.Service("hello", make_connection=False) a_get_request = dripline.core.MsgRequest.create() with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = a_service.do_get_request(a_get_request) def test_do_get_request_invalid_specifier(): - a_service = dripline.core.Service("a_service") + a_service = dripline.core.Service("a_service", make_connection=False) a_get_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "namee", "a_receiver") with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = a_service.do_get_request(a_get_request) def test_do_get_request_valid_specifier(): a_name = "a_service" - a_service = dripline.core.Service(a_name) + a_service = dripline.core.Service(a_name, make_connection=False) a_get_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") a_reply = a_service.do_get_request(a_get_request) assert(isinstance(a_reply, dripline.core.MsgReply)) @@ -63,7 +63,7 @@ def test_do_get_request_valid_specifier(): def test_do_set_request_no_specifier(): print("start test") - a_service = dripline.core.Service("hello") + a_service = dripline.core.Service("hello", make_connection=False) the_node = scarab.ParamNode() the_node.add("values", scarab.ParamArray()) the_node["values"].push_back(scarab.ParamValue("a_better_service")) @@ -72,7 +72,7 @@ def test_do_set_request_no_specifier(): a_reply = a_service.do_set_request(a_set_request) def test_do_set_request_invalid_specifier(): - a_service = dripline.core.Service("a_service") + a_service = dripline.core.Service("a_service", make_connection=False) the_node = scarab.ParamNode() the_node.add("values", scarab.ParamArray()) the_node["values"].push_back(scarab.ParamValue("a_better_service")) @@ -86,7 +86,7 @@ def test_do_set_request_valid_specifier(): ## the service base class doesn't have any settable members, create one: class ServiceWithMember(dripline.core.Service): a_value = value1 - a_service = ServiceWithMember("a_service") + a_service = ServiceWithMember("a_service", make_connection=False) the_node = scarab.ParamNode() the_node.add("values", scarab.ParamArray()) the_node["values"].push_back(scarab.ParamValue(value2)) @@ -99,18 +99,18 @@ class ServiceWithMember(dripline.core.Service): assert(a_service.a_value == value2) def test_do_cmd_request_invalid_specifier(): - a_service = dripline.core.Service("a_service") + a_service = dripline.core.Service("a_service", make_connection=False) a_cmd_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.cmd, "hey", "on_gett", "a_receiver") with pytest.raises(dripline.core.ThrowReply) as excinfo: a_reply = a_service.do_cmd_request(a_cmd_request) def test_do_cmd_request_valid_specifier(): class AnotherService(dripline.core.Service): - def __init__(self, name): - dripline.core.Service.__init__(self, name) + def __init__(self, name, **kwargs): + dripline.core.Service.__init__(self, name, **kwargs) def a_method(self, n1, n2): return n1 + n2 - a_service = AnotherService("a_service") + a_service = AnotherService("a_service", make_connection=False) the_node = scarab.ParamNode() the_node.add("values", scarab.ParamArray()) n1, n2 = 10, 13 @@ -124,7 +124,7 @@ def a_method(self, n1, n2): assert(a_reply.payload.to_python() == n1 + n2) def test_send_request(): - a_service = dripline.core.Service("a_service") + a_service = dripline.core.Service("a_service", make_connection=False) a_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.get, "hey", "on_get", "a_receiver") a_sent_msg = a_service.send(a_request) assert not a_sent_msg.successful_send From 5abb511c4c9c50124aa29ffceff929a02628b0b2 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 30 Apr 2025 11:34:02 -0700 Subject: [PATCH 50/81] Move entire dripline endpoint/service stack bindings to use classh (i.e. use py::smart_holder) --- module_bindings/dripline_core/_endpoint_pybind.hh | 4 +++- module_bindings/dripline_core/core_pybind.hh | 3 +-- module_bindings/dripline_core/scheduler_pybind.hh | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/module_bindings/dripline_core/_endpoint_pybind.hh b/module_bindings/dripline_core/_endpoint_pybind.hh index 61e44c8e..fadf1aca 100644 --- a/module_bindings/dripline_core/_endpoint_pybind.hh +++ b/module_bindings/dripline_core/_endpoint_pybind.hh @@ -17,7 +17,9 @@ namespace dripline_pybind std::list< std::string > all_items; all_items.push_back( "_Endpoint" ); - pybind11::classh< dripline::endpoint, _endpoint_trampoline >( mod, "_Endpoint", "Endpoint binding" ) + pybind11::classh< dripline::endpoint, + _endpoint_trampoline + >( mod, "_Endpoint", "Endpoint binding" ) .def( pybind11::init< const std::string& >(), DL_BIND_CALL_GUARD_STREAMS ) // mv_ properties diff --git a/module_bindings/dripline_core/core_pybind.hh b/module_bindings/dripline_core/core_pybind.hh index 65338d16..28afae00 100644 --- a/module_bindings/dripline_core/core_pybind.hh +++ b/module_bindings/dripline_core/core_pybind.hh @@ -26,8 +26,7 @@ namespace dripline_pybind ; all_items.push_back( "Core" ); - pybind11::class_< dripline::core, - std::shared_ptr< dripline::core > + pybind11::classh< dripline::core > t_core( mod, "Core", "lower-level class for AMQP message sending and receiving" ); // bind the core class diff --git a/module_bindings/dripline_core/scheduler_pybind.hh b/module_bindings/dripline_core/scheduler_pybind.hh index 4bde3b89..d089d5b7 100644 --- a/module_bindings/dripline_core/scheduler_pybind.hh +++ b/module_bindings/dripline_core/scheduler_pybind.hh @@ -20,9 +20,8 @@ namespace dripline_pybind using executor_t = dripline::simple_executor; using executable_t = std::function< void() >; using clock_t = std::chrono::system_clock; - pybind11::class_< dripline::scheduler< executor_t, clock_t >, - scarab::cancelable, - std::shared_ptr< dripline::scheduler< executor_t, clock_t > > + pybind11::classh< dripline::scheduler< executor_t, clock_t >, + scarab::cancelable >( mod, "Scheduler", "schedule future function calls" ) .def( pybind11::init<>() ) From 30d3b2a408955d8573632e03481bffb073f18a29 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 1 May 2025 17:11:05 -0700 Subject: [PATCH 51/81] More complete exception translation --- module_bindings/dripline_core/error_pybind.hh | 100 ++++++++++++++++-- 1 file changed, 93 insertions(+), 7 deletions(-) diff --git a/module_bindings/dripline_core/error_pybind.hh b/module_bindings/dripline_core/error_pybind.hh index ee3524ff..11184989 100644 --- a/module_bindings/dripline_core/error_pybind.hh +++ b/module_bindings/dripline_core/error_pybind.hh @@ -2,32 +2,118 @@ #define DRIPLINE_PYBIND_ERROR #include "dripline_exceptions.hh" +#include "message.hh" +#include "typename.hh" + #include "pybind11/pybind11.h" +#include +#include + namespace dripline_pybind { + // For use in tests of thowing exceptions on the C++ side + void throw_dripline_error( const std::string& what_msg = "" ) + { + throw dripline::dripline_error() << what_msg; + } + + // For use in tests of throwing messages on the C++ side (this is done in core::do_send()) + void throw_message() + { + throw dripline::msg_request::create(scarab::param_ptr_t(new scarab::param()), dripline::op_t::get, "hey"); + } + + // Utility function so we can load the string version of a message into an error message + template< typename T > + std::string stream_message_ptr_to_error_message( T a_ptr, const std::string& a_prefix ) + { + std::stringstream sstr; + sstr << a_prefix << *a_ptr; + std::string error_msg(sstr.str()); + return std::string(sstr.str()); + } std::list< std::string > export_error( pybind11::module& mod ) { std::list< std::string > all_items; - //TODO how do we actually want to deal with errors? - all_items.push_back( "DriplineError" ); - pybind11::register_exception< dripline::dripline_error >( mod, "DriplineError", PyExc_RuntimeError ); + // For use in tests thowing exceptions on the C++ side and catching them on the Python side + all_items.push_back( "throw_dripline_error" ); + mod.def( "throw_dripline_error", &dripline_pybind::throw_dripline_error, + "Test function for throwing a dripline_error on the C++ side" ); + + // For use in tests thowing messages on the C++ side and catching them on the Python side + all_items.push_back( "throw_message" ); + mod.def( "throw_message", &dripline_pybind::throw_message, + "Test function for throwing a message on the C++ side" ); + + // Exception and exception translator registrations go below here. + // Per Pybind11 docs, exception translation happens in reverse order from how they appear here. + // Therefore we put the register_exception_translator try/catch block first, because it has a catch-all term for unknown exception types. + // Within the try-catch block, we start with known classes that we want to catch, and finish with the `...` catch-all. + // Following that, we have registered exceptions of known type. /* + // this static definition is used for the C++ --> Python throw_reply translation + PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store throw_reply_storage; + throw_reply_storage.call_once_and_store_result( + [&]() { return dripline.core.ThrowReply; } + ); +*/ pybind11::register_exception_translator( [](std::exception_ptr p) { try { if ( p ) std::rethrow_exception( p ); } - catch ( const dripline::dripline_error &e ) +/* + catch ( const dripline::throw_reply& e ) { - // Set dripline_error as the active python error - pybind11::set_error( PyExc_Exception, e.what() ); + // Usually throw replies go Python --> C++, but this is here in case there's a C++ --> Python situation + pybind11::set_error( dripline.core.ThrowReply ) + } +*/ + catch ( const dripline::message_ptr_t& e ) + { + std::string error_msg = std::move(stream_message_ptr_to_error_message(e, "Thrown message:\n")); + //std::cerr << "Caught message thrown:\n" << error_msg << std::endl; + pybind11::set_error( PyExc_RuntimeError, error_msg.c_str() ); + } + catch ( const dripline::request_ptr_t& e ) + { + std::string error_msg = std::move(stream_message_ptr_to_error_message(e, "Thrown request:\n")); + //::cerr << "Caught request thrown:\n" << error_msg << std::endl; + pybind11::set_error( PyExc_RuntimeError, error_msg.c_str() ); + } + catch ( const dripline::reply_ptr_t& e ) + { + std::string error_msg = std::move(stream_message_ptr_to_error_message(e, "Thrown reply:\n")); + //std::cerr << "Caught reply thrown:\n" << error_msg << std::endl; + pybind11::set_error( PyExc_RuntimeError, error_msg.c_str() ); + } + catch ( const dripline::alert_ptr_t& e ) + { + std::string error_msg = std::move(stream_message_ptr_to_error_message(e, "Thrown alert:\n")); + //std::cerr << "Caught alert thrown:\n" << error_msg << std::endl; + pybind11::set_error( PyExc_RuntimeError, error_msg.c_str() ); + } + catch (...) + { + // catch-all for unknown exception types + std::string exName(abi::__cxa_current_exception_type()->name()); + std::stringstream sstr; + sstr << "Unknown exception: " << exName; + std::string error_msg(sstr.str()); + //std::cerr << error_msg << std::endl; + pybind11::set_error( PyExc_RuntimeError, error_msg.c_str() ); } } - );*/ + ); + + all_items.push_back( "DriplineError" ); + pybind11::register_exception< dripline::dripline_error >( mod, "DriplineError", PyExc_RuntimeError ); + + return all_items; } From 647e07a88eb5c0ad0c3c55963edbc98aad4af879 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 1 May 2025 17:11:15 -0700 Subject: [PATCH 52/81] Minor fix in entity.py --- dripline/core/entity.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 44333606..ae37f44b 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -183,7 +183,7 @@ def log_a_value(self, the_value): logger.info(f"value to log for {self.name} is:\n{the_value}") self._last_log_time = datetime.datetime.now(datetime.timezone.utc) the_alert = MsgAlert.create(payload=scarab.to_param(the_value), routing_key=f'{self.log_routing_key_prefix}.{self.name}') - alert_sent = self.service.send(the_alert) + _ = self.service.send(the_alert) def start_logging(self): if self._log_action_id is not None: From ce14b843c1cdafb4083e783c2595d89e20436e7b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 1 May 2025 17:11:34 -0700 Subject: [PATCH 53/81] Add test_error.py and update test_service.py --- tests/test_error.py | 21 +++++++++++++++++++++ tests/test_service.py | 41 +++++++++++++++++++++++++---------------- 2 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 tests/test_error.py diff --git a/tests/test_error.py b/tests/test_error.py new file mode 100644 index 00000000..ffba0f33 --- /dev/null +++ b/tests/test_error.py @@ -0,0 +1,21 @@ +import dripline +import pytest + +def test_raise_dripline_error(): + with pytest.raises(dripline.core.DriplineError) as excinfo: + raise dripline.core.DriplineError + assert excinfo.type is dripline.core.DriplineError + +def test_throw_dripline_error(): + message = "test_throw" + with pytest.raises(dripline.core.DriplineError, match=message) as excinfo: + dripline.core.throw_dripline_error(message) + assert excinfo.type is dripline.core.DriplineError + +# In dripline::core::do_send() we throw a dripline message object. +# Let's make sure that gets caught in a reasonable way from the Python side +def test_throw_message(): + with pytest.raises(RuntimeError) as excinfo: + dripline.core.throw_message() + print(f'Exception value: {excinfo.value}') + assert excinfo.type is RuntimeError diff --git a/tests/test_service.py b/tests/test_service.py index def3c606..7b56b394 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,4 +1,4 @@ -import scarab, dripline.core +import scarab, dripline import pytest @@ -10,22 +10,30 @@ def test_service_creation(): def test_submit_request_message(): a_name = "a_service" a_service = dripline.core.Service(a_name, make_connection=False) - a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") - a_reply = a_service.submit_request_message(a_request) - assert(isinstance(a_reply, dripline.core.MsgReply)) - assert(a_reply.return_code == 0) - assert(a_reply.correlation_id == a_request.correlation_id) - a_reply.payload.to_python()['values'] == [a_name] + a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, a_name, "name", "a_receiver") + # Should throw a reply: on_request_message results in a reply message that gets sent; in core, since the service is offline, the reply is thrown + with pytest.raises(RuntimeError) as excinfo: + a_reply = a_service.submit_request_message(a_request) + assert excinfo.type is RuntimeError + assert "Thrown reply:" in str(excinfo.value) + #assert(isinstance(a_reply, dripline.core.MsgReply)) + #assert(a_reply.return_code == 0) + #assert(a_reply.correlation_id == a_request.correlation_id) + #a_reply.payload.to_python()['values'] == [a_name] def test_on_request_message(): a_name = "a_service" a_service = dripline.core.Service(a_name, make_connection=False) - a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, "hey", "name", "a_receiver") - a_reply = a_service.on_request_message(a_request) - assert(isinstance(a_reply, dripline.core.MsgReply)) - assert(a_reply.return_code == 0) # 0 - assert(a_reply.correlation_id == a_request.correlation_id) - a_reply.payload.to_python()['values'] == [a_name] + a_request = dripline.core.MsgRequest.create(scarab.ParamValue(5), dripline.core.op_t.get, a_name, "name", "a_receiver") + # Should throw a reply: on_request_message results in a reply message that gets sent; in core, since the service is offline, the reply is thrown + with pytest.raises(RuntimeError) as excinfo: + a_reply = a_service.on_request_message(a_request) + assert excinfo.type is RuntimeError + assert "Thrown reply:" in str(excinfo.value) +# assert(isinstance(a_reply, dripline.core.MsgReply)) +# assert(a_reply.return_code == 0) # 0 +# assert(a_reply.correlation_id == a_request.correlation_id) +# a_reply.payload.to_python()['values'] == [a_name] def test_on_reply_message(): a_service = dripline.core.Service("hello", make_connection=False) @@ -126,6 +134,7 @@ def a_method(self, n1, n2): def test_send_request(): a_service = dripline.core.Service("a_service", make_connection=False) a_request = dripline.core.MsgRequest.create(scarab.Param(), dripline.core.op_t.get, "hey", "on_get", "a_receiver") - a_sent_msg = a_service.send(a_request) - assert not a_sent_msg.successful_send - assert 'Reply code: 312 NO_ROUTE' in a_sent_msg.send_error_message + with pytest.raises(RuntimeError, match="Thrown request:*") as excinfo: + a_sent_msg = a_service.send(a_request) + print(f'Exception value:\n{excinfo.value}') + assert excinfo.type is RuntimeError From d8d6efd05483685003695cf22f59a4acde48bf4c Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 1 May 2025 17:11:46 -0700 Subject: [PATCH 54/81] Add very-incomplete test_entity.py --- tests/test_entity.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tests/test_entity.py diff --git a/tests/test_entity.py b/tests/test_entity.py new file mode 100644 index 00000000..4c9b1e36 --- /dev/null +++ b/tests/test_entity.py @@ -0,0 +1,12 @@ +import dripline +import pytest + +def test_log_a_value(): + a_service = dripline.core.Service("hello", make_connection=False) + a_entity = dripline.core.Entity(name="ent") + a_service.add_child(a_entity) + with pytest.raises(RuntimeError) as excinfo: + a_entity.log_a_value(5) + assert excinfo.type is RuntimeError + assert "Thrown alert:" in excinfo.value + assert "Payload: 5" in excinfo.value From e6eaad318338bc3f67bcbf796536594b9e0038c0 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 2 May 2025 11:50:59 -0700 Subject: [PATCH 55/81] Use an alternate image for the dl-cpp base class --- .github/workflows/publish.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 5c80a043..bcc90a09 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -12,10 +12,10 @@ env: REGISTRY_OLD: docker.io BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp - DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.4' -# BASE_IMAGE_TAG: 'hf2.10.4' -# DEV_SUFFIX: '' + #DEV_SUFFIX: '-dev' + #BASE_IMAGE_TAG: 'v2.10.4' + BASE_IMAGE_TAG: 'mols' + DEV_SUFFIX: '' jobs: From 1f9ae8eb383d6468f52748e4a314400c0724a644 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 2 May 2025 15:05:35 -0700 Subject: [PATCH 56/81] Fixing test_entity test --- tests/test_entity.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_entity.py b/tests/test_entity.py index 4c9b1e36..6cfddbf0 100644 --- a/tests/test_entity.py +++ b/tests/test_entity.py @@ -8,5 +8,5 @@ def test_log_a_value(): with pytest.raises(RuntimeError) as excinfo: a_entity.log_a_value(5) assert excinfo.type is RuntimeError - assert "Thrown alert:" in excinfo.value - assert "Payload: 5" in excinfo.value + assert "Thrown alert:" in str(excinfo.value) + assert "Payload: 5" in str(excinfo.value) From ed900cfcbdd56f3ae527c517e8300effbf14b201 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 2 May 2025 17:05:08 -0700 Subject: [PATCH 57/81] Temporary change of base image --- .github/workflows/publish.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 77d0edea..b9606dc5 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -12,10 +12,10 @@ env: REGISTRY_OLD: docker.io BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp - DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.4' -# BASE_IMAGE_TAG: 'hf2.10.4' -# DEV_SUFFIX: '' +# DEV_SUFFIX: '-dev' +# BASE_IMAGE_TAG: 'v2.10.4' + BASE_IMAGE_TAG: 'cancelation' + DEV_SUFFIX: '' jobs: From 109c634ce5bcbea94e81c8529922447494f86207 Mon Sep 17 00:00:00 2001 From: "Walter C. Pettus" Date: Wed, 7 May 2025 14:55:22 -0400 Subject: [PATCH 58/81] Fix select in postgres_interface --- dripline/implementations/ethernet_scpi_service.py | 4 ++-- dripline/implementations/postgres_interface.py | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/dripline/implementations/ethernet_scpi_service.py b/dripline/implementations/ethernet_scpi_service.py index 2fbec19e..8cbcd334 100644 --- a/dripline/implementations/ethernet_scpi_service.py +++ b/dripline/implementations/ethernet_scpi_service.py @@ -52,10 +52,10 @@ def __init__(self, (ip,port) = re.findall(re_str,socket_info)[0] socket_info = (ip,int(port)) if response_terminator is None or response_terminator == '': - raise ThrowReply('service_error_invalid_value', f"Invalid response terminator: <{repr(response_terminator)}>! Expect string") + raise ValueError(f"Invalid response terminator: <{repr(response_terminator)}>! Expect string") if not isinstance(cmd_at_reconnect, list) or len(cmd_at_reconnect)==0: if cmd_at_reconnect is not None: - raise ThrowReply('service_error_invalid_value', f"Invalid cmd_at_reconnect: <{repr(cmd_at_reconnect)}>! Expect non-zero length list") + raise ValueError(f"Invalid cmd_at_reconnect: <{repr(cmd_at_reconnect)}>! Expect non-zero length list") self.alock = threading.Lock() self.socket = socket.socket() diff --git a/dripline/implementations/postgres_interface.py b/dripline/implementations/postgres_interface.py index 080fea1e..b2095cf7 100644 --- a/dripline/implementations/postgres_interface.py +++ b/dripline/implementations/postgres_interface.py @@ -131,27 +131,26 @@ def do_select(self, return_cols=[], where_eq_dict={}, where_lt_dict={}, where_gt Returns: a tuple, 1st element is list of column names, 2nd is a list of tuples of the rows that matched the select ''' if not return_cols: - return_cols = self.table.c + this_select = sqlalchemy.select(self.table) else: - return_cols = [sqlalchemy.text(col) for col in return_cols] - this_select = sqlalchemy.select(return_cols) + this_select = sqlalchemy.select(*[getattr(self.table.c,col) for col in return_cols]) for c,v in where_eq_dict.items(): this_select = this_select.where(getattr(self.table.c,c)==v) for c,v in where_lt_dict.items(): this_select = this_select.where(getattr(self.table.c,c)v) - conn = self.service.engine.connect() - result = conn.execute(this_select) + with self.service.engine.connect() as conn: + result = conn.execute(this_select) return (result.keys(), [i for i in result]) def _insert_with_return(self, insert_kv_dict, return_col_names_list): ins = self.table.insert().values(**insert_kv_dict) if return_col_names_list: ins = ins.returning(*[self.table.c[col_name] for col_name in return_col_names_list]) - conn = self.service.engine.connect() - insert_result = conn.execute(ins) - conn.commit() + with self.service.engine.connect() as conn: + insert_result = conn.execute(ins) + conn.commit() if return_col_names_list: return_values = insert_result.first() else: From 03e52107a1419187983d3ef7b2acddacfb262d5b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 15 May 2025 11:27:12 -0700 Subject: [PATCH 59/81] Update docker builds to use dl-cpp v2.10.5 --- .github/workflows/publish.yaml | 8 ++++---- Dockerfile | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index b9606dc5..f483ea64 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -12,10 +12,10 @@ env: REGISTRY_OLD: docker.io BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp -# DEV_SUFFIX: '-dev' -# BASE_IMAGE_TAG: 'v2.10.4' - BASE_IMAGE_TAG: 'cancelation' - DEV_SUFFIX: '' + DEV_SUFFIX: '-dev' + BASE_IMAGE_TAG: 'v2.10.5' +# BASE_IMAGE_TAG: 'cancelation' +# DEV_SUFFIX: '' jobs: diff --git a/Dockerfile b/Dockerfile index 2bd456da..be4bdc54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG img_user=ghcr.io/driplineorg ARG img_repo=dripline-cpp #ARG img_tag=develop -ARG img_tag=v2.10.4 +ARG img_tag=v2.10.5 FROM ${img_user}/${img_repo}:${img_tag} From 9d7363cc62f7351df240652f47e4de126109b86d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 19 May 2025 15:19:09 -0700 Subject: [PATCH 60/81] Bind a bunch of member variables in core; use alerts exchange in alert_consumer; remove alerts and requests exchange properties from _service --- dripline/core/alert_consumer.py | 2 +- module_bindings/dripline_core/_service_pybind.hh | 2 -- module_bindings/dripline_core/core_pybind.hh | 12 +++++++++++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/dripline/core/alert_consumer.py b/dripline/core/alert_consumer.py index 82926329..9434dd99 100644 --- a/dripline/core/alert_consumer.py +++ b/dripline/core/alert_consumer.py @@ -35,7 +35,7 @@ def bind_keys(self): to_return = Service.bind_keys(self); for a_key in self._alert_keys: logger.debug(f" binding alert key {a_key}") - to_return = to_return and self.bind_key("alerts", a_key) + to_return = to_return and self.bind_key(self.alerts_exchange, a_key) return to_return def on_alert_message(self, an_alert): diff --git a/module_bindings/dripline_core/_service_pybind.hh b/module_bindings/dripline_core/_service_pybind.hh index 5b3f787d..555fc156 100644 --- a/module_bindings/dripline_core/_service_pybind.hh +++ b/module_bindings/dripline_core/_service_pybind.hh @@ -65,8 +65,6 @@ namespace dripline_pybind ) .def_property( "enable_scheduling", &dripline::service::get_enable_scheduling, &dripline::service::set_enable_scheduling ) - .def_property_readonly( "alerts_exchange", (std::string& (dripline::service::*)()) &dripline::service::alerts_exchange ) - .def_property_readonly( "requests_exchange", (std::string& (dripline::service::*)()) &dripline::service::requests_exchange ) .def_property_readonly( "sync_children", (std::map& (dripline::service::*)()) &dripline::service::sync_children ) //TODO: need to deal with lr_ptr_t to bind this //.def_property_readonly( "async_children", &dripline::service::async_children ) diff --git a/module_bindings/dripline_core/core_pybind.hh b/module_bindings/dripline_core/core_pybind.hh index 80a74101..792c1954 100644 --- a/module_bindings/dripline_core/core_pybind.hh +++ b/module_bindings/dripline_core/core_pybind.hh @@ -62,7 +62,17 @@ namespace dripline_pybind DL_BIND_CALL_GUARD_STREAMS_AND_GIL, "send an alert message" ) - + //.def_property( "address", std::static_cast< const std::string& (const dripline::core::*) >( &dripline::core::address ), [](dripline::core& a_core, const std::string& a_value){a_core.address() = a_value;} ) + .def_property( "address", [](const dripline::core& a_core){return a_core.address();}, [](dripline::core& a_core, const std::string& a_value){a_core.address() = a_value;} ) + .def_property( "port", &dripline::core::get_port, &dripline::core::set_port ) + .def_property( "username", [](const dripline::core& a_core){return a_core.username();}, [](dripline::core& a_core, const std::string& a_value){a_core.username() = a_value;} ) + .def_property( "password", [](const dripline::core& a_core){return a_core.password();}, [](dripline::core& a_core, const std::string& a_value){a_core.password() = a_value;} ) + .def_property( "requests_exchange", [](const dripline::core& a_core){return a_core.requests_exchange();}, [](dripline::core& a_core, const std::string& a_value){a_core.requests_exchange() = a_value;} ) + .def_property( "alerts_exchange", [](const dripline::core& a_core){return a_core.alerts_exchange();}, [](dripline::core& a_core, const std::string& a_value){a_core.alerts_exchange() = a_value;} ) + .def_property( "heartbeat_routing_key", [](const dripline::core& a_core){return a_core.heartbeat_routing_key();}, [](dripline::core& a_core, const std::string& a_value){a_core.heartbeat_routing_key() = a_value;} ) + .def_property( "max_payload_size", &dripline::core::get_max_payload_size, &dripline::core::set_max_payload_size ) + .def_property( "make_connection", &dripline::core::get_make_connection, &dripline::core::set_make_connection ) + .def_property( "max_connection_attempts", &dripline::core::get_max_connection_attempts, &dripline::core::set_max_connection_attempts ) ; // bind core's internal types From 305c9ee94af84dbf9fd08dcfa9174b87e91c954d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 19 May 2025 15:19:42 -0700 Subject: [PATCH 61/81] Update heartbeat_monitor to have endpoints bind unique names instead of conflicting with services' names --- dripline/implementations/heartbeat_monitor.py | 42 ++++++++++++++----- .../services/heartbeat-monitor.yaml | 3 +- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index 6690a6ab..82d84ac4 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -23,10 +23,13 @@ class HeartbeatTracker(Endpoint): ''' ''' - def __init__(self, **kwargs): + def __init__(self, service_name, **kwargs): ''' + Args: + service_name (str): Name of the service to be monitored ''' Endpoint.__init__(self, **kwargs) + self.service_name = service_name self.last_timestamp = time.time() self.is_active = True self.status = HeartbeatTracker.Status.UNKNOWN @@ -34,7 +37,7 @@ def __init__(self, **kwargs): def process_heartbeat(self, timestamp): ''' ''' - logger.debug(f'New timestamp for {self.name}: {timestamp}') + logger.debug(f'New timestamp for {self.service_name}: {timestamp}') dt = datetime.fromisoformat(timestamp) posix_time = dt.timestamp() logger.debug(f'Time since epoch: {posix_time}') @@ -47,15 +50,15 @@ def check_delay(self): if self.is_active: if diff > self.service.critical_threshold_s: # report critical - logger.critical(f'Missing heartbeat: {self.name}') + logger.critical(f'Missing heartbeat: {self.service_name}') self.status = HeartbeatTracker.Status.CRITICAL else: if diff > self.service.warning_threshold_s: # report warning - logger.warning(f'Missing heartbeat: {self.name}') + logger.warning(f'Missing heartbeat: {self.service_name}') self.status = HeartbeatTracker.Status.WARNING else: - logger.debug(f'Heartbeat status ok: {self.name}') + logger.debug(f'Heartbeat status ok: {self.service_name}') self.status = HeartbeatTracker.Status.OK else: # report inactive heartbeat received @@ -75,14 +78,20 @@ class HeartbeatMonitor(AlertConsumer): An alert consumer which listens to heartbeat messages and keeps track of the time since the last was received ''' - def __init__(self, time_between_checks_s=20, warning_threshold_s=120, critical_threshold_s=300, add_unknown_heartbeats=True, **kwargs): + def __init__(self, + time_between_checks_s=20, + warning_threshold_s=120, + critical_threshold_s=300, + add_unknown_heartbeats=True, + endpoint_name_prefix='hbmon_', + **kwargs): ''' Args: time_between_checks_s (int): number of seconds between heartbeat status checks warning_threshold_s (int): warning threshold for missing heartbeats (in seconds) critical_threshold_s (int): critical threshold for missing heartbeats (in seconds) add_unknown_heartbeats (bool): whether or not to add a new endpoint if an unknown heartbeat is received - socket_timeout (int): number of seconds to wait for a reply from the device before timeout. + endpoint_name_prefix (str): prefix added to monitored-service names for hbmon endpoints ''' AlertConsumer.__init__(self, **kwargs) @@ -97,6 +106,16 @@ def __init__(self, time_between_checks_s=20, warning_threshold_s=120, critical_t self.warning_threshold_s = warning_threshold_s self.critical_threshold_s = critical_threshold_s self.add_unknown_heartbeats = add_unknown_heartbeats + self.endpoint_name_prefix = endpoint_name_prefix + + # Fill the dictionary mapping monitoring name to service name + self.monitoring_names = {} + for an_endpoint in self.sync_children.values(): + try: + self.monitoring_names[an_endpoint.service_name] = an_endpoint.name + except Exception as err: + logger.error(f'Error while attempting to fill monitoring_names: {err}') + logger.debug(f'Initial set of services monitored:\n{self.monitoring_names}') def run(self): monitor_thread = threading.Thread(target=self.monitor_heartbeats) @@ -175,15 +194,18 @@ def process_report(self, report_data): def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): service_name = a_routing_key_data['service_name'] - if not service_name in self.sync_children: + if not service_name in self.monitoring_names: logger.warning(f'received unexpected heartbeat;\npayload: {a_payload}\nrouting key data: {a_routing_key_data}\ntimestamp: {a_message_timestamp}') if self.add_unknown_heartbeats: - self.add_child(HeartbeatTracker(name=service_name)) + binding = self.endpoint_name_prefix+service_name + self.add_child(HeartbeatTracker(service_name=service_name, name=binding)) + self.monitoring_names[service_name] = binding + self.bind_key(self.requests_exchange, binding+'.#') logger.debug(f'Added endpoint for unknown heartbeat from {service_name}') return try: - self.sync_children[service_name].process_heartbeat(a_message_timestamp) + self.sync_children[self.monitoring_names[service_name]].process_heartbeat(a_message_timestamp) except Exception as err: logger.error(f'Unable to handle payload for heartbeat from service {service_name}: {err}') diff --git a/tests/integration/services/heartbeat-monitor.yaml b/tests/integration/services/heartbeat-monitor.yaml index ec1d959d..1ab4fd6f 100644 --- a/tests/integration/services/heartbeat-monitor.yaml +++ b/tests/integration/services/heartbeat-monitor.yaml @@ -10,5 +10,6 @@ alert_keys: alert_key_parser_re: 'heartbeat\.(?P\w+)' endpoints: - - name: heartbeat_monitor + - name: hbmon_heartbeat_monitor + service_name: heartbeat_monitor module: HeartbeatTracker From 31e4b705e75db144f79aeb53f87fbeb3da8dfccb Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 20 May 2025 17:25:02 -0700 Subject: [PATCH 62/81] Don't bind a key while a service is operating --- dripline/implementations/heartbeat_monitor.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dripline/implementations/heartbeat_monitor.py b/dripline/implementations/heartbeat_monitor.py index 82d84ac4..315a50e5 100644 --- a/dripline/implementations/heartbeat_monitor.py +++ b/dripline/implementations/heartbeat_monitor.py @@ -200,8 +200,14 @@ def process_payload(self, a_payload, a_routing_key_data, a_message_timestamp): binding = self.endpoint_name_prefix+service_name self.add_child(HeartbeatTracker(service_name=service_name, name=binding)) self.monitoring_names[service_name] = binding - self.bind_key(self.requests_exchange, binding+'.#') - logger.debug(f'Added endpoint for unknown heartbeat from {service_name}') + logger.debug(f'Started monitoring hearteats from {service_name}') + logger.warning(f'Heartbeat monitor is currently unable to listen for requests addressed to the endpoints of new heartbeat trackers; You will not be able to send messages to {binding}') + # We'd like to be able to bind the new end point to the service's connection. + # However, we're unable to bind while the connection is being listened on. + # We either need a way to stop the service and restart (at which point it would bind all of the endpoints, including the new one), + # or we use the endpoint as an asychronous endpoint, in which case we need a way to start its thread (there isn't a separate function in dl-cpp to do this at this point). + #self.bind_key(self.requests_exchange, binding+'.#') + #logger.debug(f'Added endpoint for unknown heartbeat from {service_name}') return try: From 2ff8c54e3464670c95dc9e044430a039042ca524 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 21 May 2025 11:05:33 -0700 Subject: [PATCH 63/81] [no ci] bumped version to 5.1.0 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6589732d..ad2576a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.5) # <3.5 is deprecated by CMake -project( DriplinePy VERSION 5.0.1 ) +project( DriplinePy VERSION 5.1.0 ) cmake_policy( SET CMP0074 NEW ) From e31e3a6a5e4a2798d0403d53e321dc4ac0591ea9 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 8 Jul 2025 17:36:04 -0700 Subject: [PATCH 64/81] Switch constants binding back to unsigned integers --- module_bindings/dripline_core/constants_pybind.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/module_bindings/dripline_core/constants_pybind.hh b/module_bindings/dripline_core/constants_pybind.hh index 9c86fa7d..2382b616 100644 --- a/module_bindings/dripline_core/constants_pybind.hh +++ b/module_bindings/dripline_core/constants_pybind.hh @@ -19,9 +19,9 @@ namespace dripline_pybind .value( "cmd", dripline::op_t::cmd ) .value( "unknown", dripline::op_t::unknown ) // helpers for type conversion - .def( "to_int", (int (*)(dripline::op_t))&dripline::to_int, "Convert an op_t to int" ) + .def( "to_uint", (unsigned (*)(dripline::op_t))&dripline::to_int, "Convert an op_t to integer" ) .def( "to_string", (std::string (*)(dripline::op_t))&dripline::to_string, "Convert an op_t to string" ) - .def_static( "to_op_t", (dripline::op_t (*)(int))&dripline::to_op_t, "Convert an int to op_t" ) + .def_static( "to_op_t", (dripline::op_t (*)(unsigned))&dripline::to_op_t, "Convert an integer to op_t" ) .def_static( "to_op_t", (dripline::op_t (*)(std::string))&dripline::to_op_t, "Convert an string to op_t" ) ; @@ -33,9 +33,9 @@ namespace dripline_pybind .value( "alert", dripline::msg_t::alert ) .value( "unknown", dripline::msg_t::unknown ) // helpers for type conversion - .def( "to_int", (int (*)(dripline::msg_t))&dripline::to_int, "Convert a msg_t to int" ) + .def( "to_uint", (unsigned (*)(dripline::msg_t))&dripline::to_uint, "Convert a msg_t to integer" ) .def( "to_string", (std::string (*)(dripline::msg_t))&dripline::to_string, "Convert a msg_t to string" ) - .def_static( "to_msg_t", (dripline::msg_t (*)(int))&dripline::to_msg_t, "Convert an int to msg_t" ) + .def_static( "to_msg_t", (dripline::msg_t (*)(unsigned))&dripline::to_msg_t, "Convert an integer to msg_t" ) .def_static( "to_msg_t", (dripline::msg_t (*)(std::string))&dripline::to_msg_t, "Convert a string to msg_t" ) ; From 685e90142d66baa231e0e78b4256f97b527d93a2 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 8 Jul 2025 17:42:39 -0700 Subject: [PATCH 65/81] Fixing the last commit --- module_bindings/dripline_core/constants_pybind.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_bindings/dripline_core/constants_pybind.hh b/module_bindings/dripline_core/constants_pybind.hh index 2382b616..1e5a5049 100644 --- a/module_bindings/dripline_core/constants_pybind.hh +++ b/module_bindings/dripline_core/constants_pybind.hh @@ -19,7 +19,7 @@ namespace dripline_pybind .value( "cmd", dripline::op_t::cmd ) .value( "unknown", dripline::op_t::unknown ) // helpers for type conversion - .def( "to_uint", (unsigned (*)(dripline::op_t))&dripline::to_int, "Convert an op_t to integer" ) + .def( "to_uint", (unsigned (*)(dripline::op_t))&dripline::to_uint, "Convert an op_t to integer" ) .def( "to_string", (std::string (*)(dripline::op_t))&dripline::to_string, "Convert an op_t to string" ) .def_static( "to_op_t", (dripline::op_t (*)(unsigned))&dripline::to_op_t, "Convert an integer to op_t" ) .def_static( "to_op_t", (dripline::op_t (*)(std::string))&dripline::to_op_t, "Convert an string to op_t" ) From 659533a17178a47b9bdd636e476e643ed6c55e12 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 8 Jul 2025 17:44:12 -0700 Subject: [PATCH 66/81] Updating the dl-cpp tag in the Dockerfile and GHA workflow --- .github/workflows/publish.yaml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 0412bd1c..58a9f668 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -13,7 +13,7 @@ env: BASE_IMAGE_USER: ghcr.io/driplineorg BASE_IMAGE_REPO: dripline-cpp DEV_SUFFIX: '-dev' - BASE_IMAGE_TAG: 'v2.10.5' + BASE_IMAGE_TAG: 'v2.10.6' # BASE_IMAGE_TAG: 'cancelation' # DEV_SUFFIX: '' diff --git a/Dockerfile b/Dockerfile index b8c17933..93ab7c9d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ARG img_user=ghcr.io/driplineorg ARG img_repo=dripline-cpp #ARG img_tag=develop -ARG img_tag=v2.10.5 +ARG img_tag=v2.10.6 FROM ${img_user}/${img_repo}:${img_tag} AS deps From c51db6e3ab9b8c0cde3030f352a931a0f7d27277 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Jul 2025 10:57:30 -0700 Subject: [PATCH 67/81] Fixing test_constants to call the right conversion functions --- tests/test_constants.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_constants.py b/tests/test_constants.py index 657998bd..313d7de7 100644 --- a/tests/test_constants.py +++ b/tests/test_constants.py @@ -3,10 +3,10 @@ def test_op_t_to_int(): item = dripline.core.op_t - assert(item.to_int(item.set) == item.set.value) - assert(item.to_int(item.get) == item.get.value) - assert(item.to_int(item.cmd) == item.cmd.value) - assert(item.to_int(item.unknown) == item.unknown.value) + assert(item.to_uint(item.set) == item.set.value) + assert(item.to_uint(item.get) == item.get.value) + assert(item.to_uint(item.cmd) == item.cmd.value) + assert(item.to_uint(item.unknown) == item.unknown.value) def test_op_t_to_string(): item = dripline.core.op_t @@ -35,10 +35,10 @@ def test_op_t_string_to_op_t(): def test_msg_t_to_int(): item = dripline.core.msg_t - assert(item.to_int(item.reply) == item.reply.value) - assert(item.to_int(item.request) == item.request.value) - assert(item.to_int(item.alert) == item.alert.value) - assert(item.to_int(item.unknown) == item.unknown.value) + assert(item.to_uint(item.reply) == item.reply.value) + assert(item.to_uint(item.request) == item.request.value) + assert(item.to_uint(item.alert) == item.alert.value) + assert(item.to_uint(item.unknown) == item.unknown.value) def test_msg_t_to_string(): item = dripline.core.msg_t From 79ff9f1655e2146255ff0548cc6e0b9995de42c1 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Jul 2025 14:08:51 -0700 Subject: [PATCH 68/81] Changed the check against max_fractional_change to be |x0-x1|/(|x0+x1|/2) and moved the return for when `result[self._check_field])` results in an exception --- dripline/core/entity.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index ae37f44b..658d7d18 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -107,8 +107,10 @@ def __init__(self, self._max_interval = max_interval self._max_fractional_change = max_fractional_change self._check_field = check_field + self._log_action_id = None self._last_log_time = None + self._last_log_value = None @property def get_on_set(self): @@ -160,22 +162,21 @@ def scheduled_log(self): try: this_value = float(result[self._check_field]) except (TypeError, ValueError): - this_value = False + logger.warning(f"cannot check value change for {self.name}") + return + # Various checks for log condition if self._last_log_time is None: - logger.debug("log because no last log") + logger.debug("Logging because this is the first logged value") elif (datetime.datetime.now(datetime.timezone.utc) - self._last_log_time).total_seconds() > self._max_interval: - logger.debug("log because too much time") - elif this_value is False: - logger.warning(f"cannot check value change for {self.name}") - return - elif ((self._last_log_value == 0 and this_value != 0) or - (self._last_log_value != 0 and\ - abs((self._last_log_value - this_value)/self._last_log_value)>self._max_fractional_change)): - logger.debug("log because change magnitude") + logger.debug("Logging because enough time has elapsed") + # this condition is |x1-x0|/(|x1+x0|/2) > max_fractional_change, but safe in case the denominator is 0 + elif 2 * abs(self._last_log_value - this_value) > self._max_fractional_change * abs(self._last_log_value + this_value): + logger.debug("Logging because the value has changed significantly") else: - logger.debug("no log condition met, not logging") + logger.debug("No log condition met, therefore not logging") return + self._last_log_value = this_value self.log_a_value(result) From d8ee30d228e8d35a85f2a7d063bb334b83a0dd98 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 11 Jul 2025 14:36:01 -0700 Subject: [PATCH 69/81] Clarified (hopefully) the docstrings in Entity.__init__(); minor changes to Entity's docstring --- dripline/core/entity.py | 49 ++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 658d7d18..635b7568 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -54,12 +54,12 @@ def wrapper(*args, **kwargs): __all__.append("Entity") class Entity(Endpoint): ''' - Subclass of Endpoint which adds logic related to logging and confirming values. + Subclass of Endpoint that adds logic related to logging and confirming values. In particular, there is support for: - get_on_set -> setting the endpoint's value returns a get() result rather than an empty success (particularly useful for devices which may round assignment values) - log_on_set -> further extends get_on_set to send an alert message in addtion to returning the value in a reply - log_interval -> leverages the scheduler class to log the on_get result at a regular cadence + get_on_set -> setting the endpoint's value returns an on_get() result rather than an empty success (particularly useful for devices that may round assignment values) + log_on_set -> further extends get_on_set to send an logging alert message in addtion to returning the value in a reply + log_interval -> leverages the scheduler class to log the on_get result at a regular cadence and if the value changes significantly ''' #check_on_set -> allows for more complex logic to confirm successful value updates # (for example, the success condition may be measuring another endpoint) @@ -75,21 +75,34 @@ def __init__(self, **kwargs): ''' Args: - get_on_set: if true, calls to on_set are immediately followed by an on_get, which is returned - log_on_set: if true, always call log_a_value() immediately after on_set + get_on_set: bool (default is False) + If true, calls to on_set() are immediately followed by an on_get(), which is returned + log_on_set: bool (default is False) + If true, always call log_a_value() immediately after on_set() **Note:** requires get_on_set be true, overrides must be equivalent - log_routing_key_prefix: first term in routing key used in alert messages which log values - log_interval: how often to check the Entity's value. If 0 then scheduled logging is disabled; - if a number, interpreted as number of seconds; if a dict, unpacked as arguments - to the datetime.time_delta initializer; if a datetime.timedelta taken as the new value - max_interval: max allowed time interval between logging, allows usage of conditional logging. If 0, - then logging values occurs every log_interval. - max_fractional_change: max allowed fractional difference between subsequent values to trigger log condition. - check_field: result field to check, 'value_cal' or 'value_raw' - calibration (string || dict) : if string, updated with raw on_get() result via str.format() in - @calibrate decorator, used to populate raw and calibrated values - fields of a result payload. If a dictionary, the raw result is used - to index the dict with the calibrated value being the dict's value. + log_routing_key_prefix: string (default is 'sensor_value') + First term in routing key used in alert messages that log values + log_interval: 0 (default), float, dict, datetime.timmedelta + Defines how often to check the Entity's value to determine if it should be logged + If 0, scheduled logging is disabled; + If a number, interpreted as number of seconds; + If a dict, unpacked as arguments to the datetime.time_delta initializer; + If a datetime.timedelta, taken as the new value + max_interval: float + Maximum time interval between logging in seconds. + Logging will take place at the next log_interval after max_interval since the last logged value. + If less than log_interval, then logging values occurs every log_interval. + max_fractional_change: float + Fractional change in the value that will trigger the value to be logged + If 0, then any change in the value will be logged + If < 0, then the value will always be logged + check_field: string + Field in the dict returned by `on_get() that's used to check for a change in the fractional value + Typically is either 'value_cal' or 'value_raw' + calibration: string or dict + If string, updated with raw on_get() result via str.format() in the @calibrate decorator, + used to populate raw and calibrated values fields of a result payload. + If a dictionary, the raw result is used to index the dict with the calibrated value being the dict's value. ''' Endpoint.__init__(self, **kwargs) From ba6664e449da3e359ea99b1b224cc01c1c7904c8 Mon Sep 17 00:00:00 2001 From: "Walter C. Pettus" Date: Tue, 22 Jul 2025 15:49:09 -0400 Subject: [PATCH 70/81] Restore dl2 FormatEntity functionality for default floatify of get reply, plus some error handling --- dripline/implementations/entity_endpoints.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dripline/implementations/entity_endpoints.py b/dripline/implementations/entity_endpoints.py index fb3d21de..a2ff7186 100644 --- a/dripline/implementations/entity_endpoints.py +++ b/dripline/implementations/entity_endpoints.py @@ -39,7 +39,7 @@ def __init__(self, ''' Entity.__init__(self, **kwargs) if base_str is None: - raise ThrowReply('service_error_invalid_value', ' is required to __init__ SimpleSCPIEntity instance') + raise ValueError(' is required to __init__ SimpleSCPIEntity instance') else: self.cmd_base = base_str @@ -105,11 +105,11 @@ def __init__(self, ''' Args: get_str (str): sent verbatim in the event of on_get; if None, getting of endpoint is disabled - get_reply_float (bool): apply special formatting to get return set_str (str): sent as set_str.format(value) in the event of on_set; if None, setting of endpoint is disabled set_value_lowercase (bool): default option to map all string set value to .lower() **WARNING**: never set to False if using a set_value_map dict set_value_map (str||dict): inverse of calibration to map raw set value to value sent; either a dictionary or an asteval-interpretable string + get_reply_float (bool): apply special default formatting to get float return extract_raw_regex (str): regular expression search pattern applied to get return. Must be constructed with an extraction group keyed with the name "value_raw" (ie r'(?P)' ) ''' Entity.__init__(self, **kwargs) @@ -120,10 +120,10 @@ def __init__(self, self._extract_raw_regex = extract_raw_regex self.evaluator = asteval.Interpreter() if set_value_map is not None and not isinstance(set_value_map, (dict,str)): - raise ThrowReply('service_error_invalid_value', f"Invalid set_value_map config for {self.name}; type is {type(set_value_map)} not dict") + raise ValueError(f"Invalid set_value_map config for {self.name}; type is {type(set_value_map)} not dict") self._set_value_lowercase = set_value_lowercase if isinstance(set_value_map, dict) and not set_value_lowercase: - raise ThrowReply('service_error_invalid_value', f"Invalid config option for {self.name} with set_value_map and set_value_lowercase=False") + raise ValueError(f"Invalid config option for {self.name} with set_value_map and set_value_lowercase=False") @calibrate() def on_get(self): @@ -131,7 +131,7 @@ def on_get(self): # exceptions.DriplineMethodNotSupportedError raise ThrowReply('message_error_invalid_method', f"endpoint '{self.name}' does not support get") result = self.service.send_to_device([self._get_str]) - logger.debug(f'result is: {result}') + logger.debug(f'raw result is: {result}') if self._extract_raw_regex is not None: first_result = result matches = re.search(self._extract_raw_regex, first_result) @@ -140,6 +140,9 @@ def on_get(self): raise ThrowReply('service_error_invalid_value', 'device returned unparsable result, [{}] has no match to input regex [{}]'.format(first_result, self._extract_raw_regex)) logger.debug(f"matches are: {matches.groupdict()}") result = matches.groupdict()['value_raw'] + elif self._get_reply_float: + result = float(re.findall("[-+]?\d+\.\d+",format(result))[0]) + logger.debug(f"formatted result is {result}") return result def on_set(self, value): From 4356fffb66a929475c4c1c90ba5d358c2b0043f5 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 6 Aug 2025 17:17:35 -0700 Subject: [PATCH 71/81] Adding a changelog.md file --- changelog.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 changelog.md diff --git a/changelog.md b/changelog.md new file mode 100644 index 00000000..4da354a0 --- /dev/null +++ b/changelog.md @@ -0,0 +1,42 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- Heartbeat Monitor (implementations.HeartbeatMonitor) +- New logic for how logging is handled by Entities + - The `log_interval` is now the interval with which an entity's value is checked, not necessarily logged + - Whether a value is logged at the `log_interval` is controlled by: + - `max_interval`: if this time is exceeded since the last log entry, then it will be logged; if 0 (default), then logging occurs every `log_interval` + - `max_fractional_change`: if the value changes by more than this since the last log entry, then it will be logged + - The field that's checked for the `max_fractional_change` is given by `check_field` + +### Changed + +- Methods for sending and receiving messages are moved to the mixin classes core.RequestHandler and core.RequestSender + to capture how dl-py handles requests for both services and endpoints +- Upgrade dl-cpp to v2.10.6 +- Docker build now separates the installation of dependencies into a separate stage + + +### Fixed + +- Postgres syntax +- Application cancelation -- can use ctrl-c or other system signals to cancel an executable +- Alerts exchange not hard-coded in the alerts consumer + +## [5.0.1] - 2023-03-05 + +### Incompatibility + +Messages sent with this version of dl-py are not compatible with: +- dl-py v5.0.0 and earlier +- dl-py v5.1.0 and later +- dl-cpp v2.10.3 and earlier +- dl-cpp v2.10.6 and later. From d6b378cd7136869a354789313d1cea13f83b7538 Mon Sep 17 00:00:00 2001 From: "Walter C. Pettus" Date: Tue, 12 Aug 2025 12:35:15 -0400 Subject: [PATCH 72/81] Fix logging conditions for string endpoints, add absolute change condition for numeric --- dripline/core/entity.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/dripline/core/entity.py b/dripline/core/entity.py index 635b7568..fae6ae3b 100644 --- a/dripline/core/entity.py +++ b/dripline/core/entity.py @@ -69,6 +69,7 @@ def __init__(self, log_routing_key_prefix='sensor_value', log_interval=0, max_interval=0, + max_absolute_change=0, max_fractional_change=0, check_field='value_cal', calibration=None, @@ -92,10 +93,14 @@ def __init__(self, Maximum time interval between logging in seconds. Logging will take place at the next log_interval after max_interval since the last logged value. If less than log_interval, then logging values occurs every log_interval. + max_absolute_change: float + Absolute change in the numeric value that will trigger the value to be logged + If 0, then any change in the value will be logged + If < 0, then the value will always be logged (recommend instead max_interval=0) max_fractional_change: float Fractional change in the value that will trigger the value to be logged If 0, then any change in the value will be logged - If < 0, then the value will always be logged + If < 0, then the value will always be logged (recommend instead max_interval=0) check_field: string Field in the dict returned by `on_get() that's used to check for a change in the fractional value Typically is either 'value_cal' or 'value_raw' @@ -118,6 +123,7 @@ def __init__(self, self.log_interval = log_interval self._max_interval = max_interval + self._max_absolute_change = max_absolute_change self._max_fractional_change = max_fractional_change self._check_field = check_field @@ -174,20 +180,30 @@ def scheduled_log(self): result = self.on_get() try: this_value = float(result[self._check_field]) - except (TypeError, ValueError): - logger.warning(f"cannot check value change for {self.name}") - return + is_float = True + except ValueError: + is_float = False + this_value = result[self._check_field] # Various checks for log condition if self._last_log_time is None: logger.debug("Logging because this is the first logged value") elif (datetime.datetime.now(datetime.timezone.utc) - self._last_log_time).total_seconds() > self._max_interval: logger.debug("Logging because enough time has elapsed") + # Treatment of non-numeric value + elif not is_float: + if this_value != self._last_log_value: + logger.debug("Logging because the value has changed") + else: + logger.debug("No log condition met for string data, therefore not logging") + return + elif abs(self._last_log_value - this_value) > self._max_absolute_change: + logger.debug("Logging because the value has changed significantly") # this condition is |x1-x0|/(|x1+x0|/2) > max_fractional_change, but safe in case the denominator is 0 elif 2 * abs(self._last_log_value - this_value) > self._max_fractional_change * abs(self._last_log_value + this_value): - logger.debug("Logging because the value has changed significantly") + logger.debug("Logging because the value has fractionally changed significantly") else: - logger.debug("No log condition met, therefore not logging") + logger.debug("No log condition met for numeric data, therefore not logging") return self._last_log_value = this_value From 8a98c77aa951eb36560850a7d60e82ef9c190b87 Mon Sep 17 00:00:00 2001 From: "Walter C. Pettus" Date: Tue, 12 Aug 2025 12:45:35 -0400 Subject: [PATCH 73/81] Update changelog for get_reply_float in FormatEntity --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index 4da354a0..b9a25fdd 100644 --- a/changelog.md +++ b/changelog.md @@ -16,6 +16,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `max_interval`: if this time is exceeded since the last log entry, then it will be logged; if 0 (default), then logging occurs every `log_interval` - `max_fractional_change`: if the value changes by more than this since the last log entry, then it will be logged - The field that's checked for the `max_fractional_change` is given by `check_field` +- FormatEntity now includes a default regex for extracting a float from device reply that may include leading and/or trailing string characters + - `get_reply_float` option activates this functionality ### Changed From e83cb0bd95a11e0403656046c654fafc4b5879ae Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 18 Aug 2025 11:31:36 -0700 Subject: [PATCH 74/81] Add changelog info to GH Release with different release step --- .github/workflows/publish.yaml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 58a9f668..434e5e04 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -204,6 +204,16 @@ jobs: tags: ${{ steps.docker_meta_integration.outputs.tags }} platforms: linux/amd64,linux/arm64,linux/arm/v7 - - name: Release - uses: softprops/action-gh-release@v2 + - name: Release with a changelog + uses: rasmus-saks/release-a-changelog-action@v1 if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + path: 'changelog.md' + title-template: 'dripline-python v{version} -- Release Notes' + tag-template: 'v{version}' + + # This should be removed if the use of rasmus-saks/release-a-changelog-action works + #- name: Release + # uses: softprops/action-gh-release@v2 + # if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} From ef3052c7727d94bb35e9dd8d7d3cc2db86b39bd6 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 18 Aug 2025 11:48:34 -0700 Subject: [PATCH 75/81] Fixing workflow step version --- .github/workflows/publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index 434e5e04..19cabb17 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -205,7 +205,7 @@ jobs: platforms: linux/amd64,linux/arm64,linux/arm/v7 - name: Release with a changelog - uses: rasmus-saks/release-a-changelog-action@v1 + uses: rasmus-saks/release-a-changelog-action@v1.2.0 if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} with: github-token: '${{ secrets.GITHUB_TOKEN }}' From f36957e549a274338c962e40ef777eb163bee67d Mon Sep 17 00:00:00 2001 From: Paul Kolbeck <66740657+pkolbeck@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:50:20 -0700 Subject: [PATCH 76/81] Update dripline/implementations/entity_endpoints.py matches all possible formats of numbers in SCPI responses of types Real and Integer. --- dripline/implementations/entity_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dripline/implementations/entity_endpoints.py b/dripline/implementations/entity_endpoints.py index a2ff7186..7768cd50 100644 --- a/dripline/implementations/entity_endpoints.py +++ b/dripline/implementations/entity_endpoints.py @@ -141,7 +141,7 @@ def on_get(self): logger.debug(f"matches are: {matches.groupdict()}") result = matches.groupdict()['value_raw'] elif self._get_reply_float: - result = float(re.findall("[-+]?\d+\.\d+",format(result))[0]) + result = float(re.findall(r"[-+]?(?:\d*\.\d+|\d+\.?)(?:[eE][-+]?\d+)?",format(result))[0]) logger.debug(f"formatted result is {result}") return result From 4b4b7d4063c859041a7791006b6c2728b380ec27 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 20 Aug 2025 16:20:41 -0700 Subject: [PATCH 77/81] [no ci] include max-absolute-change in the changelog --- changelog.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index b9a25fdd..cfd38574 100644 --- a/changelog.md +++ b/changelog.md @@ -14,8 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The `log_interval` is now the interval with which an entity's value is checked, not necessarily logged - Whether a value is logged at the `log_interval` is controlled by: - `max_interval`: if this time is exceeded since the last log entry, then it will be logged; if 0 (default), then logging occurs every `log_interval` - - `max_fractional_change`: if the value changes by more than this since the last log entry, then it will be logged - - The field that's checked for the `max_fractional_change` is given by `check_field` + - `max_absolute_change`: if the value changes by more than this since the last log entry, then it will be logged + - `max_fractional_change`: if the value changes fractional change is more than this since the last log entry, then it will be logged + - The field that's checked for the `max_fractional_change` and `max_absolute_change` is given by `check_field` - FormatEntity now includes a default regex for extracting a float from device reply that may include leading and/or trailing string characters - `get_reply_float` option activates this functionality From 586de8bad8349154745cf88214834f29a6a5fd8c Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 25 Aug 2025 13:54:35 -0700 Subject: [PATCH 78/81] Removed get_reply_float from FormatEntity --- changelog.md | 3 --- dripline/implementations/entity_endpoints.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/changelog.md b/changelog.md index cfd38574..5a17f36f 100644 --- a/changelog.md +++ b/changelog.md @@ -17,8 +17,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `max_absolute_change`: if the value changes by more than this since the last log entry, then it will be logged - `max_fractional_change`: if the value changes fractional change is more than this since the last log entry, then it will be logged - The field that's checked for the `max_fractional_change` and `max_absolute_change` is given by `check_field` -- FormatEntity now includes a default regex for extracting a float from device reply that may include leading and/or trailing string characters - - `get_reply_float` option activates this functionality ### Changed @@ -27,7 +25,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Upgrade dl-cpp to v2.10.6 - Docker build now separates the installation of dependencies into a separate stage - ### Fixed - Postgres syntax diff --git a/dripline/implementations/entity_endpoints.py b/dripline/implementations/entity_endpoints.py index 7768cd50..fb795a82 100644 --- a/dripline/implementations/entity_endpoints.py +++ b/dripline/implementations/entity_endpoints.py @@ -96,7 +96,6 @@ class FormatEntity(Entity): def __init__(self, get_str=None, - get_reply_float=False, set_str=None, set_value_lowercase=True, set_value_map=None, @@ -109,11 +108,9 @@ def __init__(self, set_value_lowercase (bool): default option to map all string set value to .lower() **WARNING**: never set to False if using a set_value_map dict set_value_map (str||dict): inverse of calibration to map raw set value to value sent; either a dictionary or an asteval-interpretable string - get_reply_float (bool): apply special default formatting to get float return extract_raw_regex (str): regular expression search pattern applied to get return. Must be constructed with an extraction group keyed with the name "value_raw" (ie r'(?P)' ) ''' Entity.__init__(self, **kwargs) - self._get_reply_float = get_reply_float self._get_str = get_str self._set_str = set_str self._set_value_map = set_value_map @@ -140,9 +137,6 @@ def on_get(self): raise ThrowReply('service_error_invalid_value', 'device returned unparsable result, [{}] has no match to input regex [{}]'.format(first_result, self._extract_raw_regex)) logger.debug(f"matches are: {matches.groupdict()}") result = matches.groupdict()['value_raw'] - elif self._get_reply_float: - result = float(re.findall(r"[-+]?(?:\d*\.\d+|\d+\.?)(?:[eE][-+]?\d+)?",format(result))[0]) - logger.debug(f"formatted result is {result}") return result def on_set(self, value): From 08c5d5d5b4b0c3ae0d358d92de6871c0d0bec379 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Mon, 25 Aug 2025 14:57:37 -0700 Subject: [PATCH 79/81] [no ci] Added new version to changelog.md --- changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.md b/changelog.md index 5a17f36f..34411306 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [5.1.0] - 2025-08-?? + ### Added - Heartbeat Monitor (implementations.HeartbeatMonitor) From 88dd4b1c161a7c42784208e12ed3a5e879a8d68d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 26 Aug 2025 10:30:05 -0700 Subject: [PATCH 80/81] [no ci] Update version in Chart.yaml --- chart/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 80e33a20..93922655 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 ## the appVersion is used as the container image tag for the main container in the pod from the deployment ## it can be overridden by a values.yaml file in image.tag -appVersion: "v5.0.0" +appVersion: "v5.1.0" description: Deploy a dripline-python microservice name: dripline-python version: 1.1.2 From 309e9379fd889c35c65ba8836a5ced1eb09214d4 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 26 Aug 2025 10:32:30 -0700 Subject: [PATCH 81/81] [no ci] Add date to changelog entry --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 34411306..decc9406 100644 --- a/changelog.md +++ b/changelog.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [5.1.0] - 2025-08-?? +## [5.1.0] - 2025-08-26 ### Added