From cb95107af2077994f044bd3159795d32e79c53e9 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 27 Nov 2025 16:01:39 +0000 Subject: [PATCH 01/16] A quick go at saving data in csv --- waveform_controller/csv_writer.py | 44 +++++++++++++++++++++++++++++++ waveform_controller/db.py | 14 +++++++--- 2 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 waveform_controller/csv_writer.py diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py new file mode 100644 index 0000000..38c8087 --- /dev/null +++ b/waveform_controller/csv_writer.py @@ -0,0 +1,44 @@ +"""Writes a frame of waveform data to a csv file.""" + +import csv +from datetime import datetime + + +def create_file_name(sourceSystem: str, observationTime: datetime, csn: str) -> str: + """Create a unique file name based on the patient contact serial number + (csn) the date, and the source system.""" + datestring = observationTime.strftime("%Y-%m-%d") + return f"{datestring}.{csn}.{sourceSystem}.csv" + + +def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: + """Appends a frame of waveform data to a csv file (creates file if it + doesn't exist. + + :return: True if write was successful. + """ + sourceSystem = waveform_message.get("sourceSystem", None) + observationTime = waveform_message.get("observationTime", False) + + if not observationTime: + raise ValueError("waveform_message is missing observationTime") + + observation_datetime = datetime.fromtimestamp(observationTime) + + filename = create_file_name(sourceSystem, observation_datetime, csn) + with open(filename, "a") as fileout: + wv_writer = csv.writer(fileout, delimiter=",") + wv_writer.writerow( + [ + csn, + mrn, + waveform_message.get("unit", "None"), + waveform_message.get("samplingRate", "None"), + observationTime, + waveform_message.get("numericValues", "NaN").get("value", "NaN"), + ] + ) + + # TODO Check write success, and clear queue if OK. + + return False diff --git a/waveform_controller/db.py b/waveform_controller/db.py index e908cd3..24c325a 100644 --- a/waveform_controller/db.py +++ b/waveform_controller/db.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta import waveform_controller.settings as settings +import waveform_controller.csv_writer as writer class starDB: @@ -31,10 +32,13 @@ def get_row(self, location_string: str, start_datetime: str, end_datetime: str): "start_datetime": start_datetime, "end_datetime": end_datetime, } - with psycopg2.connect(self.connection_string) as db_connection: - with db_connection.cursor() as curs: - curs.execute(self.sql_query, parameters) - single_row = curs.fetchone() + try: + with psycopg2.connect(self.connection_string) as db_connection: + with db_connection.cursor() as curs: + curs.execute(self.sql_query, parameters) + single_row = curs.fetchone() + except psycopg2.errors.UndefinedTable: + raise ConnectionError("There is no table in your data base") return single_row @@ -50,7 +54,9 @@ def waveform_callback(self, ch, method, properties, body): obs_time_str = observation_time.strftime("%Y-%m-%d:%H:%M:%S") start_time_str = start_time.strftime("%Y-%m-%d:%H:%M:%S") matched_mrn = self.get_row(location_string, start_time_str, obs_time_str) + # print(f"Received a waveform message {data.get('observationTime', 'NAT')}") print( f"Received a waveform message from {location_string} at {obs_time_str} with matching mrn = {matched_mrn}" ) + writer.write_frame(data, matched_mrn[2], matched_mrn[0]) From f99a192f5688402347767069e5161fd89c1840e8 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Fri, 28 Nov 2025 16:37:01 +0000 Subject: [PATCH 02/16] Dockerise and create entry point --- .dockerignore | 2 ++ .gitignore | 3 +++ Dockerfile | 10 ++++++++++ README.md | 2 +- docker-compose.yml | 11 +++++++++++ pyproject.toml | 3 +++ 6 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..18bfee6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.venv + diff --git a/.gitignore b/.gitignore index 505a3b1..0635af1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ wheels/ # Virtual environments .venv + +# IDEs +.idea/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6aea537 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.14-slim-bookworm +LABEL authors="Stephen Thompson, Jeremy Stein" +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ +WORKDIR /app +ARG UVCACHE=/root/.cache/uv +COPY pyproject.toml uv.lock* /app/ +RUN --mount=type=cache,target=${UVCACHE} uv pip install --system . +COPY . /app/ +RUN --mount=type=cache,target=${UVCACHE} uv pip install --system . +CMD ["emap-extract-waveform"] \ No newline at end of file diff --git a/README.md b/README.md index 2d63f1e..8d010d7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Once configured you can start it with emap docker up -d ``` -## 2 Install and deploy waveform reader using uv +## 2 Install and deploy waveform controller using uv ``` uv venv .waveform-controller diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b8635be --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +services: + waveform-controller: + build: + context: . + dockerfile: Dockerfile + args: + HTTP_PROXY: ${HTTP_PROXY} + http_proxy: ${http_proxy} + HTTPS_PROXY: ${HTTPS_PROXY} + https_proxy: ${https_proxy} + diff --git a/pyproject.toml b/pyproject.toml index f013aad..fc9c7bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,6 @@ dependencies = [ "pre-commit>=4.5.0", "psycopg2-binary>=2.9.11", ] + +[project.scripts] +emap-extract-waveform = "waveform_controller.controller:receiver" From 1fa829cc8750999268c9ead8c47982724f1b8946 Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Fri, 28 Nov 2025 18:44:48 +0000 Subject: [PATCH 03/16] Get secrets from the environment, not source code --- .dockerignore | 2 ++ config/.gitignore | 2 ++ docker-compose.yml | 4 +++- settings.env.EXAMPLE | 13 +++++++++++++ waveform_controller/settings.py | 30 +++++++++++++++++++----------- 5 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 config/.gitignore create mode 100644 settings.env.EXAMPLE diff --git a/.dockerignore b/.dockerignore index 18bfee6..c61a3c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,4 @@ .venv +# secrets +config diff --git a/config/.gitignore b/config/.gitignore new file mode 100644 index 0000000..a5f977a --- /dev/null +++ b/config/.gitignore @@ -0,0 +1,2 @@ +# secrets go here, do not put in git +* diff --git a/docker-compose.yml b/docker-compose.yml index b8635be..e5d87cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,4 +8,6 @@ services: http_proxy: ${http_proxy} HTTPS_PROXY: ${HTTPS_PROXY} https_proxy: ${https_proxy} - + # ideally we'd use docker secrets but it's not enabled currently + env_file: + - ./config/settings.env diff --git a/settings.env.EXAMPLE b/settings.env.EXAMPLE new file mode 100644 index 0000000..46ef5fe --- /dev/null +++ b/settings.env.EXAMPLE @@ -0,0 +1,13 @@ +# This is an EXAMPLE file, do not put real secrets in here. +# Copy it to ./config/settings.env and then DELETE THIS COMMENT. +UDS_DBNAME="fakeuds" +UDS_USERNAME="inform_user" +UDS_PASSWORD="inform" +UDS_HOST="localhost" +UDS_PORT="5433" +SCHEMA_NAME="star_dev" +RABBITMQ_USERNAME="my_name" +RABBITMQ_PASSWORD="my_pw" +RABBITMQ_HOST="localhost" +RABBITMQ_PORT=5672 +RABBITMQ_QUEUE="waveform" \ No newline at end of file diff --git a/waveform_controller/settings.py b/waveform_controller/settings.py index a6e439e..406dbf6 100644 --- a/waveform_controller/settings.py +++ b/waveform_controller/settings.py @@ -1,12 +1,20 @@ -UDS_DBNAME = "fakeuds" -UDS_USERNAME = "inform_user" -UDS_PASSWORD = "inform" -UDS_HOST = "localhost" -UDS_PORT = "5433" -SCHEMA_NAME = "star_dev" +import os +from pathlib import Path -RABBITMQ_USERNAME = "my_name" -RABBITMQ_PASSWORD = "my_pw" -RABBITMQ_HOST = "localhost" -RABBITMQ_PORT = 5672 -RABBITMQ_QUEUE = "waveform" +def get_from_env(env_var, setting_name=None): + if setting_name is None: + setting_name = env_var + globals()[setting_name] = os.environ.get(env_var) + +# read env vars into settings variables +get_from_env("UDS_DBNAME") +get_from_env("UDS_USERNAME") +get_from_env("UDS_PASSWORD") +get_from_env("UDS_HOST") +get_from_env("UDS_PORT") +get_from_env("SCHEMA_NAME") +get_from_env("RABBITMQ_USERNAME") +get_from_env("RABBITMQ_PASSWORD") +get_from_env("RABBITMQ_HOST") +get_from_env("RABBITMQ_PORT") +get_from_env("RABBITMQ_QUEUE") From 7fb4252051450dc6fbc78990c240690e4ec5d46c Mon Sep 17 00:00:00 2001 From: Jeremy Stein Date: Fri, 28 Nov 2025 19:11:56 +0000 Subject: [PATCH 04/16] Update docs --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8d010d7..d167d7c 100644 --- a/README.md +++ b/README.md @@ -36,16 +36,16 @@ emap docker up -d ## 2 Install and deploy waveform controller using uv ``` -uv venv .waveform-controller -source .waveform-controller/bin/activate -uv pip install . --active +cd .../waveform-controller +docker compose build +docker compose up -d ``` ## 3 Check if it's working -If successful you should be able to run the demo script and see waveform messaged dumped to the terminal. +Waveform messages should be dumped to the log file: ``` -python waveform_controller.py +docker compose logs ``` # Developing From dc2bcbddaedb32ee3cfc86ab62a488783b393e9f Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 1 Dec 2025 15:28:01 +0000 Subject: [PATCH 05/16] Put units in file string to make it more descriptive in the absence of source information --- waveform_controller/csv_writer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index 38c8087..7a8cd35 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -4,11 +4,13 @@ from datetime import datetime -def create_file_name(sourceSystem: str, observationTime: datetime, csn: str) -> str: +def create_file_name( + sourceSystem: str, observationTime: datetime, csn: str, units: str +) -> str: """Create a unique file name based on the patient contact serial number (csn) the date, and the source system.""" datestring = observationTime.strftime("%Y-%m-%d") - return f"{datestring}.{csn}.{sourceSystem}.csv" + return f"waveform_data/{datestring}.{csn}.{sourceSystem}.{units}.csv" def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: @@ -24,15 +26,16 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: raise ValueError("waveform_message is missing observationTime") observation_datetime = datetime.fromtimestamp(observationTime) + units = waveform_message.get("unit", "None") - filename = create_file_name(sourceSystem, observation_datetime, csn) + filename = create_file_name(sourceSystem, observation_datetime, csn, units) with open(filename, "a") as fileout: wv_writer = csv.writer(fileout, delimiter=",") wv_writer.writerow( [ csn, mrn, - waveform_message.get("unit", "None"), + units, waveform_message.get("samplingRate", "None"), observationTime, waveform_message.get("numericValues", "NaN").get("value", "NaN"), From a42d801e11d666d4c47d5502355602a480f7d572 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 1 Dec 2025 15:41:34 +0000 Subject: [PATCH 06/16] With data directory --- waveform_data/empty.csv | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 waveform_data/empty.csv diff --git a/waveform_data/empty.csv b/waveform_data/empty.csv new file mode 100644 index 0000000..e69de29 From 10c5e6c71ed341ffdd109edb5d213f8efec9cf5d Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Mon, 1 Dec 2025 16:26:26 +0000 Subject: [PATCH 07/16] Make data directory as required --- waveform_controller/csv_writer.py | 2 ++ waveform_data/empty.csv | 0 2 files changed, 2 insertions(+) delete mode 100644 waveform_data/empty.csv diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index 7a8cd35..3488256 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -2,6 +2,7 @@ import csv from datetime import datetime +from pathlib import Path def create_file_name( @@ -28,6 +29,7 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: observation_datetime = datetime.fromtimestamp(observationTime) units = waveform_message.get("unit", "None") + Path("waveform_data").mkdir(exist_ok=True) filename = create_file_name(sourceSystem, observation_datetime, csn, units) with open(filename, "a") as fileout: wv_writer = csv.writer(fileout, delimiter=",") diff --git a/waveform_data/empty.csv b/waveform_data/empty.csv deleted file mode 100644 index e69de29..0000000 From fb55f9445887f6dac889380b20a6ac866ef650d9 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 09:25:13 +0000 Subject: [PATCH 08/16] Added path and unit to csv filenname --- waveform_controller/csv_writer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index 3488256..ee3d665 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -11,7 +11,7 @@ def create_file_name( """Create a unique file name based on the patient contact serial number (csn) the date, and the source system.""" datestring = observationTime.strftime("%Y-%m-%d") - return f"waveform_data/{datestring}.{csn}.{sourceSystem}.{units}.csv" + return f"{datestring}.{csn}.{sourceSystem}.{units}.csv" def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: @@ -29,8 +29,12 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: observation_datetime = datetime.fromtimestamp(observationTime) units = waveform_message.get("unit", "None") - Path("waveform_data").mkdir(exist_ok=True) - filename = create_file_name(sourceSystem, observation_datetime, csn, units) + out_path = "waveform_data/" + Path(out_path).mkdir(exist_ok=True) + + filename = out_path + create_file_name( + sourceSystem, observation_datetime, csn, units + ) with open(filename, "a") as fileout: wv_writer = csv.writer(fileout, delimiter=",") wv_writer.writerow( From 4f22c86161d3341b20212524628d9219ba2d92f2 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 09:44:37 +0000 Subject: [PATCH 09/16] Return true if we reach the end of the function, I think it's safe to assume that if we get that far we've written the data --- waveform_controller/csv_writer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index ee3d665..4b95d7f 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -48,6 +48,4 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: ] ) - # TODO Check write success, and clear queue if OK. - - return False + return True From a25b7a1fc6f7ed250d2484e2a46cca3dff371595 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 11:47:29 +0000 Subject: [PATCH 10/16] Updated README to describe output --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2d63f1e..5226770 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,18 @@ uv pip install . --active ## 3 Check if it's working -If successful you should be able to run the demo script and see waveform messaged dumped to the terminal. +If successful you should be able to run the waveform controller with. ``` python waveform_controller.py ``` +Running the controller will create a `waveform_data` directory where waveform messages +matched to Contact Serial Number (CSN) will be saved as csv files, each containing data for +one calender day, as +`YYYY-MM-DD.CSN.sourceName.units.csv` + +Each row of the csv will contain + +`csn, mrn, units, samplingRate, observationTime, waveformData` # Developing See [developing docs](docs/develop.md) From cdf681e826d5f22158f4d443ac575e24f44f1f48 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 12:01:46 +0000 Subject: [PATCH 11/16] Add a docker volume to store the data --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e5d87cf..9a3de13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,4 +10,6 @@ services: https_proxy: ${https_proxy} # ideally we'd use docker secrets but it's not enabled currently env_file: - - ./config/settings.env + - ./config/settings.env + volumes: + - ./waveform_data:/app/waveform_data From 818e31deff34259c0849f5abf20c9972076842d3 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 12:20:30 +0000 Subject: [PATCH 12/16] Added ack to rabbitQM after file write --- waveform_controller/db.py | 17 +++++++---------- waveform_controller/settings.py | 3 ++- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/waveform_controller/db.py b/waveform_controller/db.py index 24c325a..c03f6f7 100644 --- a/waveform_controller/db.py +++ b/waveform_controller/db.py @@ -10,11 +10,11 @@ class starDB: sql_query: str = "" connection_string: str = "dbname={} user={} password={} host={} port={}".format( - settings.UDS_DBNAME, - settings.UDS_USERNAME, - settings.UDS_PASSWORD, - settings.UDS_HOST, - settings.UDS_PORT, + settings.UDS_DBNAME, # type:ignore + settings.UDS_USERNAME, # type:ignore + settings.UDS_PASSWORD, # type:ignore + settings.UDS_HOST, # type:ignore + settings.UDS_PORT, # type:ignore ) def init_query(self): @@ -55,8 +55,5 @@ def waveform_callback(self, ch, method, properties, body): start_time_str = start_time.strftime("%Y-%m-%d:%H:%M:%S") matched_mrn = self.get_row(location_string, start_time_str, obs_time_str) - # print(f"Received a waveform message {data.get('observationTime', 'NAT')}") - print( - f"Received a waveform message from {location_string} at {obs_time_str} with matching mrn = {matched_mrn}" - ) - writer.write_frame(data, matched_mrn[2], matched_mrn[0]) + if writer.write_frame(data, matched_mrn[2], matched_mrn[0]): + ch.basic_ack(method.delivery_tag) diff --git a/waveform_controller/settings.py b/waveform_controller/settings.py index 406dbf6..2ec92c9 100644 --- a/waveform_controller/settings.py +++ b/waveform_controller/settings.py @@ -1,11 +1,12 @@ import os -from pathlib import Path + def get_from_env(env_var, setting_name=None): if setting_name is None: setting_name = env_var globals()[setting_name] = os.environ.get(env_var) + # read env vars into settings variables get_from_env("UDS_DBNAME") get_from_env("UDS_USERNAME") From f76982671d0c2ec52c3f6f958e4182779eef0a3b Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Tue, 2 Dec 2025 12:30:30 +0000 Subject: [PATCH 13/16] Style fix --- .dockerignore | 1 - Dockerfile | 2 +- settings.env.EXAMPLE | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index c61a3c9..c1fb6ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ .venv # secrets config - diff --git a/Dockerfile b/Dockerfile index 6aea537..ecae0ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,4 +7,4 @@ COPY pyproject.toml uv.lock* /app/ RUN --mount=type=cache,target=${UVCACHE} uv pip install --system . COPY . /app/ RUN --mount=type=cache,target=${UVCACHE} uv pip install --system . -CMD ["emap-extract-waveform"] \ No newline at end of file +CMD ["emap-extract-waveform"] diff --git a/settings.env.EXAMPLE b/settings.env.EXAMPLE index 46ef5fe..35027b6 100644 --- a/settings.env.EXAMPLE +++ b/settings.env.EXAMPLE @@ -10,4 +10,4 @@ RABBITMQ_USERNAME="my_name" RABBITMQ_PASSWORD="my_pw" RABBITMQ_HOST="localhost" RABBITMQ_PORT=5672 -RABBITMQ_QUEUE="waveform" \ No newline at end of file +RABBITMQ_QUEUE="waveform" From 081a9f992b744e1513f64df5e49ee56be87280ec Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 3 Dec 2025 16:48:03 +0000 Subject: [PATCH 14/16] renamed save directory and moved it to parent directory to avoid problems when using setup tools to install --- README.md | 41 ++++++++++++++++++------------- docker-compose.yml | 2 +- waveform_controller/csv_writer.py | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index dc0ffd8..c7e5e27 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,33 @@ -A controller for reading waveform data from a rabbitmq queue and processing it. +A controller for reading waveform data from a rabbitmq queue and processing it. # Running the Code ## 1 Install and deploy EMAP -Follow the emap development [instructions](https://github.com/SAFEHR-data/emap/blob/main/docs/dev/core.md#deploying-a-live-version "Instructions for deploying a live version of EMAP") configure and deploy a version of EMAP. To run a local version you'll need to set +Follow the emap development [instructions](https://github.com/SAFEHR-data/emap/blob/main/docs/dev/core.md#deploying-a-live-version "Instructions for deploying a live version of EMAP") configure and deploy a version of EMAP. To run a local version you'll need to set ``` fake_uds: - enable_fake_uds: true + enable_fake_uds: true uds: - UDS_JDBC_URL: jdbc:postgresql://fakeuds:5432/fakeuds + UDS_JDBC_URL: jdbc:postgresql://fakeuds:5432/fakeuds ``` and configure and synthetic waveform generator ``` -waveform: - enable_waveform: true - enable_waveform_generator: true - CORE_WAVEFORM_RETENTION_HOURS: 24 +waveform: + enable_waveform: true + enable_waveform_generator: true + CORE_WAVEFORM_RETENTION_HOURS: 24 WAVEFORM_HL7_SOURCE_ADDRESS_ALLOW_LIST: ALL - WAVEFORM_HL7_TEST_DUMP_FILE: "" + WAVEFORM_HL7_TEST_DUMP_FILE: "" WAVEFORM_HL7_SAVE_DIRECTORY: "/waveform-saved-messages" - WAVEFORM_SYNTHETIC_NUM_PATIENTS: 2 - WAVEFORM_SYNTHETIC_WARP_FACTOR:1 - WAVEFORM_SYNTHETIC_START_DATETIME: "2024-01-02T12:00:00Z" - WAVEFORM_SYNTHETIC_END_DATETIME: "2024-01-03T12:00:00Z" + WAVEFORM_SYNTHETIC_NUM_PATIENTS: 2 + WAVEFORM_SYNTHETIC_WARP_FACTOR:1 + WAVEFORM_SYNTHETIC_START_DATETIME: "2024-01-02T12:00:00Z" + WAVEFORM_SYNTHETIC_END_DATETIME: "2024-01-03T12:00:00Z" ``` -Once configured you can start it with +Once configured you can start it with ``` emap docker up -d @@ -35,12 +35,19 @@ emap docker up -d ## 2 Install and deploy waveform controller using docker -Configuration, copy the configuration file to the config directory and edit +Configuration, copy the configuration file to the config directory and edit as necessary. ``` cp settings.env.EXAMPLE config/settings.env ``` +If it doesn't already exist you should create a directory named +`waveform-export` in the parent directory to store the saved waveform +messages. + +``` +mkdir ../waveform-export +``` Build and start the controller with docker ``` @@ -51,8 +58,8 @@ docker compose up -d ## 3 Check if it's working -Running the controller will create a `waveform_data` directory where waveform messages -matched to Contact Serial Number (CSN) will be saved as csv files, each containing data for +Running the controller will save (to `../waveform-export`) waveform messages +matched to Contact Serial Number (CSN) as csv files, each containing data for one calender day, as `YYYY-MM-DD.CSN.sourceName.units.csv` diff --git a/docker-compose.yml b/docker-compose.yml index 9a3de13..a49869c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,4 +12,4 @@ services: env_file: - ./config/settings.env volumes: - - ./waveform_data:/app/waveform_data + - ../waveform-export:/app/waveform-export diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index 4b95d7f..f65b543 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -29,7 +29,7 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: observation_datetime = datetime.fromtimestamp(observationTime) units = waveform_message.get("unit", "None") - out_path = "waveform_data/" + out_path = "waveform-export/" Path(out_path).mkdir(exist_ok=True) filename = out_path + create_file_name( From 54cee72b46a867b461c381fb34f9cefb83400259 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Wed, 3 Dec 2025 16:56:52 +0000 Subject: [PATCH 15/16] Reminder about secrets. Co-authored-by: Jeremy Stein --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c7e5e27..7de37b2 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ emap docker up -d ## 2 Install and deploy waveform controller using docker Configuration, copy the configuration file to the config directory and edit -as necessary. +as necessary. Remove the comment telling you not to put secrets in it. ``` cp settings.env.EXAMPLE config/settings.env From 08d5e94d4b4469ad2af960e00f9944ff2fb0da86 Mon Sep 17 00:00:00 2001 From: Stephen Thompson Date: Thu, 4 Dec 2025 10:15:22 +0000 Subject: [PATCH 16/16] Change missing values to empty string and avoid calling get on missing data --- waveform_controller/csv_writer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/waveform_controller/csv_writer.py b/waveform_controller/csv_writer.py index f65b543..3b8a9b1 100644 --- a/waveform_controller/csv_writer.py +++ b/waveform_controller/csv_writer.py @@ -27,7 +27,7 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: raise ValueError("waveform_message is missing observationTime") observation_datetime = datetime.fromtimestamp(observationTime) - units = waveform_message.get("unit", "None") + units = waveform_message.get("unit", "") out_path = "waveform-export/" Path(out_path).mkdir(exist_ok=True) @@ -37,14 +37,18 @@ def write_frame(waveform_message: dict, csn: str, mrn: str) -> bool: ) with open(filename, "a") as fileout: wv_writer = csv.writer(fileout, delimiter=",") + waveform_data = waveform_message.get("numericValues", "") + if waveform_data != "": + waveform_data = waveform_data.get("value", "") + wv_writer.writerow( [ csn, mrn, units, - waveform_message.get("samplingRate", "None"), + waveform_message.get("samplingRate", ""), observationTime, - waveform_message.get("numericValues", "NaN").get("value", "NaN"), + waveform_data, ] )