From 8a47894c2a9c911550c941ba3cf185c345915898 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Tue, 15 Apr 2025 22:43:54 +0200 Subject: [PATCH 01/10] test_data --- tools/test_data/efc.pump_timed.json | 760 ++++++++++++++++++++++++++++ 1 file changed, 760 insertions(+) create mode 100644 tools/test_data/efc.pump_timed.json diff --git a/tools/test_data/efc.pump_timed.json b/tools/test_data/efc.pump_timed.json new file mode 100644 index 0000000..6a94c86 --- /dev/null +++ b/tools/test_data/efc.pump_timed.json @@ -0,0 +1,760 @@ +[{ + "ts": { + "$date": "2025-03-22T16:32:49.626Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67dee6312b9ab34cba6eab58" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:46:01.523Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67dee9472b9ab34cba6eab59" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:46:09.958Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67dee9522b9ab34cba6eab5a" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:47:42.684Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67dee9ae2b9ab34cba6eab5b" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:48:03.282Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67dee9c32b9ab34cba6eab5c" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:48:10.056Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67dee9ca2b9ab34cba6eab5d" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:49:55.583Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67deea332b9ab34cba6eab5e" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:55:11.941Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67deeb702b9ab34cba6eab5f" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:56:06.804Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67deeba62b9ab34cba6eab60" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:58:06.087Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67deec1d2b9ab34cba6eab61" + } +}, +{ + "ts": { + "$date": "2025-03-22T16:59:06.773Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67deec5a2b9ab34cba6eab62" + } +}, +{ + "ts": { + "$date": "2025-03-22T17:01:06.049Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67deecd12b9ab34cba6eab63" + } +}, +{ + "ts": { + "$date": "2025-04-13T07:44:00.169Z" + }, + "id": 0, + "_id": { + "$oid": "67fb6b3feb454e4cde57df16" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T12:36:32.928Z" + }, + "id": 0, + "_id": { + "$oid": "67fbafd1eb454e4cde57df17" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T16:12:13.790Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe258eb454e4cde57df18" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T16:19:33.053Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe40feb454e4cde57df19" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T16:19:33.062Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe40feb454e4cde57df1a" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T16:26:53.057Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe5c6eb454e4cde57df1b" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T16:26:53.066Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe5c6eb454e4cde57df1c" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T16:34:13.012Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe77deb454e4cde57df1d" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T16:34:13.021Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe77deb454e4cde57df1e" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T16:36:06.847Z" + }, + "id": 0, + "_id": { + "$oid": "67fbe7f7eb454e4cde57df1f" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T17:00:05.684Z" + }, + "id": 0, + "_id": { + "$oid": "67fbed92eb454e4cde57df20" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T17:07:45.061Z" + }, + "id": 0, + "_id": { + "$oid": "67fbef5ceb454e4cde57df21" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T19:00:07.797Z" + }, + "id": 0, + "_id": { + "$oid": "67fc09b4eb454e4cde57df22" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T19:07:47.087Z" + }, + "id": 0, + "_id": { + "$oid": "67fc0b7feb454e4cde57df23" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-13T21:00:00.726Z" + }, + "id": 0, + "_id": { + "$oid": "67fc25cdeb454e4cde57df24" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-13T21:07:40.059Z" + }, + "id": 0, + "_id": { + "$oid": "67fc2797eb454e4cde57df25" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T07:00:08.476Z" + }, + "id": 0, + "_id": { + "$oid": "67fcb274eb454e4cde57df26" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T07:07:48.031Z" + }, + "id": 0, + "_id": { + "$oid": "67fcb43feb454e4cde57df27" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T09:00:05.136Z" + }, + "id": 0, + "_id": { + "$oid": "67fcce91eb454e4cde57df28" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T09:07:45.061Z" + }, + "id": 0, + "_id": { + "$oid": "67fcd05ceb454e4cde57df29" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T11:00:09.099Z" + }, + "id": 0, + "_id": { + "$oid": "67fceab5eb454e4cde57df2a" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T11:07:49.081Z" + }, + "id": 0, + "_id": { + "$oid": "67fcec80eb454e4cde57df2b" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T12:00:06.656Z" + }, + "id": 0, + "_id": { + "$oid": "67fcf8c2eb454e4cde57df2c" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T12:07:46.008Z" + }, + "id": 0, + "_id": { + "$oid": "67fcfa8deb454e4cde57df2d" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T13:00:03.507Z" + }, + "id": 0, + "_id": { + "$oid": "67fd06cfeb454e4cde57df2e" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T13:07:43.074Z" + }, + "id": 0, + "_id": { + "$oid": "67fd0899eb454e4cde57df2f" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T14:00:00.248Z" + }, + "id": 0, + "_id": { + "$oid": "67fd14ddeb454e4cde57df30" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T14:07:40.062Z" + }, + "id": 0, + "_id": { + "$oid": "67fd16a8eb454e4cde57df31" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T15:00:08.470Z" + }, + "id": 0, + "_id": { + "$oid": "67fd22f5eb454e4cde57df32" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T15:07:48.088Z" + }, + "id": 0, + "_id": { + "$oid": "67fd24c2eb454e4cde57df33" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T15:07:48.088Z" + }, + "id": 0, + "_id": { + "$oid": "67fd24c2eb454e4cde57df34" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T16:00:06.882Z" + }, + "id": 0, + "_id": { + "$oid": "67fd3103eb454e4cde57df35" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T16:07:46.065Z" + }, + "id": 0, + "_id": { + "$oid": "67fd32cdeb454e4cde57df36" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T17:00:05.733Z" + }, + "id": 0, + "_id": { + "$oid": "67fd3f12eb454e4cde57df37" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T17:07:45.083Z" + }, + "id": 0, + "_id": { + "$oid": "67fd40ddeb454e4cde57df38" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T19:00:03.499Z" + }, + "id": 0, + "_id": { + "$oid": "67fd5b2feb454e4cde57df39" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T19:07:43.057Z" + }, + "id": 0, + "_id": { + "$oid": "67fd5cfaeb454e4cde57df3a" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-14T21:00:07.591Z" + }, + "id": 0, + "_id": { + "$oid": "67fd7754eb454e4cde57df3b" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-14T21:07:47.031Z" + }, + "id": 0, + "_id": { + "$oid": "67fd791eeb454e4cde57df3c" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-15T07:00:08.748Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe03f5f481ffe83f1d061d" + } +}, +{ + "ts": { + "$date": "2025-04-15T07:07:48.061Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe05c0f481ffe83f1d061e" + } +}, +{ + "ts": { + "$date": "2025-04-15T09:00:04.448Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe2010f481ffe83f1d061f" + } +}, +{ + "ts": { + "$date": "2025-04-15T09:07:44.066Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe21dbf481ffe83f1d0620" + } +}, +{ + "ts": { + "$date": "2025-04-15T11:00:09.529Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe3c35f481ffe83f1d0621" + } +}, +{ + "ts": { + "$date": "2025-04-15T11:07:49.058Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe3dfff481ffe83f1d0622" + } +}, +{ + "ts": { + "$date": "2025-04-15T12:00:05.542Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe4a41f481ffe83f1d0623" + } +}, +{ + "ts": { + "$date": "2025-04-15T12:07:45.088Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe4c0bf481ffe83f1d0624" + } +}, +{ + "ts": { + "$date": "2025-04-15T13:00:01.701Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe584bf481ffe83f1d0625" + } +}, +{ + "ts": { + "$date": "2025-04-15T13:07:41.057Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe5a16f481ffe83f1d0626" + } +}, +{ + "ts": { + "$date": "2025-04-15T14:00:07.101Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe6662f481ffe83f1d0627" + } +}, +{ + "ts": { + "$date": "2025-04-15T14:07:47.096Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe682df481ffe83f1d0628" + } +}, +{ + "ts": { + "$date": "2025-04-15T15:00:06.140Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe7472f481ffe83f1d0629" + } +}, +{ + "ts": { + "$date": "2025-04-15T15:07:46.079Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe763df481ffe83f1d062a" + } +}, +{ + "ts": { + "$date": "2025-04-15T16:00:06.074Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe8281f481ffe83f1d062b" + } +}, +{ + "ts": { + "$date": "2025-04-15T16:07:46.035Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe844cf481ffe83f1d062c" + } +}, +{ + "ts": { + "$date": "2025-04-15T17:00:03.631Z" + }, + "id": 0, + "status": "start", + "_id": { + "$oid": "67fe908ff481ffe83f1d062d" + } +}, +{ + "ts": { + "$date": "2025-04-15T17:07:43.063Z" + }, + "id": 0, + "status": "stop", + "_id": { + "$oid": "67fe925af481ffe83f1d062e" + } +}, +{ + "ts": { + "$date": "2025-04-15T18:13:08.384Z" + }, + "id": 0, + "_id": { + "$oid": "67fea1afd717a2ba25fa68c1" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-15T18:24:10.108Z" + }, + "id": 0, + "_id": { + "$oid": "67fea44ad717a2ba25fa68c2" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-15T19:00:02.661Z" + }, + "id": 0, + "_id": { + "$oid": "67feacaed717a2ba25fa68c3" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-15T19:08:22.048Z" + }, + "id": 0, + "_id": { + "$oid": "67feaea1d717a2ba25fa68c4" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-15T19:56:00.542Z" + }, + "id": 0, + "_id": { + "$oid": "67feb9ccd717a2ba25fa68c5" + }, + "status": "start" +}, +{ + "ts": { + "$date": "2025-04-15T20:04:20.029Z" + }, + "id": 0, + "_id": { + "$oid": "67febbbed717a2ba25fa68c6" + }, + "status": "stop" +}, +{ + "ts": { + "$date": "2025-04-15T20:07:17.297Z" + }, + "id": 0, + "_id": { + "$oid": "67febc75d717a2ba25fa68c7" + }, + "status": "stop" +}] \ No newline at end of file From f4503f489ccbcb96883515e6e70810139f14b96e Mon Sep 17 00:00:00 2001 From: phofmeier Date: Tue, 15 Apr 2025 21:44:33 +0000 Subject: [PATCH 02/10] add pump times data viewer --- Dockerfile | 2 +- pyproject.toml | 3 +- src/ebb_flow_manager/config.py | 1 + src/ebb_flow_manager/database/database.py | 8 + src/ebb_flow_manager/database/mongo_db.py | 10 + .../ebb_flow_manager_data_viewer.py | 69 ++ tools/publish_test_data.py | 17 + tools/test_data/efc-pump_timed.json | 382 +++++++++ tools/test_data/efc.pump_timed.json | 760 ------------------ 9 files changed, 490 insertions(+), 762 deletions(-) create mode 100644 src/ebb_flow_manager/ebb_flow_manager_data_viewer.py create mode 100644 tools/test_data/efc-pump_timed.json delete mode 100644 tools/test_data/efc.pump_timed.json diff --git a/Dockerfile b/Dockerfile index 1613af6..8dedfe7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ WORKDIR /home/app COPY . . RUN pip install . -CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py" ] +CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py", "src/ebb_flow_manager/ebb_flow_manager_data_viewer.py" ] diff --git a/pyproject.toml b/pyproject.toml index b1b2cde..91fe232 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "paho-mqtt >= 2.1.0", "pymongo[srv] >= 4.8.0", "panel >= 1.6.0", - "watchfiles >= 1.0.3" + "watchfiles >= 1.0.3", + "plotly >= 6.0.1", ] dynamic = ["version"] diff --git a/src/ebb_flow_manager/config.py b/src/ebb_flow_manager/config.py index debbed9..8e350cf 100644 --- a/src/ebb_flow_manager/config.py +++ b/src/ebb_flow_manager/config.py @@ -20,6 +20,7 @@ def __init__(self, filename: str) -> None: "collection_config_name": "config_static", "collection_config_template_name": "config_template", "collection_used_template_name": "used_template", + "collection_pump_time_name": "pump_timed", "id_field_name": "id", }, "logging": { diff --git a/src/ebb_flow_manager/database/database.py b/src/ebb_flow_manager/database/database.py index f109644..8a85453 100644 --- a/src/ebb_flow_manager/database/database.py +++ b/src/ebb_flow_manager/database/database.py @@ -123,3 +123,11 @@ def set_used_template_of(self, id: int, template_name: str): template_name (str): name of the template used by this controller. """ self.db_impl.set_used_template_of(id, template_name.strip()) + + def get_pump_time_data(self) -> list[dict]: + """Get all pump time data. + + Returns: + list[dict]: List of dicts containing the pump time data. + """ + return self.db_impl.get_pump_time_data() diff --git a/src/ebb_flow_manager/database/mongo_db.py b/src/ebb_flow_manager/database/mongo_db.py index cf0464c..923dbbc 100644 --- a/src/ebb_flow_manager/database/mongo_db.py +++ b/src/ebb_flow_manager/database/mongo_db.py @@ -173,3 +173,13 @@ def get_all_data_from(self, database: str, collection: str) -> list[dict]: def get_all_timed_data_from(self, database: str, collection: str): return list(self.client[database][collection].find({}, {})) + + def get_pump_time_data(self) -> list[dict]: + """Get all pump time data. + + Returns: + list[dict]: List of dicts containing the pump time data. + """ + return self.get_all_timed_data_from( + self.config["database_name"], self.config["collection_pump_time_name"] + ) diff --git a/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py new file mode 100644 index 0000000..d7ebeed --- /dev/null +++ b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py @@ -0,0 +1,69 @@ +import panel as pn +import plotly.graph_objs as go + +from ebb_flow_manager.config import Config +from ebb_flow_manager.database.database import Database +from ebb_flow_manager.ebb_flow_manager_app import init_logger + +pn.extension(design="material") + +pn.extension("plotly") + + +def start_serve() -> pn.panel: + """Start the panel server and display the data viewer. + + Returns: + pn.panel: panel object containing the data viewer. + """ + config = Config("config.yml") + logger = init_logger(config.get("logging")) + logger.debug("start app") + db = Database(config.get("database")) + + print(db.get_pump_time_data()) + data = db.get_pump_time_data() + sorted_data = sorted(data, key=lambda x: x["ts"]) + timestamps = [data["ts"] for data in sorted_data] + values = [data["status"] for data in sorted_data] + + figure = go.Figure() + figure.add_trace( + go.Scatter( + x=timestamps, + y=values, + mode="lines+markers", + line={"shape": "hv"}, + name="Pump Status", + ) + ) + figure.update_layout( + title="Pump Status Over Time", + xaxis_title="Timestamp", + yaxis_title="Status", + xaxis_tickformat="%Y-%m-%d %H:%M:%S", + xaxis_tickangle=-45, + ) + figure.update_layout( + autosize=True, + margin=dict(l=20, r=20, t=20, b=20), + height=400, + width=600, + ) + plotly_pane = pn.pane.Plotly(figure) + template = pn.template.BootstrapTemplate( + title="Ebb Flow Manager - Data Viewer", + main=[plotly_pane], + ) + return template + + +def main(): + pn.serve(start_serve(), admin=True) + + +if __name__ == "__main__": + # start_serve().servable() + main() +elif __name__.startswith("bokeh_app"): + start_serve().servable() diff --git a/tools/publish_test_data.py b/tools/publish_test_data.py index 40d2448..5f38cd2 100644 --- a/tools/publish_test_data.py +++ b/tools/publish_test_data.py @@ -1,3 +1,4 @@ +import datetime import json import os @@ -36,6 +37,22 @@ def get_all_files(path="."): data, upsert=True, ) + elif type_name == "timed": + if collection not in client[database].list_collection_names(): + # Insert new collection + client[database].create_collection( + collection, + timeseries={ + "timeField": "ts", + "metaField": "id", + }, + ) + for data in all_data: + datetime_ts = datetime.datetime.fromisoformat(data["ts"]) + data["ts"] = datetime_ts + client[database][collection].insert_one( + data, + ) else: for data in all_data: client[database][collection].replace_one( diff --git a/tools/test_data/efc-pump_timed.json b/tools/test_data/efc-pump_timed.json new file mode 100644 index 0000000..3339c9c --- /dev/null +++ b/tools/test_data/efc-pump_timed.json @@ -0,0 +1,382 @@ +[ + { + "ts": "2025-03-22T16:32:49.626Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:46:01.523Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:46:09.958Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:47:42.684Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:48:03.282Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:48:10.056Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:49:55.583Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:55:11.941Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:56:06.804Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:58:06.087Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:59:06.773Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T17:01:06.049Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T07:44:00.169Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T12:36:32.928Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:12:13.790Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:19:33.053Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:19:33.062Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:26:53.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:26:53.066Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:34:13.012Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:34:13.021Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:36:06.847Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T17:00:05.684Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T17:07:45.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T19:00:07.797Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T19:07:47.087Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T21:00:00.726Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T21:07:40.059Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T07:00:08.476Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T07:07:48.031Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T09:00:05.136Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T09:07:45.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T11:00:09.099Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T11:07:49.081Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T12:00:06.656Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T12:07:46.008Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T13:00:03.507Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T13:07:43.074Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T14:00:00.248Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T14:07:40.062Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T15:00:08.470Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T15:07:48.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T15:07:48.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T16:00:06.882Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T16:07:46.065Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T17:00:05.733Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T17:07:45.083Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T19:00:03.499Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T19:07:43.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T21:00:07.591Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T21:07:47.031Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T07:00:08.748Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T07:07:48.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T09:00:04.448Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T09:07:44.066Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T11:00:09.529Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T11:07:49.058Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T12:00:05.542Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T12:07:45.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T13:00:01.701Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T13:07:41.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T14:00:07.101Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T14:07:47.096Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T15:00:06.140Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T15:07:46.079Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T16:00:06.074Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T16:07:46.035Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T17:00:03.631Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T17:07:43.063Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T18:13:08.384Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T18:24:10.108Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T19:00:02.661Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T19:08:22.048Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T19:56:00.542Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T20:04:20.029Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T20:07:17.297Z", + "id": 0, + "status": "stop" + } +] diff --git a/tools/test_data/efc.pump_timed.json b/tools/test_data/efc.pump_timed.json deleted file mode 100644 index 6a94c86..0000000 --- a/tools/test_data/efc.pump_timed.json +++ /dev/null @@ -1,760 +0,0 @@ -[{ - "ts": { - "$date": "2025-03-22T16:32:49.626Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67dee6312b9ab34cba6eab58" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:46:01.523Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67dee9472b9ab34cba6eab59" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:46:09.958Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67dee9522b9ab34cba6eab5a" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:47:42.684Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67dee9ae2b9ab34cba6eab5b" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:48:03.282Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67dee9c32b9ab34cba6eab5c" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:48:10.056Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67dee9ca2b9ab34cba6eab5d" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:49:55.583Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67deea332b9ab34cba6eab5e" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:55:11.941Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67deeb702b9ab34cba6eab5f" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:56:06.804Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67deeba62b9ab34cba6eab60" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:58:06.087Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67deec1d2b9ab34cba6eab61" - } -}, -{ - "ts": { - "$date": "2025-03-22T16:59:06.773Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67deec5a2b9ab34cba6eab62" - } -}, -{ - "ts": { - "$date": "2025-03-22T17:01:06.049Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67deecd12b9ab34cba6eab63" - } -}, -{ - "ts": { - "$date": "2025-04-13T07:44:00.169Z" - }, - "id": 0, - "_id": { - "$oid": "67fb6b3feb454e4cde57df16" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T12:36:32.928Z" - }, - "id": 0, - "_id": { - "$oid": "67fbafd1eb454e4cde57df17" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T16:12:13.790Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe258eb454e4cde57df18" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T16:19:33.053Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe40feb454e4cde57df19" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T16:19:33.062Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe40feb454e4cde57df1a" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T16:26:53.057Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe5c6eb454e4cde57df1b" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T16:26:53.066Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe5c6eb454e4cde57df1c" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T16:34:13.012Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe77deb454e4cde57df1d" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T16:34:13.021Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe77deb454e4cde57df1e" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T16:36:06.847Z" - }, - "id": 0, - "_id": { - "$oid": "67fbe7f7eb454e4cde57df1f" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T17:00:05.684Z" - }, - "id": 0, - "_id": { - "$oid": "67fbed92eb454e4cde57df20" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T17:07:45.061Z" - }, - "id": 0, - "_id": { - "$oid": "67fbef5ceb454e4cde57df21" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T19:00:07.797Z" - }, - "id": 0, - "_id": { - "$oid": "67fc09b4eb454e4cde57df22" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T19:07:47.087Z" - }, - "id": 0, - "_id": { - "$oid": "67fc0b7feb454e4cde57df23" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-13T21:00:00.726Z" - }, - "id": 0, - "_id": { - "$oid": "67fc25cdeb454e4cde57df24" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-13T21:07:40.059Z" - }, - "id": 0, - "_id": { - "$oid": "67fc2797eb454e4cde57df25" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T07:00:08.476Z" - }, - "id": 0, - "_id": { - "$oid": "67fcb274eb454e4cde57df26" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T07:07:48.031Z" - }, - "id": 0, - "_id": { - "$oid": "67fcb43feb454e4cde57df27" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T09:00:05.136Z" - }, - "id": 0, - "_id": { - "$oid": "67fcce91eb454e4cde57df28" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T09:07:45.061Z" - }, - "id": 0, - "_id": { - "$oid": "67fcd05ceb454e4cde57df29" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T11:00:09.099Z" - }, - "id": 0, - "_id": { - "$oid": "67fceab5eb454e4cde57df2a" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T11:07:49.081Z" - }, - "id": 0, - "_id": { - "$oid": "67fcec80eb454e4cde57df2b" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T12:00:06.656Z" - }, - "id": 0, - "_id": { - "$oid": "67fcf8c2eb454e4cde57df2c" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T12:07:46.008Z" - }, - "id": 0, - "_id": { - "$oid": "67fcfa8deb454e4cde57df2d" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T13:00:03.507Z" - }, - "id": 0, - "_id": { - "$oid": "67fd06cfeb454e4cde57df2e" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T13:07:43.074Z" - }, - "id": 0, - "_id": { - "$oid": "67fd0899eb454e4cde57df2f" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T14:00:00.248Z" - }, - "id": 0, - "_id": { - "$oid": "67fd14ddeb454e4cde57df30" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T14:07:40.062Z" - }, - "id": 0, - "_id": { - "$oid": "67fd16a8eb454e4cde57df31" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T15:00:08.470Z" - }, - "id": 0, - "_id": { - "$oid": "67fd22f5eb454e4cde57df32" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T15:07:48.088Z" - }, - "id": 0, - "_id": { - "$oid": "67fd24c2eb454e4cde57df33" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T15:07:48.088Z" - }, - "id": 0, - "_id": { - "$oid": "67fd24c2eb454e4cde57df34" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T16:00:06.882Z" - }, - "id": 0, - "_id": { - "$oid": "67fd3103eb454e4cde57df35" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T16:07:46.065Z" - }, - "id": 0, - "_id": { - "$oid": "67fd32cdeb454e4cde57df36" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T17:00:05.733Z" - }, - "id": 0, - "_id": { - "$oid": "67fd3f12eb454e4cde57df37" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T17:07:45.083Z" - }, - "id": 0, - "_id": { - "$oid": "67fd40ddeb454e4cde57df38" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T19:00:03.499Z" - }, - "id": 0, - "_id": { - "$oid": "67fd5b2feb454e4cde57df39" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T19:07:43.057Z" - }, - "id": 0, - "_id": { - "$oid": "67fd5cfaeb454e4cde57df3a" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-14T21:00:07.591Z" - }, - "id": 0, - "_id": { - "$oid": "67fd7754eb454e4cde57df3b" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-14T21:07:47.031Z" - }, - "id": 0, - "_id": { - "$oid": "67fd791eeb454e4cde57df3c" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-15T07:00:08.748Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe03f5f481ffe83f1d061d" - } -}, -{ - "ts": { - "$date": "2025-04-15T07:07:48.061Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe05c0f481ffe83f1d061e" - } -}, -{ - "ts": { - "$date": "2025-04-15T09:00:04.448Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe2010f481ffe83f1d061f" - } -}, -{ - "ts": { - "$date": "2025-04-15T09:07:44.066Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe21dbf481ffe83f1d0620" - } -}, -{ - "ts": { - "$date": "2025-04-15T11:00:09.529Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe3c35f481ffe83f1d0621" - } -}, -{ - "ts": { - "$date": "2025-04-15T11:07:49.058Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe3dfff481ffe83f1d0622" - } -}, -{ - "ts": { - "$date": "2025-04-15T12:00:05.542Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe4a41f481ffe83f1d0623" - } -}, -{ - "ts": { - "$date": "2025-04-15T12:07:45.088Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe4c0bf481ffe83f1d0624" - } -}, -{ - "ts": { - "$date": "2025-04-15T13:00:01.701Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe584bf481ffe83f1d0625" - } -}, -{ - "ts": { - "$date": "2025-04-15T13:07:41.057Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe5a16f481ffe83f1d0626" - } -}, -{ - "ts": { - "$date": "2025-04-15T14:00:07.101Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe6662f481ffe83f1d0627" - } -}, -{ - "ts": { - "$date": "2025-04-15T14:07:47.096Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe682df481ffe83f1d0628" - } -}, -{ - "ts": { - "$date": "2025-04-15T15:00:06.140Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe7472f481ffe83f1d0629" - } -}, -{ - "ts": { - "$date": "2025-04-15T15:07:46.079Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe763df481ffe83f1d062a" - } -}, -{ - "ts": { - "$date": "2025-04-15T16:00:06.074Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe8281f481ffe83f1d062b" - } -}, -{ - "ts": { - "$date": "2025-04-15T16:07:46.035Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe844cf481ffe83f1d062c" - } -}, -{ - "ts": { - "$date": "2025-04-15T17:00:03.631Z" - }, - "id": 0, - "status": "start", - "_id": { - "$oid": "67fe908ff481ffe83f1d062d" - } -}, -{ - "ts": { - "$date": "2025-04-15T17:07:43.063Z" - }, - "id": 0, - "status": "stop", - "_id": { - "$oid": "67fe925af481ffe83f1d062e" - } -}, -{ - "ts": { - "$date": "2025-04-15T18:13:08.384Z" - }, - "id": 0, - "_id": { - "$oid": "67fea1afd717a2ba25fa68c1" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-15T18:24:10.108Z" - }, - "id": 0, - "_id": { - "$oid": "67fea44ad717a2ba25fa68c2" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-15T19:00:02.661Z" - }, - "id": 0, - "_id": { - "$oid": "67feacaed717a2ba25fa68c3" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-15T19:08:22.048Z" - }, - "id": 0, - "_id": { - "$oid": "67feaea1d717a2ba25fa68c4" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-15T19:56:00.542Z" - }, - "id": 0, - "_id": { - "$oid": "67feb9ccd717a2ba25fa68c5" - }, - "status": "start" -}, -{ - "ts": { - "$date": "2025-04-15T20:04:20.029Z" - }, - "id": 0, - "_id": { - "$oid": "67febbbed717a2ba25fa68c6" - }, - "status": "stop" -}, -{ - "ts": { - "$date": "2025-04-15T20:07:17.297Z" - }, - "id": 0, - "_id": { - "$oid": "67febc75d717a2ba25fa68c7" - }, - "status": "stop" -}] \ No newline at end of file From 4cad89b052f89699b4871514e11f21778f54c76a Mon Sep 17 00:00:00 2001 From: phofmeier Date: Sun, 20 Apr 2025 11:25:29 +0200 Subject: [PATCH 03/10] persistence of mosquitto --- example/compose.yaml | 2 ++ example/config_manager.yml | 5 +++-- example/mosquitto/config/mosquitto.conf | 6 ++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/example/compose.yaml b/example/compose.yaml index 7024643..699819f 100644 --- a/example/compose.yaml +++ b/example/compose.yaml @@ -58,6 +58,8 @@ services: # - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf volumes: - ./mosquitto/config:/mosquitto/config + - ./mosquitto/data:/mosquitto/data + - ./mosquitto/log:/mosquitto/log volumes: mongodb-data: diff --git a/example/config_manager.yml b/example/config_manager.yml index 61b49fe..5e97411 100644 --- a/example/config_manager.yml +++ b/example/config_manager.yml @@ -1,6 +1,7 @@ database: collection_config_name: config_static collection_config_template_name: config_template + collection_pump_time_name: pump_timed collection_status_name: status_static collection_used_template_name: used_template connection_string: db:27017 @@ -10,7 +11,7 @@ logging: disable_existing_loggers: false formatters: f: - format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" + format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' handlers: h: class: logging.StreamHandler @@ -18,7 +19,7 @@ logging: level: 20 root: handlers: - - h + - h level: 20 version: 1 mqtt: diff --git a/example/mosquitto/config/mosquitto.conf b/example/mosquitto/config/mosquitto.conf index ce2c7e3..1fcb85c 100644 --- a/example/mosquitto/config/mosquitto.conf +++ b/example/mosquitto/config/mosquitto.conf @@ -3,3 +3,9 @@ protocol mqtt allow_anonymous true persistence true +persistence_location /mosquitto/data/ + +log_dest file /mosquitto/log/mosquitto.log +log_timestamp true +log_timestamp_format %Y-%m-%dT%H:%M:%S +log_dest stdout From 287c56f9c6b2018d88efd9e1aaa3f0c260c79992 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Sun, 20 Apr 2025 12:41:27 +0200 Subject: [PATCH 04/10] move config --- example/compose.yaml | 6 ++++-- example/config_manager.yml | 28 ---------------------------- example/config_mqtt2db.yml | 28 ---------------------------- 3 files changed, 4 insertions(+), 58 deletions(-) delete mode 100644 example/config_manager.yml delete mode 100644 example/config_mqtt2db.yml diff --git a/example/compose.yaml b/example/compose.yaml index 699819f..5561808 100644 --- a/example/compose.yaml +++ b/example/compose.yaml @@ -8,7 +8,8 @@ services: dockerfile: Dockerfile restart: unless-stopped volumes: - - ./config_manager.yml:/home/app/config.yml + - ./manager/config/config.yml:/home/app/config.yml + - ./manager/logs:/home/app/logs depends_on: - db - mosquitto @@ -20,7 +21,8 @@ services: restart: unless-stopped # Custom Configuration volumes: - - ./config_mqtt2db.yml:/home/app/config.yml + - ./mqtt2db/config/config.yml:/home/app/config.yml + - ./mqtt2db/logs:/home/app/logs depends_on: - db diff --git a/example/config_manager.yml b/example/config_manager.yml deleted file mode 100644 index 5e97411..0000000 --- a/example/config_manager.yml +++ /dev/null @@ -1,28 +0,0 @@ -database: - collection_config_name: config_static - collection_config_template_name: config_template - collection_pump_time_name: pump_timed - collection_status_name: status_static - collection_used_template_name: used_template - connection_string: db:27017 - database_name: efc - id_field_name: id -logging: - disable_existing_loggers: false - formatters: - f: - format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' - handlers: - h: - class: logging.StreamHandler - formatter: f - level: 20 - root: - handlers: - - h - level: 20 - version: 1 -mqtt: - broker: mosquitto - new_config_publish_topic: ef/efc/config/set - port: 1883 diff --git a/example/config_mqtt2db.yml b/example/config_mqtt2db.yml deleted file mode 100644 index d6d66a4..0000000 --- a/example/config_mqtt2db.yml +++ /dev/null @@ -1,28 +0,0 @@ -database: - connection_string: db:27017 - static: - add_received_timestamp: true - id_field_name: id - received_timestamp_name: ts_received - timed: - meta_field_name: id - time_field_name: ts -logging: - disable_existing_loggers: false - formatters: - f: - format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" - handlers: - h: - class: logging.StreamHandler - formatter: f - level: 10 - root: - handlers: - - h - level: 10 - version: 1 -mqtt: - broker: mosquitto - channel_prefix: ef - port: 1883 From 8a7ff0b81d50d81d3a7e2e33cd545d8262694f58 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Sun, 20 Apr 2025 12:43:48 +0200 Subject: [PATCH 05/10] new log and config location --- .gitignore | 2 +- example/manager/config/config.yml | 36 +++++++++++++++++++++++++++++++ example/mqtt2db/config/config.yml | 36 +++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 example/manager/config/config.yml create mode 100644 example/mqtt2db/config/config.yml diff --git a/.gitignore b/.gitignore index e8eff42..7727942 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -config.yml +./config.yml diff --git a/example/manager/config/config.yml b/example/manager/config/config.yml new file mode 100644 index 0000000..3e5e897 --- /dev/null +++ b/example/manager/config/config.yml @@ -0,0 +1,36 @@ +database: + collection_config_name: config_static + collection_config_template_name: config_template + collection_pump_time_name: pump_timed + collection_status_name: status_static + collection_used_template_name: used_template + connection_string: db:27017 + database_name: efc + id_field_name: id +logging: + disable_existing_loggers: false + formatters: + f: + format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + handlers: + file: + backupCount: 14 + class: logging.handlers.TimedRotatingFileHandler + filename: ./logs/manager.log + formatter: f + level: 20 + when: D + h: + class: logging.StreamHandler + formatter: f + level: 20 + root: + handlers: + - h + - file + level: 20 + version: 1 +mqtt: + broker: mosquitto + new_config_publish_topic: ef/efc/config/set + port: 1883 diff --git a/example/mqtt2db/config/config.yml b/example/mqtt2db/config/config.yml new file mode 100644 index 0000000..0b980d7 --- /dev/null +++ b/example/mqtt2db/config/config.yml @@ -0,0 +1,36 @@ +database: + connection_string: db:27017 + static: + add_received_timestamp: true + id_field_name: id + received_timestamp_name: ts_received + timed: + meta_field_name: id + time_field_name: ts +logging: + disable_existing_loggers: false + formatters: + f: + format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + handlers: + file: + backupCount: 14 + class: logging.handlers.TimedRotatingFileHandler + filename: ./logs/mqtt2db.log + formatter: f + level: 20 + when: D + h: + class: logging.StreamHandler + formatter: f + level: 10 + root: + handlers: + - h + - file + level: 10 + version: 1 +mqtt: + broker: mosquitto + channel_prefix: ef + port: 1883 From 64ebcd04e710461226a04719400dc19d9c4cb6bc Mon Sep 17 00:00:00 2001 From: phofmeier Date: Mon, 21 Apr 2025 13:56:29 +0000 Subject: [PATCH 06/10] fix pre-commit --- .devcontainer/docker-compose.yml | 12 ++++++------ .gitignore | 2 +- example/manager/config/config.yml | 6 +++--- example/mqtt2db/config/config.yml | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 8122e9d..fdb6c6b 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -18,21 +18,21 @@ services: # (Adding the "ports" property to this file will not forward from a Codespace.) depends_on: - - db - - mosquitto + - db-dev + - mosquitto-dev # Database - db: + db-dev: image: mongo:latest restart: unless-stopped volumes: - - mongodb-data:/data/db + - mongodb-data-dev:/data/db # MQTT brocker - mosquitto: + mosquitto-dev: image: eclipse-mosquitto:latest restart: unless-stopped network_mode: service:db volumes: - mongodb-data: + mongodb-data-dev: diff --git a/.gitignore b/.gitignore index 7727942..1e455a6 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -./config.yml +/config.yml diff --git a/example/manager/config/config.yml b/example/manager/config/config.yml index 3e5e897..0e5da54 100644 --- a/example/manager/config/config.yml +++ b/example/manager/config/config.yml @@ -11,7 +11,7 @@ logging: disable_existing_loggers: false formatters: f: - format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" handlers: file: backupCount: 14 @@ -26,8 +26,8 @@ logging: level: 20 root: handlers: - - h - - file + - h + - file level: 20 version: 1 mqtt: diff --git a/example/mqtt2db/config/config.yml b/example/mqtt2db/config/config.yml index 0b980d7..458a942 100644 --- a/example/mqtt2db/config/config.yml +++ b/example/mqtt2db/config/config.yml @@ -11,7 +11,7 @@ logging: disable_existing_loggers: false formatters: f: - format: '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' + format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" handlers: file: backupCount: 14 @@ -26,8 +26,8 @@ logging: level: 10 root: handlers: - - h - - file + - h + - file level: 10 version: 1 mqtt: From 44d4eb145209ece19d81c7038e7b940614f2a4b6 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Mon, 21 Apr 2025 13:58:55 +0000 Subject: [PATCH 07/10] updated changelog --- CHANGELOG.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d96fe..9e0c9ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,23 +15,25 @@ Use the following labels: ### Added -- [Patch] Support for configuration templates -- [Patch] New page to show and change templates -- [Patch] Create and delete templates -- [Patch] Export templates as json file +- [Patch] Added new Data Viewer page. +- [Patch] Support for configuration templates. +- [Patch] New page to show and change templates. +- [Patch] Create and delete templates. +- [Patch] Export templates as json file. ### Changed -- [Patch] Changed example config to correctly save timed messages +- [Patch] Changed example to write logs to files. +- [Patch] Changed example config to correctly save timed messages. ## [0.0.2] - 2024-12-29 ### Fixed -- [Patch] Fixed Example +- [Patch] Fixed Example. ## [0.0.1] - 2024-12-27 ### Added -- [Patch] First Version +- [Patch] First Version. From 91c385a393b139959d872da3098decfaeca2d3a8 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Mon, 21 Apr 2025 14:04:59 +0000 Subject: [PATCH 08/10] fix dev container --- .devcontainer/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index fdb6c6b..344363b 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -18,18 +18,18 @@ services: # (Adding the "ports" property to this file will not forward from a Codespace.) depends_on: - - db-dev - - mosquitto-dev + - db + - mosquitto # Database - db-dev: + db: image: mongo:latest restart: unless-stopped volumes: - - mongodb-data-dev:/data/db + - mongodb-data:/data/db # MQTT brocker - mosquitto-dev: + mosquitto: image: eclipse-mosquitto:latest restart: unless-stopped network_mode: service:db From 85acbc1c8e6fd3644a1113145a876e118c4e0405 Mon Sep 17 00:00:00 2001 From: phofmeier Date: Mon, 21 Apr 2025 14:13:09 +0000 Subject: [PATCH 09/10] fix dev container --- .devcontainer/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 344363b..8122e9d 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -35,4 +35,4 @@ services: network_mode: service:db volumes: - mongodb-data-dev: + mongodb-data: From 750832b7c307d096591e3a3233aaad33251783bf Mon Sep 17 00:00:00 2001 From: phofmeier Date: Mon, 21 Apr 2025 18:24:45 +0000 Subject: [PATCH 10/10] better plot by using a pandas dataframe --- pyproject.toml | 2 ++ src/ebb_flow_manager/database/database.py | 6 +++-- src/ebb_flow_manager/database/mongo_db.py | 25 +++++++++++++++--- .../ebb_flow_manager_data_viewer.py | 26 ++++++++----------- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 91fe232..dfc11ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,8 @@ dependencies = [ "panel >= 1.6.0", "watchfiles >= 1.0.3", "plotly >= 6.0.1", + "pymongoarrow >=1.7.1", + "pandas >= 2.2.3", ] dynamic = ["version"] diff --git a/src/ebb_flow_manager/database/database.py b/src/ebb_flow_manager/database/database.py index 8a85453..1811222 100644 --- a/src/ebb_flow_manager/database/database.py +++ b/src/ebb_flow_manager/database/database.py @@ -1,3 +1,5 @@ +import pandas as pd + from ebb_flow_manager.database.ebb_flow_controller_data import EbbFlowControllerData from ebb_flow_manager.database.mongo_db import MongoDbImpl @@ -124,10 +126,10 @@ def set_used_template_of(self, id: int, template_name: str): """ self.db_impl.set_used_template_of(id, template_name.strip()) - def get_pump_time_data(self) -> list[dict]: + def get_pump_time_data(self) -> pd.DataFrame: """Get all pump time data. Returns: - list[dict]: List of dicts containing the pump time data. + pd.DataFrame: Pandas dataframe containing the pump time data. """ return self.db_impl.get_pump_time_data() diff --git a/src/ebb_flow_manager/database/mongo_db.py b/src/ebb_flow_manager/database/mongo_db.py index 923dbbc..901c6fd 100644 --- a/src/ebb_flow_manager/database/mongo_db.py +++ b/src/ebb_flow_manager/database/mongo_db.py @@ -1,7 +1,12 @@ import logging +import pandas as pd +import pymongoarrow +import pymongoarrow.monkey from pymongo import MongoClient +pymongoarrow.monkey.patch_all() + class MongoDbImpl: """Implementation of connection to a MongoDB database.""" @@ -174,12 +179,26 @@ def get_all_data_from(self, database: str, collection: str) -> list[dict]: def get_all_timed_data_from(self, database: str, collection: str): return list(self.client[database][collection].find({}, {})) - def get_pump_time_data(self) -> list[dict]: + def get_all_timed_data_as_dataframe( + self, database: str, collection: str + ) -> pd.DataFrame: + """Get a pandas dataframe with all data. + + Args: + database (str): name of the database. + collection (str): name of the collection. + + Returns: + pd.DataFrame: Dataframe containing the data. + """ + return self.client[database][collection].find_pandas_all({}) + + def get_pump_time_data(self) -> pd.DataFrame: """Get all pump time data. Returns: - list[dict]: List of dicts containing the pump time data. + pd.DataFrame: List of dicts containing the pump time data. """ - return self.get_all_timed_data_from( + return self.get_all_timed_data_as_dataframe( self.config["database_name"], self.config["collection_pump_time_name"] ) diff --git a/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py index d7ebeed..e736692 100644 --- a/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py +++ b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py @@ -1,5 +1,5 @@ import panel as pn -import plotly.graph_objs as go +import plotly.express as px from ebb_flow_manager.config import Config from ebb_flow_manager.database.database import Database @@ -21,22 +21,18 @@ def start_serve() -> pn.panel: logger.debug("start app") db = Database(config.get("database")) - print(db.get_pump_time_data()) data = db.get_pump_time_data() - sorted_data = sorted(data, key=lambda x: x["ts"]) - timestamps = [data["ts"] for data in sorted_data] - values = [data["status"] for data in sorted_data] + data.sort_values("ts", inplace=True) - figure = go.Figure() - figure.add_trace( - go.Scatter( - x=timestamps, - y=values, - mode="lines+markers", - line={"shape": "hv"}, - name="Pump Status", - ) + figure = px.line( + data_frame=data, + x="ts", + y="status", + color="id", + markers=True, + line_shape="hv", ) + figure.update_layout( title="Pump Status Over Time", xaxis_title="Timestamp", @@ -46,7 +42,7 @@ def start_serve() -> pn.panel: ) figure.update_layout( autosize=True, - margin=dict(l=20, r=20, t=20, b=20), + margin=dict(l=20, r=20, t=40, b=20), height=400, width=600, )