diff --git a/.gitignore b/.gitignore index e8eff42..1e455a6 100644 --- a/.gitignore +++ b/.gitignore @@ -160,4 +160,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ -config.yml +/config.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 54d96fe..9e0c9ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,23 +15,25 @@ Use the following labels: ### Added -- [Patch] Support for configuration templates -- [Patch] New page to show and change templates -- [Patch] Create and delete templates -- [Patch] Export templates as json file +- [Patch] Added new Data Viewer page. +- [Patch] Support for configuration templates. +- [Patch] New page to show and change templates. +- [Patch] Create and delete templates. +- [Patch] Export templates as json file. ### Changed -- [Patch] Changed example config to correctly save timed messages +- [Patch] Changed example to write logs to files. +- [Patch] Changed example config to correctly save timed messages. ## [0.0.2] - 2024-12-29 ### Fixed -- [Patch] Fixed Example +- [Patch] Fixed Example. ## [0.0.1] - 2024-12-27 ### Added -- [Patch] First Version +- [Patch] First Version. diff --git a/Dockerfile b/Dockerfile index 1613af6..8dedfe7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ WORKDIR /home/app COPY . . RUN pip install . -CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py" ] +CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py", "src/ebb_flow_manager/ebb_flow_manager_data_viewer.py" ] diff --git a/example/compose.yaml b/example/compose.yaml index 7024643..5561808 100644 --- a/example/compose.yaml +++ b/example/compose.yaml @@ -8,7 +8,8 @@ services: dockerfile: Dockerfile restart: unless-stopped volumes: - - ./config_manager.yml:/home/app/config.yml + - ./manager/config/config.yml:/home/app/config.yml + - ./manager/logs:/home/app/logs depends_on: - db - mosquitto @@ -20,7 +21,8 @@ services: restart: unless-stopped # Custom Configuration volumes: - - ./config_mqtt2db.yml:/home/app/config.yml + - ./mqtt2db/config/config.yml:/home/app/config.yml + - ./mqtt2db/logs:/home/app/logs depends_on: - db @@ -58,6 +60,8 @@ services: # - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf volumes: - ./mosquitto/config:/mosquitto/config + - ./mosquitto/data:/mosquitto/data + - ./mosquitto/log:/mosquitto/log volumes: mongodb-data: diff --git a/example/config_manager.yml b/example/manager/config/config.yml similarity index 73% rename from example/config_manager.yml rename to example/manager/config/config.yml index 61b49fe..0e5da54 100644 --- a/example/config_manager.yml +++ b/example/manager/config/config.yml @@ -1,6 +1,7 @@ database: collection_config_name: config_static collection_config_template_name: config_template + collection_pump_time_name: pump_timed collection_status_name: status_static collection_used_template_name: used_template connection_string: db:27017 @@ -12,6 +13,13 @@ logging: f: format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" handlers: + file: + backupCount: 14 + class: logging.handlers.TimedRotatingFileHandler + filename: ./logs/manager.log + formatter: f + level: 20 + when: D h: class: logging.StreamHandler formatter: f @@ -19,6 +27,7 @@ logging: root: handlers: - h + - file level: 20 version: 1 mqtt: diff --git a/example/mosquitto/config/mosquitto.conf b/example/mosquitto/config/mosquitto.conf index ce2c7e3..1fcb85c 100644 --- a/example/mosquitto/config/mosquitto.conf +++ b/example/mosquitto/config/mosquitto.conf @@ -3,3 +3,9 @@ protocol mqtt allow_anonymous true persistence true +persistence_location /mosquitto/data/ + +log_dest file /mosquitto/log/mosquitto.log +log_timestamp true +log_timestamp_format %Y-%m-%dT%H:%M:%S +log_dest stdout diff --git a/example/config_mqtt2db.yml b/example/mqtt2db/config/config.yml similarity index 74% rename from example/config_mqtt2db.yml rename to example/mqtt2db/config/config.yml index d6d66a4..458a942 100644 --- a/example/config_mqtt2db.yml +++ b/example/mqtt2db/config/config.yml @@ -13,6 +13,13 @@ logging: f: format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s" handlers: + file: + backupCount: 14 + class: logging.handlers.TimedRotatingFileHandler + filename: ./logs/mqtt2db.log + formatter: f + level: 20 + when: D h: class: logging.StreamHandler formatter: f @@ -20,6 +27,7 @@ logging: root: handlers: - h + - file level: 10 version: 1 mqtt: diff --git a/pyproject.toml b/pyproject.toml index b1b2cde..dfc11ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,10 @@ dependencies = [ "paho-mqtt >= 2.1.0", "pymongo[srv] >= 4.8.0", "panel >= 1.6.0", - "watchfiles >= 1.0.3" + "watchfiles >= 1.0.3", + "plotly >= 6.0.1", + "pymongoarrow >=1.7.1", + "pandas >= 2.2.3", ] dynamic = ["version"] diff --git a/src/ebb_flow_manager/config.py b/src/ebb_flow_manager/config.py index debbed9..8e350cf 100644 --- a/src/ebb_flow_manager/config.py +++ b/src/ebb_flow_manager/config.py @@ -20,6 +20,7 @@ def __init__(self, filename: str) -> None: "collection_config_name": "config_static", "collection_config_template_name": "config_template", "collection_used_template_name": "used_template", + "collection_pump_time_name": "pump_timed", "id_field_name": "id", }, "logging": { diff --git a/src/ebb_flow_manager/database/database.py b/src/ebb_flow_manager/database/database.py index f109644..1811222 100644 --- a/src/ebb_flow_manager/database/database.py +++ b/src/ebb_flow_manager/database/database.py @@ -1,3 +1,5 @@ +import pandas as pd + from ebb_flow_manager.database.ebb_flow_controller_data import EbbFlowControllerData from ebb_flow_manager.database.mongo_db import MongoDbImpl @@ -123,3 +125,11 @@ def set_used_template_of(self, id: int, template_name: str): template_name (str): name of the template used by this controller. """ self.db_impl.set_used_template_of(id, template_name.strip()) + + def get_pump_time_data(self) -> pd.DataFrame: + """Get all pump time data. + + Returns: + pd.DataFrame: Pandas dataframe containing the pump time data. + """ + return self.db_impl.get_pump_time_data() diff --git a/src/ebb_flow_manager/database/mongo_db.py b/src/ebb_flow_manager/database/mongo_db.py index cf0464c..901c6fd 100644 --- a/src/ebb_flow_manager/database/mongo_db.py +++ b/src/ebb_flow_manager/database/mongo_db.py @@ -1,7 +1,12 @@ import logging +import pandas as pd +import pymongoarrow +import pymongoarrow.monkey from pymongo import MongoClient +pymongoarrow.monkey.patch_all() + class MongoDbImpl: """Implementation of connection to a MongoDB database.""" @@ -173,3 +178,27 @@ def get_all_data_from(self, database: str, collection: str) -> list[dict]: def get_all_timed_data_from(self, database: str, collection: str): return list(self.client[database][collection].find({}, {})) + + def get_all_timed_data_as_dataframe( + self, database: str, collection: str + ) -> pd.DataFrame: + """Get a pandas dataframe with all data. + + Args: + database (str): name of the database. + collection (str): name of the collection. + + Returns: + pd.DataFrame: Dataframe containing the data. + """ + return self.client[database][collection].find_pandas_all({}) + + def get_pump_time_data(self) -> pd.DataFrame: + """Get all pump time data. + + Returns: + pd.DataFrame: List of dicts containing the pump time data. + """ + return self.get_all_timed_data_as_dataframe( + self.config["database_name"], self.config["collection_pump_time_name"] + ) diff --git a/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py new file mode 100644 index 0000000..e736692 --- /dev/null +++ b/src/ebb_flow_manager/ebb_flow_manager_data_viewer.py @@ -0,0 +1,65 @@ +import panel as pn +import plotly.express as px + +from ebb_flow_manager.config import Config +from ebb_flow_manager.database.database import Database +from ebb_flow_manager.ebb_flow_manager_app import init_logger + +pn.extension(design="material") + +pn.extension("plotly") + + +def start_serve() -> pn.panel: + """Start the panel server and display the data viewer. + + Returns: + pn.panel: panel object containing the data viewer. + """ + config = Config("config.yml") + logger = init_logger(config.get("logging")) + logger.debug("start app") + db = Database(config.get("database")) + + data = db.get_pump_time_data() + data.sort_values("ts", inplace=True) + + figure = px.line( + data_frame=data, + x="ts", + y="status", + color="id", + markers=True, + line_shape="hv", + ) + + figure.update_layout( + title="Pump Status Over Time", + xaxis_title="Timestamp", + yaxis_title="Status", + xaxis_tickformat="%Y-%m-%d %H:%M:%S", + xaxis_tickangle=-45, + ) + figure.update_layout( + autosize=True, + margin=dict(l=20, r=20, t=40, b=20), + height=400, + width=600, + ) + plotly_pane = pn.pane.Plotly(figure) + template = pn.template.BootstrapTemplate( + title="Ebb Flow Manager - Data Viewer", + main=[plotly_pane], + ) + return template + + +def main(): + pn.serve(start_serve(), admin=True) + + +if __name__ == "__main__": + # start_serve().servable() + main() +elif __name__.startswith("bokeh_app"): + start_serve().servable() diff --git a/tools/publish_test_data.py b/tools/publish_test_data.py index 40d2448..5f38cd2 100644 --- a/tools/publish_test_data.py +++ b/tools/publish_test_data.py @@ -1,3 +1,4 @@ +import datetime import json import os @@ -36,6 +37,22 @@ def get_all_files(path="."): data, upsert=True, ) + elif type_name == "timed": + if collection not in client[database].list_collection_names(): + # Insert new collection + client[database].create_collection( + collection, + timeseries={ + "timeField": "ts", + "metaField": "id", + }, + ) + for data in all_data: + datetime_ts = datetime.datetime.fromisoformat(data["ts"]) + data["ts"] = datetime_ts + client[database][collection].insert_one( + data, + ) else: for data in all_data: client[database][collection].replace_one( diff --git a/tools/test_data/efc-pump_timed.json b/tools/test_data/efc-pump_timed.json new file mode 100644 index 0000000..3339c9c --- /dev/null +++ b/tools/test_data/efc-pump_timed.json @@ -0,0 +1,382 @@ +[ + { + "ts": "2025-03-22T16:32:49.626Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:46:01.523Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:46:09.958Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:47:42.684Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:48:03.282Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:48:10.056Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:49:55.583Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:55:11.941Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:56:06.804Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T16:58:06.087Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-03-22T16:59:06.773Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-03-22T17:01:06.049Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T07:44:00.169Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T12:36:32.928Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:12:13.790Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:19:33.053Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:19:33.062Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:26:53.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:26:53.066Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:34:13.012Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T16:34:13.021Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T16:36:06.847Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T17:00:05.684Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T17:07:45.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T19:00:07.797Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T19:07:47.087Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-13T21:00:00.726Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-13T21:07:40.059Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T07:00:08.476Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T07:07:48.031Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T09:00:05.136Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T09:07:45.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T11:00:09.099Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T11:07:49.081Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T12:00:06.656Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T12:07:46.008Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T13:00:03.507Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T13:07:43.074Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T14:00:00.248Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T14:07:40.062Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T15:00:08.470Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T15:07:48.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T15:07:48.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T16:00:06.882Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T16:07:46.065Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T17:00:05.733Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T17:07:45.083Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T19:00:03.499Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T19:07:43.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-14T21:00:07.591Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-14T21:07:47.031Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T07:00:08.748Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T07:07:48.061Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T09:00:04.448Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T09:07:44.066Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T11:00:09.529Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T11:07:49.058Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T12:00:05.542Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T12:07:45.088Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T13:00:01.701Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T13:07:41.057Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T14:00:07.101Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T14:07:47.096Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T15:00:06.140Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T15:07:46.079Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T16:00:06.074Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T16:07:46.035Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T17:00:03.631Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T17:07:43.063Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T18:13:08.384Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T18:24:10.108Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T19:00:02.661Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T19:08:22.048Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T19:56:00.542Z", + "id": 0, + "status": "start" + }, + { + "ts": "2025-04-15T20:04:20.029Z", + "id": 0, + "status": "stop" + }, + { + "ts": "2025-04-15T20:07:17.297Z", + "id": 0, + "status": "stop" + } +]