Data viewer

.gitignore

@@ -160,4 +160,4 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-config.yml
+/config.yml

CHANGELOG.md

@@ -15,23 +15,25 @@ Use the following labels:
 
 ### Added
 
-- [Patch] Support for configuration templates
-- [Patch] New page to show and change templates
-- [Patch] Create and delete templates
-- [Patch] Export templates as json file
+- [Patch] Added new Data Viewer page.
+- [Patch] Support for configuration templates.
+- [Patch] New page to show and change templates.
+- [Patch] Create and delete templates.
+- [Patch] Export templates as json file.
 
 ### Changed
 
-- [Patch] Changed example config to correctly save timed messages
+- [Patch] Changed example to write logs to files.
+- [Patch] Changed example config to correctly save timed messages.
 
 ## [0.0.2] - 2024-12-29
 
 ### Fixed
 
-- [Patch] Fixed Example
+- [Patch] Fixed Example.
 
 ## [0.0.1] - 2024-12-27
 
 ### Added
 
-- [Patch] First Version
+- [Patch] First Version.

Dockerfile

@@ -4,4 +4,4 @@ WORKDIR /home/app
 COPY . .
 RUN pip install .
 
-CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py" ]
+CMD [ "panel", "serve", "src/ebb_flow_manager/ebb_flow_manager_app.py", "src/ebb_flow_manager/ebb_flow_manager_templates.py", "src/ebb_flow_manager/ebb_flow_manager_data_viewer.py" ]

example/compose.yaml

@@ -8,7 +8,8 @@ services:
       dockerfile: Dockerfile
     restart: unless-stopped
     volumes:
-      - ./config_manager.yml:/home/app/config.yml
+      - ./manager/config/config.yml:/home/app/config.yml
+      - ./manager/logs:/home/app/logs
     depends_on:
       - db
       - mosquitto
@@ -20,7 +21,8 @@ services:
     restart: unless-stopped
     # Custom Configuration
     volumes:
-      - ./config_mqtt2db.yml:/home/app/config.yml
+      - ./mqtt2db/config/config.yml:/home/app/config.yml
+      - ./mqtt2db/logs:/home/app/logs
 
     depends_on:
       - db
@@ -58,6 +60,8 @@ services:
     #    - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf
     volumes:
       - ./mosquitto/config:/mosquitto/config
+      - ./mosquitto/data:/mosquitto/data
+      - ./mosquitto/log:/mosquitto/log
 
 volumes:
   mongodb-data:

example/config_manager.yml → example/manager/config/config.yml

@@ -1,6 +1,7 @@
 database:
   collection_config_name: config_static
   collection_config_template_name: config_template
+  collection_pump_time_name: pump_timed
   collection_status_name: status_static
   collection_used_template_name: used_template
   connection_string: db:27017
@@ -12,13 +13,21 @@ logging:
     f:
       format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
   handlers:
+    file:
+      backupCount: 14
+      class: logging.handlers.TimedRotatingFileHandler
+      filename: ./logs/manager.log
+      formatter: f
+      level: 20
+      when: D
     h:
       class: logging.StreamHandler
       formatter: f
       level: 20
   root:
     handlers:
       - h
+      - file
     level: 20
   version: 1
 mqtt:

example/mosquitto/config/mosquitto.conf

@@ -3,3 +3,9 @@ protocol mqtt
 allow_anonymous true
 
 persistence true
+persistence_location /mosquitto/data/
+
+log_dest file /mosquitto/log/mosquitto.log
+log_timestamp true
+log_timestamp_format %Y-%m-%dT%H:%M:%S
+log_dest stdout

example/config_mqtt2db.yml → example/mqtt2db/config/config.yml

@@ -13,13 +13,21 @@ logging:
     f:
       format: "%(asctime)s %(name)-12s %(levelname)-8s %(message)s"
   handlers:
+    file:
+      backupCount: 14
+      class: logging.handlers.TimedRotatingFileHandler
+      filename: ./logs/mqtt2db.log
+      formatter: f
+      level: 20
+      when: D
     h:
       class: logging.StreamHandler
       formatter: f
       level: 10
   root:
     handlers:
       - h
+      - file
     level: 10
   version: 1
 mqtt:

pyproject.toml

@@ -14,7 +14,10 @@ dependencies = [
     "paho-mqtt >= 2.1.0",
     "pymongo[srv] >= 4.8.0",
     "panel >= 1.6.0",
-    "watchfiles >= 1.0.3"
+    "watchfiles >= 1.0.3",
+    "plotly >= 6.0.1",
+    "pymongoarrow >=1.7.1",
+    "pandas >= 2.2.3",
 ]
 dynamic = ["version"]
 

src/ebb_flow_manager/config.py

@@ -20,6 +20,7 @@ def __init__(self, filename: str) -> None:
                 "collection_config_name": "config_static",
                 "collection_config_template_name": "config_template",
                 "collection_used_template_name": "used_template",
+                "collection_pump_time_name": "pump_timed",
                 "id_field_name": "id",
             },
             "logging": {

src/ebb_flow_manager/database/database.py

@@ -1,3 +1,5 @@
+import pandas as pd
+
 from ebb_flow_manager.database.ebb_flow_controller_data import EbbFlowControllerData
 from ebb_flow_manager.database.mongo_db import MongoDbImpl
 
@@ -123,3 +125,11 @@ def set_used_template_of(self, id: int, template_name: str):
             template_name (str): name of the template used by this controller.
         """
         self.db_impl.set_used_template_of(id, template_name.strip())
+
+    def get_pump_time_data(self) -> pd.DataFrame:
+        """Get all pump time data.
+
+        Returns:
+            pd.DataFrame: Pandas dataframe containing the pump time data.
+        """
+        return self.db_impl.get_pump_time_data()

src/ebb_flow_manager/database/mongo_db.py

@@ -1,7 +1,12 @@
 import logging
 
+import pandas as pd
+import pymongoarrow
+import pymongoarrow.monkey
 from pymongo import MongoClient
 
+pymongoarrow.monkey.patch_all()
+
 
 class MongoDbImpl:
     """Implementation of connection to a MongoDB database."""
@@ -173,3 +178,27 @@ def get_all_data_from(self, database: str, collection: str) -> list[dict]:
 
     def get_all_timed_data_from(self, database: str, collection: str):
         return list(self.client[database][collection].find({}, {}))
+
+    def get_all_timed_data_as_dataframe(
+        self, database: str, collection: str
+    ) -> pd.DataFrame:
+        """Get a pandas dataframe with all data.
+
+        Args:
+            database (str): name of the database.
+            collection (str): name of the collection.
+
+        Returns:
+            pd.DataFrame: Dataframe containing the data.
+        """
+        return self.client[database][collection].find_pandas_all({})
+
+    def get_pump_time_data(self) -> pd.DataFrame:
+        """Get all pump time data.
+
+        Returns:
+           pd.DataFrame: List of dicts containing the pump time data.
+        """
+        return self.get_all_timed_data_as_dataframe(
+            self.config["database_name"], self.config["collection_pump_time_name"]
+        )

src/ebb_flow_manager/ebb_flow_manager_data_viewer.py

@@ -0,0 +1,65 @@
+import panel as pn
+import plotly.express as px
+
+from ebb_flow_manager.config import Config
+from ebb_flow_manager.database.database import Database
+from ebb_flow_manager.ebb_flow_manager_app import init_logger
+
+pn.extension(design="material")
+
+pn.extension("plotly")
+
+
+def start_serve() -> pn.panel:
+    """Start the panel server and display the data viewer.
+
+    Returns:
+        pn.panel: panel object containing the data viewer.
+    """
+    config = Config("config.yml")
+    logger = init_logger(config.get("logging"))
+    logger.debug("start app")
+    db = Database(config.get("database"))
+
+    data = db.get_pump_time_data()
+    data.sort_values("ts", inplace=True)
+
+    figure = px.line(
+        data_frame=data,
+        x="ts",
+        y="status",
+        color="id",
+        markers=True,
+        line_shape="hv",
+    )
+
+    figure.update_layout(
+        title="Pump Status Over Time",
+        xaxis_title="Timestamp",
+        yaxis_title="Status",
+        xaxis_tickformat="%Y-%m-%d %H:%M:%S",
+        xaxis_tickangle=-45,
+    )
+    figure.update_layout(
+        autosize=True,
+        margin=dict(l=20, r=20, t=40, b=20),
+        height=400,
+        width=600,
+    )
+    plotly_pane = pn.pane.Plotly(figure)
+    template = pn.template.BootstrapTemplate(
+        title="Ebb Flow Manager - Data Viewer",
+        main=[plotly_pane],
+    )
+    return template
+
+
+def main():
+    pn.serve(start_serve(), admin=True)
+
+
+if __name__ == "__main__":
+    # start_serve().servable()
+    main()
+elif __name__.startswith("bokeh_app"):
+    start_serve().servable()

tools/publish_test_data.py

@@ -1,3 +1,4 @@
+import datetime
 import json
 import os
 
@@ -36,6 +37,22 @@ def get_all_files(path="."):
                 data,
                 upsert=True,
             )
+    elif type_name == "timed":
+        if collection not in client[database].list_collection_names():
+            # Insert new collection
+            client[database].create_collection(
+                collection,
+                timeseries={
+                    "timeField": "ts",
+                    "metaField": "id",
+                },
+            )
+        for data in all_data:
+            datetime_ts = datetime.datetime.fromisoformat(data["ts"])
+            data["ts"] = datetime_ts
+            client[database][collection].insert_one(
+                data,
+            )
     else:
         for data in all_data:
             client[database][collection].replace_one(