From 45a6fe859eb0e3a1c831a7fc10a176e7eca9e8e8 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Thu, 2 Oct 2025 14:22:49 +0200 Subject: [PATCH 01/24] Added download button to tables --- src/OpenPostbud/routes/user/forsendelser.py | 2 +- .../routes/user/tjek_tilmelding.py | 2 +- src/OpenPostbud/ui_components.py | 24 ++++++++++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/OpenPostbud/routes/user/forsendelser.py b/src/OpenPostbud/routes/user/forsendelser.py index 3314dcf..dce232d 100644 --- a/src/OpenPostbud/routes/user/forsendelser.py +++ b/src/OpenPostbud/routes/user/forsendelser.py @@ -93,7 +93,7 @@ def __init__(self, shipment_id: str) -> None: ui.label(status[0]).classes("border p-1") ui.label(status[1]).classes("border p-1") - letter_table = ui_components.SearchTable(title="Breve", rows=letter_rows, columns=LETTERS_COLUMNS, column_defaults=COLUMN_DEFAULTS, pagination=50) + letter_table = ui_components.SearchTable(title="Breve", rows=letter_rows, columns=LETTERS_COLUMNS, column_defaults=COLUMN_DEFAULTS, pagination=50, download_button=True, search_field=True) ui_components.obscure_column_values(letter_table, "recipient", 7, 4) def _download_template(self): diff --git a/src/OpenPostbud/routes/user/tjek_tilmelding.py b/src/OpenPostbud/routes/user/tjek_tilmelding.py index 5967daf..c1ba7ba 100644 --- a/src/OpenPostbud/routes/user/tjek_tilmelding.py +++ b/src/OpenPostbud/routes/user/tjek_tilmelding.py @@ -90,5 +90,5 @@ def __init__(self, job_id: int): tasks = registration_task.get_registration_tasks(job_id) rows = [task.to_row_dict() for task in tasks] - table = ui.table(title="Tilmeldinger", columns=TASK_COLUMNS, column_defaults=COLUMN_DEFAULTS, rows=rows, pagination=50) + table = ui_components.SearchTable(title="Tilmeldinger", columns=TASK_COLUMNS, column_defaults=COLUMN_DEFAULTS, rows=rows, pagination=50, search_field=True, download_button=True) ui_components.obscure_column_values(table, "registrant", 7, 4) diff --git a/src/OpenPostbud/ui_components.py b/src/OpenPostbud/ui_components.py index 8f648ab..dc65b67 100644 --- a/src/OpenPostbud/ui_components.py +++ b/src/OpenPostbud/ui_components.py @@ -1,6 +1,8 @@ """This module contains reusable UI components.""" from typing import Literal +import csv +from io import StringIO from nicegui import ui, app @@ -155,10 +157,26 @@ def add_message(self, text: str, type_: Literal["positive", "warning", "negative class SearchTable(ui.table): """An extension of ui.table that has a search field in the top slot.""" - def __init__(self, *, rows, columns = None, column_defaults = None, row_key = 'id', title = None, selection = None, pagination = None, on_select = None, on_pagination_change = None): # pylint: disable=too-many-arguments + def __init__(self, *, rows, columns = None, column_defaults = None, row_key = 'id', title = None, selection = None, pagination = None, on_select = None, on_pagination_change = None, + search_field: bool, download_button: bool): # pylint: disable=too-many-arguments super().__init__(rows=rows, columns=columns, column_defaults=column_defaults, row_key=row_key, selection=selection, pagination=pagination, on_select=on_select, on_pagination_change=on_pagination_change) with self.add_slot("top"): ui.label(title).classes("q-table__title") ui.space() - search_input = ui.input("Søg").props("clearable") - self.bind_filter_from(search_input, "value") + if download_button: + ui.button("Download liste", on_click=self._download_list).classes("mr-5") + if search_field: + search_input = ui.input("Søg").props("clearable") + self.bind_filter_from(search_input, "value") + + def _download_list(self): + """A callback function for downloading the table data as a csv file.""" + field_names = [col["field"] for col in self.columns] + field_labels = {col["field"]: col["label"] for col in self.columns} + + f = StringIO() + writer = csv.DictWriter(f, fieldnames=field_names) + writer.writerow(field_labels) + writer.writerows(self.rows) + + ui.download(f.getvalue().encode(), "Liste.csv") \ No newline at end of file From 9c18607c7f3d58038d469be6eb3d5c9981b03a5a Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 08:55:12 +0200 Subject: [PATCH 02/24] Added new env variables --- .env.example | 4 ++++ src/OpenPostbud/config.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.env.example b/.env.example index 9cf443f..95f87e4 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,10 @@ shipment_worker_sleep_time=10 sender_label=Something Corp path_to_libreoffice=somewhere/LibreOffice/program/soffice.exe +# Message broker settings +message_broker_queue_id=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +message_broker_worker_sleep_time=60 + # OIDC settings client_id=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx client_secret=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/src/OpenPostbud/config.py b/src/OpenPostbud/config.py index cd766a8..c7f24dd 100644 --- a/src/OpenPostbud/config.py +++ b/src/OpenPostbud/config.py @@ -43,6 +43,10 @@ def str_to_bool(s: str) -> bool: SENDER_LABEL = os.environ['sender_label'] PATH_TO_LIBREOFFICE = os.environ['path_to_libreoffice'] +# Message broker worker +MESSAGE_BROKER_QUEUE_ID = os.environ['message_broker_queue_id'] +MESSAGE_BROKER_WORKER_SLEEP_TIME = float(os.environ['message_broker_worker_sleep_time']) + # OIDC CLIENT_ID = os.environ['client_id'] CLIENT_SECRET = os.environ['client_secret'] From 4b1172d87fa5249c37a77e455d66368a2f372791 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 08:56:45 +0200 Subject: [PATCH 03/24] Moved set_status to Letter class + correct transaction id --- .../database/digital_post/letters.py | 30 ++++++++++++++- src/OpenPostbud/workers/shipment_worker.py | 38 ++++--------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/OpenPostbud/database/digital_post/letters.py b/src/OpenPostbud/database/digital_post/letters.py index 862d232..46aa91f 100644 --- a/src/OpenPostbud/database/digital_post/letters.py +++ b/src/OpenPostbud/database/digital_post/letters.py @@ -10,7 +10,7 @@ import re from mailmerge import MailMerge -from sqlalchemy import ForeignKey, insert, select, String +from sqlalchemy import ForeignKey, insert, select, String, update from sqlalchemy.orm import Mapped, mapped_column import requests @@ -44,7 +44,7 @@ class LetterStatus(Enum): WAITING = "Afventer" SENDING = "Behandles" SENT = "Afsendt" - RECEIVED = "Modtaget" + DELIVERED = "Leveret" FAILED = "Fejlet" @@ -93,6 +93,32 @@ def get_letter_template(self) -> Template: q = select(Template).join(Shipment).join(Letter).where(Letter.id == self.id) return session.execute(q).scalar_one() + def set_status(self, status: LetterStatus, transaction_id: str | None = None, message: str | None = None): + """Set the status of the letter in the database. + The transaction id is not overwritten if the given value is None. + + Args: + status: The status to set on the letter. + transaction_id: The transaction id from Digital Post. Defaults to None. + message: The message to set on the letter. Defaults to None. + """ + values = {} + values["status"] = status + values["updated_at"] = datetime.now() + values["message"] = message + if transaction_id: + values["transaction_id"] = transaction_id + + + with connection.get_session() as session: + q = ( + update(Letter) + .where(Letter.id == self.id) + .values(values) + ) + session.execute(q) + session.commit() + def add_letters(shipment_id: str, csv_data: list[dict[str, str]]): """Add multiple new letters to the database based diff --git a/src/OpenPostbud/workers/shipment_worker.py b/src/OpenPostbud/workers/shipment_worker.py index 079a06f..8fa52fe 100644 --- a/src/OpenPostbud/workers/shipment_worker.py +++ b/src/OpenPostbud/workers/shipment_worker.py @@ -24,7 +24,7 @@ def start_process(): """The entry point of the worker process. Raises: - RuntimeError: If any exception is raised when handling a task. + ValueError: If the Kombit certificate file couldn't be found. """ if not os.path.isfile(config.KOMBIT_CERT_PATH): raise ValueError(f"Couldn't find certificate file: {config.KOMBIT_CERT_PATH}") @@ -39,7 +39,7 @@ def start_process(): try: send_letter(letter, kombit_access) except Exception as e: # pylint: disable=broad-exception-caught - set_letter_status(letter, LetterStatus.FAILED, message="Systemfejl") + letter.set_status(LetterStatus.FAILED, message="Systemfejl") logging.error(f"Sending letter {letter.id} failed: {e}") else: logging.info(f"Sleeping for {config.SHIPMENT_WORKER_SLEEP_TIME} seconds") @@ -84,7 +84,7 @@ def send_letter(letter: Letter, kombit_access: KombitAccess): First checks if the recipient is registered to receive Digital Post. """ if not digital_post.is_registered(letter.recipient_id, 'digitalpost', kombit_access): - set_letter_status(letter, LetterStatus.FAILED, message="Ikke tilmeldt Digital Post") + letter.set_status(LetterStatus.FAILED, message="Ikke tilmeldt Digital Post") logging.info(f"Letter not sent. The recipient is not registered for Digital Post. {letter.id}") return @@ -93,10 +93,12 @@ def send_letter(letter: Letter, kombit_access: KombitAccess): label = json.loads(letter.field_data)[MemoFields.MEMO_LABEL.key] + message_uuid = str(uuid.uuid4()) + message = Message( messageHeader=MessageHeader( messageType="DIGITALPOST", - messageUUID=str(uuid.uuid4()), + messageUUID=message_uuid, label=label, sender=Sender( senderID=config.CVR, @@ -125,32 +127,8 @@ def send_letter(letter: Letter, kombit_access: KombitAccess): logging.info(f"Sending letter {letter.id}") transaction_id = digital_post.send_message("Digital Post", message, kombit_access) - set_letter_status(letter, LetterStatus.SENT, transaction_id) - logging.info(f"Letter sent {letter.id}") - - -def set_letter_status(letter: Letter, status: LetterStatus, transaction_id: str | None = None, message: str | None = None): - """Set the status of a letter in the database. - Optionally also set the transaction id of a shipped letter. - - Args: - letter: The letter to set the status on. - status: The status to set on the letter. - transaction_id: The transaction id from Digital Post. Defaults to None. - """ - with connection.get_session() as session: - q = ( - update(Letter) - .where(Letter.id == letter.id) - .values( - status=status, - updated_at=datetime.now(), - transaction_id=transaction_id, - message=message - ) - ) - session.execute(q) - session.commit() + letter.set_status(LetterStatus.SENT, message_uuid) + logging.info(f"Letter sent {letter.id} - {message_uuid=} - {transaction_id=}") if __name__ == '__main__': From f142d77ed4ae261fb3d94ebc1f770faf4bb94a55 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 08:57:20 +0200 Subject: [PATCH 04/24] Implemented message broker worker --- .../workers/message_broker_worker.py | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/OpenPostbud/workers/message_broker_worker.py diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py new file mode 100644 index 0000000..66eb28a --- /dev/null +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -0,0 +1,170 @@ +from xml.etree import ElementTree +import base64 +from datetime import datetime +import logging +import os +import time + +from sqlalchemy import select +from python_serviceplatformen.authentication import KombitAccess +from python_serviceplatformen import message_broker + +from OpenPostbud import config +from OpenPostbud.database import connection +from OpenPostbud.database.digital_post.letters import Letter, LetterStatus + + +# Sender uuids from the Beskedfordeler docs +SENDERS = { + "96514e13-afdd-44d6-95a8-adc2ca19b127": "Digital Post", + "afd21f3d-11c7-4f51-b2a6-f31d6480a9fb": "Strålfors", + "ae5b9a93-c923-40d7-a41a-1eec18374e27": "Edora", + "6b12182a-4268-4130-bd3e-159f8862c0a1": "KMD Charlie Tango" +} + +# Physical letter event uuids from the Beskedfordeler docs +EVENTS_PHYSICAL = { + "db5a6025-caa3-45c3-8e02-4e6fa8142ade": "Afsendt", + "dd98e71c-41ac-4305-a47b-8f86b00e639b": "Modtaget Fjern-print", + "e225a75c-4b63-46c4-9423-77f5c8762445": "Fejlet", + "e2b30d57-504a-4e2e-ae2d-a1394a9cb0b8": "Klar", + "e3c0a021-2070-40d4-b7b7-754aeba762e9": "Afleveret til print og kuvertering", + "e94a0a8b-60a0-42ad-8b83-52c9e15d0fb3": "Modtaget Post Danmark", + "f03904aa-df6e-417f-bd74-6a01a61adbcf": "Tilbagekaldt", + "f4e6eb11-0261-4198-8b5d-be92a9b1a35d": "Opdatering fra Post Danmark" +} + +# Digital letter events uuids from the Beskedfordeler docs +EVENTS_DIGITAL = { + "e225a75c-4b63-46c4-9423-77f5c8762445": "Fejlet", + "f7161a89-5068-4023-bc80-d7f4daad2a2e": "Afleveret Digital Post", + "eb866ca2-b871-4387-b501-12cde577bd58": "Modtaget Digital Post" +} + +# Mapping from Beskedfordeler events to OpenPostbud letter statuses +EVENT_MAP = { + # Common + "Fejlet": LetterStatus.FAILED, + + # Digital + "Afleveret Digital Post": LetterStatus.DELIVERED, + "Modtaget Digital Post": LetterStatus.DELIVERED, + + # Physical + "Afsendt": LetterStatus.SENT, + "Modtaget Fjern-print": LetterStatus.SENT, + "Klar": LetterStatus.SENT, + "Afleveret til print og kuvertering": LetterStatus.SENT, + "Modtaget Post Danmark": LetterStatus.SENT, + "Tilbagekaldt": LetterStatus.SENT, + "Opdatering fra Post Danmark": LetterStatus.SENT +} + +ENVELOPE_NAMESPACES = { + "default": "urn:oio:sagdok:3.0.0", + "kuvert": "urn:oio:besked:kuvert:1.0", + "besked": "urn:besked:kuvert:1.0" +} + +MESSAGE_NAMESPACES = { + "default": "http://serviceplatformen.dk/xml/print/PKO_PostStatus/1/types" +} + + +def start_process(): + """The entry point of the worker process. + + Raises: + ValueError: If the Kombit certificate file couldn't be found. + """ + if not os.path.isfile(config.KOMBIT_CERT_PATH): + raise ValueError(f"Couldn't find certificate file: {config.KOMBIT_CERT_PATH}") + kombit_access = KombitAccess(config.CVR, config.KOMBIT_CERT_PATH, test=config.KOMBIT_TEST_ENV) + + logging.info("Message broker worker started") + + while True: + logging.info(f"Checking queue for new messages") + for message in message_broker.iterate_queue_messages(config.MESSAGE_BROKER_QUEUE_ID, kombit_access, True): + handle_message(message.decode()) + + logging.info(f"Sleeping for {config.MESSAGE_BROKER_WORKER_SLEEP_TIME} seconds") + time.sleep(config.MESSAGE_BROKER_WORKER_SLEEP_TIME) + + +def handle_message(message: str): + """Decode an incoming message from the message broker. + Compare the sender id and event id to the known list of ids. + If the sender id or event id are not recognized log an error + and ignore the message. + + Args: + message: The XML message as a string. + """ + # Decode envelope + envelope_tree = ElementTree.fromstring(message) + + sender_uuid = envelope_tree.find("kuvert:Beskedkuvert/kuvert:Filtreringsdata/kuvert:BeskedAnsvarligAktoer/default:UUIDIdentifikator", ENVELOPE_NAMESPACES).text + sender_name = SENDERS.get(sender_uuid) + + event_uuid = envelope_tree.find("kuvert:Beskedkuvert/kuvert:Filtreringsdata/kuvert:ObjektRegistrering/kuvert:ObjektHandling/default:UUIDIdentifikator", ENVELOPE_NAMESPACES).text + event_name = EVENTS_DIGITAL.get(event_uuid) or EVENTS_PHYSICAL.get(event_uuid) + + message_time = envelope_tree.find("kuvert:Beskedkuvert/kuvert:Leveranceinformation/kuvert:Dannelsestidspunkt/default:TidsstempelDatoTid", ENVELOPE_NAMESPACES).text + message_time = datetime.fromisoformat(message_time) + + if not sender_name or not event_name: + logging.error(f"Unknown message received. Sender: {sender_name or sender_uuid} - Event: {event_name or event_uuid} - Message time: {message_time}") + return + + message_data = envelope_tree.find("kuvert:Beskeddata/besked:Base64", ENVELOPE_NAMESPACES).text + message_data = base64.b64decode(message_data).decode() + + # Decode message + message_tree = ElementTree.fromstring(message_data) + message_uuid = message_tree.find("default:MessageUUID", MESSAGE_NAMESPACES).text + error_message = message_tree.find("default:FejlDetaljer/default:FejlTekst", MESSAGE_NAMESPACES) + if error_message is not None: + error_message = error_message.text + + logging.info(f"Message received: {message_time} - {sender_name=} - {event_name=} - {message_uuid=} - {error_message=}") + + update_letter_status(message_uuid, event_name, error_message) + + +def update_letter_status(transaction_id: str, event_name: str, error: str | None): + """Update the status of a letter in the database that matches the given transaction id. + + Args: + transaction_id: The transaction id from the message broker message. + event_name: The name of the message event as defined in the event dicts. + error_message: The error from the message if any. + """ + with connection.get_session() as session: + q = select(Letter).where(Letter.transaction_id == transaction_id) + letter = session.execute(q).scalar_one_or_none() + + if not letter: + logging.error(f"No letter with transaction id {transaction_id} found in database.") + return + + letter_status = EVENT_MAP[event_name] + + if letter_status == LetterStatus.DELIVERED: + letter.set_status(LetterStatus.DELIVERED) + elif letter_status == LetterStatus.FAILED: + letter.set_status(LetterStatus.FAILED, message=error) + elif letter_status == LetterStatus.SENT: + letter.set_status(LetterStatus.SENT, message=event_name) + + logging.info(f"Status updated on letter {letter.id}") + + + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + for i in range(1): + with open(rf"C:\Users\az68933\Downloads\message_new{i}.xml") as file: + handle_message(file.read()) + print("---") From abd4f6e37d2eb4c818006b8f162cefd750cff668 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 08:57:38 +0200 Subject: [PATCH 05/24] Updated docker compose files with new worker --- docker-compose.server.yml | 16 ++++++++++++++++ docker-compose.yml | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/docker-compose.server.yml b/docker-compose.server.yml index c7ecaa3..af8aa56 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -82,6 +82,22 @@ services: environment: TZ: Europe/Copenhagen + message_broker_worker: + build: + context: . + dockerfile: .docker/app/Dockerfile + command: "python src/OpenPostbud/workers/message_broker_worker.py" + restart: unless-stopped + depends_on: + - app + networks: + - app + volumes: + - ./:/app + - ./certs:/app/certs:ro + environment: + TZ: Europe/Copenhagen + office: build: context: . diff --git a/docker-compose.yml b/docker-compose.yml index db938a1..f69c2e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,6 +80,22 @@ services: environment: TZ: Europe/Copenhagen + message_broker_worker: + build: + context: . + dockerfile: .docker/app/Dockerfile + command: "python src/OpenPostbud/workers/message_broker_worker.py" + restart: unless-stopped + depends_on: + - app + networks: + - app + volumes: + - ./:/app + - ./certs:/app/certs:ro + environment: + TZ: Europe/Copenhagen + office: build: context: . From 3a7054750a7d6fa9fd729dd56a4e0afaaca9f213 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 09:16:08 +0200 Subject: [PATCH 06/24] Moved certificate check to config.py --- src/OpenPostbud/config.py | 4 ++++ src/OpenPostbud/workers/message_broker_worker.py | 2 -- src/OpenPostbud/workers/shipment_worker.py | 2 -- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OpenPostbud/config.py b/src/OpenPostbud/config.py index c7f24dd..4910a89 100644 --- a/src/OpenPostbud/config.py +++ b/src/OpenPostbud/config.py @@ -16,8 +16,10 @@ def str_to_bool(s: str) -> bool: return s.lower() == "true" +# Set logging options for all processes logging.basicConfig(level=logging.INFO, format="%(levelname)s | %(asctime)s | %(message)s", datefmt="%m/%d/%Y %H:%M:%S%z") +# Load .env file ENV_PATH = ".env" if not os.path.isfile(ENV_PATH): @@ -33,6 +35,8 @@ def str_to_bool(s: str) -> bool: # Workers CVR = os.environ['cvr'] KOMBIT_CERT_PATH = os.environ['kombit_cert_path'] +if not os.path.isfile(KOMBIT_CERT_PATH): + raise ValueError(f"Couldn't find certificate file: {KOMBIT_CERT_PATH}") KOMBIT_TEST_ENV = str_to_bool(os.environ['Kombit_test_env']) # Registration worker diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index 66eb28a..09bd6f1 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -77,8 +77,6 @@ def start_process(): Raises: ValueError: If the Kombit certificate file couldn't be found. """ - if not os.path.isfile(config.KOMBIT_CERT_PATH): - raise ValueError(f"Couldn't find certificate file: {config.KOMBIT_CERT_PATH}") kombit_access = KombitAccess(config.CVR, config.KOMBIT_CERT_PATH, test=config.KOMBIT_TEST_ENV) logging.info("Message broker worker started") diff --git a/src/OpenPostbud/workers/shipment_worker.py b/src/OpenPostbud/workers/shipment_worker.py index 8fa52fe..312ed9a 100644 --- a/src/OpenPostbud/workers/shipment_worker.py +++ b/src/OpenPostbud/workers/shipment_worker.py @@ -26,8 +26,6 @@ def start_process(): Raises: ValueError: If the Kombit certificate file couldn't be found. """ - if not os.path.isfile(config.KOMBIT_CERT_PATH): - raise ValueError(f"Couldn't find certificate file: {config.KOMBIT_CERT_PATH}") kombit_access = KombitAccess(config.CVR, config.KOMBIT_CERT_PATH, test=config.KOMBIT_TEST_ENV) logging.info("Shipment worker started") From 5bcc92fd6f34fd6dbd2b49008418041309e76f7f Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 09:33:29 +0200 Subject: [PATCH 07/24] Added pdf converter as dependency for app --- docker-compose.server.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.server.yml b/docker-compose.server.yml index af8aa56..559cd53 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -41,6 +41,8 @@ services: dockerfile: .docker/app/Dockerfile command: "python src/OpenPostbud/main.py" restart: unless-stopped + depends_on: + - office networks: - app volumes: From 6ed1090e83a2efa51892e3c63c95b1301c7a85c6 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 09:34:20 +0200 Subject: [PATCH 08/24] Removed unused CLI commands --- README.md | 22 ---------------------- src/OpenPostbud/__main__.py | 29 ----------------------------- 2 files changed, 51 deletions(-) diff --git a/README.md b/README.md index 030fff5..91e613d 100644 --- a/README.md +++ b/README.md @@ -75,28 +75,6 @@ Under development it's possible to set some environment variables to help with t OpenPostbud adds a command line executable called `OpenPostbud`. Use `OpenPostbud -h` to see help information about the CLI. -### OpenPostbud app - -To run the app execute the following command: - -```bash -OpenPostbud run -``` - -This will start a Uvicorn server which listens on port 8000. - -### Workers - -To run the workers: - -```bash -OpenPostbud registration_worker -OpenPostbud shipment_worker -``` - -These workers will run in an infinite loop where they check the database for tasks. If there are no tasks the -workers will sleep for a set amount of time. - ## Database OpenPostbud uses SQLite and it creates an SQLite database in the current working directory called `database.db`. diff --git a/src/OpenPostbud/__main__.py b/src/OpenPostbud/__main__.py index b670cfb..4db20bb 100644 --- a/src/OpenPostbud/__main__.py +++ b/src/OpenPostbud/__main__.py @@ -2,9 +2,7 @@ import argparse -import OpenPostbud.main from OpenPostbud.middleware import authentication -from OpenPostbud.workers import registration_worker, shipment_worker # pylint: disable=unused-argument @@ -13,24 +11,6 @@ def admin_access_command(args: argparse.Namespace): authentication.grant_admin_access() -# pylint: disable=unused-argument -def run_command(args: argparse.Namespace): - """The command to run on the 'run' subcommand.""" - OpenPostbud.main.main(reload=False) - - -# pylint: disable=unused-argument -def r_worker_command(args: argparse.Namespace): - """The command to run on the 'registration_worker' subcommand.""" - registration_worker.start_process() - - -# pylint: disable=unused-argument -def s_worker_command(args: argparse.Namespace): - """The command to run on the 'shipment_worker' subcommand.""" - shipment_worker.start_process() - - def main(): """Main entry point for the CLI.""" parser = argparse.ArgumentParser( @@ -43,15 +23,6 @@ def main(): admin_parser = subparsers.add_parser("admin_access", help="Generate a single-use admin URL to the web app.") admin_parser.set_defaults(func=admin_access_command) - run_parser = subparsers.add_parser("run", help="Run the web application.") - run_parser.set_defaults(func=run_command) - - r_worker_parser = subparsers.add_parser("registration_worker", help="Start the registration worker.") - r_worker_parser.set_defaults(func=r_worker_command) - - s_worker_parser = subparsers.add_parser("shipment_worker", help="Start the shipment worker.") - s_worker_parser.set_defaults(func=s_worker_command) - args = parser.parse_args() args.func(args) From b975c6907749d55227bb334d3439a7613e876451 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 09:34:43 +0200 Subject: [PATCH 09/24] Removed libreoffice path from .env --- .env.example | 1 - src/OpenPostbud/config.py | 1 - src/OpenPostbud/workers/pdf_converter.py | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 95f87e4..f33b674 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,6 @@ registration_worker_sleep_time=10 # Shipment worker settings shipment_worker_sleep_time=10 sender_label=Something Corp -path_to_libreoffice=somewhere/LibreOffice/program/soffice.exe # Message broker settings message_broker_queue_id=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx diff --git a/src/OpenPostbud/config.py b/src/OpenPostbud/config.py index 4910a89..293990f 100644 --- a/src/OpenPostbud/config.py +++ b/src/OpenPostbud/config.py @@ -45,7 +45,6 @@ def str_to_bool(s: str) -> bool: # Shipment worker SHIPMENT_WORKER_SLEEP_TIME = float(os.environ['shipment_worker_sleep_time']) SENDER_LABEL = os.environ['sender_label'] -PATH_TO_LIBREOFFICE = os.environ['path_to_libreoffice'] # Message broker worker MESSAGE_BROKER_QUEUE_ID = os.environ['message_broker_queue_id'] diff --git a/src/OpenPostbud/workers/pdf_converter.py b/src/OpenPostbud/workers/pdf_converter.py index 4563145..7fe3abe 100644 --- a/src/OpenPostbud/workers/pdf_converter.py +++ b/src/OpenPostbud/workers/pdf_converter.py @@ -11,8 +11,6 @@ from fastapi.responses import Response import uvicorn -from OpenPostbud import config - app = FastAPI() @@ -39,7 +37,7 @@ async def convert_to_pdf(word_file: Annotated[bytes, File()]): word_path.write_bytes(word_file) - process = await asyncio.create_subprocess_exec(config.PATH_TO_LIBREOFFICE, "--headless", "--convert-to", "pdf", "--outdir", tmpdir, str(word_path), + process = await asyncio.create_subprocess_exec("libreoffice", "--headless", "--convert-to", "pdf", "--outdir", tmpdir, str(word_path), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) stdout, stderr = await process.communicate() From 7c3190cdbd19d953e2d350ac3ac9dcc6d168b298 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 10:03:07 +0200 Subject: [PATCH 10/24] Updated readme --- README.md | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 91e613d..11b7b6f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ ## Introduction OpenPostbud is a web application that makes it possible to do mail merge and mass shipment of Digital Post -using Service Platformen. +using Kombit's Serviceplatformen. OpenPostbud is split into two logical parts: The web app and the task workers. The web app is the frontend presented to the user. The workers run in separate processes @@ -11,21 +11,16 @@ performing any queued up shipment or registration tasks. ## Installation -OpenPostbud needs Python (>=3.11), pip and setuptools installed to be installed. +OpenPostbud comes with a Docker compose file that will install and start all necessary processes. -To install OpenPostbud navigate to the main folder where pyproject.toml is located and call - -```bash -pip install . -``` - -This will build and install OpenPostbud into the active environment. -It will also install any dependencies from pip. +Remember to create a `.env` file with all needed environment variables. +See below for explanations as well as `.env.example`. ### Libre Office OpenPostbud uses Libre Office to convert docx files to pdf. It does so by calling the Libre Office executable in the command line. +This is automatically installed by Docker. ## Environment variables @@ -49,17 +44,18 @@ OpenPostbud needs the following environment variables set: ### Workers -The shipment and registration workers need the following environment variables set: +The shipment, registration and message broker workers need the following environment variables set: -| Name | description | Type | -|--------------------------------|-----------------------------------------------------------------------|-------------| -| cvr | The CVR number of the organisation | String | -| kombit_cert_path | The absolute path to the certificate file used for Service Platformen | Path string | -| Kombit_test_env | Whether to use the test environment of Service Platformen | boolean | -| registration_worker_sleep_time | The number of seconds for the registration worker to idle | Integer | -| shipment_worker_sleep_time | The number of seconds for the shipment worker to idle | Integer | -| sender_label | The label to set on the sender of Digital Post | String | -| path_to_libreoffice | The absolute path to the Libre Office executable | Path string | +| Name | description | Type | +| -------------------------------- | ------------------------------------------------------------------------- | ----------- | +| cvr | The CVR number of the organisation | String | +| kombit_cert_path | The absolute path to the certificate file used for Service Platformen | Path string | +| Kombit_test_env | Whether to use the test environment of Service Platformen | boolean | +| registration_worker_sleep_time | The number of seconds for the registration worker to idle | Integer | +| shipment_worker_sleep_time | The number of seconds for the shipment worker to idle | Integer | +| sender_label | The label to set on the sender of Digital Post | String | +| message_broker_queue_id | The UUID of the message broker queue. Get this from the Kombit admin page | UUID | +| message_broker_worker_sleep_time | The number of seconds for the message broker worker to idle | Integer | ### Development @@ -83,6 +79,6 @@ Some sensitive columns in the database are encrypted using AES. ## Authentication -The OpenPostbud web app uses Microsoft OIDC to authenticate users. This needs to be set up in the Microsoft Entra. +The OpenPostbud web app uses Microsoft OIDC to authenticate users. This needs to be set up in Microsoft Entra. Admins can use the CLI command `OpenPostbud admin_access` to get a single-use login URL. From 04a374777b467215d98ad78b01148bf731d534b2 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 10:21:31 +0200 Subject: [PATCH 11/24] Updated dependency --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 606b3c4..5eea2dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "nicegui == 2.*", "python-dotenv == 1.*", "docx-mailmerge2 == 0.8.2", - "python_serviceplatformen == 3.*", + "python_serviceplatformen >= 3.1, < 4.0", "passlib == 1.7.*", "PyJWT == 2.10.*" ] From 2dcc8fee49b0e2c3b9fe904292095480b84b5108 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 10:32:52 +0200 Subject: [PATCH 12/24] Bug fix --- src/OpenPostbud/workers/message_broker_worker.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index 09bd6f1..799a5e3 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -158,11 +158,5 @@ def update_letter_status(transaction_id: str, event_name: str, error: str | None logging.info(f"Status updated on letter {letter.id}") - - if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) - for i in range(1): - with open(rf"C:\Users\az68933\Downloads\message_new{i}.xml") as file: - handle_message(file.read()) - print("---") + start_process() From fe2ab1cc979797d32515d5773a1a82fb56eb22f1 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 11:26:50 +0200 Subject: [PATCH 13/24] Linting --- .../database/check_registration/registration_job.py | 3 +++ src/OpenPostbud/database/digital_post/db_util.py | 2 +- src/OpenPostbud/workers/message_broker_worker.py | 7 +++++-- src/OpenPostbud/workers/shipment_worker.py | 1 - 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/OpenPostbud/database/check_registration/registration_job.py b/src/OpenPostbud/database/check_registration/registration_job.py index 6a4a642..9257ff8 100644 --- a/src/OpenPostbud/database/check_registration/registration_job.py +++ b/src/OpenPostbud/database/check_registration/registration_job.py @@ -17,6 +17,8 @@ class JobType(Enum): DIGITAL_POST = "digitalpost" +# We don't care about duplicate code for ORM classes. +# pylint: disable=duplicate-code class RegistrationJob(Base): """An ORM class representing a registration job. A job is a collection of multiple tasks. @@ -40,6 +42,7 @@ def to_row_dict(self) -> dict[str, str]: "created_at": self.created_at.strftime("%d-%m-%Y %H:%M:%S"), "created_by": self.created_by } +# pylint: enable=duplicate-code def add_registation_job(name: str, description: str, job_type: JobType, created_by: str) -> int: diff --git a/src/OpenPostbud/database/digital_post/db_util.py b/src/OpenPostbud/database/digital_post/db_util.py index bb48529..719e0aa 100644 --- a/src/OpenPostbud/database/digital_post/db_util.py +++ b/src/OpenPostbud/database/digital_post/db_util.py @@ -19,7 +19,7 @@ def calculate_shipment_status(shipment_id: str) -> list[tuple[str, int]]: """ with connection.get_session() as session: query = ( - select(Letter.status, func.count(Letter.status)) + select(Letter.status, func.count(Letter.status)) # pylint: disable=not-callable .where(Letter.shipment_id == shipment_id) .group_by(Letter.status) ) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index 799a5e3..e819141 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -1,8 +1,11 @@ +"""This module defines the worker process that fetches message from Kombit's Beskedfordeler. +It is spawned as a separate process next to the UI process. +""" + from xml.etree import ElementTree import base64 from datetime import datetime import logging -import os import time from sqlalchemy import select @@ -82,7 +85,7 @@ def start_process(): logging.info("Message broker worker started") while True: - logging.info(f"Checking queue for new messages") + logging.info("Checking queue for new messages") for message in message_broker.iterate_queue_messages(config.MESSAGE_BROKER_QUEUE_ID, kombit_access, True): handle_message(message.decode()) diff --git a/src/OpenPostbud/workers/shipment_worker.py b/src/OpenPostbud/workers/shipment_worker.py index 312ed9a..4bf748e 100644 --- a/src/OpenPostbud/workers/shipment_worker.py +++ b/src/OpenPostbud/workers/shipment_worker.py @@ -2,7 +2,6 @@ It is spawned as a separate process next to the UI process. """ -import os import base64 from datetime import datetime import logging From 2a53daf86b173383d0b66dbe849d668a51b44bc7 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 11:30:13 +0200 Subject: [PATCH 14/24] Lint --- src/OpenPostbud/database/digital_post/letters.py | 1 - src/OpenPostbud/routes/user/send_post.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/OpenPostbud/database/digital_post/letters.py b/src/OpenPostbud/database/digital_post/letters.py index 46aa91f..63ec596 100644 --- a/src/OpenPostbud/database/digital_post/letters.py +++ b/src/OpenPostbud/database/digital_post/letters.py @@ -109,7 +109,6 @@ def set_status(self, status: LetterStatus, transaction_id: str | None = None, me if transaction_id: values["transaction_id"] = transaction_id - with connection.get_session() as session: q = ( update(Letter) diff --git a/src/OpenPostbud/routes/user/send_post.py b/src/OpenPostbud/routes/user/send_post.py index 515cc83..0c7d906 100644 --- a/src/OpenPostbud/routes/user/send_post.py +++ b/src/OpenPostbud/routes/user/send_post.py @@ -296,10 +296,10 @@ def _verify_csv_data(fields: list[str], csv_list: list[dict], message_area: ui_c # Check for duplicate receivers if MemoFields.MEMO_MODTAGER.key in fields: - c = Counter((line[MemoFields.MEMO_MODTAGER.key] for line in csv_list)) - l = [f"{k}: {v}" for k, v in c.items() if v > 1] - if l: - message_area.add_message(f"Duplikater fundet i '{MemoFields.MEMO_MODTAGER.key}': " + " - ".join(l), type_='warning') + counter = Counter((line[MemoFields.MEMO_MODTAGER.key] for line in csv_list)) + duplicates = [f"{k}: {v}" for k, v in counter.items() if v > 1] + if duplicates: + message_area.add_message(f"Duplikater fundet i '{MemoFields.MEMO_MODTAGER.key}': " + " - ".join(duplicates), type_='warning') error = True # Check for mandatory fields From 64274c095df1db687668c76406ce6f0147dcc20b Mon Sep 17 00:00:00 2001 From: Mathias G Date: Wed, 8 Oct 2025 11:33:49 +0200 Subject: [PATCH 15/24] Lint --- src/OpenPostbud/routes/user/forsendelser.py | 2 +- src/OpenPostbud/ui_components.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OpenPostbud/routes/user/forsendelser.py b/src/OpenPostbud/routes/user/forsendelser.py index dce232d..c696287 100644 --- a/src/OpenPostbud/routes/user/forsendelser.py +++ b/src/OpenPostbud/routes/user/forsendelser.py @@ -51,7 +51,7 @@ def __init__(self) -> None: shipment_list = shipments.get_shipments() rows = [s.to_row_dict() for s in shipment_list] - table = ui_components.SearchTable(title="Forsendelser", columns=SHIPMENTS_COLUMNS, column_defaults=COLUMN_DEFAULTS, rows=rows, row_key="id", pagination=50) + table = ui_components.SearchTable(title="Forsendelser", columns=SHIPMENTS_COLUMNS, column_defaults=COLUMN_DEFAULTS, rows=rows, row_key="id", pagination=50, download_button=True, search_field=True) table.on("rowClick", self._row_click) def _row_click(self, event): diff --git a/src/OpenPostbud/ui_components.py b/src/OpenPostbud/ui_components.py index dc65b67..a14f897 100644 --- a/src/OpenPostbud/ui_components.py +++ b/src/OpenPostbud/ui_components.py @@ -157,8 +157,8 @@ def add_message(self, text: str, type_: Literal["positive", "warning", "negative class SearchTable(ui.table): """An extension of ui.table that has a search field in the top slot.""" - def __init__(self, *, rows, columns = None, column_defaults = None, row_key = 'id', title = None, selection = None, pagination = None, on_select = None, on_pagination_change = None, - search_field: bool, download_button: bool): # pylint: disable=too-many-arguments + def __init__(self, *, rows, columns = None, column_defaults = None, row_key = 'id', title = None, selection = None, pagination = None, on_select = None, on_pagination_change = None, # pylint: disable=too-many-arguments + search_field: bool, download_button: bool): super().__init__(rows=rows, columns=columns, column_defaults=column_defaults, row_key=row_key, selection=selection, pagination=pagination, on_select=on_select, on_pagination_change=on_pagination_change) with self.add_slot("top"): ui.label(title).classes("q-table__title") @@ -179,4 +179,4 @@ def _download_list(self): writer.writerow(field_labels) writer.writerows(self.rows) - ui.download(f.getvalue().encode(), "Liste.csv") \ No newline at end of file + ui.download(f.getvalue().encode(), "Liste.csv") From eea8716014d542b9fb3622f2f09ec989f8404807 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Thu, 9 Oct 2025 13:28:16 +0200 Subject: [PATCH 16/24] Removed unused folder --- certs/.gitignore | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 certs/.gitignore diff --git a/certs/.gitignore b/certs/.gitignore deleted file mode 100644 index 23f196c..0000000 --- a/certs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*.pem -*.crt From 78cc4e16fa51a79623bc4ae0b018b20d86b2f288 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Thu, 9 Oct 2025 13:28:30 +0200 Subject: [PATCH 17/24] Silenced pika logging --- src/OpenPostbud/workers/message_broker_worker.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index e819141..f6c7362 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -17,6 +17,10 @@ from OpenPostbud.database.digital_post.letters import Letter, LetterStatus +# Silence pika's DEBUG and INFO messages +logging.getLogger("pika").setLevel(logging.WARNING) + + # Sender uuids from the Beskedfordeler docs SENDERS = { "96514e13-afdd-44d6-95a8-adc2ca19b127": "Digital Post", From 595437af3d3788265ecb0e4203e1a51500ab5130 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Mon, 27 Oct 2025 12:29:27 +0100 Subject: [PATCH 18/24] Added handling of errors and physical mail messages --- .../workers/message_broker_worker.py | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index f6c7362..3b01e95 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -7,6 +7,8 @@ from datetime import datetime import logging import time +import uuid +from pathlib import Path from sqlalchemy import select from python_serviceplatformen.authentication import KombitAccess @@ -91,7 +93,11 @@ def start_process(): while True: logging.info("Checking queue for new messages") for message in message_broker.iterate_queue_messages(config.MESSAGE_BROKER_QUEUE_ID, kombit_access, True): - handle_message(message.decode()) + message = message.decode() + try: + handle_message(message) + except AttributeError: + save_failed_message(message) logging.info(f"Sleeping for {config.MESSAGE_BROKER_WORKER_SLEEP_TIME} seconds") time.sleep(config.MESSAGE_BROKER_WORKER_SLEEP_TIME) @@ -120,11 +126,29 @@ def handle_message(message: str): if not sender_name or not event_name: logging.error(f"Unknown message received. Sender: {sender_name or sender_uuid} - Event: {event_name or event_uuid} - Message time: {message_time}") + save_failed_message(message) return + logging.info(f"Message received: {message_time} - {sender_name=} - {event_name=}") + message_data = envelope_tree.find("kuvert:Beskeddata/besked:Base64", ENVELOPE_NAMESPACES).text message_data = base64.b64decode(message_data).decode() + if event_uuid in EVENTS_DIGITAL: + handle_digital_post_message(envelope_tree, message_time, sender_name, event_name, message_data) + else: + handle_physical_mail_message() + + +def handle_digital_post_message(message_time: str, sender_name: str, event_name: str, message_data: str): + """Handle a message from the Digital Post sender. + + Args: + message_time: The message time from the message. + sender_name: The sender name from the message. + event_name: The event name from the message. + message_data: The decoded base64 message data. + """ # Decode message message_tree = ElementTree.fromstring(message_data) message_uuid = message_tree.find("default:MessageUUID", MESSAGE_NAMESPACES).text @@ -137,6 +161,13 @@ def handle_message(message: str): update_letter_status(message_uuid, event_name, error_message) +def handle_physical_mail_message(): + """Handle a message from the a physical mail sender.""" + # We currently don't support physical mail. + # We don't know how to handle them properly. + logging.error("Physical mail messages are not currently supported.") + + def update_letter_status(transaction_id: str, event_name: str, error: str | None): """Update the status of a letter in the database that matches the given transaction id. @@ -165,5 +196,20 @@ def update_letter_status(transaction_id: str, event_name: str, error: str | None logging.info(f"Status updated on letter {letter.id}") +def save_failed_message(message: str): + """Save the given message to a file in the folder 'failed_messages'. + + Args: + message: The message to save to a file. + """ + folder = Path("failed_messages") + folder.mkdir(exist_ok=True) + + file_path = folder / Path(str(uuid.uuid4())).with_suffix(".xml") + file_path.write_text(message) + + logging.error(f"Error while reading message. Message saved to {file_path}") + + if __name__ == '__main__': start_process() From 2a4fdc9bc9b12081e98bf81d44344ded25e98496 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Mon, 27 Oct 2025 12:32:24 +0100 Subject: [PATCH 19/24] Lint --- src/OpenPostbud/workers/message_broker_worker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index 3b01e95..86d70b4 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -135,7 +135,12 @@ def handle_message(message: str): message_data = base64.b64decode(message_data).decode() if event_uuid in EVENTS_DIGITAL: - handle_digital_post_message(envelope_tree, message_time, sender_name, event_name, message_data) + handle_digital_post_message( + envelope_tree=envelope_tree, + message_time=message_time, + sender_name=sender_name, + event_name=event_name, + message_data=message_data) else: handle_physical_mail_message() From 797c3a29ba4682dba2c18537505a187951cd317c Mon Sep 17 00:00:00 2001 From: Mathias G Date: Mon, 27 Oct 2025 12:33:43 +0100 Subject: [PATCH 20/24] Lint --- src/OpenPostbud/database/digital_post/templates.py | 2 +- src/OpenPostbud/ui_components.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/OpenPostbud/database/digital_post/templates.py b/src/OpenPostbud/database/digital_post/templates.py index 4d348c9..59610d3 100644 --- a/src/OpenPostbud/database/digital_post/templates.py +++ b/src/OpenPostbud/database/digital_post/templates.py @@ -37,7 +37,7 @@ def add_template(file_name: str, file_data: bytes) -> int: template = Template( file_name=file_name, - file_data = file_data, + file_data=file_data, field_names=json.dumps(field_names) ) diff --git a/src/OpenPostbud/ui_components.py b/src/OpenPostbud/ui_components.py index a14f897..6e743f9 100644 --- a/src/OpenPostbud/ui_components.py +++ b/src/OpenPostbud/ui_components.py @@ -157,7 +157,7 @@ def add_message(self, text: str, type_: Literal["positive", "warning", "negative class SearchTable(ui.table): """An extension of ui.table that has a search field in the top slot.""" - def __init__(self, *, rows, columns = None, column_defaults = None, row_key = 'id', title = None, selection = None, pagination = None, on_select = None, on_pagination_change = None, # pylint: disable=too-many-arguments + def __init__(self, *, rows, columns=None, column_defaults=None, row_key='id', title=None, selection=None, pagination=None, on_select=None, on_pagination_change=None, # pylint: disable=too-many-arguments search_field: bool, download_button: bool): super().__init__(rows=rows, columns=columns, column_defaults=column_defaults, row_key=row_key, selection=selection, pagination=pagination, on_select=on_select, on_pagination_change=on_pagination_change) with self.add_slot("top"): From 73be82078f94baed4b8796ef6985a9c11417728e Mon Sep 17 00:00:00 2001 From: Mathias G Date: Tue, 28 Oct 2025 11:22:09 +0100 Subject: [PATCH 21/24] Added gotenberg to wordslist --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 61eee11..22c8e11 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,7 @@ { "cSpell.words": [ - "fastapi" + "fastapi", + "gotenberg" ], "search.useIgnoreFiles": true } \ No newline at end of file From ddefb7f2bf5a212f0f2df2b974a58618e3f607b2 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Tue, 28 Oct 2025 11:23:11 +0100 Subject: [PATCH 22/24] Changed to use Gotenberg pdf converter --- docker-compose.server.yml | 11 ++++------- docker-compose.yml | 13 ++++++------- src/OpenPostbud/database/digital_post/letters.py | 4 ++-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docker-compose.server.yml b/docker-compose.server.yml index 559cd53..6912422 100644 --- a/docker-compose.server.yml +++ b/docker-compose.server.yml @@ -42,7 +42,7 @@ services: command: "python src/OpenPostbud/main.py" restart: unless-stopped depends_on: - - office + - gotenberg networks: - app volumes: @@ -59,7 +59,7 @@ services: restart: unless-stopped depends_on: - app - - office + - gotenberg networks: - app volumes: @@ -100,11 +100,8 @@ services: environment: TZ: Europe/Copenhagen - office: - build: - context: . - dockerfile: .docker/office/Dockerfile - command: "python src/OpenPostbud/workers/pdf_converter.py" + gotenberg: + image: gotenberg/gotenberg:8 restart: unless-stopped networks: - app diff --git a/docker-compose.yml b/docker-compose.yml index f69c2e8..8501995 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - app - shipment_worker - registration_worker - - office + - gotenberg ports: - '8080' volumes: @@ -39,6 +39,8 @@ services: dockerfile: .docker/app/Dockerfile command: "python src/OpenPostbud/main.py" restart: unless-stopped + depends_on: + - gotenberg networks: - app volumes: @@ -55,7 +57,7 @@ services: restart: unless-stopped depends_on: - app - - office + - gotenberg networks: - app volumes: @@ -96,11 +98,8 @@ services: environment: TZ: Europe/Copenhagen - office: - build: - context: . - dockerfile: .docker/office/Dockerfile - command: "python src/OpenPostbud/workers/pdf_converter.py" + gotenberg: + image: gotenberg/gotenberg:8 restart: unless-stopped networks: - app diff --git a/src/OpenPostbud/database/digital_post/letters.py b/src/OpenPostbud/database/digital_post/letters.py index 63ec596..1b708f0 100644 --- a/src/OpenPostbud/database/digital_post/letters.py +++ b/src/OpenPostbud/database/digital_post/letters.py @@ -172,7 +172,7 @@ def merge_word_file(word_template: bytes, field_data: dict[str, str]) -> bytes: def convert_word_to_pdf(document: bytes) -> bytes: - """Convert a docx file to pdf using the PDF converter api. + """Convert a docx file to pdf using the Gotenberg PDF converter api. Args: document: The docx file as bytes. @@ -181,7 +181,7 @@ def convert_word_to_pdf(document: bytes) -> bytes: The converted pdf file as bytes. """ logging.info(f"Sending word file to converter. Size {len(document)}") - result = requests.post("http://office:8100", files={"word_file": document}, timeout=30) + result = requests.post("http://gotenberg:3000/forms/libreoffice/convert", files={"files": ("document.docx", document)}, timeout=30) result.raise_for_status() logging.info(f"Received pdf from converter. Size: {len(result.content)}") return result.content From d47df735ee9b9153d2166a8ad98cc06f36b456a1 Mon Sep 17 00:00:00 2001 From: Mathias G Date: Tue, 28 Oct 2025 11:23:22 +0100 Subject: [PATCH 23/24] Removed old pdf converter --- .docker/office/Dockerfile | 36 ---------------- src/OpenPostbud/workers/pdf_converter.py | 55 ------------------------ 2 files changed, 91 deletions(-) delete mode 100644 .docker/office/Dockerfile delete mode 100644 src/OpenPostbud/workers/pdf_converter.py diff --git a/.docker/office/Dockerfile b/.docker/office/Dockerfile deleted file mode 100644 index 138e880..0000000 --- a/.docker/office/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -# pull pyhton -FROM python:3.11-bookworm - -# Update PIP -RUN apt update && \ - pip install --no-cache-dir --upgrade pip - -WORKDIR /app - -# Install Libreoffice -RUN DEBIAN_FRONTEND=noninteractive \ - apt-get install -qy \ - libreoffice-common \ - libreoffice-writer \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# Install project and dependencies -RUN pip install uv -COPY pyproject.toml /app/pyproject.toml -RUN uv pip install --system -r /app/pyproject.toml -COPY src /app/src -RUN uv pip install --system -e . - -COPY .env /app/.env - -# Add deploy use to match server. -RUN addgroup --gid 1042 deploy \ - && useradd --gid 1042 --uid 1042 --home-dir /home/deploy --create-home --shell /bin/bash deploy - -# Ensure app is owned by deploy -RUN chown deploy:deploy /app - -USER deploy - -# Make port 8100 available to the world outside this container -EXPOSE 8100 diff --git a/src/OpenPostbud/workers/pdf_converter.py b/src/OpenPostbud/workers/pdf_converter.py deleted file mode 100644 index 7fe3abe..0000000 --- a/src/OpenPostbud/workers/pdf_converter.py +++ /dev/null @@ -1,55 +0,0 @@ -"""This module runs a FastApi application which exposes a single -endpoint for converting docx to pdf using LibreOffice.""" - -from typing import Annotated -import tempfile -from pathlib import Path -import asyncio -import logging - -from fastapi import FastAPI, File -from fastapi.responses import Response -import uvicorn - -app = FastAPI() - - -@app.post("/") -async def convert_to_pdf(word_file: Annotated[bytes, File()]): - """An endpoint for converting word files to pdf. - It works by writing the word file to a temp dir and then calling Libre Office - to convert to a pdf file. - Usage example: - with open("Test.docx", "rb") as file: - pdf_bytes = requests.post("http://127.0.0.1:8000", files={"word_file": file}).content - - Args: - word_file: The Word file to convert. - - Returns: - A HTTP response with the pdf files bytes as the content. - """ - logging.info(f"Word file received. Size: {len(word_file)}") - - with tempfile.TemporaryDirectory(prefix="OpenPostbud") as tmpdir: - word_path = Path(tmpdir) / Path("doc.docx") - pdf_path = word_path.with_suffix(".pdf") - - word_path.write_bytes(word_file) - - process = await asyncio.create_subprocess_exec("libreoffice", "--headless", "--convert-to", "pdf", "--outdir", tmpdir, str(word_path), - stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) - - stdout, stderr = await process.communicate() - - if process.returncode != 0: - raise RuntimeError(f"Libreoffice didn't return 0. Return code: {process.returncode} | stdout: {stdout.decode()} | stderr: {stderr.decode()}") - - pdf_bytes = pdf_path.read_bytes() - - logging.info(f"File converted. Result size: {len(pdf_bytes)}") - return Response(content=pdf_bytes) - - -if __name__ == "__main__": - uvicorn.run("pdf_converter:app", host="0.0.0.0", port=8100) From 00e4f703e8800c9304943c00c4a3675066d2aeba Mon Sep 17 00:00:00 2001 From: Mathias G Date: Tue, 28 Oct 2025 11:24:01 +0100 Subject: [PATCH 24/24] Bug fix in message broker worker --- src/OpenPostbud/workers/message_broker_worker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/OpenPostbud/workers/message_broker_worker.py b/src/OpenPostbud/workers/message_broker_worker.py index 86d70b4..b652eb4 100644 --- a/src/OpenPostbud/workers/message_broker_worker.py +++ b/src/OpenPostbud/workers/message_broker_worker.py @@ -136,7 +136,6 @@ def handle_message(message: str): if event_uuid in EVENTS_DIGITAL: handle_digital_post_message( - envelope_tree=envelope_tree, message_time=message_time, sender_name=sender_name, event_name=event_name,