From 367185725e2634a524e6b8c0822ddb4dbc6a19b1 Mon Sep 17 00:00:00 2001 From: Matteo Di Lorenzi Date: Thu, 12 Mar 2026 16:14:37 +0100 Subject: [PATCH] feat(ovpn-rw-conn-history): enhance connection history management and storage integration --- packages/ns-api/files/ns.controller | 11 +- packages/ns-api/files/ns.ovpnrw | 170 ++++++++++++++++-- packages/ns-api/files/ns.report | 19 +- packages/ns-openvpn/Makefile | 5 +- packages/ns-openvpn/README.md | 2 +- .../files/20-openvpn-merge-connections-db | 22 +++ packages/ns-openvpn/files/80-save-connection | 12 +- .../ns-openvpn/files/80-save-disconnection | 24 ++- packages/ns-openvpn/files/init-connections-db | 27 ++- .../files/openvpn-merge-connections-db | 113 ++++++++++++ packages/ns-storage/Makefile | 3 +- packages/ns-storage/files/add-storage | 5 +- packages/ns-storage/files/openvpn | 9 - 13 files changed, 381 insertions(+), 41 deletions(-) create mode 100644 packages/ns-openvpn/files/20-openvpn-merge-connections-db create mode 100644 packages/ns-openvpn/files/openvpn-merge-connections-db delete mode 100644 packages/ns-storage/files/openvpn diff --git a/packages/ns-api/files/ns.controller b/packages/ns-api/files/ns.controller index ca6d8cbcc..577536c02 100755 --- a/packages/ns-api/files/ns.controller +++ b/packages/ns-api/files/ns.controller @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2024 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -261,9 +261,14 @@ def dump_dpi_stats(): return {"data": ret} def dump_openvpn_connections(): - # Parse /var/run/openvpn/connections.db for the last 20 minutes + # Parse openvpn connections.db for the last 20 minutes. + # If /mnt/data/openvpn exists, read from there, otherwise use /var/openvpn. ret = [] - for db_file in glob.glob('/var/openvpn/*/connections.db'): + if os.path.isdir('/mnt/data/openvpn'): + pattern = '/mnt/data/openvpn/*/connections.db' + else: + pattern = '/var/openvpn/*/connections.db' + for db_file in glob.glob(pattern): instance = db_file.split('/')[-2] conn = sqlite3.connect(db_file) cursor = conn.cursor() diff --git a/packages/ns-api/files/ns.ovpnrw b/packages/ns-api/files/ns.ovpnrw index 834d66dfb..ba72e51c1 100755 --- a/packages/ns-api/files/ns.ovpnrw +++ b/packages/ns-api/files/ns.ovpnrw @@ -17,11 +17,12 @@ import ipaddress import subprocess from euci import EUci from nethsec import utils, ovpn, firewall, users, objects +import datetime as _datetime from datetime import datetime, timezone import tarfile import io import sqlite3 -import time +import time as _time import datetime import csv from zoneinfo import ZoneInfo @@ -230,6 +231,32 @@ def get_user_extra_config(u, user_id): except: return {} +def archive_connections_db(instance): + timestamp = _datetime.datetime.now().strftime('%Y%m%d%H%M%S') + var_db = f'/var/openvpn/{instance}/connections.db' + storage_db = f'/mnt/data/openvpn/{instance}/connections.db' + # archive /var DB + if os.path.exists(var_db): + archive_path = var_db.replace('connections.db', f'connections-{timestamp}.db') + try: + shutil.move(var_db, archive_path) + except: + pass + # archive /mnt/data DB if it exists + if os.path.exists(storage_db): + archive_path = storage_db.replace('connections.db', f'connections-{timestamp}.db') + try: + shutil.move(storage_db, archive_path) + except: + pass + +def get_connections_db_path(ovpninstance): + storage_db = f'/mnt/data/openvpn/{ovpninstance}/connections.db' + var_db = f'/var/openvpn/{ovpninstance}/connections.db' + if os.path.isdir('/mnt/data') and os.path.exists(storage_db): + return storage_db + return var_db + ## APIs def list_bridges(): @@ -356,6 +383,9 @@ def remove_instance(instance): remove_tap_from_bridge(u, old_bridge, old_dev) u.delete("openvpn", instance) + + # archive connection history databases before removing the instance directory + archive_connections_db(instance) shutil.rmtree(f"/etc/openvpn/{instance}", ignore_errors=True) try: # workaround: manually delete config since it's not removed from init script @@ -665,7 +695,6 @@ def disconnect_user(ovpninstance, username): return utils.generic_error("user_disconnect_failed") return {"result": "success"} - def disable_user(ovpninstance, username): u = EUci() try: @@ -966,7 +995,7 @@ def download_user_2fa(ovpninstance, username): return utils.validation_error("username", "2fa_download_failed", username) def connection_history_csv(ovpninstance, timezone): - database_path = f'/var/openvpn/{ovpninstance}/connections.db' + database_path = get_connections_db_path(ovpninstance) try: conn = sqlite3.connect(database_path) @@ -1040,9 +1069,32 @@ def connection_history_csv(ovpninstance, timezone): # Return the path of the CSV file return {"csv_path": csv_file_path} - -def connection_history(ovpninstance): - database_path = f'/var/openvpn/{ovpninstance}/connections.db' +def connection_history(ovpninstance, q='', time_range='', accounts=None, page=1, per_page=10, sort_by='startTime', desc=True): + database_path = get_connections_db_path(ovpninstance) + + # map frontend field names to database column names + sort_field_mapping = { + 'account': 'common_name', + 'startTime': 'start_time', + 'endTime': 'start_time + duration', + 'duration': 'duration', + 'virtualIpAddress': 'virtual_ip_addr', + 'remoteIpAddress': 'remote_ip_addr' + } + + # default to startTime if sort_by is invalid + if sort_by not in sort_field_mapping: + sort_by = 'startTime' + + # determine if we need to sort in Python (for IPs) or SQL (for other fields) + sort_manually = sort_by in ['virtualIpAddress', 'remoteIpAddress'] + + if not sort_manually: + sort_column = sort_field_mapping[sort_by] + sort_order = 'DESC' if desc else 'ASC' + else: + sort_column = None + sort_order = None try: conn = sqlite3.connect(database_path) @@ -1053,10 +1105,78 @@ def connection_history(ovpninstance): try: c = conn.cursor() - # Build SQL query based on whether there's a time constraint - rows = c.execute('''SELECT common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent - FROM connections - ORDER BY start_time DESC''') + conditions = [] + params = [] + + # time_range filter + now = int(_time.time()) + if time_range == 'today': + start_of_day = int(_datetime.datetime.combine(_datetime.date.today(), _datetime.time.min).timestamp()) + conditions.append('start_time >= ?') + params.append(start_of_day) + elif time_range == 'last_week': + conditions.append('start_time >= ?') + params.append(now - 7 * 86400) + elif time_range == 'last_month': + today = _datetime.date.today() + first_of_last_month = _datetime.date(today.year if today.month > 1 else today.year - 1, (today.month - 1) if today.month > 1 else 12, 1) + conditions.append('start_time >= ?') + params.append(int(_datetime.datetime.combine(first_of_last_month, _datetime.time.min).timestamp())) + elif time_range == 'last_3_months': + today = _datetime.date.today() + month = today.month - 3 + year = today.year + if month <= 0: + month += 12 + year -= 1 + first_of_3_months_ago = _datetime.date(year, month, 1) + conditions.append('start_time >= ?') + params.append(int(_datetime.datetime.combine(first_of_3_months_ago, _datetime.time.min).timestamp())) + # 'all' or empty: no filter + + # accounts filter + if accounts: + placeholders = ','.join('?' * len(accounts)) + conditions.append(f'common_name IN ({placeholders})') + params.extend(accounts) + + # free text search filter + if q: + conditions.append('(common_name LIKE ? OR virtual_ip_addr LIKE ? OR remote_ip_addr LIKE ?)') + like = f'%{q}%' + params.extend([like, like, like]) + + where = ('WHERE ' + ' AND '.join(conditions)) if conditions else '' + + total = c.execute(f'SELECT COUNT(*) FROM connections {where}', params).fetchone()[0] + total_unfiltered = c.execute('SELECT COUNT(*) FROM connections').fetchone()[0] + + if sort_manually: + # fetch all matching records for manual sorting + all_rows = c.execute( + f'''SELECT common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent + FROM connections {where}''', + params + ).fetchall() + if sort_by == 'virtualIpAddress': + all_rows = sorted(all_rows, key=lambda r: ipaddress.ip_address(r[1] or '0.0.0.0'), reverse=desc) + elif sort_by == 'remoteIpAddress': + all_rows = sorted(all_rows, key=lambda r: ipaddress.ip_address(r[2] or '0.0.0.0'), reverse=desc) + + # apply pagination + offset = (page - 1) * per_page + rows = all_rows[offset:offset + per_page] + else: + # use SQL ORDER BY for non-IP fields + offset = (page - 1) * per_page + rows = c.execute( + f'''SELECT common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent + FROM connections {where} + ORDER BY {sort_column} {sort_order} + LIMIT ? OFFSET ?''', + params + [per_page, offset] + ) + for row in rows: common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent = row end_time = start_time + duration if duration else None @@ -1071,13 +1191,28 @@ def connection_history(ovpninstance): 'bytesSent': bytes_sent }) + # collect unique accounts from the full unfiltered result for the filters field + all_accounts = [r[0] for r in c.execute('SELECT DISTINCT common_name FROM connections ORDER BY common_name').fetchall()] + except: conn.close() return utils.generic_error("database_error") conn.close() - return connections + last_page = max(1, (total + per_page - 1) // per_page) + + return { + 'connections': connections, + 'current_page': page, + 'last_page': last_page, + 'per_page': per_page, + 'total': total_unfiltered, + 'results': total, + 'filters': { + 'accounts': all_accounts + } + } def renew_server_certificate(ovpninstance): try: @@ -1142,7 +1277,7 @@ if cmd == 'list': "download-user-2fa": {"instance": "roadwarrior1", "username": "myuser"}, "download_all_user_configurations": {"instance": "roadwarrior1"}, "connection-history-csv": {"instance": "roadwarrior1", "timezone": "Europe/Rome"}, - "connection-history": {"instance": "roadwarrior1"}, + "connection-history": {"instance": "roadwarrior1", "q": "", "time_range": "all", "accounts": [], "page": 1, "per_page": 10, "sort_by": "startTime", "desc": True}, "renew-server-certificate": {"instance": "roadwarrior1"}, "regenerate-all-certificates": {"instance": "roadwarrior1"} })) @@ -1200,7 +1335,16 @@ else: elif action == "connection-history-csv": ret = connection_history_csv(args['instance'], args['timezone']) elif action == "connection-history": - ret = connection_history(args['instance']) + ret = connection_history( + args['instance'], + q=args.get('q', ''), + time_range=args.get('time_range', ''), + accounts=args.get('accounts', []), + page=int(args.get('page', 1)), + per_page=int(args.get('per_page', 10)), + sort_by=args.get('sort_by', 'startTime'), + desc=args.get('desc', True) + ) elif action == "renew-server-certificate": ret = renew_server_certificate(args["instance"]) elif action == "regenerate-all-certificates": diff --git a/packages/ns-api/files/ns.report b/packages/ns-api/files/ns.report index 85408c767..29457dfb4 100755 --- a/packages/ns-api/files/ns.report +++ b/packages/ns-api/files/ns.report @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2024 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -37,6 +37,13 @@ def get_cached_report(report_name, cache_timeout=900): return json.load(f) return None +def get_ovpnrw_db_path(instance): + storage_db = f'/mnt/data/openvpn/{instance}/connections.db' + var_db = f'/var/openvpn/{instance}/connections.db' + if os.path.isdir('/mnt/data/openvpn') and os.path.exists(storage_db): + return storage_db + return var_db + ## API def tsip_attack_report(): @@ -199,7 +206,7 @@ def mwan_report(): } def ovpnrw_list_days(instance): - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_ovpnrw_db_path(instance)) cursor = conn.cursor() try: cursor.execute(""" @@ -215,7 +222,7 @@ def ovpnrw_list_days(instance): return {"days": [day[0] for day in days]} def ovpnrw_clients_by_day(instance, day): - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_ovpnrw_db_path(instance)) cursor = conn.cursor() try: cursor.execute(""" @@ -245,7 +252,7 @@ def ovpnrw_clients_by_day(instance, day): return {"clients": client_info_list} def ovpnrw_count_clients_by_hour(instance, day): - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_ovpnrw_db_path(instance)) cursor = conn.cursor() try: cursor.execute(""" @@ -268,7 +275,7 @@ def ovpnrw_count_clients_by_hour(instance, day): return {"hours": hours_count} def ovpnrw_bytes_by_hour(instance, day): - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_ovpnrw_db_path(instance)) cursor = conn.cursor() try: cursor.execute(""" @@ -293,7 +300,7 @@ def ovpnrw_bytes_by_hour(instance, day): def ovpnrw_bytes_by_hour_and_user(instance, day, user): - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_ovpnrw_db_path(instance)) cursor = conn.cursor() try: cursor.execute(""" diff --git a/packages/ns-openvpn/Makefile b/packages/ns-openvpn/Makefile index 237dcd1e9..e3aa35242 100644 --- a/packages/ns-openvpn/Makefile +++ b/packages/ns-openvpn/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -39,6 +39,7 @@ define Package/ns-openvpn/install $(INSTALL_DIR) $(1)/usr/libexec/ns-openvpn/connect-scripts $(INSTALL_DIR) $(1)/usr/libexec/ns-openvpn/disconnect-scripts $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_DIR) $(1)/etc/hotplug.d/block $(INSTALL_BIN) ./files/ns-openvpnrw-add $(1)/usr/sbin $(INSTALL_BIN) ./files/ns-openvpnrw-init-pki $(1)/usr/sbin $(INSTALL_BIN) ./files/ns-openvpnrw-extend-crl $(1)/usr/sbin @@ -49,6 +50,8 @@ define Package/ns-openvpn/install $(INSTALL_BIN) ./files/openvpn-connect $(1)/usr/libexec/ns-openvpn/ $(INSTALL_BIN) ./files/openvpn-disconnect $(1)/usr/libexec/ns-openvpn/ $(INSTALL_BIN) ./files/init-connections-db $(1)/usr/libexec/ns-openvpn/ + $(INSTALL_BIN) ./files/openvpn-merge-connections-db $(1)/usr/libexec/ns-openvpn/ + $(INSTALL_BIN) ./files/20-openvpn-merge-connections-db $(1)/etc/hotplug.d/block/ $(INSTALL_BIN) ./files/openvpn-local-auth $(1)/usr/libexec/ns-openvpn/ $(INSTALL_BIN) ./files/openvpn-remote-auth $(1)/usr/libexec/ns-openvpn/ $(INSTALL_BIN) ./files/openvpn-otp-auth $(1)/usr/libexec/ns-openvpn/ diff --git a/packages/ns-openvpn/README.md b/packages/ns-openvpn/README.md index f71b03950..c5b5c4057 100644 --- a/packages/ns-openvpn/README.md +++ b/packages/ns-openvpn/README.md @@ -159,7 +159,7 @@ See the `delete-user` API inside the [ns-api](../ns-api/#nsovpnrw) page. ### Accounting -Every client connection is tracked inside a SQLite database saved inside `/var/openvpn//connections.db`. +Every client connection is tracked inside a SQLite database saved inside `/var/openvpn//connections.db` or inside `/mnt/data/openvpn//connections.db` if storage is configured. This allows to maintain the connection history also after a system reboot. The database is initialized as soon as the `instance` is up using the `init-connections-db` script. As default, all logs are sent to `/var/log/messages`. diff --git a/packages/ns-openvpn/files/20-openvpn-merge-connections-db b/packages/ns-openvpn/files/20-openvpn-merge-connections-db new file mode 100644 index 000000000..0657277ab --- /dev/null +++ b/packages/ns-openvpn/files/20-openvpn-merge-connections-db @@ -0,0 +1,22 @@ +#!/bin/sh +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Triggered by hotplug when a block device is added. +# If /mnt/data is mounted, sync OpenVPN connection records from /var to storage. + +[ "$ACTION" = "add" ] || exit 0 +[ "$DEVTYPE" = "partition" ] || exit 0 +[ -x /usr/libexec/ns-openvpn/openvpn-merge-connections-db ] || exit 0 + +# Wait for /mnt/data to be mounted (up to 10 seconds) +i=0 +while [ $i -lt 10 ] && ! grep -q ' /mnt/data ' /proc/mounts; do + sleep 1 + i=$((i + 1)) +done + +grep -q ' /mnt/data ' /proc/mounts || exit 0 + +/usr/libexec/ns-openvpn/openvpn-merge-connections-db diff --git a/packages/ns-openvpn/files/80-save-connection b/packages/ns-openvpn/files/80-save-connection index 63ec074ff..99cfa322b 100755 --- a/packages/ns-openvpn/files/80-save-connection +++ b/packages/ns-openvpn/files/80-save-connection @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2022 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -12,6 +12,14 @@ import sqlite3 from euci import EUci from nethsec import users +STORAGE_BASE = '/mnt/data' +VAR_BASE = '/var' + +def get_db_path(instance): + if os.path.isdir(STORAGE_BASE): + return os.path.join(STORAGE_BASE, 'openvpn', instance, 'connections.db') + return os.path.join(VAR_BASE, 'openvpn', instance, 'connections.db') + # The certificate of a migrated user is not present inside the index.txt def is_migrated(instance, user): try: @@ -48,7 +56,7 @@ except: pass try: - conn = sqlite3.connect(f'/var/openvpn/{instance}/connections.db') + conn = sqlite3.connect(get_db_path(instance)) c = conn.cursor() common_name = os.environ.get('common_name') diff --git a/packages/ns-openvpn/files/80-save-disconnection b/packages/ns-openvpn/files/80-save-disconnection index 9f42dd71e..4c1814a99 100755 --- a/packages/ns-openvpn/files/80-save-disconnection +++ b/packages/ns-openvpn/files/80-save-disconnection @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2022 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -9,7 +9,15 @@ import os import sys import sqlite3 -conn = sqlite3.connect(f'/var/openvpn/{sys.argv[1]}/connections.db') +STORAGE_BASE = '/mnt/data' +VAR_BASE = '/var' + +def get_db_path(instance): + if os.path.isdir(STORAGE_BASE): + return os.path.join(STORAGE_BASE, 'openvpn', instance, 'connections.db') + return os.path.join(VAR_BASE, 'openvpn', instance, 'connections.db') + +conn = sqlite3.connect(get_db_path(sys.argv[1])) c = conn.cursor() env = os.environ @@ -23,6 +31,18 @@ bytes_sent = int(env.get('bytes_sent', '0')) # Update connection data c.execute("UPDATE connections SET duration=?, bytes_received=?, bytes_sent=? WHERE common_name=? and start_time=(SELECT MAX(start_time) FROM connections WHERE common_name=?)", (duration, bytes_received, bytes_sent, common_name, common_name)) +if c.rowcount == 0 and not os.path.isdir(STORAGE_BASE): + # disconnection of a client whose connection record has been saved on storage that is not available now + # insert a new full record with all the data on /var + start_time = int(env.get('time_unix', '0')) + virtual_ip_addr = env.get('ifconfig_pool_remote_ip', '') + remote_ip_addr = env.get('untrusted_ip', '') + c.execute( + "INSERT INTO connections (common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + (common_name, virtual_ip_addr, remote_ip_addr, start_time, duration, bytes_received, bytes_sent) + ) + conn.commit() conn.close() sys.exit(0) diff --git a/packages/ns-openvpn/files/init-connections-db b/packages/ns-openvpn/files/init-connections-db index 9c428c066..b606b8a1c 100755 --- a/packages/ns-openvpn/files/init-connections-db +++ b/packages/ns-openvpn/files/init-connections-db @@ -1,13 +1,15 @@ #!/usr/bin/python3 # -# Copyright (C) 2022 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # import sqlite3 import os +STORAGE_BASE = '/mnt/data' + def create_connections_db(): conn = sqlite3.connect(database_path) c = conn.cursor() @@ -42,3 +44,26 @@ else: if not table_exists: create_connections_db() print('[NOTICE] Created roadwarrior connections database {}'.format(database_path)) + +# if storage is available, check that db already exists or create it +if os.path.isdir(STORAGE_BASE): + storage_dir = os.path.join(STORAGE_BASE, 'openvpn', os.environ["INSTANCE"]) + os.makedirs(storage_dir, exist_ok=True) + database_path = os.path.join(storage_dir, 'connections.db') + if not os.path.isfile(database_path): + create_connections_db() + print('[NOTICE] Created roadwarrior connections database {}'.format(database_path)) + else: + # ensure connections table exists in storage db too + conn = sqlite3.connect(database_path) + c = conn.cursor() + output = c.execute( + '''SELECT name FROM sqlite_master WHERE type="table" AND name="connections"''') + table_exists = False + output = output.fetchone() + if output and output[0] == 'connections': + table_exists = True + if not table_exists: + create_connections_db() + print('[NOTICE] Created roadwarrior connections database {}'.format(database_path)) + conn.close() diff --git a/packages/ns-openvpn/files/openvpn-merge-connections-db b/packages/ns-openvpn/files/openvpn-merge-connections-db new file mode 100644 index 000000000..27a37a556 --- /dev/null +++ b/packages/ns-openvpn/files/openvpn-merge-connections-db @@ -0,0 +1,113 @@ +#!/usr/bin/python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Merge OpenVPN connection records from /var into storage (/mnt/data). +# Called on storage mount (via add-storage script on /usr/sbin/add-storage) + +import os +import shutil +import sqlite3 +import syslog + +STORAGE_BASE = '/mnt/data' +VAR_BASE = '/var' + +def log(msg): + syslog.syslog(syslog.LOG_INFO, f"openvpn-merge-connections-db: {msg}") + +def _merge_var_into_storage(var_db, storage_db): + try: + src = sqlite3.connect(var_db) + dst = sqlite3.connect(storage_db) + src_rows = src.execute( + "SELECT common_name, virtual_ip_addr, remote_ip_addr, start_time, " + "duration, bytes_received, bytes_sent FROM connections" + ).fetchall() + for (common_name, virtual_ip_addr, remote_ip_addr, start_time, + duration, bytes_received, bytes_sent) in src_rows: + if not duration: + # client still connected + dst.execute( + "INSERT INTO connections " + "(common_name, virtual_ip_addr, remote_ip_addr, start_time, " + "duration, bytes_received, bytes_sent) VALUES (?,?,?,?,?,?,?)", + (common_name, virtual_ip_addr, remote_ip_addr, start_time, + duration, bytes_received, bytes_sent) + ) + else: + # complete record on /var, could be a client connected from storage or from /var itself + # try to make an update on the storage first (client connected from storage and disconnected from /var) + dst.execute( + "UPDATE connections SET duration=?, bytes_received=?, bytes_sent=? " + "WHERE common_name=? AND start_time=? AND (duration IS NULL OR duration=0)", + (duration, bytes_received, bytes_sent, common_name, start_time) + ) + if dst.execute("SELECT changes()").fetchone()[0] == 0: + # no record on storage, insert full record copying from /var + dst.execute( + "INSERT INTO connections " + "(common_name, virtual_ip_addr, remote_ip_addr, start_time, " + "duration, bytes_received, bytes_sent) " + "SELECT ?,?,?,?,?,?,? WHERE NOT EXISTS " + "(SELECT 1 FROM connections WHERE common_name=? AND start_time=?)", + (common_name, virtual_ip_addr, remote_ip_addr, start_time, + duration, bytes_received, bytes_sent, common_name, start_time) + ) + dst.commit() + dst.close() + src.execute("DELETE FROM connections") + src.commit() + src.close() + except Exception: + pass + +log("[OpenVPN RW merge script] Script started") + +if not os.path.isdir(STORAGE_BASE): + log(f"[OpenVPN RW merge script] Storage not available ({STORAGE_BASE} not found), exiting...") + raise SystemExit(0) + +var_openvpn = os.path.join(VAR_BASE, 'openvpn') +if not os.path.isdir(var_openvpn): + log(f"[OpenVPN RW merge script] {var_openvpn} not found, exiting...") + raise SystemExit(0) + +for instance in os.listdir(var_openvpn): + var_db = os.path.join(var_openvpn, instance, 'connections.db') + if not os.path.isfile(var_db): + continue + + storage_dir = os.path.join(STORAGE_BASE, 'openvpn', instance) + storage_db = os.path.join(storage_dir, 'connections.db') + + if not os.path.exists(storage_db): + # db on storage not existing, copy /var DB to storage and clear /var + log(f"[OpenVPN RW merge script] Instance {instance}: storage DB not found, copying from /var...") + os.makedirs(storage_dir, exist_ok=True) + shutil.copy2(var_db, storage_db) + try: + src = sqlite3.connect(var_db) + src.execute("DELETE FROM connections") + src.commit() + src.close() + except Exception: + pass + log(f"[OpenVPN RW merge script] Instance {instance}: copy done") + else: + # Storage DB exists: merge any /var rows that arrived while storage was absent + try: + _count_conn = sqlite3.connect(storage_db) + _count = _count_conn.execute("SELECT COUNT(*) FROM connections").fetchone()[0] + _count_conn.close() + log(f"[OpenVPN RW merge script] Instance {instance}: storage DB has {_count} record(s) before merge") + except Exception: + pass + log(f"[OpenVPN RW merge script] Instance {instance}: merging /var into storage") + _merge_var_into_storage(var_db, storage_db) + log(f"[OpenVPN RW merge script] Instance {instance}: merge done") + +log("[OpenVPN RW merge script] Script finished") \ No newline at end of file diff --git a/packages/ns-storage/Makefile b/packages/ns-storage/Makefile index 14e58c1ac..f9f13ce95 100644 --- a/packages/ns-storage/Makefile +++ b/packages/ns-storage/Makefile @@ -1,5 +1,5 @@ # -# Copyright (C) 2022 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -65,7 +65,6 @@ define Package/ns-storage/install $(INSTALL_BIN) ./files/remove-storage $(1)/usr/sbin $(INSTALL_BIN) ./files/rotate-messages $(1)/usr/sbin $(INSTALL_BIN) ./files/sync-data $(1)/usr/sbin - $(INSTALL_BIN) ./files/openvpn $(1)/usr/libexec/sync-data $(INSTALL_BIN) ./files/30_ns-storage $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/31-ns-storage-default.uci-default $(1)/etc/uci-defaults/31-ns-storage-default $(INSTALL_BIN) ./files/32-ns-storage-convert-uuid.uci-default $(1)/etc/uci-defaults/32-ns-storage-convert-uuid diff --git a/packages/ns-storage/files/add-storage b/packages/ns-storage/files/add-storage index 75a52933c..885950c73 100755 --- a/packages/ns-storage/files/add-storage +++ b/packages/ns-storage/files/add-storage @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -45,3 +45,6 @@ uci commit rsyslog crontab -l | grep -q '/usr/sbin/sync-data' || echo '40 1 * * * /usr/sbin/sync-data' >> /etc/crontabs/root /etc/init.d/cron restart + +# check for existing OpenVPN RW connection records in /var and merge them into storage if needed +/usr/libexec/ns-openvpn/openvpn-merge-connections-db diff --git a/packages/ns-storage/files/openvpn b/packages/ns-storage/files/openvpn deleted file mode 100644 index d3fbfc941..000000000 --- a/packages/ns-storage/files/openvpn +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -# -# Copyright (C) 2022 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -[ ! -d /mnt/data ] && exit 0 -rsync -ar /var/openvpn /mnt/data/