Config rework

.github/workflows/python-app.yml

@@ -19,21 +19,18 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.10
+    - name: Set up Python 3.13
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        python-version: "3.13"
+    - name: Install the latest version of uv
+      uses: astral-sh/setup-uv@v6
+      with:
+        version: "latest"
+
+    - name: Install the project
+      run: uv sync --locked --all-extras --dev
+
     - name: Test with pytest
       run: |
-        pytest
+        PYTHONPATH=. uv run pytest --cov=uv --cov-report=xml --cov-report=term-missing

.gitignore

@@ -1,4 +1,3 @@
-config.py
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

README.md

@@ -1,9 +1,10 @@
 Connect to dump1090 basestation port and persist messages to a REDIS data store
 
-# Install
+# Install project with uv
 ```bash
-pip3 install -r requirements.txt
-cp config.py.template config.py
+uv venv
+uv sync
+cp dotenv-sample .env
 ```
-Edit config.py - set REDIS login username/host/port/password
+Edit .env - set REDIS login username/host/port/password
 Update the hostname and port of the dump1090/Flight Aware host

config.py

@@ -0,0 +1,49 @@
+# define all the configuration variables for the application
+# by grabbing from os.environ
+import os
+import logging
+logger = logging.getLogger(__name__)
+from dotenv import load_dotenv
+load_dotenv()
+
+config_values = {
+    "REDIS_URL": os.environ.get("REDIS_URL", "redis://localhost:6379/0"),
+    "FA_HOST": os.environ.get("FA_HOST", "localhost"),
+    "MQTT_HOST": os.environ.get("MQTT_HOST", "localhost"),
+    "MQTT_TOPIC_NAME": os.environ.get("MQTT_TOPIC_NAME", "flightaware/positions"),
+    "MQTT_DISTANCE_MAX": float(os.environ.get("MQTT_DISTANCE_MAX", "1.0")),
+    "HOME_LATITUDE": float(os.environ.get("HOME_LATITUDE", "0.0")),
+    "HOME_LONGITUDE": float(os.environ.get("HOME_LONGITUDE", "0.0")),
+    "LOG_FILENAME": os.environ.get("LOG_FILENAME", "/var/log/adsb-flightaware-redis.log"),
+}
+
+def redact_url_password(url):
+    """Redact the password in a URL."""
+    if url is None:
+        return None
+    parts = url.split("://")
+    if len(parts) != 2:
+        return url  # Not a valid URL format
+    scheme, rest = parts
+    if "@" in rest:
+        user_pass, host_port = rest.split("@", 1)
+        user, _ = user_pass.split(":", 1) if ":" in user_pass else (user_pass, "")
+        return f"{scheme}://{user}:<redacted>@{host_port}"
+    return url  # No password to redact
+
+# create object where we can store the configuration
+class Config:
+    def __init__(self, config_dict):
+        for key, value in config_dict.items():
+            setattr(self, key, value)
+            # set lower case version of the key as well
+            setattr(self, key.lower(), value)
+    # make accessable as a dictionary
+    def __getitem__(self, key):
+        return getattr(self, key)
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+
+
+CONFIG = Config(config_values)

dotenv-sample

@@ -0,0 +1,17 @@
+
+REDIS_URL="redis://default:pass@redis-host.org:30036/0"
+
+FA_HOST="piaware.host.org"
+FA_PORT=30003
+
+HOME_LATITUDE="30.001"
+HOME_LONGITUDE="-100.12345"
+
+MQTT_SPLITFLAP_HOST="splitflap"
+MQTT_TOPIC_NAME="splitflap/splitflap/set"
+MQTT_HOST="host.local"
+# miles from home_lat/long 
+MQTT_DISTANCE_MAX="1.0"
+
+LOG_DIR="/tmp/adsb"
+RUNNING_IN_VSCODE="true"

flight_aware_redis.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python3
 import sys, os.path
-from config import *
+import threading
+import time
+
 import redis
 import logging
 import py1090 
@@ -9,20 +11,18 @@
 from py1090.helpers import distance_between
 from py1090 import FlightCollection
 import daemon
+from config import CONFIG, redact_url_password
 
 # configure a file logger
-if not os.path.exists(LOG_DIR):
-    os.makedirs(LOG_DIR)
-log_file = os.path.join(LOG_DIR, 'flight_aware_redis.log')
 
 FORMAT = '%(asctime)s %(levelname)-8s %(message)s'
-logging.basicConfig(level=logging.INFO, format=FORMAT, filename=log_file)
-logging.info("Connecting to %s:%d", redis_host,redis_port)
-r = redis.Redis(host=redis_host, port=redis_port, password=redis_password, decode_responses=True, 
-socket_timeout=2.0)
+logging.basicConfig(level=logging.INFO, format=FORMAT, filename=CONFIG.log_filename)
+logger = logging.getLogger(__name__)
+logging.info("Connecting to %s", redact_url_password(CONFIG.redis_url))
+r = redis.Redis.from_url(CONFIG["REDIS_URL"], decode_responses=True,socket_timeout=2.0)
 
-CALL_SIGNS = {}
 FLIGHTS = FlightCollection()
+MILES_PER_METER = 0.000621371
 
 
 def _dump_bool(value):
@@ -45,25 +45,48 @@ def to_record(message):
 
 def publish_rec(message, last_message):
     if message != last_message:
-        logging.info("queue message %s on %s", message.strip(), mqtt_topic_name)
-        publish.single(mqtt_topic_name, str(message).strip(), hostname=mqtt_host)
+        logging.info("queue message %s on %s", message.strip(), CONFIG.mqtt_topic_name)
+        publish.single(CONFIG.mqtt_topic_name, str(message).strip(), hostname=CONFIG.mqtt_host)
         return message
     return last_message
 
+def cleanup_flight_collection(max_age=3600):
+    """Remove flights that have not been updated in the last n minutes."""
+    while True:
+        time.sleep(max_age / 2)
+        logger.info("Starting cleanup of flight collection. size=%d", len(FLIGHTS))
+        time_now = datetime.datetime.now(datetime.timezone.utc)
+        time_now = time_now.replace(tzinfo=None)
+        remove_list = []
+        for flight in FLIGHTS:
+            if not flight.messages:
+                continue
+            last_message = flight.messages[-1]
+            age =  (time_now - last_message.generation_time).total_seconds()
+            if age > max_age:
+                logger.info("Removing flight %s from collection", flight.hexident)
+                remove_list.append(flight.hexident)
+        # remove from dictionary
+        for id in remove_list:
+            try:
+                del FLIGHTS._dictionary[id]
+            except KeyError:
+                logger.warning("Flight %s not found in FLIGHTS collection", id)
+
 def get_call_sign(hexident):
     flight_rec = FLIGHTS[hexident]
     if flight_rec is None:
         return None
     call_sign = None
-    distance = None
+    distance = -1
     # iterate through messages in reverse order to find the most recent call sign and distance
     for message in reversed(flight_rec.messages):
         if message.callsign:
             call_sign = message.callsign.strip()
 
         if hasattr(message, "distance"):
             distance = message.distance
-        if call_sign and distance is not None:
+        if call_sign and distance != -1:
             break
 
     return (call_sign, distance)
@@ -73,27 +96,36 @@ def record_positions_to_redis(redis_client):
     last_message = None
     msg_count = 0
     distance = 0
-    with py1090.Connection(host=fa_host) as connection:
+    with py1090.Connection(host=CONFIG.fa_host) as connection:
         for line in connection:
             message = py1090.Message.from_string(line)
 
             if message.latitude and message.longitude:
-                distance = distance_between(home_lat, home_long, message.latitude, message.longitude) * 0.000621371
+                distance = distance_between(CONFIG.home_latitude,
+                                            CONFIG.home_longitude,
+                                            message.latitude,
+                                            message.longitude) * MILES_PER_METER
+
                 message.distance = distance
-                if distance <= mqtt_distance_max and message.hexident in FLIGHTS:
+                if distance <= CONFIG.mqtt_distance_max and message.hexident in FLIGHTS:
                     call_sign, dist = get_call_sign(message.hexident)
-                    logging.info("Updating %s call_sign='%s'", message.hexident,call_sign) 
+                    logger.info("Updating %s call_sign='%s'", message.hexident,call_sign)
                     last_message = publish_rec( call_sign, last_message)
             FLIGHTS.add(message)
 
             call_sign, distance = get_call_sign(message.hexident)
             redis_client.hset(message.hexident, mapping=to_record(message))
             msg_count += 1
             if msg_count % 1000 == 0:
-                logging.info("%d %s recorded. last_dist=%d call_sign=%s", msg_count, message.hexident, distance, call_sign)
+                logging.info("%d %s recorded. last_dist=%0.2f call_sign=%s", msg_count, message.hexident, distance, call_sign)
 
 def run_loop():
     logging.info("Starting to record positions to Redis")
+    # create a background thread to execute cleanup_flight_collection
+
+    cleanup_thread = threading.Thread(target=cleanup_flight_collection, daemon=True)
+    cleanup_thread.start()
+
     while True:
         try:
             record_positions_to_redis(r)
@@ -103,8 +135,8 @@ def run_loop():
 
 if __name__ == "__main__":
 
-    if 'RUNNING_IN_VSCODE' in os.environ:
-        logging.info("Running in VSCode environment, not starting daemon")
+    if 'RUNNING_IN_IDE' in os.environ:
+        logging.info("Running in IDE environment, not starting daemon")
         run_loop()
         sys.exit(0)
 

pyproject.toml

@@ -8,5 +8,14 @@ dependencies = [
     "py1090>=1.1",
     "redis>=6.2.0",
     "paho-mqtt",
-    "python-daemon"
+    "python-daemon",
+    "python-dotenv"
+]
+
+# Optional dependencies development
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-cov",
+    "ruff",
 ]

tests/test_config.py

@@ -0,0 +1,46 @@
+import pytest
+
+from config import redact_url_password, Config
+
+
+def test_redacts_password_in_url_with_password():
+    url = "redis://user:password@localhost:6379/0"
+    redacted_url = redact_url_password(url)
+    assert redacted_url == "redis://user:<redacted>@localhost:6379/0"
+
+def test_returns_original_url_if_no_password():
+    url = "redis://localhost:6379/0"
+    redacted_url = redact_url_password(url)
+    assert redacted_url == url
+
+def test_handles_invalid_url_format():
+    url = "invalid_url"
+    redacted_url = redact_url_password(url)
+    assert redacted_url == url
+
+def test_returns_none_if_url_is_none():
+    url = None
+    redacted_url = redact_url_password(url)
+    assert redacted_url is None
+
+def test_config_object_allows_attribute_access():
+    config = Config({"TEST_KEY": "value"})
+    assert config.TEST_KEY == "value"
+    assert config.test_key == "value"
+
+def test_config_object_allows_dict_access():
+    config = Config({"TEST_KEY": "value"})
+    assert config["TEST_KEY"] == "value"
+    assert config["test_key"] == "value"
+
+def test_config_object_allows_attribute_modification():
+    config = Config({"TEST_KEY": "value"})
+    config.TEST_KEY = "new_value"
+    assert config.TEST_KEY == "new_value"
+    #assert config.test_key is None
+
+def test_config_object_allows_dict_modification():
+    config = Config({"TEST_KEY": "value"})
+    config["TEST_KEY"] = "new_value"
+    assert config.TEST_KEY == "new_value"
+    #assert config.test_key == "new_value"

tests/test_flight_aware_redis.py

@@ -0,0 +1,19 @@
+import py1090
+
+from flight_aware_redis import to_record
+
+
+def test_to_record():
+# Create a mock flight message
+    message = py1090.Message.from_string("MSG,3,111,11111,3C49CC,111111,2015/05/01,17:06:55.370,2015/05/01,17:06:55.326,,24400,,,50.65931,6.67709,,,,,,0")
+
+    # Convert the message to a record
+    record = to_record(message)
+
+    # Check if the record contains the expected fields
+    assert record['hexident'] == "3C49CC"
+    assert record['latitude'] == 50.65931
+    assert record['longitude'] == 6.67709
+    assert record['altitude'] == 24400
+    #assert record['callsign'] == "TEST123"
+    assert record["message_type"] == "MSG"