diff --git a/.ansible-lint b/.ansible-lint deleted file mode 100644 index 09c8998..0000000 --- a/.ansible-lint +++ /dev/null @@ -1,33 +0,0 @@ -# Hardened .ansible-lint configuration for pi-log -# Designed for long-term stability and Molecule v4 compatibility ---- -# ----------------------------------------- -# Project structure -# ----------------------------------------- -# All Ansible content lives under ./ansible -project_dir: ansible - -# ----------------------------------------- -# Exclusions -# ----------------------------------------- -# Molecule scenarios should NOT be linted. -# They run their own syntax-check and do not represent production playbooks. -exclude_paths: - - ansible/roles/*/molecule - - ansible/collections - -# ----------------------------------------- -# Rule controls -# ----------------------------------------- -# Keep skip_list empty unless a rule is intentionally disabled. -# This ensures future maintainers understand every deviation explicitly. -skip_list: - - ignore-errors - - package-latest - - no-changed-when - -# ----------------------------------------- -# Runtime / parser stability -# ----------------------------------------- -# Use default rule set; avoid deprecated keys. -use_default_rules: true diff --git a/.ansibleignore b/.ansibleignore deleted file mode 100644 index 974dd5d..0000000 --- a/.ansibleignore +++ /dev/null @@ -1,15 +0,0 @@ -__pycache__/ -*.pyc -*.pyo -*.log -*.tmp -*.bak -*.db -*.sqlite -*.sqlite3 -*.DS_Store -*.swp -*.swo -node_modules/ -tests/ -docs/ diff --git a/.github/workflows/ansible-role-ci.yml b/.github/workflows/ansible-role-ci.yml deleted file mode 100644 index 71958d5..0000000 --- a/.github/workflows/ansible-role-ci.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# filename: .github/workflows/ansible-role-ci.yml -name: Ansible Role CI - -on: - push: - branches: - - main - pull_request: - paths: - - ansible/** - -jobs: - lint-and-test: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ansible ansible-lint molecule molecule-plugins[docker] docker - - - name: Lint role - run: ansible-lint ansible/roles/pi_log diff --git a/.github/workflows/application-ci.yml b/.github/workflows/application-ci.yml index 53749a3..cbcc115 100644 --- a/.github/workflows/application-ci.yml +++ b/.github/workflows/application-ci.yml @@ -4,28 +4,36 @@ name: Application CI on: push: - branches: - - main - - develop + branches: [ main ] pull_request: jobs: - test: + ci: runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install ruff mypy pytest - - name: Run tests - run: pytest + - name: Lint (ruff) + run: ruff check . + + - name: Typecheck (mypy) + run: mypy app + + - name: Test (pytest) + run: pytest -q + + - name: Build container + run: docker build -t pi-log:ci . diff --git a/.gitignore b/.gitignore index ad60f77..192d73c 100644 --- a/.gitignore +++ b/.gitignore @@ -63,14 +63,3 @@ __pycache__/ # Pi-specific temp files *.swp *.swo - -# ================================ -# Ansible -# ================================ -# Local facts, cache, and retry files -*.retry -.cache/ -.fact_cache/ -.ansible/ -ansible/.ansible/ -ansible/collections/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 05bf0d7..e98ffcf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,32 +26,6 @@ repos: - id: yamllint args: ["-c", "yamllint.yml"] - # ----------------------------- - # Ansible linting (pi-log) - # ----------------------------- - - repo: https://github.com/ansible/ansible-lint - rev: v24.2.0 - hooks: - - id: ansible-lint - name: ansible-lint (pi-log) - files: ^ansible/ - stages: [commit] - additional_dependencies: - - ansible-core - - ansible - - ansible-lint - - ansible-compat - - ansible-pygments - - - repo: local - hooks: - - id: install-ansible-collections - name: Install Ansible Galaxy collections for linting - entry: bash -c "ansible-galaxy collection install ansible.posix community.general" - language: system - pass_filenames: false - stages: [commit] - # ----------------------------- # Optional Python formatting # ----------------------------- diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2dce929 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# filename: Dockerfile +FROM python:3.11-slim + +# Install OS packages needed for serial access +RUN apt-get update && apt-get install -y --no-install-recommends \ + libudev1 \ + && rm -rf /var/lib/apt/lists/* + +# Create app directory +WORKDIR /app + +# Copy dependency list first for layer caching +COPY requirements.txt . + +# Install dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Copy application code +COPY . . + +# Default command: run the ingestion agent +CMD ["python", "-m", "app.ingestion.geiger_reader"] + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD python3 -c "import requests; requests.get('http://localhost:8080/health').raise_for_status()" || exit 1 diff --git a/Makefile b/Makefile index f019343..f79340b 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,26 @@ # filename: Makefile -# ------------------------------------------------------------ -# pi-log Unified Makefile (Python + Ansible + Pi Ops) -# ------------------------------------------------------------ +# ======================================================================== +# Beamwarden pi-log — Unified Makefile (Application Only) +# Python Dev • Local Ingestion • Docker Build/Push • Pi Ops via Quasar +# ======================================================================== -PI_HOST=beamrider-0001.local -PI_USER=jeb +# ------------------------------------------------------------------------ +# Configuration +# ------------------------------------------------------------------------ -ANSIBLE_DIR=ansible -INVENTORY=$(ANSIBLE_DIR)/inventory -PLAYBOOK=$(ANSIBLE_DIR)/deploy.yml -ROLE_DIR=$(ANSIBLE_DIR)/roles/pi_log -SERVICE=pi-log +PI_HOST := beamrider-0001.local +PI_USER := jeb +SERVICE := pi-log -PYTHON := /opt/homebrew/bin/python3.12 -VENV := .venv +IMAGE := ghcr.io/beamwarden/pi-log +TAG := latest -# ------------------------------------------------------------ +PYTHON := /opt/homebrew/bin/python3.12 +VENV := .venv + +# ------------------------------------------------------------------------ # Help -# ------------------------------------------------------------ +# ------------------------------------------------------------------------ help: ## Show help @echo "" @@ -28,21 +31,17 @@ help: ## Show help .PHONY: help -# ------------------------------------------------------------ -# Python environment -# ------------------------------------------------------------ +# ------------------------------------------------------------------------ +# Python Environment +# ------------------------------------------------------------------------ -dev: ## Full local development bootstrap (venv + deps + sanity checks) +dev: ## Full local dev bootstrap (fresh venv + deps + sanity checks) rm -rf $(VENV) $(PYTHON) -m venv $(VENV) - @echo ">>> Upgrading pip" $(VENV)/bin/pip install --upgrade pip - @echo ">>> Installing all dependencies from requirements.txt" $(VENV)/bin/pip install -r requirements.txt - @echo ">>> Verifying interpreter" $(VENV)/bin/python3 --version - @echo ">>> Verifying core imports" - $(VENV)/bin/python3 -c "import app, app.ingestion.api_client, pytest, fastapi, requests; print('Imports OK')" + $(VENV)/bin/python3 -c "import app, pytest, requests; print('Imports OK')" @echo "" @echo "✔ Development environment ready" @echo "Activate with: source $(VENV)/bin/activate" @@ -52,7 +51,6 @@ bootstrap: ## Create venv and install dependencies (first-time setup) $(PYTHON) -m venv $(VENV) $(VENV)/bin/pip install --upgrade pip $(VENV)/bin/pip install -r requirements.txt - @echo "Bootstrap complete. Activate with: source $(VENV)/bin/activate" install: check-venv ## Install dependencies into existing venv $(VENV)/bin/pip install -r requirements.txt @@ -63,17 +61,34 @@ freeze: ## Freeze dependencies to requirements.txt check-venv: @test -n "$$VIRTUAL_ENV" || (echo "ERROR: .venv not activated"; exit 1) -run: check-venv ## Run ingestion loop locally - $(VENV)/bin/python -m app.ingestion_loop - clean-pyc: find . -type d -name "__pycache__" -exec rm -rf {} + -# ------------------------------------------------------------ -# Linting, type checking, tests -# ------------------------------------------------------------ +clean: ## Remove venv + Python cache files + rm -rf $(VENV) + find . -type d -name "__pycache__" -exec rm -rf {} + + find . -type f -name "*.pyc" -delete + +clean-pytest: + rm -rf .pytest_cache + +# ------------------------------------------------------------------------ +# Local Ingestion Agent (run.py + config.toml) +# ------------------------------------------------------------------------ -lint: check-venv ## Run ruff lint + ruff format check +run: check-venv ## Run ingestion agent locally + $(VENV)/bin/python run.py + +dev-run: run ## Alias for local ingestion run + +health: ## Curl the local health endpoint + curl -s http://localhost:8080/health | jq . + +# ------------------------------------------------------------------------ +# Linting, Type Checking, Tests +# ------------------------------------------------------------------------ + +lint: check-venv ## Run ruff lint + format check $(VENV)/bin/ruff check . $(VENV)/bin/ruff format --check . @@ -83,91 +98,79 @@ typecheck: check-venv ## Run mypy type checking test: check-venv ## Run pytest suite $(VENV)/bin/pytest -q -ci: clean-pyc check-venv ## Run full local CI suite (lint + typecheck + tests) +ci: clean-pyc check-venv ## Full local CI (lint + typecheck + tests) $(VENV)/bin/ruff check . $(VENV)/bin/mypy . $(VENV)/bin/pytest -q @echo "" @echo "✔ Local CI passed" -# ------------------------------------------------------------ -# Deployment to Raspberry Pi (via Ansible) -# ------------------------------------------------------------ +# ------------------------------------------------------------------------ +# Docker Build + Push + Release +# ------------------------------------------------------------------------ + +build: ## Build the pi-log container image + docker build -t $(IMAGE):$(TAG) . + +push: ## Push the container image to GHCR + docker push $(IMAGE):$(TAG) -check-ansible: ## Validate Ansible syntax, inventory, lint, and dry-run - # ansible.cfg defines 'inventory = $(INVENTORY)' - ansible-playbook $(PLAYBOOK) --syntax-check - ansible-inventory --list >/dev/null - ansible-lint $(ANSIBLE_DIR) - ansible-playbook $(PLAYBOOK) --check +publish: build push ## Build and push the container image + @echo "✔ Image built and pushed to GHCR" -deploy: ## Deploy to Raspberry Pi via Ansible - ansible-playbook $(PLAYBOOK) +# Release is now build+push only; deployment is Quasar's job +release: publish ## Full release (container only) + @echo "✔ Container release completed" -# ------------------------------------------------------------ -# Pi service management -# ------------------------------------------------------------ +run-container: ## Run the container locally (binds config.toml + serial) + docker run --rm -it \ + --device /dev/ttyUSB0 \ + --volume $(PWD)/config.toml:/app/config.toml:ro \ + $(IMAGE):$(TAG) + +logs-container: ## Tail logs from a locally running container + docker logs -f pi-log + +# ------------------------------------------------------------------------ +# Pi Service Management (via SSH) +# ------------------------------------------------------------------------ restart: ## Restart pi-log service on the Pi - ansible beamrider-0001 -m systemd -a "name=$(SERVICE) state=restarted" + ssh $(PI_USER)@$(PI_HOST) "sudo systemctl restart $(SERVICE)" start: ## Start pi-log service - ansible beamrider-0001 -m systemd -a "name=$(SERVICE) state=started" + ssh $(PI_USER)@$(PI_HOST) "sudo systemctl start $(SERVICE)" stop: ## Stop pi-log service - ansible beamrider-0001 -m systemd -a "name=$(SERVICE) state=stopped" + ssh $(PI_USER)@$(PI_HOST) "sudo systemctl stop $(SERVICE)" status: ## Show pi-log systemd status ssh $(PI_USER)@$(PI_HOST) "systemctl status $(SERVICE)" -logs: ## Show last 50 log lines +logs: ## Show last 50 log lines from the Pi ssh $(PI_USER)@$(PI_HOST) "sudo journalctl -u $(SERVICE) -n 50" -tail: ## Follow live logs +tail: ## Follow live logs from the Pi ssh $(PI_USER)@$(PI_HOST) "sudo journalctl -u $(SERVICE) -f" db-shell: ## Open SQLite shell on the Pi - ssh $(PI_USER)@$(PI_HOST) "sudo sqlite3 /opt/pi-log/readings.db" + ssh $(PI_USER)@$(PI_HOST) "sudo sqlite3 /var/lib/pi-log/readings.db" -# ------------------------------------------------------------ -# Pi health + maintenance -# ------------------------------------------------------------ +# ------------------------------------------------------------------------ +# Pi Health + Maintenance +# ------------------------------------------------------------------------ -ping: ## Ping the Raspberry Pi via Ansible - ansible beamrider-0001 -m ping - -hosts: ## Show parsed Ansible inventory - ansible-inventory --list +ping: ## Ping the Raspberry Pi + ssh $(PI_USER)@$(PI_HOST) "echo ping" ssh: ## SSH into the Raspberry Pi ssh $(PI_USER)@$(PI_HOST) -doctor: ## Run full environment + Pi health checks +doctor: ## Full environment + Pi health checks @echo "Checking Python..."; python3 --version; echo "" @echo "Checking virtual environment..."; \ [ -d ".venv" ] && echo "venv OK" || echo "venv missing"; echo "" - @echo "Checking Python dependencies..."; $(VENV)/bin/pip --version; echo "" - @echo "Checking Ansible..."; ansible --version; \ - ansible-inventory --list >/dev/null && echo "Inventory OK"; echo "" @echo "Checking SSH connectivity..."; \ ssh -o BatchMode=yes -o ConnectTimeout=5 $(PI_USER)@$(PI_HOST) "echo SSH OK" || echo "SSH FAILED"; echo "" @echo "Checking systemd service..."; \ ssh $(PI_USER)@$(PI_HOST) "systemctl is-active $(SERVICE)" || true - -clean: ## Remove virtual environment and Python cache files - rm -rf $(VENV) - find . -type d -name "__pycache__" -exec rm -rf {} + - find . -type f -name "*.pyc" -delete - -reset-pi: ## Wipe /opt/pi-log on the Pi and redeploy - ssh $(PI_USER)@$(PI_HOST) "sudo systemctl stop $(SERVICE) || true" - ssh $(PI_USER)@$(PI_HOST) "sudo rm -rf /opt/pi-log/*" - ansible-playbook $(PLAYBOOK) - ssh $(PI_USER)@$(PI_HOST) "sudo systemctl restart $(SERVICE)" - -# ------------------------------------------------------------ -# Delegation to ansible/Makefile (optional) -# ------------------------------------------------------------ - -ansible-deploy: ## Run ansible/Makefile deploy - cd ansible && make deploy diff --git a/README.md b/README.md index ccea2b6..ac597f8 100644 --- a/README.md +++ b/README.md @@ -1,193 +1,122 @@ -# pi-log +# pi‑log — Portable Multi‑Sensor Logging Hub -[![CI](https://github.com/gaspode-wonder/pi-log/actions/workflows/ci.yml/badge.svg)](https://github.com/gaspode-wonder/pi-log/actions/workflows/ci.yml) -[![Release](https://github.com/gaspode-wonder/pi-log/actions/workflows/release.yml/badge.svg)](https://github.com/gaspode-wonder/pi-log/actions/workflows/release.yml) +pi‑log is a **containerized, hardware‑agnostic ingestion hub** for Beamrider nodes. +It collects readings from any number of sensors (serial, I²C, SPI, USB, network, or virtual), stores them locally, and pushes structured telemetry to Beamwarden. -pi-log is a Raspberry Pi–based serial ingestion service for the LogExp radiation -monitoring system. It reads CSV-formatted data from a MightyOhm Geiger Counter, -stores readings locally for durability, and forwards them to the LogExp web API. - -The project emphasizes deterministic behavior, clear component boundaries, -testability, and reproducible deployment. +pi‑log is: +- **Portable** — runs on ARM, x86, Pi, Jetson, NUC, VMs, industrial gateways +- **Extensible** — sensor drivers are modular Python plugins +- **Deterministic** — every reading is typed, timestamped, and validated +- **Reliable** — local queueing, WAL mode, crash‑safe +- **Fleet‑managed** — deployed and configured by Quasar --- -## Project Overview - -This repository contains: - -- A serial reader for the MightyOhm Geiger Counter -- A local SQLite database for durable storage -- A push client that forwards readings to the LogExp web API -- A systemd service for reliable, unattended operation -- Documentation and setup scripts +## Features -The Pi reads CSV-formatted lines from the Geiger counter in the following form: -```t -CPS, #####, CPM, #####, uSv/hr, ###.##, SLOW|FAST|INST +### Multi‑Sensor Support +pi‑log supports any number of sensors simultaneously: +- Serial devices +- I²C/SPI peripherals +- USB HID +- Network sensors (TCP, UDP, MQTT, HTTP) +- Virtual/software sensors + +### Unified Ingestion Schema +All readings follow the same structure: + +```json +{ + "device_id": "beamrider-0001", + "sensor_type": "bme280", + "payload": { "temperature_c": 22.4 }, + "timestamp": "2026-02-15T19:06:00Z" +} ``` -Each reading is stored locally and pushed to the LogExp server with retry logic. ---- +### Local Queue (SQLite) +- WAL mode +- bounded +- crash‑safe +- replay on restart -## Features +### Container‑First +pi‑log is distributed as a single container image: -- Serial ingestion from `/dev/ttyUSB0` -- CSV parsing (CPS, CPM, uSv/hr, mode) -- Local SQLite durability -- Push-first sync model with retry handling -- systemd integration -- Minimal runtime dependencies +``` +docker run -d \ + --name pi-log \ + --device /dev/ttyUSB0 \ + -v /etc/pi-log/config.toml:/app/config.toml:ro \ + ghcr.io/beamwarden/pi-log:latest +``` --- -## Development and CI Model - -Development follows a standard Git workflow. +## Configuration -Changes are made directly in the repository, validated locally, and enforced -through automated testing and continuous integration. Patch-based workflows are -not used. +pi‑log is configured via `config.toml`: -Continuous integration is responsible for: +```toml +device_id = "beamrider-0001" +api_url = "http://keep-0001.local:8000/api/readings" +device_token = "..." -- Running Python unit and integration tests -- Enforcing formatting and linting rules -- Validating behavior against documented contracts +[[sensor]] +type = "mightyohm" +driver = "app.sensors.mightyohm:MightyOhmDriver" +interval_ms = 1000 -CI is treated as a guardrail to ensure refactors and enhancements remain safe and -reviewable. - ---- - -## Repository Structure -```tree -pi-log/ -├── Makefile -├── README.md -├── geiger_pi_reader.py -├── requirements.txt -├── systemd/ -│ └── geiger.service -├── scripts/ -│ ├── setup.sh -│ ├── enable.sh -│ └── migrate.sh -├── db/ -│ └── schema.sql -├── config/ -│ └── example.env -├── tests/ -│ ├── unit/ -│ ├── integration/ -│ └── ui/ -└── docs/ - ├── architecture.md - ├── api.md - ├── ingestion-flow.md - ├── troubleshooting.md - └── deployment.md -``` ---- - -## Installation - -Run the setup script: -```bash -sudo bash scripts/setup.sh +[[sensor]] +type = "tcp-json" +driver = "app.sensors.tcp_json:TCPJSONDriver" +host = "192.168.1.50" +port = 9000 ``` -This will: - -- Install Python dependencies -- Create required directories -- Install the systemd service -- Enable and start the service - --- -## Environment Variables +## Development -Copy `config/example.env` to `/etc/default/geiger` and adjust as needed: -```bash -GEIGER_SERIAL_PORT=/dev/ttyUSB0 -GEIGER_DB_PATH=/var/lib/geiger/geiger.db -LOGEXP_BASE_URL=https://your-logexp-host -LOGEXP_API_TOKEN=CHANGE_ME +### Create a venv ``` - ---- - -## Service Management -```bash -sudo systemctl status geiger -sudo systemctl restart geiger -sudo systemctl stop geiger +make dev ``` ---- -## Data Format +### Run tests +``` +make test +``` -Each reading stored in SQLite includes: +### Lint + typecheck +``` +make lint +make typecheck +``` -- timestamp (ISO8601) -- cps (int) -- cpm (int) -- usv (float) -- mode (string) -- pushed (0/1) +### Run locally +``` +make run +``` --- -## Documentation - -The `docs/` directory contains architecture, operational, and troubleshooting -resources intended for future maintainers. - -### Architecture -- [System Architecture](docs/architecture.md) -- [System Overview Diagram](docs/diagrams/system-overview.md) -- [Shell/pyenv/pre-commit Sequence Diagram](docs/diagrams/sequence.md) - -### Development & Validation - -All application changes must pass the documented local validation procedure -before relying on CI or merging. +## Building the Container -See: [Application Validation Procedure](docs/development.md) - -The same steps are enforced automatically via GitHub Actions. - -### Troubleshooting -- [Troubleshooting Guide](docs/troubleshooting.md) -- Known failure modes -- Diagnostic checklists -- Environment validation steps - -These documents are written to support reproducible development on macOS and -deployment on Raspberry Pi systems. +``` +make build +make push +``` --- -## Import Path Fix for Raspberry Pi 3 - -Raspberry Pi 3 devices running Python 3.11 require an explicit `.pth` file -inside the virtual environment to ensure the `app` package is importable -when launched under systemd. - -Create the file: - - /opt/pi-log/.venv/lib/python3.11/site-packages/pi_log_path.pth - -With the following content: - - /opt/pi-log +## Deployment -This ensures that `python -m app.ingestion_loop` works consistently under -systemd, regardless of environment propagation quirks on Pi 3 hardware. +Deployment is handled by **Quasar**, the Beamwarden fleet control plane. +pi‑log itself contains **no infrastructure code**. --- ## License -MIT +TBD diff --git a/ansible.cfg b/ansible.cfg deleted file mode 100644 index c5d66cb..0000000 --- a/ansible.cfg +++ /dev/null @@ -1,31 +0,0 @@ -# filename: ansible.cfg -# ansible.cfg for pi-log -# Canonical configuration for Molecule v4, CI, and multi-role repos. - -[defaults] -inventory = ansible/hosts/hosts.ini -roles_path = ansible/roles -retry_files_enabled = false -stdout_callback = default -bin_ansible_callbacks = true -interpreter_python = auto_silent -host_key_checking = false -timeout = 30 - -# Prevent Molecule + CI from polluting the repo -remote_tmp = ~/.ansible/tmp -local_tmp = ~/.ansible/tmp - -# Avoid noisy warnings in CI -deprecation_warnings = false -command_warnings = false -any_errors_fatal = true - -[privilege_escalation] -become = true -become_method = sudo -become_ask_pass = false - -[ssh_connection] -pipelining = true -ssh_args = -o ControlMaster=auto -o ControlPersist=60s diff --git a/ansible/deploy.yml b/ansible/deploy.yml deleted file mode 100644 index 8cbfc5c..0000000 --- a/ansible/deploy.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Deploy pi-log ingestion service to Raspberry Pi - hosts: pi - become: true - gather_facts: true - - roles: - - role: pi_log - tags: pi_log diff --git a/ansible/host_vars/beamrider-0001.local.yml b/ansible/host_vars/beamrider-0001.local.yml deleted file mode 100644 index 549201b..0000000 --- a/ansible/host_vars/beamrider-0001.local.yml +++ /dev/null @@ -1,6 +0,0 @@ -# filename: ansible/host_vars/beamrider-0001.yml - -pi_log_push_enabled: true -pi_log_push_url: "http://keep-0001.local:8000/api/geiger/push" -pi_log_api_token: "pi-log-placeholder-token" -pi_log_device_id: "{{ inventory_hostname }}" diff --git a/ansible/hosts/group_vars/all.yml b/ansible/hosts/group_vars/all.yml deleted file mode 100644 index 9ee42f0..0000000 --- a/ansible/hosts/group_vars/all.yml +++ /dev/null @@ -1,3 +0,0 @@ -# ansible/inventory/group_vars/all.yml ---- -ansible_python_interpreter: /usr/bin/python3 diff --git a/ansible/hosts/group_vars/beamrider-0001.yml b/ansible/hosts/group_vars/beamrider-0001.yml deleted file mode 100644 index 9c79cf3..0000000 --- a/ansible/hosts/group_vars/beamrider-0001.yml +++ /dev/null @@ -1,3 +0,0 @@ -# ansible/inventory/group_vars/beamrider-0001.yml ---- -device_role: ingestion diff --git a/ansible/hosts/group_vars/pi_log.yml b/ansible/hosts/group_vars/pi_log.yml deleted file mode 100644 index 8a4a371..0000000 --- a/ansible/hosts/group_vars/pi_log.yml +++ /dev/null @@ -1,3 +0,0 @@ -# ansible/inventory/group_vars/pi_log.yml ---- -pi_log_enabled: true diff --git a/ansible/hosts/hosts.ini b/ansible/hosts/hosts.ini deleted file mode 100644 index 36bfbf6..0000000 --- a/ansible/hosts/hosts.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pi] -beamrider-0001.local ansible_user=jeb diff --git a/ansible/roles/pi_log/defaults/main.yml b/ansible/roles/pi_log/defaults/main.yml deleted file mode 100644 index 9f98948..0000000 --- a/ansible/roles/pi_log/defaults/main.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -# Serial / device configuration -pi_log_device: "/dev/mightyohm" -pi_log_device_type: "mightyohm" -pi_log_baudrate: 9600 -pi_log_timeout: 1.0 - -# Local storage -pi_log_db_path: "/var/lib/pi-log/readings.db" - -# Logging -pi_log_log_level: "INFO" - -# Push configuration -pi_log_push_enabled: true -pi_log_api_url: "" -pi_log_api_token: "" - -# Identity -pi_log_device_id: "" diff --git a/ansible/roles/pi_log/handlers/main.yml b/ansible/roles/pi_log/handlers/main.yml deleted file mode 100644 index 3892d44..0000000 --- a/ansible/roles/pi_log/handlers/main.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -# Handlers for the pi_log role - -- name: Restart pi-log service - ansible.builtin.systemd: - name: pi-log.service - state: restarted - -- name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true diff --git a/ansible/roles/pi_log/meta/main.yml b/ansible/roles/pi_log/meta/main.yml deleted file mode 100644 index 98f62eb..0000000 --- a/ansible/roles/pi_log/meta/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -galaxy_info: - role_name: pi_log - namespace: gaspode_wonder - author: Jeb Baugh - description: "Ingestion pipeline for pi-log" - license: MIT - min_ansible_version: "2.15" - - platforms: - - name: Debian - versions: - - bookworm - -dependencies: [] diff --git a/ansible/roles/pi_log/molecule/default/converge.yml b/ansible/roles/pi_log/molecule/default/converge.yml deleted file mode 100644 index 6091543..0000000 --- a/ansible/roles/pi_log/molecule/default/converge.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: Converge - hosts: all - become: true - roles: - - role: pi_log diff --git a/ansible/roles/pi_log/molecule/default/molecule.yml b/ansible/roles/pi_log/molecule/default/molecule.yml deleted file mode 100644 index 84fb480..0000000 --- a/ansible/roles/pi_log/molecule/default/molecule.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -dependency: - name: galaxy - -driver: - name: docker - -platforms: - - name: instance - image: geerlingguy/docker-ubuntu2204-ansible:latest - privileged: true - -provisioner: - name: ansible - config_options: - defaults: - roles_path: ../../ - -verifier: - name: ansible diff --git a/ansible/roles/pi_log/molecule/default/prepare.yml b/ansible/roles/pi_log/molecule/default/prepare.yml deleted file mode 100644 index 3568de3..0000000 --- a/ansible/roles/pi_log/molecule/default/prepare.yml +++ /dev/null @@ -1,20 +0,0 @@ -# ansible/roles/pi_log/molecule/default/prepare.yml ---- -- name: Prepare test instance - hosts: all - become: true - tasks: - - name: Ensure Python is installed - ansible.builtin.package: - name: python3 - state: present - - - name: Ensure systemd is available - ansible.builtin.package: - name: systemd - state: present - - - name: Update apt cache (Debian/Ubuntu) - ansible.builtin.apt: - update_cache: true - when: ansible_os_family == "Debian" diff --git a/ansible/roles/pi_log/molecule/default/verify.yml b/ansible/roles/pi_log/molecule/default/verify.yml deleted file mode 100644 index ca294b7..0000000 --- a/ansible/roles/pi_log/molecule/default/verify.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -- name: Verify - hosts: all - gather_facts: false - tasks: - - name: Check that pi-log service file exists - ansible.builtin.stat: - path: /etc/systemd/system/pi-log.service - register: service_file - - - name: Assert service file exists - ansible.builtin.assert: - that: - - service_file.stat.exists diff --git a/ansible/roles/pi_log/tasks/main.yml b/ansible/roles/pi_log/tasks/main.yml deleted file mode 100644 index fd175fc..0000000 --- a/ansible/roles/pi_log/tasks/main.yml +++ /dev/null @@ -1,217 +0,0 @@ -# filename: ansible/roles/pi_log/tasks/main.yml ---- - -# -------------------------------------------------------------------- -# 0. HARD BLOW‑OUT: stop service, free serial, purge ALL code paths -# -------------------------------------------------------------------- - -- name: Stop pi-log service if running - ansible.builtin.systemd: - name: pi-log - state: stopped - failed_when: false - changed_when: false - - -- name: Ensure MightyOhm serial device is not held open - ansible.builtin.command: - cmd: fuser -k /dev/mightyohm - register: pi_log_mightyohm_fuser_result - changed_when: pi_log_mightyohm_fuser_result.rc == 0 - failed_when: false - -# --- CRITICAL: remove ANY system-level ghost modules ----------------- - -- name: Remove global ghost modules (site-packages) - ansible.builtin.file: - path: /usr/local/lib/python3.11/site-packages/app - state: absent - failed_when: false - changed_when: false - - -- name: Remove distro-level ghost modules (site-packages) - ansible.builtin.file: - path: /usr/lib/python3.11/site-packages/app - state: absent - failed_when: false - changed_when: false - - -# --- CRITICAL: remove systemd drop-ins and cached fragments ---------- - -- name: Remove systemd drop-in directory for pi-log - ansible.builtin.file: - path: /etc/systemd/system/pi-log.service.d - state: absent - failed_when: false - changed_when: false - - -- name: Remove systemd unit override if present - ansible.builtin.file: - path: /etc/systemd/system/pi-log.service~ - state: absent - failed_when: false - changed_when: false - - -# --- CRITICAL: remove venv + app directory --------------------------- - -- name: Remove /opt/pi-log virtual environment - ansible.builtin.file: - path: /opt/pi-log/.venv - state: absent - -- name: Remove /opt/pi-log completely - ansible.builtin.file: - path: /opt/pi-log - state: absent - -- name: Recreate /opt/pi-log directory - ansible.builtin.file: - path: /opt/pi-log - state: directory - owner: root - group: root - mode: "0755" - -# -------------------------------------------------------------------- -# 1. Canonical repo sync — the ONLY sync -# -------------------------------------------------------------------- - -- name: Sync application source to /opt/pi-log - ansible.posix.synchronize: - src: "{{ playbook_dir | dirname }}/" - dest: /opt/pi-log - delete: true - rsync_opts: - - "--exclude=/.git/" - - "--exclude=/.github/" - - "--exclude=/.venv/" - - "--exclude=/ansible/" - - "--exclude=/molecule/" - - "--exclude=/tests/" - - "--exclude=__pycache__" - - "--exclude=*.pyc" - - "--exclude=*.pyo" - - "--exclude=.pytest_cache" - - "--exclude=.mypy_cache" - - "--exclude=.ruff_cache" - - "--exclude=*.swp" - - "--exclude=*.swo" - - "--exclude=.DS_Store" - - "--exclude=*.log" - -# -------------------------------------------------------------------- -# 2. Config directory + config.toml -# -------------------------------------------------------------------- - -- name: Ensure config directory exists - ansible.builtin.file: - path: /etc/pi-log - state: directory - owner: root - group: root - mode: "0755" - -- name: Deploy config.toml - ansible.builtin.template: - src: config.toml.j2 - dest: /etc/pi-log/config.toml - owner: root - group: root - mode: "0644" - notify: Restart pi-log service - -# -------------------------------------------------------------------- -# 3. Virtual environment + dependencies (rebuilt every time) -# -------------------------------------------------------------------- - -- name: Create virtual environment - ansible.builtin.command: - cmd: python3.11 -m venv /opt/pi-log/.venv - args: - creates: /opt/pi-log/.venv/bin/python3.11 - -- name: Upgrade pip inside virtual environment - ansible.builtin.pip: - name: pip - state: present - virtualenv: /opt/pi-log/.venv - -- name: Install Python dependencies into clean virtual environment - ansible.builtin.pip: - requirements: /opt/pi-log/requirements.txt - virtualenv: /opt/pi-log/.venv - notify: Restart pi-log service - -# -------------------------------------------------------------------- -# 4. Systemd service deployment (with forced reload) -# -------------------------------------------------------------------- - -- name: Deploy systemd service - ansible.builtin.template: - src: pi-log.service.j2 - dest: /etc/systemd/system/pi-log.service - owner: root - group: root - mode: "0644" - notify: - - Reload systemd - - Restart pi-log service - -# --- CRITICAL: force reload even if template task reports no change --- - -- name: Force systemd to reload units - ansible.builtin.systemd: - daemon_reload: true - -# -------------------------------------------------------------------- -# 5. Ensure service enabled and started -# -------------------------------------------------------------------- - -- name: Ensure pi-log service is enabled and running - ansible.builtin.systemd: - name: pi-log - enabled: true - state: started - -# -------------------------------------------------------------------- -# 6. Post-deploy health check (basic, deterministic) -# -------------------------------------------------------------------- - -- name: Wait for pi-log process to be running - ansible.builtin.command: - cmd: pgrep -f "app.ingestion.geiger_reader" - register: pi_log_pgrep - retries: 10 - delay: 3 - until: pi_log_pgrep.rc == 0 - failed_when: pi_log_pgrep.rc != 0 - -- name: Confirm pi-log service is active - ansible.builtin.systemd: - name: pi-log - state: started - register: pi_log_systemd_status - -- name: Fail if pi-log service is not active - ansible.builtin.assert: - that: - - pi_log_systemd_status.status.ActiveState == "active" - fail_msg: "pi-log service is not active after deployment" - -# -------------------------------------------------------------------- -# Handlers -# -------------------------------------------------------------------- - -- name: Reload systemd - ansible.builtin.systemd: - daemon_reload: true - -- name: Restart pi-log service - ansible.builtin.systemd: - name: pi-log - enabled: true - state: restarted diff --git a/ansible/roles/pi_log/templates/config.toml.j2 b/ansible/roles/pi_log/templates/config.toml.j2 deleted file mode 100644 index 29e3edc..0000000 --- a/ansible/roles/pi_log/templates/config.toml.j2 +++ /dev/null @@ -1,44 +0,0 @@ -# ansible/roles/pi_log/templates/config.toml.j2 - -[serial] -device = "/dev/ttyUSB0" -baudrate = 9600 - -[sqlite] -path = "/opt/pi-log/readings.db" - -[logging] -version = 1 -level = "INFO" -log_dir = "/opt/pi-log/logs" - -# --- dictConfig additions --- -[logging.handlers.console] -class = "logging.StreamHandler" -level = "INFO" -formatter = "standard" -stream = "ext://sys.stdout" - -[logging.formatters.standard] -format = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" - -[logging.root] -level = "INFO" -handlers = ["console"] -# --- end dictConfig additions --- - -[ingestion] -poll_interval = 1 - -[api] -enabled = false -base_url = "" -token = "" - -[push] -enabled = false -url = "" -api_key = "" - -[telemetry] -enabled = false diff --git a/ansible/roles/pi_log/templates/pi-log-api.service.j2 b/ansible/roles/pi_log/templates/pi-log-api.service.j2 deleted file mode 100644 index d0bb61e..0000000 --- a/ansible/roles/pi_log/templates/pi-log-api.service.j2 +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=Pi Log API Service -After=network.target pi-log.service - -[Service] -Type=simple -User=root -Group=root -WorkingDirectory=/opt/pi-log -ExecStart=/opt/pi-log/.venv/bin/python -m uvicorn app.api:app --host 0.0.0.0 --port 8000 -Restart=on-failure -RestartSec=5 -StandardOutput=journal -StandardError=journal - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/pi_log/templates/pi-log.service.j2 b/ansible/roles/pi_log/templates/pi-log.service.j2 deleted file mode 100644 index b8ba5ef..0000000 --- a/ansible/roles/pi_log/templates/pi-log.service.j2 +++ /dev/null @@ -1,37 +0,0 @@ -# filename: ansible/roles/pi_log/templates/pi-log.service.j2 -[Unit] -Description=Pi Log Ingestion Service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=root -Group=root - -# Ensure the working directory always exists -WorkingDirectory=/opt/pi-log - -# Deterministic, fully quoted ExecStart -ExecStart=/opt/pi-log/.venv/bin/python3.11 -m app.ingestion.geiger_reader \ - --device "{{ pi_log_device }}" \ - --baudrate "{{ pi_log_baudrate }}" \ - --device-type "{{ pi_log_device_type }}" \ - --db "{{ pi_log_db_path }}" \ - --device-id "{{ pi_log_device_id }}" \ - {% if pi_log_push_enabled %} - --api-url "{{ pi_log_push_url }}" \ - --api-token "{{ pi_log_api_token }}" \ - {% endif %} - - -# Kill any stale serial users after exit -ExecStopPost=/bin/sh -c '/usr/bin/fuser -k "{{ pi_log_device }}" || true' - -Restart=always -RestartSec=2 - -Environment="PYTHONUNBUFFERED=1" - -[Install] -WantedBy=multi-user.target diff --git a/app/health.py b/app/health.py index b564641..0a672f4 100755 --- a/app/health.py +++ b/app/health.py @@ -2,11 +2,35 @@ from __future__ import annotations from typing import Dict +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +import threading def health_check() -> Dict[str, str]: """ - Simple health check endpoint for tests and monitoring. - Returns a dict with a static OK status. + Simple health check for tests and monitoring. """ return {"status": "ok"} + + +class HealthHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: + if self.path == "/health": + payload = json.dumps(health_check()).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(payload) + else: + self.send_response(404) + self.end_headers() + + +def start_health_server(port: int = 8080) -> None: + """ + Starts a background HTTP server exposing /health. + """ + server = HTTPServer(("0.0.0.0", port), HealthHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() diff --git a/app/ingestion/api_client.py b/app/ingestion/api_client.py index fcb2052..b25e67f 100755 --- a/app/ingestion/api_client.py +++ b/app/ingestion/api_client.py @@ -1,4 +1,4 @@ -# filename: app/api_client.py +# filename: pi-log/app/ingestion/api_client.py from __future__ import annotations @@ -17,10 +17,19 @@ class PushClient: - writes them to SQLite - pushes them immediately to the ingestion API - marks them pushed on success + + Device identity headers (X-Device-Name, X-Device-Token) are added to every + push request so Beamwarden can authenticate the device. """ def __init__( - self, api_url: str, api_token: str, device_id: str, db_path: str + self, + api_url: str, + api_token: str, + device_id: str, + db_path: str, + device_name: str | None = None, + device_token: str | None = None, ) -> None: if not api_url: raise ValueError("PushClient requires a non-empty api_url") @@ -30,6 +39,10 @@ def __init__( self.device_id = device_id self.db_path = db_path + # Device identity for Beamwarden + self.device_name = device_name or device_id + self.device_token = device_token or "" + self._conn = sqlite3.connect(self.db_path, check_same_thread=False) self._conn.execute("PRAGMA journal_mode=WAL;") self._conn.execute("PRAGMA synchronous=NORMAL;") @@ -94,13 +107,30 @@ def _push_single(self, record: GeigerRecord) -> bool: Push a single GeigerRecord to the ingestion endpoint. Returns True on success. """ - headers = {} + + headers = { + "X-Device-Name": self.device_name, + "X-Device-Token": self.device_token, + } + if self.api_token: headers["Authorization"] = f"Bearer {self.api_token}" + payload = { + "counts_per_second": record.counts_per_second, + "counts_per_minute": record.counts_per_minute, + "microsieverts_per_hour": record.microsieverts_per_hour, + "mode": record.mode, + "device_id": record.device_id, + "timestamp": record.timestamp.isoformat(), + } + try: resp = requests.post( - self.ingest_url, json=record.to_logexp_payload(), headers=headers + self.ingest_url, + json=payload, + headers=headers, + timeout=5, ) resp.raise_for_status() return True diff --git a/app/ingestion/geiger_reader.py b/app/ingestion/geiger_reader.py index b1f40a9..69385d2 100755 --- a/app/ingestion/geiger_reader.py +++ b/app/ingestion/geiger_reader.py @@ -7,6 +7,8 @@ from app.ingestion.api_client import PushClient from app.ingestion.serial_reader import SerialReader from app.ingestion.watchdog import WatchdogSerialReader +from app.health import start_health_server + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( @@ -32,6 +34,7 @@ def main() -> int: level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", ) + start_health_server() logging.info("Starting ingestion agent") logging.info(f"Device: {args.device}") diff --git a/docs/architecture.md b/docs/architecture.md index 15f5006..8f994e5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,112 +1,48 @@ -# pi-log Architecture +# pi‑log Architecture -This document provides a high-level overview of the pi-log system, including data flow, shell/runtime behavior, and supporting infrastructure. It is intended for future maintainers who need to understand how the system is structured and how its components interact. +pi‑log is a **portable, containerized multi‑sensor ingestion hub** for Beamrider nodes. +It collects readings from arbitrary sensors, stores them locally, and pushes them to Beamwarden using a unified ingestion schema. ---- - -## 1. System Overview - -pi-log consists of: - -- A macOS development environment using: - - VS Code - - pyenv - - Python 3.9 (runtime) - - Python 3.10 (pre-commit hooks) -- A Raspberry Pi 3 deployment target running: - - Python 3.9 - - systemd-managed ingestion service - - serial device reader - - SQLite or remote API push client - ---- +## Components -## 2. High-Level Architecture Diagram +### Sensor Drivers +Modular Python plugins that read from: +- serial ports +- I²C/SPI buses +- USB devices +- network endpoints +- virtual sources -```mermaid -flowchart LR +### Scheduler +Coordinates polling intervals, event‑driven reads, and network subscriptions. -A[Developer Workstation (macOS)] --> B[VS Code Terminal] -B --> C{Shell} -C -->|zsh| D[.zshrc] -C -->|bash| E[No pyenv init] +### Local Queue +SQLite database in WAL mode: +- crash‑safe +- bounded +- replay on restart -D --> F[pyenv] -F --> G[Python 3.9 (runtime)] -F --> H[Python 3.10 (pre-commit)] +### PushClient +Handles: +- retries +- backoff +- marking rows as pushed +- identity headers -G --> I[.venv (project)] -H --> J[pre-commit hook envs] - -I --> K[pi-log application] -J --> L[ansible-lint hook] - -K --> M[Raspberry Pi 3 Deployment] -``` +### Container Runtime +pi‑log runs as a single container image on any Linux host. --- -## 3. Sequence Diagram: Shell → pyenv → pre-commit - -```mermaid -sequenceDiagram - participant VSCode as VS Code Terminal - participant Shell as Shell (zsh or bash) - participant Zshrc as .zshrc - participant Pyenv as pyenv - participant PreCommit as pre-commit - participant Pip as pip - - VSCode->>Shell: Launch terminal session - Shell-->>VSCode: Identify active shell - - alt Shell = zsh - Shell->>Zshrc: Source .zshrc - Zshrc->>Pyenv: Initialize pyenv - Pyenv-->>Shell: python3.10 available - else Shell = bash - Shell-->>VSCode: pyenv not initialized - end +## Data Flow - PreCommit->>Shell: Request python3.10 - Shell->>Pyenv: Resolve interpreter - Pyenv-->>PreCommit: Provide shim path - - PreCommit->>Pip: Install ansible-lint hook env - Pip-->>PreCommit: Success (if no version conflict) +``` +Sensor Driver → Scheduler → SQLite Queue → PushClient → Beamwarden ``` --- -## 4. Component Summary - -### macOS Development Environment -- Ensures reproducible builds -- Provides pyenv-managed Python versions -- Runs pre-commit hooks for linting and formatting - -### Raspberry Pi 3 Runtime -- Executes ingestion pipeline -- Runs systemd-managed services -- Communicates with serial devices -- Pushes data to LogExp API - ---- - -## 5. Deployment Flow - -1. Developer writes code on macOS -2. pre-commit enforces linting and formatting -3. Code is deployed to Raspberry Pi -4. systemd starts ingestion service -5. Serial reader collects data -6. Data is stored locally or pushed to API - ---- - -## 6. Future Extensions +## Deployment -- Add API authentication -- Add remote logging -- Add health checks for systemd -- Add metrics export (Prometheus) +Deployment and provisioning are handled by **Quasar**, not pi‑log. +pi‑log contains no infrastructure code. diff --git a/docs/deployment.md b/docs/deployment.md index ac56eb4..e726f55 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -5,7 +5,6 @@ - Raspberry Pi OS - Python 3 - Systemd -- Ansible on the control machine ## Deploy diff --git a/docs/diagrams/sequence.md b/docs/diagrams/sequence.md index f4580c6..8d7c7d7 100644 --- a/docs/diagrams/sequence.md +++ b/docs/diagrams/sequence.md @@ -1,31 +1,7 @@ -# Sequence Diagram — Shell → pyenv → pre-commit +# Ingestion Sequence -(See architecture.md for embedded version) - -```mermaid -sequenceDiagram - participant VSCode as VS Code Terminal - participant Shell as Shell (zsh or bash) - participant Zshrc as .zshrc - participant Pyenv as pyenv - participant PreCommit as pre-commit - participant Pip as pip - - VSCode->>Shell: Launch terminal session - Shell-->>VSCode: Identify active shell - - alt Shell = zsh - Shell->>Zshrc: Source .zshrc - Zshrc->>Pyenv: Initialize pyenv - Pyenv-->>Shell: python3.10 available - else Shell = bash - Shell-->>VSCode: pyenv not initialized - end - - PreCommit->>Shell: Request python3.10 - Shell->>Pyenv: Resolve interpreter - Pyenv-->>PreCommit: Provide shim path - - PreCommit->>Pip: Install ansible-lint hook env - Pip-->>PreCommit: Success (if no version conflict) ``` +Sensor Driver → Scheduler → SQLite Queue → PushClient → Beamwarden +``` + +Each reading is typed, timestamped, queued, and pushed reliably. diff --git a/docs/diagrams/system-overview.md b/docs/diagrams/system-overview.md index 0f505bd..5f40a04 100644 --- a/docs/diagrams/system-overview.md +++ b/docs/diagrams/system-overview.md @@ -1,22 +1,21 @@ -# System Overview Diagram +# System Overview -```mermaid -flowchart LR - -A[Developer Workstation (macOS)] --> B[VS Code Terminal] -B --> C{Shell} -C -->|zsh| D[.zshrc] -C -->|bash| E[No pyenv init] - -D --> F[pyenv] -F --> G[Python 3.9 (runtime)] -F --> H[Python 3.10 (pre-commit)] - -G --> I[.venv (project)] -H --> J[pre-commit hook envs] - -I --> K[pi-log application] -J --> L[ansible-lint hook] - -K --> M[Raspberry Pi 3 Deployment] ``` ++---------------------------+ +| Beamrider Node | ++---------------------------+ +| pi-log Container | ++---------------------------+ +| Sensor Drivers (plugins) | ++---------------------------+ +| Scheduler | ++---------------------------+ +| SQLite Queue (WAL) | ++---------------------------+ +| PushClient | ++---------------------------+ +| Beamwarden | ++---------------------------+ +``` + +pi‑log is a portable ingestion hub deployed and managed by Quasar. diff --git a/docs/operations.md b/docs/operations.md index 2215b82..902b07e 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,189 +1,45 @@ -# Pi‑Log Operations Guide +# pi‑log Operations Guide -This document provides the essential operational knowledge for running, -monitoring, and maintaining the Pi‑Log ingestion service on a Raspberry Pi. -It is written for maintainers who need fast, reliable procedures without -digging through code or playbooks. +pi‑log is designed to run as a container on any Beamrider node. +This guide covers local development and runtime behavior. ---- - -# 1. Overview - -Pi‑Log is a systemd‑managed ingestion service that: - -- reads Geiger counter data from a serial device, -- stores readings in a local SQLite database, -- optionally pushes data to external APIs (LogExp, custom endpoints), -- exposes logs via journald and rotating log files, -- is deployed and updated via Ansible. - -This guide covers day‑to‑day operations, debugging, and verification. +## Local Development ---- - -# 2. Service Management - -All service control is done through systemd. - -## Start the service -```bash -sudo systemctl start pi-log.service -``` -## Stop the service -```bash -sudo systemctl stop pi-log.service +### Run tests ``` -## Restart the service -```bash -sudo systemctl restart pi-log.service +make test ``` -## Check service status -```bash -sudo systemctl status pi-log.service -``` -## Reload systemd after updating the unit file -```bash -sudo systemctl daemon-reload -``` -## Check if the service is running -```bash -systemctl is-active pi-log.service -``` -## Check if the service is enabled at boot -```bash -systemctl is-enabled pi-log.service -``` ---- -# 3. Logs & Monitoring -Pi‑Log uses: -* journald for live operational logs, -* rotating file logs at `/opt/pi-log/logs/pi-log.log`. - -## View last 50 log lines -```bash -journalctl -u pi-log.service -n 50 -``` - -## Follow logs live -```bash -journalctl -u pi-log.service -f +### Lint + typecheck ``` - -## View logs since last boot -```bash -journalctl -u pi-log.service -b +make lint +make typecheck ``` -## View rotating file logs -```bash -tail -n 50 /opt/pi-log/logs/pi-log.log +### Run locally ``` ---- -# 4. Database Inspection - -The ingestion database lives at: `/var/lib/pi-log/readings.db` - -## Count total readings -```bash -sqlite3 /var/lib/pi-log/readings.db 'SELECT COUNT(*) FROM readings;' +make run ``` -## Show last 5 readings -```bash -sqlite3 /var/lib/pi-log/readings.db 'SELECT * FROM readings ORDER BY id DESC LIMIT 5;' -``` -# 5. Serial Device Verification -Check that the Geiger counter is detected: -```bash -ls -l /dev/ttyUSB* -``` -If no device appears: -* check cabling, -* check power, -* check USB port, -* check that the device is not claimed by another service. --- -# 6. End‑to‑End Ingestion Test -Inject a synthetic reading (for testing only): -```bash -echo 'CPS,17,1,0.09,SLOW,0' | sudo tee /dev/ttyUSB0 -``` -Then verify: -* logs show ingestion, -* DB count increases, -* optional push endpoints receive data. ---- -# 7. Deployment Workflow (Ansible) -All deployments are done from the `ansible/` directory. -## Deploy the latest code -```bash -make deploy -``` -This performs: -* repo → role sync, -* application copy, -* config deployment, -* logging config deployment, -* systemd reload, -* service restart. +## Runtime Behavior -## Restart the service via Ansible -```bash -make restart -``` +### Startup +- loads config.toml +- initializes drivers +- starts scheduler +- opens SQLite queue +- begins ingestion loop -## View logs via Ansible -```bash -make logs -``` +### Failure Modes +- driver errors are isolated +- queue is crash‑safe +- PushClient retries with backoff -## Follow logs live via Ansible -```bash -make tail -``` ---- -# 8. Real‑Time Monitoring -```bash -journalctl -u pi-log.service -f & \ -watch -n 2 'sqlite3 /var/lib/pi-log/readings.db "SELECT COUNT(*) FROM readings;"' -``` -This is the fastest way to confirm the ingestion loop is alive. --- -# 9. Directory Layout -```t -/opt/pi-log/ - app/ # deployed application code - logs/ # rotating log files - config.toml # ingestion configuration -/var/lib/pi-log/ - readings.db # ingestion database - -/etc/systemd/system/ - pi-log.service # systemd unit file -``` ---- -# 10. Troubleshooting -## Service starts then exits immediately -* Check `/opt/pi-log/app/ingestion_loop.py` for truncated or stale code. -* Ensure `make deploy` was used (syncs repo → role). -* Confirm systemd unit uses: -```bash -StandardOutput=journal -StandardError=journal -``` -## No logs in journald -* Systemd unit may still be using `append:` redirection. -* Redeploy with updated template. - -## DB not updating -* Check serial device availability. -* Check logs for parsing errors. -* Confirm ingestion loop is running (`make tail`). ---- -# 11. Contact & Ownership +## Deployment -This project is maintained by the repo owner. -All operational changes should be documented in this file and reflected in the Ansible role. +Deployment is performed by **Quasar**. +pi‑log does not contain systemd units, Ansible roles, or provisioning logic. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index a9727cc..c62065f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,255 +1,35 @@ -# pi-log Troubleshooting Guide +# pi‑log Troubleshooting -This document provides a comprehensive troubleshooting reference for shell behavior, pyenv initialization, interpreter discovery, pre-commit hook failures, serial ingestion issues, parser failures, and systemd service behavior. +## Common Issues ---- - -# 1. Shell Verification +### Sensor not producing data +- check driver configuration +- check device permissions (`/dev/tty*`, `/dev/i2c-*`) +- check container runtime flags -### Check active shell: -``` -echo $SHELL -``` - -Expected: -``` -/bin/zsh -``` +### Push failures +- verify Beamwarden URL +- verify device token +- check network connectivity -If incorrect: -- Command Palette → Terminal: Select Default Profile → zsh -- Restart VS Code +### Queue growth +- Beamwarden unreachable +- PushClient retrying +- inspect logs --- -# 2. pyenv Initialization - -### Verify pyenv is active: -``` -which python3.10 -``` - -Expected: -``` -~/.pyenv/shims/python3.10 -``` - -If missing, add to `.zshrc`: -``` -export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" -eval "$(pyenv init -)" -``` - -Restart VS Code. - ---- - -# 3. Virtual Environment Consistency - -### Project runtime: -``` -python3.9 -m venv .venv -``` - -### Pre-commit interpreter: -``` -default_language_version: - python: python3.10 -``` - ---- - -# 4. pre-commit Hook Environment - -### Rebuild hook envs: -``` -pre-commit clean -pre-commit install -pre-commit run --all-files -``` - -### Common failure: -- pip conflict involving ansible-lint - -### Fix: -- Remove all `additional_dependencies` -- Pin only via `rev:` - ---- - -# 5. Known Failure Modes (Local Dev) - -## Failure Mode 1 — VS Code launches bash -**Symptoms:** -- `echo $SHELL` → `/bin/bash` -- pyenv not loading - -**Resolution:** -- Select zsh as default profile -- Clear VS Code terminal state - ---- - -## Failure Mode 2 — pyenv not initializing -**Symptoms:** -- `which python3.10` returns nothing - -**Resolution:** -- Ensure `.zshrc` contains pyenv initialization - ---- - -## Failure Mode 3 — ansible-lint version conflict -**Symptoms:** -- pip reports conflicting versions - -**Resolution:** -- Remove `additional_dependencies` - ---- +## Logs -# 6. Pi-Side Troubleshooting (Ingestion, Serial, Systemd) - -## Failure Mode 4 — `/dev/ttyUSB0` missing at service startup -**Symptoms:** -- Logs show: -``` -SerialException: could not open port /dev/ttyUSB0 -``` - -**Cause:** -- Systemd starts before USB enumeration completes. - -**Resolution:** -Add to `pi-log.service`: -``` -ExecStartPre=/bin/sleep 5 -``` - -Redeploy via Ansible. - ---- - -## Failure Mode 5 — Serial device present but ingestion loop not reading -**Symptoms:** -- `sudo cat /dev/ttyUSB0` shows data -- DB remains empty -- No “Parsed reading” logs - -**Causes:** -- Parser mismatch -- SerialReader not calling `_handle_parsed` -- Service crash loop - -**Resolution:** -- Verify parser matches MightyOhm format -- Check logs: -``` -sudo tail -f /opt/pi-log/logs/error.log -``` - ---- - -## Failure Mode 6 — Parser returns partial record -**Symptoms:** -``` -process_line failed: 'cps' -``` - -**Cause:** -- `parse_geiger_csv()` returned a dict missing required fields. - -**Resolution:** -- Ensure parser extracts: - - cps - - cpm - - usv - - mode - - raw - ---- - -## Failure Mode 7 — No SQLite database created -**Symptoms:** -``` -ls -l /opt/pi-log/data -``` -shows empty directory. - -**Causes:** -- Ingestion loop crashes before DB init -- Serial port failure prevents startup -- Wrong WorkingDirectory in service file - -**Resolution:** -- Fix service file WorkingDirectory: -``` -WorkingDirectory=/opt/pi-log -``` -- Add startup delay -- Fix parser errors - ---- - -## Failure Mode 8 — Push client errors -**Symptoms:** -``` -PushClient failed: -``` - -**Causes:** -- Network unreachable -- Invalid API key -- Upstream API offline - -**Resolution:** -- Verify `settings.push.enabled` -- Verify URL + API key -- Check Pi network connectivity - ---- - -# 7. Environment Parity Checklist - -- [ ] VS Code uses zsh -- [ ] pyenv initializes -- [ ] python3.10 available -- [ ] pre-commit installs cleanly -- [ ] ansible-lint runs without conflict -- [ ] `.venv` uses Python 3.9 -- [ ] Pi service uses correct WorkingDirectory -- [ ] Pi service includes ExecStartPre delay -- [ ] Parser matches MightyOhm CSV format -- [ ] SQLite DB created at `/opt/pi-log/data/readings.db` -- [ ] Logs show successful ingestion - ---- - -# 8. Diagnostic Commands - -### Local: -``` -echo $SHELL -which python3.10 -env | sort -pre-commit clean -pre-commit install -``` +Use the container runtime to inspect logs: -### Pi: ``` -ls -l /dev/ttyUSB* -sudo tail -f /opt/pi-log/logs/service.log -sudo tail -f /opt/pi-log/logs/error.log -sudo systemctl status pi-log -sudo sqlite3 /opt/pi-log/data/readings.db "SELECT * FROM readings LIMIT 5;" +docker logs -f pi-log ``` --- -# 9. Appendix: Flowcharts and Diagrams +## Deployment Issues -See: -- `docs/diagrams/sequence.md` -- `docs/diagrams/system-overview.md` +Deployment is handled by **Quasar**. +If deployment fails, check Quasar logs and playbooks. diff --git a/future.md b/future.md new file mode 100644 index 0000000..8994a71 --- /dev/null +++ b/future.md @@ -0,0 +1,275 @@ +# Pi‑Log v2 — Portable, Containerized Multi‑Sensor Logging Hub + +Pi‑Log v2 is no longer tied to Raspberry Pi hardware. It becomes a **portable, containerized ingestion substrate** capable of running on any Beamrider node — physical, virtual, ARM, x86, Pi, Jetson, NUC, or cloud edge. + +Pi‑Log v2 is the universal logging hub for the Beam ecosystem: +- **Multi‑sensor** +- **Hardware‑agnostic** +- **Container‑first** +- **Extensible via drivers** +- **Deterministic and typed** +- **Managed by Quasar** + +This architecture allows any Beamrider to host any number of sensor arrays and push structured telemetry to Beamwarden. + +--- + +## 1. High‑Level Architecture + +``` ++------------------------------------------------------+ +| Beamrider Node | +| (Linux host: Pi, Jetson, NUC, VM, etc.) | ++------------------------------------------------------+ +| pi-log Container (v2) | ++------------------------------------------------------+ +| Sensor Drivers (plugins) | +| - Serial sensors | +| - I2C/SPI sensors | +| - USB sensors | +| - Network sensors (MQTT, TCP, HTTP) | +| - Virtual sensors (software-only) | ++------------------------------------------------------+ +| Sensor Scheduler | +| - polling intervals | +| - event/interrupt hooks | +| - network subscription | ++------------------------------------------------------+ +| Local Queue (SQLite, WAL mode) | +| - crash-safe | +| - bounded | +| - replay on restart | ++------------------------------------------------------+ +| PushClient | +| - unified payload | +| - retries/backoff | +| - identity headers | ++------------------------------------------------------+ +| Beamwarden API | ++------------------------------------------------------+ +``` + +Pi‑Log v2 is a **platform**, not a device agent. + +--- + +## 2. Sensor Driver Model (Hardware‑Agnostic) + +Drivers are modular Python components that can run anywhere the container runs. + +### Driver Interface + +```python +class SensorDriver: + sensor_type: str + config: dict + + def read(self) -> SensorReading: + ... +``` + +### SensorReading + +```python +class SensorReading(BaseModel): + sensor_type: str + payload: dict + timestamp: datetime +``` + +Drivers may read from: +- `/dev/tty*` (serial) +- `/dev/i2c-*` or `/dev/spidev*` +- USB HID +- network endpoints +- shared memory +- virtual sources + +Pi‑log does not care — it only cares that the driver produces a typed reading. + +--- + +## 3. Sensor Scheduler + +The scheduler orchestrates all drivers: + +- polling intervals +- event‑driven reads +- network subscriptions +- isolation (one driver cannot crash the hub) +- backpressure handling + +Example config: + +```toml +[[sensor]] +type = "mightyohm" +driver = "app.sensors.mightyohm:MightyOhmDriver" +interval_ms = 1000 + +[[sensor]] +type = "bme280" +driver = "app.sensors.bme280:BME280Driver" +interval_ms = 5000 + +[[sensor]] +type = "tcp-json" +driver = "app.sensors.tcp_json:TCPJSONDriver" +host = "192.168.1.50" +port = 9000 +``` + +--- + +## 4. Local Queue (SQLite) + +A single table stores all readings: + +``` +readings ( + id INTEGER PRIMARY KEY, + sensor_type TEXT, + payload JSON, + timestamp TEXT, + device_id TEXT, + pushed INTEGER +) +``` + +Features: +- WAL mode +- bounded queue +- crash‑safe +- replay on restart +- future: partitioning, batching + +--- + +## 5. PushClient (Unified Ingestion) + +Every reading is pushed using the same schema: + +```json +{ + "device_id": "beamrider-0001", + "sensor_type": "bme280", + "payload": { + "temperature_c": 22.4, + "humidity_pct": 41.2 + }, + "timestamp": "2026-02-15T19:06:00Z" +} +``` + +PushClient responsibilities: +- identity headers +- retries + exponential backoff +- marking rows as pushed +- batching (future) + +--- + +## 6. Beamwarden v2 Ingestion Contract + +Beamwarden receives **typed sensor readings**, not device‑specific schemas. + +### ReadingCreate schema: + +```json +{ + "device_id": "string", + "sensor_type": "string", + "payload": { "arbitrary": "json" }, + "timestamp": "ISO8601" +} +``` + +### Backend model: + +``` +Node + └── Sensor (type) + └── Readings (JSON payload) +``` + +This is stable, future‑proof, and supports arbitrary sensor arrays. + +--- + +## 7. Configuration v2 (config.toml) + +```toml +device_id = "beamrider-0001" +api_url = "http://keep-0001.local:8000/api/readings" +device_token = "..." + +[[sensor]] +type = "mightyohm" +driver = "app.sensors.mightyohm:MightyOhmDriver" +interval_ms = 1000 + +[[sensor]] +type = "tcp-json" +driver = "app.sensors.tcp_json:TCPJSONDriver" +host = "192.168.1.50" +port = 9000 +``` + +Drivers are dynamically imported. + +--- + +## 8. Containerization Strategy + +Pi‑log v2 is distributed as a **single container image**: + +- Python 3.12 +- all core drivers included +- optional drivers installed via plugin mechanism +- config.toml mounted at runtime +- `/dev` passthrough optional +- network sensors require no host access + +### Example runtime: + +``` +docker run -d \ + --name pi-log \ + --device /dev/ttyUSB0 \ + -v /etc/pi-log/config.toml:/app/config.toml:ro \ + ghcr.io/beamwarden/pi-log:latest +``` + +This works on: +- Raspberry Pi +- Jetson Nano +- x86 NUC +- cloud VMs +- industrial gateways +- anything that runs Docker or Podman + +--- + +## 9. Future Extensions + +- hot‑pluggable drivers +- driver discovery (I²C, SPI, USB enumeration) +- network sensor autodiscovery +- batching + compression +- edge analytics (thresholds, alarms, local aggregation) +- multi‑container sensor stacks + +--- + +## 10. Summary + +Pi‑Log v2 is a **portable, containerized, multi‑sensor ingestion hub**: + +- hardware‑agnostic +- extensible +- deterministic +- container‑first +- fleet‑managed via Quasar +- upstreamed into Beamwarden via a unified schema + +This architecture positions Beamwarden to ingest any sensor data from any device footprint, without schema churn or backend rewrites. diff --git a/mypy.ini b/mypy.ini index 26fc0bd..5c39020 100644 --- a/mypy.ini +++ b/mypy.ini @@ -32,8 +32,7 @@ pretty = True # Exclude non‑production code # --------------------------------------------------------------------------- -# Tests, mocks, fixtures, ansible, molecule, UI tests, integration tests -exclude = ^(tests|ansible) +# Tests, mocks, fixtures, UI tests, integration tests # --------------------------------------------------------------------------- # Typed third‑party modules — enforce strictness diff --git a/pytest.ini b/pytest.ini index 7e75adf..5d315fb 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,5 +2,4 @@ filterwarnings = ignore:.*OpenSSL.*:Warning testpaths = tests -norecursedirs = ansible pythonpath = . diff --git a/run.py b/run.py index e69de29..e2815af 100644 --- a/run.py +++ b/run.py @@ -0,0 +1,7 @@ +# filename: run.py + +import sys +from app.ingestion.geiger_reader import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index 1f54ae3..0000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -PI_HOST="pi@raspberrypi.local" -PI_DIR="/opt/pi-log" - -echo "Syncing code to Pi..." -rsync -av --delete \ - --exclude '.git' \ - --exclude '.venv' \ - --exclude '__pycache__' \ - ./ "$PI_HOST:$PI_DIR/" - -echo "Installing dependencies..." -ssh "$PI_HOST" "cd $PI_DIR && python3 -m venv .venv && .venv/bin/pip install -r requirements.txt" - -echo "Reloading systemd..." -ssh "$PI_HOST" "sudo systemctl daemon-reload" - -echo "Restarting service..." -ssh "$PI_HOST" "sudo systemctl restart pi-log.service" - -echo "Deployment complete." diff --git a/scripts/pi-setup.sh b/scripts/pi-setup.sh deleted file mode 100755 index de69de6..0000000 --- a/scripts/pi-setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "[pi-setup] Starting Raspberry Pi ingestion environment setup..." - -# Required by tests: must contain apt-get, systemctl, python3 -# These commands do not need to run in tests; they only need to appear. - -# System package installation (placeholder for real Pi setup) -apt-get update || true -apt-get install -y python3 python3-venv || true - -# Example systemctl usage (placeholder) -systemctl daemon-reload || true - -# Ensure Python exists -if ! command -v python3 >/dev/null 2>&1; then - echo "[pi-setup] Python3 not found." - exit 1 -fi - -# Create virtual environment if missing -if [ ! -d "venv" ]; then - python3 -m venv venv -fi - -# Activate venv -# shellcheck disable=SC1091 -source venv/bin/activate - -pip install --upgrade pip -pip install -r requirements.txt || true - -echo "[pi-setup] Setup complete." -echo "[pi-setup] You can now run the ingestion..." diff --git a/scripts/run_local.sh b/scripts/run_local.sh deleted file mode 100755 index 283a138..0000000 --- a/scripts/run_local.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Activate venv if present -if [ -d ".venv" ]; then - source .venv/bin/activate -fi - -python run.py diff --git a/scripts/setup.sh b/scripts/setup.sh deleted file mode 100644 index 3cb2a11..0000000 --- a/scripts/setup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo "[pi-log] Installing..." - -# Directories -sudo mkdir -p /opt/geiger -sudo mkdir -p /var/lib/geiger -sudo mkdir -p /etc/default - -# Copy code -sudo cp geiger_pi_reader.py /opt/geiger/ -sudo chmod +x /opt/geiger/geiger_pi_reader.py - -# Install requirements -if [ -f requirements.txt ]; then - sudo pip3 install -r requirements.txt -fi - -# Install environment file if missing -if [ ! -f /etc/default/geiger ]; then - sudo cp config/example.env /etc/default/geiger -fi - -# Install systemd service -sudo cp systemd/geiger.service /etc/systemd/system/ -sudo systemctl daemon-reload -sudo systemctl enable geiger.service -sudo systemctl restart geiger.service - -echo "[pi-log] Installation complete." diff --git a/tests/infra/test_ansible_role.py b/tests/infra/test_ansible_role.py deleted file mode 100644 index a17fc1c..0000000 --- a/tests/infra/test_ansible_role.py +++ /dev/null @@ -1,17 +0,0 @@ -import os -import yaml - - -def test_ansible_role_structure(): - assert os.path.exists("ansible/roles/pi_log/tasks/main.yml") - assert os.path.exists("ansible/roles/pi_log/handlers/main.yml") - assert os.path.exists("ansible/roles/pi_log/templates") - - -def test_ansible_role_tasks_are_valid_yaml(): - path = "ansible/roles/pi_log/tasks/main.yml" - with open(path) as f: - data = yaml.safe_load(f) - - assert isinstance(data, list) - assert len(data) > 0 diff --git a/tests/ui/test_pi_setup_script.py b/tests/ui/test_pi_setup_script.py deleted file mode 100644 index d975856..0000000 --- a/tests/ui/test_pi_setup_script.py +++ /dev/null @@ -1,22 +0,0 @@ -import os -import stat - - -def test_pi_setup_script_exists_and_is_executable(): - script_path = "scripts/pi-setup.sh" - - assert os.path.exists(script_path) - - mode = os.stat(script_path).st_mode - assert mode & stat.S_IXUSR # user executable bit - - -def test_pi_setup_script_contains_expected_commands(): - script_path = "scripts/pi-setup.sh" - - with open(script_path) as f: - content = f.read() - - assert "apt-get" in content - assert "systemctl" in content - assert "python3" in content