Better Python Practices and Delphi Improvements

delphi/.dockerignore

@@ -14,7 +14,7 @@ build/
 
 # Virtual environments
 polis_env/
-delphi_env/
+delphi-env/
 delphi-dev-env/
 venv/
 ENV/

delphi/.github/workflows/ci.yml

@@ -0,0 +1,215 @@
+name: CI
+
+on:
+  push:
+    branches: [main, edge, develop]
+    paths:
+      - 'delphi/**'
+      - '.github/workflows/ci.yml'
+  pull_request:
+    branches: [main, edge, develop]
+    paths:
+      - 'delphi/**'
+      - '.github/workflows/ci.yml'
+
+defaults:
+  run:
+    working-directory: delphi
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run ruff
+        run: ruff check .
+
+      - name: Run black
+        run: black --check .
+
+      - name: Run mypy
+        run: mypy polismath umap_narrative
+        continue-on-error: true  # MyPy might need gradual adoption
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.12', '3.13', '3.14']
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: polis_test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      dynamodb:
+        image: amazon/dynamodb-local:latest
+        ports:
+          - 8000:8000
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cache pip
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-${{ matrix.python-version }}-pip-${{ hashFiles('**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.python-version }}-pip-
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libpq-dev
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Set up test environment
+        run: |
+          cp example.env .env
+          # Override with test-specific values
+          echo "DATABASE_HOST=localhost" >> .env
+          echo "DATABASE_NAME=polis_test" >> .env
+          echo "DATABASE_USER=postgres" >> .env
+          echo "DATABASE_PASSWORD=postgres" >> .env
+          echo "DYNAMODB_ENDPOINT=http://localhost:8000" >> .env
+          echo "AWS_ACCESS_KEY_ID=dummy" >> .env
+          echo "AWS_SECRET_ACCESS_KEY=dummy" >> .env
+          echo "AWS_REGION=us-east-1" >> .env
+
+      - name: Create DynamoDB tables
+        run: |
+          python create_dynamodb_tables.py --endpoint-url http://localhost:8000
+
+      - name: Run unit tests
+        run: |
+          pytest tests/ -v --cov --cov-report=xml -m "not slow and not real_data"
+
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v3
+        with:
+          file: ./delphi/coverage.xml
+          flags: unittests
+          name: codecov-umbrella
+
+  integration-test:
+    runs-on: ubuntu-latest
+    needs: [lint, test]
+    if: github.event_name == 'push' || github.event.pull_request.draft == false
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DB: polis_test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      dynamodb:
+        image: amazon/dynamodb-local:latest
+        ports:
+          - 8000:8000
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Set up test environment
+        run: |
+          cp example.env .env
+          echo "DATABASE_HOST=localhost" >> .env
+          echo "DATABASE_NAME=polis_test" >> .env
+          echo "DATABASE_USER=postgres" >> .env
+          echo "DATABASE_PASSWORD=postgres" >> .env
+          echo "DYNAMODB_ENDPOINT=http://localhost:8000" >> .env
+
+      - name: Create DynamoDB tables
+        run: |
+          python create_dynamodb_tables.py --endpoint-url http://localhost:8000
+
+      - name: Run integration tests
+        run: |
+          pytest tests/ -v -m "integration and not real_data" --tb=short
+
+  docker:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: delphi
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/delphi:latest
+            ghcr.io/${{ github.repository_owner }}/delphi:${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

delphi/.pre-commit-config.yaml

@@ -0,0 +1,60 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-json
+      - id: check-toml
+      - id: debug-statements
+
+  - repo: https://github.com/psf/black
+    rev: 25.9.0
+    hooks:
+      - id: black
+        language_version: python3
+        args: [--config=delphi/pyproject.toml]
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.14.0
+    hooks:
+      - id: ruff-check
+        args: [--fix, --exit-non-zero-on-fix, --config=delphi/pyproject.toml]
+
+  # - repo: https://github.com/pre-commit/mirrors-mypy
+  #   rev: v1.18.2
+  #   hooks:
+  #     - id: mypy
+  #       additional_dependencies: [
+  #         types-requests,
+  #         types-psycopg2,
+  #         "boto3-stubs[dynamodb,s3]",
+  #         pydantic,
+  #       ]
+  #       exclude: ^(tests/|scripts/)
+
+  # - repo: https://github.com/PyCQA/bandit
+  #   rev: 1.8.6
+  #   hooks:
+  #     - id: bandit
+  #       args: ["-c", "delphi/pyproject.toml"]
+  #       additional_dependencies: ["bandit[toml]"]
+  #       exclude: ^tests/
+
+  # Note: isort and flake8 functionality now handled by ruff above
+
+ci:
+  autofix_commit_msg: |
+    [pre-commit.ci] auto fixes from pre-commit.com hooks
+
+    for more information, see https://pre-commit.ci
+  autofix_prs: true
+  autoupdate_branch: ''
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+  autoupdate_schedule: weekly
+  skip: []
+  submodules: false

delphi/CLAUDE.md

@@ -11,6 +11,7 @@ For a comprehensive list of all documentation files with descriptions, see:
 
 delphi/docs/JOB_QUEUE_SCHEMA.md
 delphi/docs/DISTRIBUTED_SYSTEM_ROADMAP.md
+delphi/docs/BETTER_PYTHON_TODO.md
 
 ## Helpful terminology
 
@@ -57,8 +58,8 @@ Always use the commands above to determine the most substantial conversation whe
 
 ### Environment Files
 
-- Main project uses a `.env` file in the parent directory (`/Users/colinmegill/polis/.env`)
-- Example environment file is available at `/Users/colinmegill/polis/delphi/example.env`
+- Main project uses a `.env` file in the parent directory (`$HOME/polis/.env`)
+- Example environment file is available at `$HOME/polis/delphi/example.env`
 
 ### Key Environment Variables
 
@@ -72,7 +73,6 @@ Always use the commands above to determine the most substantial conversation whe
 
 - **Docker Configuration**:
 
-  - `PYTHONPATH=/app` is set in the container
   - DynamoDB local endpoint: `http://dynamodb-local:8000`
   - Ollama endpoint: `http://ollama:11434`
 
@@ -124,28 +124,26 @@ Always use the commands above to determine the most substantial conversation whe
 
 The system uses Docker Compose with three main services:
 
-1. `dynamodb-local`: Local DynamoDB instance for development
+1. `dynamodb`: Local DynamoDB instance for development
 2. `ollama`: Ollama service for local LLM processing
 3. `polis-dev-delphi-1`: Main application container
 
 ## DynamoDB Configuration
 
-### Docker Services
-
-- The primary DynamoDB service is defined in the main `/docker-compose.yml` file
+- The primary DynamoDB service is defined in the parent project's `/docker-compose.yml` file
 - Service name is `dynamodb` and container name is `polis-dynamodb-local`
 - Exposed on port 8000
 - Uses persistent storage via Docker volume `dynamodb-data`
 - Access URL from the host: `http://localhost:8000`
 - Access URL from Delphi containers: `http://host.docker.internal:8000`
 
-**Important Update:** The Delphi-specific DynamoDB service (`dynamodb-local` in delphi/docker-compose.yml) has been deprecated. All DynamoDB operations now use the centralized instance from the main docker-compose.yml file.
+**Important Update:** The Delphi-specific DynamoDB service (`dynamodb-local` in delphi/docker-compose.yml) has been deprecated. All DynamoDB operations now use the centralized instance from the parent project's `/docker-compose.yml` file.
 
 ### Connection Details
 
 When connecting to DynamoDB from the Delphi container, use these settings:
 
-```
+```txt
 DYNAMODB_ENDPOINT=http://host.docker.internal:8000
 AWS_ACCESS_KEY_ID=dummy
 AWS_SECRET_ACCESS_KEY=dummy
@@ -219,7 +217,7 @@ Delphi now includes a distributed job queue system built on DynamoDB:
 - `Delphi_CollectiveStatement` - Collective statements generated for topics
 
 > **Note:** All table names now use the `Delphi_` prefix for consistency.
-> For complete documentation on the table renaming, see `/Users/colinmegill/polis/delphi/docs/DATABASE_NAMING_PROPOSAL.md`
+> For complete documentation on the table renaming, see `$HOME/polis/delphi/docs/DATABASE_NAMING_PROPOSAL.md`
 
 ## Reset Single Conversation