Add docker support for local development

.dockerignore

@@ -0,0 +1,10 @@
+.dockerignore
+.editorconfig
+.env
+.git
+.github
+.gitignore
+README.md
+
+docker/Docker*
+docker/docker-compose*

.gitignore

@@ -14,6 +14,7 @@ dist/*
 **/*.h5
 **/*.csv.gz
 .env
+.python-version
 
 # Ignore generated credentials from google-github-actions/auth
 gha-creds-*.json

Makefile

@@ -1,24 +1,31 @@
-install:
+.PHONY: help
+help:  ## Print this message
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: install
+install: ## Install dependencies and setup env
 	pip install -e ".[dev]" --config-settings editable_mode=compat
 	bash .github/setup_env.sh
 
-debug:
-	FLASK_APP=policyengine_api.api FLASK_DEBUG=1 flask run --without-threads
+.PHONY: debug
+debug: ## Run the Flask app in debug mode
+	FLASK_APP=policyengine_api.api FLASK_DEBUG=1 flask run --without-threads --host=0.0.0.0
 
-test-env-vars:
+.PHONY: test-env-vars
+test-env-vars: ## Test environment variables
 	pytest tests/env_variables
 
-test:
+test: ## Run tests
 	MAX_HOUSEHOLDS=1000 coverage run -a --branch -m pytest tests/to_refactor tests/unit --disable-pytest-warnings
 	coverage xml -i
 
-debug-test:
+debug-test: ## Run tests with debug verbosity
 	MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests
 
-format:
+format: ## Run the black formmater
 	black . -l 79
 
-deploy:
+deploy: ## Deploy to GCP
 	python gcp/export.py
 	gcloud config set app/cloud_build_timeout 2400
 	cp gcp/policyengine_api/* .
@@ -28,9 +35,39 @@ deploy:
 	rm .gac.json
 	rm .dbpw
 
-changelog:
+changelog: ## Build the changelog
 	build-changelog changelog.yaml --output changelog.yaml --update-last-date --start-from 0.1.0 --append-file changelog_entry.yaml
 	build-changelog changelog.yaml --org PolicyEngine --repo policyengine-api --output CHANGELOG.md --template .github/changelog_template.md
 	bump-version changelog.yaml setup.py policyengine_api/constants.py
 	rm changelog_entry.yaml || true
-	touch changelog_entry.yaml 
+	touch changelog_entry.yaml 
+
+
+COMPOSE_FILE := docker/docker-compose.yml
+DOCKER_IMG=policyengine:policyengine-api
+DOCKER_NAME=policyengine-api
+ifeq (, $(shell which docker))
+DOCKER_CONTAINER_ID := docker-is-not-installed
+else
+DOCKER_CONTAINER_ID := $(shell docker ps --filter ancestor=$(DOCKER_IMG) --format "{{.ID}}")
+endif
+
+.PHONY: docker-build
+docker-build: ## Build the docker image
+	docker compose --file $(COMPOSE_FILE) build --force-rm
+
+.PHONY: docker-run
+docker-run:  ## Run the app as docker container with supporing services
+	docker compose --file $(COMPOSE_FILE) up
+
+.PHONY: services-start
+services-start:  ## Run the docker containers for supporting services (e.g. Redis)
+	docker compose --file $(COMPOSE_FILE) up -d redis
+
+.PHONY: docker-console
+docker-console:  ## Open a one-off container bash session
+	@docker run -p 8080:5000 -v $(PWD):/code \
+   --network policyengine-api_default \
+   --rm --name policyengine-api-console -it \
+   $(DOCKER_IMG) bash
+	@docker rm policyengine-api-console

docker/Dockerfile.api

@@ -0,0 +1,27 @@
+FROM python:3.11
+
+# use root to install pip libs
+USER root
+
+WORKDIR /code
+
+COPY . /code/
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+# can't use make install because we ignore .github/ via .dockerignore
+# and env is set via docker-compose.yml or injection at container run time
+RUN pip install -e ".[dev]" --config-settings editable_mode=compat
+
+# switch to app user
+RUN groupadd policyapi && \
+  useradd -g policyapi policyapi && \
+  apt-get purge -y --auto-remove build-essential && \
+  apt-get -y install make && \
+  chown -R policyapi:policyapi /code
+
+# TODO cannot switch because policyengine_core writes to data/storage
+RUN mkdir /usr/local/lib/python3.11/site-packages/policyengine_core/country_template/data/storage
+
+# USER policyapi

docker/docker-compose.yml

@@ -0,0 +1,34 @@
+services:
+  redis:
+    image: "redis:alpine"
+    expose:
+      - "6379"
+    volumes:
+      - redis-data:/data
+    networks:
+      - my_network
+
+  policyengine:
+    build:
+      context: ../
+      dockerfile: docker/Dockerfile.api
+    command: ["/bin/bash", "/code/docker/start.sh"]
+    image: policyengine:policyengine-api
+    depends_on:
+      - redis
+    env_file:
+      - ../.env
+    expose:
+      - 8080
+    ports:
+      - "8080:8080"
+    networks:
+      - my_network
+
+volumes:
+  redis-data:
+
+networks:
+  my_network:
+    name: policyengine-api_default
+    external: true

docker/start.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# Environment variables
+PORT="${PORT:-8080}"
+WORKER_COUNT="${WORKER_COUNT:-3}"
+REDIS_PORT="${REDIS_PORT:-6379}"
+
+# Start the API
+gunicorn -b :"$PORT" policyengine_api.api --timeout 300 --workers 5 --preload &
+
+# Start multiple workers using POSIX-compliant loop
+# TODO worker.py does not seem to exist?
+#i=1
+#while [ $i -le "$WORKER_COUNT" ]
+#do
+#    echo "Starting worker $i..."
+#    python3 policyengine_api/worker.py &
+#    i=$((i + 1))
+#done
+
+# Keep the script running and handle shutdown gracefully
+trap "pkill -P $$; exit 1" INT TERM
+
+wait

policyengine_api/api.py

@@ -4,6 +4,7 @@
 
 import time
 import sys
+import os
 
 start_time = time.time()
 
@@ -89,7 +90,7 @@ def log_timing(message):
     {
         "CACHE_TYPE": "RedisCache",
         "CACHE_KEY_PREFIX": "policyengine",
-        "CACHE_REDIS_HOST": "127.0.0.1",
+        "CACHE_REDIS_HOST": os.getenv("CACHE_REDIS_HOST", "127.0.0.1"),
         "CACHE_REDIS_PORT": 6379,
         "CACHE_DEFAULT_TIMEOUT": 300,
     }

policyengine_api/gcp_logging.py

@@ -1,3 +1,26 @@
+import os
+import logging
 from google.cloud.logging import Client
 
-logger = Client().logger("policyengine-api")
+if os.environ.get("FLASK_DEBUG") == "1":
+    logger = logging.getLogger(__name__)
+
+    # shims to make default logger act like google's
+    levels = {
+        "DEBUG": logging.DEBUG,
+        "INFO": logging.INFO,
+        "WARNING": logging.WARNING,
+        "ERROR": logging.ERROR,
+        "CRITICAL": logging.CRITICAL,
+    }
+
+    def log_struct(msg, severity="INFO"):
+        logger.log(levels.get(severity, "INFO"), msg)
+
+    def log_text(msg, severity="INFO"):
+        logger.log(levels.get(severity, "INFO"), msg)
+
+    logger.log_struct = log_struct
+    logger.log_text = log_text
+else:
+    logger = Client().logger("policyengine-api")

policyengine_api/libs/simulation_api.py

@@ -25,9 +25,9 @@ def __init__(self):
                 "GOOGLE_APPLICATION_CREDENTIALS not set; unable to run simulation API.",
                 severity="ERROR",
             )
-            raise ValueError(
-                "GOOGLE_APPLICATION_CREDENTIALS not set; unable to run simulation API."
-            )
+
+            return
+
         self.project = "prod-api-v2-c4d5"
         self.location = "us-central1"
         self.workflow = "simulation-workflow"

policyengine_api/routes/error_routes.py

@@ -3,10 +3,14 @@
 from werkzeug.exceptions import (
     HTTPException,
 )
+import logging
 
 error_bp = Blueprint("error", __name__)
 
 
+logger = logging.getLogger(__name__)
+
+
 @error_bp.app_errorhandler(404)
 def response_404(error) -> Response:
     """Specific handler for 404 Not Found errors"""
@@ -54,6 +58,8 @@ def make_error_response(
     status_code: int,
 ) -> Response:
     """Create a generic error response"""
+    logger.error(str(error))
+
     return Response(
         json.dumps(
             {