diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..c36476013 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.dockerignore +.editorconfig +.env +.git +.github +.gitignore +README.md + +docker/Docker* +docker/docker-compose* diff --git a/.gitignore b/.gitignore index 91af8c3c1..786aa9681 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ dist/* **/*.h5 **/*.csv.gz .env +.python-version # Ignore generated credentials from google-github-actions/auth gha-creds-*.json diff --git a/Makefile b/Makefile index fef3abcf6..685f77212 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,31 @@ -install: +.PHONY: help +help: ## Print this message + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-24s\033[0m %s\n", $$1, $$2}' + +.PHONY: install +install: ## Install dependencies and setup env pip install -e ".[dev]" --config-settings editable_mode=compat bash .github/setup_env.sh -debug: - FLASK_APP=policyengine_api.api FLASK_DEBUG=1 flask run --without-threads +.PHONY: debug +debug: ## Run the Flask app in debug mode + FLASK_APP=policyengine_api.api FLASK_DEBUG=1 flask run --without-threads --host=0.0.0.0 -test-env-vars: +.PHONY: test-env-vars +test-env-vars: ## Test environment variables pytest tests/env_variables -test: +test: ## Run tests MAX_HOUSEHOLDS=1000 coverage run -a --branch -m pytest tests/to_refactor tests/unit --disable-pytest-warnings coverage xml -i -debug-test: +debug-test: ## Run tests with debug verbosity MAX_HOUSEHOLDS=1000 FLASK_DEBUG=1 pytest -vv --durations=0 tests -format: +format: ## Run the black formmater black . -l 79 -deploy: +deploy: ## Deploy to GCP python gcp/export.py gcloud config set app/cloud_build_timeout 2400 cp gcp/policyengine_api/* . @@ -28,9 +35,39 @@ deploy: rm .gac.json rm .dbpw -changelog: +changelog: ## Build the changelog build-changelog changelog.yaml --output changelog.yaml --update-last-date --start-from 0.1.0 --append-file changelog_entry.yaml build-changelog changelog.yaml --org PolicyEngine --repo policyengine-api --output CHANGELOG.md --template .github/changelog_template.md bump-version changelog.yaml setup.py policyengine_api/constants.py rm changelog_entry.yaml || true - touch changelog_entry.yaml \ No newline at end of file + touch changelog_entry.yaml + + +COMPOSE_FILE := docker/docker-compose.yml +DOCKER_IMG=policyengine:policyengine-api +DOCKER_NAME=policyengine-api +ifeq (, $(shell which docker)) +DOCKER_CONTAINER_ID := docker-is-not-installed +else +DOCKER_CONTAINER_ID := $(shell docker ps --filter ancestor=$(DOCKER_IMG) --format "{{.ID}}") +endif + +.PHONY: docker-build +docker-build: ## Build the docker image + docker compose --file $(COMPOSE_FILE) build --force-rm + +.PHONY: docker-run +docker-run: ## Run the app as docker container with supporing services + docker compose --file $(COMPOSE_FILE) up + +.PHONY: services-start +services-start: ## Run the docker containers for supporting services (e.g. Redis) + docker compose --file $(COMPOSE_FILE) up -d redis + +.PHONY: docker-console +docker-console: ## Open a one-off container bash session + @docker run -p 8080:5000 -v $(PWD):/code \ + --network policyengine-api_default \ + --rm --name policyengine-api-console -it \ + $(DOCKER_IMG) bash + @docker rm policyengine-api-console \ No newline at end of file diff --git a/docker/Dockerfile.api b/docker/Dockerfile.api new file mode 100644 index 000000000..bb88bf170 --- /dev/null +++ b/docker/Dockerfile.api @@ -0,0 +1,27 @@ +FROM python:3.11 + +# use root to install pip libs +USER root + +WORKDIR /code + +COPY . /code/ + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +# can't use make install because we ignore .github/ via .dockerignore +# and env is set via docker-compose.yml or injection at container run time +RUN pip install -e ".[dev]" --config-settings editable_mode=compat + +# switch to app user +RUN groupadd policyapi && \ + useradd -g policyapi policyapi && \ + apt-get purge -y --auto-remove build-essential && \ + apt-get -y install make && \ + chown -R policyapi:policyapi /code + +# TODO cannot switch because policyengine_core writes to data/storage +RUN mkdir /usr/local/lib/python3.11/site-packages/policyengine_core/country_template/data/storage + +# USER policyapi \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..82518b9ae --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,34 @@ +services: + redis: + image: "redis:alpine" + expose: + - "6379" + volumes: + - redis-data:/data + networks: + - my_network + + policyengine: + build: + context: ../ + dockerfile: docker/Dockerfile.api + command: ["/bin/bash", "/code/docker/start.sh"] + image: policyengine:policyengine-api + depends_on: + - redis + env_file: + - ../.env + expose: + - 8080 + ports: + - "8080:8080" + networks: + - my_network + +volumes: + redis-data: + +networks: + my_network: + name: policyengine-api_default + external: true \ No newline at end of file diff --git a/docker/start.sh b/docker/start.sh new file mode 100644 index 000000000..b264a0039 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Environment variables +PORT="${PORT:-8080}" +WORKER_COUNT="${WORKER_COUNT:-3}" +REDIS_PORT="${REDIS_PORT:-6379}" + +# Start the API +gunicorn -b :"$PORT" policyengine_api.api --timeout 300 --workers 5 --preload & + +# Start multiple workers using POSIX-compliant loop +# TODO worker.py does not seem to exist? +#i=1 +#while [ $i -le "$WORKER_COUNT" ] +#do +# echo "Starting worker $i..." +# python3 policyengine_api/worker.py & +# i=$((i + 1)) +#done + +# Keep the script running and handle shutdown gracefully +trap "pkill -P $$; exit 1" INT TERM + +wait diff --git a/policyengine_api/api.py b/policyengine_api/api.py index b22529b31..f98200caf 100644 --- a/policyengine_api/api.py +++ b/policyengine_api/api.py @@ -4,6 +4,7 @@ import time import sys +import os start_time = time.time() @@ -89,7 +90,7 @@ def log_timing(message): { "CACHE_TYPE": "RedisCache", "CACHE_KEY_PREFIX": "policyengine", - "CACHE_REDIS_HOST": "127.0.0.1", + "CACHE_REDIS_HOST": os.getenv("CACHE_REDIS_HOST", "127.0.0.1"), "CACHE_REDIS_PORT": 6379, "CACHE_DEFAULT_TIMEOUT": 300, } diff --git a/policyengine_api/gcp_logging.py b/policyengine_api/gcp_logging.py index 1696af5d0..9f6632af2 100644 --- a/policyengine_api/gcp_logging.py +++ b/policyengine_api/gcp_logging.py @@ -1,3 +1,26 @@ +import os +import logging from google.cloud.logging import Client -logger = Client().logger("policyengine-api") +if os.environ.get("FLASK_DEBUG") == "1": + logger = logging.getLogger(__name__) + + # shims to make default logger act like google's + levels = { + "DEBUG": logging.DEBUG, + "INFO": logging.INFO, + "WARNING": logging.WARNING, + "ERROR": logging.ERROR, + "CRITICAL": logging.CRITICAL, + } + + def log_struct(msg, severity="INFO"): + logger.log(levels.get(severity, "INFO"), msg) + + def log_text(msg, severity="INFO"): + logger.log(levels.get(severity, "INFO"), msg) + + logger.log_struct = log_struct + logger.log_text = log_text +else: + logger = Client().logger("policyengine-api") diff --git a/policyengine_api/libs/simulation_api.py b/policyengine_api/libs/simulation_api.py index 1fbd12b48..0a5f9a7ee 100644 --- a/policyengine_api/libs/simulation_api.py +++ b/policyengine_api/libs/simulation_api.py @@ -25,9 +25,9 @@ def __init__(self): "GOOGLE_APPLICATION_CREDENTIALS not set; unable to run simulation API.", severity="ERROR", ) - raise ValueError( - "GOOGLE_APPLICATION_CREDENTIALS not set; unable to run simulation API." - ) + + return + self.project = "prod-api-v2-c4d5" self.location = "us-central1" self.workflow = "simulation-workflow" diff --git a/policyengine_api/routes/error_routes.py b/policyengine_api/routes/error_routes.py index e9fced1c0..f07355c4a 100644 --- a/policyengine_api/routes/error_routes.py +++ b/policyengine_api/routes/error_routes.py @@ -3,10 +3,14 @@ from werkzeug.exceptions import ( HTTPException, ) +import logging error_bp = Blueprint("error", __name__) +logger = logging.getLogger(__name__) + + @error_bp.app_errorhandler(404) def response_404(error) -> Response: """Specific handler for 404 Not Found errors""" @@ -54,6 +58,8 @@ def make_error_response( status_code: int, ) -> Response: """Create a generic error response""" + logger.error(str(error)) + return Response( json.dumps( {