eoAPI Kubernetes v0.7.0 Release (#221) #923
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| types: [ opened, reopened, synchronize, labeled ] | |
| env: | |
| HELM_VERSION: v3.15.2 | |
| PGO_VERSION: 5.7.4 | |
| jobs: | |
| helm-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: d3adb5/helm-unittest-action@v2 | |
| with: | |
| helm-version: ${{ env.HELM_VERSION }} | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: | | |
| cd helm-chart | |
| helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml | |
| k3s-integration-tests: | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| needs: helm-tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Start a local k3s cluster | |
| uses: jupyterhub/action-k3s-helm@v4 | |
| with: | |
| # See available: | |
| # - k3s release channels at https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml | |
| # - k3s versions at https://github.com/k3s-io/k3s/tags | |
| # - helm versions at https://github.com/helm/helm/tags | |
| k3s-channel: latest | |
| helm-version: ${{ env.HELM_VERSION }} | |
| metrics-enabled: false | |
| docker-enabled: true | |
| - name: last commit sha if PR | |
| if: ${{ github.event_name == 'pull_request' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> ${GITHUB_ENV} | |
| - name: last commit sha if push | |
| if: ${{ github.event_name == 'push' }} | |
| shell: bash | |
| run: | | |
| echo "LAST_COMMIT_SHA=${GITHUB_SHA}" >> ${GITHUB_ENV} | |
| - name: set k8s .release.name suffix | |
| run: | | |
| # salt for randomness per test run | |
| COMMITSHA=$(echo $LAST_COMMIT_SHA | cut -c 1-6) | |
| SALT=$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -c1-3) | |
| echo "RELEASE_NAME=eoapi$COMMITSHA$SALT" >> $GITHUB_ENV | |
| - name: helm install crunchydata postgres operator | |
| run: | | |
| helm upgrade --install \ | |
| --set disable_check_for_upgrades=true \ | |
| pgo \ | |
| oci://registry.developers.crunchydata.com/crunchydata/pgo \ | |
| --version ${{ env.PGO_VERSION }} | |
| - id: helm-render-install-eoapi-templates | |
| name: helm render/install eoapi templates | |
| continue-on-error: true | |
| run: | | |
| export GITSHA='${{github.sha}}' | |
| cd helm-chart | |
| helm dependency build eoapi | |
| helm install $RELEASE_NAME \ | |
| -f ./eoapi/values.yaml \ | |
| -f ./eoapi/test-k3s-unittest-values.yaml \ | |
| ./eoapi | |
| exit $? | |
| - name: debug pgstac-migrate job failure | |
| if: steps.helm-render-install-eoapi-templates.outcome == 'failure' | |
| continue-on-error: true | |
| run: | | |
| echo "Extracting pgstac-migrate job info and logs for debugging..." | |
| # Get job details | |
| echo "===== pgstac-migrate Job Details =====" | |
| kubectl get job pgstac-migrate -o yaml || echo "Could not get pgstac-migrate job details" | |
| # Get pod details | |
| echo "===== pgstac-migrate Pod Details =====" | |
| kubectl get pods -l app=pgstac-migrate --all-namespaces || echo "Could not find pgstac-migrate pods" | |
| # Extract logs from pgstac-migrate pod(s) | |
| echo "===== pgstac-migrate Pod Logs =====" | |
| echo "Looking for completed pods from pgstac-migrate job..." | |
| COMPLETED_PODS=$(kubectl get pods --selector=job-name=pgstac-migrate --field-selector=status.phase=Succeeded,status.phase=Failed -o jsonpath='{.items[*].metadata.name}' 2>/dev/null) | |
| if [ -n "$COMPLETED_PODS" ]; then | |
| echo "Found completed pods from pgstac-migrate job. Extracting logs from each:" | |
| for POD in $COMPLETED_PODS; do | |
| echo "--- Logs from completed pod $POD ---" | |
| kubectl logs pod/$POD || echo "Could not get logs from pod $POD" | |
| done | |
| else | |
| echo "No completed pods found for pgstac-migrate job" | |
| fi | |
| # Get details about the database pods/services | |
| echo "===== Database Pod/Service Details =====" | |
| # Find database service | |
| kubectl get svc | grep "db" || echo "Could not find database services" | |
| # Find database pods | |
| kubectl get pods | grep "db-" || echo "Could not find database pods" | |
| # Check for any events related to the job or pods | |
| echo "===== Related Kubernetes Events =====" | |
| kubectl get events | grep -E "pgstac|db" || echo "No relevant events found" | |
| - id: watchservices | |
| name: watch services boot | |
| timeout-minutes: 3 | |
| continue-on-error: true | |
| run: | | |
| # Now wait for services to boot up | |
| while [[ -z "$(kubectl get pod | grep "^raster-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do | |
| echo "still waiting for raster service to start..." | |
| sleep 1 | |
| done | |
| echo "raster service has started, moving on..." | |
| while [[ -z "$(kubectl get pod | grep "^vector-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/healthz" | head -n 1)" ]]; do | |
| echo "still waiting for vector service to start..." | |
| sleep 1 | |
| done | |
| echo "vector service has started, moving on..." | |
| while [[ -z "$(kubectl get pod | grep "^stac-$RELEASE_NAME-.*$" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} | grep "GET /.*/_mgmt/ping" | head -n 1)" ]]; do | |
| echo "still waiting for stac service to start..." | |
| sleep 1 | |
| done | |
| echo "all services have started, moving on..." | |
| - name: cleanup if services fail to boot | |
| if: steps.watchservices.outcome == 'failure' | |
| run: | | |
| echo "The watchservices step failed or timed out. Extracting pod logs for debugging..." | |
| # Get and display all pods status | |
| echo "===== Pod Status =====" | |
| kubectl get pods | |
| # Extract logs from raster pod init container (wait-for-pgstacbootstrap) | |
| echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" | |
| # Extract logs from raster pod main container | |
| echo "===== Raster Pod Main Container Logs =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get raster main container logs" | |
| # Extract logs from vector pod | |
| echo "===== Vector Pod Logs =====" | |
| kubectl get pod | grep "^vector-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get vector logs" | |
| # Extract logs from stac pod | |
| echo "===== STAC Pod Logs =====" | |
| kubectl get pod | grep "^stac-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get STAC logs" | |
| # Check if pods are in pending state or have issues | |
| echo "===== Pod Descriptions for Troubleshooting =====" | |
| kubectl get pod | grep "$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl describe pod/{} || echo "Could not describe pods" | |
| # force GH action to show failed result | |
| exit 128 | |
| - name: install python unit-test dependencies | |
| run: | | |
| python -m pip install pytest httpx | |
| - name: run the tests | |
| id: testrunner | |
| # continue-on-error: true | |
| run: | | |
| kubectl get svc --all-namespaces | |
| kubectl get ingress --all-namespaces -o jsonpath='{range .items[0]}kubectl describe ingress {.metadata.name} -n {.metadata.namespace}{end}' | sh | |
| kubectl get middleware.traefik.io --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers | while read -r namespace name; do kubectl describe middleware.traefik.io "$name" -n "$namespace"; done | |
| # Get the IP address of the Traefik service | |
| PUBLICIP_VALUE=$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') | |
| PUBLICIP=http://eoapi.local | |
| export VECTOR_ENDPOINT=$PUBLICIP/vector | |
| export STAC_ENDPOINT=$PUBLICIP/stac | |
| export RASTER_ENDPOINT=$PUBLICIP/raster | |
| # Add entry to /etc/hosts for eoapi.local | |
| echo "Adding eoapi.local to /etc/hosts with IP: $PUBLICIP_VALUE" | |
| echo "$PUBLICIP_VALUE eoapi.local" | sudo tee -a /etc/hosts | |
| echo '#################################' | |
| echo $VECTOR_ENDPOINT | |
| echo $STAC_ENDPOINT | |
| echo $RASTER_ENDPOINT | |
| echo '#################################' | |
| # Run tests with proper failure propagation | |
| set -e # Make sure any command failure causes the script to exit with error | |
| pytest .github/workflows/tests/test_vector.py || { kubectl logs svc/vector; exit 1; } | |
| pytest .github/workflows/tests/test_stac.py || { kubectl logs svc/stac; exit 1; } | |
| # TODO: fix raster tests | |
| #pytest .github/workflows/tests/test_raster.py || { kubectl logs svc/raster; exit 1; } | |
| - name: error if tests failed | |
| if: steps.testrunner.outcome == 'failure' | |
| run: | | |
| echo "The tests failed. Extracting pod logs for debugging..." | |
| # Get and display all pods status | |
| echo "===== Pod Status =====" | |
| kubectl get pods | |
| # Extract logs from raster pod init container (wait-for-pgstacbootstrap) | |
| echo "===== Raster Pod Init Container Logs (wait-for-pgstacbootstrap) =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstacbootstrap --tail=100 || echo "Could not get raster init container logs" | |
| # Extract logs from raster pod main container | |
| echo "===== Raster Pod Main Container Logs =====" | |
| kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get raster main container logs" | |
| # Extract logs from vector pod | |
| echo "===== Vector Pod Logs =====" | |
| kubectl get pod | grep "^vector-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get vector logs" | |
| # Extract logs from stac pod | |
| echo "===== STAC Pod Logs =====" | |
| kubectl get pod | grep "^stac-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get STAC logs" | |
| # Check if pods are in pending state or have issues | |
| echo "===== Pod Descriptions for Troubleshooting =====" | |
| kubectl get pod | grep "$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl describe pod/{} || echo "Could not describe pods" | |
| # force GH action to show failed result | |
| exit 128 | |
| - name: helm uninstall eoapi templates | |
| run: | | |
| helm uninstall $RELEASE_NAME |