diff --git a/.github/workflows/api-ci.yml b/.github/workflows/api-ci.yml index e92a37fb4e..98cd5afa45 100644 --- a/.github/workflows/api-ci.yml +++ b/.github/workflows/api-ci.yml @@ -17,9 +17,9 @@ jobs: api-lint: name: Run Lint (API) if: ${{ inputs.skip == false }} - uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.36 with: - working_directory: apps/codecov-api + make_target_prefix: api. api-mypy: name: Patch typing (API) @@ -28,72 +28,98 @@ jobs: with: working_directory: apps/codecov-api + # Once we cut over to umbrella, we can just replace the default cache key in build-app.yml + # and self-hosted.yml and get rid of this step. + api-compute-reqs-cache-key: + name: Compute cache key for requirements image + runs-on: ubuntu-latest + outputs: + reqs-cache-key: ${{ steps.compute.outputs.cache-key }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - id: compute + run: | + echo cache-key=${{ hashFiles('apps/codecov-api/uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} >> "$GITHUB_OUTPUT" + api-build: name: Build App (API) if: ${{ inputs.skip == false }} - uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.31 + needs: [api-compute-reqs-cache-key] + uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.36 secrets: inherit with: repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api + make_target_prefix: api. + reqs_cache_key: ${{ needs.api-compute-reqs-cache-key.outputs.reqs-cache-key }} api-test: name: Test (API) if: ${{ inputs.skip == false }} needs: [api-build] - uses: codecov/gha-workflows/.github/workflows/run-tests.yml@v1.2.35 + uses: codecov/gha-workflows/.github/workflows/run-tests.yml@v1.2.36 secrets: inherit with: run_integration: false repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api flag_prefix: api pytest_rootdir: /app + make_target_prefix: api. api-build-self-hosted: name: Build Self Hosted (API) if: ${{ inputs.skip == false }} needs: [api-build, api-test] - uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.36 secrets: inherit with: repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api + make_target_prefix: api. + reqs_cache_key: ${{ needs.api-compute-reqs-cache-key.outputs.reqs-cache-key }} api-staging: name: Push Staging Image (API) needs: [api-build, api-test] if: ${{ inputs.skip == false && github.event_name == 'push' && github.event.ref == 'refs/heads/staging' && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.36 secrets: inherit with: environment: staging repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api sentry_project: api + make_target_prefix: api. api-production: name: Push Production Image (API) needs: [api-build, api-test] if: ${{ inputs.skip == false && github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.36 secrets: inherit with: environment: production repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api sentry_project: api + make_target_prefix: api. api-self-hosted: name: Push Self Hosted Image (API) needs: [api-build-self-hosted, api-test] secrets: inherit if: ${{ inputs.skip == false && github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.36 with: push_rolling: true repo: ${{ vars.CODECOV_API_IMAGE_V2 || vars.CODECOV_API_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-api' }} - working_directory: apps/codecov-api + output_directory: apps/codecov-api + make_target_prefix: api. + reqs_cache_key: ${{ needs.api-compute-reqs-cache-key.outputs.reqs-cache-key }} # This job works around a strange interaction between reusable workflows and # path filters. diff --git a/.github/workflows/shared-ci.yml b/.github/workflows/shared-ci.yml index 1a4981cb46..5a128c8e39 100644 --- a/.github/workflows/shared-ci.yml +++ b/.github/workflows/shared-ci.yml @@ -12,9 +12,9 @@ jobs: shared-lint: name: Run Lint (Shared) if: ${{ inputs.skip == false }} - uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.36 with: - working_directory: libs/shared + make_target_prefix: shared. shared-benchmark: name: Benchmarks (Shared) @@ -138,7 +138,6 @@ jobs: # The coverage action will have installed codecovcli with pip. The # actual binary will be found in $PATH. binary: codecovcli - recurse_submodules: true # This job works around a strange interaction between reusable workflows and # path filters. diff --git a/.github/workflows/worker-ci.yml b/.github/workflows/worker-ci.yml index adbcf7e9bd..934447da90 100644 --- a/.github/workflows/worker-ci.yml +++ b/.github/workflows/worker-ci.yml @@ -2,7 +2,7 @@ name: Worker CI on: workflow_call: - inputs: + inputs: skip: type: boolean default: false @@ -17,9 +17,9 @@ jobs: worker-lint: name: Run Lint (Worker) if: ${{ inputs.skip == false }} - uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/lint.yml@v1.2.36 with: - working_directory: apps/worker + make_target_prefix: worker. worker-mypy: name: Patch typing (Worker) @@ -28,71 +28,97 @@ jobs: with: working_directory: apps/worker + # Once we cut over to umbrella, we can just replace the default cache key in build-app.yml + # and self-hosted.yml and get rid of this step. + worker-compute-reqs-cache-key: + name: Compute cache key for requirements image + runs-on: ubuntu-latest + outputs: + reqs-cache-key: ${{ steps.compute.outputs.cache-key }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - id: compute + run: | + echo cache-key=${{ hashFiles('apps/worker/uv.lock') }}-${{ hashFiles('docker/Dockerfile.requirements') }}-${{ hashFiles('libs/shared/**') }} >> "$GITHUB_OUTPUT" + worker-build: name: Build App (Worker) if: ${{ inputs.skip == false }} - uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.31 + needs: [worker-compute-reqs-cache-key] + uses: codecov/gha-workflows/.github/workflows/build-app.yml@v1.2.36 secrets: inherit with: repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker + make_target_prefix: worker. + reqs_cache_key: ${{ needs.worker-compute-reqs-cache-key.outputs.reqs-cache-key }} worker-test: name: Test (Worker) if: ${{ inputs.skip == false }} needs: [worker-build] - uses: codecov/gha-workflows/.github/workflows/run-tests.yml@v1.2.35 + uses: codecov/gha-workflows/.github/workflows/run-tests.yml@v1.2.36 secrets: inherit with: repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker flag_prefix: worker pytest_rootdir: /app + make_target_prefix: worker. worker-build-self-hosted: name: Build Self Hosted (Worker) if: ${{ inputs.skip == false }} needs: [worker-build, worker-test] - uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.36 secrets: inherit with: repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker + make_target_prefix: worker. + reqs_cache_key: ${{ needs.worker-compute-reqs-cache-key.outputs.reqs-cache-key }} worker-staging: name: Push Staging Image (Worker) needs: [worker-build, worker-test] if: ${{ inputs.skip == false && github.event_name == 'push' && (github.event.ref == 'refs/heads/main' || github.event.ref == 'refs/heads/staging') && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.36 secrets: inherit with: environment: staging repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker sentry_project: worker + make_target_prefix: worker. worker-production: name: Push Production Image (Worker) needs: [worker-build, worker-test] if: ${{ inputs.skip == false && github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/push-env.yml@v1.2.36 secrets: inherit with: environment: production repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker sentry_project: worker + make_target_prefix: worker. worker-self-hosted: name: Push Self Hosted Image (Worker) - needs: [worker-build-self-hosted, worker-test] + needs: [worker-compute-reqs-cache-key, worker-build-self-hosted, worker-test] secrets: inherit if: ${{ inputs.skip == false && github.event_name == 'push' && github.event.ref == 'refs/heads/main' && github.repository_owner == 'codecov' }} - uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.31 + uses: codecov/gha-workflows/.github/workflows/self-hosted.yml@v1.2.36 with: push_rolling: true repo: ${{ vars.CODECOV_WORKER_IMAGE_V2 || vars.CODECOV_WORKER_IMAGE_V2_SELF_HOSTED || 'codecov/self-hosted-worker' }} - working_directory: apps/worker + output_directory: apps/worker + make_target_prefix: worker. + reqs_cache_key: ${{ needs.worker-compute-reqs-cache-key.outputs.reqs-cache-key }} # This job works around a strange interaction between reusable workflows and # path filters. diff --git a/Makefile b/Makefile index 61b7e65ddf..94d85ce54b 100644 --- a/Makefile +++ b/Makefile @@ -1,280 +1,102 @@ -sha := $(shell git rev-parse --short=7 HEAD) -long_sha := $(shell git rev-parse HEAD) -release_version := `cat VERSION` -build_date ?= $(shell git show -s --date=iso8601-strict --pretty=format:%cd $$sha) -branch = $(shell git branch | grep \* | cut -f2 -d' ') -epoch := $(shell date +"%s") -AR_REPO ?= codecov/umbrella -DOCKERHUB_REPO ?= codecov/self-hosted-api -REQUIREMENTS_TAG := requirements-v1-$(shell sha1sum requirements.txt | cut -d ' ' -f 1)-$(shell sha1sum docker/Dockerfile.requirements | cut -d ' ' -f 1) -VERSION := release-${sha} -CODECOV_UPLOAD_TOKEN ?= "notset" -CODECOV_STATIC_TOKEN ?= "notset" -TIMESERIES_ENABLED ?= "true" -CODECOV_URL ?= "https://api.codecov.io" -export DOCKER_BUILDKIT=1 -export API_DOCKER_REPO=${AR_REPO} -export API_DOCKER_VERSION=${VERSION} -export CODECOV_TOKEN=${CODECOV_UPLOAD_TOKEN} -API_DOMAIN ?= api -PROXY_NETWORK ?= api_default - -# Codecov CLI version to use -CODECOV_CLI_VERSION := 0.5.1 - -include tools/devenv/Makefile.devenv - -update-reqs: - cd codecov-api && git pull - cd worker && git pull - cd shared && git pull - cat worker/requirements.in codecov-api/requirements.in | sort -u > requirements.in - pip-compile requirements.in - -build: - make build.requirements - make build.app - -check-for-migration-conflicts: - python manage.py check_for_migration_conflicts +export sha := $(shell git rev-parse --short=7 HEAD) +export full_sha := $(shell git rev-parse HEAD) +export long_sha := ${full_sha} +export merge_sha := $(shell git merge-base HEAD^ origin/main) +export VERSION := release-${sha} -test: - COVERAGE_CORE=sysmon python -m pytest --cov=./ --junitxml=junit.xml +export build_date ?= $(shell git show -s --date=iso8601-strict --pretty=format:%cd $$sha) +export branch := $(shell git branch | grep \* | cut -f2 -d' ') -test.unit: - COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml - -test.integration: - COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml - -lint: - make lint.install - make lint.run - -lint.install: - echo "Installing..." - pip install -Iv ruff - -lint.run: - ruff check - ruff format +export DOCKER_BUILDKIT=1 -lint.check: - echo "Linting..." - ruff check - echo "Formatting..." - ruff format --check +export SHARED_SHA := $(shell git archive --format=tar HEAD libs/shared | sha1sum | head -c 40) +export DOCKER_REQS_SHA := $(shell sha1sum docker/Dockerfile.requirements | head -c 40) -build.requirements: - # if docker pull succeeds, we have already build this version of - # requirements.txt. Otherwise, build and push a version tagged - # with the hash of this requirements.txt - touch .testenv +# Generic target for building a requirements image. You probably want +# `worker.build.requirements` or `api.build.requirements`. +_build.requirements: docker pull ${AR_REPO}:${REQUIREMENTS_TAG} || docker build \ - -f docker/Dockerfile.requirements . \ - -t ${AR_REPO}:${REQUIREMENTS_TAG} \ - -t codecov/umbrella-ci-requirements:${REQUIREMENTS_TAG} - -build.local: - docker build -f docker/Dockerfile . \ - -t ${AR_REPO}:latest \ - -t ${AR_REPO}:${VERSION} \ - --build-arg REQUIREMENTS_IMAGE=${AR_REPO}:${REQUIREMENTS_TAG} \ - --build-arg BUILD_ENV=local - -build.app: - docker build -f docker/Dockerfile . \ - -t ${AR_REPO}:latest \ - -t ${AR_REPO}:${VERSION} \ - --label "org.label-schema.vendor"="Codecov" \ - --label "org.label-schema.version"="${release_version}-${sha}" \ - --label "org.opencontainers.image.revision"="$(long_sha)" \ - --label "org.opencontainers.image.source"="github.com/codecov/umbrella" \ - --build-arg REQUIREMENTS_IMAGE=${AR_REPO}:${REQUIREMENTS_TAG} \ - --build-arg RELEASE_VERSION=${VERSION} \ - --build-arg BUILD_ENV=cloud - -build.self-hosted: - make build.self-hosted-base - make build.self-hosted-runtime - -build.self-hosted-base: - docker build -f docker/Dockerfile . \ - -t ${DOCKERHUB_REPO}:latest-no-dependencies \ - -t ${DOCKERHUB_REPO}:${VERSION}-no-dependencies \ - --build-arg REQUIREMENTS_IMAGE=${AR_REPO}:${REQUIREMENTS_TAG} \ - --build-arg RELEASE_VERSION=${VERSION} \ - --build-arg BUILD_ENV=self-hosted - -build.self-hosted-runtime: - docker build -f docker/Dockerfile . \ - -t ${DOCKERHUB_REPO}:latest \ - -t ${DOCKERHUB_REPO}:${VERSION} \ - --label "org.label-schema.vendor"="Codecov" \ - --label "org.label-schema.version"="${release_version}-${sha}" \ - --build-arg REQUIREMENTS_IMAGE=${AR_REPO}:${REQUIREMENTS_TAG} \ - --build-arg RELEASE_VERSION=${VERSION} \ - --build-arg BUILD_ENV=self-hosted-runtime - -tag.latest: - docker tag ${AR_REPO}:${VERSION} ${AR_REPO}:latest - -tag.staging: - docker tag ${AR_REPO}:${VERSION} ${AR_REPO}:staging-${VERSION} - -tag.production: - docker tag ${AR_REPO}:${VERSION} ${AR_REPO}:production-${VERSION} - -tag.self-hosted-rolling: - docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${DOCKERHUB_REPO}:rolling_no_dependencies - docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:rolling - -tag.self-hosted-release: - docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${DOCKERHUB_REPO}:${release_version}_no_dependencies - docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${DOCKERHUB_REPO}:latest_calver_no_dependencies - docker tag ${DOCKERHUB_REPO}:${VERSION}-no-dependencies ${DOCKERHUB_REPO}:latest_stable_no_dependencies - docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:${release_version} - docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-stable - docker tag ${DOCKERHUB_REPO}:${VERSION} ${DOCKERHUB_REPO}:latest-calver - -load.requirements: - docker load --input requirements.tar - docker tag codecov/umbrella-ci-requirements:${REQUIREMENTS_TAG} ${AR_REPO}:${REQUIREMENTS_TAG} - -load.self-hosted: - docker load --input self-hosted-runtime.tar - docker load --input self-hosted.tar - -save.app: - docker save -o app.tar ${AR_REPO}:${VERSION} - -save.requirements: - docker tag ${AR_REPO}:${REQUIREMENTS_TAG} codecov/umbrella-ci-requirements:${REQUIREMENTS_TAG} - docker save -o requirements.tar codecov/umbrella-ci-requirements:${REQUIREMENTS_TAG} - -save.self-hosted: - make save.self-hosted-base - make save.self-hosted-runtime - -save.self-hosted-base: - docker save -o self-hosted.tar ${DOCKERHUB_REPO}:${VERSION}-no-dependencies - -save.self-hosted-runtime: - docker save -o self-hosted-runtime.tar ${DOCKERHUB_REPO}:${VERSION} - -push.latest: - docker push ${AR_REPO}:latest - -push.staging: - docker push ${AR_REPO}:staging-${VERSION} - -push.production: - docker push ${AR_REPO}:production-${VERSION} - -push.requirements: - docker push ${AR_REPO}:${REQUIREMENTS_TAG} - -push.self-hosted-release: - docker push ${DOCKERHUB_REPO}:${release_version}_no_dependencies - docker push ${DOCKERHUB_REPO}:latest_calver_no_dependencies - docker push ${DOCKERHUB_REPO}:latest_stable_no_dependencies - docker push ${DOCKERHUB_REPO}:${release_version} - docker push ${DOCKERHUB_REPO}:latest-stable - docker push ${DOCKERHUB_REPO}:latest-calver - -push.self-hosted-rolling: - docker push ${DOCKERHUB_REPO}:rolling_no_dependencies - docker push ${DOCKERHUB_REPO}:rolling - -test_env.up: - env | grep GITHUB > .testenv; true - TIMESERIES_ENABLED=${TIMESERIES_ENABLED} docker-compose up -d - -test_env.prepare: - docker-compose exec api make test_env.container_prepare - -test_env.check_db: - docker-compose exec umbrella make test_env.container_check_db - make test_env.check-for-migration-conflicts - -test_env.install_cli: - pip install codecov-cli==$(CODECOV_CLI_VERSION) - -test_env.container_prepare: - apt-get -y install git build-essential netcat-traditional - make test_env.install_cli - git config --global --add safe.directory /app - -test_env.container_check_db: - while ! nc -vz postgres 5432; do sleep 1; echo "waiting for postgres"; done - while ! nc -vz timescale 5432; do sleep 1; echo "waiting for timescale"; done - -test_env.run_unit: - docker-compose exec umbrella make test.unit - -test_env.run_integration: - #docker-compose exec umbrella make test.integration - echo "Skipping. No Tests" - -test_env.check-for-migration-conflicts: - docker-compose exec umbrella python manage.py check_for_migration_conflicts - -test_env.upload: - docker-compose exec umbrella make test_env.container_upload CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL} - docker-compose exec umbrella make test_env.container_upload_test_results CODECOV_UPLOAD_TOKEN=${CODECOV_UPLOAD_TOKEN} CODECOV_URL=${CODECOV_URL} - -test_env.container_upload: - codecovcli -u ${CODECOV_URL} upload-process --flag unit-latest-uploader --flag unit \ - --coverage-files-search-exclude-folder=graphql_api/types/** \ - --coverage-files-search-exclude-folder=api/internal/tests/unit/views/cassetes/** - -test_env.container_upload_test_results: - codecovcli -u ${CODECOV_URL} do-upload --report-type "test_results" \ - --files-search-exclude-folder=graphql_api/types/** \ - --files-search-exclude-folder=api/internal/tests/unit/views/cassetes/** || true - -test_env: - make test_env.up - make test_env.prepare - make test_env.check_db - make test_env.run_unit - make test_env.check-for-migration-conflicts - - -### START Proxy Commands -.PHONY: proxy.build -proxy.build: # Used to build the proxy -proxy.build: - docker build -f docker/Dockerfile-proxy . -t ${API_DOCKER_REPO}/proxy:latest -t ${API_DOCKER_REPO}/proxy:${release_version}-${sha} \ - --label "org.label-schema.build-date"="$(build_date)" \ - --label "org.label-schema.name"="API Proxy" \ - --label "org.label-schema.vendor"="api" \ - --label "org.label-schema.version"="${release_version}" - -.PHONY: proxy.run -proxy.run: # Used to run the proxy -proxy.run: - make proxy.build - make proxy.down - docker run --rm --network ${PROXY_NETWORK} -e FRP_TOKEN=${FRP_TOKEN} -e DOMAIN=${API_DOMAIN} --name api-proxy ${API_DOCKER_REPO}/proxy:latest - sleep 3 - make proxy.logs - # You should see "[api] start proxy success" - # If no logs then proxy failed to start. Check if you are on VPN. If you get a 404, check if you are on VPN - -.PHONY: proxy.logs -proxy.logs: # Used to logs the proxy -proxy.logs: - docker logs api-proxy - -.PHONY: proxy.shell -proxy.shell: # Used to shell the proxy -proxy.shell: - docker exec -it api-proxy sh - -.PHONY: proxy.down -proxy.down: # Used to down the proxy -proxy.down: - docker kill api-proxy || true - -### END PROXY Commands + -f docker/Dockerfile.requirements . \ + --build-arg APP_DIR=${APP_DIR} \ + -t ${AR_REPO}:${REQUIREMENTS_TAG} \ + -t ${CI_REQS_REPO}:${REQUIREMENTS_TAG} + +###### +# codecov-api targets +###### +API_UV_LOCK_SHA := $(shell sha1sum apps/codecov-api/uv.lock | head -c 40) +API_REQS_TAG := reqs-${API_UV_LOCK_SHA}-${DOCKER_REQS_SHA}-${SHARED_SHA} + +define api_rule_prefix +.PHONY: $(1) +$(1): export APP_DIR := apps/codecov-api +$(1): export REQUIREMENTS_TAG := ${API_REQS_TAG} +$(1): export AR_REPO ?= codecov/api-umbrella +$(1): export DOCKERHUB_REPO ?= codecov/self-hosted-api-umbrella +$(1): export CI_REQS_REPO ?= codecov/api-ci-requirements +endef + +# umbrella builds a special requirements image for api that installs shared properly. +$(eval $(call api_rule_prefix,api.build.requirements)) +api.build.requirements: + $(MAKE) _build.requirements + +# This target calls `make build.requirements` for api so we have to make sure it calls our +# custom `build.requirements` instead. +$(eval $(call api_rule_prefix,api.build)) +api.build: + $(MAKE) api.build.requirements + $(MAKE) -C apps/codecov-api build.local + +# Any other target starting with `api.` should be forwarded to `apps/codecov-api`. +# The `$*` variable is the string caught by the `%` in the rule pattern. +$(eval $(call api_rule_prefix,api.%)) +api.%: + $(MAKE) -C apps/codecov-api $* + +###### +# worker targets +###### +WORKER_UV_LOCK_SHA := $(shell sha1sum apps/worker/uv.lock | head -c 40) +WORKER_REQS_TAG := reqs-${WORKER_UV_LOCK_SHA}-${DOCKER_REQS_SHA}-${SHARED_SHA} + +define worker_rule_prefix +.PHONY: $(1) +$(1): export APP_DIR := apps/worker +$(1): export REQUIREMENTS_TAG := ${WORKER_REQS_TAG} +$(1): export AR_REPO ?= codecov/worker-umbrella +$(1): export DOCKERHUB_REPO ?= codecov/self-hosted-worker-umbrella +$(1): export CI_REQS_REPO ?= codecov/worker-ci-requirements +endef + +# umbrella builds a special requirements image for worker that installs shared properly. +$(eval $(call worker_rule_prefix,worker.build.requirements)) +worker.build.requirements: + $(MAKE) _build.requirements + +# This target calls `make build.requirements` for worker so we have to make sure it calls our +# custom `build.requirements` instead. +$(eval $(call worker_rule_prefix,worker.build)) +worker.build: + $(MAKE) worker.build.requirements + $(MAKE) -C apps/worker build.local + +# Any other target starting with `worker.` should be forwarded to `apps/worker`. +# The `$*` variable is the string caught by the `%` in the rule pattern. +$(eval $(call worker_rule_prefix,worker.%)) +worker.%: + $(MAKE) -C apps/worker $* + +###### +# shared targets +###### + +# No need to override any of shared's targets. Just run make in `libs/shared`. +.PHONY: shared.% +shared.%: + $(MAKE) -C libs/shared $* + +###### +# Development environment targets +###### +include tools/devenv/Makefile.devenv diff --git a/apps/codecov-api b/apps/codecov-api index defc6a5acf..6bb21bf03c 160000 --- a/apps/codecov-api +++ b/apps/codecov-api @@ -1 +1 @@ -Subproject commit defc6a5acf7c03dda8ef757c47b42daa3468aeb1 +Subproject commit 6bb21bf03c1fd90c3ca9c00c1e9bbbee0b5ec763 diff --git a/apps/worker b/apps/worker index ad52223de9..09f11c32b2 160000 --- a/apps/worker +++ b/apps/worker @@ -1 +1 @@ -Subproject commit ad52223de9313e23dc140a3bf6248dab4646beb7 +Subproject commit 09f11c32b23019b6badc0f6f8e690d43f0970a7c diff --git a/docker-compose.yml b/docker-compose.yml index f168da37ca..80db8c4aa3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: - codecov api: - image: codecov/api + image: codecov/api-umbrella entrypoint: sh -c "/devenv-scripts/start-api.sh" environment: - RUN_ENV=DEV @@ -53,7 +53,7 @@ services: - redis worker: - image: codecov/worker + image: codecov/worker-umbrella entrypoint: sh -c "/devenv-scripts/start-worker.sh" environment: - RUN_ENV=DEV @@ -124,7 +124,7 @@ services: - codecov minio: - image: minio/minio:RELEASE.2020-04-15T00-39-01Z + image: minio/minio:latest command: server /export ports: - "${CODECOV_MINIO_PORT-9000}:9000" diff --git a/docker/Dockerfile.requirements b/docker/Dockerfile.requirements index c27b8369d1..9eec9d3793 100644 --- a/docker/Dockerfile.requirements +++ b/docker/Dockerfile.requirements @@ -1,16 +1,14 @@ # syntax=docker/dockerfile:1.4 -ARG PYTHON_IMAGE=python:3.12-slim-bookworm +ARG PYTHON_IMAGE=ghcr.io/astral-sh/uv:python3.13-bookworm-slim -# BUILD STAGE - Download dependencies from GitHub that require SSH access -FROM $PYTHON_IMAGE as build +ARG APP_DIR +# BUILD STAGE +FROM $PYTHON_IMAGE as build -# Pinning a specific nightly version so that builds don't suddenly break if a -# "this feature is now stabilized" warning is promoted to an error or something. -# We would like to keep up with nightly if we can. -ARG RUST_VERSION=nightly-2024-02-22 -ENV RUST_VERSION=${RUST_VERSION} +ARG APP_DIR +# Install all system packages needed by either worker OR codecov-api RUN apt-get update RUN apt-get install -y \ build-essential \ @@ -18,37 +16,64 @@ RUN apt-get install -y \ libpq-dev \ libxml2-dev \ libxslt-dev \ + git \ curl -# Install Rust +# Install the Rust toolchain which we need to build test-results-parser +ARG RUST_VERSION=stable +ENV RUST_VERSION=${RUST_VERSION} RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | bash -s -- -y --default-toolchain $RUST_VERSION + | bash -s -- -y --profile minimal --default-toolchain $RUST_VERSION ENV PATH="/root/.cargo/bin:$PATH" -COPY requirements.txt / -WORKDIR /pip-packages/ -RUN pip wheel -r /requirements.txt -RUN rm -rf /pip-packages/src +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + UV_PYTHON_DOWNLOADS=never \ + UV_PYTHON=python \ + UV_PROJECT_ENVIRONMENT=/app + +# Export a requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=${APP_DIR}/pyproject.toml,target=${APP_DIR}/pyproject.toml \ + --mount=type=bind,source=${APP_DIR}/uv.lock,target=${APP_DIR}/uv.lock \ + --mount=type=bind,source=libs/shared,target=libs/shared \ + uv --directory ${APP_DIR} export --no-hashes --frozen --format requirements-txt > requirements.txt -ENV PYCURL_SSL_LIBRARY=openssl +# The resulting requirements.txt includes ourselves as a dependency we should install +# "editably". Filter that out, leaving only external dependencies. +RUN grep -v '^-e ' requirements.txt > ${APP_DIR}/requirements.remote.txt -# RUNTIME STAGE - Copy packages from build stage and install runtime dependencies +# Fetch/build wheels for all external dependencies. +RUN --mount=type=bind,source=libs/shared,target=libs/shared \ + (cd ${APP_DIR} && pip wheel -w /wheels --find-links /wheels -r requirements.remote.txt) + +# Build a wheel for ourselves +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=${APP_DIR}/pyproject.toml,target=${APP_DIR}/pyproject.toml \ + --mount=type=bind,source=${APP_DIR}/uv.lock,target=${APP_DIR}/uv.lock \ + --mount=type=bind,source=libs/shared,target=libs/shared \ + uv build --directory ${APP_DIR} --all-packages --wheel -o /wheels + +# RUNTIME STAGE FROM $PYTHON_IMAGE -# Our postgres driver psycopg2 requires libpq-dev as a runtime dependency +ARG APP_DIR + RUN apt-get update RUN apt-get install -y \ - libpq-dev \ libxml2-dev \ libxslt-dev \ - make \ - curl \ - && pip install --upgrade pip + libpq-dev \ + libexpat1 \ + make -WORKDIR /pip-packages/ -COPY --from=build /pip-packages/ /pip-packages/ +COPY --from=build /wheels/ /wheels/ -RUN pip install --no-deps --no-index --find-links=/pip-packages/ /pip-packages/* +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,source=${APP_DIR}/pyproject.toml,target=${APP_DIR}/pyproject.toml \ + --mount=type=bind,source=${APP_DIR}/uv.lock,target=${APP_DIR}/uv.lock \ + --mount=type=bind,source=libs/shared,target=libs/shared \ + uv --directory ${APP_DIR} pip install --no-deps --no-index --find-links=/wheels /wheels/* --system RUN addgroup --system application \ - && adduser --system codecov --ingroup application --home /home/codecov \ No newline at end of file + && adduser --system codecov --ingroup application --home /home/codecov diff --git a/libs/shared b/libs/shared index d591317cb5..ece4366f3b 160000 --- a/libs/shared +++ b/libs/shared @@ -1 +1 @@ -Subproject commit d591317cb558fa499e81e52d9eacf61a2ff52744 +Subproject commit ece4366f3bce7fab2216704f85e365fbbe0f147d diff --git a/tools/devenv/Makefile.devenv b/tools/devenv/Makefile.devenv index 23753b09dc..be5a876594 100644 --- a/tools/devenv/Makefile.devenv +++ b/tools/devenv/Makefile.devenv @@ -1,10 +1,10 @@ .PHONY: devenv.build.worker devenv.build.worker: - $(MAKE) -C apps/worker build + $(MAKE) worker.build .PHONY: devenv.build.api devenv.build.api: - $(MAKE) -C apps/codecov-api build + $(MAKE) api.build .PHONY: devenv.build.shared devenv.build.shared: