From 6495a0c2e4b72b870561747c5d6408f1a7631e85 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Mon, 8 Sep 2025 15:09:20 +0200 Subject: [PATCH 1/5] Use uv instead of pip, bump core from 4.0.4 to 4.4.0rc3 --- .github/workflows/check-uv-lock.yaml | 19 +++++++++ docker-compose.yml | 3 +- docker/orchestrator/entrypoint.sh | 8 ++-- pyproject.toml | 2 +- requirements.txt | 4 -- uv.lock | 60 +++++++++++++++++++--------- 6 files changed, 69 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/check-uv-lock.yaml delete mode 100644 requirements.txt diff --git a/.github/workflows/check-uv-lock.yaml b/.github/workflows/check-uv-lock.yaml new file mode 100644 index 0000000..ce2a7ff --- /dev/null +++ b/.github/workflows/check-uv-lock.yaml @@ -0,0 +1,19 @@ +name: Check uv.lock + +on: + pull_request: + branches: [ master ] + paths: + - 'pyproject.toml' + - 'uv.lock' + +jobs: + check-uv-lock: + runs-on: ubuntu-latest + container: ghcr.io/astral-sh/uv:python3.13-bookworm-slim + steps: + - uses: actions/checkout@v4 + + - name: Check uv.lock + run: | + uv lock --check diff --git a/docker-compose.yml b/docker-compose.yml index dcdfec1..95bcc0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -179,7 +179,8 @@ services: - ./graphql_federation.py:/home/orchestrator/graphql_federation.py - ./utils:/home/orchestrator/utils - ./services:/home/orchestrator/services - - ./requirements.txt:/home/orchestrator/requirements.txt + - ./pyproject.toml:/home/orchestrator/pyproject.toml + - ./uv.lock:/home/orchestrator/uv.lock - ./alembic.ini:/home/orchestrator/alembic.ini - ./translations:/home/orchestrator/translations - ./templates:/home/orchestrator/templates diff --git a/docker/orchestrator/entrypoint.sh b/docker/orchestrator/entrypoint.sh index 9a547d2..1a12ded 100755 --- a/docker/orchestrator/entrypoint.sh +++ b/docker/orchestrator/entrypoint.sh @@ -4,11 +4,13 @@ PATH=$PATH:~/.local/bin # Install extra requirements for orchestrator -pip install -r requirements.txt +pip install uv +uv sync +source .venv/bin/activate if [ -f ${CORE_OVERRIDE}/pyproject.toml ]; then echo "⏭️ Use editable install of orchestrator-core with dev and test dependencies" - pip install -e $CORE_OVERRIDE[dev,test] + uv pip install -e $CORE_OVERRIDE[dev,test] # Run any missing migrations on the database python main.py db upgrade heads @@ -18,6 +20,6 @@ else # Run any missing migrations on the database python main.py db upgrade heads - echo "⏭️ Use pip installed orchestrator-core $(pip freeze | grep orchestrator-core)" + echo "⏭️ Use orchestrator-core as specified in pyproject.toml $(uv pip freeze | grep orchestrator-core)" uvicorn --host 0.0.0.0 --port 8080 $UVICORN_ARGS main:app --reload --proxy-headers fi diff --git a/pyproject.toml b/pyproject.toml index 03d21d5..b0e1a11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" requires-python = ">=3.12" dependencies = [ "deepdiff==8.0.1", - "orchestrator-core==4.0.4", + "orchestrator-core==4.4.0rc3", "pynetbox==7.4.1", "rich==13.9.4", ] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ed8eb28..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -orchestrator-core==4.0.4 -deepdiff==8.0.1 -rich==13.9.4 -pynetbox==7.4.1 diff --git a/uv.lock b/uv.lock index e041399..98f5515 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12" resolution-markers = [ "python_full_version >= '3.13'", @@ -43,6 +43,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/12/e5e0282d673bb9746bacfb6e2dba8719989d3660cdb2ea79aee9a9651afb/anyio-4.10.0-py3-none-any.whl", hash = "sha256:60e474ac86736bbfd6f210f7a61218939c318f43f9972497381f1c5e930ed3d1", size = 107213, upload-time = "2025-08-04T08:54:24.882Z" }, ] +[[package]] +name = "apscheduler" +version = "3.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/00/6d6814ddc19be2df62c8c898c4df6b5b1914f3bd024b780028caa392d186/apscheduler-3.11.0.tar.gz", hash = "sha256:4c622d250b0955a65d5d0eb91c33e6d43fd879834bf541e0a18661ae60460133", size = 107347, upload-time = "2024-11-24T19:39:26.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/ae/9a053dd9229c0fde6b1f1f33f609ccff1ee79ddda364c756a924c6d8563b/APScheduler-3.11.0-py3-none-any.whl", hash = "sha256:fc134ca32e50f5eadcc4938e3a4545ab19131435e851abb40b34d63d5141c6da", size = 64004, upload-time = "2024-11-24T19:39:24.442Z" }, +] + [[package]] name = "asyncstdlib" version = "3.13.1" @@ -273,7 +285,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "deepdiff", specifier = "==8.0.1" }, - { name = "orchestrator-core", specifier = "==4.0.4" }, + { name = "orchestrator-core", specifier = "==4.4.0rc3" }, { name = "pynetbox", specifier = "==7.4.1" }, { name = "rich", specifier = "==13.9.4" }, ] @@ -601,11 +613,12 @@ wheels = [ [[package]] name = "orchestrator-core" -version = "4.0.4" +version = "4.4.0rc3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "alembic" }, { name = "anyio" }, + { name = "apscheduler" }, { name = "click" }, { name = "deepmerge" }, { name = "deprecated" }, @@ -626,7 +639,6 @@ dependencies = [ { name = "python-rapidjson" }, { name = "pytz" }, { name = "redis" }, - { name = "schedule" }, { name = "semver" }, { name = "sentry-sdk", extra = ["fastapi"] }, { name = "sqlalchemy" }, @@ -637,9 +649,9 @@ dependencies = [ { name = "typer" }, { name = "uvicorn", extra = ["standard"] }, ] -sdist = { url = "https://files.pythonhosted.org/packages/49/5a/3cbf21fa70442d48262b707cac2256a60828cd533212a03636b63e227031/orchestrator_core-4.0.4.tar.gz", hash = "sha256:999984db8c75dfe47a129ee5c5974dde92d4889dcbcba61b9ab2e7c9886a5bec", size = 2673275, upload-time = "2025-06-03T07:01:04.709Z" } +sdist = { url = "https://files.pythonhosted.org/packages/65/18/47714f43d98757a3064a2595c22271f3234abf73b535b37106f94af6a522/orchestrator_core-4.4.0rc3.tar.gz", hash = "sha256:cc96fe8c82815a288618928683890dbad1dcd6fad166b29c1b164907fbfb0a45", size = 254097, upload-time = "2025-09-08T10:17:38.142Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/56/2a85f6bf842de3ae3a71f44242a45a3bbec253bf45781983505e0c4143b3/orchestrator_core-4.0.4-py3-none-any.whl", hash = "sha256:594f82698a5477663acf6dbd0a3b3352cc179dd2be160415aba65e8fcde9aa1d", size = 409251, upload-time = "2025-06-03T07:01:02.692Z" }, + { url = "https://files.pythonhosted.org/packages/5d/e0/944612bf29dc4c9515c7d81867033124e7f22961e763e1b767e2190204fd/orchestrator_core-4.4.0rc3-py3-none-any.whl", hash = "sha256:eb8cd0d3a8079012a22e8d48ca4a2009bec98b9f9d2ff4ca65de3bc3c04ec276", size = 426680, upload-time = "2025-09-08T10:17:36.224Z" }, ] [[package]] @@ -700,11 +712,11 @@ wheels = [ [[package]] name = "prometheus-client" -version = "0.22.0" +version = "0.22.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5b/5a/3fa1fa7e91a203759aaf316be394f70f2ef598d589b9785a8611b6094c00/prometheus_client-0.22.0.tar.gz", hash = "sha256:18da1d2241ac2d10c8d2110f13eedcd5c7c0c8af18c926e8731f04fc10cd575c", size = 74443, upload-time = "2025-05-16T20:50:18.333Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/cf/40dde0a2be27cc1eb41e333d1a674a74ce8b8b0457269cc640fd42b07cf7/prometheus_client-0.22.1.tar.gz", hash = "sha256:190f1331e783cf21eb60bca559354e0a4d4378facecf78f5428c39b675d20d28", size = 69746, upload-time = "2025-06-02T14:29:01.152Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/c7/cee159ba3d7192e84a4c166ec1752f44a5fa859ac0eeda2d73a1da65ab47/prometheus_client-0.22.0-py3-none-any.whl", hash = "sha256:c8951bbe64e62b96cd8e8f5d917279d1b9b91ab766793f33d4dce6c228558713", size = 62658, upload-time = "2025-05-16T20:50:16.978Z" }, + { url = "https://files.pythonhosted.org/packages/32/ae/ec06af4fe3ee72d16973474f122541746196aaa16cea6f66d18b963c6177/prometheus_client-0.22.1-py3-none-any.whl", hash = "sha256:cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094", size = 58694, upload-time = "2025-06-02T14:29:00.068Z" }, ] [[package]] @@ -988,15 +1000,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/71/39c7c0d87f8d4e6c020a393182060eaefeeae6c01dab6a84ec346f2567df/rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90", size = 242424, upload-time = "2024-11-01T16:43:55.817Z" }, ] -[[package]] -name = "schedule" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a8/b5/a291a4c0faa491fd5baefa6d89011ece581cff47b23c0a39b42a63383358/schedule-1.1.0.tar.gz", hash = "sha256:e6ca13585e62c810e13a08682e0a6a8ad245372e376ba2b8679294f377dfc8e4", size = 18290, upload-time = "2021-04-10T10:48:20.598Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/3b/040bd180eaef427dd160562ee66adc9f4f67088185c272edcdb899c609c7/schedule-1.1.0-py2.py3-none-any.whl", hash = "sha256:617adce8b4bf38c360b781297d59918fbebfb2878f1671d189f4f4af5d0567a4", size = 10589, upload-time = "2021-04-10T10:48:19.5Z" }, -] - [[package]] name = "semver" version = "3.0.4" @@ -1174,6 +1177,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, ] +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "tzlocal" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/2e/c14812d3d4d9cd1773c6be938f89e5735a1f11a9f184ac3639b93cef35d5/tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd", size = 30761, upload-time = "2025-03-05T21:17:41.549Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, +] + [[package]] name = "urllib3" version = "2.5.0" From 77b848bdccfb00bba07feeda6ed9224059f28e5c Mon Sep 17 00:00:00 2001 From: Mark90 Date: Mon, 8 Sep 2025 15:46:08 +0200 Subject: [PATCH 2/5] Make uvicorn reload when using core override --- docker/orchestrator/entrypoint.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docker/orchestrator/entrypoint.sh b/docker/orchestrator/entrypoint.sh index 1a12ded..83185d8 100755 --- a/docker/orchestrator/entrypoint.sh +++ b/docker/orchestrator/entrypoint.sh @@ -15,7 +15,13 @@ if [ -f ${CORE_OVERRIDE}/pyproject.toml ]; then # Run any missing migrations on the database python main.py db upgrade heads - uvicorn --host 0.0.0.0 --port 8080 $UVICORN_ARGS main:app --reload --proxy-headers --reload-dir $CORE_OVERRIDE + uvicorn --host 0.0.0.0 --port 8080 $UVICORN_ARGS main:app --reload --proxy-headers \ + --reload-dir $CORE_OVERRIDE \ + --reload-dir products \ + --reload-dir services \ + --reload-dir translations \ + --reload-dir utils \ + --reload-dir workflows else # Run any missing migrations on the database python main.py db upgrade heads From 96c1248147b1427247333e7d25b7256590fdf99f Mon Sep 17 00:00:00 2001 From: Mark90 Date: Mon, 8 Sep 2025 15:46:40 +0200 Subject: [PATCH 3/5] Fix error in modify_l2vpn --- workflows/l2vpn/modify_l2vpn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/l2vpn/modify_l2vpn.py b/workflows/l2vpn/modify_l2vpn.py index 691e739..cfc2bb3 100644 --- a/workflows/l2vpn/modify_l2vpn.py +++ b/workflows/l2vpn/modify_l2vpn.py @@ -62,7 +62,7 @@ def update_subscription_description(subscription: L2vpn) -> State: @step("Update L2VPN in NRM") -def update_l2vpn_in_nrm(subscription: L2vpnProvisioning) -> State: +def update_l2vpn_in_nrm(subscription: L2vpn) -> State: """Dummy step, replace with actual call to NRM.""" return {"subscription": subscription} From 2c3bfb5320df77d62d673607dda4bfefc195f979 Mon Sep 17 00:00:00 2001 From: Mark90 Date: Mon, 8 Sep 2025 15:47:04 +0200 Subject: [PATCH 4/5] Add github CI workflow test-migrations --- .github/workflows/test-migrations.yaml | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/test-migrations.yaml diff --git a/.github/workflows/test-migrations.yaml b/.github/workflows/test-migrations.yaml new file mode 100644 index 0000000..cd85ee1 --- /dev/null +++ b/.github/workflows/test-migrations.yaml @@ -0,0 +1,48 @@ +name: Test migrations + +on: + push: + branches: [master] + workflow_call: + pull_request: + +env: + UV_FROZEN: true # Do not update the lockfile + +jobs: + test_migrations: + name: Test migrations + runs-on: ubuntu-latest + container: ghcr.io/astral-sh/uv:python3.13-bookworm-slim + env: + DATABASE_URI: postgresql://nwa:nwa@postgres/orchestrator-core + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: nwa + POSTGRES_USER: nwa + POSTGRES_DB: orchestrator-core + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Install dependencies + run: uv sync + + - name: Test migrations + run: uv run main.py db upgrade heads + + - name: Verify number of heads + run: | + num_heads=$(uv run main.py db heads | grep -c '(head)') + if [ "$num_heads" -ne 2 ]; then + echo "Found $num_heads DB heads instead of 2, check for duplicate down_revisions" + exit 1 + fi From 449d697ab7c9ae5dabc2b396cb73f2adbfce901b Mon Sep 17 00:00:00 2001 From: Mark90 Date: Mon, 8 Sep 2025 16:16:38 +0200 Subject: [PATCH 5/5] Fix down_revision for bc54616fefcf --- .../schema/2025-08-26_bc54616fefcf_update_validate_targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migrations/versions/schema/2025-08-26_bc54616fefcf_update_validate_targets.py b/migrations/versions/schema/2025-08-26_bc54616fefcf_update_validate_targets.py index 42c18a9..4a816d2 100644 --- a/migrations/versions/schema/2025-08-26_bc54616fefcf_update_validate_targets.py +++ b/migrations/versions/schema/2025-08-26_bc54616fefcf_update_validate_targets.py @@ -1,7 +1,7 @@ """Update validate targets. Revision ID: bc54616fefcf -Revises: d946c20663d3 +Revises: 0e8d17ce0f06 Create Date: 2025-08-26 22:55:28.955536 """ @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "bc54616fefcf" -down_revision = "d946c20663d3" +down_revision = "0e8d17ce0f06" branch_labels = None depends_on = None