From 5bbe6dba95de0fe91398d74f189801c62b3ecc85 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 29 Jul 2025 11:53:51 +0200 Subject: [PATCH 01/12] Add s390x integration tests --- .github/workflows/integration_test.yaml | 11 ++++++++-- spread.yaml | 28 +++++++++++++++++++++---- tests/integration/markers.py | 3 +++ tests/integration/test_architecture.py | 23 +++++++++++++++++++- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index b7d035f37..9b668a63a 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -42,7 +42,9 @@ jobs: _, runner, task, variant = job.split(":") # Example: "test_charm.py" task = task.removeprefix("tests/spread/") - if runner.endswith("-arm"): + if "s390x" in runner: + architecture = "s390x" + elif runner.endswith("-arm"): architecture = "arm64" else: architecture = "amd64" @@ -86,8 +88,9 @@ jobs: runs-on: ${{ matrix.job.runner }} timeout-minutes: 217 # Sum of steps `timeout-minutes` + 5 steps: - - name: Free up disk space + - name: (GitHub hosted) Free up disk space timeout-minutes: 1 + if: ${{ !contains(matrix.job.runner, 'self-hosted') }} run: | printf '\nDisk usage before cleanup\n' df --human-readable @@ -95,6 +98,10 @@ jobs: rm -r /opt/hostedtoolcache/ printf '\nDisk usage after cleanup\n' df --human-readable + - name: (IS hosted) Disk usage + timeout-minutes: 1 + if: ${{ contains(matrix.job.runner, 'self-hosted') }} + run: df --human-readable - name: Checkout timeout-minutes: 3 uses: actions/checkout@v4 diff --git a/spread.yaml b/spread.yaml index 7cd6e1a52..72ba391d3 100644 --- a/spread.yaml +++ b/spread.yaml @@ -93,6 +93,18 @@ backends: username: runner variants: - -juju29 + - self-hosted-linux-s390x-noble-edge: + username: ubuntu + environment: + # Several python packages (e.g. cryptography, bcrypt) do not have s390x builds on PyPI, + # so they must be built from source. pkg-config & rust toolchain needed + # (These packages are being built when installing poetry or integration test Python + # dependencies) + CONCIERGE_EXTRA_DEBS: pipx,pkg-config,rustup + # s390x only available on edge + CONCIERGE_MICROK8S_CHANNEL: 1.32-strict/edge + variants: + - -juju29 suites: tests/spread/: @@ -109,10 +121,18 @@ prepare: | snap refresh --hold chown -R root:root "$SPREAD_PATH" cd "$SPREAD_PATH" - snap install --classic concierge + # Install via snap after https://github.com/canonical/concierge/pull/81 released + go install github.com/canonical/concierge@latest + + # Install charmcraft & pipx on lxd-vm backend and install pipx on IS-hosted runners + ~/go/bin/concierge prepare --trace - # Install charmcraft & pipx (on lxd-vm backend) - concierge prepare --trace + if [[ $SPREAD_SYSTEM == *"s390x"* ]] + then + rustup set profile minimal + # TODO add renovate comment for rust version + rustup default 1.88.0 + fi pipx install tox poetry prepare-each: | @@ -124,7 +144,7 @@ prepare-each: | poetry add --lock --group integration juju@^2 fi # `concierge prepare` needs to be run for each spread job in case Juju version changed - concierge prepare --trace + ~/go/bin/concierge prepare --trace # Unable to set constraint on all models because of Juju bug: # https://bugs.launchpad.net/juju/+bug/2065050 diff --git a/tests/integration/markers.py b/tests/integration/markers.py index 46d84271d..d38ea4a4c 100644 --- a/tests/integration/markers.py +++ b/tests/integration/markers.py @@ -15,3 +15,6 @@ arm64_only = pytest.mark.skipif( architecture.architecture != "arm64", reason="Requires arm64 architecture" ) +s390x_only = pytest.mark.skipif( + architecture.architecture != "s390x", reason="Requires s390x architecture" +) diff --git a/tests/integration/test_architecture.py b/tests/integration/test_architecture.py index b7044c8c5..fee7850a9 100644 --- a/tests/integration/test_architecture.py +++ b/tests/integration/test_architecture.py @@ -61,4 +61,25 @@ async def test_amd_charm_on_arm_host(ops_test: OpsTest) -> None: ) -# TODO: add s390x test +@markers.arm64_only +async def test_amd_charm_on_s390x_host(ops_test: OpsTest) -> None: + """Tries deploying an amd64 charm on s390x host.""" + charm = "./mysql-router-k8s_ubuntu@22.04-amd64.charm" + + resources = { + "mysql-router-image": METADATA["resources"]["mysql-router-image"]["upstream-source"] + } + + await ops_test.model.deploy( + charm, + application_name=MYSQL_ROUTER_APP_NAME, + num_units=1, + resources=resources, + base="ubuntu@22.04", + ) + + await ops_test.model.wait_for_idle( + apps=[MYSQL_ROUTER_APP_NAME], + status="error", + raise_on_error=False, + ) From e940b6e99801feb5140efbbfc2875aad66385951 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Tue, 29 Jul 2025 12:22:38 +0200 Subject: [PATCH 02/12] Docker hub mirror --- spread.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/spread.yaml b/spread.yaml index 72ba391d3..322d1a055 100644 --- a/spread.yaml +++ b/spread.yaml @@ -86,6 +86,7 @@ backends: # Manually pass specific environment variables environment: CI: '$(HOST: echo $CI)' + DOCKERHUB_MIRROR: '$(HOST: echo $DOCKERHUB_MIRROR)' systems: - ubuntu-24.04: username: runner @@ -123,6 +124,26 @@ prepare: | cd "$SPREAD_PATH" # Install via snap after https://github.com/canonical/concierge/pull/81 released go install github.com/canonical/concierge@latest + + if [[ -n "$DOCKERHUB_MIRROR" ]] + then + # Running on IS-hosted runner; configure microk8s to use Docker Hub mirror + # Run before concierge prepare because of https://github.com/canonical/concierge/issues/75 + + snap install microk8s --channel "$CONCIERGE_MICROK8S_CHANNEL" + + # Wait for microk8s to populate iptables + # https://chat.canonical.com/canonical/pl/jo5cg6wqjjrudqd5ybj6hhttee + microk8s status --wait-ready + + tee /var/snap/microk8s/current/args/certs.d/docker.io/hosts.toml << EOF + server = "$DOCKERHUB_MIRROR" + [host."${DOCKERHUB_MIRROR#'https://'}"] + capabilities = ["pull", "resolve"] + EOF + microk8s stop + microk8s start + fi # Install charmcraft & pipx on lxd-vm backend and install pipx on IS-hosted runners ~/go/bin/concierge prepare --trace From 6c97d9012065dd86e5582535f24b7893559b459d Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 11:57:58 +0200 Subject: [PATCH 03/12] Use non-strict microk8s on IS-hosted runners --- spread.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spread.yaml b/spread.yaml index 322d1a055..bc3017225 100644 --- a/spread.yaml +++ b/spread.yaml @@ -102,8 +102,11 @@ backends: # (These packages are being built when installing poetry or integration test Python # dependencies) CONCIERGE_EXTRA_DEBS: pipx,pkg-config,rustup - # s390x only available on edge - CONCIERGE_MICROK8S_CHANNEL: 1.32-strict/edge + # s390x only available on edge risk of microk8s snap + # IS-hosted GitHub runners do not work with the strictly confined microk8s snap: + # https://github.com/canonical/microk8s/issues/5082 + # https://chat.canonical.com/canonical/pl/i6yydsx5ifrepp56khq5fq5dke + CONCIERGE_MICROK8S_CHANNEL: latest/edge variants: - -juju29 @@ -130,7 +133,7 @@ prepare: | # Running on IS-hosted runner; configure microk8s to use Docker Hub mirror # Run before concierge prepare because of https://github.com/canonical/concierge/issues/75 - snap install microk8s --channel "$CONCIERGE_MICROK8S_CHANNEL" + snap install microk8s --channel "$CONCIERGE_MICROK8S_CHANNEL" --classic # Wait for microk8s to populate iptables # https://chat.canonical.com/canonical/pl/jo5cg6wqjjrudqd5ybj6hhttee From 004847872f8121129d57c46a225d8106ea6333e1 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 12:30:51 +0200 Subject: [PATCH 04/12] temp disable unit needs --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c6b88a70d..ffe1cbcee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,7 +59,6 @@ jobs: name: Integration test charm needs: - lint - - unit-test - build uses: ./.github/workflows/integration_test.yaml with: From d4d1ba7d3d02f004765aceedfad7d4fb25f20eb8 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 13:52:08 +0200 Subject: [PATCH 05/12] try bootstrap with add-k8s & without concierge to use classic microk8s snap --- concierge.yaml | 2 +- spread.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/concierge.yaml b/concierge.yaml index 361973cb8..95ee838d4 100644 --- a/concierge.yaml +++ b/concierge.yaml @@ -4,7 +4,7 @@ juju: providers: microk8s: enable: true - bootstrap: true + bootstrap: false addons: - dns - hostpath-storage diff --git a/spread.yaml b/spread.yaml index bc3017225..aee72049b 100644 --- a/spread.yaml +++ b/spread.yaml @@ -151,6 +151,9 @@ prepare: | # Install charmcraft & pipx on lxd-vm backend and install pipx on IS-hosted runners ~/go/bin/concierge prepare --trace + microk8s config | juju add-k8s my-k8s --client + juju bootstrap my-k8s concierge-microk8s + if [[ $SPREAD_SYSTEM == *"s390x"* ]] then rustup set profile minimal From 8982ba285ad475d5c5bf97f2a4df955b2fb37f2c Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 14:20:27 +0200 Subject: [PATCH 06/12] add model --- spread.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/spread.yaml b/spread.yaml index aee72049b..ac72e8599 100644 --- a/spread.yaml +++ b/spread.yaml @@ -153,6 +153,7 @@ prepare: | microk8s config | juju add-k8s my-k8s --client juju bootstrap my-k8s concierge-microk8s + juju add-model testing if [[ $SPREAD_SYSTEM == *"s390x"* ]] then From d76fdea23c557c3b8d100322d84c696a5bf54ecf Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 15:11:48 +0200 Subject: [PATCH 07/12] fix marker --- tests/integration/test_architecture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_architecture.py b/tests/integration/test_architecture.py index fee7850a9..b8dd06f70 100644 --- a/tests/integration/test_architecture.py +++ b/tests/integration/test_architecture.py @@ -61,7 +61,7 @@ async def test_amd_charm_on_arm_host(ops_test: OpsTest) -> None: ) -@markers.arm64_only +@markers.s390x_only async def test_amd_charm_on_s390x_host(ops_test: OpsTest) -> None: """Tries deploying an amd64 charm on s390x host.""" charm = "./mysql-router-k8s_ubuntu@22.04-amd64.charm" From 859843437675a6c462e706b543536a0c28f475dd Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 15:23:17 +0200 Subject: [PATCH 08/12] fix tasks --- tests/spread/test_exporter.py/task.yaml | 1 + tests/spread/test_exporter_with_tls.py/task.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/spread/test_exporter.py/task.yaml b/tests/spread/test_exporter.py/task.yaml index 800f0c4a2..e2f7ff6c2 100644 --- a/tests/spread/test_exporter.py/task.yaml +++ b/tests/spread/test_exporter.py/task.yaml @@ -7,3 +7,4 @@ artifacts: - allure-results systems: - -ubuntu-24.04-arm + - -self-hosted-linux-s390x-noble-edge diff --git a/tests/spread/test_exporter_with_tls.py/task.yaml b/tests/spread/test_exporter_with_tls.py/task.yaml index 4d7dce7fc..70d2c33b9 100644 --- a/tests/spread/test_exporter_with_tls.py/task.yaml +++ b/tests/spread/test_exporter_with_tls.py/task.yaml @@ -7,3 +7,4 @@ artifacts: - allure-results systems: - -ubuntu-24.04-arm + - -self-hosted-linux-s390x-noble-edge From c65200a489d605efc6cd9b4c2af9744bd06ffef0 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Thu, 31 Jul 2025 15:24:07 +0200 Subject: [PATCH 09/12] temp remove already completed tests --- tests/spread/test_charm.py/task.yaml | 7 ------- tests/spread/test_log_rotation.py/task.yaml | 7 ------- tests/spread/test_tls.py/task.yaml | 7 ------- tests/spread/test_upgrade.py/task.yaml | 7 ------- 4 files changed, 28 deletions(-) delete mode 100644 tests/spread/test_charm.py/task.yaml delete mode 100644 tests/spread/test_log_rotation.py/task.yaml delete mode 100644 tests/spread/test_tls.py/task.yaml delete mode 100644 tests/spread/test_upgrade.py/task.yaml diff --git a/tests/spread/test_charm.py/task.yaml b/tests/spread/test_charm.py/task.yaml deleted file mode 100644 index 96450bdc3..000000000 --- a/tests/spread/test_charm.py/task.yaml +++ /dev/null @@ -1,7 +0,0 @@ -summary: test_charm.py -environment: - TEST_MODULE: test_charm.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results diff --git a/tests/spread/test_log_rotation.py/task.yaml b/tests/spread/test_log_rotation.py/task.yaml deleted file mode 100644 index 8e8900110..000000000 --- a/tests/spread/test_log_rotation.py/task.yaml +++ /dev/null @@ -1,7 +0,0 @@ -summary: test_log_rotation.py -environment: - TEST_MODULE: test_log_rotation.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results diff --git a/tests/spread/test_tls.py/task.yaml b/tests/spread/test_tls.py/task.yaml deleted file mode 100644 index a60574491..000000000 --- a/tests/spread/test_tls.py/task.yaml +++ /dev/null @@ -1,7 +0,0 @@ -summary: test_tls.py -environment: - TEST_MODULE: test_tls.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results diff --git a/tests/spread/test_upgrade.py/task.yaml b/tests/spread/test_upgrade.py/task.yaml deleted file mode 100644 index a53e8a4ee..000000000 --- a/tests/spread/test_upgrade.py/task.yaml +++ /dev/null @@ -1,7 +0,0 @@ -summary: test_upgrade.py -environment: - TEST_MODULE: test_upgrade.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results From 14991c105c8e71cfe2c7f03d8aa73b282d2bdb98 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 1 Aug 2025 11:47:46 +0200 Subject: [PATCH 10/12] self-signed-certificates channel s390x --- tests/integration/test_exporter_with_tls.py | 4 ++-- tests/integration/test_expose_external.py | 4 ++-- tests/integration/test_tls.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_exporter_with_tls.py b/tests/integration/test_exporter_with_tls.py index b872bb8c1..2667e8963 100644 --- a/tests/integration/test_exporter_with_tls.py +++ b/tests/integration/test_exporter_with_tls.py @@ -34,8 +34,8 @@ if juju_.is_3_or_higher: tls_app_name = "self-signed-certificates" - if architecture.architecture == "arm64": - tls_channel = "latest/edge" + if architecture.architecture == "s390x": + tls_channel = "1/edge" else: tls_channel = "latest/stable" tls_config = {"ca-common-name": "Test CA"} diff --git a/tests/integration/test_expose_external.py b/tests/integration/test_expose_external.py index 92f1348cc..39945c9c7 100644 --- a/tests/integration/test_expose_external.py +++ b/tests/integration/test_expose_external.py @@ -36,8 +36,8 @@ TLS_SETUP_SLEEP_TIME = 30 if juju_.is_3_or_higher: TLS_APP_NAME = "self-signed-certificates" - if architecture.architecture == "arm64": - TLS_CHANNEL = "latest/edge" + if architecture.architecture == "s390x": + TLS_CHANNEL = "1/edge" else: TLS_CHANNEL = "latest/stable" TLS_CONFIG = {"ca-common-name": "Test CA"} diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 00fb17f95..6e687fb26 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -30,8 +30,8 @@ if juju_.is_3_or_higher: tls_app_name = "self-signed-certificates" - if architecture.architecture == "arm64": - tls_channel = "latest/edge" + if architecture.architecture == "s390x": + tls_channel = "1/edge" else: tls_channel = "latest/stable" tls_config = {"ca-common-name": "Test CA"} From a06735d7b96d338bfc3060072ed15930b5eec9b7 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 1 Aug 2025 11:48:37 +0200 Subject: [PATCH 11/12] temp remove already completed tests --- tests/spread/test_architecture.py/task.yaml | 9 --------- tests/spread/test_exporter_with_tls.py/task.yaml | 10 ---------- .../spread/{test_exporter.py => test_tls.py}/task.yaml | 7 ++----- 3 files changed, 2 insertions(+), 24 deletions(-) delete mode 100644 tests/spread/test_architecture.py/task.yaml delete mode 100644 tests/spread/test_exporter_with_tls.py/task.yaml rename tests/spread/{test_exporter.py => test_tls.py}/task.yaml (57%) diff --git a/tests/spread/test_architecture.py/task.yaml b/tests/spread/test_architecture.py/task.yaml deleted file mode 100644 index cad863364..000000000 --- a/tests/spread/test_architecture.py/task.yaml +++ /dev/null @@ -1,9 +0,0 @@ -summary: test_architecture.py -environment: - TEST_MODULE: test_architecture.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results -backends: - - -lxd-vm # This task requires charm built on different architecture from host diff --git a/tests/spread/test_exporter_with_tls.py/task.yaml b/tests/spread/test_exporter_with_tls.py/task.yaml deleted file mode 100644 index 70d2c33b9..000000000 --- a/tests/spread/test_exporter_with_tls.py/task.yaml +++ /dev/null @@ -1,10 +0,0 @@ -summary: test_exporter_with_tls.py -environment: - TEST_MODULE: test_exporter_with_tls.py -execute: | - tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" -artifacts: - - allure-results -systems: - - -ubuntu-24.04-arm - - -self-hosted-linux-s390x-noble-edge diff --git a/tests/spread/test_exporter.py/task.yaml b/tests/spread/test_tls.py/task.yaml similarity index 57% rename from tests/spread/test_exporter.py/task.yaml rename to tests/spread/test_tls.py/task.yaml index e2f7ff6c2..a60574491 100644 --- a/tests/spread/test_exporter.py/task.yaml +++ b/tests/spread/test_tls.py/task.yaml @@ -1,10 +1,7 @@ -summary: test_exporter.py +summary: test_tls.py environment: - TEST_MODULE: test_exporter.py + TEST_MODULE: test_tls.py execute: | tox run -e integration -- "tests/integration/$TEST_MODULE" --model testing --alluredir="$SPREAD_TASK/allure-results" artifacts: - allure-results -systems: - - -ubuntu-24.04-arm - - -self-hosted-linux-s390x-noble-edge From d75243071e9a98e5de4fe105b33404a7efff51d9 Mon Sep 17 00:00:00 2001 From: Carl Csaposs Date: Fri, 1 Aug 2025 12:24:45 +0200 Subject: [PATCH 12/12] fix base --- tests/integration/test_exporter_with_tls.py | 1 - tests/integration/test_expose_external.py | 1 - tests/integration/test_tls.py | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/integration/test_exporter_with_tls.py b/tests/integration/test_exporter_with_tls.py index 2667e8963..bbc7601e1 100644 --- a/tests/integration/test_exporter_with_tls.py +++ b/tests/integration/test_exporter_with_tls.py @@ -138,7 +138,6 @@ async def test_exporter_endpoint(ops_test: OpsTest, charm) -> None: application_name=tls_app_name, channel=tls_channel, config=tls_config, - base="ubuntu@22.04", ) await ops_test.model.wait_for_idle([tls_app_name], status="active", timeout=SLOW_TIMEOUT) diff --git a/tests/integration/test_expose_external.py b/tests/integration/test_expose_external.py index 39945c9c7..32b2f93c4 100644 --- a/tests/integration/test_expose_external.py +++ b/tests/integration/test_expose_external.py @@ -197,7 +197,6 @@ async def test_expose_external_with_tls(ops_test: OpsTest) -> None: TLS_APP_NAME, channel=TLS_CHANNEL, config=TLS_CONFIG, - base="ubuntu@22.04", ) async with ops_test.fast_forward("60s"): await ops_test.model.wait_for_idle( diff --git a/tests/integration/test_tls.py b/tests/integration/test_tls.py index 6e687fb26..e21a434ba 100644 --- a/tests/integration/test_tls.py +++ b/tests/integration/test_tls.py @@ -79,7 +79,6 @@ async def test_deploy_and_relate(ops_test: OpsTest, charm) -> None: application_name=tls_app_name, channel=tls_channel, config=tls_config, - base="ubuntu@22.04", ), ops_test.model.deploy( TEST_APP_NAME,