From 063a2393fe33cc8584eda5d7dcd951eac32bf87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 16 Nov 2023 13:52:03 +0000 Subject: [PATCH] many: run nested tests for UC22 Backported from https://github.com/snapcore/core-base/pull/153 --- .github/workflows/tests.yaml | 10 ++++- spread.yaml | 19 +++++++-- tests/lib/nested.sh | 71 ++++++++++++++++++------------- tests/lib/prepare-utils.sh | 19 +++++++-- tests/spread/main/basic/task.yaml | 2 +- 5 files changed, 83 insertions(+), 38 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 0de16d44..b39b32de 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,11 +23,16 @@ jobs: mkdir "${{ github.workspace }}" - uses: actions/checkout@v2 - - name: Build snap + - name: x86 build run: | spread -artifacts=./artifacts google-nested:tests/spread/build/ find ./artifacts -type f -name "*.artifact" -exec cp {} "${{ github.workspace }}" \; + - name: arm64 build + run: | + spread-arm -artifacts=./artifacts google-nested-arm:tests/spread/build/ + find ./artifacts -type f -name "*.artifact" -exec cp {} "${{ github.workspace }}" \; + - uses: actions/upload-artifact@v2 with: name: core-snap @@ -94,6 +99,9 @@ jobs: - name: Build image working-directory: '${{ github.workspace }}' run: | + PROJECT_PATH=$PWD/core-base + # For os.query and others + PATH=$PATH:$PROJECT_PATH/tests/lib/external/snapd-testing-tools/tools/ . "core-base/tests/lib/prepare-utils.sh" echo "************* INSTALLING DEPS *************" diff --git a/spread.yaml b/spread.yaml index 8091fac3..e1f981ed 100644 --- a/spread.yaml +++ b/spread.yaml @@ -3,7 +3,7 @@ project: core22 environment: SETUPDIR: /home/core22 PROJECT_PATH: $SETUPDIR - PATH: $PATH:$PROJECT_PATH/tests/bin + PATH: $PATH:$PROJECT_PATH/tests/bin:$PROJECT_PATH/tests/lib/external/snapd-testing-tools/tools/ TESTSLIB: $PROJECT_PATH/tests/lib SNAP_BRANCH: "edge" # stable/edge/beta UC_VERSION: 22 @@ -16,13 +16,26 @@ backends: type: google key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")' location: snapd-spread/us-east1-b - plan: n2-standard-2 + plan: n2-standard-4 halt-timeout: 2h systems: - ubuntu-22.04-64: + workers: 4 image: ubuntu-2204-64-virt-enabled storage: 20G + google-nested-arm: + type: google + key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")' + location: snapd-spread/us-central1-a + plan: t2a-standard-4 + halt-timeout: 2h + systems: + - ubuntu-22.04-arm-64: + workers: 4 + image: ubuntu-2204-arm-64-virt-enabled + storage: 25G + qemu-nested: type: qemu # TODO:UC22: uc22 needs 2G or grub will not loopback the kernel snap @@ -83,7 +96,7 @@ prepare: | unset REBOOT warn-timeout: 40m -kill-timeout: 50m +kill-timeout: 60m suites: tests/spread/build/: diff --git a/tests/lib/nested.sh b/tests/lib/nested.sh index fded5379..64c22b59 100644 --- a/tests/lib/nested.sh +++ b/tests/lib/nested.sh @@ -11,7 +11,7 @@ execute_remote(){ wait_for_ssh(){ local service_name="$1" - retry=800 + retry=1800 wait=1 while ! execute_remote true; do if ! systemctl is-active "$service_name"; then @@ -54,7 +54,7 @@ cleanup_nested_core_vm(){ # remove the swtpm # TODO: we could just remove/reset the swtpm instead of removing the snap # wholesale - snap remove swtpm-mvo + snap remove test-snapd-swtpm fi # delete the image file @@ -74,13 +74,10 @@ start_nested_core_vm_unit(){ cp "${SETUPDIR}/pc.img" "${IMAGE_FILE}" # use only 2G of RAM for qemu-nested - if [ "${SPREAD_BACKEND}" = "google-nested" ]; then - # Do not enable SMP on GCE as it will cause boot issues. There is most likely - # a bug in the combination of the kernel version used in GCE images, combined with - # a new qemu version (v6) and OVMF - # TODO try again to enable more cores in the future to see if it is fixed + if [ "${SPREAD_BACKEND}" = "google-nested" ] || + [ "${SPREAD_BACKEND}" = "google-nested-arm" ]; then PARAM_MEM="-m 4096" - PARAM_SMP="-smp 1" + PARAM_SMP="-smp 4" elif [ "${SPREAD_BACKEND}" = "lxd-nested" ]; then PARAM_MEM="-m 4096" PARAM_SMP="-smp 2" @@ -103,40 +100,55 @@ start_nested_core_vm_unit(){ PARAM_SERIAL="-serial file:${WORK_DIR}/serial.log" PARAM_TPM="" - ATTR_KVM="" - if [ "$ENABLE_KVM" = "true" ]; then - ATTR_KVM=",accel=kvm" - # CPU can be defined just when kvm is enabled - PARAM_CPU="-cpu host" - fi - # TODO: enable ms key booting for i.e. nightly edge jobs ? - OVMF_CODE="" - OVMF_VARS="" + VMF_CODE="" + VMF_VARS="" if [ "${ENABLE_SECURE_BOOT:-false}" = "true" ]; then - OVMF_CODE=".secboot" + VMF_CODE=".ms" fi if [ "${ENABLE_OVMF_SNAKEOIL:-false}" = "true" ]; then - OVMF_VARS=".snakeoil" + VMF_VARS=".snakeoil" fi mkdir -p "${WORK_DIR}/image/" - cp -f "/usr/share/OVMF/OVMF_VARS${OVMF_VARS}.fd" "${WORK_DIR}/image/OVMF_VARS${OVMF_VARS}.fd" - PARAM_BIOS="-drive file=/usr/share/OVMF/OVMF_CODE${OVMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/OVMF_VARS${OVMF_VARS}.fd,if=pflash,format=raw" - PARAM_MACHINE="-machine q35${ATTR_KVM} -global ICH9-LPC.disable_s3=1" + if os.query is-pc-amd64; then + ATTR_KVM="" + if [ "$ENABLE_KVM" = "true" ]; then + ATTR_KVM=",accel=kvm" + # CPU can be defined just when kvm is enabled + PARAM_CPU="-cpu host" + fi + QEMU_BIN=qemu-system-x86_64 + PARAM_MACHINE="-machine q35${ATTR_KVM} -global ICH9-LPC.disable_s3=1" + PARAM_BIOS="-drive file=/usr/share/OVMF/OVMF_CODE${VMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/OVMF_VARS${VMF_VARS}.fd,if=pflash,format=raw" + TPM_DEVICE=tpm-tis + cp -f "/usr/share/OVMF/OVMF_VARS${VMF_VARS}.fd" "${WORK_DIR}/image/OVMF_VARS${VMF_VARS}.fd" + elif os.query is-arm64; then + # Assume arm64 + # Unfortunately gce does not offer kvm enabled arm64 VMs + PARAM_CPU="-cpu cortex-a57" + QEMU_BIN=qemu-system-aarch64 + PARAM_MACHINE="-machine virt" + PARAM_BIOS="-drive file=/usr/share/AAVMF/AAVMF_CODE${VMF_CODE}.fd,if=pflash,format=raw,unit=0,readonly=on -drive file=${WORK_DIR}/image/AAVMF_VARS${VMF_VARS}.fd,if=pflash,format=raw" + TPM_DEVICE=tpm-tis-device + cp -f "/usr/share/AAVMF/AAVMF_VARS${VMF_VARS}.fd" "${WORK_DIR}/image/AAVMF_VARS${VMF_VARS}.fd" + else + printf "ERROR: unsupported archtecture\n" + exit 1 + fi - # Unfortunately the swtpm-mvo snap does not work correctly in lxd container. It's not possible + # Unfortunately the test-snapd-swtpm snap does not work correctly in lxd container. It's not possible # for the socket to come up due to being containerized. if [ "${ENABLE_TPM:-false}" = "true" ]; then - TPMSOCK_PATH="/var/snap/swtpm-mvo/current/swtpm-sock" + TPMSOCK_PATH="/var/snap/test-snapd-swtpm/current/swtpm-sock" if [ "${SPREAD_BACKEND}" = "lxd-nested" ]; then mkdir -p /tmp/qtpm swtpm socket --tpmstate dir=/tmp/qtpm --ctrl type=unixio,path=/tmp/qtpm/sock --tpm2 -d -t TPMSOCK_PATH="/tmp/qtpm/sock" - elif ! snap list swtpm-mvo > /dev/null; then - snap install swtpm-mvo --beta + elif ! snap list test-snapd-swtpm > /dev/null; then + snap install test-snapd-swtpm --beta retry=60 - while ! test -S /var/snap/swtpm-mvo/current/swtpm-sock; do + while ! test -S /var/snap/test-snapd-swtpm/current/swtpm-sock; do retry=$(( retry - 1 )) if [ $retry -le 0 ]; then echo "Timed out waiting for the swtpm socket. Aborting!" @@ -145,14 +157,15 @@ start_nested_core_vm_unit(){ sleep 1 done fi - PARAM_TPM="-chardev socket,id=chrtpm,path=${TPMSOCK_PATH} -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis,tpmdev=tpm0" + PARAM_TPM="-chardev socket,id=chrtpm,path=${TPMSOCK_PATH} -tpmdev emulator,id=tpm0,chardev=chrtpm -device $TPM_DEVICE,tpmdev=tpm0" fi PARAM_IMAGE="-drive file=${IMAGE_FILE},cache=none,format=raw,id=disk1,if=none -device virtio-blk-pci,drive=disk1,bootindex=1" SVC_NAME="nested-vm-$(systemd-escape "${SPREAD_JOB:-unknown}")" + # shellcheck disable=SC2086 if ! systemd-run --service-type=simple --unit="${SVC_NAME}" -- \ - qemu-system-x86_64 \ + "$QEMU_BIN" \ ${PARAM_SMP} \ ${PARAM_CPU} \ ${PARAM_MEM} \ diff --git a/tests/lib/prepare-utils.sh b/tests/lib/prepare-utils.sh index 84e42d9f..caefc1be 100644 --- a/tests/lib/prepare-utils.sh +++ b/tests/lib/prepare-utils.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -set -x +set -x SSH_PORT=${SSH_PORT:-8022} MON_PORT=${MON_PORT:-8888} @@ -114,9 +114,20 @@ start_snapd_core_vm() { nested_wait_for_snap_command } +get_arch() { + if os.query is-pc-amd64; then + printf amd64 + elif os.query is-arm64; then + printf arm64 + else + printf "ERROR: unsupported archtecture\n" + exit 1 + fi +} + get_core_snap_name() { printf -v date '%(%Y%m%d)T' -1 - echo "core22_${date}_amd64.snap" + echo "core22_${date}_$(get_arch).snap" } install_core22_deps() { @@ -141,7 +152,7 @@ download_core22_snaps() { local snap_branch="$1" # get the model - curl -o ubuntu-core-amd64-dangerous.model https://raw.githubusercontent.com/snapcore/models/master/ubuntu-core-22-amd64-dangerous.model + curl -o ubuntu-core-dangerous.model https://raw.githubusercontent.com/snapcore/models/master/ubuntu-core-22-$(get_arch)-dangerous.model # download neccessary images snap download pc-kernel --channel=22/${snap_branch} --basename=upstream-pc-kernel @@ -209,5 +220,5 @@ build_core22_image() { --snap upstream-snapd.snap \ --snap upstream-pc-kernel.snap \ --snap upstream-pc-gadget.snap \ - ubuntu-core-amd64-dangerous.model + ubuntu-core-dangerous.model } diff --git a/tests/spread/main/basic/task.yaml b/tests/spread/main/basic/task.yaml index b8660710..a8601288 100644 --- a/tests/spread/main/basic/task.yaml +++ b/tests/spread/main/basic/task.yaml @@ -4,7 +4,7 @@ prepare: | # for various utilities . "$TESTSLIB/nested.sh" - # Start the nested UC20 VM + # Start the nested UC VM start_nested_core_vm_unit # At this point we are able to SSH to the nested VM, so things probably worked