From 5a1ec146bb5eb774dda371270a9c3419518709ce Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@edpnet.be>
Date: Wed, 13 Sep 2023 09:05:39 +0200
Subject: [PATCH 01/10] Increase cross-compile compatibility

Some user distros do not use standard triplets, but instead, the triplet has another name.
Move the triplet name into a gitignored config file,
and fix a few trivial error messages breaking a cross-compile build:
- use headers-generic instead of headers-$arch
- if ulimit isn't set, don't do it.

Signed-off-by: jpsollie <janpieter.sollie@edpnet.be>
---
 .gitignore      |  1 +
 cross.conf      |  8 ++++++++
 lib/common.sh   | 10 +++++++---
 lib/libktest.sh |  2 +-
 root_image      |  2 +-
 5 files changed, 18 insertions(+), 5 deletions(-)
 create mode 100644 cross.conf

diff --git a/.gitignore b/.gitignore
index d82a2a50..0e86b513 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ cscope*
 tags
 ktest-out
 *.tags
+cross.conf
 
 lib/lwip-connect
 lib/supervisor
diff --git a/cross.conf b/cross.conf
new file mode 100644
index 00000000..28968856
--- /dev/null
+++ b/cross.conf
@@ -0,0 +1,8 @@
+# this file specifies target triplets for cross-compiling
+# whenever these need to be changed (some distributions prefer ARCHITECTURE-VENDOR-OS-LIBC), 
+# change the triplet here
+
+ARCH_TRIPLE_X86=x86-linux-gnu
+ARCH_TRIPLE_X86_64=x86_64-linux-gnu
+ARCH_TRIPLE_ARM64=aarch64-linux-gnu
+
diff --git a/lib/common.sh b/lib/common.sh
index 77860915..078f512b 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -3,6 +3,10 @@ set -o nounset
 set -o errtrace
 set -o pipefail
 
+[[ -v ktest_dir ]] || ktest_dir=$(dirname ${BASH_SOURCE})/..
+
+. "$ktest_dir/cross.conf"
+
 trap 'echo "Error $? at $BASH_SOURCE $LINENO from: $BASH_COMMAND, exiting"' ERR
 
 ktest_tmp=${ktest_tmp:-""}
@@ -85,7 +89,7 @@ parse_arch()
 	x86|i386)
 	    ktest_arch=x86
 	    DEBIAN_ARCH=i386
-	    ARCH_TRIPLE=x86-linux-gnu
+	    ARCH_TRIPLE=${ARCH_TRIPLE_X86}
 
 	    KERNEL_ARCH=x86
 	    BITS=32
@@ -96,7 +100,7 @@ parse_arch()
 	x86_64|amd64)
 	    ktest_arch=x86_64
 	    DEBIAN_ARCH=amd64
-	    ARCH_TRIPLE=x86_64-linux-gnu
+	    ARCH_TRIPLE=${ARCH_TRIPLE_X86_64}
 
 	    KERNEL_ARCH=x86
 	    BITS=64
@@ -107,7 +111,7 @@ parse_arch()
 	aarch64|arm64)
 	    ktest_arch=aarch64
 	    DEBIAN_ARCH=arm64
-	    ARCH_TRIPLE=aarch64-linux-gnu
+	    ARCH_TRIPLE=${ARCH_TRIPLE_ARM64}
 
 	    KERNEL_ARCH=arm64
 	    BITS=64
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 95d4b77f..9b4931d7 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -438,7 +438,7 @@ start_vm()
 	qemu_pmem mem-path="$file",size=$size
     done
 
-    ulimit -n 65535
+    [ "$(ulimit)" == "unlimited" ] || ulimit -n 65535
     qemu_cmd+=("${ktest_qemu_append[@]}")
 
     set +o errexit
diff --git a/root_image b/root_image
index ece986a6..26c99fee 100755
--- a/root_image
+++ b/root_image
@@ -118,7 +118,7 @@ PACKAGES+=(cryptsetup)
 PACKAGES+=(multipath-tools sg3-utils srptools)
 
 # ZFS support
-PACKAGES+=("linux-headers-$DEBIAN_ARCH" dkms zfsutils-linux zfs-dkms)
+PACKAGES+=("linux-headers-generic" dkms zfsutils-linux zfs-dkms)
 
 # suspend testing:
 # [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)

From b513eff3d27127f4877489075778a72f424d1b26 Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Tue, 14 Nov 2023 19:27:36 +0100
Subject: [PATCH 02/10] Run bcachefs tests on emulated architectures

Allow the tests to run on a non-ABI-compatible cpu,
verify code when the architecture isn't physically available.
in the future, bcachefs-tools should be cross-compiled,
and the list of CPU types should be expanded

Signed-off-by: jpsollie <janpieter.sollie@edpnet.be>
---
 build-test-kernel                    |  8 ++------
 lib/common.sh                        | 12 +++++++++---
 lib/libktest.sh                      |  8 ++++++--
 lib/testrunner                       |  4 ++++
 root_image                           |  8 +++++++-
 tests/bcachefs/bcachefs-test-libs.sh |  8 ++++++++
 tests/kconfig.sh                     |  2 ++
 7 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/build-test-kernel b/build-test-kernel
index 72990bae..8cd40865 100755
--- a/build-test-kernel
+++ b/build-test-kernel
@@ -107,7 +107,7 @@ ktest_kernel_source=$(readlink -e "$ktest_kernel_source")
 ktest_kernel_build="$ktest_out/kernel_build.$ktest_arch"
 mkdir -p "$ktest_kernel_build"
 
-if [[ -n $CROSS_COMPILE ]]; then
+if [[ -n ${CROSS_COMPILE} ]]; then
     checkdep "$ARCH_TRIPLE-gcc" "gcc-$ARCH_TRIPLE"
 fi
 
@@ -121,10 +121,6 @@ run_ktest()
 
 do_make()
 {
-    if [[ -n $CROSS_COMPILE ]]; then
-	export ARCH="$KERNEL_ARCH"
-	export CROSS_COMPILE="$ARCH_TRIPLE-"
-    fi
 
     make --jobs="$ktest_njobs"			\
 	--directory="$ktest_kernel_source"    	\
@@ -192,7 +188,7 @@ build_kernel()
 
     log_verbose "kernel_config_require: ${ktest_kernel_config_require[@]}"
 
-    MAKEARGS+=("LOCALVERSION=-ktest")
+    MAKEARGS+=("LOCALVERSION=-ktest" ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-")
 
     for opt in "${ktest_kernel_config_require[@]}"; do
 	[[ -n $opt ]] && kernel_opt set "$opt"
diff --git a/lib/common.sh b/lib/common.sh
index 078f512b..a6068c6a 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -80,11 +80,9 @@ join_by()
     echo "$*"
 }
 
-ktest_arch=$(uname -m)
-CROSS_COMPILE=""
-
 parse_arch()
 {
+    CROSS_COMPILE=
     case $1 in
 	x86|i386)
 	    ktest_arch=x86
@@ -191,6 +189,14 @@ parse_arch()
     if [[ $ktest_arch != $(uname -m) ]]; then
 	CROSS_COMPILE=1
     fi
+
+    export DEBIAN_ARCH
+    export MIRROR
+    export ARCH_TRIPLE
+    export KERNEL_ARCH
+    export QEMU_PACKAGE
+    export QEMU_BIN
+    export ktest_arch
 }
 
 find_command() {
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 9b4931d7..8ead78bf 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -85,6 +85,7 @@ parse_ktest_arg()
 
 parse_args_post()
 {
+    [ -z ${ktest_arch:+x} ] && ktest_arch=$(uname -m)
     parse_arch "$ktest_arch"
 
     ktest_out=$(readlink -f "$ktest_out")
@@ -306,12 +307,15 @@ start_vm()
     kernelargs+=("${ktest_kernel_append[@]}")
 
     local qemu_cmd=("$QEMU_BIN" -nodefaults -nographic)
+    local accel=kvm
+    local cputype=host
+    [[ $(uname -m) == $ktest_arch ]] || accel=tcg && cputype=max
     case $ktest_arch in
 	x86|x86_64)
-	    qemu_cmd+=(-cpu host -machine type=q35,accel=kvm,nvdimm=on)
+	    qemu_cmd+=(-cpu $cputype -machine type=q35,accel=$accel,nvdimm=on)
 	    ;;
 	aarch64)
-	    qemu_cmd+=(-cpu host -machine type=virt,gic-version=max,accel=kvm)
+	    qemu_cmd+=(-cpu $cputype -machine type=virt,gic-version=max,accel=$accel)
 	    ;;
 	mips)
 	    qemu_cmd+=(-cpu 24Kf -machine malta)
diff --git a/lib/testrunner b/lib/testrunner
index acbc45f7..4eb73f2c 100755
--- a/lib/testrunner
+++ b/lib/testrunner
@@ -20,6 +20,10 @@ ktest_out="/host/$ktest_out"
 ln -sf $ktest_dir /ktest
 ln -sf $ktest_out /ktest-out
 
+. /ktest/lib/common.sh
+
+parse_arch $(uname -m)
+
 # Some home directories are in weird places:
 mkdir -p $(dirname $home)
 ln -sf /host/$home $home
diff --git a/root_image b/root_image
index 26c99fee..a8817142 100755
--- a/root_image
+++ b/root_image
@@ -118,7 +118,12 @@ PACKAGES+=(cryptsetup)
 PACKAGES+=(multipath-tools sg3-utils srptools)
 
 # ZFS support
-PACKAGES+=("linux-headers-generic" dkms zfsutils-linux zfs-dkms)
+PACKAGES+=("linux-headers-generic")
+# unless no other option when cross-compiling, ignore ZFS
+# DKMS needs to cross-compile the module,
+# against a different kernel on a different CPUarchitecture.
+# this has to cause errors
+[[ -n $CROSS_COMPILE ] && PACKAGES+=(dkms zfsutils-linux zfs-dkms)
 
 # suspend testing:
 # [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)
@@ -302,6 +307,7 @@ cmd_create()
 	--components='main,contrib,non-free,bullseye-backports' \
 	bullseye "$MNT" "$MIRROR"
 
+    [[ -n ${CROSS_COMPILE} ]] || cp $(which qemu-${ktest_arch}) ${MNT}$(which qemu-${ktest_arch})
     _chroot "$MNT" /debootstrap/debootstrap --second-stage
     _chroot "$MNT" dpkg --configure -a
 
diff --git a/tests/bcachefs/bcachefs-test-libs.sh b/tests/bcachefs/bcachefs-test-libs.sh
index 71678563..05996b11 100644
--- a/tests/bcachefs/bcachefs-test-libs.sh
+++ b/tests/bcachefs/bcachefs-test-libs.sh
@@ -5,8 +5,16 @@
 #
 
 . $(dirname $(readlink -e "${BASH_SOURCE[0]}"))/../test-libs.sh
+bch_loc=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-tools
 
 require-git http://evilpiepirate.org/git/bcachefs-tools.git
+if [[ ! -f "${bch_loc}/.last_arch_for_compile" || "$(cat ${bch_loc}/.last_arch_for_compile)" != $ktest_arch ]]; then
+	make -C ${bch_loc} clean >/dev/null 2>&1;
+	rm -rf "${bch_loc}/rust-src/target/*";
+	find ${bch_loc} -name "*.o" -exec rm {} \;
+	find ${bch_loc} -name "*.a" -exec rm {} \;
+	echo $ktest_arch > ${bch_loc}/.last_arch_for_compile
+fi
 require-make bcachefs-tools
 
 require-kernel-config BCACHEFS_FS
diff --git a/tests/kconfig.sh b/tests/kconfig.sh
index 68005fc9..73e93db1 100644
--- a/tests/kconfig.sh
+++ b/tests/kconfig.sh
@@ -4,6 +4,8 @@ have_kvmguest=0
 have_virtio=0
 have_suspend=0
 
+[ -z ${ktest_arch:+x} ] && ktest_arch=$(uname -m)
+
 case $ktest_arch in
     x86)
 	require-kernel-config SMP

From 3704efe85b60e9981f54fd9bdc69bebe29eca31d Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Wed, 15 Nov 2023 10:12:57 +0100
Subject: [PATCH 03/10] ktest: upgrade to debian trixie channel

Due to bcachefs being mainlined,
it's worth considering a more recent update channel,
which may include bcachefs features in the future.

Necessary adjustments have been made for the packages,
but kernel requirements may still need some extra tweaks

Signed-off-by: jpsollie <janpieter.sollie@edpnet.be>

ktest: enable debian sid fallback

RISC-V is currently moving from debian-ports to debian,
in order to protect from unreleased architectures etc,
a fallback scenario is needed:
in case preconfigured settings do not work,
apply a default sid fallback

Signed-Off-By: jpsollie <janpieter.sollie@edpnet.be>
---
 lib/common.sh   |  8 +++++---
 lib/libktest.sh |  9 ++++++++-
 root_image      | 35 ++++++++++++++++++++++++++++++-----
 3 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/lib/common.sh b/lib/common.sh
index a6068c6a..e977af48 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -84,7 +84,7 @@ parse_arch()
 {
     CROSS_COMPILE=
     case $1 in
-	x86|i386)
+	x86|i386|i686)
 	    ktest_arch=x86
 	    DEBIAN_ARCH=i386
 	    ARCH_TRIPLE=${ARCH_TRIPLE_X86}
@@ -93,7 +93,7 @@ parse_arch()
 	    BITS=32
 
 	    QEMU_PACKAGE=qemu-system-x86
-	    QEMU_BIN=qemu-system-x86_64
+	    QEMU_BIN=qemu-system-i386
 	    ;;
 	x86_64|amd64)
 	    ktest_arch=x86_64
@@ -189,7 +189,8 @@ parse_arch()
     if [[ $ktest_arch != $(uname -m) ]]; then
 	CROSS_COMPILE=1
     fi
-
+    #special case: x86_64 is able to run i386 code.  this isn't always the case for armv8 -> armv7 (cortex A35)
+    [[ $DEBIAN_ARCH == "i386" && "$(uname -m)" == "x86_64" ]] && CROSS_COMPILE=
     export DEBIAN_ARCH
     export MIRROR
     export ARCH_TRIPLE
@@ -197,6 +198,7 @@ parse_arch()
     export QEMU_PACKAGE
     export QEMU_BIN
     export ktest_arch
+    export BITS
 }
 
 find_command() {
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 8ead78bf..75679b82 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -309,7 +309,7 @@ start_vm()
     local qemu_cmd=("$QEMU_BIN" -nodefaults -nographic)
     local accel=kvm
     local cputype=host
-    [[ $(uname -m) == $ktest_arch ]] || accel=tcg && cputype=max
+    [[ -n ${CROSS_COMPILE} ]] && accel=tcg && cputype=max
     case $ktest_arch in
 	x86|x86_64)
 	    qemu_cmd+=(-cpu $cputype -machine type=q35,accel=$accel,nvdimm=on)
@@ -326,7 +326,14 @@ start_vm()
 	    ;;
     esac
 
+    #we assign 4GB of ram, but for gcc jobs on target, >512MiB / GCC job is recommended
+    (( $ktest_cpus > 8 )) && ktest_mem=8G
+
     local maxmem=$(awk '/MemTotal/ {printf "%dG\n", $2/1024/1024}' /proc/meminfo 2>/dev/null) || maxmem="1T"
+    local memconfig="$ktest_mem,slots=8,maxmem=$maxmem"
+
+    #do not be fancy on 32-bit hardware.  if it works, it's fine
+    [[ $BITS == 32 ]] &&  memconfig="3G" && ktest_cpus=$((min($ktest_cpus,4)))
 
     qemu_cmd+=(								\
 	-m		"$ktest_mem,slots=8,maxmem=$maxmem"		\
diff --git a/root_image b/root_image
index a8817142..c91faf65 100755
--- a/root_image
+++ b/root_image
@@ -45,6 +45,7 @@ if [[ $# = 0 ]]; then
 fi
 
 ktest_image=""
+ktest_arch="$(uname -m)"
 CMD="cmd_$1"
 shift
 
@@ -100,7 +101,7 @@ PACKAGES+=(libudev-dev libldap2-dev)
 PACKAGES+=(acct bsdextrautils xfsprogs xfslibs-dev quota libcap2-bin)
 PACKAGES+=(libattr1-dev libaio-dev libgdbm-dev libacl1-dev gettext)
 PACKAGES+=(libssl-dev libgdbm-dev libgdbm-compat-dev liburing-dev)
-PACKAGES+=(duperemove thin-provisioning-tools fsverity)
+PACKAGES+=(duperemove fsverity)
 
 # xfsprogs:
 PACKAGES+=(libinih-dev)
@@ -123,7 +124,7 @@ PACKAGES+=("linux-headers-generic")
 # DKMS needs to cross-compile the module,
 # against a different kernel on a different CPUarchitecture.
 # this has to cause errors
-[[ -n $CROSS_COMPILE ] && PACKAGES+=(dkms zfsutils-linux zfs-dkms)
+[[ -n ${CROSS_COMPILE} ]] || PACKAGES+=(dkms zfsutils-linux zfs-dkms)
 
 # suspend testing:
 # [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)
@@ -241,6 +242,7 @@ update_packages()
     chmod 755 "$MNT"/tmp/rustup.sh
 
     _chroot "$MNT" /tmp/rustup.sh -y
+    echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
 }
 
 trim_image()
@@ -299,15 +301,38 @@ cmd_create()
     mkfs.ext4 -F "$ktest_image"
     mount "$ktest_image" "$MNT"
 
+   local debian_release="trixie" #general release
+   local keyring=""
+
+    if [[ $MIRROR == *"debian-ports"* ]]; then
+	debian_release="sid" #unofficial ports don't have named releases
+	echo ""
+	echo "WARNING: $ktest_arch is unsupported, using SID release for packages"
+	echo "*******************************************************************"
+	echo "PLEASE NOTE: this often has dependency problems between packages, "
+	echo "and can prevent the install due to a dependency conflict "
+	echo "If so, contact the debian maintainer for this architecture to fix it:"
+	echo "${ktest_arch} : ${DEBIAN_ARCH}@buildd.debian.org"
+	echo "*******************************************************************"
+	echo ""
+    fi
+
+    #fallback: if the architecture can't be found, try official sid:
+    if [[ -z $(curl -qq -I "${MIRROR}/dists/${debian_release}/main/binary-${DEBIAN_ARCH}/Release" 2>/dev/null | grep "HTTP/2 200") ]]; then
+	echo "WARNING: $DEBIAN_ARCH $debian_release could not be found at ${MIRROR} (where it should be).  Falling back to standard sid"
+        MIRROR="https://deb.debian.org/debian/"
+	debian_release=sid
+    fi
+
     DEBOOTSTRAP_DIR=$ktest_dir/debootstrap $debootstrap	\
 	--no-check-gpg					\
 	--arch="$DEBIAN_ARCH"				\
 	--exclude=$(join_by , "${EXCLUDE[@]}")		\
 	--foreign					\
-	--components='main,contrib,non-free,bullseye-backports' \
-	bullseye "$MNT" "$MIRROR"
+	--components='main,contrib,non-free' \
+	trixie "$MNT" "$MIRROR"
 
-    [[ -n ${CROSS_COMPILE} ]] || cp $(which qemu-${ktest_arch}) ${MNT}$(which qemu-${ktest_arch})
+    [[ -n ${CROSS_COMPILE} ]] && cp $(which qemu-${ktest_arch}) ${MNT}$(which qemu-${ktest_arch})
     _chroot "$MNT" /debootstrap/debootstrap --second-stage
     _chroot "$MNT" dpkg --configure -a
 

From d6aa59bf04f318cf44c25923b60df751c14239a5 Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@edpnet.be>
Date: Fri, 22 Sep 2023 21:17:33 +0200
Subject: [PATCH 04/10] enable ARMv7 support

Ktest has now been expanded with cross-binary armv7 (hard float).
this should allow some testing on more legacy armv7 hardware,
which may still be present in NAS devices.
However, due to the nature of the ecosystem, a limit is in place:
- 3GB ram for 32 bit systems
- 4 CPUs for 32 bit systems
I think if you need more, you'd better think of going 64-bit.
These constraints allow the ARM emulator to run smoothly.

Signed-off-by:  jpsollie <janpieter.sollie@edpnet.be>
---
 build-test-kernel | 19 +++++++++++--------
 cross.conf        |  2 +-
 lib/common.sh     | 11 +++++++++++
 lib/libktest.sh   |  8 +++++---
 tests/kconfig.sh  | 25 ++++++++++++++++++++++---
 5 files changed, 50 insertions(+), 15 deletions(-)

diff --git a/build-test-kernel b/build-test-kernel
index 8cd40865..9a35aad3 100755
--- a/build-test-kernel
+++ b/build-test-kernel
@@ -137,7 +137,7 @@ new_config()
     local config_tool="$ktest_kernel_source/scripts/config"
 
     if [[ ! -f $kconfig ]]; then
-	do_make allnoconfig
+	do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" allnoconfig
 
 	# Really undefine everything:
 	sed -i -e 's/\(CONFIG_.*\)=.*/# \1 is not set/' "$kconfig"
@@ -184,17 +184,17 @@ build_kernel()
 	mv "$kconfig" "$kconfig".bak
     fi
 
+    MAKEARGS+=("LOCALVERSION=-ktest" ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-")
+
     new_config
 
     log_verbose "kernel_config_require: ${ktest_kernel_config_require[@]}"
 
-    MAKEARGS+=("LOCALVERSION=-ktest" ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-")
-
     for opt in "${ktest_kernel_config_require[@]}"; do
 	[[ -n $opt ]] && kernel_opt set "$opt"
     done
 
-    do_make olddefconfig
+    do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" olddefconfig
 
     for opt in "${ktest_kernel_config_require[@]}"; do
 	[[ -n $opt ]] && kernel_opt check "$opt"
@@ -207,10 +207,10 @@ build_kernel()
 
     case $KERNEL_ARCH in
 	mips)
-	    do_make -k vmlinuz
+	    do_make $MAKEARGS -k vmlinuz
 	    ;;
 	*)
-	    do_make -k
+	    do_make $MAKEARGS -k
 	    ;;
     esac
 
@@ -220,6 +220,9 @@ build_kernel()
 	x86*)
 	    install -m0644 "$BOOT/bzImage"	"$ktest_kernel_binary/vmlinuz"
 	    ;;
+	arm)
+	    install -m0644 "$BOOT/zImage"	"$ktest_kernel_binary/vmlinuz"
+	    ;;
 	aarch64)
 	    install -m0644 "$BOOT/Image"	"$ktest_kernel_binary/vmlinuz"
 	    ;;
@@ -279,13 +282,13 @@ cmd_boot()
 cmd_oldconfig()
 {
     new_config
-    do_make oldconfig
+    do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" oldconfig
 }
 
 cmd_config()
 {
     new_config
-    do_make nconfig
+    do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" nconfig
 }
 
 cmd_faddr2line()
diff --git a/cross.conf b/cross.conf
index 28968856..d98219e9 100644
--- a/cross.conf
+++ b/cross.conf
@@ -5,4 +5,4 @@
 ARCH_TRIPLE_X86=x86-linux-gnu
 ARCH_TRIPLE_X86_64=x86_64-linux-gnu
 ARCH_TRIPLE_ARM64=aarch64-linux-gnu
-
+ARCH_TRIPLE_ARMV7=arm-linux-gnueabihf
diff --git a/lib/common.sh b/lib/common.sh
index e977af48..d167017d 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -117,6 +117,17 @@ parse_arch()
 	    QEMU_PACKAGE=qemu-system-arm
 	    QEMU_BIN=qemu-system-aarch64
 	    ;;
+	armhf|armv7|armv7l|arm)
+	    ktest_arch=arm
+	    DEBIAN_ARCH=armhf
+	    ARCH_TRIPLE=${ARCH_TRIPLE_ARMV7}
+
+	    KERNEL_ARCH=arm
+	    BITS=32
+
+	    QEMU_PACKAGE=qemu-system-arm
+	    QEMU_BIN=qemu-system-arm
+	    ;;
 	mips)
 	    DEBIAN_ARCH=mips
 	    ARCH_TRIPLE=mips-linux-gnu
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 75679b82..62dd3b30 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -223,7 +223,7 @@ ktest_kgdb()
 
 ktest_mon()
 {
-    exec socat UNIX-CONNECT:"$ktest_out/vm/on" STDIO
+    exec socat UNIX-CONNECT:"$ktest_out/vm/mon" STDIO
     exec nc "$ktest_out/vm/mon"
 }
 
@@ -314,7 +314,7 @@ start_vm()
 	x86|x86_64)
 	    qemu_cmd+=(-cpu $cputype -machine type=q35,accel=$accel,nvdimm=on)
 	    ;;
-	aarch64)
+	aarch64|arm)
 	    qemu_cmd+=(-cpu $cputype -machine type=virt,gic-version=max,accel=$accel)
 	    ;;
 	mips)
@@ -336,7 +336,7 @@ start_vm()
     [[ $BITS == 32 ]] &&  memconfig="3G" && ktest_cpus=$((min($ktest_cpus,4)))
 
     qemu_cmd+=(								\
-	-m		"$ktest_mem,slots=8,maxmem=$maxmem"		\
+	-m		"$memconfig"					\
 	-smp		"$ktest_cpus"					\
 	-kernel		"$ktest_kernel_binary/vmlinuz"			\
 	-append		"$(join_by " " ${kernelargs[@]})"		\
@@ -350,6 +350,8 @@ start_vm()
 	-virtfs		local,path=/,mount_tag=host,security_model=none,multidevs=remap	\
     )
 
+
+
     if [[ -f $ktest_kernel_binary/initramfs ]]; then
 	qemu_cmd+=(-initrd 	"$ktest_kernel_binary/initramfs")
     fi
diff --git a/tests/kconfig.sh b/tests/kconfig.sh
index 73e93db1..abb70124 100644
--- a/tests/kconfig.sh
+++ b/tests/kconfig.sh
@@ -12,6 +12,7 @@ case $ktest_arch in
 	require-kernel-config MCORE2	# optimize for core2
 	require-kernel-config IO_DELAY_0XED
 	require-kernel-config 64BIT=n
+	require-kernel-config COMPAT_32BIT_TIME
 	require-kernel-config ACPI	# way slower without it, do not know why
 	require-kernel-config UNWINDER_FRAME_POINTER
 	require-kernel-config HARDLOCKUP_DETECTOR
@@ -38,11 +39,31 @@ case $ktest_arch in
 	have_virtio=1
 	have_suspend=1
 
+	require-kernel-append console=hvc0
+	;;
+    arm)
+	require-kernel-config ARCH_VIRT
+	require-kernel-config SMP
+	require-kernel-config VFP
+	require-kernel-config NEON
+	require-kernel-config ARM_LPAE
+	require-kernel-config MMU
+	require-kernel-config HAVE_PCI
+	require-kernel-config PCI_HOST_GENERIC
+	require-kernel-config ARM_AMBA
+	require-kernel-config COMPAT_32BIT_TIME
+	require-kernel-config RTC_DRV_PL031
+
+	have_virtio=1
+
 	require-kernel-append console=hvc0
 	;;
     aarch64)
 	require-kernel-config PCI_HOST_GENERIC
 	require-kernel-config RTC_DRV_PL031
+	require-kernel-config COMPAT_32BIT_TIME
+	require-kernel-config IOMMU_SUPPORT
+	require-kernel-config PARAVIRT
 
 	have_virtio=1
 
@@ -70,7 +91,7 @@ case $ktest_arch in
 	require-kernel-append console=hvc0
 	;;
     *)
-	echo "Kernel architecture not supported by kconfig.sh"
+	echo "Kernel architecture $ktest_arch not supported by kconfig.sh"
 	exit 1
 	;;
 esac
@@ -95,8 +116,6 @@ require-kernel-config BINFMT_SCRIPT
 
 require-kernel-config COMPACTION	# virtfs doesn't do well without it
 
-require-kernel-config PROC_KCORE	# XXX Needed?
-
 require-kernel-config TTY
 require-kernel-config VT
 

From 7f4ee0a07f3babe3eed454d431e19031fb92005a Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Wed, 15 Nov 2023 14:18:59 +0100
Subject: [PATCH 05/10] Enable rust support cross-arch

Make the rust compiler instsallable via rustup even though it's not in the vm yet.
This avoids rust compile errors on emulated CPUs,
rustup installs for host cpu kernel architecture instead of VM architecture.

Signed-off-by: jpsollie <janpieter.sollie@edpnet.be>
---
 lib/common.sh | 4 ++++
 root_image    | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/common.sh b/lib/common.sh
index d167017d..53a097f5 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -88,6 +88,7 @@ parse_arch()
 	    ktest_arch=x86
 	    DEBIAN_ARCH=i386
 	    ARCH_TRIPLE=${ARCH_TRIPLE_X86}
+	    RUST_TRIPLE=i686-unknown-linux-gnu
 
 	    KERNEL_ARCH=x86
 	    BITS=32
@@ -99,6 +100,7 @@ parse_arch()
 	    ktest_arch=x86_64
 	    DEBIAN_ARCH=amd64
 	    ARCH_TRIPLE=${ARCH_TRIPLE_X86_64}
+	    RUST_TRIPLE=x86_64-unknown-linux-gnu
 
 	    KERNEL_ARCH=x86
 	    BITS=64
@@ -110,6 +112,7 @@ parse_arch()
 	    ktest_arch=aarch64
 	    DEBIAN_ARCH=arm64
 	    ARCH_TRIPLE=${ARCH_TRIPLE_ARM64}
+	    RUST_TRIPLE=aarch64-unknown-linux-gnu
 
 	    KERNEL_ARCH=arm64
 	    BITS=64
@@ -121,6 +124,7 @@ parse_arch()
 	    ktest_arch=arm
 	    DEBIAN_ARCH=armhf
 	    ARCH_TRIPLE=${ARCH_TRIPLE_ARMV7}
+	    RUST_TRIPLE=armv7-unknown-linux-gnueabihf
 
 	    KERNEL_ARCH=arm
 	    BITS=32
diff --git a/root_image b/root_image
index c91faf65..2504bd01 100755
--- a/root_image
+++ b/root_image
@@ -241,7 +241,7 @@ update_packages()
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
     chmod 755 "$MNT"/tmp/rustup.sh
 
-    _chroot "$MNT" /tmp/rustup.sh -y
+    _chroot "$MNT" /tmp/rustup.sh --default-host $RUST_TRIPLE -y
     echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
 }
 

From 721ed7ba037a8c2505fdf222a6aa7df8e0dd25d0 Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@edpnet.be>
Date: Fri, 29 Sep 2023 13:54:39 +0200
Subject: [PATCH 06/10] Ktest: re-introduce foreign unsupported architectures

Whereas MIPS and MIPS64 are abandoned my manufacturer,
SPARC and POWER seem to be still alive,
and upcoming RISCV also deserves some attention,
so add those to the list of supported architectures.

Signed-off-by: Janpieter Sollie <janpieter.sollie@edpnet.be>
---
 build-test-kernel |  9 +++++---
 cross.conf        |  3 +++
 lib/common.sh     | 59 ++++++++++++++---------------------------------
 lib/libktest.sh   | 12 ++++++----
 root_image        | 17 ++++++++------
 tests/kconfig.sh  | 27 +++++++++++++---------
 6 files changed, 59 insertions(+), 68 deletions(-)

diff --git a/build-test-kernel b/build-test-kernel
index 9a35aad3..916d3e12 100755
--- a/build-test-kernel
+++ b/build-test-kernel
@@ -228,11 +228,14 @@ build_kernel()
 	    ;;
 	mips)
 	    install -m0644 "$BOOT/vmlinux.strip"	"$ktest_kernel_binary/vmlinuz"
-	    #install -m0644 "$ktest_kernel_build/vmlinux"	"$ktest_kernel_binary/vmlinuz"
 	    ;;
 	default)
-	    echo "Don't know how to install kernel"
-	    exit 1
+	    if [ -f "$BOOT/Image" ]; then
+		 install -m0644 "$BOOT/Image"	"$ktest_kernel_binary/vmlinuz"
+	    else
+		echo "Don't know how to install kernel"
+	    	exit 1
+	    fi
 	    ;;
     esac
 
diff --git a/cross.conf b/cross.conf
index d98219e9..a78bffd4 100644
--- a/cross.conf
+++ b/cross.conf
@@ -6,3 +6,6 @@ ARCH_TRIPLE_X86=x86-linux-gnu
 ARCH_TRIPLE_X86_64=x86_64-linux-gnu
 ARCH_TRIPLE_ARM64=aarch64-linux-gnu
 ARCH_TRIPLE_ARMV7=arm-linux-gnueabihf
+ARCH_TRIPLE_PPC64=powerpc64-linux-gnu
+ARCH_TRIPLE_SPARC64=sparc64-linux-gnu
+ARCH_TRIPLE_RISCV64=riscv64-linux-gnu
diff --git a/lib/common.sh b/lib/common.sh
index 53a097f5..3ba0a9f4 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -132,39 +132,23 @@ parse_arch()
 	    QEMU_PACKAGE=qemu-system-arm
 	    QEMU_BIN=qemu-system-arm
 	    ;;
-	mips)
-	    DEBIAN_ARCH=mips
-	    ARCH_TRIPLE=mips-linux-gnu
-
-	    KERNEL_ARCH=mips
-	    BITS=32
-
-	    QEMU_PACKAGE=qemu-system-mips
-	    QEMU_BIN=qemu-system-mips
-	    ;;
-	mips64)
-	    DEBIAN_ARCH=mips
-	    ARCH_TRIPLE=mips-linux-gnu
+	riscv64)
+	    DEBIAN_ARCH=riscv64
+	    ARCH_TRIPLE=${ARCH_TRIPLE_RISCV64}
+	    MIRROR=http://deb.debian.org/debian-ports
+	    RUST_TRIPLE=riscv64gc-unknown-linux-gnu
 
-	    KERNEL_ARCH=mips
+	    KERNEL_ARCH=riscv
 	    BITS=64
 
-	    QEMU_PACKAGE=qemu-system-mips
-	    QEMU_BIN=qemu-system-mips64
-	    ;;
-	sparc)
-	    DEBIAN_ARCH=sparc
-	    ARCH_TRIPLE=sparc64-linux-gnu
-
-	    KERNEL_ARCH=sparc
-	    BITS=32
-
-	    QEMU_PACKAGE=qemu-system-sparc
-	    QEMU_BIN=qemu-system-sparc
+	    QEMU_PACKAGE=qemu-system-riscv
+	    QEMU_BIN=qemu-system-riscv64
 	    ;;
 	sparc64)
-	    DEBIAN_ARCH=sparc
-	    ARCH_TRIPLE=sparc64-linux-gnu
+	    DEBIAN_ARCH=sparc64
+	    ARCH_TRIPLE=${ARCH_TRIPLE_SPARC64}
+	    MIRROR=http://deb.debian.org/debian-ports
+	    RUST_TRIPLE=sparc64-unknown-linux-gnu
 
 	    KERNEL_ARCH=sparc
 	    BITS=64
@@ -172,23 +156,13 @@ parse_arch()
 	    QEMU_PACKAGE=qemu-system-sparc
 	    QEMU_BIN=qemu-system-sparc64
 	    ;;
-	ppc|powerpc)
-	    DEBIAN_ARCH=powerpc
-	    MIRROR=http://deb.debian.org/debian-ports
-
-	    ARCH_TRIPLE=powerpc-linux-gnu
-
-	    KERNEL_ARCH=powerpc
-	    BITS=32
-
-	    QEMU_PACKAGE=qemu-system-ppc
-	    QEMU_BIN=qemu-system-ppc
-	    ;;
-	ppc64)
+	ppc64|powerpc)
+	    ktest_arch=ppc64
 	    DEBIAN_ARCH=ppc64
 	    MIRROR=http://deb.debian.org/debian-ports
 
-	    ARCH_TRIPLE=powerpc-linux-gnu
+	    ARCH_TRIPLE=${ARCH_TRIPLE_PPC64}
+	    RUST_TRIPLE=powerpc64-unknown-linux-gnu
 
 	    KERNEL_ARCH=powerpc
 	    BITS=64
@@ -214,6 +188,7 @@ parse_arch()
     export QEMU_BIN
     export ktest_arch
     export BITS
+    export RUST_TRIPLE
 }
 
 find_command() {
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 62dd3b30..a385a08b 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -317,12 +317,14 @@ start_vm()
 	aarch64|arm)
 	    qemu_cmd+=(-cpu $cputype -machine type=virt,gic-version=max,accel=$accel)
 	    ;;
-	mips)
-	    qemu_cmd+=(-cpu 24Kf -machine malta)
-	    ktest_cpus=1
+	ppc64)
+	    qemu_cmd+=(-cpu power9)
 	    ;;
-	mips64)
-	    qemu_cmd+=(-cpu MIPS64R2-generic -machine malta)
+	sparc64)
+	    qemu_cmd+=(-cpu default)
+	    ;;
+	riscv64)
+	    qemu_cmd+=(-cpu any)
 	    ;;
     esac
 
diff --git a/root_image b/root_image
index 2504bd01..764e4103 100755
--- a/root_image
+++ b/root_image
@@ -34,7 +34,7 @@ usage()
     echo
     echo "options:"
     echo "  -h			Display this help and exit"
-    echo "  -a <arch>		Architecture for vm image"
+    echo "  -a <arch>		Architecture for vm image (x86_64,armhf,i386,aarch64,sparc64,ppc64,riscv64)"
     echo "  -m <mirror>		Debian mirror"
     echo '  -i <image>		Image to create/update, defaults to /var/lib/ktest/root.$arch'
 }
@@ -233,9 +233,11 @@ update_packages()
     mkdir -p "$MNT"/run/user/0
     cp /etc/resolv.conf "$MNT/etc/resolv.conf"
     _chroot "$MNT" mount -t proc none /proc
-    _chroot "$MNT" apt-get -qq update
-    _chroot "$MNT" apt-get -qq upgrade
-    _chroot "$MNT" apt-get -qq install --no-install-recommends "${PACKAGES[@]}"
+    [[ $MIRROR == *"debian-ports"* ]] && _chroot "$MNT" apt-get -qq install debian-ports-archive-keyring
+    _chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
+    _chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
+    _chroot "$MNT" apt-get -qq install -f
+    _chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}"
     rm -f "$MNT/var/cache/apt/archives/*.deb"
 
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
@@ -328,14 +330,15 @@ cmd_create()
 	--no-check-gpg					\
 	--arch="$DEBIAN_ARCH"				\
 	--exclude=$(join_by , "${EXCLUDE[@]}")		\
+	$keyring					\
 	--foreign					\
 	--components='main,contrib,non-free' \
-	trixie "$MNT" "$MIRROR"
+	$debian_release "$MNT" "$MIRROR"
 
-    [[ -n ${CROSS_COMPILE} ]] && cp $(which qemu-${ktest_arch}) ${MNT}$(which qemu-${ktest_arch})
+    statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static)
+    [[ -n ${CROSS_COMPILE} ]] && cp $statichelper ${MNT}$statichelper
     _chroot "$MNT" /debootstrap/debootstrap --second-stage
     _chroot "$MNT" dpkg --configure -a
-
     update_packages
     update_files
 
diff --git a/tests/kconfig.sh b/tests/kconfig.sh
index abb70124..9a30094f 100644
--- a/tests/kconfig.sh
+++ b/tests/kconfig.sh
@@ -69,24 +69,29 @@ case $ktest_arch in
 
 	require-kernel-append console=hvc0
 	;;
-    powerpc)
-	require-kernel-config ADVANCED_OPTIONS
+    ppc64)
+	require-kernel-config PPC64
+
+	have_virtio=1
+
+	require-kernel-append console=hvc0
+	;;
+    sparc64)
+	require-kernel-config 64BIT
+	require-kernel-config SMP
+	require-kernel-config VIRTIO_MENU
+	require-kernel-config PCI
 
-	have_kvmguest=1
 	have_virtio=1
-	have_suspend=1
 
 	require-kernel-append console=hvc0
 	;;
-    mips)
-	require-kernel-config MIPS_MALTA
-	require-kernel-config CPU_MIPS${BITS}_R2
-	require-kernel-config CPU_BIG_ENDIAN=y
-	require-kernel-config CPU_LITTLE_ENDIAN=n
-	require-kernel-config 32BIT
+    riscv64)
+	require-kernel-config SOC_VIRT
+	require-kernel-config VIRTIO_MENU
+	require-kernel-config PCI
 
 	have_virtio=1
-	ktest_storage_bus=piix4-ide
 
 	require-kernel-append console=hvc0
 	;;

From 856c6b333566c0f1f8ec919468934f89ad59fd4a Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Fri, 6 Oct 2023 15:31:11 +0200
Subject: [PATCH 07/10] fix a number of issues with cross-architecture testing

This commit fixes a _LOT_ of issues with the previous commits,
but at least there's one supported big endian architecture now,
which is capable of booting and running ktests: s390x

For unsupported architectures, a number of root_create features:
- enabled snapshot lookups for debian archives,
  in case of missing dependencies, it may still be in there
- enabled possibility to build the package from source,
  this is painful
- enabled the possibility during root_create to intervene,
  this is only available on unsupported architectures.

So far, it seems the qemu E6500 emulator for powerPC isn't stable,
and rustup doesn't know about sparc / sparc64,
so both are currently broken.

Re-organised the cross.conf file as the first one was a bit messy.

Signed-Off-by: Janpieter Sollie <janpieter.sollie@edpnet.be>
---
 build-test-kernel                    | 19 ++++----
 cross.conf                           | 17 ++++++-
 lib/common.sh                        | 11 +++++
 lib/libktest.sh                      | 12 +++--
 root_image                           | 71 ++++++++++++++++++++++++++--
 tests/bcachefs/bcachefs-test-libs.sh |  4 ++
 tests/kconfig.sh                     | 30 ++++++++++--
 7 files changed, 137 insertions(+), 27 deletions(-)

diff --git a/build-test-kernel b/build-test-kernel
index 916d3e12..91aeaba4 100755
--- a/build-test-kernel
+++ b/build-test-kernel
@@ -205,14 +205,7 @@ build_kernel()
 	mv "$kconfig".bak "$kconfig"
     fi
 
-    case $KERNEL_ARCH in
-	mips)
-	    do_make $MAKEARGS -k vmlinuz
-	    ;;
-	*)
-	    do_make $MAKEARGS -k
-	    ;;
-    esac
+    do_make $MAKEARGS -k
 
     local BOOT=$ktest_kernel_build/arch/$KERNEL_ARCH/boot
 
@@ -226,12 +219,16 @@ build_kernel()
 	aarch64)
 	    install -m0644 "$BOOT/Image"	"$ktest_kernel_binary/vmlinuz"
 	    ;;
-	mips)
-	    install -m0644 "$BOOT/vmlinux.strip"	"$ktest_kernel_binary/vmlinuz"
+	ppc64)
+	    install -m0644 "$BOOT/zImage"	"$ktest_kernel_binary/vmlinuz"
 	    ;;
-	default)
+	*)
 	    if [ -f "$BOOT/Image" ]; then
 		 install -m0644 "$BOOT/Image"	"$ktest_kernel_binary/vmlinuz"
+	    elif [ -f "$BOOT/zImage" ]; then
+		 install -m0644 "$BOOT/zImage"	"$ktest_kernel_binary/vmlinuz"
+	    elif [ -f "$BOOT/bzImage" ]; then
+		 install -m0644 "$BOOT/bzImage"	"$ktest_kernel_binary/vmlinuz"
 	    else
 		echo "Don't know how to install kernel"
 	    	exit 1
diff --git a/cross.conf b/cross.conf
index a78bffd4..952d4341 100644
--- a/cross.conf
+++ b/cross.conf
@@ -2,10 +2,23 @@
 # whenever these need to be changed (some distributions prefer ARCHITECTURE-VENDOR-OS-LIBC), 
 # change the triplet here
 
+#32 bit architectures
 ARCH_TRIPLE_X86=x86-linux-gnu
+ARCH_TRIPLE_ARMV7=arm-linux-gnueabihf
+
+#64 bit architectures
 ARCH_TRIPLE_X86_64=x86_64-linux-gnu
 ARCH_TRIPLE_ARM64=aarch64-linux-gnu
-ARCH_TRIPLE_ARMV7=arm-linux-gnueabihf
-ARCH_TRIPLE_PPC64=powerpc64-linux-gnu
+ARCH_TRIPLE_S390X=s390x-linux-gnu
+
+#currently unsupported (but maintained) debian architectures
+ARCH_TRIPLE_PPC64=powerpc-linux-gnu
 ARCH_TRIPLE_SPARC64=sparc64-linux-gnu
 ARCH_TRIPLE_RISCV64=riscv64-linux-gnu
+
+#here you can specify up snapshots which are used to fix sid dependencies:
+#debian SID is updated frequently, having a lot of broken packages due to dependencies
+#view https://snapshot.debian.org/archive/debian-ports/ for a list
+#as always: the older, the more dangerous ...
+
+SID_SNAPSHOTS=("20220501T101242Z")
diff --git a/lib/common.sh b/lib/common.sh
index 3ba0a9f4..c74791ab 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -132,6 +132,17 @@ parse_arch()
 	    QEMU_PACKAGE=qemu-system-arm
 	    QEMU_BIN=qemu-system-arm
 	    ;;
+	s390x)
+	    DEBIAN_ARCH=s390x
+	    ARCH_TRIPLE=${ARCH_TRIPLE_S390X}
+	    RUST_TRIPLE=s390x-unknown-linux-gnu
+
+	    KERNEL_ARCH=s390
+	    BITS=64
+
+	    QEMU_PACKAGE=qemu-system-s390x
+	    QEMU_BIN=qemu-system-s390x
+	    ;;
 	riscv64)
 	    DEBIAN_ARCH=riscv64
 	    ARCH_TRIPLE=${ARCH_TRIPLE_RISCV64}
diff --git a/lib/libktest.sh b/lib/libktest.sh
index a385a08b..d1908a05 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -85,7 +85,7 @@ parse_ktest_arg()
 
 parse_args_post()
 {
-    [ -z ${ktest_arch:+x} ] && ktest_arch=$(uname -m)
+    [[ -z ${ktest_arch} ]] && ktest_arch=$(uname -m)
     parse_arch "$ktest_arch"
 
     ktest_out=$(readlink -f "$ktest_out")
@@ -318,13 +318,17 @@ start_vm()
 	    qemu_cmd+=(-cpu $cputype -machine type=virt,gic-version=max,accel=$accel)
 	    ;;
 	ppc64)
-	    qemu_cmd+=(-cpu power9)
+	    qemu_cmd+=(-machine ppce500 -cpu e6500 -accel tcg)
+	    ;;
+	s390x)
+	    qemu_cmd+=(-cpu max -machine s390-ccw-virtio -accel tcg)
 	    ;;
 	sparc64)
-	    qemu_cmd+=(-cpu default)
+	    qemu_cmd+=(-machine sun4u -accel tcg)
+	    ktest_cpus=1; #sparc64 currently supports only 1 cpu
 	    ;;
 	riscv64)
-	    qemu_cmd+=(-cpu any)
+	    qemu_cmd+=(-machine virt -cpu rv64 -accel tcg)
 	    ;;
     esac
 
diff --git a/root_image b/root_image
index 764e4103..af4f1798 100755
--- a/root_image
+++ b/root_image
@@ -34,11 +34,68 @@ usage()
     echo
     echo "options:"
     echo "  -h			Display this help and exit"
-    echo "  -a <arch>		Architecture for vm image (x86_64,armhf,i386,aarch64,sparc64,ppc64,riscv64)"
+    echo "  -a <arch>		Architecture for vm (x86_64,armhf,i386,aarch64,sparc64,s390x,ppc64,riscv64)"
     echo "  -m <mirror>		Debian mirror"
     echo '  -i <image>		Image to create/update, defaults to /var/lib/ktest/root.$arch'
 }
 
+intervene()
+{
+	local yn="N";
+	echo "Installing $2 failed: both binary install and $1";
+	echo "You can start a shell to check whether any dependency issues can be fixed"
+	echo "if you do, and exit the shell, it is assumed to be fixed and installing will continue"
+	read -p "Do you wish to start a shell to investigate? (y/N)" yn
+	case $yn in
+		[Yy]* )
+			_chroot "$MNT" /tmpbuild/executer.sh /bin/bash;
+		;;
+		* ) return 1;;
+	esac
+}
+
+install_fix_deps()
+{
+	trap 'echo "Could not install packages! please revert to the included torrent file to download a working root image"; umount $MNT/tmpbuild; umount_image; rm "$ktest_image"' EXIT
+	#get necessary certificates / pgp keyrings to work with unofficial and outdated snapshots
+	_chroot "$MNT" apt-get -qq install debian-ports-archive-keyring ca-certificates
+	#disable unwanted checks - we know the snapshots are not valid
+	echo "Acquire::Check-Valid-Until false;" > $MNT/etc/apt/apt.conf.d/10-nocheckvalid
+	#create a sources.list with more options to work around missing packages
+	echo 'deb [trusted=yes] http://deb.debian.org/debian-ports unstable main non-free' > $MNT/etc/apt/sources.list
+	echo 'deb-src [trusted=yes] http://deb.debian.org/debian-ports unstable main non-free' >> $MNT/etc/apt/sources.list
+	echo 'deb [trusted=yes] http://deb.debian.org/debian-ports unreleased main non-free' >> $MNT/etc/apt/sources.list
+	echo 'deb-src [trusted=yes] http://deb.debian.org/debian-ports unreleased main non-free' >> $MNT/etc/apt/sources.list
+	for i in "${SID_SNAPSHOTS[@]}"; do
+	echo "deb [trusted=yes] https://snapshot.debian.org/archive/debian-ports/$i sid main" >> $MNT/etc/apt/sources.list
+	echo "deb-src [trusted=yes] https://snapshot.debian.org/archive/debian-ports/$i sid main" >> $MNT/etc/apt/sources.list
+	done
+	echo "deb-src http://deb.debian.org/debian unstable main contrib non-free" >> $MNT/etc/apt/sources.list
+	touch $MNT/etc/passwd
+	touch $MNT/etc/shadow
+	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
+	_chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
+	_chroot "$MNT" apt-get -qq install -f
+	_chroot "$MNT" apt-get -qq install build-essential
+	#if we can install them all at once, do so, otherwise, try one by one
+	_chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}" || for i in "${PACKAGES[@]}"; do
+		if [[ ! $(_chroot "$MNT" apt-cache show $i | grep Version) == "" ]]; then
+		_chroot "$MNT" apt-get -qq install --allow-unauthenticated --no-install-recommends $i && continue;
+		fi
+		#installing binary failed.  Try to install from source.  Do it in a tmpfs folder so the image doesn't get overwhelmed:
+		mkdir ${MNT}/tmpbuild;
+		mount -t tmpfs none ${MNT}/tmpbuild;
+		echo '#!/bin/bash' > "$MNT"/tmpbuild/executer.sh
+		echo 'cd /tmpbuild/; $@' >> "$MNT"/tmpbuild/executer.sh
+		chmod ago+x "$MNT"/tmpbuild/executer.sh
+		_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq build-dep --allow-unauthenticated --no-install-recommends $i || intervene install-deps $i;
+		_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq -b source --allow-unauthenticated $i || intervene build $i;
+		[ -f "${MNT}/tmpbuild/${i}*.deb" ] && _chroot "$MNT" apt-get -qq install /tmpbuild/${i}*.deb;
+		umount ${MNT}/tmpbuild;
+		rmdir ${MNT}/tmpbuild;
+	done
+}
+
 if [[ $# = 0 ]]; then
     usage
     exit 1
@@ -130,7 +187,7 @@ PACKAGES+=("linux-headers-generic")
 # [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp)
 
 EXCLUDE=(dmidecode nano rsyslog logrotate cron		\
-    iptables nfacct					\
+    iptables nfacct vim-tiny				\
     debconf-i18n info gnupg libpam-systemd)
 
 SYSTEMD_MASK=(dev-hvc0.device				\
@@ -233,11 +290,14 @@ update_packages()
     mkdir -p "$MNT"/run/user/0
     cp /etc/resolv.conf "$MNT/etc/resolv.conf"
     _chroot "$MNT" mount -t proc none /proc
-    [[ $MIRROR == *"debian-ports"* ]] && _chroot "$MNT" apt-get -qq install debian-ports-archive-keyring
+    if [[ $MIRROR == *"debian-ports"* ]]; then
+	install_fix_deps
+    else
     _chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
     _chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
     _chroot "$MNT" apt-get -qq install -f
     _chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}"
+    fi
     rm -f "$MNT/var/cache/apt/archives/*.deb"
 
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
@@ -293,11 +353,11 @@ cmd_create()
 	echo "$ktest_image already exists"
 	exit 1
     fi
-
+    set -o xtrace
     (cd "$ktest_dir"; git submodule update --init debootstrap)
 
     MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)
-    trap 'umount_image; rm "$ktest_image"' EXIT
+    trap '/bin/bash; umount_image; rm "$ktest_image"' EXIT
 
     fallocate -l "$IMAGE_SIZE" "$ktest_image"
     mkfs.ext4 -F "$ktest_image"
@@ -337,6 +397,7 @@ cmd_create()
 
     statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static)
     [[ -n ${CROSS_COMPILE} ]] && cp $statichelper ${MNT}$statichelper
+
     _chroot "$MNT" /debootstrap/debootstrap --second-stage
     _chroot "$MNT" dpkg --configure -a
     update_packages
diff --git a/tests/bcachefs/bcachefs-test-libs.sh b/tests/bcachefs/bcachefs-test-libs.sh
index 05996b11..90c36b6f 100644
--- a/tests/bcachefs/bcachefs-test-libs.sh
+++ b/tests/bcachefs/bcachefs-test-libs.sh
@@ -28,8 +28,12 @@ else
     require-kernel-config BCACHEFS_NO_LATENCY_ACCT=y
 fi
 
+if [[ ! $ktest_arch == ppc64 ]]; then
+
 require-kernel-config TRANSPARENT_HUGEPAGE
 
+fi
+
 if [[ $ktest_arch = x86_64 ]]; then
     require-kernel-config CRYPTO_CRC32C_INTEL
     require-kernel-config CRYPTO_POLY1305_X86_64
diff --git a/tests/kconfig.sh b/tests/kconfig.sh
index 9a30094f..1088311d 100644
--- a/tests/kconfig.sh
+++ b/tests/kconfig.sh
@@ -4,8 +4,6 @@ have_kvmguest=0
 have_virtio=0
 have_suspend=0
 
-[ -z ${ktest_arch:+x} ] && ktest_arch=$(uname -m)
-
 case $ktest_arch in
     x86)
 	require-kernel-config SMP
@@ -69,8 +67,14 @@ case $ktest_arch in
 
 	require-kernel-append console=hvc0
 	;;
+
     ppc64)
 	require-kernel-config PPC64
+	require-kernel-config PPC_BOOK3E_64
+	require-kernel-config PPC_QEMU_E500
+	require-kernel-config E6500_CPU
+	require-kernel-config SMP
+	require-kernel-config KVM_GUEST
 
 	have_virtio=1
 
@@ -79,7 +83,6 @@ case $ktest_arch in
     sparc64)
 	require-kernel-config 64BIT
 	require-kernel-config SMP
-	require-kernel-config VIRTIO_MENU
 	require-kernel-config PCI
 
 	have_virtio=1
@@ -93,6 +96,17 @@ case $ktest_arch in
 
 	have_virtio=1
 
+	require-kernel-append console=hvc0
+	;;
+    s390x)
+	require-kernel-config S390_GUEST
+	require-kernel-config MARCH_Z13
+	require-kernel-config CMM
+	require-kernel-config PCI
+	require-kernel-config DCSSBLK
+
+	have_virtio=1
+
 	require-kernel-append console=hvc0
 	;;
     *)
@@ -183,13 +197,15 @@ require-kernel-config PCI
 require-kernel-config HW_RANDOM
 
 # Clock:
+if [[ ! $ktest_arch == *"s390"* ]]; then
 require-kernel-config RTC_CLASS
 require-kernel-config RTC_HCTOSYS
-
+fi
 # Console:
+if [[ ! $ktest_arch == *"s390"* ]]; then
 require-kernel-config SERIAL_8250	# XXX can probably drop
 require-kernel-config SERIAL_8250_CONSOLE
-
+fi
 # Block devices:
 require-kernel-config SCSI
 require-kernel-config SCSI_LOWLEVEL	# what's this for?
@@ -230,8 +246,10 @@ require-kernel-config 9P_FS
 #fi
 
 # KGDB:
+if [[ ! $ktest_arch == *"s390"* ]]; then
 require-kernel-config KGDB
 require-kernel-config KGDB_SERIAL_CONSOLE
+fi
 require-kernel-config VMAP_STACK=n
 require-kernel-config RANDOMIZE_BASE=n
 require-kernel-config RANDOMIZE_MEMORY=n
@@ -254,7 +272,9 @@ require-kernel-config FUNCTION_TRACER
 #require-kernel-config ENABLE_DEFAULT_TRACERS
 
 require-kernel-config PANIC_ON_OOPS
+if [[ ! $ktest_arch == *"s390"* ]]; then
 require-kernel-config SOFTLOCKUP_DETECTOR
+fi
 require-kernel-config DETECT_HUNG_TASK
 #require-kernel-config DEFAULT_HUNG_TASK_TIMEOUT=30
 require-kernel-config WQ_WATCHDOG

From 81000df585c89c4d89dbf8415159138fe6015fed Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Sun, 8 Oct 2023 12:56:08 +0200
Subject: [PATCH 08/10] Fall back to sid rustc if rustup isn't available

Rustup only has a limited number of architectures,
when the root_image we are creating isn't supported,
try to install the debian rustc compiler.
this will be older, but >95% of time, it should be ok.

Signed-off-by: Janpieter Sollie <janpieter.sollie@edpnet.be>
---
 root_image | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/root_image b/root_image
index af4f1798..f6e2f82b 100755
--- a/root_image
+++ b/root_image
@@ -89,8 +89,11 @@ install_fix_deps()
 		echo 'cd /tmpbuild/; $@' >> "$MNT"/tmpbuild/executer.sh
 		chmod ago+x "$MNT"/tmpbuild/executer.sh
 		_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq build-dep --allow-unauthenticated --no-install-recommends $i || intervene install-deps $i;
-		_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq -b source --allow-unauthenticated $i || intervene build $i;
-		[ -f "${MNT}/tmpbuild/${i}*.deb" ] && _chroot "$MNT" apt-get -qq install /tmpbuild/${i}*.deb;
+		#in case the intervention also installed it, continue:
+		if [[ $(_chroot "$MNT" dpkg -l | grep "^ii  $i") == "" ]]; then
+			_chroot "$MNT" /tmpbuild/executer.sh apt-get -qq -b source --allow-unauthenticated $i || intervene build $i;
+			[ -f "${MNT}/tmpbuild/${i}*.deb" ] && _chroot "$MNT" apt-get -qq install /tmpbuild/${i}*.deb;
+		fi
 		umount ${MNT}/tmpbuild;
 		rmdir ${MNT}/tmpbuild;
 	done
@@ -286,25 +289,32 @@ ZZ
 
 update_packages()
 {
+    NO_RUSTUP=0
     # systemd... !?
     mkdir -p "$MNT"/run/user/0
     cp /etc/resolv.conf "$MNT/etc/resolv.conf"
+
+    [[ $(curl -qq -I "https://static.rust-lang.org/rustup/${RUST_TRIPLE}/rustup-init" 2>/dev/null | grep "HTTP/2 200") == "" ]] && NO_RUSTUP=1
+    [ $NO_RUSTUP ] && PACKAGES+=(rustc rustc-dbgsym rustfmt rustfmt-dbgsym) #don't do rustup on unsupported architectures, just try from debian repositories
+
     _chroot "$MNT" mount -t proc none /proc
     if [[ $MIRROR == *"debian-ports"* ]]; then
 	install_fix_deps
     else
-    _chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
-    _chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
-    _chroot "$MNT" apt-get -qq install -f
-    _chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}"
+	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
+    	_chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade
+    	_chroot "$MNT" apt-get -qq install -f
+    	_chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}"
     fi
     rm -f "$MNT/var/cache/apt/archives/*.deb"
 
+    if [ ! $NO_RUSTUP ]; then
     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
     chmod 755 "$MNT"/tmp/rustup.sh
 
     _chroot "$MNT" /tmp/rustup.sh --default-host $RUST_TRIPLE -y
     echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
+    fi;
 }
 
 trim_image()
@@ -353,7 +363,6 @@ cmd_create()
 	echo "$ktest_image already exists"
 	exit 1
     fi
-    set -o xtrace
     (cd "$ktest_dir"; git submodule update --init debootstrap)
 
     MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX)

From be4a189380d59497b393e169a75568f695bdd2f0 Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Tue, 31 Oct 2023 13:47:41 +0100
Subject: [PATCH 09/10] fixup!: - fix malfunctioning variables - provide sid
 fallback for missing architectures - make the extended installer default for
 sid

Signed-off-by: Janpieter Sollie <janpieter.sollie@edpnet.be>
---
 lib/common.sh   |  5 +++-
 lib/libktest.sh |  6 ++---
 root_image      | 68 +++++++++++++++++++++++++++++--------------------
 3 files changed, 48 insertions(+), 31 deletions(-)

diff --git a/lib/common.sh b/lib/common.sh
index c74791ab..b32a8e9e 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -188,8 +188,10 @@ parse_arch()
 
     if [[ $ktest_arch != $(uname -m) ]]; then
 	CROSS_COMPILE=1
+    else
+	CROSS_COMPILE=
     fi
-    #special case: x86_64 is able to run i386 code.  this isn't always the case for armv8 -> armv7 (cortex A35)
+    #special case: x86_64 is able to run i386 code. we can use KVM.
     [[ $DEBIAN_ARCH == "i386" && "$(uname -m)" == "x86_64" ]] && CROSS_COMPILE=
     export DEBIAN_ARCH
     export MIRROR
@@ -200,6 +202,7 @@ parse_arch()
     export ktest_arch
     export BITS
     export RUST_TRIPLE
+    export CROSS_COMPILE
 }
 
 find_command() {
diff --git a/lib/libktest.sh b/lib/libktest.sh
index d1908a05..9cb9aea8 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -27,6 +27,7 @@ ktest_ssh_port=0
 ktest_networking=user
 ktest_dio=off
 ktest_nice=0
+ktest_arch=
 
 checkdep socat
 checkdep brotli
@@ -286,7 +287,6 @@ start_vm()
     ln -s "$ktest_tmp" "$ktest_out/vm"
 
     local kernelargs=()
-
     case $ktest_storage_bus in
 	virtio-blk)
 	    ktest_root_dev="/dev/vda"
@@ -296,7 +296,7 @@ start_vm()
 	    ;;
     esac
 
-    kernelargs+=(root=$ktest_root_dev rw log_buf_len=8M)
+    kernelargs+=(root=$ktest_root_dev rw log_buf_len=8M rootwait)
     kernelargs+=(mitigations=off)
     kernelargs+=("ktest.dir=$ktest_dir")
     kernelargs+=(ktest.env=$(readlink -f "$ktest_out/vm/env"))
@@ -339,7 +339,7 @@ start_vm()
     local memconfig="$ktest_mem,slots=8,maxmem=$maxmem"
 
     #do not be fancy on 32-bit hardware.  if it works, it's fine
-    [[ $BITS == 32 ]] &&  memconfig="3G" && ktest_cpus=$((min($ktest_cpus,4)))
+    [[ $BITS == 32 ]] &&  memconfig="3G" && ktest_cpus=$(($ktest_cpus > 4 ? 4 : $ktest_cpus))
 
     qemu_cmd+=(								\
 	-m		"$memconfig"					\
diff --git a/root_image b/root_image
index f6e2f82b..770e719f 100755
--- a/root_image
+++ b/root_image
@@ -23,7 +23,8 @@ checkdep mkfs.ext4 e2fsprogs
 checkdep curl 
 
 IMAGE_SIZE="10G"
-MIRROR=https://deb.debian.org/debian/
+#http is preferred over https here: on emulated CPUs, the encryption simply isn't worth it
+MIRROR=http://deb.debian.org/debian/
 
 usage()
 {
@@ -44,33 +45,43 @@ intervene()
 	local yn="N";
 	echo "Installing $2 failed: both binary install and $1";
 	echo "You can start a shell to check whether any dependency issues can be fixed"
-	echo "if you do, and exit the shell, it is assumed to be fixed and installing will continue"
-	read -p "Do you wish to start a shell to investigate? (y/N)" yn
+	echo "if you do (y), and exit the shell, it is assumed to be fixed and installing will continue"
+	echo "if you don't, we give up."
+        echo "you can also choose to skip package installation (s), though this is not recommended"
+	read -p "Do you wish to start a shell to investigate? (y/s/N)" yn
 	case $yn in
 		[Yy]* )
 			_chroot "$MNT" /tmpbuild/executer.sh /bin/bash;
 		;;
+		[sS]* )
+			return 0;
+		;;
 		* ) return 1;;
 	esac
 }
 
 install_fix_deps()
 {
-	trap 'echo "Could not install packages! please revert to the included torrent file to download a working root image"; umount $MNT/tmpbuild; umount_image; rm "$ktest_image"' EXIT
+	trap 'echo "Could not install packages! consider switching to a supported architecture"; umount_image; rm "$ktest_image"' EXIT
+	#debian has a separate dist for "experimental" / broken packages, in debian-ports it is called "unreleased".
+	experimental_dist="experimental" && [[ ${MIRROR} == *"debian-ports"* ]] && experimental_dist=unreleased;
+        SNAPSHOT_MIRROR="https://snapshot.debian.org/archive/debian" && [[ ${MIRROR} == *"debian-ports"* ]] && SNAPSHOT_MIRROR+="-ports";
 	#get necessary certificates / pgp keyrings to work with unofficial and outdated snapshots
 	_chroot "$MNT" apt-get -qq install debian-ports-archive-keyring ca-certificates
 	#disable unwanted checks - we know the snapshots are not valid
 	echo "Acquire::Check-Valid-Until false;" > $MNT/etc/apt/apt.conf.d/10-nocheckvalid
 	#create a sources.list with more options to work around missing packages
-	echo 'deb [trusted=yes] http://deb.debian.org/debian-ports unstable main non-free' > $MNT/etc/apt/sources.list
-	echo 'deb-src [trusted=yes] http://deb.debian.org/debian-ports unstable main non-free' >> $MNT/etc/apt/sources.list
-	echo 'deb [trusted=yes] http://deb.debian.org/debian-ports unreleased main non-free' >> $MNT/etc/apt/sources.list
-	echo 'deb-src [trusted=yes] http://deb.debian.org/debian-ports unreleased main non-free' >> $MNT/etc/apt/sources.list
+	echo "deb [trusted=yes] $MIRROR sid main contrib" > $MNT/etc/apt/sources.list
+	echo "deb-src [trusted=yes] $MIRROR sid main contrib" >> $MNT/etc/apt/sources.list
+	echo "deb [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list
+	echo "deb-src [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list
+        #add snapshots specified in cross.conf
 	for i in "${SID_SNAPSHOTS[@]}"; do
-	echo "deb [trusted=yes] https://snapshot.debian.org/archive/debian-ports/$i sid main" >> $MNT/etc/apt/sources.list
-	echo "deb-src [trusted=yes] https://snapshot.debian.org/archive/debian-ports/$i sid main" >> $MNT/etc/apt/sources.list
+	echo "deb [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list
+	echo "deb-src [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list
 	done
-	echo "deb-src http://deb.debian.org/debian unstable main contrib non-free" >> $MNT/etc/apt/sources.list
+        #when working with debian-ports, add the general sources as a last hope ... there *may* be a package compiling for this archwe need
+	[[ ${MIRROR} == *"debian-ports"* ]] && echo "deb-src [trusted=yes] http://deb.debian.org/debian sid main contrib" >> $MNT/etc/apt/sources.list
 	touch $MNT/etc/passwd
 	touch $MNT/etc/shadow
 	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
@@ -83,7 +94,7 @@ install_fix_deps()
 		_chroot "$MNT" apt-get -qq install --allow-unauthenticated --no-install-recommends $i && continue;
 		fi
 		#installing binary failed.  Try to install from source.  Do it in a tmpfs folder so the image doesn't get overwhelmed:
-		mkdir ${MNT}/tmpbuild;
+		[[ ! -d ${MNT}/tmpbuild ]] && mkdir ${MNT}/tmpbuild;
 		mount -t tmpfs none ${MNT}/tmpbuild;
 		echo '#!/bin/bash' > "$MNT"/tmpbuild/executer.sh
 		echo 'cd /tmpbuild/; $@' >> "$MNT"/tmpbuild/executer.sh
@@ -289,16 +300,17 @@ ZZ
 
 update_packages()
 {
-    NO_RUSTUP=0
+    NO_RUSTUP=
     # systemd... !?
     mkdir -p "$MNT"/run/user/0
     cp /etc/resolv.conf "$MNT/etc/resolv.conf"
-
-    [[ $(curl -qq -I "https://static.rust-lang.org/rustup/${RUST_TRIPLE}/rustup-init" 2>/dev/null | grep "HTTP/2 200") == "" ]] && NO_RUSTUP=1
-    [ $NO_RUSTUP ] && PACKAGES+=(rustc rustc-dbgsym rustfmt rustfmt-dbgsym) #don't do rustup on unsupported architectures, just try from debian repositories
+    #don't do rustup on unsupported architectures, just try to install from debian repositories
+    [[ -z $(curl -qq -I "https://static.rust-lang.org/rustup/dist/${RUST_TRIPLE}/rustup-init" 2>/dev/null | grep "HTTP/2 200") ]] && NO_RUSTUP=1
+    [[ -n $NO_RUSTUP ]] && PACKAGES+=(rustc rustc-dbgsym rustfmt rustfmt-dbgsym)
 
     _chroot "$MNT" mount -t proc none /proc
-    if [[ $MIRROR == *"debian-ports"* ]]; then
+    _chroot "$MNT" mount -t devpts none /dev/pts
+    if [[ $MIRROR == *"debian-ports"* || $1 == "sid" ]]; then
 	install_fix_deps
     else
 	_chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing
@@ -308,12 +320,12 @@ update_packages()
     fi
     rm -f "$MNT/var/cache/apt/archives/*.deb"
 
-    if [ ! $NO_RUSTUP ]; then
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
-    chmod 755 "$MNT"/tmp/rustup.sh
+    if [[ -z $NO_RUSTUP ]]; then
+	curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh
+	chmod 755 "$MNT"/tmp/rustup.sh
 
-    _chroot "$MNT" /tmp/rustup.sh --default-host $RUST_TRIPLE -y
-    echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
+	_chroot "$MNT" /tmp/rustup.sh --default-host ${RUST_TRIPLE} -y
+	echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh
     fi;
 }
 
@@ -349,7 +361,7 @@ cmd_update()
     cp "$ktest_image" "$ktest_image".new
     mount "$ktest_image".new "$MNT"
 
-    update_packages
+    update_packages none
     update_files
 
     umount_image
@@ -391,7 +403,7 @@ cmd_create()
     #fallback: if the architecture can't be found, try official sid:
     if [[ -z $(curl -qq -I "${MIRROR}/dists/${debian_release}/main/binary-${DEBIAN_ARCH}/Release" 2>/dev/null | grep "HTTP/2 200") ]]; then
 	echo "WARNING: $DEBIAN_ARCH $debian_release could not be found at ${MIRROR} (where it should be).  Falling back to standard sid"
-        MIRROR="https://deb.debian.org/debian/"
+        MIRROR="http://deb.debian.org/debian/"
 	debian_release=sid
     fi
 
@@ -404,12 +416,14 @@ cmd_create()
 	--components='main,contrib,non-free' \
 	$debian_release "$MNT" "$MIRROR"
 
-    statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static)
-    [[ -n ${CROSS_COMPILE} ]] && cp $statichelper ${MNT}$statichelper
+    if [[ -n ${CROSS_COMPILE} ]]; then
+       statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static)
+       cp ${statichelper} ${MNT}${statichelper}
+    fi
 
     _chroot "$MNT" /debootstrap/debootstrap --second-stage
     _chroot "$MNT" dpkg --configure -a
-    update_packages
+    update_packages $debian_release
     update_files
 
     umount_image

From 48e153e858a021aaba74bbe5491469c3811e2db3 Mon Sep 17 00:00:00 2001
From: Janpieter Sollie <janpieter.sollie@kabelmail.de>
Date: Sat, 18 Nov 2023 14:37:51 +0100
Subject: [PATCH 10/10] ktest: use overlays to store arch-specific
 bcachefs-tools build

when running tests, eg test1 on arch 1, and then on arch 2,
the whole bcachefs-tools package needs to be rebuilt:
all binaries of arch 1 need to be erased, and recompiled for arch 2
when going back to arch 1, it happens again

This patch fixes the issue by applying a arch-specific overlay,
so bcachefs-tools doesn't need a full rebuild every time.
The overlay is mounted via fuse-overlay.
If this isn't available, it will fall back to rsync,
causing a larger waste of disk space,
and not-applied-immediately git commits.

Signed-of-by: jpsollie <janpieter.sollie@edpnet.be>
---
 lib/common.sh                        |  2 ++
 tests/bcachefs/bcachefs-test-libs.sh | 22 +++++++++++++++++++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/lib/common.sh b/lib/common.sh
index b32a8e9e..e5acf143 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -19,6 +19,8 @@ ktest_exit()
     fi
 
     [[ -n $ktest_tmp ]] && rm -rf "$ktest_tmp"
+    #umount overlayfs of bcachefs-tools if needed
+    [[ -n $ktest_tmp && $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]] && umount $(grep bcachefs-tools-$ktest_arch /proc/mounts | cut -d' ' -f 2)
     true
 }
 
diff --git a/tests/bcachefs/bcachefs-test-libs.sh b/tests/bcachefs/bcachefs-test-libs.sh
index 90c36b6f..f6461b5f 100644
--- a/tests/bcachefs/bcachefs-test-libs.sh
+++ b/tests/bcachefs/bcachefs-test-libs.sh
@@ -7,16 +7,32 @@
 . $(dirname $(readlink -e "${BASH_SOURCE[0]}"))/../test-libs.sh
 bch_loc=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-tools
 
-require-git http://evilpiepirate.org/git/bcachefs-tools.git
-if [[ ! -f "${bch_loc}/.last_arch_for_compile" || "$(cat ${bch_loc}/.last_arch_for_compile)" != $ktest_arch ]]; then
+declare -A ov_dirs=(["tmp"]="${bch_loc}/../.tmp-bcachefs-tools-overlay" \
+              ["diff"]="${bch_loc}/../.bcachefs-tools-overlay-${ktest_arch}" \
+              ["merge"]="${bch_loc}/../bcachefs-tools-${ktest_arch}")
+
+for dir_overlay in ${ov_dirs[@]}; do
+  if [[ ! -d ${dir_overlay} ]]; then mkdir -p ${dir_overlay}; fi
+done
+
+if [[ $(which fuse-overlayfs) && ! $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]]; then
+	fuse-overlayfs -o lowerdir=${bch_loc},upperdir=${ov_dirs["diff"]},workdir=${ov_dirs["tmp"]} ${ov_dirs["merge"]}
+elif [[ ! $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]]; then
+	rsync -av ${bch_loc}/ ${ov_dirs["merge"]}/
+fi
+
+bch_loc=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-tools-${ktest_arch}
+
+if [[ ! -f "${bch_loc}/.last_arch_for_compile" || "$(cat ${bch_loc}/.last_arch_for_compile)" != ${ktest_arch} ]]; then
 	make -C ${bch_loc} clean >/dev/null 2>&1;
 	rm -rf "${bch_loc}/rust-src/target/*";
 	find ${bch_loc} -name "*.o" -exec rm {} \;
 	find ${bch_loc} -name "*.a" -exec rm {} \;
 	echo $ktest_arch > ${bch_loc}/.last_arch_for_compile
 fi
-require-make bcachefs-tools
 
+require-git http://evilpiepirate.org/git/bcachefs-tools.git
+require-make bcachefs-tools-${ktest_arch}
 require-kernel-config BCACHEFS_FS
 
 if [[ ! -v NO_BCACHEFS_DEBUG ]]; then