diff --git a/.gitignore b/.gitignore index d82a2a50..0e86b513 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ cscope* tags ktest-out *.tags +cross.conf lib/lwip-connect lib/supervisor diff --git a/build-test-kernel b/build-test-kernel index 72990bae..91aeaba4 100755 --- a/build-test-kernel +++ b/build-test-kernel @@ -107,7 +107,7 @@ ktest_kernel_source=$(readlink -e "$ktest_kernel_source") ktest_kernel_build="$ktest_out/kernel_build.$ktest_arch" mkdir -p "$ktest_kernel_build" -if [[ -n $CROSS_COMPILE ]]; then +if [[ -n ${CROSS_COMPILE} ]]; then checkdep "$ARCH_TRIPLE-gcc" "gcc-$ARCH_TRIPLE" fi @@ -121,10 +121,6 @@ run_ktest() do_make() { - if [[ -n $CROSS_COMPILE ]]; then - export ARCH="$KERNEL_ARCH" - export CROSS_COMPILE="$ARCH_TRIPLE-" - fi make --jobs="$ktest_njobs" \ --directory="$ktest_kernel_source" \ @@ -141,7 +137,7 @@ new_config() local config_tool="$ktest_kernel_source/scripts/config" if [[ ! -f $kconfig ]]; then - do_make allnoconfig + do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" allnoconfig # Really undefine everything: sed -i -e 's/\(CONFIG_.*\)=.*/# \1 is not set/' "$kconfig" @@ -188,17 +184,17 @@ build_kernel() mv "$kconfig" "$kconfig".bak fi + MAKEARGS+=("LOCALVERSION=-ktest" ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-") + new_config log_verbose "kernel_config_require: ${ktest_kernel_config_require[@]}" - MAKEARGS+=("LOCALVERSION=-ktest") - for opt in "${ktest_kernel_config_require[@]}"; do [[ -n $opt ]] && kernel_opt set "$opt" done - do_make olddefconfig + do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" olddefconfig for opt in "${ktest_kernel_config_require[@]}"; do [[ -n $opt ]] && kernel_opt check "$opt" @@ -209,14 +205,7 @@ build_kernel() mv "$kconfig".bak "$kconfig" fi - case $KERNEL_ARCH in - mips) - do_make -k vmlinuz - ;; - *) - do_make -k - ;; - esac + do_make $MAKEARGS -k local BOOT=$ktest_kernel_build/arch/$KERNEL_ARCH/boot @@ -224,16 +213,26 @@ build_kernel() x86*) install -m0644 "$BOOT/bzImage" "$ktest_kernel_binary/vmlinuz" ;; + arm) + install -m0644 "$BOOT/zImage" "$ktest_kernel_binary/vmlinuz" + ;; aarch64) install -m0644 "$BOOT/Image" "$ktest_kernel_binary/vmlinuz" ;; - mips) - install -m0644 "$BOOT/vmlinux.strip" "$ktest_kernel_binary/vmlinuz" - #install -m0644 "$ktest_kernel_build/vmlinux" "$ktest_kernel_binary/vmlinuz" + ppc64) + install -m0644 "$BOOT/zImage" "$ktest_kernel_binary/vmlinuz" ;; - default) - echo "Don't know how to install kernel" - exit 1 + *) + if [ -f "$BOOT/Image" ]; then + install -m0644 "$BOOT/Image" "$ktest_kernel_binary/vmlinuz" + elif [ -f "$BOOT/zImage" ]; then + install -m0644 "$BOOT/zImage" "$ktest_kernel_binary/vmlinuz" + elif [ -f "$BOOT/bzImage" ]; then + install -m0644 "$BOOT/bzImage" "$ktest_kernel_binary/vmlinuz" + else + echo "Don't know how to install kernel" + exit 1 + fi ;; esac @@ -283,13 +282,13 @@ cmd_boot() cmd_oldconfig() { new_config - do_make oldconfig + do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" oldconfig } cmd_config() { new_config - do_make nconfig + do_make ARCH="$KERNEL_ARCH" CROSS_COMPILE="$ARCH_TRIPLE-" nconfig } cmd_faddr2line() diff --git a/cross.conf b/cross.conf new file mode 100644 index 00000000..952d4341 --- /dev/null +++ b/cross.conf @@ -0,0 +1,24 @@ +# this file specifies target triplets for cross-compiling +# whenever these need to be changed (some distributions prefer ARCHITECTURE-VENDOR-OS-LIBC), +# change the triplet here + +#32 bit architectures +ARCH_TRIPLE_X86=x86-linux-gnu +ARCH_TRIPLE_ARMV7=arm-linux-gnueabihf + +#64 bit architectures +ARCH_TRIPLE_X86_64=x86_64-linux-gnu +ARCH_TRIPLE_ARM64=aarch64-linux-gnu +ARCH_TRIPLE_S390X=s390x-linux-gnu + +#currently unsupported (but maintained) debian architectures +ARCH_TRIPLE_PPC64=powerpc-linux-gnu +ARCH_TRIPLE_SPARC64=sparc64-linux-gnu +ARCH_TRIPLE_RISCV64=riscv64-linux-gnu + +#here you can specify up snapshots which are used to fix sid dependencies: +#debian SID is updated frequently, having a lot of broken packages due to dependencies +#view https://snapshot.debian.org/archive/debian-ports/ for a list +#as always: the older, the more dangerous ... + +SID_SNAPSHOTS=("20220501T101242Z") diff --git a/lib/common.sh b/lib/common.sh index 77860915..e5acf143 100644 --- a/lib/common.sh +++ b/lib/common.sh @@ -3,6 +3,10 @@ set -o nounset set -o errtrace set -o pipefail +[[ -v ktest_dir ]] || ktest_dir=$(dirname ${BASH_SOURCE})/.. + +. "$ktest_dir/cross.conf" + trap 'echo "Error $? at $BASH_SOURCE $LINENO from: $BASH_COMMAND, exiting"' ERR ktest_tmp=${ktest_tmp:-""} @@ -15,6 +19,8 @@ ktest_exit() fi [[ -n $ktest_tmp ]] && rm -rf "$ktest_tmp" + #umount overlayfs of bcachefs-tools if needed + [[ -n $ktest_tmp && $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]] && umount $(grep bcachefs-tools-$ktest_arch /proc/mounts | cut -d' ' -f 2) true } @@ -76,27 +82,27 @@ join_by() echo "$*" } -ktest_arch=$(uname -m) -CROSS_COMPILE="" - parse_arch() { + CROSS_COMPILE= case $1 in - x86|i386) + x86|i386|i686) ktest_arch=x86 DEBIAN_ARCH=i386 - ARCH_TRIPLE=x86-linux-gnu + ARCH_TRIPLE=${ARCH_TRIPLE_X86} + RUST_TRIPLE=i686-unknown-linux-gnu KERNEL_ARCH=x86 BITS=32 QEMU_PACKAGE=qemu-system-x86 - QEMU_BIN=qemu-system-x86_64 + QEMU_BIN=qemu-system-i386 ;; x86_64|amd64) ktest_arch=x86_64 DEBIAN_ARCH=amd64 - ARCH_TRIPLE=x86_64-linux-gnu + ARCH_TRIPLE=${ARCH_TRIPLE_X86_64} + RUST_TRIPLE=x86_64-unknown-linux-gnu KERNEL_ARCH=x86 BITS=64 @@ -107,7 +113,8 @@ parse_arch() aarch64|arm64) ktest_arch=aarch64 DEBIAN_ARCH=arm64 - ARCH_TRIPLE=aarch64-linux-gnu + ARCH_TRIPLE=${ARCH_TRIPLE_ARM64} + RUST_TRIPLE=aarch64-unknown-linux-gnu KERNEL_ARCH=arm64 BITS=64 @@ -115,39 +122,46 @@ parse_arch() QEMU_PACKAGE=qemu-system-arm QEMU_BIN=qemu-system-aarch64 ;; - mips) - DEBIAN_ARCH=mips - ARCH_TRIPLE=mips-linux-gnu + armhf|armv7|armv7l|arm) + ktest_arch=arm + DEBIAN_ARCH=armhf + ARCH_TRIPLE=${ARCH_TRIPLE_ARMV7} + RUST_TRIPLE=armv7-unknown-linux-gnueabihf - KERNEL_ARCH=mips + KERNEL_ARCH=arm BITS=32 - QEMU_PACKAGE=qemu-system-mips - QEMU_BIN=qemu-system-mips + QEMU_PACKAGE=qemu-system-arm + QEMU_BIN=qemu-system-arm ;; - mips64) - DEBIAN_ARCH=mips - ARCH_TRIPLE=mips-linux-gnu + s390x) + DEBIAN_ARCH=s390x + ARCH_TRIPLE=${ARCH_TRIPLE_S390X} + RUST_TRIPLE=s390x-unknown-linux-gnu - KERNEL_ARCH=mips + KERNEL_ARCH=s390 BITS=64 - QEMU_PACKAGE=qemu-system-mips - QEMU_BIN=qemu-system-mips64 + QEMU_PACKAGE=qemu-system-s390x + QEMU_BIN=qemu-system-s390x ;; - sparc) - DEBIAN_ARCH=sparc - ARCH_TRIPLE=sparc64-linux-gnu + riscv64) + DEBIAN_ARCH=riscv64 + ARCH_TRIPLE=${ARCH_TRIPLE_RISCV64} + MIRROR=http://deb.debian.org/debian-ports + RUST_TRIPLE=riscv64gc-unknown-linux-gnu - KERNEL_ARCH=sparc - BITS=32 + KERNEL_ARCH=riscv + BITS=64 - QEMU_PACKAGE=qemu-system-sparc - QEMU_BIN=qemu-system-sparc + QEMU_PACKAGE=qemu-system-riscv + QEMU_BIN=qemu-system-riscv64 ;; sparc64) - DEBIAN_ARCH=sparc - ARCH_TRIPLE=sparc64-linux-gnu + DEBIAN_ARCH=sparc64 + ARCH_TRIPLE=${ARCH_TRIPLE_SPARC64} + MIRROR=http://deb.debian.org/debian-ports + RUST_TRIPLE=sparc64-unknown-linux-gnu KERNEL_ARCH=sparc BITS=64 @@ -155,23 +169,13 @@ parse_arch() QEMU_PACKAGE=qemu-system-sparc QEMU_BIN=qemu-system-sparc64 ;; - ppc|powerpc) - DEBIAN_ARCH=powerpc - MIRROR=http://deb.debian.org/debian-ports - - ARCH_TRIPLE=powerpc-linux-gnu - - KERNEL_ARCH=powerpc - BITS=32 - - QEMU_PACKAGE=qemu-system-ppc - QEMU_BIN=qemu-system-ppc - ;; - ppc64) + ppc64|powerpc) + ktest_arch=ppc64 DEBIAN_ARCH=ppc64 MIRROR=http://deb.debian.org/debian-ports - ARCH_TRIPLE=powerpc-linux-gnu + ARCH_TRIPLE=${ARCH_TRIPLE_PPC64} + RUST_TRIPLE=powerpc64-unknown-linux-gnu KERNEL_ARCH=powerpc BITS=64 @@ -186,7 +190,21 @@ parse_arch() if [[ $ktest_arch != $(uname -m) ]]; then CROSS_COMPILE=1 + else + CROSS_COMPILE= fi + #special case: x86_64 is able to run i386 code. we can use KVM. + [[ $DEBIAN_ARCH == "i386" && "$(uname -m)" == "x86_64" ]] && CROSS_COMPILE= + export DEBIAN_ARCH + export MIRROR + export ARCH_TRIPLE + export KERNEL_ARCH + export QEMU_PACKAGE + export QEMU_BIN + export ktest_arch + export BITS + export RUST_TRIPLE + export CROSS_COMPILE } find_command() { diff --git a/lib/libktest.sh b/lib/libktest.sh index 95d4b77f..9cb9aea8 100644 --- a/lib/libktest.sh +++ b/lib/libktest.sh @@ -27,6 +27,7 @@ ktest_ssh_port=0 ktest_networking=user ktest_dio=off ktest_nice=0 +ktest_arch= checkdep socat checkdep brotli @@ -85,6 +86,7 @@ parse_ktest_arg() parse_args_post() { + [[ -z ${ktest_arch} ]] && ktest_arch=$(uname -m) parse_arch "$ktest_arch" ktest_out=$(readlink -f "$ktest_out") @@ -222,7 +224,7 @@ ktest_kgdb() ktest_mon() { - exec socat UNIX-CONNECT:"$ktest_out/vm/on" STDIO + exec socat UNIX-CONNECT:"$ktest_out/vm/mon" STDIO exec nc "$ktest_out/vm/mon" } @@ -285,7 +287,6 @@ start_vm() ln -s "$ktest_tmp" "$ktest_out/vm" local kernelargs=() - case $ktest_storage_bus in virtio-blk) ktest_root_dev="/dev/vda" @@ -295,7 +296,7 @@ start_vm() ;; esac - kernelargs+=(root=$ktest_root_dev rw log_buf_len=8M) + kernelargs+=(root=$ktest_root_dev rw log_buf_len=8M rootwait) kernelargs+=(mitigations=off) kernelargs+=("ktest.dir=$ktest_dir") kernelargs+=(ktest.env=$(readlink -f "$ktest_out/vm/env")) @@ -306,26 +307,42 @@ start_vm() kernelargs+=("${ktest_kernel_append[@]}") local qemu_cmd=("$QEMU_BIN" -nodefaults -nographic) + local accel=kvm + local cputype=host + [[ -n ${CROSS_COMPILE} ]] && accel=tcg && cputype=max case $ktest_arch in x86|x86_64) - qemu_cmd+=(-cpu host -machine type=q35,accel=kvm,nvdimm=on) + qemu_cmd+=(-cpu $cputype -machine type=q35,accel=$accel,nvdimm=on) + ;; + aarch64|arm) + qemu_cmd+=(-cpu $cputype -machine type=virt,gic-version=max,accel=$accel) ;; - aarch64) - qemu_cmd+=(-cpu host -machine type=virt,gic-version=max,accel=kvm) + ppc64) + qemu_cmd+=(-machine ppce500 -cpu e6500 -accel tcg) ;; - mips) - qemu_cmd+=(-cpu 24Kf -machine malta) - ktest_cpus=1 + s390x) + qemu_cmd+=(-cpu max -machine s390-ccw-virtio -accel tcg) ;; - mips64) - qemu_cmd+=(-cpu MIPS64R2-generic -machine malta) + sparc64) + qemu_cmd+=(-machine sun4u -accel tcg) + ktest_cpus=1; #sparc64 currently supports only 1 cpu + ;; + riscv64) + qemu_cmd+=(-machine virt -cpu rv64 -accel tcg) ;; esac + #we assign 4GB of ram, but for gcc jobs on target, >512MiB / GCC job is recommended + (( $ktest_cpus > 8 )) && ktest_mem=8G + local maxmem=$(awk '/MemTotal/ {printf "%dG\n", $2/1024/1024}' /proc/meminfo 2>/dev/null) || maxmem="1T" + local memconfig="$ktest_mem,slots=8,maxmem=$maxmem" + + #do not be fancy on 32-bit hardware. if it works, it's fine + [[ $BITS == 32 ]] && memconfig="3G" && ktest_cpus=$(($ktest_cpus > 4 ? 4 : $ktest_cpus)) qemu_cmd+=( \ - -m "$ktest_mem,slots=8,maxmem=$maxmem" \ + -m "$memconfig" \ -smp "$ktest_cpus" \ -kernel "$ktest_kernel_binary/vmlinuz" \ -append "$(join_by " " ${kernelargs[@]})" \ @@ -339,6 +356,8 @@ start_vm() -virtfs local,path=/,mount_tag=host,security_model=none,multidevs=remap \ ) + + if [[ -f $ktest_kernel_binary/initramfs ]]; then qemu_cmd+=(-initrd "$ktest_kernel_binary/initramfs") fi @@ -438,7 +457,7 @@ start_vm() qemu_pmem mem-path="$file",size=$size done - ulimit -n 65535 + [ "$(ulimit)" == "unlimited" ] || ulimit -n 65535 qemu_cmd+=("${ktest_qemu_append[@]}") set +o errexit diff --git a/lib/testrunner b/lib/testrunner index acbc45f7..4eb73f2c 100755 --- a/lib/testrunner +++ b/lib/testrunner @@ -20,6 +20,10 @@ ktest_out="/host/$ktest_out" ln -sf $ktest_dir /ktest ln -sf $ktest_out /ktest-out +. /ktest/lib/common.sh + +parse_arch $(uname -m) + # Some home directories are in weird places: mkdir -p $(dirname $home) ln -sf /host/$home $home diff --git a/root_image b/root_image index ece986a6..770e719f 100755 --- a/root_image +++ b/root_image @@ -23,7 +23,8 @@ checkdep mkfs.ext4 e2fsprogs checkdep curl IMAGE_SIZE="10G" -MIRROR=https://deb.debian.org/debian/ +#http is preferred over https here: on emulated CPUs, the encryption simply isn't worth it +MIRROR=http://deb.debian.org/debian/ usage() { @@ -34,17 +35,88 @@ usage() echo echo "options:" echo " -h Display this help and exit" - echo " -a Architecture for vm image" + echo " -a Architecture for vm (x86_64,armhf,i386,aarch64,sparc64,s390x,ppc64,riscv64)" echo " -m Debian mirror" echo ' -i Image to create/update, defaults to /var/lib/ktest/root.$arch' } +intervene() +{ + local yn="N"; + echo "Installing $2 failed: both binary install and $1"; + echo "You can start a shell to check whether any dependency issues can be fixed" + echo "if you do (y), and exit the shell, it is assumed to be fixed and installing will continue" + echo "if you don't, we give up." + echo "you can also choose to skip package installation (s), though this is not recommended" + read -p "Do you wish to start a shell to investigate? (y/s/N)" yn + case $yn in + [Yy]* ) + _chroot "$MNT" /tmpbuild/executer.sh /bin/bash; + ;; + [sS]* ) + return 0; + ;; + * ) return 1;; + esac +} + +install_fix_deps() +{ + trap 'echo "Could not install packages! consider switching to a supported architecture"; umount_image; rm "$ktest_image"' EXIT + #debian has a separate dist for "experimental" / broken packages, in debian-ports it is called "unreleased". + experimental_dist="experimental" && [[ ${MIRROR} == *"debian-ports"* ]] && experimental_dist=unreleased; + SNAPSHOT_MIRROR="https://snapshot.debian.org/archive/debian" && [[ ${MIRROR} == *"debian-ports"* ]] && SNAPSHOT_MIRROR+="-ports"; + #get necessary certificates / pgp keyrings to work with unofficial and outdated snapshots + _chroot "$MNT" apt-get -qq install debian-ports-archive-keyring ca-certificates + #disable unwanted checks - we know the snapshots are not valid + echo "Acquire::Check-Valid-Until false;" > $MNT/etc/apt/apt.conf.d/10-nocheckvalid + #create a sources.list with more options to work around missing packages + echo "deb [trusted=yes] $MIRROR sid main contrib" > $MNT/etc/apt/sources.list + echo "deb-src [trusted=yes] $MIRROR sid main contrib" >> $MNT/etc/apt/sources.list + echo "deb [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list + echo "deb-src [trusted=yes] $MIRROR ${experimental_dist} main contrib" >> $MNT/etc/apt/sources.list + #add snapshots specified in cross.conf + for i in "${SID_SNAPSHOTS[@]}"; do + echo "deb [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list + echo "deb-src [trusted=yes] ${SNAPSHOT_MIRROR}/$i sid main" >> $MNT/etc/apt/sources.list + done + #when working with debian-ports, add the general sources as a last hope ... there *may* be a package compiling for this archwe need + [[ ${MIRROR} == *"debian-ports"* ]] && echo "deb-src [trusted=yes] http://deb.debian.org/debian sid main contrib" >> $MNT/etc/apt/sources.list + touch $MNT/etc/passwd + touch $MNT/etc/shadow + _chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing + _chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade + _chroot "$MNT" apt-get -qq install -f + _chroot "$MNT" apt-get -qq install build-essential + #if we can install them all at once, do so, otherwise, try one by one + _chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}" || for i in "${PACKAGES[@]}"; do + if [[ ! $(_chroot "$MNT" apt-cache show $i | grep Version) == "" ]]; then + _chroot "$MNT" apt-get -qq install --allow-unauthenticated --no-install-recommends $i && continue; + fi + #installing binary failed. Try to install from source. Do it in a tmpfs folder so the image doesn't get overwhelmed: + [[ ! -d ${MNT}/tmpbuild ]] && mkdir ${MNT}/tmpbuild; + mount -t tmpfs none ${MNT}/tmpbuild; + echo '#!/bin/bash' > "$MNT"/tmpbuild/executer.sh + echo 'cd /tmpbuild/; $@' >> "$MNT"/tmpbuild/executer.sh + chmod ago+x "$MNT"/tmpbuild/executer.sh + _chroot "$MNT" /tmpbuild/executer.sh apt-get -qq build-dep --allow-unauthenticated --no-install-recommends $i || intervene install-deps $i; + #in case the intervention also installed it, continue: + if [[ $(_chroot "$MNT" dpkg -l | grep "^ii $i") == "" ]]; then + _chroot "$MNT" /tmpbuild/executer.sh apt-get -qq -b source --allow-unauthenticated $i || intervene build $i; + [ -f "${MNT}/tmpbuild/${i}*.deb" ] && _chroot "$MNT" apt-get -qq install /tmpbuild/${i}*.deb; + fi + umount ${MNT}/tmpbuild; + rmdir ${MNT}/tmpbuild; + done +} + if [[ $# = 0 ]]; then usage exit 1 fi ktest_image="" +ktest_arch="$(uname -m)" CMD="cmd_$1" shift @@ -100,7 +172,7 @@ PACKAGES+=(libudev-dev libldap2-dev) PACKAGES+=(acct bsdextrautils xfsprogs xfslibs-dev quota libcap2-bin) PACKAGES+=(libattr1-dev libaio-dev libgdbm-dev libacl1-dev gettext) PACKAGES+=(libssl-dev libgdbm-dev libgdbm-compat-dev liburing-dev) -PACKAGES+=(duperemove thin-provisioning-tools fsverity) +PACKAGES+=(duperemove fsverity) # xfsprogs: PACKAGES+=(libinih-dev) @@ -118,13 +190,18 @@ PACKAGES+=(cryptsetup) PACKAGES+=(multipath-tools sg3-utils srptools) # ZFS support -PACKAGES+=("linux-headers-$DEBIAN_ARCH" dkms zfsutils-linux zfs-dkms) +PACKAGES+=("linux-headers-generic") +# unless no other option when cross-compiling, ignore ZFS +# DKMS needs to cross-compile the module, +# against a different kernel on a different CPUarchitecture. +# this has to cause errors +[[ -n ${CROSS_COMPILE} ]] || PACKAGES+=(dkms zfsutils-linux zfs-dkms) # suspend testing: # [[ $KERNEL_ARCH = x86 ]] && PACKAGES+=(uswsusp) EXCLUDE=(dmidecode nano rsyslog logrotate cron \ - iptables nfacct \ + iptables nfacct vim-tiny \ debconf-i18n info gnupg libpam-systemd) SYSTEMD_MASK=(dev-hvc0.device \ @@ -223,19 +300,33 @@ ZZ update_packages() { + NO_RUSTUP= # systemd... !? mkdir -p "$MNT"/run/user/0 cp /etc/resolv.conf "$MNT/etc/resolv.conf" + #don't do rustup on unsupported architectures, just try to install from debian repositories + [[ -z $(curl -qq -I "https://static.rust-lang.org/rustup/dist/${RUST_TRIPLE}/rustup-init" 2>/dev/null | grep "HTTP/2 200") ]] && NO_RUSTUP=1 + [[ -n $NO_RUSTUP ]] && PACKAGES+=(rustc rustc-dbgsym rustfmt rustfmt-dbgsym) + _chroot "$MNT" mount -t proc none /proc - _chroot "$MNT" apt-get -qq update - _chroot "$MNT" apt-get -qq upgrade - _chroot "$MNT" apt-get -qq install --no-install-recommends "${PACKAGES[@]}" + _chroot "$MNT" mount -t devpts none /dev/pts + if [[ $MIRROR == *"debian-ports"* || $1 == "sid" ]]; then + install_fix_deps + else + _chroot "$MNT" apt-get -qq --allow-unauthenticated --allow-insecure-repositories update --fix-missing + _chroot "$MNT" apt-get -qq --allow-unauthenticated upgrade + _chroot "$MNT" apt-get -qq install -f + _chroot "$MNT" apt-get -qq install -m --allow-unauthenticated --no-install-recommends "${PACKAGES[@]}" + fi rm -f "$MNT/var/cache/apt/archives/*.deb" - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh - chmod 755 "$MNT"/tmp/rustup.sh + if [[ -z $NO_RUSTUP ]]; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > "$MNT"/tmp/rustup.sh + chmod 755 "$MNT"/tmp/rustup.sh - _chroot "$MNT" /tmp/rustup.sh -y + _chroot "$MNT" /tmp/rustup.sh --default-host ${RUST_TRIPLE} -y + echo 'export PATH="$HOME/.cargo/bin:$PATH"' > $MNT/etc/profile.d/rustup.sh + fi; } trim_image() @@ -270,7 +361,7 @@ cmd_update() cp "$ktest_image" "$ktest_image".new mount "$ktest_image".new "$MNT" - update_packages + update_packages none update_files umount_image @@ -284,28 +375,55 @@ cmd_create() echo "$ktest_image already exists" exit 1 fi - (cd "$ktest_dir"; git submodule update --init debootstrap) MNT=$(mktemp --tmpdir -d $(basename "$0")-XXXXXXXXXX) - trap 'umount_image; rm "$ktest_image"' EXIT + trap '/bin/bash; umount_image; rm "$ktest_image"' EXIT fallocate -l "$IMAGE_SIZE" "$ktest_image" mkfs.ext4 -F "$ktest_image" mount "$ktest_image" "$MNT" + local debian_release="trixie" #general release + local keyring="" + + if [[ $MIRROR == *"debian-ports"* ]]; then + debian_release="sid" #unofficial ports don't have named releases + echo "" + echo "WARNING: $ktest_arch is unsupported, using SID release for packages" + echo "*******************************************************************" + echo "PLEASE NOTE: this often has dependency problems between packages, " + echo "and can prevent the install due to a dependency conflict " + echo "If so, contact the debian maintainer for this architecture to fix it:" + echo "${ktest_arch} : ${DEBIAN_ARCH}@buildd.debian.org" + echo "*******************************************************************" + echo "" + fi + + #fallback: if the architecture can't be found, try official sid: + if [[ -z $(curl -qq -I "${MIRROR}/dists/${debian_release}/main/binary-${DEBIAN_ARCH}/Release" 2>/dev/null | grep "HTTP/2 200") ]]; then + echo "WARNING: $DEBIAN_ARCH $debian_release could not be found at ${MIRROR} (where it should be). Falling back to standard sid" + MIRROR="http://deb.debian.org/debian/" + debian_release=sid + fi + DEBOOTSTRAP_DIR=$ktest_dir/debootstrap $debootstrap \ --no-check-gpg \ --arch="$DEBIAN_ARCH" \ --exclude=$(join_by , "${EXCLUDE[@]}") \ + $keyring \ --foreign \ - --components='main,contrib,non-free,bullseye-backports' \ - bullseye "$MNT" "$MIRROR" + --components='main,contrib,non-free' \ + $debian_release "$MNT" "$MIRROR" + + if [[ -n ${CROSS_COMPILE} ]]; then + statichelper=$(which qemu-${ktest_arch}) || statichelper=$(which qemu-${ktest_arch}-static) + cp ${statichelper} ${MNT}${statichelper} + fi _chroot "$MNT" /debootstrap/debootstrap --second-stage _chroot "$MNT" dpkg --configure -a - - update_packages + update_packages $debian_release update_files umount_image diff --git a/tests/bcachefs/bcachefs-test-libs.sh b/tests/bcachefs/bcachefs-test-libs.sh index 71678563..f6461b5f 100644 --- a/tests/bcachefs/bcachefs-test-libs.sh +++ b/tests/bcachefs/bcachefs-test-libs.sh @@ -5,10 +5,34 @@ # . $(dirname $(readlink -e "${BASH_SOURCE[0]}"))/../test-libs.sh +bch_loc=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-tools -require-git http://evilpiepirate.org/git/bcachefs-tools.git -require-make bcachefs-tools +declare -A ov_dirs=(["tmp"]="${bch_loc}/../.tmp-bcachefs-tools-overlay" \ + ["diff"]="${bch_loc}/../.bcachefs-tools-overlay-${ktest_arch}" \ + ["merge"]="${bch_loc}/../bcachefs-tools-${ktest_arch}") + +for dir_overlay in ${ov_dirs[@]}; do + if [[ ! -d ${dir_overlay} ]]; then mkdir -p ${dir_overlay}; fi +done + +if [[ $(which fuse-overlayfs) && ! $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]]; then + fuse-overlayfs -o lowerdir=${bch_loc},upperdir=${ov_dirs["diff"]},workdir=${ov_dirs["tmp"]} ${ov_dirs["merge"]} +elif [[ ! $(grep bcachefs-tools-$ktest_arch /proc/mounts) ]]; then + rsync -av ${bch_loc}/ ${ov_dirs["merge"]}/ +fi + +bch_loc=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))/bcachefs-tools-${ktest_arch} +if [[ ! -f "${bch_loc}/.last_arch_for_compile" || "$(cat ${bch_loc}/.last_arch_for_compile)" != ${ktest_arch} ]]; then + make -C ${bch_loc} clean >/dev/null 2>&1; + rm -rf "${bch_loc}/rust-src/target/*"; + find ${bch_loc} -name "*.o" -exec rm {} \; + find ${bch_loc} -name "*.a" -exec rm {} \; + echo $ktest_arch > ${bch_loc}/.last_arch_for_compile +fi + +require-git http://evilpiepirate.org/git/bcachefs-tools.git +require-make bcachefs-tools-${ktest_arch} require-kernel-config BCACHEFS_FS if [[ ! -v NO_BCACHEFS_DEBUG ]]; then @@ -20,8 +44,12 @@ else require-kernel-config BCACHEFS_NO_LATENCY_ACCT=y fi +if [[ ! $ktest_arch == ppc64 ]]; then + require-kernel-config TRANSPARENT_HUGEPAGE +fi + if [[ $ktest_arch = x86_64 ]]; then require-kernel-config CRYPTO_CRC32C_INTEL require-kernel-config CRYPTO_POLY1305_X86_64 diff --git a/tests/kconfig.sh b/tests/kconfig.sh index 68005fc9..1088311d 100644 --- a/tests/kconfig.sh +++ b/tests/kconfig.sh @@ -10,6 +10,7 @@ case $ktest_arch in require-kernel-config MCORE2 # optimize for core2 require-kernel-config IO_DELAY_0XED require-kernel-config 64BIT=n + require-kernel-config COMPAT_32BIT_TIME require-kernel-config ACPI # way slower without it, do not know why require-kernel-config UNWINDER_FRAME_POINTER require-kernel-config HARDLOCKUP_DETECTOR @@ -36,39 +37,80 @@ case $ktest_arch in have_virtio=1 have_suspend=1 + require-kernel-append console=hvc0 + ;; + arm) + require-kernel-config ARCH_VIRT + require-kernel-config SMP + require-kernel-config VFP + require-kernel-config NEON + require-kernel-config ARM_LPAE + require-kernel-config MMU + require-kernel-config HAVE_PCI + require-kernel-config PCI_HOST_GENERIC + require-kernel-config ARM_AMBA + require-kernel-config COMPAT_32BIT_TIME + require-kernel-config RTC_DRV_PL031 + + have_virtio=1 + require-kernel-append console=hvc0 ;; aarch64) require-kernel-config PCI_HOST_GENERIC require-kernel-config RTC_DRV_PL031 + require-kernel-config COMPAT_32BIT_TIME + require-kernel-config IOMMU_SUPPORT + require-kernel-config PARAVIRT have_virtio=1 require-kernel-append console=hvc0 ;; - powerpc) - require-kernel-config ADVANCED_OPTIONS - have_kvmguest=1 + ppc64) + require-kernel-config PPC64 + require-kernel-config PPC_BOOK3E_64 + require-kernel-config PPC_QEMU_E500 + require-kernel-config E6500_CPU + require-kernel-config SMP + require-kernel-config KVM_GUEST + have_virtio=1 - have_suspend=1 require-kernel-append console=hvc0 ;; - mips) - require-kernel-config MIPS_MALTA - require-kernel-config CPU_MIPS${BITS}_R2 - require-kernel-config CPU_BIG_ENDIAN=y - require-kernel-config CPU_LITTLE_ENDIAN=n - require-kernel-config 32BIT + sparc64) + require-kernel-config 64BIT + require-kernel-config SMP + require-kernel-config PCI + + have_virtio=1 + + require-kernel-append console=hvc0 + ;; + riscv64) + require-kernel-config SOC_VIRT + require-kernel-config VIRTIO_MENU + require-kernel-config PCI + + have_virtio=1 + + require-kernel-append console=hvc0 + ;; + s390x) + require-kernel-config S390_GUEST + require-kernel-config MARCH_Z13 + require-kernel-config CMM + require-kernel-config PCI + require-kernel-config DCSSBLK have_virtio=1 - ktest_storage_bus=piix4-ide require-kernel-append console=hvc0 ;; *) - echo "Kernel architecture not supported by kconfig.sh" + echo "Kernel architecture $ktest_arch not supported by kconfig.sh" exit 1 ;; esac @@ -93,8 +135,6 @@ require-kernel-config BINFMT_SCRIPT require-kernel-config COMPACTION # virtfs doesn't do well without it -require-kernel-config PROC_KCORE # XXX Needed? - require-kernel-config TTY require-kernel-config VT @@ -157,13 +197,15 @@ require-kernel-config PCI require-kernel-config HW_RANDOM # Clock: +if [[ ! $ktest_arch == *"s390"* ]]; then require-kernel-config RTC_CLASS require-kernel-config RTC_HCTOSYS - +fi # Console: +if [[ ! $ktest_arch == *"s390"* ]]; then require-kernel-config SERIAL_8250 # XXX can probably drop require-kernel-config SERIAL_8250_CONSOLE - +fi # Block devices: require-kernel-config SCSI require-kernel-config SCSI_LOWLEVEL # what's this for? @@ -204,8 +246,10 @@ require-kernel-config 9P_FS #fi # KGDB: +if [[ ! $ktest_arch == *"s390"* ]]; then require-kernel-config KGDB require-kernel-config KGDB_SERIAL_CONSOLE +fi require-kernel-config VMAP_STACK=n require-kernel-config RANDOMIZE_BASE=n require-kernel-config RANDOMIZE_MEMORY=n @@ -228,7 +272,9 @@ require-kernel-config FUNCTION_TRACER #require-kernel-config ENABLE_DEFAULT_TRACERS require-kernel-config PANIC_ON_OOPS +if [[ ! $ktest_arch == *"s390"* ]]; then require-kernel-config SOFTLOCKUP_DETECTOR +fi require-kernel-config DETECT_HUNG_TASK #require-kernel-config DEFAULT_HUNG_TASK_TIMEOUT=30 require-kernel-config WQ_WATCHDOG