From 61a9d64ebfe4db97cc49bd6dccef53a1cb1edb06 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sat, 8 Nov 2025 15:45:57 -0500 Subject: [PATCH 1/7] Revert "bootlinux: fix os detection for 9p build dependency installation" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I've observed that the "build kernel on a separate node" tasks are now failing when my controller is Fedora 42 and the selected target nodes are running Debian 11. Daniel has confirmed this: TASK: Install packages we care about [kci-19183690762-181-nvme] FAILED - RETRYING: [kci-19183690762-181-nvme -> localhost]: Install packages we care about (3 retries left). FAILED - RETRYING: [kci-19183690762-181-nvme -> localhost]: Install packages we care about (2 retries left). FAILED - RETRYING: [kci-19183690762-181-nvme -> localhost]: Install packages we care about (1 retries left). ⠀⠀FAILED: [kci-19183690762-181-nvme] => ['Could not detect which major revision of dnf is in use, which is required to determine module backend.', 'You should manually specify use_backend to tell the module whether to use the dnf4 or dnf5 backend})'] Error: : Task failed: Action failed: ('Could not detect which major revision of dnf is in use, which is required to determine module backend.', 'You should manually specify use_backend to tell the module whether to use the dnf4 or dnf5 backend})') Origin: /data4-xfs/gh/actions-runner/kdevops-ci-0019/_work/kdevops/kdevops/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml:8:3 We believe that commit 75f870367e16 ("bootlinux: Update grub on Debian after installing debs") is the culprit. Revert that commit so we can find a fix that doesn't break the bootlinux_builder tasks. Acked-by: Daniel Gomez Signed-off-by: Chuck Lever --- playbooks/roles/bootlinux/defaults/main.yml | 6 ------ playbooks/roles/bootlinux/tasks/build/9p.yml | 2 +- playbooks/roles/bootlinux/tasks/install-deps/main.yml | 6 +++--- .../roles/bootlinux/tasks/install-deps/redhat/main.yml | 4 ++-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/playbooks/roles/bootlinux/defaults/main.yml b/playbooks/roles/bootlinux/defaults/main.yml index 78e4e28ff..1ae70b2c1 100644 --- a/playbooks/roles/bootlinux/defaults/main.yml +++ b/playbooks/roles/bootlinux/defaults/main.yml @@ -141,9 +141,3 @@ bootlinux_fragment_x86: false bootlinux_fragment_xarray: false bootlinux_fragment_xarray_no_multi: false - -# Distro controls -distro_debian_based: false -distro_fedora: false -distro_redhat_based: false -distro_suse_based: false diff --git a/playbooks/roles/bootlinux/tasks/build/9p.yml b/playbooks/roles/bootlinux/tasks/build/9p.yml index 809d342ae..f390f028d 100644 --- a/playbooks/roles/bootlinux/tasks/build/9p.yml +++ b/playbooks/roles/bootlinux/tasks/build/9p.yml @@ -14,7 +14,7 @@ when: - target_linux_install_b4 is defined - target_linux_install_b4 - - not distro_debian_based|bool + - ansible_facts['os_family']|lower != 'debian' run_once: true delegate_to: localhost diff --git a/playbooks/roles/bootlinux/tasks/install-deps/main.yml b/playbooks/roles/bootlinux/tasks/install-deps/main.yml index 058f39268..51a207416 100644 --- a/playbooks/roles/bootlinux/tasks/install-deps/main.yml +++ b/playbooks/roles/bootlinux/tasks/install-deps/main.yml @@ -2,14 +2,14 @@ - name: Debian-specific setup ansible.builtin.import_tasks: debian/main.yml when: - - distro_debian_based|bool + - ansible_os_family == "Debian" - name: SuSE-specific setup ansible.builtin.import_tasks: suse/main.yml when: - - distro_suse_based|bool + - ansible_os_family == "Suse" - name: Red Hat-specific setup ansible.builtin.import_tasks: redhat/main.yml when: - - distro_redhat_based|bool + - ansible_os_family == "RedHat" diff --git a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml index e9a0c6e23..7f1955bf4 100644 --- a/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml +++ b/playbooks/roles/bootlinux/tasks/install-deps/redhat/main.yml @@ -3,7 +3,7 @@ ansible.builtin.include_role: name: epel-release when: - - not distro_fedora|bool + - ansible_distribution != "Fedora" - name: Install packages we care about become: true @@ -67,7 +67,7 @@ vars: packages: - btrfs-progs - when: distro_fedora|bool + when: ansible_distribution == 'Fedora' - name: Install clang become: true From 30e557e81af8c5a7cdbf3dbc20ab1127f6793c77 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 7 Nov 2025 21:10:54 -0500 Subject: [PATCH 2/7] bootlinux: Update install-deps for Debian Building bindeb-pkg requires the rsync package on some Debian systems. Enabling BTF (as the Debian 11 kernel config does) requires pahole. Signed-off-by: Chuck Lever --- playbooks/roles/bootlinux/tasks/install-deps/debian/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/playbooks/roles/bootlinux/tasks/install-deps/debian/main.yml b/playbooks/roles/bootlinux/tasks/install-deps/debian/main.yml index e59110298..7e8cdcc1e 100644 --- a/playbooks/roles/bootlinux/tasks/install-deps/debian/main.yml +++ b/playbooks/roles/bootlinux/tasks/install-deps/debian/main.yml @@ -40,4 +40,7 @@ - libncurses-dev - b4 - ccache + - rsync + - dwarves + - lz4 state: present From 8b8c70e3a5984b2122fa1f2671c8daac328fbe90 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 7 Nov 2025 14:46:27 -0500 Subject: [PATCH 3/7] bootlinux: Define artifact_paths on Debian This task got lost somewhere. Without artifact_paths, one or more subsequent tasks fail with "undefined variable" when building the Linux kernel on a separate builder node. Signed-off-by: Chuck Lever --- playbooks/roles/bootlinux/tasks/build/builder.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/playbooks/roles/bootlinux/tasks/build/builder.yml b/playbooks/roles/bootlinux/tasks/build/builder.yml index 30118b88e..79ff8c8d5 100644 --- a/playbooks/roles/bootlinux/tasks/build/builder.yml +++ b/playbooks/roles/bootlinux/tasks/build/builder.yml @@ -176,6 +176,11 @@ when: - ansible_os_family == "Debian" block: + - name: Build the list of artifacts directories on the builder + ansible.builtin.set_fact: + artifact_paths: + - "{{ target_linux_dir_path }}/.." + - name: Make the bindeb-pkg target community.general.make: chdir: "{{ target_linux_dir_path }}" From 1d06620315993f8ebf90321eb9b17ca1868ab96f Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 7 Nov 2025 22:47:21 -0500 Subject: [PATCH 4/7] bootlinux: Add Debian support for installing kernel packages packages.yml handles kernel RPM installation already. Now add similar support for installing kernel debs on Debian systems. Generated-by: Claude AI Signed-off-by: Chuck Lever --- .../bootlinux/tasks/install/packages.yml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/playbooks/roles/bootlinux/tasks/install/packages.yml b/playbooks/roles/bootlinux/tasks/install/packages.yml index 3395e2c0c..e9823d742 100644 --- a/playbooks/roles/bootlinux/tasks/install/packages.yml +++ b/playbooks/roles/bootlinux/tasks/install/packages.yml @@ -39,3 +39,45 @@ loop: "{{ kernel_packages }}" loop_control: label: "Installing {{ item }}" + changed_when: true + +- name: Install the built kernel debs on the target nodes + when: + - ansible_os_family == "Debian" + block: + - name: Find the kernel build artifacts on the control host + delegate_to: localhost + ansible.builtin.find: + paths: "{{ bootlinux_artifacts_dir }}" + patterns: "*.deb" + file_type: file + recurse: true + register: found_debs + + - name: Upload the kernel build artifacts to the target nodes + ansible.builtin.copy: + src: "{{ item.path }}" + dest: "/tmp" + mode: "u=rw,g=r,o=r" + loop: "{{ found_debs.files }}" + loop_control: + label: "Uploading {{ item.path | basename }} ..." + + - name: Add the core kernel package + ansible.builtin.set_fact: + kernel_packages: "{{ kernel_packages + ['/tmp/' + item.path | basename] }}" + when: + - "'headers' not in item.path" + loop: "{{ found_debs.files }}" + loop_control: + label: "Adding {{ item.path | basename }} ..." + + - name: Install the selected kernel build artifacts on the target nodes + become: true + become_method: ansible.builtin.sudo + ansible.builtin.command: + cmd: "dpkg -i {{ item }}" + loop: "{{ kernel_packages }}" + loop_control: + label: "Installing {{ item }}" + changed_when: true From 7d2b939cd1176b808b4650f0e54e464e26671076 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Fri, 7 Nov 2025 21:37:16 -0500 Subject: [PATCH 5/7] bootlinux: Update grub on Debian after installing debs Unlike the kernel RPMs built by bin-rpmpkg, the Debian debs do not update the grub configuration. That has to be handled explicitly. Generated-by: Claude AI Signed-off-by: Chuck Lever --- .../bootlinux/tasks/update-grub/install.yml | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/playbooks/roles/bootlinux/tasks/update-grub/install.yml b/playbooks/roles/bootlinux/tasks/update-grub/install.yml index dc280031a..fac56179f 100644 --- a/playbooks/roles/bootlinux/tasks/update-grub/install.yml +++ b/playbooks/roles/bootlinux/tasks/update-grub/install.yml @@ -212,3 +212,66 @@ when: - kernel_release_file.stat.exists + # GRUB detection logic for packaged kernels on Debian + - name: Construct the command line to determine the default boot entry (package) + tags: + - saved + ansible.builtin.set_fact: + determine_default_kernel_id: >- + awk -F\' '/menuentry / {print $2}' /boot/grub/grub.cfg | + awk '{print NR-1" ... "$0}' | + grep {{ kernelrelease }} | + head -1 | + awk '{print $1}' + when: + - kernel_release_file.stat.exists + - kernelrelease is defined + - kernelrelease != "unknown" + - ansible_facts['os_family']|lower == 'debian' + + - name: Determine the target kernel's GRUB boot entry number (package) # noqa: command-instead-of-shell + tags: + - saved + become: true + become_flags: "su - -c" + become_method: ansible.builtin.sudo + ansible.builtin.shell: + cmd: "{{ determine_default_kernel_id }}" + register: grub_boot_number_cmd + changed_when: false + when: + - kernel_release_file.stat.exists + - kernelrelease is defined + - kernelrelease != "unknown" + - determine_default_kernel_id is defined + + - name: Set the target kernel to be booted by default (package) + become: true + become_flags: "su - -c" + become_method: ansible.builtin.sudo + ansible.builtin.command: + cmd: "/usr/sbin/grub-set-default \"{{ grub_boot_number_cmd.stdout_lines.0 }}\"" + changed_when: true + tags: ["saved"] + when: + - grub_boot_number_cmd is defined + - grub_boot_number_cmd.rc is defined + - grub_boot_number_cmd.rc == 0 + - grub_boot_number_cmd.stdout is defined + - grub_boot_number_cmd.stdout != "" + + - name: Itemize kernel and GRUB entry selected (package) + ansible.builtin.debug: + msg: >- + {{ target_kernel }} determined to be {{ grub_boot_number_cmd.stdout_lines.0 }} + on the GRUB2 flat menu. + Ran: grub-set-default {{ grub_boot_number_cmd.stdout_lines.0 }} + vars: + target_kernel: "{{ kernelrelease }}" + tags: ["saved"] + when: + - grub_boot_number_cmd is defined + - grub_boot_number_cmd.rc is defined + - grub_boot_number_cmd.rc == 0 + - grub_boot_number_cmd.stdout is defined + - grub_boot_number_cmd.stdout != "" From 2d47a773532d9a5aa15223d3b08b878796722d14 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 9 Nov 2025 16:18:19 -0500 Subject: [PATCH 6/7] bootlinux: Ensure nfsd host gets an updated Linux kernel Commit 434df33309e7 ("workflows: linux: remove explicit inventory in ansible wrappers") added "--limit 'baseline:dev' to the bootlinux Makefile invocation to exclude localhost from the list of hosts that get a kernel update. This, however, excludes the nfsd host. When it is part of a test workflow, NFSD needs to get the test kernel too. I tried adding nfsd to [baseline], but then "make nfstest" set up the test programs on the nfsd host as well as the clients. Fixes: 434df33309e7 ("workflows: linux: remove explicit inventory in ansible wrappers") Signed-off-by: Chuck Lever --- workflows/linux/Makefile | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/workflows/linux/Makefile b/workflows/linux/Makefile index 23a599681..5cbf92ca8 100644 --- a/workflows/linux/Makefile +++ b/workflows/linux/Makefile @@ -69,6 +69,16 @@ endif BOOTLINUX_ARGS += bootlinux_cxl_test=$(CONFIG_ENABLE_CXL_TEST) WORKFLOW_ARGS += $(BOOTLINUX_ARGS) +# Default host limit for bootlinux operations +BOOTLINUX_LIMIT := baseline:dev + +# When NFS server is enabled, include it in kernel updates so that workflows +# like fstests, gitr, nfstest, and pynfs can test the kernel's NFS server +# implementation (knfsd). +ifeq (y,$(CONFIG_KDEVOPS_SETUP_NFSD)) +BOOTLINUX_LIMIT := baseline:dev:nfsd +endif + PHONY += linux-help-menu linux-help-menu: @echo "Linux git kernel development options" @@ -102,14 +112,14 @@ linux: linux-baseline linux-dev else linux: $(KDEVOPS_NODES) $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) endif else linux: $(KDEVOPS_NODES) $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) endif @@ -143,7 +153,7 @@ endif PHONY += linux-mount linux-mount: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags vars,9p_mount \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) @@ -151,7 +161,7 @@ linux-mount: PHONY += linux-deploy linux-deploy: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags vars,build-linux,install-linux,manual-update-grub,saved,vars,reboot \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) @@ -159,7 +169,7 @@ linux-deploy: PHONY += linux-build linux-build: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags vars,build-linux,saved,vars \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) @@ -167,7 +177,7 @@ linux-build: PHONY += linux-install linux-install: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags vars,build-linux,install-linux \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) @@ -175,7 +185,7 @@ linux-install: PHONY += linux-uninstall linux-uninstall: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags uninstall-linux,vars \ --extra-vars '{ "uninstall_kernel_enable": "True", $(LINUX_DYNAMIC_RUNTIME_VARS) }' \ @@ -183,7 +193,7 @@ linux-uninstall: linux-clone-clients: $(KDEVOPS_NODES) $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) \ --tags vars,clone @@ -202,20 +212,20 @@ linux-clone: $(KDEVOPS_NODES) $(LINUX_CLONE_DEFAULT_TYPE) PHONY += linux-grub-setup linux-grub-setup: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) --tags manual-update-grub,saved,vars PHONY += linux-reboot linux-reboot: $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) --tags vars,reboot PHONY += uname uname: - $(Q)ansible 'baseline:dev' -b -m command -a "uname -r" -o \ + $(Q)ansible '$(BOOTLINUX_LIMIT)' -b -m command -a "uname -r" -o \ | awk -F '|' '{gsub(/^ +| +$$/, "", $$2); printf "%-30s %s\n", $$1, $$4}' \ | sed -e 's|(stdout)||' @@ -223,7 +233,7 @@ ifeq (y,$(CONFIG_KDEVOPS_WORKFLOW_ENABLE_CXL)) PHONY += linux-cxl linux-cxl: $(KDEVOPS_NODES) $(Q)ansible-playbook $(ANSIBLE_VERBOSE) \ - --limit 'baseline:dev' \ + --limit '$(BOOTLINUX_LIMIT)' \ $(KDEVOPS_PLAYBOOKS_DIR)/bootlinux.yml \ --tags 'vars,cxl-build,cxl-install' \ --extra-vars="$(BOOTLINUX_ARGS)" $(LIMIT_HOSTS) From bef466104f515ba23acdefdf3b75907e2e1d5d03 Mon Sep 17 00:00:00 2001 From: Chuck Lever Date: Sun, 9 Nov 2025 11:34:46 -0500 Subject: [PATCH 7/7] pynfs: Fix the xdrlib dependency for Debian When running "make pynfs" in the current directory, I get to the "Install pynfs build dependencies" task, which fails with the message "No package matching 'python3-standard-xdrlib' is available". The test system is running Debian 11. Python 3.13 removed xdrlib (PEP 594), and we recently adjusted install-deps/redhat/main.yml to accommodate that. I found that the xdrlib module is already part of Python 3.9's standard library on Debian 11. The package python3-standard-xdrlib doesn't exist on Debian 11 because xdrlib is included by default. Remove the python3-standard-xdrlib package and adopt the solution used by install-deps/redhat/main.yml, which is to pull in xdrlib3 when xdrlib is not included in the standard Python library (this should be only on Debian 13). Signed-off-by: Chuck Lever --- .../pynfs/tasks/install-deps/debian/main.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/pynfs/tasks/install-deps/debian/main.yml b/playbooks/roles/pynfs/tasks/install-deps/debian/main.yml index f0e7f9c61..23ae6eb0c 100644 --- a/playbooks/roles/pynfs/tasks/install-deps/debian/main.yml +++ b/playbooks/roles/pynfs/tasks/install-deps/debian/main.yml @@ -1,7 +1,7 @@ --- - name: Install pynfs build dependencies become: true - become_method: sudo + become_method: ansible.builtin.sudo ansible.builtin.apt: name: - gcc @@ -13,7 +13,18 @@ - swig - python3-gssapi - python3-ply - - python3-standard-xdrlib state: present update_cache: true tags: ["pynfs", "deps"] + +# xdrlib was removed from Python 3.13; Debian packages it as python3-mda-xdrlib +- name: Install xdrlib from Debian package on Debian 13+ + become: true + become_method: ansible.builtin.sudo + ansible.builtin.apt: + name: + - python3-mda-xdrlib + state: present + tags: ["pynfs", "deps"] + when: + - ansible_distribution_major_version | int >= 13