diff --git a/Kconfig b/Kconfig index 357341be1..988782a9d 100644 --- a/Kconfig +++ b/Kconfig @@ -30,6 +30,7 @@ config NEEDS_LOCAL_DEVELOPMENT_PATH config KDEVOPS_FIRST_RUN bool "Is this your first time running kdevops on this system?" + output yaml default n help When you run kdevops for the first time we can enable options diff --git a/empty b/empty new file mode 100644 index 000000000..c6cac6926 --- /dev/null +++ b/empty @@ -0,0 +1 @@ +empty diff --git a/kconfigs/Kconfig.guestfs b/kconfigs/Kconfig.guestfs index f618fc30f..d309436fa 100644 --- a/kconfigs/Kconfig.guestfs +++ b/kconfigs/Kconfig.guestfs @@ -1,5 +1,20 @@ if GUESTFS +config STORAGE_POOL_PATH + string + output yaml + default LIBVIRT_STORAGE_POOL_PATH + +config GUESTFS_STORAGE_DIR + string + output yaml + default "{{ kdevops_storage_pool_path }}/kdevops/guestfs" + +config GUESTFS_BASE_IMAGE_DIR + string + output yaml + default "{{ guestfs_storage_dir }}/base_images" + config GUESTFS_HAS_CUSTOM_RAW_IMAGE bool diff --git a/kconfigs/Kconfig.libvirt b/kconfigs/Kconfig.libvirt index 1ed967423..cba8abf1e 100644 --- a/kconfigs/Kconfig.libvirt +++ b/kconfigs/Kconfig.libvirt @@ -155,13 +155,20 @@ config LIBVIRT_QEMU_GROUP default "qemu" if !DISTRO_DEBIAN && !DISTRO_UBUNTU default "libvirt-qemu" if DISTRO_DEBIAN || DISTRO_UBUNTU -config KDEVOPS_STORAGE_POOL_PATH + +config LIBVIRT_STORAGE_POOL_PATH string + output yaml default LIBVIRT_STORAGE_POOL_PATH_AUTO if LIBVIRT && !LIBVIRT_STORAGE_POOL_PATH_CUSTOM_MANUAL default LIBVIRT_STORAGE_POOL_PATH_AUTO if LIBVIRT && LIBVIRT_STORAGE_POOL_PATH_CUSTOM_CWD default LIBVIRT_STORAGE_POOL_PATH_CUSTOM if LIBVIRT && LIBVIRT_STORAGE_POOL_PATH_CUSTOM_MANUAL default VIRTUALBOX_STORAGE_POOL_PATH_CUSTOM if VAGRANT_VIRTUALBOX +config KDEVOPS_STORAGE_POOL_PATH + string + output yaml + default "{{ libvirt_storage_pool_path }}/kdevops" + config QEMU_BIN_PATH string default QEMU_BIN_PATH_LIBVIRT if LIBVIRT @@ -1052,6 +1059,7 @@ endif config LIBVIRT_STORAGE_POOL_CREATE bool "Should we build a custom storage pool for you?" + output yaml default n if !LIBVIRT_STORAGE_POOL_PATH_INFER_ADVANCED default $(shell, ./scripts/get_libvirsh_pool_enabled.sh) if LIBVIRT_STORAGE_POOL_PATH_INFER_ADVANCED help @@ -1063,6 +1071,7 @@ config LIBVIRT_STORAGE_POOL_CREATE config LIBVIRT_STORAGE_POOL_NAME string "Libvirt storage pool name" + output yaml depends on LIBVIRT_STORAGE_POOL_CREATE default "default" if !LIBVIRT_STORAGE_POOL_PATH_INFER_ADVANCED default $(shell, ./scripts/get_libvirsh_pool_name.sh) if LIBVIRT_STORAGE_POOL_PATH_INFER_ADVANCED diff --git a/playbooks/roles/bringup_guestfs/tasks/main.yml b/playbooks/roles/bringup_guestfs/tasks/main.yml index dcbbaef02..947d7dbc0 100644 --- a/playbooks/roles/bringup_guestfs/tasks/main.yml +++ b/playbooks/roles/bringup_guestfs/tasks/main.yml @@ -42,6 +42,65 @@ when: guestfs_subdirectories.matched == 0 tags: [ 'config-check' ] +- name: Create kdevops guestfs storage directory if missing (libvirt session uri) + file: + path: "{{ guestfs_base_image_dir }}" + state: directory + mode: '0755' + tags: ['storage-pool-path'] + when: + - 'not libvirt_uri_system|bool' + +- name: Create kdevops guestfs storage directory if missing (libvirt system uri) + become: yes + become_flags: 'su - -c' + become_method: sudo + file: + path: "{{ guestfs_base_image_dir }}" + state: directory + mode: '0775' + group: "{{ libvirt_qemu_group }}" + tags: ['storage-pool-path'] + when: + - 'libvirt_uri_system|bool' + +- name: Check if directory is owned by the correct group (libvirt system uri) + become: yes + become_flags: 'su - -c' + become_method: sudo + command: stat -c '%G' "{{ libvirt_storage_pool_path }}" + register: dir_group + changed_when: false + tags: ['storage-pool-path'] + when: + - 'libvirt_uri_system|bool' + +- name: Check if directory has group write permissions (libvirt system uri) + become: yes + become_flags: 'su - -c' + become_method: sudo + command: stat -c '%A' "{{ libvirt_storage_pool_path }}" + register: dir_perms + changed_when: false + tags: ['storage-pool-path'] + when: + - 'libvirt_uri_system|bool' + +- name: Verify storage pool path directory is group-writable (libvirt system uri) + become: yes + become_flags: 'su - -c' + become_method: sudo + fail: + msg: | + The permissions for {{ libvirt_storage_pool_path }} should be group + writeable by the group used by libvirt: {{ libvirt_qemu_group }} + Current group: {{ dir_group.stdout }} + Current permissions: {{ dir_perms.stdout }} + tags: ['storage-pool-path'] + when: + - 'libvirt_uri_system|bool' + - (dir_group.stdout != libvirt_qemu_group) or (dir_perms.stdout[5] != 'w') + - name: Check for dnsmasq configuration files stat: path: "{{ item }}" diff --git a/playbooks/roles/devconfig/tasks/install-deps/debian/main.yml b/playbooks/roles/devconfig/tasks/install-deps/debian/main.yml index 5624a9f84..954f0aede 100644 --- a/playbooks/roles/devconfig/tasks/install-deps/debian/main.yml +++ b/playbooks/roles/devconfig/tasks/install-deps/debian/main.yml @@ -1,4 +1,26 @@ --- +- name: Check if unattended-upgrades is installed + command: dpkg-query -W -f='${Status}' unattended-upgrades + register: unattended_upgrade_status + ignore_errors: true + changed_when: false + +- name: Set fact if unattended-upgrades is installed + set_fact: + unattended_upgrades_installed: "{{ 'install ok installed' in unattended_upgrade_status.stdout }}" + +- name: Verify unattended-upgrades is not installed + fail: + msg: | + The unattended-upgrades package is installed on the base image, this + can cause tons of issues with CIs. Fix this by running the following + commands: + + make cleancache + make bringup + when: + - unattended_upgrades_installed|bool + - name: Upgrade Packages become: yes become_method: sudo diff --git a/playbooks/roles/gen_nodes/tasks/main.yml b/playbooks/roles/gen_nodes/tasks/main.yml index d541dcbf1..8c6a1f705 100644 --- a/playbooks/roles/gen_nodes/tasks/main.yml +++ b/playbooks/roles/gen_nodes/tasks/main.yml @@ -18,6 +18,11 @@ command: "id -g -n" register: my_group +- name: Fail if kdevops_host_prefix contains a dash + fail: + msg: "Invalid kdevops_host_prefix '{{ kdevops_host_prefix }}'. The prefix cannot contain a dash ('-')." + when: kdevops_host_prefix is search("-") + - name: Create guestfs directory ansible.builtin.file: path: "{{ guestfs_path }}" diff --git a/scripts/bringup_guestfs.sh b/scripts/bringup_guestfs.sh index 716684230..bc0176f8f 100755 --- a/scripts/bringup_guestfs.sh +++ b/scripts/bringup_guestfs.sh @@ -14,7 +14,7 @@ IMG_FMT="qcow2" if [ "${CONFIG_LIBVIRT_EXTRA_DRIVE_FORMAT_RAW}" = "y" ]; then IMG_FMT="raw" fi -STORAGETOPDIR="${CONFIG_KDEVOPS_STORAGE_POOL_PATH}" +STORAGETOPDIR="${CONFIG_LIBVIRT_STORAGE_POOL_PATH}" STORAGEDIR="${STORAGETOPDIR}/kdevops/guestfs" QEMU_GROUP=$CONFIG_LIBVIRT_QEMU_GROUP GUESTFSDIR="${TOPDIR}/guestfs" @@ -223,6 +223,7 @@ firstboot-command systemctl stop ssh firstboot-command DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true dpkg-reconfigure -p low --force openssh-server firstboot-command systemctl start ssh firstboot-command apt update && apt upgrade --yes +uninstall unattended-upgrades _EOT # CONFIG_GUESTFS_COPY_SOURCES_FROM_HOST_TO_GUEST will not work # if /etc/nsswitch.conf has a line like this: @@ -270,9 +271,6 @@ if [[ "$CONFIG_LIBVIRT_URI_SYSTEM" == "y" ]]; then USE_SUDO="sudo " fi -$USE_SUDO mkdir -p $STORAGEDIR -$USE_SUDO mkdir -p $BASE_IMAGE_DIR - cmdfile=$(mktemp) if [ ! -f $BASE_IMAGE ]; then diff --git a/scripts/bringup_vagrant.sh b/scripts/bringup_vagrant.sh index 4d30c2312..4e163871d 100755 --- a/scripts/bringup_vagrant.sh +++ b/scripts/bringup_vagrant.sh @@ -55,7 +55,7 @@ vagrant_check_dups() # instances *and* we know one does not exist in another # directory for this user. - kdevops_pool_path="$CONFIG_KDEVOPS_STORAGE_POOL_PATH" + kdevops_pool_path="$CONFIG_LIBVIRT_STORAGE_POOL_PATH" # For libvirt we can do one more global sanity check if [[ "$CONFIG_LIBVIRT" == "y" ]]; then possible_image="${kdevops_pool_path}/vagrant_${instance}.img" diff --git a/scripts/destroy_guestfs.sh b/scripts/destroy_guestfs.sh index 58dca78d8..ee5dc2b57 100755 --- a/scripts/destroy_guestfs.sh +++ b/scripts/destroy_guestfs.sh @@ -7,7 +7,7 @@ source ${TOPDIR}/scripts/lib.sh export LIBVIRT_DEFAULT_URI=$CONFIG_LIBVIRT_URI -STORAGEDIR="${CONFIG_KDEVOPS_STORAGE_POOL_PATH}/kdevops/guestfs" +STORAGEDIR="${CONFIG_LIBVIRT_STORAGE_POOL_PATH}/kdevops/guestfs" GUESTFSDIR="${TOPDIR}/guestfs" if [ -f "$GUESTFSDIR/kdevops_nodes.yaml" ]; then diff --git a/scripts/destroy_vagrant.sh b/scripts/destroy_vagrant.sh index 4e5bb9d64..bd5e43d00 100755 --- a/scripts/destroy_vagrant.sh +++ b/scripts/destroy_vagrant.sh @@ -18,7 +18,7 @@ rm -rf .vagrant # doing so we don't check for global dups or anything like that. UNINIT_CURRENT_INSTANCES=$(vagrant status --machine-readable | grep ",state," | grep not_created | awk -F "," '{print $2}') for i in $UNINIT_CURRENT_INSTANCES; do - UNINIT_INSTANCE_SPARE_DRIVE_DIR="${CONFIG_KDEVOPS_STORAGE_POOL_PATH}/kdevops/$i" + UNINIT_INSTANCE_SPARE_DRIVE_DIR="${CONFIG_LIBVIRT_STORAGE_POOL_PATH}/kdevops/$i" if [[ -d $UNINIT_INSTANCE_SPARE_DRIVE_DIR ]]; then echo "Found unitialized (possibly old) instance spare drive directory, removing it ... $i" rm -rf $UNINIT_INSTANCE_SPARE_DRIVE_DIR diff --git a/scripts/gen-nodes.Makefile b/scripts/gen-nodes.Makefile index 8bee2db57..775ec5c49 100644 --- a/scripts/gen-nodes.Makefile +++ b/scripts/gen-nodes.Makefile @@ -42,12 +42,6 @@ GEN_NODES_EXTRA_ARGS += libvirt_session_management_network_device='$(subst ",,$( GEN_NODES_EXTRA_ARGS += libvirt_session_public_network_dev='$(subst ",,$(CONFIG_LIBVIRT_SESSION_PUBLIC_NETWORK_DEV))' endif -ifeq (y,$(CONFIG_LIBVIRT_STORAGE_POOL_CREATE)) -GEN_NODES_EXTRA_ARGS += libvirt_storage_pool_create='True' -GEN_NODES_EXTRA_ARGS += libvirt_storage_pool_name='$(subst ",,$(CONFIG_LIBVIRT_STORAGE_POOL_NAME))' -GEN_NODES_EXTRA_ARGS += libvirt_storage_pool_path='$(subst ",,$(CONFIG_KDEVOPS_STORAGE_POOL_PATH))' -endif - GEN_NODES_EXTRA_ARGS += libvirt_extra_storage_aio_mode='$(subst ",,$(CONFIG_LIBVIRT_AIO_MODE))' GEN_NODES_EXTRA_ARGS += libvirt_extra_storage_aio_cache_mode='$(subst ",,$(CONFIG_LIBVIRT_AIO_CACHE_MODE))' diff --git a/scripts/guestfs.Makefile b/scripts/guestfs.Makefile index 8b1b4e9fc..e1cf25d62 100644 --- a/scripts/guestfs.Makefile +++ b/scripts/guestfs.Makefile @@ -18,11 +18,6 @@ QEMU_GROUP:=$(subst ",,$(CONFIG_LIBVIRT_QEMU_GROUP)) GUESTFS_ARGS += kdevops_storage_pool_group='$(QEMU_GROUP)' GUESTFS_ARGS += storage_pool_group='$(QEMU_GROUP)' -STORAGE_POOL_PATH:=$(subst ",,$(CONFIG_KDEVOPS_STORAGE_POOL_PATH)) -KDEVOPS_STORAGE_POOL_PATH:=$(STORAGE_POOL_PATH)/kdevops -GUESTFS_ARGS += storage_pool_path=$(STORAGE_POOL_PATH) -GUESTFS_ARGS += kdevops_storage_pool_path=$(KDEVOPS_STORAGE_POOL_PATH) - 9P_HOST_CLONE := ifeq (y,$(CONFIG_BOOTLINUX_9P)) 9P_HOST_CLONE := 9p_linux_clone @@ -88,7 +83,7 @@ bringup_guestfs: $(GUESTFS_BRINGUP_DEPS) playbooks/bringup_guestfs.yml \ -e 'ansible_python_interpreter=/usr/bin/python3' \ --extra-vars=@./extra_vars.yaml \ - --tags config-check,network + --tags config-check,network,storage-pool-path $(Q)$(TOPDIR)/scripts/bringup_guestfs.sh $(Q)ansible-playbook $(ANSIBLE_VERBOSE) --connection=local \ --inventory localhost, \ @@ -109,4 +104,4 @@ destroy_guestfs: PHONY += destroy_guestfs cleancache: - $(Q)rm -f $(subst ",,$(CONFIG_KDEVOPS_STORAGE_POOL_PATH))/kdevops/guestfs/base_images/* + $(Q)rm -f $(subst ",,$(CONFIG_LIBVIRT_STORAGE_POOL_PATH))/kdevops/guestfs/base_images/* diff --git a/scripts/prune_stale_vagrant.sh b/scripts/prune_stale_vagrant.sh index 95b88911c..61ac0e86b 100755 --- a/scripts/prune_stale_vagrant.sh +++ b/scripts/prune_stale_vagrant.sh @@ -37,7 +37,7 @@ if [[ "$CONFIG_LIBVIRT" != "y" ]]; then fi if [[ $# -eq 0 ]]; then - KDEVOPS_POOL_PATH="$CONFIG_KDEVOPS_STORAGE_POOL_PATH" + KDEVOPS_POOL_PATH="$CONFIG_LIBVIRT_STORAGE_POOL_PATH" elif [[ $# -eq 1 ]]; then if [[ "$1" == "--help" ]]; then echo "Usage: $0" diff --git a/scripts/vagrant.Makefile b/scripts/vagrant.Makefile index 368c2f84a..664c8f9cf 100644 --- a/scripts/vagrant.Makefile +++ b/scripts/vagrant.Makefile @@ -44,11 +44,6 @@ ifeq (y,$(CONFIG_VAGRANT_VIRTUALBOX)) VAGRANT_ARGS += virtualbox_provider=True endif -STORAGE_POOL_PATH:=$(subst ",,$(CONFIG_KDEVOPS_STORAGE_POOL_PATH)) -KDEVOPS_STORAGE_POOL_PATH:=$(STORAGE_POOL_PATH)/kdevops -VAGRANT_ARGS += storage_pool_path=$(STORAGE_POOL_PATH) -VAGRANT_ARGS += kdevops_storage_pool_path=$(KDEVOPS_STORAGE_POOL_PATH) - VAGRANT_9P_HOST_CLONE := ifeq (y,$(CONFIG_BOOTLINUX_9P)) VAGRANT_9P_HOST_CLONE := vagrant_9p_linux_clone