From 494d92e53c9e0812c4666cf02e97090e2335e7e1 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Tue, 25 Nov 2025 09:28:34 +0100 Subject: [PATCH 01/15] incusd/storage/drivers: Add utils for qcow2 manipulation Signed-off-by: Piotr Resztak --- .../server/storage/drivers/utils_qcow2.go | 295 ++++++++++++++++++ 1 file changed, 295 insertions(+) create mode 100644 internal/server/storage/drivers/utils_qcow2.go diff --git a/internal/server/storage/drivers/utils_qcow2.go b/internal/server/storage/drivers/utils_qcow2.go new file mode 100644 index 00000000000..7c9421d0ecf --- /dev/null +++ b/internal/server/storage/drivers/utils_qcow2.go @@ -0,0 +1,295 @@ +package drivers + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/lxc/incus/v6/internal/linux" + "github.com/lxc/incus/v6/internal/server/operations" + internalUtil "github.com/lxc/incus/v6/internal/util" + "github.com/lxc/incus/v6/shared/api" + "github.com/lxc/incus/v6/shared/subprocess" +) + +// Type of the block volume. +const ( + BlockVolumeTypeRaw = "raw" + BlockVolumeTypeQcow2 = "qcow2" +) + +// ImageInfo contains information about a qcow2 image. +type ImageInfo struct { + BackingFilename string `json:"backing-filename"` + Format string `json:"format"` + VirtualSize int `json:"virtual-size"` +} + +// Qcow2Create creates a qcow2-formatted image. +func Qcow2Create(path string, backingPath string, size int64) error { + args := []string{ + "create", + "-f", + "qcow2", + } + + if backingPath != "" { + args = append(args, "-b", backingPath) + args = append(args, "-F", "qcow2") + } + + args = append(args, path) + + if size > 0 { + args = append(args, fmt.Sprintf("%db", size)) + } + + _, err := subprocess.RunCommand("qemu-img", args...) + if err != nil { + return err + } + + return nil +} + +// Qcow2Rebase changes the backing file of a qcow2 image. +func Qcow2Rebase(path string, backingPath string) error { + _, err := subprocess.RunCommand("qemu-img", "rebase", "-u", "-b", backingPath, "-F", "qcow2", path) + if err != nil { + return err + } + + return nil +} + +// Qcow2Convert creates an image using the contents of backingPath and path. +func Qcow2Convert(path string, backingPath string, outputPath string) error { + args := []string{ + "convert", + "-O", + "qcow2", + } + + if backingPath != "" { + args = append(args, "-B", backingPath) + args = append(args, "-F", "qcow2") + } + + args = append(args, path, outputPath) + + _, err := subprocess.RunCommand("qemu-img", args...) + if err != nil { + return err + } + + return nil +} + +// Qcow2Info returns information about a qcow2 image. +func Qcow2Info(path string) (*ImageInfo, error) { + imgJSON, err := subprocess.RunCommand("qemu-img", "info", "--output=json", path) + if err != nil { + return nil, err + } + + imgInfo := ImageInfo{} + + err = json.Unmarshal([]byte(imgJSON), &imgInfo) + if err != nil { + return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) + } + + return &imgInfo, nil +} + +// Qcow2BackingChain returns information about the backing chain of a qcow2 image. +func Qcow2BackingChain(path string) ([]string, error) { + result := []string{} + imgJSON, err := subprocess.RunCommand("qemu-img", "info", "--backing-chain", "--output=json", path) + if err != nil { + return nil, err + } + + imgInfo := []struct { + BackingFilename string `json:"backing-filename"` + }{} + + err = json.Unmarshal([]byte(imgJSON), &imgInfo) + if err != nil { + return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) + } + + for _, info := range imgInfo { + if info.BackingFilename == "" { + break + } + + result = append(result, info.BackingFilename) + } + + return result, nil +} + +// Qcow2MountConfigTask mounts the config filesystem volume with its snapshots and performs the task specified by the parameter. +func Qcow2MountConfigTask(vol Volume, op *operations.Operation, task func(mountPath string) error) error { + mountPath := fmt.Sprintf("%s%s", vol.MountPath(), tmpVolSuffix) + mountVol := NewVolume(vol.driver, vol.driver.Name(), vol.volType, vol.contentType, vol.name, vol.config, vol.poolConfig) + mountVol.mountFullFilesystem = true + mountVol.mountCustomPath = mountPath + wasMounted := linux.IsMountPoint(mountPath) + + err := mountVol.MountTask(func(mountPath string, op *operations.Operation) error { + taskErr := task(mountVol.MountPath()) + + // Return task error if failed. + if taskErr != nil { + return taskErr + } + + return nil + }, op) + if err != nil { + return err + } + + // MountTask delegates unmounting to UnmountVolume(), which calculates the + // refCount based on the volume name and type. Since a volume can be mounted + // at multiple paths, it is only unmounted when the refCount drops to zero. + // In this case, we unmount from customPath if the mount is no longer needed. + if !wasMounted && linux.IsMountPoint(mountPath) { + err = TryUnmount(mountPath, 0) + if err != nil { + return fmt.Errorf("Failed to unmount logical volume: %w", err) + } + } + + // Remove temporary mount path. + isEmpty, err := internalUtil.PathIsEmpty(mountPath) + if err != nil { + return err + } + + if isEmpty { + err := os.Remove(mountPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) + } + } + + return nil +} + +// Qcow2CreateConfig creates the btrfs config filesystem associated with the QCOW2 block volume. +func Qcow2CreateConfig(vol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + volPath := filepath.Join(mountPath, vol.Name()) + // Create the volume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "create", volPath) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2CreateConfigSnapshot creates the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2CreateConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + dstPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + + _, err := subprocess.RunCommand("btrfs", "subvolume", "snapshot", filepath.Join(mountPath, vol.Name()), dstPath) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2RestoreConfigSnapshot restores the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2RestoreConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + snapPath := fmt.Sprintf("%s-%s", parent, snapName) + + // Delete the subvolume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", filepath.Join(mountPath, parent)) + if err != nil { + return err + } + + _, err = subprocess.RunCommand("btrfs", "subvolume", "snapshot", filepath.Join(mountPath, snapPath), filepath.Join(mountPath, parent)) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2RenameConfigSnapshot renames the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2RenameConfigSnapshot(vol Volume, snapVol Volume, newName string, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + oldPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + newPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, newName)) + + err := os.Rename(oldPath, newPath) + if err != nil { + return fmt.Errorf("Failed to rename %q to %q: %w", oldPath, newPath, err) + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2DeleteConfigSnapshot deletes the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2DeleteConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + path := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + + // Delete the subvolume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", path) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// isQcow2Block checks whether a volume is a QCOW2 block device. +func isQcow2Block(vol Volume) bool { + return vol.Config()["block.type"] == BlockVolumeTypeQcow2 && vol.ContentType() == ContentTypeBlock +} From 1841449a63d52c65aa8c22aa197f9845d61c82cc Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Tue, 28 Oct 2025 14:41:52 +0100 Subject: [PATCH 02/15] incusd/storage/drivers: Add 'block.type' config and additional validation checks * Implement a new block.type read-only volume option which would support raw or qcow2 * Disallow snapshot creation on lvmcluster volumes that have the 'security.shared' property set to true * Change lvmcluster so that all new instance VM volumes get the qcow2 block.type * Prevent the creation of snapshots on lvmcluster volumes of type VM or custom (block) that don't have the qcow2 'block.type' Signed-off-by: Piotr Resztak --- internal/server/storage/drivers/driver_lvm.go | 5 +++ .../storage/drivers/driver_lvm_volumes.go | 35 +++++++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm.go b/internal/server/storage/drivers/driver_lvm.go index 250b3d30724..95d868cc2fb 100644 --- a/internal/server/storage/drivers/driver_lvm.go +++ b/internal/server/storage/drivers/driver_lvm.go @@ -725,6 +725,11 @@ func (d *lvm) Update(changedConfig map[string]string) error { return errors.New("volume.lvm.stripes.size cannot be changed when using thin pool") } + _, changed = changedConfig["volume.block.type"] + if changed { + return errors.New("volume.block.type cannot be changed after creation") + } + if changedConfig["lvm.vg_name"] != "" { _, err := subprocess.TryRunCommand("vgrename", d.config["lvm.vg_name"], changedConfig["lvm.vg_name"]) if err != nil { diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index dd29229394b..331b17669fd 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -295,6 +295,13 @@ func (d *lvm) FillVolumeConfig(vol Volume) error { } } + if d.clustered && vol.ContentType() == ContentTypeBlock && vol.Type() == VolumeTypeVM { + if vol.config["block.type"] == "" { + // Unchangeable volume property: Set unconditionally. + vol.config["block.type"] = BlockVolumeTypeQcow2 + } + } + // Inherit stripe settings from pool if not set and not using thin pool. if !d.usesThinpool() { if vol.config["lvm.stripes"] == "" { @@ -311,7 +318,7 @@ func (d *lvm) FillVolumeConfig(vol Volume) error { // commonVolumeRules returns validation rules which are common for pool and volume. func (d *lvm) commonVolumeRules() map[string]func(value string) error { - return map[string]func(value string) error{ + rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_lvm, group=common, key=block.mount_options) // // --- @@ -348,6 +355,19 @@ func (d *lvm) commonVolumeRules() map[string]func(value string) error { // shortdesc: Size of stripes to use (at least 4096 bytes and multiple of 512 bytes) "lvm.stripes.size": validate.Optional(validate.IsSize), } + + if d.clustered { + // gendoc:generate(entity=storage_lvm, group=common, key=block.type) + // + // --- + // type:string + // condition: block-based volume + // default: same as `volume.block.type` + // shortdesc: Type of the block volume + rules["block.type"] = validate.Optional(validate.IsOneOf(BlockVolumeTypeRaw, BlockVolumeTypeQcow2)) + } + + return rules } // ValidateVolume validates the supplied volume config. @@ -495,6 +515,11 @@ func (d *lvm) UpdateVolume(vol Volume, changedConfig map[string]string) error { return errors.New("lvm.stripes.size cannot be changed") } + _, changed = changedConfig["block.type"] + if changed { + return errors.New("block.type cannot be changed after creation") + } + return nil } @@ -1133,7 +1158,13 @@ func (d *lvm) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, _ b func (d *lvm) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { // Perform validation if d.isRemote() && snapVol.ContentType() == ContentTypeBlock { - return fmt.Errorf("lvmcluster doesn't currently support snapshot creation") + if util.IsTrue(snapVol.ExpandedConfig("security.shared")) { + return fmt.Errorf(`Snapshots of shared custom storage volumes aren't supported on "lvmcluster"`) + } + + if snapVol.ExpandedConfig("block.type") != BlockVolumeTypeQcow2 { + return fmt.Errorf(`Snapshots of raw block volumes aren't supported on "lvmcluster"`) + } } parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) From 8b48af5a5609ff45df749d4480c60d12d46456f6 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Tue, 28 Oct 2025 14:59:21 +0100 Subject: [PATCH 03/15] incusd/storage: Implement the creation of qcow2 formatted volumes when on lvmcluster Signed-off-by: Piotr Resztak --- internal/server/storage/backend.go | 12 +++--- internal/server/storage/drivers/driver_lvm.go | 3 ++ .../storage/drivers/driver_lvm_volumes.go | 24 +++++++++++ .../server/storage/drivers/driver_types.go | 3 +- .../server/storage/drivers/generic_vfs.go | 2 +- internal/server/storage/utils.go | 43 ++++++++++++++++++- 6 files changed, 77 insertions(+), 10 deletions(-) diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index 7f19f1d73d7..5b99d5f4c07 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -685,7 +685,7 @@ func (b *backend) CreateInstance(inst instance.Instance, op *operations.Operatio var filler *drivers.VolumeFiller if inst.Type() == instancetype.Container { filler = &drivers.VolumeFiller{ - Fill: func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { + Fill: func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { // Create an empty rootfs. err := os.Mkdir(filepath.Join(vol.MountPath(), "rootfs"), 0o755) if err != nil && !os.IsExist(err) { @@ -1707,8 +1707,8 @@ func (b *backend) RefreshInstance(inst instance.Instance, src instance.Instance, // imageFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will unpack the specified image archive into the specified mount path // provided, and for VM images, a raw root block path is required to unpack the qcow2 image into. -func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { - return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { +func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { + return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { var tracker *ioprogress.ProgressTracker if op != nil { // Not passed when being done as part of pre-migration setup. metadata := make(map[string]any) @@ -1721,15 +1721,15 @@ func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func } imageFile := internalUtil.VarPath("images", fingerprint) - return ImageUnpack(imageFile, vol, rootBlockPath, b.state.OS, allowUnsafeResize, targetIsZero, tracker) + return ImageUnpack(imageFile, vol, rootBlockPath, b.state.OS, allowUnsafeResize, targetIsZero, tracker, targetFormat) } } // isoFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will copy the ISO content into the specified mount path // provided. -func (b *backend) isoFiller(data io.Reader) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { - return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { +func (b *backend) isoFiller(data io.Reader) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { + return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { f, err := os.OpenFile(rootBlockPath, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return -1, err diff --git a/internal/server/storage/drivers/driver_lvm.go b/internal/server/storage/drivers/driver_lvm.go index 95d868cc2fb..81b70786e11 100644 --- a/internal/server/storage/drivers/driver_lvm.go +++ b/internal/server/storage/drivers/driver_lvm.go @@ -141,8 +141,10 @@ func (d *lvm) isRemote() bool { // Info returns info about the driver and its environment. func (d *lvm) Info() Info { name := "lvm" + targetFormat := BlockVolumeTypeRaw if d.clustered { name = "lvmcluster" + targetFormat = BlockVolumeTypeQcow2 } return Info{ @@ -163,6 +165,7 @@ func (d *lvm) Info() Info { Buckets: !d.isRemote(), Deactivate: d.isRemote(), ZeroUnpack: !d.usesThinpool(), + TargetFormat: targetFormat, } } diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index 331b17669fd..0afa987a397 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -58,6 +58,30 @@ func (d *lvm) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Oper reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } + // Format LV as qcow2 (lvmcluster). + if vol.ContentType() == ContentTypeBlock && vol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 { + // Get the device path. + devPath, err := d.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + qcow2SizeBytes, err := d.roundedSizeBytesString(vol.ConfigSize()) + if err != nil { + return err + } + + err = Qcow2Create(devPath, "", qcow2SizeBytes) + if err != nil { + return err + } + } else if vol.ContentType() == ContentTypeFS && vol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 { + err = Qcow2CreateConfig(vol, op) + if err != nil { + return err + } + } + err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { diff --git a/internal/server/storage/drivers/driver_types.go b/internal/server/storage/drivers/driver_types.go index 92549171aa8..03e003bb62d 100644 --- a/internal/server/storage/drivers/driver_types.go +++ b/internal/server/storage/drivers/driver_types.go @@ -21,11 +21,12 @@ type Info struct { MountedRoot bool // Whether the pool directory itself is a mount. Deactivate bool // Whether an unmount action is required prior to removing the pool. ZeroUnpack bool // Whether to write zeroes (no discard) during unpacking. + TargetFormat string // Whether the output image format should be raw or qcow2. } // VolumeFiller provides a struct for filling a volume. type VolumeFiller struct { - Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) // Function to fill the volume. + Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) // Function to fill the volume. Size int64 // Size of the unpacked volume in bytes. Fingerprint string // If the Filler will unpack an image, it should be this fingerprint. diff --git a/internal/server/storage/drivers/generic_vfs.go b/internal/server/storage/drivers/generic_vfs.go index 825431e0c7c..2a15276145a 100644 --- a/internal/server/storage/drivers/generic_vfs.go +++ b/internal/server/storage/drivers/generic_vfs.go @@ -1210,7 +1210,7 @@ func genericRunFiller(d Driver, vol Volume, devPath string, filler *VolumeFiller } vol.driver.Logger().Debug("Running filler function", logger.Ctx{"dev": devPath, "path": vol.MountPath()}) - volSize, err := filler.Fill(vol, devPath, allowUnsafeResize, !d.Info().ZeroUnpack) + volSize, err := filler.Fill(vol, devPath, allowUnsafeResize, !d.Info().ZeroUnpack, d.Info().TargetFormat) if err != nil { return err } diff --git a/internal/server/storage/utils.go b/internal/server/storage/utils.go index d67fd1b326c..ea55f0ccf6f 100644 --- a/internal/server/storage/utils.go +++ b/internal/server/storage/utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -572,7 +573,7 @@ func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error // VM Format A: Separate metadata tarball and root qcow2 file. // - Unpack metadata tarball into mountPath. // - Check rootBlockPath is a file and convert qcow2 file into raw format in rootBlockPath. -func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sysOS *sys.OS, allowUnsafeResize bool, targetIsZero bool, tracker *ioprogress.ProgressTracker) (int64, error) { +func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sysOS *sys.OS, allowUnsafeResize bool, targetIsZero bool, tracker *ioprogress.ProgressTracker, targetFormat string) (int64, error) { l := logger.Log.AddContext(logger.Ctx{"imageFile": imageFile, "volName": vol.Name()}) l.Info("Image unpack started") defer l.Info("Image unpack stopped") @@ -660,7 +661,7 @@ func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sys } // Belt and braces qcow2 check. - if imgInfo.Format != "qcow2" { + if imgInfo.Format != drivers.BlockVolumeTypeQcow2 { return -1, fmt.Errorf("Unexpected image format %q", imgInfo.Format) } @@ -695,6 +696,44 @@ func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sys } } + if targetFormat == drivers.BlockVolumeTypeQcow2 { + l.Debug("Writing qcow2 image to disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath}) + + // Attempt to deref all paths. + imgFullPath, err := filepath.EvalSymlinks(imgPath) + if err == nil { + imgPath = imgFullPath + } + + if dstPath != "" { + dstFullPath, err := filepath.EvalSymlinks(dstPath) + if err == nil { + dstPath = dstFullPath + } + } + + from, err := os.OpenFile(imgPath, unix.O_RDONLY, 0) + if err != nil { + return -1, err + } + + defer from.Close() + + to, err := os.OpenFile(dstPath, unix.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0) + if err != nil { + return -1, err + } + + defer to.Close() + + _, err = io.Copy(to, from) + if err != nil { + return -1, err + } + + return imgInfo.VirtualSize, nil + } + // Convert the qcow2 format to a raw block device. l.Debug("Converting qcow2 image to raw disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath}) From 91eeff4434af0cfe8ae69b9a65579c872486a821 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 19:52:00 +0100 Subject: [PATCH 04/15] incusd/instance/drivers/qmp: Add QueryNamedBlockNodes and ChangeBackingFile Signed-off-by: Piotr Resztak --- .../server/instance/drivers/qmp/commands.go | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/internal/server/instance/drivers/qmp/commands.go b/internal/server/instance/drivers/qmp/commands.go index 720f8435ee0..20332e3b60c 100644 --- a/internal/server/instance/drivers/qmp/commands.go +++ b/internal/server/instance/drivers/qmp/commands.go @@ -1128,6 +1128,47 @@ func (m *Monitor) NBDBlockExportAdd(deviceNodeName string) error { return nil } +// QueryNamedBlockNodes returns block nodes names. +func (m *Monitor) QueryNamedBlockNodes() ([]string, error) { + var resp struct { + Return []struct { + NodeName string `json:"node-name"` + } `json:"return"` + } + + err := m.Run("query-named-block-nodes", nil, &resp) + if err != nil { + return nil, err + } + + result := []string{} + for _, r := range resp.Return { + result = append(result, r.NodeName) + } + + return result, nil +} + +// ChangeBackingFile changes backing file name for node. +func (m *Monitor) ChangeBackingFile(nodeName string, backingFilename string) error { + var args struct { + Device string `json:"device"` + NodeName string `json:"image-node-name"` + BackingFile string `json:"backing-file"` + } + + args.Device = nodeName + args.NodeName = nodeName + args.BackingFile = backingFilename + + err := m.Run("change-backing-file", args, nil) + if err != nil { + return err + } + + return nil +} + // BlockDevSnapshot creates a snapshot of a device using the specified snapshot device. func (m *Monitor) BlockDevSnapshot(deviceNodeName string, snapshotNodeName string) error { var args struct { From 654b4d38fe345fa8a0e217bc5a4efd35e2d11815 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:01:15 +0100 Subject: [PATCH 05/15] incusd/storage/drivers: Add support for activating and deactivating qcow2-formatted volumes Signed-off-by: Piotr Resztak --- internal/server/storage/drivers/driver_lvm_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm_utils.go b/internal/server/storage/drivers/driver_lvm_utils.go index 6377362567c..c32a26d21fe 100644 --- a/internal/server/storage/drivers/driver_lvm_utils.go +++ b/internal/server/storage/drivers/driver_lvm_utils.go @@ -871,7 +871,7 @@ func (d *lvm) parseLogicalVolumeSnapshot(parent Volume, lvmVolName string) strin func (d *lvm) activateVolume(vol Volume) (bool, error) { var volPath string - if d.usesThinpool() { + if d.usesThinpool() || isQcow2Block(vol) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as activating the parent volume also activates its snapshots. @@ -915,7 +915,7 @@ func (d *lvm) activateVolume(vol Volume) (bool, error) { func (d *lvm) deactivateVolume(vol Volume) (bool, error) { var volPath string - if d.usesThinpool() { + if d.usesThinpool() || isQcow2Block(vol) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as deactivating the parent volume also activates its snapshots. From 8e29e15cf1a555c82d2356cdad624b32019fee41 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Wed, 3 Dec 2025 17:35:01 +0100 Subject: [PATCH 06/15] incusd/storage/drivers: Add support for the qcow2 config filesystem snapshots Signed-off-by: Piotr Resztak --- .../storage/drivers/driver_lvm_volumes.go | 138 +++++++++++------- internal/server/storage/drivers/volume.go | 73 +++++++++ 2 files changed, 156 insertions(+), 55 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index 0afa987a397..054641db2bd 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -1311,74 +1311,102 @@ func (d *lvm) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) erro mountVol := snapVol mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(mountVol.ConfigBlockMountOptions(), ",")) - // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting - // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need - // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. - // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be - // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly - // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time - // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then - // mount that. - regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) - if regenerateFSUUID { - // Instantiate a new volume to be the temporary writable snapshot. - tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) - tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) - - // Create writable snapshot from source snapshot named with a tmpVolSuffix suffix. - _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, tmpVol, false, d.usesThinpool()) + isQcow2 := snapVol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 + if isQcow2 { + parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) + mountVol = NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) + + // Activate volume if needed. + _, err = d.activateVolume(mountVol) if err != nil { - return fmt.Errorf("Error creating temporary LVM logical volume snapshot: %w", err) + return err } - reverter.Add(func() { - _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], tmpVol.volType, tmpVol.contentType, tmpVol.name)) - }) + // Get volume path. + volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) - // We are going to mount the temporary volume instead. - mountVol = tmpVol - } + volDevPath, err := d.lvmDevPath(volPath) + if err != nil { + return err + } - // Activate volume if needed. - _, err = d.activateVolume(mountVol) - if err != nil { - return err - } + // Finally attempt to mount the volume that needs mounting. + err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) + if err != nil { + return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) + } - // Get volume path. - volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) + d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + } else { + // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting + // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need + // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. + // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be + // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly + // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time + // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then + // mount that. + regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) + if regenerateFSUUID { + // Instantiate a new volume to be the temporary writable snapshot. + tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) + tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) + + // Create writable snapshot from source snapshot named with a tmpVolSuffix suffix. + _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, tmpVol, false, d.usesThinpool()) + if err != nil { + return fmt.Errorf("Error creating temporary LVM logical volume snapshot: %w", err) + } - volDevPath, err := d.lvmDevPath(volPath) - if err != nil { - return err - } + reverter.Add(func() { + _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], tmpVol.volType, tmpVol.contentType, tmpVol.name)) + }) - if regenerateFSUUID { - tmpVolFsType := mountVol.ConfigBlockFilesystem() + // We are going to mount the temporary volume instead. + mountVol = tmpVol + } - // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully - // regenerating the filesystem UUID. - if tmpVolFsType == "xfs" { - idx := strings.Index(mountOptions, "nouuid") - if idx < 0 { - mountOptions += ",nouuid" - } - } else { - d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) - err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volDevPath) - if err != nil { - return err + // Activate volume if needed. + _, err = d.activateVolume(mountVol) + if err != nil { + return err + } + + // Get volume path. + volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) + + volDevPath, err := d.lvmDevPath(volPath) + if err != nil { + return err + } + + if regenerateFSUUID { + tmpVolFsType := mountVol.ConfigBlockFilesystem() + + // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully + // regenerating the filesystem UUID. + if tmpVolFsType == "xfs" { + idx := strings.Index(mountOptions, "nouuid") + if idx < 0 { + mountOptions += ",nouuid" + } + } else { + d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) + err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volDevPath) + if err != nil { + return err + } } } - } - // Finally attempt to mount the volume that needs mounting. - err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) - if err != nil { - return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) - } + // Finally attempt to mount the volume that needs mounting. + err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) + if err != nil { + return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) + } - d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + } } else if snapVol.contentType == ContentTypeBlock { // Activate volume if needed. _, err = d.activateVolume(snapVol) diff --git a/internal/server/storage/drivers/volume.go b/internal/server/storage/drivers/volume.go index eb0a179b649..7b2a3aad303 100644 --- a/internal/server/storage/drivers/volume.go +++ b/internal/server/storage/drivers/volume.go @@ -122,6 +122,7 @@ type Volume struct { driver Driver mountCustomPath string // Mount the filesystem volume at a custom location. mountFilesystemProbe bool // Probe filesystem type when mounting volume (when needed). + mountFullFilesystem bool // Whether the whole volume, including snapshots data, should be mounted. It is used by the VM config filesystem. hasSource bool // Whether the volume is created from a source volume. isDeleted bool // Whether we're dealing with a hidden volume (kept until all references are gone). } @@ -364,6 +365,62 @@ func (v Volume) MountTask(task func(mountPath string, op *operations.Operation) return nil } +// MountWithSnapshotsTask runs the supplied task after mounting the volume with its snapshots if needed. If the volume was mounted +// for this then it is unmounted when the task finishes. +func (v Volume) MountWithSnapshotsTask(task func(mountPath string, op *operations.Operation) error, op *operations.Operation) error { + var err error + + if v.IsSnapshot() { + return fmt.Errorf("Volume cannot be snapshot") + } + + err = v.driver.MountVolume(v, op) + if err != nil { + return err + } + + defer func() { _, _ = v.driver.UnmountVolume(v, false, op) }() + + snapshots, err := v.Snapshots(op) + if err != nil { + return err + } + + for _, s := range snapshots { + err = v.driver.MountVolumeSnapshot(s, op) + if err != nil { + return err + } + + defer func() { _, _ = v.driver.UnmountVolumeSnapshot(s, op) }() + + snapDiskPath, err := v.driver.GetVolumeDiskPath(s) + if err != nil { + return err + } + + backingPath, err := v.driver.GetQcow2BackingFilePath(s) + if err != nil { + return err + } + + // Check whether the backing path and the LVM volume resolve to the same /dev/dm-X device. + target, err := filepath.EvalSymlinks(backingPath) + if target != snapDiskPath { + return fmt.Errorf("LVM is in an inconsistent state") + } + } + + taskErr := task(v.MountPath(), op) + + // Return task error if failed. + if taskErr != nil { + return taskErr + } + + return nil +} + // UnmountTask runs the supplied task after unmounting the volume if needed. // If the volume was unmounted for this then it is mounted when the task finishes. // keepBlockDev indicates if backing block device should be not be deactivated if volume is unmounted. @@ -515,6 +572,11 @@ func (v Volume) SetConfigStateSize(size string) { // ConfigBlockFilesystem returns the filesystem to use for block volumes. Returns config value "block.filesystem" // if defined in volume or pool's volume config, otherwise the DefaultFilesystem. func (v Volume) ConfigBlockFilesystem() string { + blockType := v.ExpandedConfig("block.type") + if blockType != "" && blockType == BlockVolumeTypeQcow2 && v.contentType == ContentTypeFS { + return "btrfs" + } + fs := v.ExpandedConfig("block.filesystem") if fs != "" { return fs @@ -526,6 +588,17 @@ func (v Volume) ConfigBlockFilesystem() string { // ConfigBlockMountOptions returns the filesystem mount options to use for block volumes. Returns config value // "block.mount_options" if defined in volume or pool's volume config, otherwise defaultFilesystemMountOptions. func (v Volume) ConfigBlockMountOptions() string { + blockType := v.ExpandedConfig("block.type") + if blockType != "" && blockType == BlockVolumeTypeQcow2 && !v.mountFullFilesystem { + parent, snapName, isSnap := api.GetParentAndSnapshotName(v.name) + subvol := parent + if isSnap { + subvol = fmt.Sprintf("%s-%s", parent, snapName) + } + + return fmt.Sprintf("subvol=%s", subvol) + } + fs := v.ExpandedConfig("block.mount_options") if fs != "" { return fs From cbde72ad19f14d868aff7dd833dcaa37fb379917 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:04:03 +0100 Subject: [PATCH 07/15] incusd/storage/drivers: Add support for creating and renaming qcow2 volume snapshots Signed-off-by: Piotr Resztak --- .../storage/drivers/driver_lvm_volumes.go | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index 054641db2bd..9536f2e2dd1 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -9,6 +9,7 @@ import ( "math" "os" "os/exec" + "path/filepath" "strings" "golang.org/x/sys/unix" @@ -1180,7 +1181,10 @@ func (d *lvm) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, _ b // CreateVolumeSnapshot creates a snapshot of a volume. func (d *lvm) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { - // Perform validation + parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) + parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) + snapPath := snapVol.MountPath() + if d.isRemote() && snapVol.ContentType() == ContentTypeBlock { if util.IsTrue(snapVol.ExpandedConfig("security.shared")) { return fmt.Errorf(`Snapshots of shared custom storage volumes aren't supported on "lvmcluster"`) @@ -1189,11 +1193,36 @@ func (d *lvm) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) err if snapVol.ExpandedConfig("block.type") != BlockVolumeTypeQcow2 { return fmt.Errorf(`Snapshots of raw block volumes aren't supported on "lvmcluster"`) } - } - parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) - parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) - snapPath := snapVol.MountPath() + parentVolPath := d.lvmPath(d.config["lvm.vg_name"], parentVol.volType, parentVol.contentType, parentName) + snapVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) + + releaseParent, err := d.acquireExclusive(parentVol) + if err != nil { + return err + } + + defer releaseParent() + + err = d.renameLogicalVolume(parentVolPath, snapVolPath) + if err != nil { + return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) + } + + releaseSnap, err := d.acquireExclusive(snapVol) + if err != nil { + return err + } + + defer releaseSnap() + + err = d.createLogicalVolume(d.config["lvm.vg_name"], d.thinpoolName(), parentVol, d.usesThinpool()) + if err != nil { + return fmt.Errorf("Error creating LVM logical volume: %w", err) + } + + return nil + } // Create the parent directory. err := createParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) @@ -1738,7 +1767,15 @@ func (d *lvm) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *o parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) newSnapVolName := GetSnapshotVolumeName(parentName, newSnapshotName) newVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, newSnapVolName) - err := d.renameLogicalVolume(volPath, newVolPath) + + release, err := d.acquireExclusive(snapVol) + if err != nil { + return err + } + + defer release() + + err = d.renameLogicalVolume(volPath, newVolPath) if err != nil { return fmt.Errorf("Error renaming LVM logical volume: %w", err) } From 740ff59643f7f6c7fdb54938c10764bae5b57bb5 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:05:16 +0100 Subject: [PATCH 08/15] incusd/storage/drivers: Add GetQcow2BackingFilePath Signed-off-by: Piotr Resztak --- internal/server/storage/drivers/driver_common.go | 5 +++++ internal/server/storage/drivers/driver_lvm_volumes.go | 6 ++++++ internal/server/storage/drivers/interface.go | 1 + 3 files changed, 12 insertions(+) diff --git a/internal/server/storage/drivers/driver_common.go b/internal/server/storage/drivers/driver_common.go index 04de280fcc1..81b5ff68a7c 100644 --- a/internal/server/storage/drivers/driver_common.go +++ b/internal/server/storage/drivers/driver_common.go @@ -557,3 +557,8 @@ func (d *common) filesystemFreeze(path string) (func() error, error) { func (d *common) CacheVolumeSnapshots(vol Volume) error { return nil } + +// GetQcow2BackingFilePath generates the backing file path for the specified volume. +func (d *common) GetQcow2BackingFilePath(vol Volume) (string, error) { + return "", ErrNotSupported +} diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index 9536f2e2dd1..b0b86494077 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -1789,3 +1789,9 @@ func (d *lvm) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *o return nil } + +// GetQcow2BackingFilePath generates the backing file path for the specified volume. +func (d *lvm) GetQcow2BackingFilePath(vol Volume) (string, error) { + pathName := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) + return filepath.Join("/dev", pathName), nil +} diff --git a/internal/server/storage/drivers/interface.go b/internal/server/storage/drivers/interface.go index 9f5c2e7b7c8..73b2cbc316e 100644 --- a/internal/server/storage/drivers/interface.go +++ b/internal/server/storage/drivers/interface.go @@ -99,6 +99,7 @@ type Driver interface { CacheVolumeSnapshots(vol Volume) error CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error + GetQcow2BackingFilePath(vol Volume) (string, error) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) From 4e3950c7713b041d48eb2be501a968d96249f34e Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:14:30 +0100 Subject: [PATCH 09/15] incusd/device/config: Add 'BackingPath' to track backing chain for qcow2 volumes Signed-off-by: Piotr Resztak --- .../server/device/config/device_runconfig.go | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/server/device/config/device_runconfig.go b/internal/server/device/config/device_runconfig.go index 3efb04af0f5..738d650abc8 100644 --- a/internal/server/device/config/device_runconfig.go +++ b/internal/server/device/config/device_runconfig.go @@ -21,16 +21,17 @@ type RunConfigItem struct { // MountEntryItem represents a single mount entry item. type MountEntryItem struct { - DevName string // The internal name for the device. - DevPath string // Describes the block special device or remote filesystem to be mounted. - TargetPath string // Describes the mount point (target) for the filesystem. - FSType string // Describes the type of the filesystem. - Opts []string // Describes the mount options associated with the filesystem. - Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present. - PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present. - OwnerShift string // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic. - Limits *DiskLimits // Disk limits. - Size int64 // Expected disk size in bytes. + DevName string // The internal name for the device. + DevPath string // Describes the block special device or remote filesystem to be mounted. + BackingPath []string // Describes the block special device to be mounted as backing drive for qcow2. + TargetPath string // Describes the mount point (target) for the filesystem. + FSType string // Describes the type of the filesystem. + Opts []string // Describes the mount options associated with the filesystem. + Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present. + PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present. + OwnerShift string // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic. + Limits *DiskLimits // Disk limits. + Size int64 // Expected disk size in bytes. } // RootFSEntryItem represents the root filesystem options for an Instance. From 79636c2855bf8141c7b4bb29ff987ace09a4a0d3 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:14:50 +0100 Subject: [PATCH 10/15] incusd/storage: Add 'BackingPath' to track backing chain for qcow2 volumes Signed-off-by: Piotr Resztak --- internal/server/storage/pool_interface.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/server/storage/pool_interface.go b/internal/server/storage/pool_interface.go index c871fa17130..e9ccce7a51f 100644 --- a/internal/server/storage/pool_interface.go +++ b/internal/server/storage/pool_interface.go @@ -26,8 +26,9 @@ type VolumeUsage struct { // MountInfo represents info about the result of a mount operation. type MountInfo struct { - DiskPath string // The location of the block disk (if supported). - PostHooks []func(inst instance.Instance) error // Hooks to be called following a mount. + DiskPath string // The location of the block disk (if supported). + BackingPath []string // The location of the block disk (backing disk for qcow2). + PostHooks []func(inst instance.Instance) error // Hooks to be called following a mount. } // Type represents an Incus storage pool type. From a3f94bd0c869842d1c9fa10dbaca6da1354b7d31 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:17:06 +0100 Subject: [PATCH 11/15] incusd/storage: Add support for creating, renaming, restoring and deleting qcow2 instance volumes Signed-off-by: Piotr Resztak --- internal/server/storage/backend.go | 519 ++++++++++++++++++++++++++++- 1 file changed, 510 insertions(+), 9 deletions(-) diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index 5b99d5f4c07..a4029e87379 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -54,6 +54,7 @@ import ( "github.com/lxc/incus/v6/shared/ioprogress" "github.com/lxc/incus/v6/shared/logger" "github.com/lxc/incus/v6/shared/revert" + "github.com/lxc/incus/v6/shared/subprocess" "github.com/lxc/incus/v6/shared/units" "github.com/lxc/incus/v6/shared/util" ) @@ -2027,6 +2028,10 @@ func (b *backend) CreateInstanceFromMigration(inst instance.Instance, conn io.Re // Create new volume database records when the storage pool is changed or // when it is not a remote cluster move. if !isRemoteClusterMove || args.StoragePool != "" { + if vol.ExpandedConfig("block.type") == drivers.BlockVolumeTypeQcow2 { + return errors.New("Qcow2 instance migration is not supported") + } + for i, snapshot := range args.Snapshots { snapName := snapshot.GetName() newSnapshotName := drivers.GetSnapshotVolumeName(inst.Name(), snapName) @@ -2542,6 +2547,10 @@ func (b *backend) MigrateInstance(inst instance.Instance, conn io.ReadWriteClose return err } + if args.StorageMove && vol.ExpandedConfig("block.type") == drivers.BlockVolumeTypeQcow2 { + return errors.New("Qcow2 instance migration is not supported") + } + args.Name = inst.Name() // Override args.Name to ensure instance volume is sent. // Send migration index header frame with volume info and wait for receipt if not doing final sync. @@ -2879,10 +2888,36 @@ func (b *backend) MountInstance(inst instance.Instance, op *operations.Operation return nil, fmt.Errorf("Failed getting disk path: %w", err) } + backingPaths := []string{} + if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return nil, err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) + err = b.driver.MountVolumeSnapshot(currentSnapVol, op) + if err != nil { + return nil, err + } + } + + // Fetch backing chain for a qcow2 formatted volume. + backingPaths, err = b.qcow2BackingPaths(vol, diskPath, inst.Project().Name) + if err != nil { + return nil, err + } + } + mountInfo := &MountInfo{ - DiskPath: diskPath, + DiskPath: diskPath, + BackingPath: backingPaths, } + l.Debug("MountInstance mountInfo", logger.Ctx{"mountInfo": mountInfo}) + reverter.Success() // From here on it is up to caller to call UnmountInstance() when done. // Handle delegation. @@ -2937,6 +2972,22 @@ func (b *backend) UnmountInstance(inst instance.Instance, op *operations.Operati vol = b.GetVolume(volType, contentType, volStorageName, nil) } + if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) + _, err = b.driver.UnmountVolumeSnapshot(currentSnapVol, op) + if err != nil && !errors.Is(err, drivers.ErrInUse) { + return err + } + } + } + _, err = b.driver.UnmountVolume(vol, false, op) return err @@ -3065,8 +3116,11 @@ func (b *backend) CreateInstanceSnapshot(inst instance.Instance, src instance.In volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Get the volume. - // There's no need to pass config as it's not needed when creating volume snapshots. - vol := b.GetVolume(volType, contentType, volStorageName, nil) + vol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) + err = b.applyInstanceRootDiskOverrides(inst, &vol) + if err != nil { + return err + } // Lock this operation to ensure that the only one snapshot is made at the time. // Other operations will wait for this one to finish. @@ -3082,6 +3136,18 @@ func (b *backend) CreateInstanceSnapshot(inst instance.Instance, src instance.In return err } + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get the parent volume. + parentVol := b.GetVolume(volType, contentType, src.Name(), srcDBVol.Config) + + // parentVol should already be prepared as an overlay by CreateVolumeSnapshot. + // vol will be used as the base. + err = b.qcow2CreateSnapshot(parentVol, vol, src, op) + if err != nil { + return err + } + } + err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err @@ -3124,18 +3190,33 @@ func (b *backend) RenameInstanceSnapshot(inst instance.Instance, newName string, return errors.New("Volume name must be a snapshot") } + newVolName := drivers.GetSnapshotVolumeName(parentName, newName) + + // Load storage volume from database. + srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) + if err != nil { + return err + } + contentType := InstanceContentType(inst) volStorageName := project.Instance(inst.Project().Name, inst.Name()) - // Rename storage volume snapshot. No need to pass config as it's not needed when renaming a volume. - snapVol := b.GetVolume(volType, contentType, volStorageName, nil) + // Rename storage volume snapshot. + snapVol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) + + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + parentVol := b.GetVolume(volType, contentType, parentName, srcDBVol.Config) + err = b.qcow2RenameSnapshot(parentVol, snapVol, newVolName, inst.Type(), inst.Project().Name, op) + if err != nil { + return err + } + } + err = b.driver.RenameVolumeSnapshot(snapVol, newName, op) if err != nil { return err } - newVolName := drivers.GetSnapshotVolumeName(parentName, newName) - reverter.Add(func() { // Revert rename. No need to pass config as it's not needed when renaming a volume. newSnapVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, newVolName), nil) @@ -3195,8 +3276,13 @@ func (b *backend) DeleteInstanceSnapshot(inst instance.Instance, op *operations. snapVolName := drivers.GetSnapshotVolumeName(parentStorageName, snapName) - // There's no need to pass config as it's not needed when deleting a volume snapshot. - vol := b.GetVolume(volType, contentType, snapVolName, nil) + // Load storage volume from database. + srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) + if err != nil { + return err + } + + vol := b.GetVolume(volType, contentType, snapVolName, srcDBVol.Config) volExists, err := b.driver.HasVolume(vol) if err != nil { @@ -3204,6 +3290,40 @@ func (b *backend) DeleteInstanceSnapshot(inst instance.Instance, op *operations. } if volExists { + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + randomSuffix, err := internalUtil.RandomHexString(10) + if err != nil { + return fmt.Errorf("Failed creating temporary file name: %w", err) + } + + parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) + tmpSnapName := fmt.Sprintf("%s_%s", parent, randomSuffix) + tmpVol := b.GetVolume(vol.Type(), vol.ContentType(), tmpSnapName, vol.Config()) + + err = b.applyInstanceRootDiskOverrides(inst, &tmpVol) + if err != nil { + return err + } + + // Create temporary volume. + err = b.driver.CreateVolume(tmpVol, nil, op) + if err != nil { + return err + } + + parentVol := b.GetVolume(volType, contentType, parentName, srcDBVol.Config) + err = b.qcow2DeleteSnapshot(parentVol, vol, tmpVol, inst, op) + if err != nil { + return err + } + + // Delete temporary volume. + err = b.driver.DeleteVolume(tmpVol, op) + if err != nil { + return err + } + } + err = b.driver.DeleteVolumeSnapshot(vol, op) if err != nil { return err @@ -3305,6 +3425,16 @@ func (b *backend) RestoreInstanceSnapshot(inst instance.Instance, src instance.I }) } + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + snapVol := b.GetVolume(volType, contentType, src.Name(), srcDBVol.Config) + err = b.qcow2RestoreSnapshot(vol, snapVol, inst.Type(), inst.Project().Name, op) + if err != nil { + return err + } + + return nil + } + err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots @@ -7506,3 +7636,374 @@ func (b *backend) getFirstAdminStorageBucketPoolKey(projectName string, bucketNa return bucketKey, nil } + +// qcow2CreateSnapshot creates the QCOW2 volume snapshot. +func (b *backend) qcow2CreateSnapshot(vol drivers.Volume, snapVol drivers.Volume, src instance.Instance, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + if src.Type() == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err := drivers.Qcow2CreateConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + // For a running instance, mount the snapshot to increase the volume's refCount. + // The increased refCount protects against deallocating a volume that is + // currently in use by the instance. + if src.IsRunning() { + err := b.driver.MountVolumeSnapshot(snapVol, op) + if err != nil { + return err + } + } + + snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + err = vol.MountWithSnapshotsTask(func(parentMountPath string, parentOp *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + if !src.IsRunning() { + err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) + if err != nil { + return err + } + } else { + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + err = drivers.Qcow2Create(parentDiskPath, "", int64(imgInfo.VirtualSize)) + if err != nil { + return err + } + } + + return nil + }, op) + if err != nil { + return err + } + + if src.IsRunning() { + _, snapshotName, isSnap := api.GetParentAndSnapshotName(snapVol.Name()) + if !isSnap { + return errors.New("Volume name must be a snapshot") + } + + err = src.CreateQcow2Snapshot(snapshotName, snapVolDevPath) + if err != nil { + return err + } + } + + return nil +} + +// qcow2RestoreSnapshot restores the QCOW2 volume snapshot. +func (b *backend) qcow2RestoreSnapshot(vol drivers.Volume, snapVol drivers.Volume, instanceType instancetype.Type, projectName string, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + // Restoring is allowed only for the most recent snapshot. + if imgInfo.BackingFilename != snapVolDevPath { + return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s).", snapVol.Name()) + } + + err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) + if err != nil { + return err + } + + return nil + }, op) + if err != nil { + return err + } + + if instanceType == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err = drivers.Qcow2RestoreConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2RenameSnapshot renames the QCOW2 volume snapshot. +func (b *backend) qcow2RenameSnapshot(vol drivers.Volume, snapVol drivers.Volume, newVolName string, instanceType instancetype.Type, projectName string, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + newSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, newVolName), nil) + newSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(newSnapVol) + if err != nil { + return err + } + + snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, projectName, vol.Name(), vol.Type()) + if err != nil { + return err + } + + // Update the metadata of the parent if it points to a renamed volume. + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + err = drivers.Qcow2Rebase(parentDiskPath, newSnapVolDiskPath) + if err != nil { + return err + } + } + + // Update the metadata of the snapshot which points to a renamed volume. + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + err = drivers.Qcow2Rebase(currentSnapVolDiskPath, newSnapVolDiskPath) + if err != nil { + return err + } + } + } + + return nil + }, op) + if err != nil { + return err + } + + if instanceType == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + _, newName, _ := api.GetParentAndSnapshotName(newVolName) + err = drivers.Qcow2RenameConfigSnapshot(fsParentVol, fsVol, newName, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2DeleteSnapshot deletes the QCOW2 volume snapshot. +func (b *backend) qcow2DeleteSnapshot(vol drivers.Volume, snapVol drivers.Volume, tmpVol drivers.Volume, inst instance.Instance, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + var destinationVol string + var baseVol string + + _, snapName, _ := api.GetParentAndSnapshotName(inst.Name()) + + snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + // Get parent volume backing file + parentVolDiskPath, err := b.driver.GetQcow2BackingFilePath(vol) + if err != nil { + return err + } + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return err + } + + // Check if the main volume is the parent of the snapshot to be deleted. + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + destinationVol = parentVolDiskPath + } + + // Check which snapshot is the parent of the snapshot to be deleted. + // Record information about the snapshot that will be deleted. + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return err + } + + _, currentSnapName, _ := api.GetParentAndSnapshotName(snap.Name) + + imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + destinationVol = currentSnapVolDiskPath + } + + if snapName == currentSnapName { + baseVol = imgInfo.BackingFilename + } + } + + tmpVolDiskPath, err := b.driver.GetQcow2BackingFilePath(tmpVol) + if err != nil { + return err + } + + // Move the data from the deleted snapshot to temporary volume. + err = drivers.Qcow2Convert(destinationVol, baseVol, tmpVolDiskPath) + if err != nil { + return err + } + + // Copy data from the temporary volume to the parent of the deleted snapshot. + _, err = subprocess.RunCommand("dd", "bs=4M", fmt.Sprintf("if=%s", tmpVolDiskPath), fmt.Sprintf("of=%s", destinationVol)) + if err != nil { + return err + } + + return nil + }, op) + if err != nil { + return err + } + + if inst.Type() == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err = drivers.Qcow2DeleteConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2BackingPaths returns information about the backing chain of a qcow2 image. +func (b *backend) qcow2BackingPaths(vol drivers.Volume, diskPath string, projectName string) ([]string, error) { + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil, nil + } + + backingPaths := []string{} + diskPathSnapshot := map[string]string{} + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, projectName, vol.Name(), vol.Type()) + if err != nil { + return nil, err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return nil, err + } + + _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) + diskPathSnapshot[currentSnapVolDiskPath] = snapName + } + + b.logger.Debug("Disk path snapshots map:", logger.Ctx{"diskPathSnapshots": diskPathSnapshot}) + + var chainPaths []string + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + chainPaths, err = drivers.Qcow2BackingChain(diskPath) + if err != nil { + return nil + } + + for _, p := range chainPaths { + snapName := diskPathSnapshot[p] + snapVolName := drivers.GetSnapshotVolumeName(vol.Name(), snapName) + snapVol := b.GetVolume(vol.Type(), vol.ContentType(), snapVolName, nil) + + volDiskPath, err := b.driver.GetVolumeDiskPath(snapVol) + if err != nil { + return err + } + + backingPaths = append(backingPaths, volDiskPath) + } + + return nil + }, nil) + if err != nil { + return nil, err + } + + return backingPaths, nil +} From a1e9a61e6aceae86de351d9347fab25ee534fae6 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:19:58 +0100 Subject: [PATCH 12/15] incusd/instance/drivers: Add support for running instances from a backing chain Signed-off-by: Piotr Resztak --- .../server/instance/drivers/driver_qemu.go | 103 ++++++++++++++++-- 1 file changed, 96 insertions(+), 7 deletions(-) diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 95bec89ea73..c882a4a76df 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -4337,11 +4337,12 @@ func (d *qemu) addRootDriveConfig(qemuDev map[string]any, mountInfo *storagePool // Generate a new device config with the root device path expanded. driveConf := deviceConfig.MountEntryItem{ - DevName: rootDriveConf.DevName, - DevPath: mountInfo.DiskPath, - Opts: rootDriveConf.Opts, - TargetPath: rootDriveConf.TargetPath, - Limits: rootDriveConf.Limits, + DevName: rootDriveConf.DevName, + DevPath: mountInfo.DiskPath, + BackingPath: mountInfo.BackingPath, + Opts: rootDriveConf.Opts, + TargetPath: rootDriveConf.TargetPath, + Limits: rootDriveConf.Limits, } if d.storagePool.Driver().Info().Remote { @@ -4540,13 +4541,14 @@ func (d *qemu) addDriveConfig(qemuDev map[string]any, bootIndexes map[string]int } var isBlockDev bool + var srcDevPath string // Detect device caches and I/O modes. if isRBDImage { // For RBD, we want writeback to allow for the system-configured "rbd cache" to take effect if present. cacheMode = "writeback" } else { - srcDevPath := driveConf.DevPath // This should not be used for passing to QEMU, only for probing. + srcDevPath = driveConf.DevPath // This should not be used for passing to QEMU, only for probing. // Detect if existing file descriptor format is being supplied. if strings.HasPrefix(driveConf.DevPath, fmt.Sprintf("%s:", device.DiskFileDescriptorMountPrefix)) { @@ -4852,7 +4854,40 @@ func (d *qemu) addDriveConfig(qemuDev map[string]any, bootIndexes map[string]int _ = m.RemoveFDFromFDSet(nodeName) }) - blockDev["filename"] = fmt.Sprintf("/dev/fdset/%d", info.ID) + isQcow2, err := d.isQCOW2(srcDevPath) + if err != nil { + return fmt.Errorf("Failed checking disk format: %w", err) + } + + if isQcow2 { + blockDev = map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": d.blockNodeName(escapedDeviceName), + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + "aio": aioMode, + "cache": map[string]any{ + "direct": directCache, + "no-flush": noFlushCache, + }, + }, + } + + // If there are any children, load block information about them. + if len(driveConf.BackingPath) > 0 { + backingBlockDev, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, driveConf.BackingPath, 0) + if err != nil { + return nil + } + + blockDev["backing"] = backingBlockDev + } + } else { + blockDev["filename"] = fmt.Sprintf("/dev/fdset/%d", info.ID) + } } err := m.AddBlockDevice(blockDev, qemuDev, bus == "usb") @@ -10424,3 +10459,57 @@ func (d *qemu) GuestOS() string { return "unknown" } + +func (d *qemu) isQCOW2(devPath string) (bool, error) { + imgInfo, err := storageDrivers.Qcow2Info(devPath) + if err != nil { + return false, err + } + + return imgInfo.Format == storageDrivers.BlockVolumeTypeQcow2, nil +} + +func (d *qemu) qcow2BlockDev(m *qmp.Monitor, nodeName string, aioMode string, directCache bool, noFlushCache bool, permissions int, readonly bool, backingPaths []string, iter int) (map[string]any, error) { + devName := backingPaths[0] + backingNodeName := fmt.Sprintf("%s_backing%d", nodeName, iter) + + f, err := os.OpenFile(devName, permissions, 0) + if err != nil { + return nil, fmt.Errorf("Failed opening file descriptor for disk device %q: %w", devName, err) + } + + defer func() { _ = f.Close() }() + + info, err := m.SendFileWithFDSet(backingNodeName, f, readonly) + if err != nil { + return nil, fmt.Errorf("Failed sending file descriptor of %q for disk device %q: %w", f.Name(), devName, err) + } + + blockDev := map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": backingNodeName, + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + "aio": aioMode, + "cache": map[string]any{ + "direct": directCache, + "no-flush": noFlushCache, + }, + }, + } + + // If there are any children, load block information about them. + if len(backingPaths) > 1 { + backingBlockDev, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, backingPaths[1:], iter+1) + if err != nil { + return nil, err + } + + blockDev["backing"] = backingBlockDev + } + + return blockDev, nil +} From cb540dea55351568197cf056b24581359c0d56ae Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Sat, 22 Nov 2025 20:21:45 +0100 Subject: [PATCH 13/15] incusd/instance: Add support for creating qcow2 snapshots while instance is running Signed-off-by: Piotr Resztak --- .../server/instance/drivers/driver_lxc.go | 5 + .../server/instance/drivers/driver_qemu.go | 111 ++++++++++++++++++ .../server/instance/instance_interface.go | 1 + 3 files changed, 117 insertions(+) diff --git a/internal/server/instance/drivers/driver_lxc.go b/internal/server/instance/drivers/driver_lxc.go index cccc27c5afc..d38017250f7 100644 --- a/internal/server/instance/drivers/driver_lxc.go +++ b/internal/server/instance/drivers/driver_lxc.go @@ -9384,3 +9384,8 @@ func (d *lxc) setupCredentials(update bool) error { func (d *lxc) GuestOS() string { return "linux" } + +// CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. Not supported by containers. +func (d *lxc) CreateQcow2Snapshot(snapName string, backingFilename string) error { + return nil +} diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index c882a4a76df..0e1a1d03bba 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "slices" "sort" "strconv" @@ -10460,6 +10461,97 @@ func (d *qemu) GuestOS() string { return "unknown" } +// CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. +func (d *qemu) CreateQcow2Snapshot(snapshotName string, backingFilename string) error { + monitor, err := d.qmpConnect() + if err != nil { + return err + } + + snap, err := instance.LoadByProjectAndName(d.state, d.project.Name, fmt.Sprintf("%s/%s", d.name, snapshotName)) + if err != nil { + return fmt.Errorf("Load by project and name: %w", err) + } + + pool, err := storagePools.LoadByInstance(d.state, snap) + if err != nil { + return fmt.Errorf("Load by instance: %w", err) + } + + mountInfoRoot, err := pool.MountInstance(d, d.op) + if err != nil { + return fmt.Errorf("Mount instance: %w", err) + } + + defer func() { _ = pool.UnmountInstance(d, d.op) }() + + f, err := os.OpenFile(mountInfoRoot.DiskPath, unix.O_RDWR, 0) + if err != nil { + return fmt.Errorf("Failed opening file descriptor for disk device %s: %w", mountInfoRoot.DiskPath, err) + } + + defer func() { _ = f.Close() }() + + // Fetch information about block devices. + blockdevNames, err := monitor.QueryNamedBlockNodes() + if err != nil { + return fmt.Errorf("Failed fetching block nodes names: %w", err) + } + + rootDevName, _, err := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) + if err != nil { + return fmt.Errorf("Failed getting instance root disk: %w", err) + } + + escapedDeviceName := linux.PathNameEncode(rootDevName) + rootNodeName := d.blockNodeName(escapedDeviceName) + + // Fetch the current maximum overlay index. + overlayNodeIndex := currentQcow2OverlayIndex(blockdevNames, rootNodeName) + nextOverlayName := fmt.Sprintf("%s_overlay%d", rootNodeName, overlayNodeIndex+1) + + currentOverlayName := rootNodeName + if overlayNodeIndex >= 0 { + currentOverlayName = fmt.Sprintf("%s_overlay%d", rootNodeName, overlayNodeIndex) + } + + info, err := monitor.SendFileWithFDSet(nextOverlayName, f, false) + if err != nil { + return fmt.Errorf("Failed sending file descriptor of %q for disk device: %w", f.Name(), err) + } + + blockDev := map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": nextOverlayName, + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + }, + } + + // Add overlay block dev. + err = monitor.AddBlockDevice(blockDev, nil, false) + if err != nil { + return fmt.Errorf("Fail to add block device: %w", err) + } + + // Take a snapshot of the root disk and redirect writes to the snapshot disk. + err = monitor.BlockDevSnapshot(currentOverlayName, nextOverlayName) + if err != nil { + return fmt.Errorf("Failed taking storage snapshot: %w", err) + } + + // Update metadata of the backing file. + err = monitor.ChangeBackingFile(nextOverlayName, backingFilename) + if err != nil { + return fmt.Errorf("Failed changing backing file: %w", err) + } + + return nil +} + func (d *qemu) isQCOW2(devPath string) (bool, error) { imgInfo, err := storageDrivers.Qcow2Info(devPath) if err != nil { @@ -10513,3 +10605,22 @@ func (d *qemu) qcow2BlockDev(m *qmp.Monitor, nodeName string, aioMode string, di return blockDev, nil } + +// currentQcow2OverlayIndex returns the current maximum overlay index. +func currentQcow2OverlayIndex(names []string, prefix string) int { + re := regexp.MustCompile(fmt.Sprintf(`^%s_overlay(\d+)$`, prefix)) + + maxIndex := -1 + + for _, name := range names { + m := re.FindStringSubmatch(name) + if len(m) == 2 { + n, err := strconv.Atoi(m[1]) + if err == nil && n > maxIndex { + maxIndex = n + } + } + } + + return maxIndex +} diff --git a/internal/server/instance/instance_interface.go b/internal/server/instance/instance_interface.go index e23d43f60af..531ef8d66b8 100644 --- a/internal/server/instance/instance_interface.go +++ b/internal/server/instance/instance_interface.go @@ -98,6 +98,7 @@ type Instance interface { Backups() ([]backup.InstanceBackup, error) UpdateBackupFile() error CanLiveMigrate() bool + CreateQcow2Snapshot(snapshotName string, backingFilename string) error // Config handling. Rename(newName string, applyTemplateTrigger bool) error From 7ff9f604df90893c5b12c6d4573aef8604dea1ea Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Thu, 4 Dec 2025 13:30:03 +0100 Subject: [PATCH 14/15] api: Add storage_lvmcluster_qcow2 extension Signed-off-by: Piotr Resztak --- doc/api-extensions.md | 4 ++++ internal/version/api.go | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 8e457bd8e79..24b60e37a27 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -2954,3 +2954,7 @@ This adds serial device tracking to the resources API. ## `ovn_nic_limits` This adds support for `limits.egress`, `limits.ingress`, `limits.max` and `limits.priority` on `ovn` type network devices. + +## `storage_lvmcluster_qcow2` + +This adds support for running virtual machines from `QCOW2` images. Support is currently limited to `lvmcluster` storage driver. diff --git a/internal/version/api.go b/internal/version/api.go index 70aef3863ea..cc7a6237ef8 100644 --- a/internal/version/api.go +++ b/internal/version/api.go @@ -511,6 +511,7 @@ var APIExtensions = []string{ "device_pci_firmware", "resources_serial", "ovn_nic_limits", + "storage_lvmcluster_qcow2", } // APIExtensionsCount returns the number of available API extensions. From 763e4ed63c0f8b88a6930793c27f54ae2433bdb0 Mon Sep 17 00:00:00 2001 From: Piotr Resztak Date: Thu, 4 Dec 2025 14:57:18 +0100 Subject: [PATCH 15/15] doc: Update metadata Signed-off-by: Piotr Resztak --- doc/config_options.txt | 7 +++++++ internal/server/metadata/configuration.json | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/doc/config_options.txt b/doc/config_options.txt index b2b8f27031b..50c1acac596 100644 --- a/doc/config_options.txt +++ b/doc/config_options.txt @@ -5685,6 +5685,13 @@ This value is required by some providers. +```{config:option} block.type storage_lvm-common +:condition: "block-based volume" +:default: "same as `volume.block.type`" +:shortdesc: "Type of the block volume" + +``` + ```{config:option} lvm.metadata_size storage_lvm-common :default: "`0` (auto)" :scope: "global" diff --git a/internal/server/metadata/configuration.json b/internal/server/metadata/configuration.json index 00f714eab70..fd551c0feae 100644 --- a/internal/server/metadata/configuration.json +++ b/internal/server/metadata/configuration.json @@ -6452,6 +6452,14 @@ "storage_lvm": { "common": { "keys": [ + { + "block.type": { + "condition": "block-based volume", + "default": "same as `volume.block.type`", + "longdesc": "", + "shortdesc": "Type of the block volume" + } + }, { "lvm.metadata_size": { "default": "`0` (auto)",