diff --git a/doc/config_options.txt b/doc/config_options.txt index e27a59af879..9bd1b2e2a14 100644 --- a/doc/config_options.txt +++ b/doc/config_options.txt @@ -5648,6 +5648,13 @@ This value is required by some providers. +```{config:option} block.type storage_lvm-common +:condition: "block-based volume" +:default: "same as `volume.block.type`" +:shortdesc: "Type of the block volume" + +``` + ```{config:option} lvm.metadata_size storage_lvm-common :default: "`0` (auto)" :scope: "global" diff --git a/internal/server/device/config/device_runconfig.go b/internal/server/device/config/device_runconfig.go index 3efb04af0f5..738d650abc8 100644 --- a/internal/server/device/config/device_runconfig.go +++ b/internal/server/device/config/device_runconfig.go @@ -21,16 +21,17 @@ type RunConfigItem struct { // MountEntryItem represents a single mount entry item. type MountEntryItem struct { - DevName string // The internal name for the device. - DevPath string // Describes the block special device or remote filesystem to be mounted. - TargetPath string // Describes the mount point (target) for the filesystem. - FSType string // Describes the type of the filesystem. - Opts []string // Describes the mount options associated with the filesystem. - Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present. - PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present. - OwnerShift string // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic. - Limits *DiskLimits // Disk limits. - Size int64 // Expected disk size in bytes. + DevName string // The internal name for the device. + DevPath string // Describes the block special device or remote filesystem to be mounted. + BackingPath []string // Describes the block special device to be mounted as backing drive for qcow2. + TargetPath string // Describes the mount point (target) for the filesystem. + FSType string // Describes the type of the filesystem. + Opts []string // Describes the mount options associated with the filesystem. + Freq int // Used by dump(8) to determine which filesystems need to be dumped. Defaults to zero (don't dump) if not present. + PassNo int // Used by fsck(8) to determine the order in which filesystem checks are done at boot time. Defaults to zero (don't fsck) if not present. + OwnerShift string // Ownership shifting mode, use constants MountOwnerShiftNone, MountOwnerShiftStatic or MountOwnerShiftDynamic. + Limits *DiskLimits // Disk limits. + Size int64 // Expected disk size in bytes. } // RootFSEntryItem represents the root filesystem options for an Instance. diff --git a/internal/server/instance/drivers/driver_lxc.go b/internal/server/instance/drivers/driver_lxc.go index 898991daac6..c207f460579 100644 --- a/internal/server/instance/drivers/driver_lxc.go +++ b/internal/server/instance/drivers/driver_lxc.go @@ -9379,3 +9379,8 @@ func (d *lxc) setupCredentials(update bool) error { func (d *lxc) GuestOS() string { return "linux" } + +// CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. Not supported by containers. +func (d *lxc) CreateQcow2Snapshot(snapName string, backingFilename string) error { + return nil +} diff --git a/internal/server/instance/drivers/driver_qemu.go b/internal/server/instance/drivers/driver_qemu.go index 9bfd1a1ad97..49fae5326ab 100644 --- a/internal/server/instance/drivers/driver_qemu.go +++ b/internal/server/instance/drivers/driver_qemu.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "regexp" "slices" "sort" "strconv" @@ -4337,11 +4338,12 @@ func (d *qemu) addRootDriveConfig(qemuDev map[string]any, mountInfo *storagePool // Generate a new device config with the root device path expanded. driveConf := deviceConfig.MountEntryItem{ - DevName: rootDriveConf.DevName, - DevPath: mountInfo.DiskPath, - Opts: rootDriveConf.Opts, - TargetPath: rootDriveConf.TargetPath, - Limits: rootDriveConf.Limits, + DevName: rootDriveConf.DevName, + DevPath: mountInfo.DiskPath, + BackingPath: mountInfo.BackingPath, + Opts: rootDriveConf.Opts, + TargetPath: rootDriveConf.TargetPath, + Limits: rootDriveConf.Limits, } if d.storagePool.Driver().Info().Remote { @@ -4540,13 +4542,14 @@ func (d *qemu) addDriveConfig(qemuDev map[string]any, bootIndexes map[string]int } var isBlockDev bool + var srcDevPath string // Detect device caches and I/O modes. if isRBDImage { // For RBD, we want writeback to allow for the system-configured "rbd cache" to take effect if present. cacheMode = "writeback" } else { - srcDevPath := driveConf.DevPath // This should not be used for passing to QEMU, only for probing. + srcDevPath = driveConf.DevPath // This should not be used for passing to QEMU, only for probing. // Detect if existing file descriptor format is being supplied. if strings.HasPrefix(driveConf.DevPath, fmt.Sprintf("%s:", device.DiskFileDescriptorMountPrefix)) { @@ -4852,7 +4855,40 @@ func (d *qemu) addDriveConfig(qemuDev map[string]any, bootIndexes map[string]int _ = m.RemoveFDFromFDSet(nodeName) }) - blockDev["filename"] = fmt.Sprintf("/dev/fdset/%d", info.ID) + isQcow2, err := d.isQCOW2(srcDevPath) + if err != nil { + return fmt.Errorf("Failed checking disk format: %w", err) + } + + if isQcow2 { + blockDev = map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": d.blockNodeName(escapedDeviceName), + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + "aio": aioMode, + "cache": map[string]any{ + "direct": directCache, + "no-flush": noFlushCache, + }, + }, + } + + // If there are any children, load block information about them. + if len(driveConf.BackingPath) > 0 { + backingBlockDev, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, driveConf.BackingPath, 0) + if err != nil { + return nil + } + + blockDev["backing"] = backingBlockDev + } + } else { + blockDev["filename"] = fmt.Sprintf("/dev/fdset/%d", info.ID) + } } err := m.AddBlockDevice(blockDev, qemuDev, bus == "usb") @@ -10419,3 +10455,167 @@ func (d *qemu) GuestOS() string { return "unknown" } + +// CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. +func (d *qemu) CreateQcow2Snapshot(snapshotName string, backingFilename string) error { + monitor, err := d.qmpConnect() + if err != nil { + return err + } + + snap, err := instance.LoadByProjectAndName(d.state, d.project.Name, fmt.Sprintf("%s/%s", d.name, snapshotName)) + if err != nil { + return fmt.Errorf("Load by project and name: %w", err) + } + + pool, err := storagePools.LoadByInstance(d.state, snap) + if err != nil { + return fmt.Errorf("Load by instance: %w", err) + } + + mountInfoRoot, err := pool.MountInstance(d, d.op) + if err != nil { + return fmt.Errorf("Mount instance: %w", err) + } + + defer func() { _ = pool.UnmountInstance(d, d.op) }() + + f, err := os.OpenFile(mountInfoRoot.DiskPath, unix.O_RDWR, 0) + if err != nil { + return fmt.Errorf("Failed opening file descriptor for disk device %s: %w", mountInfoRoot.DiskPath, err) + } + + defer func() { _ = f.Close() }() + + // Fetch information about block devices. + blockdevNames, err := monitor.QueryNamedBlockNodes() + if err != nil { + return fmt.Errorf("Failed fetching block nodes names: %w", err) + } + + rootDevName, _, err := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative()) + if err != nil { + return fmt.Errorf("Failed getting instance root disk: %w", err) + } + + escapedDeviceName := linux.PathNameEncode(rootDevName) + rootNodeName := d.blockNodeName(escapedDeviceName) + + // Fetch the current maximum overlay index. + overlayNodeIndex := currentQcow2OverlayIndex(blockdevNames, rootNodeName) + nextOverlayName := fmt.Sprintf("%s_overlay%d", rootNodeName, overlayNodeIndex+1) + + currentOverlayName := rootNodeName + if overlayNodeIndex >= 0 { + currentOverlayName = fmt.Sprintf("%s_overlay%d", rootNodeName, overlayNodeIndex) + } + + info, err := monitor.SendFileWithFDSet(nextOverlayName, f, false) + if err != nil { + return fmt.Errorf("Failed sending file descriptor of %q for disk device: %w", f.Name(), err) + } + + blockDev := map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": nextOverlayName, + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + }, + } + + // Add overlay block dev. + err = monitor.AddBlockDevice(blockDev, nil, false) + if err != nil { + return fmt.Errorf("Fail to add block device: %w", err) + } + + // Take a snapshot of the root disk and redirect writes to the snapshot disk. + err = monitor.BlockDevSnapshot(currentOverlayName, nextOverlayName) + if err != nil { + return fmt.Errorf("Failed taking storage snapshot: %w", err) + } + + // Update metadata of the backing file. + err = monitor.ChangeBackingFile(nextOverlayName, backingFilename) + if err != nil { + return fmt.Errorf("Failed changing backing file: %w", err) + } + + return nil +} + +func (d *qemu) isQCOW2(devPath string) (bool, error) { + imgInfo, err := storageDrivers.Qcow2Info(devPath) + if err != nil { + return false, err + } + + return imgInfo.Format == storageDrivers.BlockVolumeTypeQcow2, nil +} + +func (d *qemu) qcow2BlockDev(m *qmp.Monitor, nodeName string, aioMode string, directCache bool, noFlushCache bool, permissions int, readonly bool, backingPaths []string, iter int) (map[string]any, error) { + devName := backingPaths[0] + backingNodeName := fmt.Sprintf("%s_backing%d", nodeName, iter) + + f, err := os.OpenFile(devName, permissions, 0) + if err != nil { + return nil, fmt.Errorf("Failed opening file descriptor for disk device %q: %w", devName, err) + } + + defer func() { _ = f.Close() }() + + info, err := m.SendFileWithFDSet(backingNodeName, f, readonly) + if err != nil { + return nil, fmt.Errorf("Failed sending file descriptor of %q for disk device %q: %w", f.Name(), devName, err) + } + + blockDev := map[string]any{ + "driver": "qcow2", + "discard": "unmap", // Forward as an unmap request. This is the same as `discard=on` in the qemu config file. + "node-name": backingNodeName, + "read-only": false, + "file": map[string]any{ + "driver": "host_device", + "filename": fmt.Sprintf("/dev/fdset/%d", info.ID), + "aio": aioMode, + "cache": map[string]any{ + "direct": directCache, + "no-flush": noFlushCache, + }, + }, + } + + // If there are any children, load block information about them. + if len(backingPaths) > 1 { + backingBlockDev, err := d.qcow2BlockDev(m, nodeName, aioMode, directCache, noFlushCache, permissions, readonly, backingPaths[1:], iter+1) + if err != nil { + return nil, err + } + + blockDev["backing"] = backingBlockDev + } + + return blockDev, nil +} + +// currentQcow2OverlayIndex returns the current maximum overlay index. +func currentQcow2OverlayIndex(names []string, prefix string) int { + re := regexp.MustCompile(fmt.Sprintf(`^%s_overlay(\d+)$`, prefix)) + + maxIndex := -1 + + for _, name := range names { + m := re.FindStringSubmatch(name) + if len(m) == 2 { + n, err := strconv.Atoi(m[1]) + if err == nil && n > maxIndex { + maxIndex = n + } + } + } + + return maxIndex +} diff --git a/internal/server/instance/drivers/qmp/commands.go b/internal/server/instance/drivers/qmp/commands.go index 720f8435ee0..20332e3b60c 100644 --- a/internal/server/instance/drivers/qmp/commands.go +++ b/internal/server/instance/drivers/qmp/commands.go @@ -1128,6 +1128,47 @@ func (m *Monitor) NBDBlockExportAdd(deviceNodeName string) error { return nil } +// QueryNamedBlockNodes returns block nodes names. +func (m *Monitor) QueryNamedBlockNodes() ([]string, error) { + var resp struct { + Return []struct { + NodeName string `json:"node-name"` + } `json:"return"` + } + + err := m.Run("query-named-block-nodes", nil, &resp) + if err != nil { + return nil, err + } + + result := []string{} + for _, r := range resp.Return { + result = append(result, r.NodeName) + } + + return result, nil +} + +// ChangeBackingFile changes backing file name for node. +func (m *Monitor) ChangeBackingFile(nodeName string, backingFilename string) error { + var args struct { + Device string `json:"device"` + NodeName string `json:"image-node-name"` + BackingFile string `json:"backing-file"` + } + + args.Device = nodeName + args.NodeName = nodeName + args.BackingFile = backingFilename + + err := m.Run("change-backing-file", args, nil) + if err != nil { + return err + } + + return nil +} + // BlockDevSnapshot creates a snapshot of a device using the specified snapshot device. func (m *Monitor) BlockDevSnapshot(deviceNodeName string, snapshotNodeName string) error { var args struct { diff --git a/internal/server/instance/instance_interface.go b/internal/server/instance/instance_interface.go index e23d43f60af..531ef8d66b8 100644 --- a/internal/server/instance/instance_interface.go +++ b/internal/server/instance/instance_interface.go @@ -98,6 +98,7 @@ type Instance interface { Backups() ([]backup.InstanceBackup, error) UpdateBackupFile() error CanLiveMigrate() bool + CreateQcow2Snapshot(snapshotName string, backingFilename string) error // Config handling. Rename(newName string, applyTemplateTrigger bool) error diff --git a/internal/server/metadata/configuration.json b/internal/server/metadata/configuration.json index 8369d294891..2bcd078ae47 100644 --- a/internal/server/metadata/configuration.json +++ b/internal/server/metadata/configuration.json @@ -6410,6 +6410,14 @@ "storage_lvm": { "common": { "keys": [ + { + "block.type": { + "condition": "block-based volume", + "default": "same as `volume.block.type`", + "longdesc": "", + "shortdesc": "Type of the block volume" + } + }, { "lvm.metadata_size": { "default": "`0` (auto)", diff --git a/internal/server/storage/backend.go b/internal/server/storage/backend.go index 7f19f1d73d7..300dee2eeb1 100644 --- a/internal/server/storage/backend.go +++ b/internal/server/storage/backend.go @@ -54,6 +54,7 @@ import ( "github.com/lxc/incus/v6/shared/ioprogress" "github.com/lxc/incus/v6/shared/logger" "github.com/lxc/incus/v6/shared/revert" + "github.com/lxc/incus/v6/shared/subprocess" "github.com/lxc/incus/v6/shared/units" "github.com/lxc/incus/v6/shared/util" ) @@ -685,7 +686,7 @@ func (b *backend) CreateInstance(inst instance.Instance, op *operations.Operatio var filler *drivers.VolumeFiller if inst.Type() == instancetype.Container { filler = &drivers.VolumeFiller{ - Fill: func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { + Fill: func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { // Create an empty rootfs. err := os.Mkdir(filepath.Join(vol.MountPath(), "rootfs"), 0o755) if err != nil && !os.IsExist(err) { @@ -1707,8 +1708,8 @@ func (b *backend) RefreshInstance(inst instance.Instance, src instance.Instance, // imageFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will unpack the specified image archive into the specified mount path // provided, and for VM images, a raw root block path is required to unpack the qcow2 image into. -func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { - return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { +func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { + return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { var tracker *ioprogress.ProgressTracker if op != nil { // Not passed when being done as part of pre-migration setup. metadata := make(map[string]any) @@ -1721,15 +1722,15 @@ func (b *backend) imageFiller(fingerprint string, op *operations.Operation) func } imageFile := internalUtil.VarPath("images", fingerprint) - return ImageUnpack(imageFile, vol, rootBlockPath, b.state.OS, allowUnsafeResize, targetIsZero, tracker) + return ImageUnpack(imageFile, vol, rootBlockPath, b.state.OS, allowUnsafeResize, targetIsZero, tracker, targetFormat) } } // isoFiller returns a function that can be used as a filler function with CreateVolume(). // The function returned will copy the ISO content into the specified mount path // provided. -func (b *backend) isoFiller(data io.Reader) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { - return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) { +func (b *backend) isoFiller(data io.Reader) func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { + return func(vol drivers.Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) { f, err := os.OpenFile(rootBlockPath, os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return -1, err @@ -2879,10 +2880,36 @@ func (b *backend) MountInstance(inst instance.Instance, op *operations.Operation return nil, fmt.Errorf("Failed getting disk path: %w", err) } + backingPaths := []string{} + if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return nil, err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) + err = b.driver.MountVolumeSnapshot(currentSnapVol, op) + if err != nil { + return nil, err + } + } + + // Fetch backing chain for a qcow2 formatted volume. + backingPaths, err = b.qcow2BackingPaths(vol, diskPath, inst.Project().Name) + if err != nil { + return nil, err + } + } + mountInfo := &MountInfo{ - DiskPath: diskPath, + DiskPath: diskPath, + BackingPath: backingPaths, } + l.Debug("MountInstance mountInfo", logger.Ctx{"mountInfo": mountInfo}) + reverter.Success() // From here on it is up to caller to call UnmountInstance() when done. // Handle delegation. @@ -2937,6 +2964,22 @@ func (b *backend) UnmountInstance(inst instance.Instance, op *operations.Operati vol = b.GetVolume(volType, contentType, volStorageName, nil) } + if vol.Config()["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), vol.Config()) + _, err = b.driver.UnmountVolumeSnapshot(currentSnapVol, op) + if err != nil && !errors.Is(err, drivers.ErrInUse) { + return err + } + } + } + _, err = b.driver.UnmountVolume(vol, false, op) return err @@ -3065,8 +3108,11 @@ func (b *backend) CreateInstanceSnapshot(inst instance.Instance, src instance.In volStorageName := project.Instance(inst.Project().Name, inst.Name()) // Get the volume. - // There's no need to pass config as it's not needed when creating volume snapshots. - vol := b.GetVolume(volType, contentType, volStorageName, nil) + vol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) + err = b.applyInstanceRootDiskOverrides(inst, &vol) + if err != nil { + return err + } // Lock this operation to ensure that the only one snapshot is made at the time. // Other operations will wait for this one to finish. @@ -3082,6 +3128,18 @@ func (b *backend) CreateInstanceSnapshot(inst instance.Instance, src instance.In return err } + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + // Get the parent volume. + parentVol := b.GetVolume(volType, contentType, src.Name(), srcDBVol.Config) + + // parentVol should already be prepared as an overlay by CreateVolumeSnapshot. + // vol will be used as the base. + err = b.qcow2CreateSnapshot(parentVol, vol, src, op) + if err != nil { + return err + } + } + err = b.ensureInstanceSnapshotSymlink(inst.Type(), inst.Project().Name, inst.Name()) if err != nil { return err @@ -3124,18 +3182,33 @@ func (b *backend) RenameInstanceSnapshot(inst instance.Instance, newName string, return errors.New("Volume name must be a snapshot") } + newVolName := drivers.GetSnapshotVolumeName(parentName, newName) + + // Load storage volume from database. + srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) + if err != nil { + return err + } + contentType := InstanceContentType(inst) volStorageName := project.Instance(inst.Project().Name, inst.Name()) - // Rename storage volume snapshot. No need to pass config as it's not needed when renaming a volume. - snapVol := b.GetVolume(volType, contentType, volStorageName, nil) + // Rename storage volume snapshot. + snapVol := b.GetVolume(volType, contentType, volStorageName, srcDBVol.Config) + + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + parentVol := b.GetVolume(volType, contentType, parentName, srcDBVol.Config) + err = b.qcow2RenameSnapshot(parentVol, snapVol, newVolName, inst.Type(), inst.Project().Name, op) + if err != nil { + return err + } + } + err = b.driver.RenameVolumeSnapshot(snapVol, newName, op) if err != nil { return err } - newVolName := drivers.GetSnapshotVolumeName(parentName, newName) - reverter.Add(func() { // Revert rename. No need to pass config as it's not needed when renaming a volume. newSnapVol := b.GetVolume(volType, contentType, project.Instance(inst.Project().Name, newVolName), nil) @@ -3195,8 +3268,13 @@ func (b *backend) DeleteInstanceSnapshot(inst instance.Instance, op *operations. snapVolName := drivers.GetSnapshotVolumeName(parentStorageName, snapName) - // There's no need to pass config as it's not needed when deleting a volume snapshot. - vol := b.GetVolume(volType, contentType, snapVolName, nil) + // Load storage volume from database. + srcDBVol, err := VolumeDBGet(b, inst.Project().Name, inst.Name(), volType) + if err != nil { + return err + } + + vol := b.GetVolume(volType, contentType, snapVolName, srcDBVol.Config) volExists, err := b.driver.HasVolume(vol) if err != nil { @@ -3204,6 +3282,40 @@ func (b *backend) DeleteInstanceSnapshot(inst instance.Instance, op *operations. } if volExists { + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + randomSuffix, err := internalUtil.RandomHexString(10) + if err != nil { + return fmt.Errorf("Failed creating temporary file name: %w", err) + } + + parent, _, _ := api.GetParentAndSnapshotName(vol.Name()) + tmpSnapName := fmt.Sprintf("%s_%s", parent, randomSuffix) + tmpVol := b.GetVolume(vol.Type(), vol.ContentType(), tmpSnapName, vol.Config()) + + err = b.applyInstanceRootDiskOverrides(inst, &tmpVol) + if err != nil { + return err + } + + // Create temporary volume. + err = b.driver.CreateVolume(tmpVol, nil, op) + if err != nil { + return err + } + + parentVol := b.GetVolume(volType, contentType, parentName, srcDBVol.Config) + err = b.qcow2DeleteSnapshot(parentVol, vol, tmpVol, inst, op) + if err != nil { + return err + } + + // Delete temporary volume. + err = b.driver.DeleteVolume(tmpVol, op) + if err != nil { + return err + } + } + err = b.driver.DeleteVolumeSnapshot(vol, op) if err != nil { return err @@ -3305,6 +3417,16 @@ func (b *backend) RestoreInstanceSnapshot(inst instance.Instance, src instance.I }) } + if srcDBVol.Config["block.type"] == drivers.BlockVolumeTypeQcow2 { + snapVol := b.GetVolume(volType, contentType, src.Name(), srcDBVol.Config) + err = b.qcow2RestoreSnapshot(vol, snapVol, inst.Type(), inst.Project().Name, op) + if err != nil { + return err + } + + return nil + } + err = b.driver.RestoreVolume(vol, snapshotName, op) if err != nil { var snapErr drivers.ErrDeleteSnapshots @@ -7506,3 +7628,364 @@ func (b *backend) getFirstAdminStorageBucketPoolKey(projectName string, bucketNa return bucketKey, nil } + +// qcow2CreateSnapshot creates the QCOW2 volume snapshot. +func (b *backend) qcow2CreateSnapshot(vol drivers.Volume, snapVol drivers.Volume, src instance.Instance, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + if src.Type() == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err := drivers.Qcow2CreateConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + err = vol.MountWithSnapshotsTask(func(parentMountPath string, parentOp *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + if !src.IsRunning() { + err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) + if err != nil { + return err + } + } else { + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + err = drivers.Qcow2Create(parentDiskPath, "", int64(imgInfo.VirtualSize)) + if err != nil { + return err + } + } + + return nil + }, op) + if err != nil { + return err + } + + if src.IsRunning() { + _, snapshotName, isSnap := api.GetParentAndSnapshotName(snapVol.Name()) + if !isSnap { + return errors.New("Volume name must be a snapshot") + } + + err = src.CreateQcow2Snapshot(snapshotName, snapVolDevPath) + if err != nil { + return err + } + } + + return nil +} + +// qcow2RestoreSnapshot restores the QCOW2 volume snapshot. +func (b *backend) qcow2RestoreSnapshot(vol drivers.Volume, snapVol drivers.Volume, instanceType instancetype.Type, projectName string, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + snapVolDevPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + // Restoring is allowed only for the most recent snapshot. + if imgInfo.BackingFilename != snapVolDevPath { + return fmt.Errorf("Snapshot %q cannot be restored due to subsequent snapshot(s).", snapVol.Name()) + } + + err = drivers.Qcow2Create(parentDiskPath, snapVolDevPath, 0) + if err != nil { + return err + } + + return nil + }, op) + if err != nil { + return err + } + + if instanceType == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err = drivers.Qcow2RestoreConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2RenameSnapshot renames the QCOW2 volume snapshot. +func (b *backend) qcow2RenameSnapshot(vol drivers.Volume, snapVol drivers.Volume, newVolName string, instanceType instancetype.Type, projectName string, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + newSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, newVolName), nil) + newSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(newSnapVol) + if err != nil { + return err + } + + snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, projectName, vol.Name(), vol.Type()) + if err != nil { + return err + } + + // Update the metadata of the parent if it points to a renamed volume. + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + err = drivers.Qcow2Rebase(parentDiskPath, newSnapVolDiskPath) + if err != nil { + return err + } + } + + // Update the metadata of the snapshot which points to a renamed volume. + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + err = drivers.Qcow2Rebase(currentSnapVolDiskPath, newSnapVolDiskPath) + if err != nil { + return err + } + } + } + + return nil + }, op) + if err != nil { + return err + } + + if instanceType == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + _, newName, _ := api.GetParentAndSnapshotName(newVolName) + err = drivers.Qcow2RenameConfigSnapshot(fsParentVol, fsVol, newName, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2DeleteSnapshot deletes the QCOW2 volume snapshot. +func (b *backend) qcow2DeleteSnapshot(vol drivers.Volume, snapVol drivers.Volume, tmpVol drivers.Volume, inst instance.Instance, op *operations.Operation) error { + // Return if this is not a qcow2 image. + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil + } + + var destinationVol string + var baseVol string + + _, snapName, _ := api.GetParentAndSnapshotName(inst.Name()) + + snapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(snapVol) + if err != nil { + return err + } + + // Get parent volume backing file + parentVolDiskPath, err := b.driver.GetQcow2BackingFilePath(vol) + if err != nil { + return err + } + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, inst.Project().Name, vol.Name(), vol.Type()) + if err != nil { + return err + } + + // Check if the main volume is the parent of the snapshot to be deleted. + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + parentDiskPath, err := b.driver.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + imgInfo, err := drivers.Qcow2Info(parentDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + destinationVol = parentVolDiskPath + } + + // Check which snapshot is the parent of the snapshot to be deleted. + // Record information about the snapshot that will be deleted. + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(inst.Project().Name, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return err + } + + _, currentSnapName, _ := api.GetParentAndSnapshotName(snap.Name) + + imgInfo, err := drivers.Qcow2Info(currentSnapVolDiskPath) + if err != nil { + return err + } + + if imgInfo.BackingFilename == snapVolDiskPath { + destinationVol = currentSnapVolDiskPath + } + + if snapName == currentSnapName { + baseVol = imgInfo.BackingFilename + } + } + + tmpVolDiskPath, err := b.driver.GetQcow2BackingFilePath(tmpVol) + if err != nil { + return err + } + + // Move the data from the deleted snapshot to temporary volume. + err = drivers.Qcow2Convert(destinationVol, baseVol, tmpVolDiskPath) + if err != nil { + return err + } + + // Copy data from the temporary volume to the parent of the deleted snapshot. + _, err = subprocess.RunCommand("dd", "bs=4M", fmt.Sprintf("if=%s", tmpVolDiskPath), fmt.Sprintf("of=%s", destinationVol)) + if err != nil { + return err + } + + return nil + }, op) + if err != nil { + return err + } + + if inst.Type() == instancetype.VM { + fsParentVol := vol.NewVMBlockFilesystemVolume() + fsVol := snapVol.NewVMBlockFilesystemVolume() + err = drivers.Qcow2DeleteConfigSnapshot(fsParentVol, fsVol, op) + if err != nil { + return err + } + } + + return nil +} + +// qcow2BackingPaths returns information about the backing chain of a qcow2 image. +func (b *backend) qcow2BackingPaths(vol drivers.Volume, diskPath string, projectName string) ([]string, error) { + if vol.Config()["block.type"] != drivers.BlockVolumeTypeQcow2 { + return nil, nil + } + + backingPaths := []string{} + diskPathSnapshot := map[string]string{} + + // Get snapshots. + volSnaps, err := VolumeDBSnapshotsGet(b, projectName, vol.Name(), vol.Type()) + if err != nil { + return nil, err + } + + for _, snap := range volSnaps { + currentSnapVol := b.GetVolume(vol.Type(), vol.ContentType(), project.Instance(projectName, snap.Name), nil) + currentSnapVolDiskPath, err := b.driver.GetQcow2BackingFilePath(currentSnapVol) + if err != nil { + return nil, err + } + + _, snapName, _ := api.GetParentAndSnapshotName(snap.Name) + diskPathSnapshot[currentSnapVolDiskPath] = snapName + } + + b.logger.Debug("Disk path snapshots map:", logger.Ctx{"diskPathSnapshots": diskPathSnapshot}) + + var chainPaths []string + err = vol.MountWithSnapshotsTask(func(mountPath string, op *operations.Operation) error { + chainPaths, err = drivers.Qcow2BackingChain(diskPath) + if err != nil { + return nil + } + + for _, p := range chainPaths { + snapName := diskPathSnapshot[p] + snapVolName := drivers.GetSnapshotVolumeName(vol.Name(), snapName) + snapVol := b.GetVolume(vol.Type(), vol.ContentType(), snapVolName, nil) + + volDiskPath, err := b.driver.GetVolumeDiskPath(snapVol) + if err != nil { + return err + } + + backingPaths = append(backingPaths, volDiskPath) + } + + return nil + }, nil) + if err != nil { + return nil, err + } + + return backingPaths, nil +} diff --git a/internal/server/storage/drivers/driver_common.go b/internal/server/storage/drivers/driver_common.go index 04de280fcc1..81b5ff68a7c 100644 --- a/internal/server/storage/drivers/driver_common.go +++ b/internal/server/storage/drivers/driver_common.go @@ -557,3 +557,8 @@ func (d *common) filesystemFreeze(path string) (func() error, error) { func (d *common) CacheVolumeSnapshots(vol Volume) error { return nil } + +// GetQcow2BackingFilePath generates the backing file path for the specified volume. +func (d *common) GetQcow2BackingFilePath(vol Volume) (string, error) { + return "", ErrNotSupported +} diff --git a/internal/server/storage/drivers/driver_lvm.go b/internal/server/storage/drivers/driver_lvm.go index 250b3d30724..8e68dc1b7f9 100644 --- a/internal/server/storage/drivers/driver_lvm.go +++ b/internal/server/storage/drivers/driver_lvm.go @@ -141,8 +141,10 @@ func (d *lvm) isRemote() bool { // Info returns info about the driver and its environment. func (d *lvm) Info() Info { name := "lvm" + targetFormat := BlockVolumeTypeRaw if d.clustered { name = "lvmcluster" + targetFormat = BlockVolumeTypeQcow2 } return Info{ @@ -163,6 +165,7 @@ func (d *lvm) Info() Info { Buckets: !d.isRemote(), Deactivate: d.isRemote(), ZeroUnpack: !d.usesThinpool(), + TargetFormat: targetFormat, } } @@ -725,6 +728,11 @@ func (d *lvm) Update(changedConfig map[string]string) error { return errors.New("volume.lvm.stripes.size cannot be changed when using thin pool") } + _, changed = changedConfig["volume.block.type"] + if changed { + return errors.New("volume.block.type cannot be changed") + } + if changedConfig["lvm.vg_name"] != "" { _, err := subprocess.TryRunCommand("vgrename", d.config["lvm.vg_name"], changedConfig["lvm.vg_name"]) if err != nil { diff --git a/internal/server/storage/drivers/driver_lvm_utils.go b/internal/server/storage/drivers/driver_lvm_utils.go index 6377362567c..e7839e1f48e 100644 --- a/internal/server/storage/drivers/driver_lvm_utils.go +++ b/internal/server/storage/drivers/driver_lvm_utils.go @@ -871,7 +871,7 @@ func (d *lvm) parseLogicalVolumeSnapshot(parent Volume, lvmVolName string) strin func (d *lvm) activateVolume(vol Volume) (bool, error) { var volPath string - if d.usesThinpool() { + if d.usesThinpool() || (vol.Config()["block.type"] == BlockVolumeTypeQcow2 && vol.ContentType() == ContentTypeBlock) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as activating the parent volume also activates its snapshots. @@ -915,7 +915,7 @@ func (d *lvm) activateVolume(vol Volume) (bool, error) { func (d *lvm) deactivateVolume(vol Volume) (bool, error) { var volPath string - if d.usesThinpool() { + if d.usesThinpool() || (vol.Config()["block.type"] == BlockVolumeTypeQcow2 && vol.ContentType() == ContentTypeBlock) { volPath = d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) } else { // Use parent for non-thinpool vols as deactivating the parent volume also activates its snapshots. diff --git a/internal/server/storage/drivers/driver_lvm_volumes.go b/internal/server/storage/drivers/driver_lvm_volumes.go index dd29229394b..267ab7ab4ae 100644 --- a/internal/server/storage/drivers/driver_lvm_volumes.go +++ b/internal/server/storage/drivers/driver_lvm_volumes.go @@ -9,6 +9,7 @@ import ( "math" "os" "os/exec" + "path/filepath" "strings" "golang.org/x/sys/unix" @@ -58,6 +59,30 @@ func (d *lvm) CreateVolume(vol Volume, filler *VolumeFiller, op *operations.Oper reverter.Add(func() { _ = d.DeleteVolume(fsVol, op) }) } + // Format LV as qcow2 (lvmcluster). + if d.clustered && vol.ContentType() == ContentTypeBlock { + // Get the device path. + devPath, err := d.GetVolumeDiskPath(vol) + if err != nil { + return err + } + + qcow2SizeBytes, err := d.roundedSizeBytesString(vol.ConfigSize()) + if err != nil { + return err + } + + err = Qcow2Create(devPath, "", qcow2SizeBytes) + if err != nil { + return err + } + } else if d.clustered && vol.ContentType() == ContentTypeFS { + err = Qcow2CreateConfig(vol, op) + if err != nil { + return err + } + } + err = vol.MountTask(func(mountPath string, op *operations.Operation) error { // Run the volume filler function if supplied. if filler != nil && filler.Fill != nil { @@ -295,6 +320,13 @@ func (d *lvm) FillVolumeConfig(vol Volume) error { } } + if d.clustered && vol.ContentType() == ContentTypeBlock { + if vol.config["block.type"] == "" { + // Unchangeable volume property: Set unconditionally. + vol.config["block.type"] = BlockVolumeTypeQcow2 + } + } + // Inherit stripe settings from pool if not set and not using thin pool. if !d.usesThinpool() { if vol.config["lvm.stripes"] == "" { @@ -311,7 +343,7 @@ func (d *lvm) FillVolumeConfig(vol Volume) error { // commonVolumeRules returns validation rules which are common for pool and volume. func (d *lvm) commonVolumeRules() map[string]func(value string) error { - return map[string]func(value string) error{ + rules := map[string]func(value string) error{ // gendoc:generate(entity=storage_volume_lvm, group=common, key=block.mount_options) // // --- @@ -348,6 +380,19 @@ func (d *lvm) commonVolumeRules() map[string]func(value string) error { // shortdesc: Size of stripes to use (at least 4096 bytes and multiple of 512 bytes) "lvm.stripes.size": validate.Optional(validate.IsSize), } + + if d.clustered { + // gendoc:generate(entity=storage_lvm, group=common, key=block.type) + // + // --- + // type:string + // condition: block-based volume + // default: same as `volume.block.type` + // shortdesc: Type of the block volume + rules["block.type"] = validate.Optional(validate.IsOneOf(BlockVolumeTypeRaw, BlockVolumeTypeQcow2)) + } + + return rules } // ValidateVolume validates the supplied volume config. @@ -495,6 +540,11 @@ func (d *lvm) UpdateVolume(vol Volume, changedConfig map[string]string) error { return errors.New("lvm.stripes.size cannot be changed") } + _, changed = changedConfig["block.type"] + if changed { + return errors.New("block.type cannot be changed") + } + return nil } @@ -750,11 +800,7 @@ func (d *lvm) SetVolumeQuota(vol Volume, size string, allowUnsafeResize bool, op // GetVolumeDiskPath returns the location of a disk volume. func (d *lvm) GetVolumeDiskPath(vol Volume) (string, error) { - if vol.IsVMBlock() || (vol.volType == VolumeTypeCustom && IsContentBlock(vol.contentType)) { - return d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) - } - - return "", ErrNotSupported + return d.lvmDevPath(d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name)) } // ListVolumes returns a list of volumes in storage pool. @@ -1131,15 +1177,42 @@ func (d *lvm) BackupVolume(vol Volume, writer instancewriter.InstanceWriter, _ b // CreateVolumeSnapshot creates a snapshot of a volume. func (d *lvm) CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error { - // Perform validation - if d.isRemote() && snapVol.ContentType() == ContentTypeBlock { - return fmt.Errorf("lvmcluster doesn't currently support snapshot creation") - } - parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) parentVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) snapPath := snapVol.MountPath() + if d.isRemote() && snapVol.ContentType() == ContentTypeBlock { + if util.IsTrue(snapVol.ExpandedConfig("security.shared")) { + return fmt.Errorf("Creating an lvmcluster snapshot with the 'security.shared' flag enabled is prohibited.") + } + + if snapVol.ExpandedConfig("block.type") != BlockVolumeTypeQcow2 { + return fmt.Errorf("Creating an lvmcluster snapshot with the 'block.type' different than 'qcow2' is prohibited.") + } + + parentVolPath := d.lvmPath(d.config["lvm.vg_name"], parentVol.volType, parentVol.contentType, parentName) + snapVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, snapVol.name) + + release, err := d.acquireExclusive(parentVol) + if err != nil { + return err + } + + defer release() + + err = d.renameLogicalVolume(parentVolPath, snapVolPath) + if err != nil { + return fmt.Errorf("Error temporarily renaming original LVM logical volume: %w", err) + } + + err = d.createLogicalVolume(d.config["lvm.vg_name"], d.thinpoolName(), parentVol, d.usesThinpool()) + if err != nil { + return fmt.Errorf("Error creating LVM logical volume: %w", err) + } + + return nil + } + // Create the parent directory. err := createParentSnapshotDirIfMissing(d.name, snapVol.volType, parentName) if err != nil { @@ -1256,74 +1329,102 @@ func (d *lvm) MountVolumeSnapshot(snapVol Volume, op *operations.Operation) erro mountVol := snapVol mountFlags, mountOptions := linux.ResolveMountOptions(strings.Split(mountVol.ConfigBlockMountOptions(), ",")) - // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting - // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need - // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. - // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be - // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly - // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time - // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then - // mount that. - regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) - if regenerateFSUUID { - // Instantiate a new volume to be the temporary writable snapshot. - tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) - tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) - - // Create writable snapshot from source snapshot named with a tmpVolSuffix suffix. - _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, tmpVol, false, d.usesThinpool()) + isQcow2 := snapVol.ExpandedConfig("block.type") == BlockVolumeTypeQcow2 + if isQcow2 { + parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) + mountVol = NewVolume(d, d.name, snapVol.volType, snapVol.contentType, parentName, snapVol.config, snapVol.poolConfig) + + // Activate volume if needed. + _, err = d.activateVolume(mountVol) if err != nil { - return fmt.Errorf("Error creating temporary LVM logical volume snapshot: %w", err) + return err } - reverter.Add(func() { - _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], tmpVol.volType, tmpVol.contentType, tmpVol.name)) - }) + // Get volume path. + volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) - // We are going to mount the temporary volume instead. - mountVol = tmpVol - } + volDevPath, err := d.lvmDevPath(volPath) + if err != nil { + return err + } - // Activate volume if needed. - _, err = d.activateVolume(mountVol) - if err != nil { - return err - } + // Finally attempt to mount the volume that needs mounting. + err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) + if err != nil { + return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) + } - // Get volume path. - volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) + d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + } else { + // Regenerate filesystem UUID if needed. This is because some filesystems do not allow mounting + // multiple volumes that share the same UUID. As snapshotting a volume will copy its UUID we need + // to potentially regenerate the UUID of the snapshot now that we are trying to mount it. + // This is done at mount time rather than snapshot time for 2 reasons; firstly snapshots need to be + // as fast as possible, and on some filesystems regenerating the UUID is a slow process, secondly + // we do not want to modify a snapshot in case it is corrupted for some reason, so at mount time + // we take another snapshot of the snapshot, regenerate the temporary snapshot's UUID and then + // mount that. + regenerateFSUUID := renegerateFilesystemUUIDNeeded(snapVol.ConfigBlockFilesystem()) + if regenerateFSUUID { + // Instantiate a new volume to be the temporary writable snapshot. + tmpVolName := fmt.Sprintf("%s%s", snapVol.name, tmpVolSuffix) + tmpVol := NewVolume(d, d.name, snapVol.volType, snapVol.contentType, tmpVolName, snapVol.config, snapVol.poolConfig) + + // Create writable snapshot from source snapshot named with a tmpVolSuffix suffix. + _, err = d.createLogicalVolumeSnapshot(d.config["lvm.vg_name"], snapVol, tmpVol, false, d.usesThinpool()) + if err != nil { + return fmt.Errorf("Error creating temporary LVM logical volume snapshot: %w", err) + } - volDevPath, err := d.lvmDevPath(volPath) - if err != nil { - return err - } + reverter.Add(func() { + _ = d.removeLogicalVolume(d.lvmPath(d.config["lvm.vg_name"], tmpVol.volType, tmpVol.contentType, tmpVol.name)) + }) - if regenerateFSUUID { - tmpVolFsType := mountVol.ConfigBlockFilesystem() + // We are going to mount the temporary volume instead. + mountVol = tmpVol + } - // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully - // regenerating the filesystem UUID. - if tmpVolFsType == "xfs" { - idx := strings.Index(mountOptions, "nouuid") - if idx < 0 { - mountOptions += ",nouuid" - } - } else { - d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) - err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volDevPath) - if err != nil { - return err + // Activate volume if needed. + _, err = d.activateVolume(mountVol) + if err != nil { + return err + } + + // Get volume path. + volPath := d.lvmPath(d.config["lvm.vg_name"], mountVol.volType, mountVol.contentType, mountVol.name) + + volDevPath, err := d.lvmDevPath(volPath) + if err != nil { + return err + } + + if regenerateFSUUID { + tmpVolFsType := mountVol.ConfigBlockFilesystem() + + // When mounting XFS filesystems temporarily we can use the nouuid option rather than fully + // regenerating the filesystem UUID. + if tmpVolFsType == "xfs" { + idx := strings.Index(mountOptions, "nouuid") + if idx < 0 { + mountOptions += ",nouuid" + } + } else { + d.logger.Debug("Regenerating filesystem UUID", logger.Ctx{"dev": volPath, "fs": tmpVolFsType}) + err = regenerateFilesystemUUID(mountVol.ConfigBlockFilesystem(), volDevPath) + if err != nil { + return err + } } } - } - // Finally attempt to mount the volume that needs mounting. - err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) - if err != nil { - return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) - } + // Finally attempt to mount the volume that needs mounting. + err = TryMount(volDevPath, mountPath, mountVol.ConfigBlockFilesystem(), mountFlags|unix.MS_RDONLY, mountOptions) + if err != nil { + return fmt.Errorf("Failed to mount LVM snapshot volume: %w", err) + } - d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + d.logger.Debug("Mounted logical volume snapshot", logger.Ctx{"dev": volPath, "path": mountPath, "options": mountOptions}) + } } else if snapVol.contentType == ContentTypeBlock { // Activate volume if needed. _, err = d.activateVolume(snapVol) @@ -1655,7 +1756,15 @@ func (d *lvm) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *o parentName, _, _ := api.GetParentAndSnapshotName(snapVol.name) newSnapVolName := GetSnapshotVolumeName(parentName, newSnapshotName) newVolPath := d.lvmPath(d.config["lvm.vg_name"], snapVol.volType, snapVol.contentType, newSnapVolName) - err := d.renameLogicalVolume(volPath, newVolPath) + + release, err := d.acquireExclusive(snapVol) + if err != nil { + return err + } + + defer release() + + err = d.renameLogicalVolume(volPath, newVolPath) if err != nil { return fmt.Errorf("Error renaming LVM logical volume: %w", err) } @@ -1669,3 +1778,9 @@ func (d *lvm) RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *o return nil } + +// GetQcow2BackingFilePath generates the backing file path for the specified volume. +func (d *lvm) GetQcow2BackingFilePath(vol Volume) (string, error) { + pathName := d.lvmPath(d.config["lvm.vg_name"], vol.volType, vol.contentType, vol.name) + return filepath.Join("/dev", pathName), nil +} diff --git a/internal/server/storage/drivers/driver_types.go b/internal/server/storage/drivers/driver_types.go index 92549171aa8..03e003bb62d 100644 --- a/internal/server/storage/drivers/driver_types.go +++ b/internal/server/storage/drivers/driver_types.go @@ -21,11 +21,12 @@ type Info struct { MountedRoot bool // Whether the pool directory itself is a mount. Deactivate bool // Whether an unmount action is required prior to removing the pool. ZeroUnpack bool // Whether to write zeroes (no discard) during unpacking. + TargetFormat string // Whether the output image format should be raw or qcow2. } // VolumeFiller provides a struct for filling a volume. type VolumeFiller struct { - Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool) (int64, error) // Function to fill the volume. + Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool, targetIsZero bool, targetFormat string) (int64, error) // Function to fill the volume. Size int64 // Size of the unpacked volume in bytes. Fingerprint string // If the Filler will unpack an image, it should be this fingerprint. diff --git a/internal/server/storage/drivers/generic_vfs.go b/internal/server/storage/drivers/generic_vfs.go index 825431e0c7c..2a15276145a 100644 --- a/internal/server/storage/drivers/generic_vfs.go +++ b/internal/server/storage/drivers/generic_vfs.go @@ -1210,7 +1210,7 @@ func genericRunFiller(d Driver, vol Volume, devPath string, filler *VolumeFiller } vol.driver.Logger().Debug("Running filler function", logger.Ctx{"dev": devPath, "path": vol.MountPath()}) - volSize, err := filler.Fill(vol, devPath, allowUnsafeResize, !d.Info().ZeroUnpack) + volSize, err := filler.Fill(vol, devPath, allowUnsafeResize, !d.Info().ZeroUnpack, d.Info().TargetFormat) if err != nil { return err } diff --git a/internal/server/storage/drivers/interface.go b/internal/server/storage/drivers/interface.go index 9f5c2e7b7c8..73b2cbc316e 100644 --- a/internal/server/storage/drivers/interface.go +++ b/internal/server/storage/drivers/interface.go @@ -99,6 +99,7 @@ type Driver interface { CacheVolumeSnapshots(vol Volume) error CreateVolumeSnapshot(snapVol Volume, op *operations.Operation) error + GetQcow2BackingFilePath(vol Volume) (string, error) DeleteVolumeSnapshot(snapVol Volume, op *operations.Operation) error RenameVolumeSnapshot(snapVol Volume, newSnapshotName string, op *operations.Operation) error VolumeSnapshots(vol Volume, op *operations.Operation) ([]string, error) diff --git a/internal/server/storage/drivers/utils_qcow2.go b/internal/server/storage/drivers/utils_qcow2.go new file mode 100644 index 00000000000..c942a9f9cef --- /dev/null +++ b/internal/server/storage/drivers/utils_qcow2.go @@ -0,0 +1,276 @@ +package drivers + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/lxc/incus/v6/internal/server/operations" + internalUtil "github.com/lxc/incus/v6/internal/util" + "github.com/lxc/incus/v6/shared/api" + "github.com/lxc/incus/v6/shared/subprocess" +) + +// Type of the block volume. +const ( + BlockVolumeTypeRaw = "raw" + BlockVolumeTypeQcow2 = "qcow2" +) + +// ImageInfo contains information about a qcow2 image. +type ImageInfo struct { + BackingFilename string `json:"backing-filename"` + Format string `json:"format"` + VirtualSize int `json:"virtual-size"` +} + +// Qcow2Create creates a qcow2-formatted image. +func Qcow2Create(path string, backingPath string, size int64) error { + args := []string{ + "create", + "-f", + "qcow2", + } + + if backingPath != "" { + args = append(args, "-b", backingPath) + args = append(args, "-F", "qcow2") + } + + args = append(args, path) + + if size > 0 { + args = append(args, fmt.Sprintf("%db", size)) + } + + _, err := subprocess.RunCommand("qemu-img", args...) + if err != nil { + return err + } + + return nil +} + +// Qcow2Rebase changes the backing file of a qcow2 image. +func Qcow2Rebase(path string, backingPath string) error { + _, err := subprocess.RunCommand("qemu-img", "rebase", "-u", "-b", backingPath, "-F", "qcow2", path) + if err != nil { + return err + } + + return nil +} + +// Qcow2Convert creates an image using the contents of backingPath and path. +func Qcow2Convert(path string, backingPath string, outputPath string) error { + args := []string{ + "convert", + "-O", + "qcow2", + } + + if backingPath != "" { + args = append(args, "-B", backingPath) + args = append(args, "-F", "qcow2") + } + + args = append(args, path, outputPath) + + _, err := subprocess.RunCommand("qemu-img", args...) + if err != nil { + return err + } + + return nil +} + +// Qcow2Info returns information about a qcow2 image. +func Qcow2Info(path string) (*ImageInfo, error) { + imgJSON, err := subprocess.RunCommand("qemu-img", "info", "--output=json", path) + if err != nil { + return nil, err + } + + imgInfo := ImageInfo{} + + err = json.Unmarshal([]byte(imgJSON), &imgInfo) + if err != nil { + return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) + } + + return &imgInfo, nil +} + +// Qcow2BackingChain returns information about the backing chain of a qcow2 image. +func Qcow2BackingChain(path string) ([]string, error) { + result := []string{} + imgJSON, err := subprocess.RunCommand("qemu-img", "info", "--backing-chain", "--output=json", path) + if err != nil { + return nil, err + } + + imgInfo := []struct { + BackingFilename string `json:"backing-filename"` + }{} + + err = json.Unmarshal([]byte(imgJSON), &imgInfo) + if err != nil { + return nil, fmt.Errorf("Failed unmarshalling image info %q: %w (%q)", path, err, imgJSON) + } + + for _, info := range imgInfo { + if info.BackingFilename == "" { + break + } + + result = append(result, info.BackingFilename) + } + + return result, nil +} + +// Qcow2MountConfigTask mounts the config filesystem volume with its snapshots and performs the task specified by the parameter. +func Qcow2MountConfigTask(vol Volume, op *operations.Operation, task func(mountPath string) error) error { + mountPath := fmt.Sprintf("%s%s", vol.MountPath(), tmpVolSuffix) + mountVol := NewVolume(vol.driver, vol.driver.Name(), vol.volType, vol.contentType, vol.name, vol.config, vol.poolConfig) + mountVol.mountFullFilesystem = true + mountVol.mountCustomPath = mountPath + err := mountVol.MountTask(func(mountPath string, op *operations.Operation) error { + taskErr := task(mountVol.MountPath()) + + // Return task error if failed. + if taskErr != nil { + return taskErr + } + + return nil + }, op) + if err != nil { + return err + } + + // Remove temporary mount path. + isEmpty, err := internalUtil.PathIsEmpty(mountPath) + if err != nil { + return err + } + + if isEmpty { + err := os.Remove(mountPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("Failed to remove '%s': %w", mountPath, err) + } + } + + return nil +} + +// Qcow2CreateConfig creates the btrfs config filesystem associated with the QCOW2 block volume. +func Qcow2CreateConfig(vol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + volPath := filepath.Join(mountPath, vol.Name()) + // Create the volume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "create", volPath) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2CreateConfigSnapshot creates the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2CreateConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + dstPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + + _, err := subprocess.RunCommand("btrfs", "subvolume", "snapshot", filepath.Join(mountPath, vol.Name()), dstPath) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2RestoreConfigSnapshot restores the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2RestoreConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + snapPath := fmt.Sprintf("%s-%s", parent, snapName) + + // Delete the subvolume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", filepath.Join(mountPath, parent)) + if err != nil { + return err + } + + _, err = subprocess.RunCommand("btrfs", "subvolume", "snapshot", filepath.Join(mountPath, snapPath), filepath.Join(mountPath, parent)) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2RenameConfigSnapshot renames the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2RenameConfigSnapshot(vol Volume, snapVol Volume, newName string, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + oldPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + newPath := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, newName)) + + err := os.Rename(oldPath, newPath) + if err != nil { + return fmt.Errorf("Failed to rename %q to %q: %w", oldPath, newPath, err) + } + + return nil + }) + if err != nil { + return err + } + + return nil +} + +// Qcow2DeleteConfigSnapshot deletes the btrfs snapshot of the config filesystem associated with the QCOW2 block volume. +func Qcow2DeleteConfigSnapshot(vol Volume, snapVol Volume, op *operations.Operation) error { + err := Qcow2MountConfigTask(vol, op, func(mountPath string) error { + parent, snapName, _ := api.GetParentAndSnapshotName(snapVol.Name()) + path := filepath.Join(mountPath, fmt.Sprintf("%s-%s", parent, snapName)) + + // Delete the subvolume itself. + _, err := subprocess.RunCommand("btrfs", "subvolume", "delete", path) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + + return nil +} diff --git a/internal/server/storage/drivers/volume.go b/internal/server/storage/drivers/volume.go index eb0a179b649..c14c153ce73 100644 --- a/internal/server/storage/drivers/volume.go +++ b/internal/server/storage/drivers/volume.go @@ -122,6 +122,7 @@ type Volume struct { driver Driver mountCustomPath string // Mount the filesystem volume at a custom location. mountFilesystemProbe bool // Probe filesystem type when mounting volume (when needed). + mountFullFilesystem bool // MountWholeVolume indicates whether the whole volume, including snapshots data, should be mounted. It is used by the VM config filesystem. hasSource bool // Whether the volume is created from a source volume. isDeleted bool // Whether we're dealing with a hidden volume (kept until all references are gone). } @@ -364,6 +365,57 @@ func (v Volume) MountTask(task func(mountPath string, op *operations.Operation) return nil } +// MountWithSnapshotsTask runs the supplied task after mounting the volume with its snapshots if needed. If the volume was mounted +// for this then it is unmounted when the task finishes. +func (v Volume) MountWithSnapshotsTask(task func(mountPath string, op *operations.Operation) error, op *operations.Operation) error { + var err error + + if v.IsSnapshot() { + return fmt.Errorf("Volume cannot be snapshot") + } + + err = v.driver.MountVolume(v, op) + if err != nil { + return err + } + + snapshots, err := v.Snapshots(op) + if err != nil { + return err + } + + for _, s := range snapshots { + err = v.driver.MountVolumeSnapshot(s, op) + if err != nil { + return err + } + } + + taskErr := task(v.MountPath(), op) + + // Try and unmount, even on task error. + for _, s := range snapshots { + _, err = v.driver.UnmountVolumeSnapshot(s, op) + if err != nil && !errors.Is(err, ErrInUse) { + return err + } + } + + _, err = v.driver.UnmountVolume(v, false, op) + + // Return task error if failed. + if taskErr != nil { + return taskErr + } + + // Return unmount error if failed. + if err != nil && !errors.Is(err, ErrInUse) { + return err + } + + return nil +} + // UnmountTask runs the supplied task after unmounting the volume if needed. // If the volume was unmounted for this then it is mounted when the task finishes. // keepBlockDev indicates if backing block device should be not be deactivated if volume is unmounted. @@ -515,6 +567,11 @@ func (v Volume) SetConfigStateSize(size string) { // ConfigBlockFilesystem returns the filesystem to use for block volumes. Returns config value "block.filesystem" // if defined in volume or pool's volume config, otherwise the DefaultFilesystem. func (v Volume) ConfigBlockFilesystem() string { + blockType := v.ExpandedConfig("block.type") + if blockType != "" && blockType == BlockVolumeTypeQcow2 && v.contentType == ContentTypeFS { + return "btrfs" + } + fs := v.ExpandedConfig("block.filesystem") if fs != "" { return fs @@ -526,6 +583,17 @@ func (v Volume) ConfigBlockFilesystem() string { // ConfigBlockMountOptions returns the filesystem mount options to use for block volumes. Returns config value // "block.mount_options" if defined in volume or pool's volume config, otherwise defaultFilesystemMountOptions. func (v Volume) ConfigBlockMountOptions() string { + blockType := v.ExpandedConfig("block.type") + if blockType != "" && blockType == BlockVolumeTypeQcow2 && !v.mountFullFilesystem { + parent, snapName, isSnap := api.GetParentAndSnapshotName(v.name) + subvol := parent + if isSnap { + subvol = fmt.Sprintf("%s-%s", parent, snapName) + } + + return fmt.Sprintf("subvol=%s", subvol) + } + fs := v.ExpandedConfig("block.mount_options") if fs != "" { return fs diff --git a/internal/server/storage/pool_interface.go b/internal/server/storage/pool_interface.go index c871fa17130..e9ccce7a51f 100644 --- a/internal/server/storage/pool_interface.go +++ b/internal/server/storage/pool_interface.go @@ -26,8 +26,9 @@ type VolumeUsage struct { // MountInfo represents info about the result of a mount operation. type MountInfo struct { - DiskPath string // The location of the block disk (if supported). - PostHooks []func(inst instance.Instance) error // Hooks to be called following a mount. + DiskPath string // The location of the block disk (if supported). + BackingPath []string // The location of the block disk (backing disk for qcow2). + PostHooks []func(inst instance.Instance) error // Hooks to be called following a mount. } // Type represents an Incus storage pool type. diff --git a/internal/server/storage/utils.go b/internal/server/storage/utils.go index d67fd1b326c..ea55f0ccf6f 100644 --- a/internal/server/storage/utils.go +++ b/internal/server/storage/utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "io/fs" "os" "path/filepath" @@ -572,7 +573,7 @@ func validateVolumeCommonRules(vol drivers.Volume) map[string]func(string) error // VM Format A: Separate metadata tarball and root qcow2 file. // - Unpack metadata tarball into mountPath. // - Check rootBlockPath is a file and convert qcow2 file into raw format in rootBlockPath. -func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sysOS *sys.OS, allowUnsafeResize bool, targetIsZero bool, tracker *ioprogress.ProgressTracker) (int64, error) { +func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sysOS *sys.OS, allowUnsafeResize bool, targetIsZero bool, tracker *ioprogress.ProgressTracker, targetFormat string) (int64, error) { l := logger.Log.AddContext(logger.Ctx{"imageFile": imageFile, "volName": vol.Name()}) l.Info("Image unpack started") defer l.Info("Image unpack stopped") @@ -660,7 +661,7 @@ func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sys } // Belt and braces qcow2 check. - if imgInfo.Format != "qcow2" { + if imgInfo.Format != drivers.BlockVolumeTypeQcow2 { return -1, fmt.Errorf("Unexpected image format %q", imgInfo.Format) } @@ -695,6 +696,44 @@ func ImageUnpack(imageFile string, vol drivers.Volume, destBlockFile string, sys } } + if targetFormat == drivers.BlockVolumeTypeQcow2 { + l.Debug("Writing qcow2 image to disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath}) + + // Attempt to deref all paths. + imgFullPath, err := filepath.EvalSymlinks(imgPath) + if err == nil { + imgPath = imgFullPath + } + + if dstPath != "" { + dstFullPath, err := filepath.EvalSymlinks(dstPath) + if err == nil { + dstPath = dstFullPath + } + } + + from, err := os.OpenFile(imgPath, unix.O_RDONLY, 0) + if err != nil { + return -1, err + } + + defer from.Close() + + to, err := os.OpenFile(dstPath, unix.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0) + if err != nil { + return -1, err + } + + defer to.Close() + + _, err = io.Copy(to, from) + if err != nil { + return -1, err + } + + return imgInfo.VirtualSize, nil + } + // Convert the qcow2 format to a raw block device. l.Debug("Converting qcow2 image to raw disk", logger.Ctx{"imgPath": imgPath, "dstPath": dstPath})