From 25dd45c143e0d49ecf36ac6cc0e15dbecb15beaa Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 8 Apr 2026 15:35:34 +0200 Subject: [PATCH 01/44] tests: add sbuild-schroot install for debian sid to fix nightly sbuild test (#16880) --- tests/lib/pkgdb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/lib/pkgdb.sh b/tests/lib/pkgdb.sh index 3332f8bed7c..0ffac2e3296 100755 --- a/tests/lib/pkgdb.sh +++ b/tests/lib/pkgdb.sh @@ -707,6 +707,7 @@ pkg_dependencies_ubuntu_classic(){ net-tools packagekit sbuild + sbuild-schroot schroot strace systemd-timesyncd From 0b89960d36ba623e05df6f719240b37a441ddb9b Mon Sep 17 00:00:00 2001 From: Oliver Calder Date: Wed, 8 Apr 2026 12:14:45 -0500 Subject: [PATCH 02/44] tests: avoid installing prompting-client camera dependencies (#16877) Signed-off-by: Oliver Calder --- tests/nightly/prompting-client-integration-tests/task.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/nightly/prompting-client-integration-tests/task.yaml b/tests/nightly/prompting-client-integration-tests/task.yaml index 8a29ffc8540..97c795268d4 100644 --- a/tests/nightly/prompting-client-integration-tests/task.yaml +++ b/tests/nightly/prompting-client-integration-tests/task.yaml @@ -44,10 +44,11 @@ prepare: | popd echo "Install dependencies for local tests" - apt install -y protobuf-compiler "linux-modules-extra-$(uname -r)" v4l-utils + apt install -y protobuf-compiler # Skip video device setup since we're skipping camera tests below #echo "Ensure virtual camera and audio devices will work" + #apt install -y "linux-modules-extra-$(uname -r)" v4l-utils #modprobe v4l2loopback devices=1 video_nr=0 card_label="Placeholder Card" exclusive_caps=1 #modprobe snd-aloop #adduser test video && newgrp video From 72e402ea2901dda31665a980f42385e3f4815627 Mon Sep 17 00:00:00 2001 From: Ondra Kubik Date: Thu, 9 Apr 2026 13:14:01 +0100 Subject: [PATCH 03/44] core-initrd: fix mount with uboot env in partition (#16765) --- core-initrd/26.04/factory/usr/lib/the-modeenv | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core-initrd/26.04/factory/usr/lib/the-modeenv b/core-initrd/26.04/factory/usr/lib/the-modeenv index 92c9f704234..312d817e066 100755 --- a/core-initrd/26.04/factory/usr/lib/the-modeenv +++ b/core-initrd/26.04/factory/usr/lib/the-modeenv @@ -16,7 +16,8 @@ if [ "${mode}" = "run" ]; then echo '/run/mnt/ubuntu-boot/EFI/ubuntu /boot/grub none bind 0 0' >> /run/image.fstab # ensure ESP efi dir is available for fwupdate (LP: 1892392) echo '/run/mnt/ubuntu-seed/ /boot/efi none bind 0 0' >> /run/image.fstab - elif [ -f /run/mnt/ubuntu-boot/uboot/ubuntu/boot.sel ]; then + elif [ -d /run/mnt/ubuntu-boot/uboot/ubuntu ]; then + # uboot and ubootpart bootloaders echo '/run/mnt/ubuntu-boot/uboot/ubuntu /boot/uboot none bind 0 0' >> /run/image.fstab elif [ -f /run/mnt/ubuntu-seed/piboot/ubuntu/piboot.conf ]; then echo '/run/mnt/ubuntu-seed/piboot/ubuntu /boot/piboot none bind 0 0' >> /run/image.fstab From 094e7ecbacdf71905aadae2886943951f0bf8455 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Tue, 31 Mar 2026 09:00:17 +0200 Subject: [PATCH 04/44] cmd/snap: show component details in snap info output Replace the simple 'components: N/M' count with a new section that, for each component, shows output with the same information that tracks display name, version, install date, (revision), installed size and notes. Two notes are used: one for not-installed (to explain the empty fields better) and one for component type other than standard. Signed-off-by: Zygmunt Krynicki --- cmd/snap/cmd_info.go | 72 +++++++++++++---- cmd/snap/cmd_info_test.go | 96 ++++++++++++++++------- tests/main/snap-info-components/task.yaml | 36 ++++++--- 3 files changed, 151 insertions(+), 53 deletions(-) diff --git a/cmd/snap/cmd_info.go b/cmd/snap/cmd_info.go index e0927edcc62..b211a1d0671 100644 --- a/cmd/snap/cmd_info.go +++ b/cmd/snap/cmd_info.go @@ -186,26 +186,66 @@ func (iw *infoWriter) setupSnap(localSnap, remoteSnap *client.Snap, resInfo *cli } func (iw *infoWriter) maybePrintComponents() { - totalCount := 0 - localCount := 0 - - if iw.localSnap != nil { - for _, comp := range iw.localSnap.Components { - if !comp.Revision.Unset() { - localCount++ - } - } + if iw.theSnap == nil || len(iw.theSnap.Components) == 0 { + return } - if iw.theSnap != nil { - totalCount = len(iw.theSnap.Components) - } + iw.Flush() + defer iw.Flush() + fmt.Fprintln(iw, "components:") - if totalCount == 0 && localCount == 0 { - return + sort.Slice(iw.theSnap.Components, componentsByInstallStatusAndSnapName(iw.theSnap.Components)) + + releasedfmt := "2006-01-02" + if iw.absTime { + releasedfmt = time.RFC3339 } - fmt.Fprintf(iw, "components: %d/%d\n", localCount, totalCount) + for idx := range iw.theSnap.Components { + comp := &iw.theSnap.Components[idx] + // This mimics the design of track display, on purpose. + fmt.Fprintf(iw, " +%s:\t", comp.Name) + if comp.Version != "" { + fmt.Fprintf(iw, "%s\t", comp.Version) + } else { + fmt.Fprintf(iw, "%s\t", iw.esc.dash) + } + if comp.InstallDate != nil { + fmt.Fprintf(iw, "%s\t", comp.InstallDate.Format(releasedfmt)) + } else { + fmt.Fprintf(iw, "%s\t", iw.esc.dash) + } + if !comp.Revision.Unset() { + // NOTE: Surprisingly component revisions are not unique within a snap, so + // for a given snap revision, we may see all the components corresponding + // to it share one revision number. This may be confusing to users. + // For the moment we DO print the component revision, but over time we + // may revise that decision and just omit it in this view. + fmt.Fprintf(iw, "%s\t", fmt.Sprintf("(%s)", comp.Revision)) + } else { + fmt.Fprintf(iw, "%s\t", iw.esc.dash) + } + if comp.InstalledSize != 0 { + fmt.Fprintf(iw, "%s\t", fmtSize(comp.InstalledSize)) + } else { + fmt.Fprintf(iw, "%s\t", iw.esc.dash) + } + + var notes []string + + if comp.Type != snap.StandardComponent { + notes = append(notes, string(comp.Type)) + } + if comp.InstallDate == nil { + notes = append(notes, "not installed") + } + if len(notes) > 0 { + fmt.Fprintf(iw, "%s", strings.Join(notes, ", ")) + } else { + fmt.Fprintf(iw, "%s", iw.esc.dash) + } + fmt.Fprintln(iw) + } } func (iw *infoWriter) maybePrintPrice() { @@ -709,6 +749,7 @@ func (x *infoCmd) Execute([]string) error { iw.printDescr() iw.maybePrintCommands() iw.maybePrintServices() + iw.maybePrintComponents() iw.maybePrintNotes() // stops the notes etc trying to be aligned with channels iw.Flush() @@ -720,7 +761,6 @@ func (x *infoCmd) Execute([]string) error { iw.maybePrintTrackingChannel() iw.maybePrintRefreshInfo() iw.maybePrintChinfo() - iw.maybePrintComponents() } w.Flush() diff --git a/cmd/snap/cmd_info_test.go b/cmd/snap/cmd_info_test.go index c787b3585fd..700851459a5 100644 --- a/cmd/snap/cmd_info_test.go +++ b/cmd/snap/cmd_info_test.go @@ -97,60 +97,102 @@ func (s *infoSuite) TestMaybePrintServicesNoServices(c *check.C) { } } -func (s *infoSuite) TestMaybePrintComponents(c *check.C) { +func (s *infoSuite) TestMaybePrintComponentsNoneInstalled(c *check.C) { var buf flushBuffer iw := snap.NewInfoWriter(&buf) - // c1 is "installed" (revision != 0), c2 is "not installed" (revision == 0) - c1 := client.Component{Name: "comp-1", Revision: snaplib.R(10)} - c2 := client.Component{Name: "comp-2", Revision: snaplib.R(0)} - - remoteSnap := &client.Snap{Components: []client.Component{c1, c2}} - snap.SetupSnap(iw, nil, remoteSnap, nil) + // both components available in the store but not installed (revision unset) + c1 := client.Component{Name: "comp-1", Type: "standard"} + c2 := client.Component{Name: "comp-2", Type: "standard"} + snap.SetupSnap(iw, nil, &client.Snap{ + Name: "foo", + Components: []client.Component{c1, c2}, + }, nil) snap.MaybePrintComponents(iw) - c.Check(buf.String(), check.Equals, "components: 0/2\n") + c.Check(buf.String(), check.Equals, "components:\n"+ + " +comp-1:\t--\t--\t--\t--\tnot installed\n"+ + " +comp-2:\t--\t--\t--\t--\tnot installed\n") +} + +func (s *infoSuite) TestMaybePrintComponentsMixedInstall(c *check.C) { + var buf flushBuffer + iw := snap.NewInfoWriter(&buf) - buf.Reset() + // c1 is installed with revision, c2 is not installed (revision unset) + c1 := client.Component{ + Name: "comp-1", + Type: "standard", + Version: "1.0", + Revision: snaplib.R(10), + InstalledSize: 1024, + InstallDate: func() *time.Time { t := time.Date(2022, time.August, 19, 12, 34, 56, 0, time.UTC); return &t }(), + } + c2 := client.Component{Name: "comp-2", Type: "standard"} snap.SetupSnap(iw, &client.Snap{ Name: "foo", Components: []client.Component{c1, c2}, - }, remoteSnap, nil) - + }, nil, nil) snap.MaybePrintComponents(iw) - c.Check(buf.String(), check.Equals, "components: 1/2\n") - - buf.Reset() - - snap.SetupSnap(iw, nil, &client.Snap{Components: []client.Component{}}, nil) - snap.MaybePrintComponents(iw) - - c.Check(buf.String(), check.Equals, "") + c.Check(buf.String(), check.Equals, "components:\n"+ + " +comp-1:\t1.0\t2022-08-19\t(10)\t1024B\t--\n"+ + " +comp-2:\t--\t--\t--\t--\tnot installed\n") } func (s *infoSuite) TestMaybePrintComponentsBothInstalled(c *check.C) { var buf flushBuffer iw := snap.NewInfoWriter(&buf) - c1 := client.Component{Name: "comp-1", Revision: snaplib.R(10)} - c2 := client.Component{Name: "comp-2", Revision: snaplib.R(20)} - - remoteSnap := &client.Snap{Components: []client.Component{c1, c2}} - snap.SetupSnap(iw, nil, remoteSnap, nil) + c1 := client.Component{ + Name: "comp-1", + Type: "standard", + Version: "1.0", + Revision: snaplib.R(10), + InstalledSize: 1024, + InstallDate: func() *time.Time { t := time.Date(2022, time.August, 19, 12, 34, 56, 0, time.UTC); return &t }(), + } + c2 := client.Component{ + Name: "comp-2", + Type: "kernel-modules", + Version: "2.0", + Revision: snaplib.R(20), + InstalledSize: 65536, + InstallDate: func() *time.Time { t := time.Date(2022, time.August, 19, 12, 34, 56, 0, time.UTC); return &t }(), + } snap.SetupSnap(iw, &client.Snap{ Name: "foo", Components: []client.Component{c1, c2}, - }, remoteSnap, nil) + }, nil, nil) + snap.MaybePrintComponents(iw) + + c.Check(buf.String(), check.Equals, "components:\n"+ + " +comp-1:\t1.0\t2022-08-19\t(10)\t1024B\t--\n"+ + " +comp-2:\t2.0\t2022-08-19\t(20)\t65.5kB\tkernel-modules\n") +} +func (s *infoSuite) TestMaybePrintComponentsEmpty(c *check.C) { + var buf flushBuffer + iw := snap.NewInfoWriter(&buf) + + snap.SetupSnap(iw, nil, &client.Snap{Components: []client.Component{}}, nil) snap.MaybePrintComponents(iw) - c.Check(buf.String(), check.Equals, "components: 2/2\n") + c.Check(buf.String(), check.Equals, "") +} - buf.Reset() +func (s *infoSuite) TestMaybePrintComponents(c *check.C) { + // Keep a variant that exercises theSnap == nil path (no snap set up yet) + // which is covered by SetupSnap leaving theSnap as nil when both snaps are nil. + var buf flushBuffer + iw := snap.NewInfoWriter(&buf) + + snap.MaybePrintComponents(iw) + + c.Check(buf.String(), check.Equals, "") } func (s *infoSuite) TestMaybePrintCommands(c *check.C) { diff --git a/tests/main/snap-info-components/task.yaml b/tests/main/snap-info-components/task.yaml index fdda9849069..d9e6589c9d2 100644 --- a/tests/main/snap-info-components/task.yaml +++ b/tests/main/snap-info-components/task.yaml @@ -1,12 +1,12 @@ summary: Test that snap info command handles components as expected details: | - This test verifies that snap info prints components installed/available - counts correctly for various scenarios: - - No components installed using test-snap-with-components - - Snap installed but no components installed using test-snap-with-components - - 2/3 (installed/total) using test-snap-with-components - - No components anywhere using test-snapd-sh + This test verifies that snap info prints a `components:` section with + tabular per-component entries. Scenarios: + - No components section when snap is only available in the store. + - Components are shown in an abbreviated form `+comp` instead of `snap+comp`. + - Version, revision and installed size are only shown when available (when installed). + - No components section for a snap that has no components (test-snapd-sh). prepare: | snap install test-snapd-sh @@ -16,13 +16,29 @@ restore: | snap remove --purge test-snap-with-components || true execute: | - # The snap-info command does not print component info if it is not installed - snap info test-snap-with-components | NOMATCH 'components: 0/3' + # The snap-info command does not print a components section for a store-only snap + snap info test-snap-with-components | NOMATCH 'components:' snap info test-snapd-sh | NOMATCH 'components:' snap install test-snap-with-components - snap info test-snap-with-components | MATCH 'components: 0/3' + snap info test-snap-with-components | MATCH 'components:' + # All components show 'not installed' before any are installed + snap info test-snap-with-components | MATCH '\+one:[[:space:]]*-- -- -- -- test, not installed' + snap info test-snap-with-components | MATCH '\+two:[[:space:]]*-- -- -- -- test, not installed' + snap info test-snap-with-components | MATCH '\+three:[[:space:]]*-- -- -- -- test, not installed' snap install test-snap-with-components+one+two - snap info test-snap-with-components | MATCH 'components: 2/3' + # Installed components show revision and size + snap info test-snap-with-components | MATCH '\+one:[[:space:]]*[[:digit:]]+\.[[:digit:]]+ [[:digit:]]+-[[:digit:]]+-[[:digit:]]+ \([[:digit:]]+\) [[:digit:]]+\.[[:digit:]]+kB test$' + snap info test-snap-with-components | MATCH '\+two:[[:space:]]*[[:digit:]]+\.[[:digit:]]+ [[:digit:]]+-[[:digit:]]+-[[:digit:]]+ \([[:digit:]]+\) [[:digit:]]+\.[[:digit:]]+kB test$' + + # Exactly two components are installed + snap info test-snap-with-components | grep -c -E '[[:space:]]*[+].*\([[:digit:]]+\)' | MATCH '^2$' + # Installed components do not show 'not installed' + snap info test-snap-with-components | NOMATCH '\+one.*test, not installed' + snap info test-snap-with-components | NOMATCH '\+two.*test, not installed' + # Uninstalled component still shows 'not installed' + snap info test-snap-with-components | MATCH '\+three.*test, not installed' + # Exactly one component is not installed + snap info test-snap-with-components | grep -c 'not installed' | MATCH '^1$' From 71e64cb4b9c503eaf7e8b81bedcb5434f44208f7 Mon Sep 17 00:00:00 2001 From: Valentin David Date: Tue, 27 Jan 2026 12:12:58 +0100 Subject: [PATCH 05/44] boot,overlord/fdestate: move FDE auto-repair and provide state --- boot/boot.go | 5 +- boot/boot_test.go | 47 --- boot/bootstate16.go | 2 +- boot/bootstate20.go | 15 +- boot/kernel_os.go | 3 +- boot/unlocked_state.go | 17 - daemon/api_general_test.go | 11 +- overlord/devicestate/devicemgr.go | 60 ++- .../devicestate/devicestate_cloudinit_test.go | 9 +- overlord/devicestate/devicestate_test.go | 106 ++--- overlord/devicestate/export_test.go | 8 + overlord/fdestate/activate_state.go | 13 + overlord/fdestate/activate_state_test.go | 58 ++- overlord/fdestate/autorepair.go | 198 +++++++++ overlord/fdestate/autorepair_test.go | 392 ++++++++++++++++++ overlord/fdestate/backend/reseal.go | 25 +- overlord/fdestate/backend/reseal_test.go | 172 -------- overlord/fdestate/export_test.go | 17 + secboot/secboot_nosb.go | 4 + secboot/secboot_sb.go | 85 ++++ secboot/secboot_sb_test.go | 66 +++ .../core-recover-from-recovery/task.yaml | 44 +- 22 files changed, 994 insertions(+), 363 deletions(-) create mode 100644 overlord/fdestate/autorepair.go create mode 100644 overlord/fdestate/autorepair_test.go diff --git a/boot/boot.go b/boot/boot.go index aea917e5c0e..08d735eb990 100644 --- a/boot/boot.go +++ b/boot/boot.go @@ -343,7 +343,7 @@ func GetCurrentBoot(t snap.Type, dev snap.Device) (snap.PlaceInfo, error) { // bootStateUpdate carries the state for an on-going boot state update. // At the end it can be used to commit it. type bootStateUpdate interface { - commit(markedSuccesful bool) error + commit() error } // MarkBootSuccessful marks the current boot as successful. This means @@ -403,8 +403,7 @@ func MarkBootSuccessful(dev snap.Device) error { } if u != nil { - const markedSuccessful = true - if err := u.commit(markedSuccessful); err != nil { + if err := u.commit(); err != nil { return fmt.Errorf(errPrefix, err) } } diff --git a/boot/boot_test.go b/boot/boot_test.go index 2229a951402..4a63bfc53e7 100644 --- a/boot/boot_test.go +++ b/boot/boot_test.go @@ -20,7 +20,6 @@ package boot_test import ( - "encoding/json" "errors" "fmt" "os" @@ -5268,49 +5267,3 @@ func (s *bootenv20Suite) TestMarkBootSuccessfulClassModes(c *C) { c.Check(m2.Base, Equals, "") c.Check(m2.TryBase, Equals, "") } - -func (s *bootenv20Suite) TestMarkBootSuccessfulAutoRepair(c *C) { - m := &boot.Modeenv{ - Mode: "run", - CurrentKernels: []string{s.kern1.Filename()}, - } - defer setupUC20Bootenv( - c, - s.bootloader, - &bootenv20Setup{ - modeenv: m, - kern: s.kern1, - kernStatus: boot.DefaultStatus, - }, - )() - - data := map[string]any{ - "ubuntu-data": map[string]any{ - "unlock-key": "recovery", - }, - "ubuntu-save": map[string]any{ - "unlock-key": "run", - }, - } - jsonData, err := json.Marshal(data) - c.Assert(err, IsNil) - - err = os.MkdirAll(filepath.Join(s.rootdir, "run/snapd/snap-bootstrap"), 0755) - c.Assert(err, IsNil) - err = os.WriteFile(filepath.Join(s.rootdir, "run/snapd/snap-bootstrap/unlocked.json"), jsonData, 0644) - c.Assert(err, IsNil) - - resealCalls := 0 - defer boot.MockResealKeyToModeenv(func(rootdir string, modeenv *boot.Modeenv, opts boot.ResealKeyToModeenvOptions, unlocker boot.Unlocker) error { - resealCalls++ - c.Check(opts.Force, Equals, true) - c.Check(opts.EnsureProvisioned, Equals, true) - return nil - })() - - dev := boottest.MockClassicWithModesDevice("", nil) - - err = boot.MarkBootSuccessful(dev) - c.Assert(err, IsNil) - c.Check(resealCalls, Equals, 1) -} diff --git a/boot/bootstate16.go b/boot/bootstate16.go index 96f6afa534c..457180dee2e 100644 --- a/boot/bootstate16.go +++ b/boot/bootstate16.go @@ -114,7 +114,7 @@ func newBootStateUpdate16(u bootStateUpdate, names ...string) (*bootStateUpdate1 return &bootStateUpdate16{bl: bl, env: m, toCommit: make(map[string]string)}, nil } -func (u16 *bootStateUpdate16) commit(markedSuccessful bool) error { +func (u16 *bootStateUpdate16) commit() error { if len(u16.toCommit) == 0 { // nothing to do return nil diff --git a/boot/bootstate20.go b/boot/bootstate20.go index d9f3855823d..10bdf670218 100644 --- a/boot/bootstate20.go +++ b/boot/bootstate20.go @@ -21,7 +21,6 @@ package boot import ( "fmt" - "os" "path/filepath" "sync" "sync/atomic" @@ -189,7 +188,7 @@ func newBootStateUpdate20(m *Modeenv) (*bootStateUpdate20, error) { } // commit will write out boot state persistently to disk. -func (u20 *bootStateUpdate20) commit(markedSuccessful bool) error { +func (u20 *bootStateUpdate20) commit() error { if !isModeenvLocked() { return fmt.Errorf("internal error: cannot commit modeenv without the lock") } @@ -225,18 +224,6 @@ func (u20 *bootStateUpdate20) commit(markedSuccessful bool) error { resealOpts.ExpectReseal = resealExpectedByModeenvChange(u20.writeModeenv, u20.modeenv) } - if markedSuccessful { - autoRepair, err := isUnlockedWithRecoveryKey() - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else if autoRepair { - resealOpts.Force = true - resealOpts.EnsureProvisioned = true - } - } - // next reseal using the modeenv values, we do this before any // post-modeenv tasks so if we are rebooted at any point after // the reseal even before the post tasks are completed, we diff --git a/boot/kernel_os.go b/boot/kernel_os.go index 902d08f535a..1f6c7045055 100644 --- a/boot/kernel_os.go +++ b/boot/kernel_os.go @@ -48,8 +48,7 @@ func (bp *coreBootParticipant) SetNextBoot(bootCtx NextBootContext) (rebootInfo } if u != nil { - const markedSuccessful = false - if err := u.commit(markedSuccessful); err != nil { + if err := u.commit(); err != nil { return RebootInfo{RebootRequired: false}, fmt.Errorf(errPrefix, err) } } diff --git a/boot/unlocked_state.go b/boot/unlocked_state.go index 6f5f6f5edb7..828936b8f4a 100644 --- a/boot/unlocked_state.go +++ b/boot/unlocked_state.go @@ -95,20 +95,3 @@ func LoadDiskUnlockState(name string) (*DiskUnlockState, error) { return ret, nil } - -// isUnlockedWithRecoveryKey tells whether a recovery key has been -// typed to unlock a disk during boot. -func isUnlockedWithRecoveryKey() (bool, error) { - state, err := LoadDiskUnlockState(UnlockedStateFileName) - if err != nil { - return false, err - } - - if state.State != nil { - return state.State.NumActivatedContainersWithRecoveryKey() != 0, nil - } else { - // This is a case of an old snap-boostrap that does not provide the activate state API result. - // We still can guess based on the old status. - return state.UbuntuData.UnlockKey == "recovery" || state.UbuntuSave.UnlockKey == "recovery", nil - } -} diff --git a/daemon/api_general_test.go b/daemon/api_general_test.go index 41604b03770..3415e702a43 100644 --- a/daemon/api_general_test.go +++ b/daemon/api_general_test.go @@ -1242,15 +1242,22 @@ func (s *generalSuite) TestSysInfoStorageEncHappy(c *check.C) { setExpectedStatus := func(status string) { expectedStatus = status expectedResponse["status"] = status + expectedResponse["auto-repair-result"] = "not-initialized" } defer daemon.MockFdestateSystemState(func(*state.State) (*fdestate.FDESystemState, error) { switch expectedStatus { case "active": - return &fdestate.FDESystemState{Status: fdestate.FDEStatusActive}, nil + return &fdestate.FDESystemState{ + Status: fdestate.FDEStatusActive, + AutoRepairResult: fdestate.AutoRepairNotInitialized, + }, nil case "inactive": - return &fdestate.FDESystemState{Status: fdestate.FDEStatusInactive}, nil + return &fdestate.FDESystemState{ + Status: fdestate.FDEStatusInactive, + AutoRepairResult: fdestate.AutoRepairNotInitialized, + }, nil } return nil, errors.New("cannot set unsupported expected status") diff --git a/overlord/devicestate/devicemgr.go b/overlord/devicestate/devicemgr.go index 89368b58d5b..070652bc559 100644 --- a/overlord/devicestate/devicemgr.go +++ b/overlord/devicestate/devicemgr.go @@ -46,6 +46,7 @@ import ( "github.com/snapcore/snapd/overlord/auth" "github.com/snapcore/snapd/overlord/configstate/config" "github.com/snapcore/snapd/overlord/devicestate/internal" + "github.com/snapcore/snapd/overlord/fdestate" "github.com/snapcore/snapd/overlord/hookstate" "github.com/snapcore/snapd/overlord/install" "github.com/snapcore/snapd/overlord/restart" @@ -75,6 +76,8 @@ var ( secbootMarkSuccessful = secboot.MarkSuccessful osutilBootID = osutil.BootID + + fdestateAttemptAutoRepairIfNeeded = fdestate.AttemptAutoRepairIfNeeded ) var ( @@ -89,6 +92,7 @@ func init() { swfeats.RegisterEnsure("DeviceManager", "ensureSeeded") swfeats.RegisterEnsure("DeviceManager", "ensureAutoImportAssertions") swfeats.RegisterEnsure("DeviceManager", "ensureSerialBoundSystemUserAssertionsProcessed") + swfeats.RegisterEnsure("DeviceManager", "ensureFDE") swfeats.RegisterEnsure("DeviceManager", "ensureBootOk") swfeats.RegisterEnsure("DeviceManager", "ensureCloudInitRestricted") swfeats.RegisterEnsure("DeviceManager", "ensureInstalled") @@ -159,6 +163,7 @@ type DeviceManager struct { newStore func(storecontext.DeviceBackend) snapstate.StoreService bootRevisionsUpdated bool + fdeRan bool seedTimings *timings.Timings // this is used during early phases until seeding is under way @@ -1244,6 +1249,40 @@ func (m *DeviceManager) ensureSerialBoundSystemUserAssertionsProcessed() error { return nil } +func (m *DeviceManager) ensureFDE() error { + m.state.Lock() + defer m.state.Unlock() + + if m.SystemMode(SysAny) != "run" { + return nil + } + + if m.fdeRan { + return nil + } + + // Auto-repair should be attempted only once. + m.fdeRan = true + + logger.Trace("ensure", "manager", "DeviceManager", "func", "ensureFDE") + + // FIXME: we should rename to something like "reset lockout" + lockoutResetErr := secbootMarkSuccessful() + + // TODO:FDEM: with new APIs of lockout reset we will get so + // more statuses that we will need to react to and + // provide to the status API. + + // FIXME: we need to check that a try kernel was attempted here and not attempt + // repair in that case. + + if err := fdestateAttemptAutoRepairIfNeeded(m.state, lockoutResetErr); err != nil { + return err + } + + return nil +} + var bootOkRanForBootID = bootOkRanForBootIDImpl func bootOkRanForBootIDImpl(st *state.State, currentBootID string) (bool, error) { @@ -1287,12 +1326,12 @@ func (m *DeviceManager) ensureBootOk() error { return err } if err == nil && deviceCtx.Model().KernelSnap() != nil { + // FIXME: we should check if recovery keys + // were used and in that case do not mark the + // boot successful. if err := boot.MarkBootSuccessful(deviceCtx); err != nil { return err } - if err := secbootMarkSuccessful(); err != nil { - return err - } } } else { // a reseal already ran, nothing to do @@ -2103,9 +2142,18 @@ func (m *DeviceManager) Ensure() error { errs = append(errs, err) } - // XXX: This might trigger a reseal but it should not affect - // resealing tasks since it is run at most once during startup - // before TaskRunner.Ensure() is called. + // XXX: This might trigger a reseal (auto-repair) but + // it should not affect resealing tasks since it is + // run at most once during startup before + // TaskRunner.Ensure() is called. + if err := m.ensureFDE(); err != nil { + errs = append(errs, err) + } + + // XXX: This might trigger a reseal (removal of "try" + // entries in modeenv) but it should not affect + // resealing tasks since it is run at most once during + // startup before TaskRunner.Ensure() is called. if err := m.ensureBootOk(); err != nil { errs = append(errs, err) } diff --git a/overlord/devicestate/devicestate_cloudinit_test.go b/overlord/devicestate/devicestate_cloudinit_test.go index bbb2702ad07..3e2a99f1088 100644 --- a/overlord/devicestate/devicestate_cloudinit_test.go +++ b/overlord/devicestate/devicestate_cloudinit_test.go @@ -15,6 +15,7 @@ import ( "github.com/snapcore/snapd/overlord/auth" "github.com/snapcore/snapd/overlord/devicestate" "github.com/snapcore/snapd/overlord/devicestate/devicestatetest" + "github.com/snapcore/snapd/overlord/state" "github.com/snapcore/snapd/release" "github.com/snapcore/snapd/sysconfig" "github.com/snapcore/snapd/testutil" @@ -42,6 +43,10 @@ func (s *cloudInitBaseSuite) SetUpTest(c *C) { r := release.MockOnClassic(false) defer r() + s.AddCleanup(devicestate.MockFdestateAttemptAutoRepairIfNeeded(func(st *state.State, lockoutResetErr error) error { + return nil + })) + st := s.o.State() st.Lock() st.Set("seeded", true) @@ -1135,7 +1140,7 @@ fi`, cloudInitScriptStateFile)) c.Assert(restrictCalls, Equals, 1) // we now have a message about restricting - c.Assert(strings.TrimSpace(s.logbuf.String()), Matches, `.*System initialized, cloud-init reported to be done, set datasource_list to \[ NoCloud \] and disabled auto-import by filesystem label`) + c.Assert(strings.TrimSpace(s.logbuf.String()), Matches, `.*System initialized, cloud-init reported to be done, set datasource_list to \[ NoCloud \] and disabled auto-import by filesystem label.*`) } func (s *cloudInitSuite) TestCloudInitHappyNotFound(c *C) { // pretend that cloud-init was not found on PATH @@ -1161,5 +1166,5 @@ func (s *cloudInitSuite) TestCloudInitHappyNotFound(c *C) { c.Assert(err, IsNil) c.Assert(statusCalls, Equals, 1) c.Assert(restrictCalls, Equals, 1) - c.Assert(strings.TrimSpace(s.logbuf.String()), Matches, `.*System initialized, cloud-init not found, disabled permanently`) + c.Assert(strings.TrimSpace(s.logbuf.String()), Matches, `.*System initialized, cloud-init not found, disabled permanently.*`) } diff --git a/overlord/devicestate/devicestate_test.go b/overlord/devicestate/devicestate_test.go index 58254229ddf..14e13cf2083 100644 --- a/overlord/devicestate/devicestate_test.go +++ b/overlord/devicestate/devicestate_test.go @@ -650,70 +650,6 @@ func (s *deviceMgrSuite) switchDevManagerToClassicWithModes(c *C) { c.Assert(err, IsNil) } -func (s *deviceMgrSuite) TestDeviceManagerEnsureBootOkRunsOnClassicWithModes(c *C) { - s.switchDevManagerToClassicWithModes(c) - s.setPCModelInState(c) - - secbootMarkSuccessfulCalled := 0 - r := devicestate.MockSecbootMarkSuccessful(func() error { - secbootMarkSuccessfulCalled++ - return nil - }) - defer r() - - err := devicestate.EnsureBootOk(s.mgr) - c.Assert(err, IsNil) - c.Check(secbootMarkSuccessfulCalled, Equals, 1) -} - -func (s *deviceMgrSuite) TestDeviceManagerEnsureBootOkRunsOncePerBoot(c *C) { - s.setPCModelInState(c) - - logbuf, restore := logger.MockLogger() - defer restore() - - restore = devicestate.MockOsutilBootID("boot-id-1") - defer restore() - - secbootMarkSuccessfulCalled := 0 - r := devicestate.MockSecbootMarkSuccessful(func() error { - secbootMarkSuccessfulCalled++ - return nil - }) - defer r() - - var lastBootID string - s.state.Lock() - err := s.state.Get("ensure-boot-ok-boot-id", &lastBootID) - s.state.Unlock() - // ensure-boot-ok-boot-id is unset - c.Assert(errors.Is(err, state.ErrNoState), Equals, true) - - // last reseal boot ID does not match current boot id so a reseal - // is expected - err = devicestate.EnsureBootOk(s.mgr) - c.Assert(err, IsNil) - c.Check(secbootMarkSuccessfulCalled, Equals, 1) - c.Check(logbuf.String(), Equals, "") - logbuf.Reset() - - s.state.Lock() - err = s.state.Get("ensure-boot-ok-boot-id", &lastBootID) - s.state.Unlock() - c.Assert(err, IsNil) - c.Check(lastBootID, Equals, "boot-id-1") - - // mimic a snapd restart - devicestate.SetEnsureBootOkRan(s.mgr, false) - - // now last reseal boot ID matches current boot id so no reseal - // is expected - err = devicestate.EnsureBootOk(s.mgr) - c.Assert(err, IsNil) - c.Check(secbootMarkSuccessfulCalled, Equals, 1) // unchanged - c.Check(logbuf.String(), testutil.Contains, `skipping boot ok check since it already ran for boot-id "boot-id-1"`) -} - func (s *deviceMgrSuite) TestDeviceManagerEnsureSeededHappyWithModeenv(c *C) { n := 0 @@ -753,13 +689,6 @@ func (s *deviceMgrSuite) TestDeviceManagerEnsureSeededHappyWithModeenv(c *C) { func (s *deviceMgrSuite) TestDeviceManagerEnsureBootOkBootloaderHappy(c *C) { s.setPCModelInState(c) - secbootMarkSuccessfulCalled := 0 - r := devicestate.MockSecbootMarkSuccessful(func() error { - secbootMarkSuccessfulCalled++ - return nil - }) - defer r() - s.bootloader.SetBootVars(map[string]string{ "snap_mode": boot.TryingStatus, "snap_try_core": "core_1.snap", @@ -779,7 +708,6 @@ func (s *deviceMgrSuite) TestDeviceManagerEnsureBootOkBootloaderHappy(c *C) { err := devicestate.EnsureBootOk(s.mgr) s.state.Lock() c.Assert(err, IsNil) - c.Check(secbootMarkSuccessfulCalled, Equals, 1) m, err := s.bootloader.GetBootVars("snap_mode") c.Assert(err, IsNil) @@ -3443,3 +3371,37 @@ func (s *deviceMgrSuite) TestDeviceManagerStartupCallbacks(c *C) { c.Check(callA.called, Equals, 1) c.Check(callB.called, Equals, 1) } + +func (s *deviceMgrSuite) TestDeviceManagerEnsureFDE(c *C) { + defer devicestate.MockSecbootMarkSuccessful(func() error { + return fmt.Errorf("MarkSuccessful did not work") + })() + + called := 0 + defer devicestate.MockFdestateAttemptAutoRepairIfNeeded(func(st *state.State, lockoutResetErr error) error { + c.Check(lockoutResetErr, ErrorMatches, `MarkSuccessful did not work`) + called++ + return nil + })() + + devicestate.SetSystemMode(s.mgr, "run") + err := devicestate.EnsureFDE(s.mgr) + c.Assert(err, IsNil) + c.Check(called, Equals, 1) +} + +func (s *deviceMgrSuite) TestDeviceManagerEnsureFDEInstall(c *C) { + defer devicestate.MockSecbootMarkSuccessful(func() error { + c.Errorf("unexpected call") + return fmt.Errorf("unexpected call") + })() + + defer devicestate.MockFdestateAttemptAutoRepairIfNeeded(func(st *state.State, lockoutResetErr error) error { + c.Errorf("unexpected call") + return fmt.Errorf("unexpected call") + })() + + devicestate.SetSystemMode(s.mgr, "install") + err := devicestate.EnsureFDE(s.mgr) + c.Assert(err, IsNil) +} diff --git a/overlord/devicestate/export_test.go b/overlord/devicestate/export_test.go index 6b63fcc1b82..45f38b8b9cf 100644 --- a/overlord/devicestate/export_test.go +++ b/overlord/devicestate/export_test.go @@ -263,6 +263,10 @@ func MockProcessAutoImportAssertion(f func(*state.State, seed.Seed, asserts.RODa } } +func EnsureFDE(m *DeviceManager) error { + return m.ensureFDE() +} + func EnsureBootOk(m *DeviceManager) error { return m.ensureBootOk() } @@ -746,3 +750,7 @@ func MockSnapstateGadgetInfo(f func(st *state.State, deviceCtx snapstate.DeviceC func MockOsutilBootID(bootID string) (restore func()) { return testutil.Mock(&osutilBootID, func() (string, error) { return bootID, nil }) } + +func MockFdestateAttemptAutoRepairIfNeeded(f func(st *state.State, locktoutResetErr error) error) (restore func()) { + return testutil.Mock(&fdestateAttemptAutoRepairIfNeeded, f) +} diff --git a/overlord/fdestate/activate_state.go b/overlord/fdestate/activate_state.go index 1e2543731f3..8c1f3f30ccd 100644 --- a/overlord/fdestate/activate_state.go +++ b/overlord/fdestate/activate_state.go @@ -92,6 +92,9 @@ type FDESystemState struct { // Status gives a summary on whether encrypted disks have been // activated and whether any recovery key was used. Status FDEStatus `json:"status"` + + // AutoRepairResult is the status of the auto-repair attempt + AutoRepairResult AutoRepairResult `json:"auto-repair-result"` } // SystemState returns a json serializable FDE state of the booted @@ -99,6 +102,16 @@ type FDESystemState struct { func SystemState(st *state.State) (*FDESystemState, error) { ret := &FDESystemState{} + repairResult, err := getRepairAttemptResult(st) + if err != nil { + return nil, err + } + if repairResult == nil { + ret.AutoRepairResult = AutoRepairNotInitialized + } else { + ret.AutoRepairResult = repairResult.Result + } + s, err := getActivateState(st) if err == errNoActivateState { // We are probably in a case where snap-bootstrap is diff --git a/overlord/fdestate/activate_state_test.go b/overlord/fdestate/activate_state_test.go index 4477b482b1b..37fb274f615 100644 --- a/overlord/fdestate/activate_state_test.go +++ b/overlord/fdestate/activate_state_test.go @@ -97,7 +97,48 @@ func (s *activateStateSuite) TestActivateStateHappy(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "active", + "status": "active", + "auto-repair-result": "not-initialized", + }) + + // Let's check the file is cached + _, err = fdestate.SystemState(st) + c.Assert(err, IsNil) + c.Check(calls, Equals, 1) +} + +func (s *activateStateSuite) TestActivateStateHappyWithAutoRepairState(c *C) { + calls := 0 + defer fdestate.MockBootLoadDiskUnlockState(func(name string) (*boot.DiskUnlockState, error) { + calls += 1 + c.Check(name, Equals, "unlocked.json") + + s := &secboot.ActivateState{} + s.Activations = map[string]*sb.ContainerActivateState{ + "data-cred-id": { + Status: sb.ActivationSucceededWithPlatformKey, + }, + "save-cred-id": { + Status: sb.ActivationSucceededWithPlatformKey, + }, + } + return &boot.DiskUnlockState{ + State: s, + }, nil + })() + + st := state.New(nil) + st.Lock() + defer st.Unlock() + + fdestate.SetRepairAttemptResult(st, &fdestate.RepairState{Result: fdestate.AutoRepairSuccess}) + + systemState, err := fdestate.SystemState(st) + c.Assert(err, IsNil) + + c.Check(systemState, SerializesTo, map[string]any{ + "status": "active", + "auto-repair-result": "success", }) // Let's check the file is cached @@ -137,7 +178,8 @@ func (s *activateStateSuite) TestActivateStateDegraded(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "degraded", + "status": "degraded", + "auto-repair-result": "not-initialized", }) // Let's check the file is cached @@ -165,7 +207,8 @@ func (s *activateStateSuite) TestActivateStateInactive(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "inactive", + "status": "inactive", + "auto-repair-result": "not-initialized", }) } @@ -195,7 +238,8 @@ func (s *activateStateSuite) TestActivateStateRecovery(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "recovery", + "status": "recovery", + "auto-repair-result": "not-initialized", }) } @@ -213,7 +257,8 @@ func (s *activateStateSuite) TestActivateStateNoActivateState(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "indeterminate", + "status": "indeterminate", + "auto-repair-result": "not-initialized", }) } @@ -231,7 +276,8 @@ func (s *activateStateSuite) TestActivateStateNoUnlockedJSON(c *C) { c.Assert(err, IsNil) c.Check(systemState, SerializesTo, map[string]any{ - "status": "indeterminate", + "status": "indeterminate", + "auto-repair-result": "not-initialized", }) } diff --git a/overlord/fdestate/autorepair.go b/overlord/fdestate/autorepair.go new file mode 100644 index 00000000000..b00ed24ca16 --- /dev/null +++ b/overlord/fdestate/autorepair.go @@ -0,0 +1,198 @@ +// -*- Mode: Go; indent-tabs-mode: t -*- + +/* + * Copyright (C) 2026 Canonical Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package fdestate + +import ( + "errors" + "fmt" + "os" + + "github.com/snapcore/snapd/boot" + "github.com/snapcore/snapd/dirs" + "github.com/snapcore/snapd/gadget/device" + "github.com/snapcore/snapd/logger" + "github.com/snapcore/snapd/osutil" + "github.com/snapcore/snapd/overlord/state" + "github.com/snapcore/snapd/secboot" +) + +var ( + secbootProvisionTPM = secboot.ProvisionTPM + secbootShouldAttemptRepair = secboot.ShouldAttemptRepair + + osutilBootID = osutil.BootID +) + +type AutoRepairResult string + +const ( + AutoRepairNotInitialized AutoRepairResult = "not-initialized" + AutoRepairNotAttempted AutoRepairResult = "not-attempted" + AutoRepairFailedPlatformInit AutoRepairResult = "failed-platform-init" + AutoRepairFailedKeyslots AutoRepairResult = "failed-keyslots" + AutoRepairFailedEncryptionSupport AutoRepairResult = "failed-encryption-support" + AutoRepairSuccess AutoRepairResult = "success" +) + +type repairState struct { + Result AutoRepairResult `json:"result"` +} + +type repairStateForBoot struct { + BootID string `json:"boot-id"` + State *repairState `json:"state"` +} + +const fdeRepairStateKey = "fde-repair-state" + +func setRepairAttemptResult(st *state.State, rs *repairState) error { + bootId, err := osutilBootID() + if err != nil { + return err + } + st.Set(fdeRepairStateKey, &repairStateForBoot{ + BootID: bootId, + State: rs, + }) + return nil +} + +func getRepairAttemptResult(st *state.State) (*repairState, error) { + var rs repairStateForBoot + if err := st.Get(fdeRepairStateKey, &rs); err != nil { + if errors.Is(err, state.ErrNoState) { + return nil, nil + } else { + return nil, err + } + } + + bootId, err := osutilBootID() + if err != nil { + return nil, err + } + + if rs.BootID != bootId { + st.Set(fdeRepairStateKey, nil) + return nil, nil + } + + return rs.State, nil +} + +func autoRepair(st *state.State) (AutoRepairResult, error) { + method, err := device.SealedKeysMethod(dirs.GlobalRootDir) + if err != nil { + return AutoRepairNotAttempted, err + } + + switch method { + case device.SealingMethodFDESetupHook: + case device.SealingMethodTPM, device.SealingMethodLegacyTPM: + // FIXME: re-run platform checks (post install checks?) + // Then maybe return AutoRepairFailedEncryptionSupport + + lockoutAuthFile := device.TpmLockoutAuthUnder(boot.InstallHostFDESaveDir) + if err := secbootProvisionTPM(secboot.TPMPartialReprovision, lockoutAuthFile); err != nil { + logger.Noticef("WARNING: could not repair platform: %v", err) + return AutoRepairFailedPlatformInit, nil + } + default: + return AutoRepairNotAttempted, fmt.Errorf("unknown key sealing method: %q", method) + } + + mgr := fdeMgr(st) + wrapped := &unlockedStateManager{ + FDEManager: mgr, + unlocker: st.Unlocker(), + } + err = boot.WithBootChains(func(bc boot.BootChains) error { + params := boot.ResealKeyForBootChainsParams{ + BootChains: bc, + Options: boot.ResealKeyToModeenvOptions{Force: true}, + } + return backendResealKeyForBootChains(wrapped, method, dirs.GlobalRootDir, ¶ms) + }, method) + + if err != nil { + logger.Noticef("WARNING: could not auto repair keyslots: %v", err) + return AutoRepairFailedKeyslots, nil + } + + return AutoRepairSuccess, nil +} + +// AttemptAutoRepairIfNeeded looks at the activation state and status +// of lockout reset and may attempt to repair keyslots. If the +// auto-repair attempted has already occurred during the current boot, +// this will do nothing. +func AttemptAutoRepairIfNeeded(st *state.State, lockoutResetErr error) error { + if lockoutResetErr != nil { + // FIXME: we need to either try repair in some cases and save the + // error for the status API + return lockoutResetErr + } + + // let's get the result from previous attempt during the + // current boot + previousResult, err := getRepairAttemptResult(st) + if err != nil { + return err + } + if previousResult != nil { + return nil + } + + s, err := getActivateState(st) + + if err == errNoActivateState { + logger.Noticef("WARNING: the system booted with an old initrd without using activation API") + unlockedState, err := bootLoadDiskUnlockState("unlocked.json") + if err != nil { + // errNoActivateState means the file must exist + return err + } + if unlockedState.UbuntuData.UnlockKey != "recovery" && unlockedState.UbuntuSave.UnlockKey != "recovery" { + setRepairAttemptResult(st, &repairState{Result: AutoRepairNotAttempted}) + return nil + } + } else if os.IsNotExist(err) { + logger.Noticef("WARNING: the system booted with an old initrd without unlocked status reporting") + setRepairAttemptResult(st, &repairState{Result: AutoRepairNotAttempted}) + return nil + } else if err != nil { + logger.Noticef("WARNING: error while getting activation state: %v", err) + setRepairAttemptResult(st, &repairState{Result: AutoRepairNotAttempted}) + return nil + } else { + if !secbootShouldAttemptRepair(s) { + setRepairAttemptResult(st, &repairState{Result: AutoRepairNotAttempted}) + return nil + } + } + + result, err := autoRepair(st) + if err != nil { + return err + } + setRepairAttemptResult(st, &repairState{Result: result}) + + return nil +} diff --git a/overlord/fdestate/autorepair_test.go b/overlord/fdestate/autorepair_test.go new file mode 100644 index 00000000000..3afc24cbe4b --- /dev/null +++ b/overlord/fdestate/autorepair_test.go @@ -0,0 +1,392 @@ +// -*- Mode: Go; indent-tabs-mode: t -*- +//go:build !nosecboot + +/* + * Copyright (C) 2026 Canonical Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 3 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package fdestate_test + +import ( + "fmt" + "os" + + . "gopkg.in/check.v1" + + sb "github.com/snapcore/secboot" + "github.com/snapcore/snapd/boot" + "github.com/snapcore/snapd/dirs" + "github.com/snapcore/snapd/gadget/device" + "github.com/snapcore/snapd/logger" + "github.com/snapcore/snapd/overlord/fdestate" + "github.com/snapcore/snapd/overlord/fdestate/backend" + "github.com/snapcore/snapd/secboot" + "github.com/snapcore/snapd/testutil" +) + +type autoRepairSuite struct { + fdeMgrSuite + + bootId string +} + +var _ = Suite(&autoRepairSuite{}) + +func (s *autoRepairSuite) SetUpTest(c *C) { + s.fdeMgrSuite.SetUpTest(c) + + s.bootId = "547730db-9e31-4c33-b418-1bce4e03f467" + s.AddCleanup(fdestate.MockOsutilBootID(func() (string, error) { + return s.bootId, nil + })) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairNeeded(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + s.createUnlockedState(c, sb.ActivationSucceededWithPlatformKey) + + reprovisioned := 0 + defer fdestate.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { + c.Check(mode, Equals, secboot.TPMPartialReprovision) + reprovisioned += 1 + return nil + })() + + defer fdestate.MockSecbootShouldAttemptRepair(func(as *secboot.ActivateState) bool { + return true + })() + + s.mockBootAssetsStateForModeenv(c) + + resealed := 0 + defer fdestate.MockBackendResealKeyForBootChains(func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error { + resealed += 1 + c.Check(params.Options.Force, Equals, true) + return nil + })() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + c.Check(reprovisioned, Equals, 1) + c.Check(resealed, Equals, 1) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("success")) + + // Try again it should do nothing + err = fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + c.Check(reprovisioned, Equals, 1) + c.Check(resealed, Equals, 1) + + // And keep the same result + result, err = fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("success")) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairNotNeeded(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + s.createUnlockedState(c, sb.ActivationSucceededWithPlatformKey) + + defer fdestate.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { + c.Errorf("Unexpected call") + return fmt.Errorf("Unexpected call") + })() + + defer fdestate.MockSecbootShouldAttemptRepair(func(as *secboot.ActivateState) bool { + return false + })() + + s.mockBootAssetsStateForModeenv(c) + + defer fdestate.MockBackendResealKeyForBootChains(func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error { + c.Errorf("Unexpected call") + return fmt.Errorf("Unexpected call") + })() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("not-attempted")) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairNeededBadReprovision(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + s.createUnlockedState(c, sb.ActivationSucceededWithPlatformKey) + + reprovisioned := 0 + defer fdestate.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { + reprovisioned += 1 + return fmt.Errorf("some error") + })() + + defer fdestate.MockSecbootShouldAttemptRepair(func(as *secboot.ActivateState) bool { + return true + })() + + s.mockBootAssetsStateForModeenv(c) + + defer fdestate.MockBackendResealKeyForBootChains(func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error { + c.Errorf("Unexpected call") + return fmt.Errorf("Unexpected call") + })() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + c.Check(reprovisioned, Equals, 1) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("failed-platform-init")) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairErrorNoActivateState(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + defer fdestate.MockBootLoadDiskUnlockState(func(name string) (*boot.DiskUnlockState, error) { + c.Check(name, Equals, "unlocked.json") + return &boot.DiskUnlockState{}, nil + })() + + logbuf, restore := logger.MockLogger() + defer restore() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("not-attempted")) + + c.Check(logbuf.String(), testutil.Contains, `WARNING: the system booted with an old initrd without using activation API`) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairErrorNoActivateStateRecovery(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + reprovisioned := 0 + defer fdestate.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { + c.Check(mode, Equals, secboot.TPMPartialReprovision) + reprovisioned += 1 + return nil + })() + + defer fdestate.MockSecbootShouldAttemptRepair(func(as *secboot.ActivateState) bool { + return true + })() + + s.mockBootAssetsStateForModeenv(c) + + resealed := 0 + defer fdestate.MockBackendResealKeyForBootChains(func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error { + resealed += 1 + c.Check(params.Options.Force, Equals, true) + return nil + })() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + defer fdestate.MockBootLoadDiskUnlockState(func(name string) (*boot.DiskUnlockState, error) { + c.Check(name, Equals, "unlocked.json") + return &boot.DiskUnlockState{ + UbuntuData: boot.PartitionState{ + UnlockKey: boot.KeyRecovery, + }, + }, nil + })() + + logbuf, restore := logger.MockLogger() + defer restore() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("success")) + + c.Check(logbuf.String(), testutil.Contains, `WARNING: the system booted with an old initrd without using activation API`) + + c.Check(reprovisioned, Equals, 1) + c.Check(resealed, Equals, 1) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairErrorActivateState(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + defer fdestate.MockBootLoadDiskUnlockState(func(name string) (*boot.DiskUnlockState, error) { + c.Check(name, Equals, "unlocked.json") + return nil, fmt.Errorf("cannot read state") + })() + + logbuf, restore := logger.MockLogger() + defer restore() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("not-attempted")) + + c.Check(logbuf.String(), testutil.Contains, `WARNING: error while getting activation state: cannot read state`) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairErrorNoFileActivateState(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + defer fdestate.MockBootLoadDiskUnlockState(func(name string) (*boot.DiskUnlockState, error) { + c.Check(name, Equals, "unlocked.json") + return nil, os.ErrNotExist + })() + + logbuf, restore := logger.MockLogger() + defer restore() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("not-attempted")) + + c.Check(logbuf.String(), testutil.Contains, `WARNING: the system booted with an old initrd without unlocked status reporting`) +} + +func (s *autoRepairSuite) TestAttemptAutoRepairNeededBadReseal(c *C) { + const onClassic = false + s.startedManager(c, onClassic) + + s.st.Lock() + defer s.st.Unlock() + + c.Assert(device.StampSealedKeys(dirs.GlobalRootDir, device.SealingMethodTPM), IsNil) + + s.createUnlockedState(c, sb.ActivationSucceededWithPlatformKey) + + reprovisioned := 0 + defer fdestate.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { + c.Check(mode, Equals, secboot.TPMPartialReprovision) + reprovisioned += 1 + return nil + })() + + defer fdestate.MockSecbootShouldAttemptRepair(func(as *secboot.ActivateState) bool { + return true + })() + + s.mockBootAssetsStateForModeenv(c) + + resealed := 0 + defer fdestate.MockBackendResealKeyForBootChains(func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error { + resealed += 1 + c.Check(params.Options.Force, Equals, true) + return fmt.Errorf("some error") + })() + + err := fdestate.AttemptAutoRepairIfNeeded(s.st, nil) + c.Assert(err, IsNil) + + c.Check(reprovisioned, Equals, 1) + c.Check(resealed, Equals, 1) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + + c.Check(result.Result, Equals, fdestate.AutoRepairResult("failed-keyslots")) +} + +func (s *autoRepairSuite) TestIgnoreOldAutoRepairResult(c *C) { + s.st.Lock() + defer s.st.Unlock() + + err := fdestate.SetRepairAttemptResult(s.st, &fdestate.RepairState{Result: fdestate.AutoRepairFailedPlatformInit}) + c.Assert(err, IsNil) + + result, err := fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + c.Check(result.Result, Equals, fdestate.AutoRepairResult("failed-platform-init")) + + s.bootId = "e6784d27-c31b-4d1f-be44-582702fd6250" + result, err = fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + c.Check(result, IsNil) + + err = fdestate.SetRepairAttemptResult(s.st, &fdestate.RepairState{Result: fdestate.AutoRepairFailedPlatformInit}) + c.Assert(err, IsNil) + + result, err = fdestate.GetRepairAttemptResult(s.st) + c.Assert(err, IsNil) + c.Check(result.Result, Equals, fdestate.AutoRepairResult("failed-platform-init")) +} diff --git a/overlord/fdestate/backend/reseal.go b/overlord/fdestate/backend/reseal.go index 5ad6e1f2114..94f4d7e99b0 100644 --- a/overlord/fdestate/backend/reseal.go +++ b/overlord/fdestate/backend/reseal.go @@ -631,11 +631,10 @@ func ResealKeyForBootChains(manager FDEStateManager, method device.SealingMethod bootChains: params.BootChains, }, resealOptions{ - ExpectReseal: params.Options.ExpectReseal, - Force: params.Options.Force, - EnsureProvisioned: params.Options.EnsureProvisioned, - IgnoreFDEHooks: params.Options.IgnoreFDEHooks, - Revoke: params.Options.RevokeOldKeys, + ExpectReseal: params.Options.ExpectReseal, + Force: params.Options.Force, + IgnoreFDEHooks: params.Options.IgnoreFDEHooks, + Revoke: params.Options.RevokeOldKeys, }) } @@ -669,11 +668,10 @@ type resealInputs struct { } type resealOptions struct { - ExpectReseal bool - Force bool - EnsureProvisioned bool - Revoke bool - IgnoreFDEHooks bool + ExpectReseal bool + Force bool + Revoke bool + IgnoreFDEHooks bool } func resealKeys( @@ -688,12 +686,7 @@ func resealKeys( } case device.SealingMethodTPM, device.SealingMethodLegacyTPM: - if opts.EnsureProvisioned { - lockoutAuthFile := device.TpmLockoutAuthUnder(boot.InstallHostFDESaveDir) - if err := secbootProvisionTPM(secboot.TPMPartialReprovision, lockoutAuthFile); err != nil { - return err - } - } + default: return fmt.Errorf("unknown key sealing method: %q", method) } diff --git a/overlord/fdestate/backend/reseal_test.go b/overlord/fdestate/backend/reseal_test.go index 434905b5540..a8b14ef7062 100644 --- a/overlord/fdestate/backend/reseal_test.go +++ b/overlord/fdestate/backend/reseal_test.go @@ -2598,175 +2598,3 @@ func (s *resealTestSuite) TestResealKeyForSignatureDBUpdate(c *C) { c.Check(buildProfileCalls, Equals, 3) c.Check(resealKeysCalls, Equals, 3) } - -func (s *resealTestSuite) TestTPMResealEnsureProvisioned(c *C) { - bl := bootloadertest.Mock("trusted", "").WithTrustedAssets() - bootloader.Force(bl) - defer bootloader.Force(nil) - - bl.TrustedAssetsMap = map[string]string{ - "asset": "asset", - } - recoveryKernel := bootloader.NewBootFile("/var/lib/snapd/seed/snaps/pc-kernel_1.snap", "kernel.efi", bootloader.RoleRecovery) - runKernel := bootloader.NewBootFile(filepath.Join(s.rootdir, "var/lib/snapd/snaps/pc-kernel_500.snap"), "kernel.efi", bootloader.RoleRunMode) - - bl.RecoveryBootChainList = []bootloader.BootFile{ - bootloader.NewBootFile("", "asset", bootloader.RoleRecovery), - recoveryKernel, - } - bl.BootChainList = []bootloader.BootFile{ - bootloader.NewBootFile("", "asset", bootloader.RoleRunMode), - runKernel, - } - - c.Assert(os.MkdirAll(filepath.Join(dirs.SnapBootAssetsDir, "trusted"), 0755), IsNil) - for _, name := range []string{ - "asset-runassethash", - "asset-assethash", - } { - err := os.WriteFile(filepath.Join(dirs.SnapBootAssetsDir, "trusted", name), nil, 0644) - c.Assert(err, IsNil) - } - - model := boottest.MakeMockUC20Model() - bootChains := boot.BootChains{ - RunModeBootChains: []boot.BootChain{ - { - BrandID: model.BrandID(), - Model: model.Model(), - Classic: model.Classic(), - Grade: model.Grade(), - ModelSignKeyID: model.SignKeyID(), - - AssetChain: []boot.BootAsset{ - { - Role: bootloader.RoleRecovery, - Name: "asset", - Hashes: []string{ - "assethash", - }, - }, - { - Role: bootloader.RoleRunMode, - Name: "asset", - Hashes: []string{ - "runassethash", - }, - }, - }, - - Kernel: "kernel.efi", - KernelRevision: "500", - KernelCmdlines: []string{ - "mode=run", - }, - KernelBootFile: runKernel, - }, - }, - - RecoveryBootChainsForRunKey: []boot.BootChain{ - { - BrandID: model.BrandID(), - Model: model.Model(), - Classic: model.Classic(), - Grade: model.Grade(), - ModelSignKeyID: model.SignKeyID(), - - AssetChain: []boot.BootAsset{ - { - Role: bootloader.RoleRecovery, - Name: "asset", - Hashes: []string{ - "assethash", - }, - }, - }, - - Kernel: "kernel.efi", - KernelRevision: "1", - KernelCmdlines: []string{ - "mode=recover", - }, - KernelBootFile: recoveryKernel, - }, - }, - - RecoveryBootChains: []boot.BootChain{ - { - BrandID: model.BrandID(), - Model: model.Model(), - Classic: model.Classic(), - Grade: model.Grade(), - ModelSignKeyID: model.SignKeyID(), - - AssetChain: []boot.BootAsset{ - { - Role: bootloader.RoleRecovery, - Name: "asset", - Hashes: []string{ - "assethash", - }, - }, - }, - - Kernel: "kernel.efi", - KernelRevision: "1", - KernelCmdlines: []string{ - "mode=recover", - }, - KernelBootFile: recoveryKernel, - }, - }, - - RoleToBlName: map[bootloader.Role]string{ - bootloader.RoleRecovery: "trusted", - bootloader.RoleRunMode: "trusted", - }, - } - - buildPCRProtectionProfileCalls := 0 - defer backend.MockSecbootBuildPCRProtectionProfile(func(modelParams []*secboot.SealKeyModelParams, checkResult *secboot.PreinstallCheckResult, allowInsufficientDmaProtection bool) (secboot.SerializedPCRProfile, error) { - buildPCRProtectionProfileCalls++ - return []byte(`"serialized-pcr-profile"`), nil - })() - - resealCalls := 0 - defer backend.MockSecbootResealKey(func(key secboot.KeyDataLocation, params *secboot.ResealKeyParams) (secboot.UpdatedKeys, error) { - resealCalls++ - return nil, nil - })() - - myState := &fakeState{} - myState.EncryptedContainers = []backend.EncryptedContainer{ - &encryptedContainer{ - uuid: "123", - containerRole: "system-data", - legacyKeys: map[string]string{ - "default": filepath.Join(s.rootdir, "run/mnt/ubuntu-boot/device/fde/ubuntu-data.sealed-key"), - "default-fallback": filepath.Join(s.rootdir, "run/mnt/ubuntu-seed/device/fde/ubuntu-data.recovery.sealed-key"), - }, - }, - &encryptedContainer{ - uuid: "456", - containerRole: "system-save", - legacyKeys: map[string]string{ - "default-fallback": filepath.Join(s.rootdir, "run/mnt/ubuntu-seed/device/fde/ubuntu-save.recovery.sealed-key"), - }, - }, - } - - provisioned := 0 - defer backend.MockSecbootProvisionTPM(func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error { - provisioned++ - c.Check(mode, Equals, secboot.TPMPartialReprovision) - c.Check(lockoutAuthFile, Equals, filepath.Join(s.rootdir, "/run/mnt/ubuntu-save/device/fde/tpm-lockout-auth")) - return nil - })() - - opts := boot.ResealKeyToModeenvOptions{ExpectReseal: true, EnsureProvisioned: true} - err := backend.ResealKeyForBootChains(myState, device.SealingMethodTPM, s.rootdir, &boot.ResealKeyForBootChainsParams{BootChains: bootChains, Options: opts}) - c.Assert(err, IsNil) - - c.Check(resealCalls, Equals, 3) - c.Check(provisioned, Equals, 1) -} diff --git a/overlord/fdestate/export_test.go b/overlord/fdestate/export_test.go index 5899ee6b806..1a0a16704b6 100644 --- a/overlord/fdestate/export_test.go +++ b/overlord/fdestate/export_test.go @@ -53,10 +53,15 @@ var ( CheckFDEChangeConflict = checkFDEChangeConflict CheckFDEParametersChangeConflicts = checkFDEParametersChangeConflicts + + SetRepairAttemptResult = setRepairAttemptResult + GetRepairAttemptResult = getRepairAttemptResult ) type ExternalOperation = externalOperation +type RepairState = repairState + func MockBackendResealKeyForBootChains(f func(manager backend.FDEStateManager, method device.SealingMethod, rootdir string, params *boot.ResealKeyForBootChainsParams) error) (restore func()) { restore = testutil.Backup(&backendResealKeyForBootChains) backendResealKeyForBootChains = f @@ -170,3 +175,15 @@ func MockBootLoadDiskUnlockState(f func(name string) (*boot.DiskUnlockState, err } type CachedActivateStateKey = cachedActivateStateKey + +func MockSecbootProvisionTPM(f func(mode secboot.TPMProvisionMode, lockoutAuthFile string) error) (restore func()) { + return testutil.Mock(&secbootProvisionTPM, f) +} + +func MockOsutilBootID(f func() (string, error)) (restore func()) { + return testutil.Mock(&osutilBootID, f) +} + +func MockSecbootShouldAttemptRepair(f func(as *secboot.ActivateState) bool) (restore func()) { + return testutil.Mock(&secbootShouldAttemptRepair, f) +} diff --git a/secboot/secboot_nosb.go b/secboot/secboot_nosb.go index 122552ca76c..b3217881c6b 100644 --- a/secboot/secboot_nosb.go +++ b/secboot/secboot_nosb.go @@ -207,6 +207,10 @@ func (*ActivateState) NumActivatedContainersWithRecoveryKey() uint { return 0 } +func ShouldAttemptRepair(a *ActivateState) bool { + return false +} + type ActivateContext interface { State() *ActivateState } diff --git a/secboot/secboot_sb.go b/secboot/secboot_sb.go index f07ba6c19c3..0d87d2ec4a8 100644 --- a/secboot/secboot_sb.go +++ b/secboot/secboot_sb.go @@ -108,6 +108,91 @@ func LockSealedKeys() error { type ActivateState = sb.ActivateState +func shouldAttemptRepairOnFailure(a *ActivateState) bool { + for _, activation := range a.Activations { + for _, errorType := range activation.KeyslotErrors { + switch errorType { + case sb.KeyslotErrorPlatformFailure: + return false + case sb.KeyslotErrorIncorrectUserAuth: + return false + case sb.KeyslotErrorInvalidKeyData: + // FIXME: for now when not using tokens, we get this error. We should get + // a different error to ignore when we use external keydata + // return false + case sb.KeyslotErrorInvalidPrimaryKey: + return false + case sb.KeyslotErrorUnknown: + return false + case sb.KeyslotErrorNone: + // This is not really clear if that should happen. + return false + // FIXME: add this case after updating secboot when we have this error + //case sb.KeyslotErrorIncorrectRoleParams: + // return false + case sb.KeyslotErrorIncompatibleRoleParams: + // FIXME: we should ignore this case only if the given keyslot is not expected + // to work for the boot mode. For now we just ignore it for every keyslot. + } + } + } + // We only encountered IncompatibleRoleParams errors. That + // means it could be repaired. + return true +} + +// ShouldAttemptRepair reads an activate state and decides whether +// a repair should be attempted. +func ShouldAttemptRepair(a *ActivateState) bool { + // First case: recovery. We do attempt repair if all keyslots + // of any disk unlocked with recovery key failed with + // IncompatibleRoleParams + for _, activation := range a.Activations { + if activation.Status == sb.ActivationSucceededWithRecoveryKey || activation.Status == sb.ActivationFailed { + return shouldAttemptRepairOnFailure(a) + } + } + + // Second case: degraded. All disk were unlocked + // successfully. But we check for some specific errors. + // * We ignore errors due to wrong user auth (failed attempt). + // * Some errors can be repaired: + // - If the role params are incompatible and we detect that this key should have been actually used. + // - If the role params were incorrect. + // * Other error point to more complicated issues that will need reprovision instead. + needAutoRepair := false + for _, activation := range a.Activations { + for _, errorType := range activation.KeyslotErrors { + switch errorType { + // No error + case sb.KeyslotErrorNone: + case sb.KeyslotErrorIncorrectUserAuth: + + // Require reprovision, auto-repair is not enough + case sb.KeyslotErrorInvalidKeyData: + return false + case sb.KeyslotErrorInvalidPrimaryKey: + return false + case sb.KeyslotErrorPlatformFailure: + return false + case sb.KeyslotErrorUnknown: + return false + + // Repair + case sb.KeyslotErrorIncompatibleRoleParams: + // FIXME: we need to verify the keyslot is expected to work in the current mode. + // For now, it is likely we attempted the "default" keyslots first and we are in run mode. + needAutoRepair = true + // FIXME: add this case after updating secboot when we have this error + //case sb.KeyslotErrorIncorrectRoleParams: + // needAutoRepair = true + } + } + } + + return needAutoRepair +} + // ActivateStateHasDegradedErrors looks for any error that is not // ignorable and should be reported on an ActivateState. // This function assumes all activations have been unlocked using diff --git a/secboot/secboot_sb_test.go b/secboot/secboot_sb_test.go index 07280712ce1..04d08730a0c 100644 --- a/secboot/secboot_sb_test.go +++ b/secboot/secboot_sb_test.go @@ -5843,3 +5843,69 @@ func (s *secbootSuite) TestActivateStateDegraded(c *C) { c.Check(secboot.ActivateStateHasDegradedErrors(state), Equals, true) } } + +func (s *secbootSuite) TestShouldAttemptRepairDegraded(c *C) { + state := &secboot.ActivateState{} + state.Activations = map[string]*sb.ContainerActivateState{ + "a": { + Status: sb.ActivationSucceededWithPlatformKey, + KeyslotErrors: map[string]sb.KeyslotErrorType{ + "a": sb.KeyslotErrorNone, + }, + }, + } + + // Incorrect user auth should not cause repair + state.Activations["a"].KeyslotErrors["b"] = sb.KeyslotErrorIncorrectUserAuth + c.Check(secboot.ShouldAttemptRepair(state), Equals, false) + + for _, failure := range []sb.KeyslotErrorType{ + // No repair cases since we need reprovision instead + sb.KeyslotErrorInvalidKeyData, + sb.KeyslotErrorInvalidPrimaryKey, + sb.KeyslotErrorPlatformFailure, + sb.KeyslotErrorUnknown, + } { + state.Activations["a"].KeyslotErrors["b"] = failure + + c.Check(secboot.ShouldAttemptRepair(state), Equals, false) + } + + state.Activations["a"].KeyslotErrors["b"] = sb.KeyslotErrorIncompatibleRoleParams + + c.Check(secboot.ShouldAttemptRepair(state), Equals, true) +} + +func (s *secbootSuite) TestShouldAttemptRepairWithRecovery(c *C) { + state := &secboot.ActivateState{} + state.Activations = map[string]*sb.ContainerActivateState{ + "a": { + Status: sb.ActivationSucceededWithRecoveryKey, + KeyslotErrors: map[string]sb.KeyslotErrorType{ + "a": sb.KeyslotErrorIncorrectUserAuth, + }, + }, + "ignored": { + Status: sb.ActivationSucceededWithPlatformKey, + KeyslotErrors: map[string]sb.KeyslotErrorType{ + "a": sb.KeyslotErrorIncompatibleRoleParams, + }, + }, + } + + c.Check(secboot.ShouldAttemptRepair(state), Equals, false) + + for _, failure := range []sb.KeyslotErrorType{ + // No repair cases since we need reprovision instead + //sb.KeyslotErrorInvalidKeyData, + sb.KeyslotErrorInvalidPrimaryKey, + sb.KeyslotErrorPlatformFailure, + sb.KeyslotErrorUnknown, + } { + state.Activations["a"].KeyslotErrors["a"] = failure + c.Check(secboot.ShouldAttemptRepair(state), Equals, false) + } + + state.Activations["a"].KeyslotErrors["a"] = sb.KeyslotErrorIncompatibleRoleParams + c.Check(secboot.ShouldAttemptRepair(state), Equals, true) +} diff --git a/tests/nested/manual/core-recover-from-recovery/task.yaml b/tests/nested/manual/core-recover-from-recovery/task.yaml index a60006428b0..e427d7579cd 100644 --- a/tests/nested/manual/core-recover-from-recovery/task.yaml +++ b/tests/nested/manual/core-recover-from-recovery/task.yaml @@ -43,7 +43,34 @@ prepare: | "bad-key.esl" "dbx-update.auth" fi +debug: | + cat initial-state.json || true + cat pre-repair-state.json || true + cat post-repair-state.json || true + execute: | + api_get_v2_system_info_storage_encrypted() { + remote.exec "sudo snap debug api /v2/system-info/storage-encrypted 2>/dev/null" + } + + wait_for_auto_repair_state() { + save_state="${1}" + expected="${2}" + # shellcheck disable=SC2034 + for try in {0..120}; do + api_get_v2_system_info_storage_encrypted >"${save_state}" + gojq -r '.result."auto-repair-result"' <"${save_state}" >last-autorepair-result + if MATCH "${expected}" initial-state.json + gojq -r '.result.status' recovery.out tests.nested vm set-recovery-key "$(sed '/recovery: */{;s///;q;};d' recovery.out)" @@ -81,12 +113,13 @@ execute: | # We must have been able to unlock with the plain key test "$(gojq -r '."ubuntu-save"."unlock-key"' pre-repair-state.json + gojq -r '.result.status' post-repair-state.json + gojq -r '.result.status' Date: Fri, 3 Apr 2026 15:22:12 +0200 Subject: [PATCH 06/44] o/snapstate: do not forget components in doLinkSnap We found that snap enable / snap disable cycle forgets the components of a given snap by erasing them from the state. The component mount points and everything else still exists, but snapd is confused about it. Fix doLinkSnap to retain component information and add a simple unit test. Signed-off-by: Zygmunt Krynicki --- overlord/snapstate/handlers.go | 11 +++-- overlord/snapstate/handlers_link_test.go | 53 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) diff --git a/overlord/snapstate/handlers.go b/overlord/snapstate/handlers.go index 18fdceaf58c..149f94b1137 100644 --- a/overlord/snapstate/handlers.go +++ b/overlord/snapstate/handlers.go @@ -2392,10 +2392,15 @@ func (m *SnapManager) doLinkSnap(t *state.Task, _ *tomb.Tomb) (retErr error) { // find if the snap is already installed before we modify snapst below isInstalled := snapst.IsInstalled() - cand := sequence.NewRevisionSideState(snapsup.SideInfo, nil) - m.backend.Candidate(cand.Snap) + oldCandidateIndex := snapst.LastIndex(snapsup.SideInfo.Revision) + + var candidateComponents []*sequence.ComponentState + if oldCandidateIndex >= 0 { + candidateComponents = snapst.Sequence.Revisions[oldCandidateIndex].Components + } - oldCandidateIndex := snapst.LastIndex(cand.Snap.Revision) + cand := sequence.NewRevisionSideState(snapsup.SideInfo, candidateComponents) + m.backend.Candidate(cand.Snap) var oldRevsBeforeCand []snap.Revision if oldCandidateIndex < 0 { diff --git a/overlord/snapstate/handlers_link_test.go b/overlord/snapstate/handlers_link_test.go index 1627db3f3d5..37743d83cfb 100644 --- a/overlord/snapstate/handlers_link_test.go +++ b/overlord/snapstate/handlers_link_test.go @@ -2125,6 +2125,59 @@ func (s *linkSnapSuite) TestDoUndoLinkSnapSequenceHadCandidate(c *C) { c.Check(t.Status(), Equals, state.UndoneStatus) } +func (s *linkSnapSuite) TestDoLinkSnapSequenceHadCandidateRetainsComponents(c *C) { + s.state.Lock() + defer s.state.Unlock() + + si1 := &snap.SideInfo{ + RealName: "foo", + Revision: snap.R(1), + } + si2 := &snap.SideInfo{ + RealName: "foo", + Revision: snap.R(2), + } + + compSI := &snap.ComponentSideInfo{ + Component: naming.NewComponentRef("foo", "comp"), + Revision: snap.R(11), + } + + snapstate.Set(s.state, "foo", &snapstate.SnapState{ + Sequence: snapstatetest.NewSequenceFromRevisionSideInfos([]*sequence.RevisionSideState{ + sequence.NewRevisionSideState(si1, []*sequence.ComponentState{ + sequence.NewComponentState(compSI, snap.StandardComponent), + }), + sequence.NewRevisionSideState(si2, nil), + }), + Current: si2.Revision, + }) + + t := s.state.NewTask("link-snap", "test") + t.Set("snap-setup", &snapstate.SnapSetup{ + SideInfo: si1, + Channel: "beta", + }) + s.state.NewChange("sample", "...").AddTask(t) + + s.state.Unlock() + s.se.Ensure() + s.se.Wait() + s.state.Lock() + + var snapst snapstate.SnapState + err := snapstate.Get(s.state, "foo", &snapst) + c.Assert(err, IsNil) + + comps := snapst.Sequence.ComponentsForRevision(si1.Revision) + c.Assert(comps, HasLen, 1) + c.Check(comps[0].SideInfo.Component, Equals, compSI.Component) + c.Check(comps[0].SideInfo.Revision, Equals, compSI.Revision) + c.Check(comps[0].CompType, Equals, snap.StandardComponent) + + c.Check(t.Status(), Equals, state.DoneStatus) +} + func (s *linkSnapSuite) TestDoUndoUnlinkCurrentSnapCore(c *C) { restore := release.MockOnClassic(true) defer restore() From 1c437dbe140e4facd10052d75dcc464c3d45b069 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Wed, 8 Apr 2026 10:50:12 +0200 Subject: [PATCH 07/44] tests: check that disable/enable retains components Signed-off-by: Zygmunt Krynicki --- tests/main/snap-info-components/task.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/main/snap-info-components/task.yaml b/tests/main/snap-info-components/task.yaml index d9e6589c9d2..9879294c9bf 100644 --- a/tests/main/snap-info-components/task.yaml +++ b/tests/main/snap-info-components/task.yaml @@ -42,3 +42,8 @@ execute: | snap info test-snap-with-components | MATCH '\+three.*test, not installed' # Exactly one component is not installed snap info test-snap-with-components | grep -c 'not installed' | MATCH '^1$' + + # Disabling and re-enabling the snap does NOT forget components anymore. + snap disable test-snap-with-components + snap enable test-snap-with-components + snap info test-snap-with-components | grep -c 'not installed' | MATCH '^1$' From 050297456c259991d523d0b6a913c5936ec53222 Mon Sep 17 00:00:00 2001 From: Zygmunt Krynicki Date: Fri, 10 Apr 2026 09:01:26 +0200 Subject: [PATCH 08/44] packaging: refactor and cleanup for snapd.mk (#16506) * packaging/debian: remove vendor-specific logic The code used to have both Debian and Ubuntu vendor checks. Remove the vendor check and act as if it is built for Debian all the time. This realistically disables --with-host-arch-32bit-triplet= and a number of statically-linked packages. The 32bit triplet is related to support for nvidia userspace driver for 32bit applications. Since this was never supported in Debian, this is not a regression. Signed-off-by: Zygmunt Krynicki * packaging: remove trailing spaces Signed-off-by: Zygmunt Krynicki * packaging: share build tree pruning in snapd.mk * cmd/snap: rename test-data directory to testdata Rename cmd/snap/test-data to cmd/snap/testdata to follow Go conventions and update all references in test files and packaging scripts. * packaging: remove redundant testdata copying dh-golang automatically installs testdata directories since version 1.31, so the manual mkdir and cp commands are no longer needed. * secboot: rename test-data directory to testdata Rename secboot/test-data to secboot/testdata to follow Go conventions and update all references in secboot_sb_test.go. * cmd/snap-bootstrap/blkid: rename test-data directory to testdata Rename cmd/snap-bootstrap/blkid/test-data to testdata to follow Go conventions and update reference in blkid_test.go. * packaging: refactor trusted account key checks into snapd.mk Extract the duplicated public-key-sha3-384 checking logic from debian-sid/rules and ubuntu-16.04/rules into a new reusable check-trusted-account-keys target in snapd.mk. The new target: - Checks snapd, snap-bootstrap, snap-preseed binaries for 2 expected keys - Checks snap-repair for 3 expected keys (2 common + 1 repair-root) - Only checks binaries that exist (handles differences between distros) - Provides clear error messages for debugging This reduces code duplication and makes it easier to maintain the security checks across different packaging files. * packaging: refactor static binary checks into snapd.mk Extract duplicated static linking verification logic from debian-sid/rules and ubuntu-16.04/rules into a new reusable check-static-binaries target in snapd.mk. The new target verifies that snap-exec, snap-update-ns, and snapctl are statically linked, as these binaries execute inside mount namespaces and cannot depend on external libraries. This matches the approach already used in openSUSE packaging which checks for 'statically linked|not a dynamic executable' in ldd output. Benefits: - Reduces code duplication across packaging files - Provides clearer error messages when checks fail - Makes it easier to maintain consistent checks across distributions * packaging: use check-static-binaries in debian rules Update debian-sid/rules and ubuntu-16.04/rules to use the new check-static-binaries target from snapd.mk instead of inline shell-based ldd checks. This reduces duplication and provides consistent error reporting. * packaging/opensuse: use snapd.mk static check * packaging/debian: generate snapd.defines.mk and pass to snapd.mk targets * rename install_dummy.go to install_placeholder.go * packaging/ubuntu: generate snapd.defines.mk and pass to snapd.mk targets * packaging/fedora: remove manual testdata copying The testdata directory is now handled automatically by the Go build system, so manual copying is no longer needed. * packaging/snapd.mk: clarify prepare-build-tree comment * packaging/debian: remove redundant nocheck test * packaging/debian: build snap via snapd.mk * packaging/snapd.mk: add with_vendor and with_static_pie to vars These variables were used in snapd.mk but not included in the vars validation list, which could lead to silent failures if they weren't defined in snapd.defines.mk. * packaging/debian: add with_vendor=0 to snapd.defines.mk Debian builds without using the vendor directory. * packaging/ubuntu: add with_vendor=1 to snapd.defines.mk Ubuntu builds use the vendor directory. * packaging/arch: add with_vendor and with_static_pie to PKGBUILD - with_vendor=1: Arch builds with Go vendor dependencies - with_static_pie=0: maintaining the status quo of undefined variable * packaging/fedora: add with_static_pie to snapd.spec - with_vendor already uses %{with_bundled} conditional - with_static_pie=0: maintaining the status quo of undefined variable * packaging/opensuse: add with_vendor=1 to snapd.spec - with_vendor=1: openSUSE builds with Go vendor dependencies * packaging: rename store key variables for clarity Rename SNAPD_STORE_KEY_1/2 to SNAPD_STORE_ROOT_KEY and SNAPD_STORE_GENERIC_MODELS_KEY respectively to better reflect their actual purpose and usage. * packaging: add snap binary check to check-trusted-account-keys Add validation for the snap binary in the check-trusted-account-keys target. The snap binary should also contain exactly 2 trusted keys (store root key and generic models key). * packaging: clarify trusted account keys comment Change 'should be' to 'must be' to better convey that these keys are required in production builds. * packaging/opensuse: fix pair of typos Signed-off-by: Zygmunt Krynicki * packaging: move checks for static pie binaries into snapd.mk Signed-off-by: Zygmunt Krynicki * packaging: rename prepare-build-tree to prepare-debian-build-tree Clarify that this target is specific to Debian builds by renaming it from the generic prepare-build-tree to prepare-debian-build-tree. Update all references in snapd.mk and debian-sid/rules. Signed-off-by: Zygmunt Krynicki * packaging/fedora: set BASH_XTRACEFD= when calling dnf DNF closes incoming file descriptors so this didn't work and issued a bunch of noisy warnings. Signed-off-by: Zygmunt Krynicki * packaging/fedora: copy sources earlier Signed-off-by: Zygmunt Krynicki * packaging/arch: use snapd.mk for checks Signed-off-by: Zygmunt Krynicki * packaging: add snap-gdbserver-shim to static binaries list Signed-off-by: Zygmunt Krynicki * packaging: explicitly ship testdata in -devel package Signed-off-by: Zygmunt Krynicki * packaging: bump dh-golang to >1.31 for testdata support For details look for "testdata" in https://manpages.debian.org/testing/dh-golang/Debian::Debhelper::Buildsystem::golang.3pm.en.html Signed-off-by: Zygmunt Krynicki * packaging: always check snapd and snap keys Those binaries are always expected so instead of making the test conditional on binary presence, make it unconditional. This can help unmask problems that would otherwise be silently ignored. Signed-off-by: Zygmunt Krynicki * packaging: use xargs -r to not fail when given no input Signed-off-by: Zygmunt Krynicki * packaging: respect GO111MODULE=off Signed-off-by: Zygmunt Krynicki * packaging: make all the first goal Signed-off-by: Zygmunt Krynicki * packaging: ensure that snap{,d} binary exist before checking keys Signed-off-by: Zygmunt Krynicki * packaging: print errors to stderr Signed-off-by: Zygmunt Krynicki * packaging/ubuntu-16.04: set builddir= in snapd.defines.mk Signed-off-by: Zygmunt Krynicki * packaging/debian-sid: set builddir= in snapd.defines.mk Signed-off-by: Zygmunt Krynicki * packaging: pass -e to bash in the container Signed-off-by: Zygmunt Krynicki * Revert "packaging: bump dh-golang to >1.31 for testdata support" This reverts commit 5e643e5f46e4f85536294f677dd1c6d1ffe4b005. * packaging: retain manual testdata copy for xenial * packaging: fix copying of debian/ directory --------- Signed-off-by: Zygmunt Krynicki --- cmd/snap-bootstrap/blkid/blkid_test.go | 2 +- .../repart.d/10-ext4.conf | 0 cmd/snap/cmd_keys_test.go | 2 +- cmd/snap/cmd_sign_build_test.go | 4 +- cmd/snap/{test-data => testdata}/pubring.gpg | Bin cmd/snap/{test-data => testdata}/secring.gpg | Bin cmd/snap/{test-data => testdata}/trustdb.gpg | Bin ...nstall_dummy.go => install_placeholder.go} | 0 packaging/arch/PKGBUILD | 20 +- packaging/arch/README.md | 2 +- packaging/debian-sid/README.md | 6 +- packaging/debian-sid/rules | 83 ++++----- packaging/fedora/README.md | 13 +- packaging/fedora/snapd.spec | 14 +- packaging/opensuse/README.md | 2 +- packaging/opensuse/snapd.spec | 27 +-- packaging/snapd.mk | 176 +++++++++++++++++- packaging/ubuntu-16.04/README.md | 6 +- packaging/ubuntu-16.04/rules | 55 ++++-- secboot/secboot_sb_test.go | 14 +- secboot/{test-data => testdata}/keydata | 0 secboot/{test-data => testdata}/keydata2 | 0 secboot/{test-data => testdata}/keyfile | Bin 23 files changed, 294 insertions(+), 132 deletions(-) rename cmd/snap-bootstrap/blkid/{test-data => testdata}/repart.d/10-ext4.conf (100%) rename cmd/snap/{test-data => testdata}/pubring.gpg (100%) rename cmd/snap/{test-data => testdata}/secring.gpg (100%) rename cmd/snap/{test-data => testdata}/trustdb.gpg (100%) rename gadget/install/{install_dummy.go => install_placeholder.go} (100%) rename secboot/{test-data => testdata}/keydata (100%) rename secboot/{test-data => testdata}/keydata2 (100%) rename secboot/{test-data => testdata}/keyfile (100%) diff --git a/cmd/snap-bootstrap/blkid/blkid_test.go b/cmd/snap-bootstrap/blkid/blkid_test.go index 3049cb53065..3b5bd837be1 100644 --- a/cmd/snap-bootstrap/blkid/blkid_test.go +++ b/cmd/snap-bootstrap/blkid/blkid_test.go @@ -52,7 +52,7 @@ func (s *blkidSuite) SetUpTest(c *C) { tmp := c.MkDir() image := filepath.Join(tmp, "image") - cmd := exec.Command(systemdRepart, "--offline=yes", "--size=64M", "--empty=create", "--definitions=test-data/repart.d", image) + cmd := exec.Command(systemdRepart, "--offline=yes", "--size=64M", "--empty=create", "--definitions=testdata/repart.d", image) err = cmd.Run() if err != nil { c.Skip("systemd-repart is not working") diff --git a/cmd/snap-bootstrap/blkid/test-data/repart.d/10-ext4.conf b/cmd/snap-bootstrap/blkid/testdata/repart.d/10-ext4.conf similarity index 100% rename from cmd/snap-bootstrap/blkid/test-data/repart.d/10-ext4.conf rename to cmd/snap-bootstrap/blkid/testdata/repart.d/10-ext4.conf diff --git a/cmd/snap/cmd_keys_test.go b/cmd/snap/cmd_keys_test.go index 356a6358510..e70c778c486 100644 --- a/cmd/snap/cmd_keys_test.go +++ b/cmd/snap/cmd_keys_test.go @@ -69,7 +69,7 @@ func (s *SnapKeysSuite) SetUpTest(c *C) { s.tempdir = c.MkDir() for _, fileName := range []string{"pubring.gpg", "secring.gpg", "trustdb.gpg"} { - data, err := os.ReadFile(filepath.Join("test-data", fileName)) + data, err := os.ReadFile(filepath.Join("testdata", fileName)) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(s.tempdir, fileName), data, 0644) c.Assert(err, IsNil) diff --git a/cmd/snap/cmd_sign_build_test.go b/cmd/snap/cmd_sign_build_test.go index a5979078b80..1a2de5aeb53 100644 --- a/cmd/snap/cmd_sign_build_test.go +++ b/cmd/snap/cmd_sign_build_test.go @@ -78,7 +78,7 @@ func (s *SnapSignBuildSuite) TestSignBuildWorks(c *C) { tempdir := c.MkDir() for _, fileName := range []string{"pubring.gpg", "secring.gpg", "trustdb.gpg"} { - data, err := os.ReadFile(filepath.Join("test-data", fileName)) + data, err := os.ReadFile(filepath.Join("testdata", fileName)) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(tempdir, fileName), data, 0644) c.Assert(err, IsNil) @@ -113,7 +113,7 @@ func (s *SnapSignBuildSuite) TestSignBuildWorksDevelGrade(c *C) { tempdir := c.MkDir() for _, fileName := range []string{"pubring.gpg", "secring.gpg", "trustdb.gpg"} { - data, err := os.ReadFile(filepath.Join("test-data", fileName)) + data, err := os.ReadFile(filepath.Join("testdata", fileName)) c.Assert(err, IsNil) err = os.WriteFile(filepath.Join(tempdir, fileName), data, 0644) c.Assert(err, IsNil) diff --git a/cmd/snap/test-data/pubring.gpg b/cmd/snap/testdata/pubring.gpg similarity index 100% rename from cmd/snap/test-data/pubring.gpg rename to cmd/snap/testdata/pubring.gpg diff --git a/cmd/snap/test-data/secring.gpg b/cmd/snap/testdata/secring.gpg similarity index 100% rename from cmd/snap/test-data/secring.gpg rename to cmd/snap/testdata/secring.gpg diff --git a/cmd/snap/test-data/trustdb.gpg b/cmd/snap/testdata/trustdb.gpg similarity index 100% rename from cmd/snap/test-data/trustdb.gpg rename to cmd/snap/testdata/trustdb.gpg diff --git a/gadget/install/install_dummy.go b/gadget/install/install_placeholder.go similarity index 100% rename from gadget/install/install_dummy.go rename to gadget/install/install_placeholder.go diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index 9d1b6d2bcf9..741d62f9843 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -63,6 +63,8 @@ with_core_bits = 0 with_alt_snap_mount_dir = 1 with_apparmor = 1 with_testkeys = ${WITH_TEST_KEYS:-0} +with_vendor = 1 +with_static_pie = 0 EXTRA_GO_BUILD_FLAGS = -trimpath EXTRA_GO_LDFLAGS = -w -s __DEFINES__ @@ -99,19 +101,11 @@ __DEFINES__ check() { cd "$pkgname-$pkgver" - # make sure the binaries that need to be built statically really are - for binary in snap-exec snap-update-ns snapctl; do - if ! LC_ALL=C ldd "$srcdir/_go_build/$binary" 2>&1 | grep -q 'not a dynamic executable'; then - echo "$binary is not a static binary" - exit 1 - fi - done - - SKIP_UNCLEAN=1 IGNORE_MISSING=1 ./run-checks --unit - # XXX: Static checks choke on autotools generated cruft. Let's not run them - # here as they are designed to pass on a clean tree, before anything else is - # done, not after building the tree. - # ./run-checks --static + make -f packaging/snapd.mk \ + SNAPD_DEFINES_DIR="$srcdir" \ + check \ + check-static-binaries \ + check-trusted-account-keys TMPDIR=/tmp make -C cmd -k check TMPDIR=/tmp make -C data -k check } diff --git a/packaging/arch/README.md b/packaging/arch/README.md index 149bb57e548..53c85d12968 100644 --- a/packaging/arch/README.md +++ b/packaging/arch/README.md @@ -49,7 +49,7 @@ podman run \ -v "snapd-gomod-cache:/var/cache/gomod" \ -w /build \ docker.io/archlinux/archlinux:latest \ - /bin/bash -x -u + /bin/bash -x -e -u ``` ## Host Script diff --git a/packaging/debian-sid/README.md b/packaging/debian-sid/README.md index 8bf3fbb16a9..8ee8a006377 100644 --- a/packaging/debian-sid/README.md +++ b/packaging/debian-sid/README.md @@ -52,7 +52,7 @@ podman run \ -v "snapd-gomod-cache:/var/cache/gomod" \ -w /build \ docker.io/debian:sid \ - /bin/bash -x -u + /bin/bash -x -e -u ``` ## Host Script @@ -116,7 +116,9 @@ tar -C /src -c \ # Copy packaging files to the build directory. tar -Jxf /build/snapd_"$version".no-vendor.tar.xz -C /build -cp -a /src/packaging/debian-sid /build/snapd-"$version"/debian +# The debian-sid directory contains .build/ which would be copied recursively, exclude it by using a glob. +mkdir /build/snapd-"$version"/debian +cp -a /src/packaging/debian-sid/* /build/snapd-"$version"/debian # Discover and install build dependencies. apt-get --yes build-dep /build/snapd-"$version" diff --git a/packaging/debian-sid/rules b/packaging/debian-sid/rules index e1673f3726a..0831a0c9d6a 100755 --- a/packaging/debian-sid/rules +++ b/packaging/debian-sid/rules @@ -79,30 +79,8 @@ BUILT_USING_PACKAGES=libc-dev-bin # DPKG_EXPORT_BUILDFLAGS = 1 # include /usr/share/dpkg/buildflags.mk -# Currently, we enable confinement for Ubuntu only, not for derivatives, -# because derivatives may have different kernels that don't support all the -# required confinement features and we don't to mislead anyone about the -# security of the system. Discuss a proper approach to this for downstreams -# if and when they approach us. -ifeq ($(shell dpkg-vendor --query Vendor),Ubuntu) - # On Ubuntu 16.04 we need to produce a build that can be used on wide - # variety of systems. As such we prefer static linking over dynamic linking - # for stability, predicability and easy of deployment. We need to link some - # things dynamically though: udev has no stable IPC protocol between - # libudev and udevd so we need to link with it dynamically. - VENDOR_ARGS=--enable-nvidia-multiarch --enable-static-libcap --enable-static-libapparmor --enable-static-libseccomp --with-host-arch-triplet=$(DEB_HOST_MULTIARCH) -ifeq ($(shell dpkg-architecture -qDEB_HOST_ARCH),amd64) - VENDOR_ARGS+= --with-host-arch-32bit-triplet=$(shell dpkg-architecture -f -ai386 -qDEB_HOST_MULTIARCH) -endif - BUILT_USING_PACKAGES+=libcap-dev libapparmor-dev libseccomp-dev -else -ifeq ($(shell dpkg-vendor --query Vendor),Debian) - VENDOR_ARGS=--enable-nvidia-multiarch - BUILT_USING_PACKAGES+=libcap-dev -else - VENDOR_ARGS=--disable-apparmor -endif -endif +VENDOR_ARGS=--enable-nvidia-multiarch +BUILT_USING_PACKAGES+=libcap-dev BUILT_USING=$(shell dpkg-query -f '$${source:Package} (= $${source:Version}), ' -W $(BUILT_USING_PACKAGES)) %: @@ -128,6 +106,7 @@ override_dh_clean: $(MAKE) -C data clean # XXX: hacky $(MAKE) -C cmd distclean || true + rm -f snapd.defines.mk version_from_changelog := $(shell dpkg-parsechangelog --show-field Version) $(info version_from_changelog is $(version_from_changelog)) @@ -150,33 +129,51 @@ SNAPD_APPARMOR_REEXEC=0 SNAPD_ASSERTS_FORMATS='{"account-key":1,"snap-declaration":6,"system-user":2}' endef -override_dh_auto_build: +# Generate snapd.defines.mk for use by snapd.mk makefile targets. +# This file contains variable definitions that match Debian's FHS conventions. +define snapd_defines_mk_content +# This file is generated by Debian's snapd.rules +# Directory variables +prefix = /usr +bindir = /usr/bin +sbindir = /usr/sbin +libexecdir = /usr/lib/snapd +mandir = /usr/share/man +datadir = /usr/share +localstatedir = /var +sharedstatedir = /var/lib +unitdir = $(shell pkg-config --variable=systemdsystemunitdir systemd) +# Build configuration +builddir=$(CURDIR)/_build/bin +with_core_bits = 0 +with_alt_snap_mount_dir = 0 +with_apparmor = 1 +with_testkeys = $(if $(findstring testkeys,$(DEB_BUILD_OPTIONS)),1,0) +with_static_pie = 0 +with_vendor = 0 +endef + +snapd.defines.mk: + $(file >$@,$(snapd_defines_mk_content)) + +override_dh_auto_build: snapd.defines.mk # We used to call mkversion.sh, but since it builds and executes go code # from this package it is significantly easier to do by hand. $(file >_build/src/$(DH_GOPKG)/snapdtool/version_generated.go,$(snapdtool__version_generated.go)) $(file >cmd/VERSION,$(cmd__VERSION)) $(file >data/info,$(data__info)) # Build golang bits - mkdir -p _build/src/$(DH_GOPKG)/cmd/snap/test-data - cp -a cmd/snap/test-data/*.gpg _build/src/$(DH_GOPKG)/cmd/snap/test-data/ cp -a bootloader/assets/data _build/src/$(DH_GOPKG)/bootloader/assets - # exclude certain parts that won't be used by debian - find _build/src/$(DH_GOPKG)/cmd/snap-bootstrap -name "*.go" | xargs rm -f - find _build/src/$(DH_GOPKG)/cmd/snap-fde-keymgr -name "*.go" | xargs rm -f - find _build/src/$(DH_GOPKG)/gadget/install -name "*.go" | grep -vE '(params\.go|install_dummy\.go|kernel\.go)'| xargs rm -f - # XXX: once dh-golang understands go build tags this would not be needed - find _build/src/$(DH_GOPKG)/secboot/ -name "*.go" | grep -E '(.*_sb(_test)?\.go|.*_tpm(_test)?\.go|secboot_hooks.go|auth_requestor.go|keymgr/)' | xargs rm -f - mv _build/src/$(DH_GOPKG)/secboot/keys/plainkey.go _build/src/$(DH_GOPKG)/secboot/keys/plainkey_sb.go - mv _build/src/$(DH_GOPKG)/secboot/keys/plainkey_test.go _build/src/$(DH_GOPKG)/secboot/keys/plainkey_sb_test.go - find _build/src/$(DH_GOPKG)/secboot/keys/ -name "*.go" | grep -E '(.*_sb(_test)?\.go)' | xargs rm -f - find _build/src/$(DH_GOPKG)/boot/ -name "*.go" | grep -E '(.*_sb(_test)?\.go)' | xargs rm -f + # Prepare the build tree by removing code not used for Debian builds + $(MAKE) -f packaging/snapd.mk prepare-debian-build-tree sourcedir=_build/src/$(DH_GOPKG) SNAPD_DEFINES_DIR=$(CURDIR) + # and build, we cannot use modules as packaging on Debian requires us to use # dependencies from the distro, and this would require further updates to the # go.mod file GO111MODULE=off dh_auto_build -- $(BUILDFLAGS) -tags "$(TAGS)" $(GCCGOFLAGS) - (cd _build/bin && GO111MODULE=off GOPATH=$$(pwd)/.. go build $(BUILDFLAGS) $(GCCGOFLAGS) -tags "$(SNAP_TAGS)" $(DH_GOPKG)/cmd/snap) + GO111MODULE=off $(MAKE) -f packaging/snapd.mk $(CURDIR)/_build/bin/snap SNAPD_DEFINES_DIR=$(CURDIR) # (static linking on powerpc with cgo is broken) ifneq ($(shell dpkg-architecture -qDEB_HOST_ARCH),powerpc) @@ -188,9 +185,7 @@ ifneq ($(shell dpkg-architecture -qDEB_HOST_ARCH),powerpc) (cd _build/bin && GO111MODULE=off GOPATH=$$(pwd)/.. go build -tags "$(TAGS)" --ldflags '-extldflags "-static"' $(GCCGOFLAGS) -pkgdir=$$(pwd)/std $(DH_GOPKG)/cmd/snap-update-ns) # ensure we generated a static build - $(shell if ldd _build/bin/snap-exec; then false "need static build"; fi) - $(shell if ldd _build/bin/snap-update-ns; then false "need static build"; fi) - $(shell if ldd _build/bin/snapctl; then false "need static build"; fi) + $(MAKE) -f packaging/snapd.mk check-static-binaries SNAPD_DEFINES_DIR=$(CURDIR) endif # ensure snap-seccomp is build with a static libseccomp on Ubuntu @@ -220,11 +215,7 @@ override_dh_auto_test: # a tested default (production) build should have no test keys ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) # check that only the main trusted account-keys are included - [ $$(strings _build/bin/snapd|grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}") -eq 2 ] - strings _build/bin/snapd|grep -c "^public-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswHNiEB9Lxk$$" - strings _build/bin/snapd|grep -c "^public-key-sha3-384: d-JcZF9nD9eBw7bwMnH61x-bklnQOhQud1Is6o_cn2wTj8EYDi9musrIT9z2MdAa$$" -endif -ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) + $(MAKE) -f packaging/snapd.mk check-trusted-account-keys SNAPD_DEFINES_DIR=$(CURDIR) # run the snap-confine tests $(MAKE) -C cmd check endif diff --git a/packaging/fedora/README.md b/packaging/fedora/README.md index 032fcf35d4d..714eef4da20 100644 --- a/packaging/fedora/README.md +++ b/packaging/fedora/README.md @@ -49,7 +49,7 @@ podman run \ -v "snapd-gomod-cache:/var/cache/gomod:Z" \ -w /build \ registry.fedoraproject.org/fedora:latest \ - /bin/bash -x -u + /bin/bash -x -e -u ``` ## Host Script @@ -80,10 +80,13 @@ chmod 1777 /var/cache/gomod export GOMODCACHE=/var/cache/gomod # Install bootstrap packages. -dnf --assumeyes install --setopt=install_weak_deps=False --setopt=keepcache=True \ +BASH_XTRACEFD= dnf --assumeyes install --setopt=install_weak_deps=False --setopt=keepcache=True \ bash coreutils findutils gawk git gzip make rpm-build \ rpm-devel systemd-rpm-macros tar xz golang +# Copy packaging files to the build directory. +install -t /build/SPECS/ /src/packaging/fedora/snapd.spec + # Determine the version of the package. version=$(rpmspec -q --qf "%{VERSION}\n" /build/SPECS/snapd.spec | head -n1) @@ -106,13 +109,9 @@ tar -C /src -c \ # Create the no-vendor and only-vendor source archives. ( cd /src-rw && ./packaging/pack-source -v "$version" -o /build/SOURCES ) -# Copy packaging files to the build directory. -ls -lh /build -install -t /build/SPECS/ /src/packaging/fedora/snapd.spec - # Discover and install build dependencies. rpmspec -q --buildrequires /build/SPECS/snapd.spec >/tmp/buildreqs.txt -xargs -r -d "\n" dnf --assumeyes install --setopt=keepcache=True > unit-test-devel.file-list done - -# Install additional testdata -install -d %{buildroot}/%{gopath}/src/%{import_path}/cmd/snap/test-data/ -cp -pav cmd/snap/test-data/* %{buildroot}/%{gopath}/src/%{import_path}/cmd/snap/test-data/ -echo "%%{gopath}/src/%%{import_path}/cmd/snap/test-data" >> unit-test-devel.file-list +if [ -d cmd/snap/testdata ]; then + echo "%%dir %%{gopath}/src/%%{import_path}/cmd/snap/testdata" >> devel.file-list + install -d -p %{buildroot}/%{gopath}/src/%{import_path}/cmd/snap/testdata + for file in cmd/snap/testdata/*; do + cp -pav $file %{buildroot}/%{gopath}/src/%{import_path}/$file + echo "%%{gopath}/src/%%{import_path}/$file" >> unit-test-devel.file-list + done +fi %endif %if 0%{?with_devel} diff --git a/packaging/opensuse/README.md b/packaging/opensuse/README.md index 4513b0d6f5c..a1b0da0cb4f 100644 --- a/packaging/opensuse/README.md +++ b/packaging/opensuse/README.md @@ -50,7 +50,7 @@ podman run \ -v "snapd-gomod-cache:/var/cache/gomod" \ -w /build \ registry.opensuse.org/opensuse/tumbleweed:latest \ - /bin/bash -x -u + /bin/bash -x -e -u ``` ## Host Script diff --git a/packaging/opensuse/snapd.spec b/packaging/opensuse/snapd.spec index f53998e89fc..8c1fa7b4555 100644 --- a/packaging/opensuse/snapd.spec +++ b/packaging/opensuse/snapd.spec @@ -169,7 +169,7 @@ Requires: (snapd-selinux = %{version} if selinux-policy-%{selinuxtype}) %endif # Old versions of xdg-document-portal can expose data belonging to -# other confied apps. Older OpenSUSE releases are unlikely to change, +# other confined apps. Older OpenSUSE releases are unlikely to change, # so for now limit this to Tumbleweed. %if 0%{?suse_version} >= 1550 || 0%{?sle_version} >= 150300 Conflicts: xdg-desktop-portal < 0.11 @@ -260,12 +260,14 @@ localstatedir = %{_localstatedir} sharedstatedir = %{_sharedstatedir} unitdir = %{_unitdir} builddir = %{_builddir} +sourcedir = %{indigo_srcdir} # Build configuration with_core_bits = 0 with_alt_snap_mount_dir = %{!?with_alt_snap_mount_dir:0}%{?with_alt_snap_mount_dir:1} with_apparmor = %{with apparmor} with_testkeys = %{!?with_testkeys:0}%{?with_testkeys:1} with_static_pie = $build_with_static_pie +with_vendor = 1 EXTRA_GO_BUILD_FLAGS = -v -x # fix broken debuginfo bsc#1215402 EXTRA_GO_LDFLAGS = -compressdwarf=false @@ -278,7 +280,7 @@ popd # Sanity check, ensure that systemd system generator directory is in agreement between the build system and packaging. if [ "$(pkg-config --variable=systemdsystemgeneratordir systemd)" != "%{_systemdgeneratordir}" ]; then - echo "pkg-confing and rpm macros disagree about the location of systemd system generator directory" + echo "pkg-config and rpm macros disagree about the location of systemd system generator directory" exit 1 fi @@ -339,25 +341,8 @@ M4PARAM='-D distro_opensuse' %make_build -C %{indigo_srcdir}/data/selinux USE_ALT_SNAP_MOUNT_DIR=true %check - -static_pie= -if [ -e build-with-static-pie ]; then - static_pie=1 -fi - -# These binaries execute inside the mount namespace thus they must be built statically -pushd %{buildroot}/%{_libexecdir}/snapd/ -for binary in snap-exec snap-update-ns snapctl snap-gdbserver-shim; do - ldd $binary 2>&1 | grep 'statically linked\|not a dynamic executable' -done - -if [ -n "$static_pie" ]; then - for binary in snap-exec snap-update-ns snapctl snap-gdbserver-shim; do - file $binary | grep -F pie - done -fi - -popd +# Verify that statically linked binaries are indeed static +%make_build -f %{indigo_srcdir}/packaging/snapd.mk SNAPD_DEFINES_DIR=%{_builddir} check-static-binaries export CFLAGS="$RPM_OPT_FLAGS -fpie" export CXXFLAGS="$RPM_OPT_FLAGS -fpie" diff --git a/packaging/snapd.mk b/packaging/snapd.mk index b040f3a7d84..b037341a550 100644 --- a/packaging/snapd.mk +++ b/packaging/snapd.mk @@ -25,7 +25,9 @@ vars += bindir sbindir libexecdir mandir datadir localstatedir sharedstatedir un # with_apparmor: set to 1 to build snapd with apparmor support # with_core_bits: set to 1 to build snapd with things needed for the core/snapd snap # with_alt_snap_mount_dir: set to 1 to build snapd with alternate snap mount directory -vars += with_testkeys with_apparmor with_core_bits with_alt_snap_mount_dir +# with_vendor: set to 1 to build snapd using the vendor directory for dependencies +# with_static_pie: set to 1 to build static binaries in PIE mode if +vars += with_testkeys with_apparmor with_core_bits with_alt_snap_mount_dir with_vendor with_static_pie # Verify that none of the variables are empty. This may happen if snapd.mk and # distribution packaging generating snapd.defines.mk get out of sync. @@ -38,6 +40,13 @@ $(foreach var,$(vars),$(if $(value $(var)),,$(error $(var) is empty or unset, ch # Import path of snapd. import_path = github.com/snapcore/snapd +# Trusted account keys that must be present in production builds. +# These are used by check-trusted-account-keys target to verify that +# test keys are not accidentally included in production builds. +SNAPD_STORE_ROOT_KEY = -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswHNiEB9Lxk +SNAPD_STORE_GENERIC_MODELS_KEY = d-JcZF9nD9eBw7bwMnH61x-bklnQOhQud1Is6o_cn2wTj8EYDi9musrIT9z2MdAa +SNAPD_REPAIR_ROOT_KEY = nttW6NfBXI_E-00u38W-KH6eiksfQNXuI7IiumoV49_zkbhM0sYTzSnFlwZC-W4t + # This is usually set by %make_install. It is defined here to avoid warnings or # errors from referencing undefined variables. @@ -64,10 +73,14 @@ endif # Any additional tags common to all targets GO_TAGS += $(EXTRA_GO_BUILD_TAGS) +ifeq ($(GO111MODULE),off) +GO_MOD= +else GO_MOD=-mod=vendor ifeq ($(with_vendor),0) GO_MOD=-mod=readonly endif +endif GO_STATIC_BUILDMODE = default GO_STATIC_EXTLDFLAG = -static @@ -81,10 +94,36 @@ endif # Go -ldflags settings for static build. Can be overridden in snapd.defines.mk. EXTRA_GO_STATIC_LDFLAGS ?= -linkmode external -extldflags="$(GO_STATIC_EXTLDFLAG)" $(EXTRA_GO_LDFLAGS) +# sourcedir is the path to the source directory tree (where the go source files are). +# This is used by prepare-debian-build-tree to remove unnecessary code, and by +# check-static-binaries to locate C binaries built by the autotools cmd/ build. +# Can be set in snapd.defines.mk or on the make command line; defaults to $(CURDIR). +# For Debian/dh-golang, this would be: sourcedir=_build/src/github.com/snapcore/snapd +sourcedir ?= $(CURDIR) + # NOTE: This *depends* on building out of tree. Some of the built binaries # conflict with directory names in the tree. .PHONY: all -all: $(go_binaries) +all: $(go_binaries) + +# Prepare the build tree by removing code that is not used in non-embedded builds. +# This removes snap-bootstrap, snap-fde-keymgr, and secboot-related code that +# is only needed for embedded systems and UC20+ builds. This could be somewhat +# avoided if we had all the dependencies in Debian OR if dh-golang supported +# build tags properly. +.PHONY: prepare-debian-build-tree +prepare-debian-build-tree: + # exclude certain parts that won't be used by debian + find $(sourcedir)/cmd/snap-bootstrap -name "*.go" 2>/dev/null | xargs -r rm -f + find $(sourcedir)/cmd/snap-fde-keymgr -name "*.go" 2>/dev/null | xargs -r rm -f + find $(sourcedir)/gadget/install -name "*.go" -not -name "params.go" -not -name "install_placeholder.go" -not -name "kernel.go" 2>/dev/null | xargs -r rm -f + # XXX: once dh-golang understands go build tags this would not be needed + find $(sourcedir)/secboot/ -name "*.go" 2>/dev/null | grep -E '(.*_sb(_test)?\.go|.*_tpm(_test)?\.go|secboot_hooks.go|auth_requestor.go|keymgr/)' | xargs -r rm -f + # Rename plainkey files to indicate they are secboot variants + if [ -f $(sourcedir)/secboot/keys/plainkey.go ]; then mv $(sourcedir)/secboot/keys/plainkey.go $(sourcedir)/secboot/keys/plainkey_sb.go; fi + if [ -f $(sourcedir)/secboot/keys/plainkey_test.go ]; then mv $(sourcedir)/secboot/keys/plainkey_test.go $(sourcedir)/secboot/keys/plainkey_sb_test.go; fi + find $(sourcedir)/secboot/keys/ -name "*.go" 2>/dev/null | grep -E '(.*_sb(_test)?\.go)' | xargs -r rm -f + find $(sourcedir)/boot/ -name "*.go" 2>/dev/null | grep -E '(.*_sb(_test)?\.go)' | xargs -r rm -f # FIXME: not all Go toolchains we build with support '-B gobuildid', replace a # random GNU build ID with something more predictable, use something similar to @@ -109,6 +148,52 @@ $(builddir)/snap-update-ns $(builddir)/snap-exec $(builddir)/snapctl: $(EXTRA_GO_BUILD_FLAGS) \ $(import_path)/cmd/$(notdir $@) +# Check that critical binaries are statically linked. +# These binaries execute inside mount namespaces and cannot depend on external libraries. +# builddir: the directory containing the built Go binaries (e.g., _build/bin) +# sourcedir: the root of the snapd source tree (used to locate C binaries built by autotools) +.PHONY: check-static-binaries +check-static-binaries: + @echo "Checking that critical binaries are statically linked..." + @for binary in snap-exec snap-update-ns snapctl; do \ + if [ -f "$(builddir)/$$binary" ]; then \ + if ! file "$(builddir)/$$binary" | grep -q -F static; then \ + echo "ERROR: $$binary is dynamically linked, must be static"; \ + ldd "$(builddir)/$$binary"; \ + exit 1; \ + fi; \ + if [ "$(with_static_pie)" = 1 ] && ! file $(builddir)/$$binary | grep -q -F pie; then \ + echo "ERROR: $$binary is not a static PIE"; \ + exit 1; \ + fi; \ + echo " $$binary: OK (static)"; \ + fi; \ + done + @# snap-gdbserver-shim is a C binary built by the autotools cmd/ build, not by the Go + @# build rules, so it will not appear in $(builddir). Search several well-known locations + @# in order: the Go output dir (unlikely but harmless), the autotools in-tree build + @# directory, and the installed location when DESTDIR is set. + @shim=""; \ + for candidate in \ + "$(builddir)/snap-gdbserver-shim" \ + "$(sourcedir)/cmd/snap-gdb-shim/snap-gdbserver-shim" \ + "$(DESTDIR)$(libexecdir)/snapd/snap-gdbserver-shim"; do \ + if [ -f "$$candidate" ]; then shim="$$candidate"; break; fi; \ + done; \ + if [ -n "$$shim" ]; then \ + if ! file "$$shim" | grep -q -F static; then \ + echo "ERROR: snap-gdbserver-shim is dynamically linked, must be static"; \ + ldd "$$shim"; \ + exit 1; \ + fi; \ + if [ "$(with_static_pie)" = 1 ] && ! file "$$shim" | grep -q -F pie; then \ + echo "ERROR: snap-gdbserver-shim is not a static PIE"; \ + exit 1; \ + fi; \ + echo " snap-gdbserver-shim: OK (static) [$$shim]"; \ + fi + @echo "All static binary checks passed." + # XXX see the note about build ID in rule for building 'snap' # Snapd can be built with test keys. This is only used by the internal test # suite to add test assertions. Do not enable this in distribution packages. @@ -185,7 +270,7 @@ install:: | $(DESTDIR)$(snap_mount_dir) install:: install -m 755 -d $(DESTDIR)$(snap_mount_dir)/bin -# Install misc directories: +# Install misc directories: install:: install -m 755 -d $(DESTDIR)$(localstatedir)/cache/snapd install -m 755 -d $(DESTDIR)$(datadir)/polkit-1/actions @@ -225,4 +310,87 @@ check: .PHONY: clean clean: - rm -f $(go_binaries) + rm -f $(go_binaries) + +# Check that production builds contain only the expected trusted account keys. +# This verifies that test keys are not accidentally included in production builds. +# builddir: the directory containing the built binaries (e.g., _build/bin) +.PHONY: check-trusted-account-keys +check-trusted-account-keys: + @echo "Checking trusted account keys in snapd and related binaries..." + @# Check snapd binary (2 keys expected) + @if [ ! -f "$(builddir)/snapd" ]; then \ + echo "ERROR: snapd binary not found at $(builddir)/snapd" >&2; \ + exit 1; \ + fi + @if true; then \ + count=$$(strings $(builddir)/snapd | grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}"); \ + if [ "$$count" -ne 2 ]; then \ + echo "ERROR: Expected 2 public keys in snapd, found $$count" >&2; \ + exit 1; \ + fi; \ + strings $(builddir)/snapd | grep -q "^public-key-sha3-384: $(SNAPD_STORE_ROOT_KEY)$$" || \ + { echo "ERROR: snapd store root key not found" >&2; exit 1; }; \ + strings $(builddir)/snapd | grep -q "^public-key-sha3-384: $(SNAPD_STORE_GENERIC_MODELS_KEY)$$" || \ + { echo "ERROR: snapd store generic models key not found" >&2; exit 1; }; \ + echo " snapd: OK (2 keys)"; \ + fi + @# Check snap binary (2 keys expected) + @if [ ! -f "$(builddir)/snap" ]; then \ + echo "ERROR: snap binary not found at $(builddir)/snap" >&2; \ + exit 1; \ + fi + @if true; then \ + count=$$(strings $(builddir)/snap | grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}"); \ + if [ "$$count" -ne 2 ]; then \ + echo "ERROR: Expected 2 public keys in snap, found $$count" >&2; \ + exit 1; \ + fi; \ + strings $(builddir)/snap | grep -q "^public-key-sha3-384: $(SNAPD_STORE_ROOT_KEY)$$" || \ + { echo "ERROR: snap store root key not found" &>2; exit 1; }; \ + strings $(builddir)/snap | grep -q "^public-key-sha3-384: $(SNAPD_STORE_GENERIC_MODELS_KEY)$$" || \ + { echo "ERROR: snap store generic models key not found" >&2; exit 1; }; \ + echo " snap: OK (2 keys)"; \ + fi + @# Check snap-bootstrap if it exists (Ubuntu 16.04+) + @if [ -f "$(builddir)/snap-bootstrap" ]; then \ + count=$$(strings $(builddir)/snap-bootstrap | grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}"); \ + if [ "$$count" -ne 2 ]; then \ + echo "ERROR: Expected 2 public keys in snap-bootstrap, found $$count" >&2; \ + exit 1; \ + fi; \ + strings $(builddir)/snap-bootstrap | grep -q "^public-key-sha3-384: $(SNAPD_STORE_ROOT_KEY)$$" || \ + { echo "ERROR: snap-bootstrap store root key not found" >&2; exit 1; }; \ + strings $(builddir)/snap-bootstrap | grep -q "^public-key-sha3-384: $(SNAPD_STORE_GENERIC_MODELS_KEY)$$" || \ + { echo "ERROR: snap-bootstrap store generic models key not found" >&2; exit 1; }; \ + echo " snap-bootstrap: OK (2 keys)"; \ + fi + @# Check snap-preseed if it exists (Ubuntu 16.04+) + @if [ -f "$(builddir)/snap-preseed" ]; then \ + count=$$(strings $(builddir)/snap-preseed | grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}"); \ + if [ "$$count" -ne 2 ]; then \ + echo "ERROR: Expected 2 public keys in snap-preseed, found $$count" >&2; \ + exit 1; \ + fi; \ + strings $(builddir)/snap-preseed | grep -q "^public-key-sha3-384: $(SNAPD_STORE_ROOT_KEY)$$" || \ + { echo "ERROR: snap-preseed store root key not found" >&2; exit 1; }; \ + strings $(builddir)/snap-preseed | grep -q "^public-key-sha3-384: $(SNAPD_STORE_GENERIC_MODELS_KEY)$$" || \ + { echo "ERROR: snap-preseed store generic models key not found" >&2; exit 1; }; \ + echo " snap-preseed: OK (2 keys)"; \ + fi + @# Check snap-repair (3 keys expected: 2 common + 1 repair-root) + @if [ -f "$(builddir)/snap-repair" ]; then \ + count=$$(strings $(builddir)/snap-repair | grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}"); \ + if [ "$$count" -ne 3 ]; then \ + echo "ERROR: Expected 3 public keys in snap-repair, found $$count" >&2; \ + exit 1; \ + fi; \ + strings $(builddir)/snap-repair | grep -q "^public-key-sha3-384: $(SNAPD_STORE_ROOT_KEY)$$" || \ + { echo "ERROR: snap-repair store root key not found" >&2; exit 1; }; \ + strings $(builddir)/snap-repair | grep -q "^public-key-sha3-384: $(SNAPD_STORE_GENERIC_MODELS_KEY)$$" || \ + { echo "ERROR: snap-repair store generic models key not found" >&2; exit 1; }; \ + strings $(builddir)/snap-repair | grep -q "^public-key-sha3-384: $(SNAPD_REPAIR_ROOT_KEY)$$" || \ + { echo "ERROR: snap-repair repair-root key not found" >&2; exit 1; }; \ + echo " snap-repair: OK (3 keys)"; \ + fi + @echo "All trusted account key checks passed." diff --git a/packaging/ubuntu-16.04/README.md b/packaging/ubuntu-16.04/README.md index e45f75ecb48..4e3bf887eed 100644 --- a/packaging/ubuntu-16.04/README.md +++ b/packaging/ubuntu-16.04/README.md @@ -58,7 +58,7 @@ podman run \ -v "snapd-gomod-cache:/var/cache/gomod" \ -w /build \ docker.io/ubuntu:noble \ - /bin/bash -x -u + /bin/bash -x -e -u ``` ## Host Script @@ -133,7 +133,9 @@ tar -C /src -c \ # Unpack the source archive and install the packaging directory. tar -Jxf /build/snapd_"$version".no-vendor.tar.xz -C /build tar -Jxf /build/snapd_"$version".only-vendor.tar.xz -C /build -cp -a /src/packaging/ubuntu-16.04 /build/snapd-"$version"/debian +# The ubuntu-16.04 directory contains .build/ which would be copied recursively, exclude it by using a glob. +mkdir /build/snapd-"$version"/debian +cp -a /src/packaging/ubuntu-16.04/* /build/snapd-"$version"/debian # Discover and install build dependencies. DEBIAN_FRONTEND=noninteractive apt-get --yes build-dep /build/snapd-"$version" diff --git a/packaging/ubuntu-16.04/rules b/packaging/ubuntu-16.04/rules index e4e0e67849c..7545f98834f 100755 --- a/packaging/ubuntu-16.04/rules +++ b/packaging/ubuntu-16.04/rules @@ -195,8 +195,36 @@ override_dh_clean: $(MAKE) -C cmd distclean || true # XXX: hacky^2 (cd c-vendor/squashfuse && rm -f snapfuse && make distclean || true ) - -override_dh_auto_build: + rm -f snapd.defines.mk + +# Generate snapd.defines.mk for use by snapd.mk makefile targets. +# This file contains variable definitions that match Ubuntu's FHS conventions. +define snapd_defines_mk_content +# This file is generated by Ubuntu's snapd.rules +# Directory variables +prefix = /usr +bindir = /usr/bin +sbindir = /usr/sbin +libexecdir = /usr/lib/snapd +mandir = /usr/share/man +datadir = /usr/share +localstatedir = /var +sharedstatedir = /var/lib +unitdir = $(shell pkg-config --variable=systemdsystemunitdir systemd) +# Build configuration +builddir=$(CURDIR)/_build/bin +with_core_bits = 0 +with_alt_snap_mount_dir = 0 +with_apparmor = 1 +with_testkeys = $(if $(findstring testkeys,$(DEB_BUILD_OPTIONS)),1,0) +with_static_pie = 0 +with_vendor = 1 +endef + +snapd.defines.mk: + $(file >$@,$(snapd_defines_mk_content)) + +override_dh_auto_build: snapd.defines.mk # very ugly test for FIPS variant of a toolchain # see https://warthogs.atlassian.net/browse/FR-8860 ifeq (${FIPSBUILD},1) @@ -213,10 +241,11 @@ endif # ensure auto-generated version is also in the build-tree (cd _build/src/$(DH_GOPKG)/ && ../../../../../mkversion.sh) # Build golang bits - mkdir -p _build/src/$(DH_GOPKG)/cmd/snap/test-data - cp -a cmd/snap/test-data/*.gpg _build/src/$(DH_GOPKG)/cmd/snap/test-data/ cp -a bootloader/assets/data _build/src/$(DH_GOPKG)/bootloader/assets + # Copy test-data manually for old dh-golang. + mkdir -p _build/src/$(DH_GOPKG)/cmd/snap/testdata + cp -a cmd/snap/testdata/*.gpg _build/src/$(DH_GOPKG)/cmd/snap/testdata/ # this is the main go build # note that dh-golang invokes go generate as the first step, which may invoke @@ -236,9 +265,7 @@ endif (cd _build/bin && GOPATH=$$(pwd)/.. go build -mod=vendor --ldflags '-extldflags "-static"' $(GCCGOFLAGS) $(DH_GOPKG)/cmd/snap-update-ns) # ensure we generated a static build - $(shell if ldd _build/bin/snap-exec; then false "need static build"; fi) - $(shell if ldd _build/bin/snap-update-ns; then false "need static build"; fi) - $(shell if ldd _build/bin/snapctl; then false "need static build"; fi) + $(MAKE) -f packaging/snapd.mk check-static-binaries SNAPD_DEFINES_DIR=$(CURDIR) # ensure snap-seccomp is build with a static libseccomp on Ubuntu ifeq ($(shell dpkg-vendor --query Vendor),Ubuntu) @@ -273,18 +300,8 @@ endif # a tested default (production) build should have no test keys ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) # check that only the main trusted account-keys are included - for b in _build/bin/snapd _build/bin/snap-bootstrap _build/bin/snap-preseed; do \ - [ $$(strings $$b |grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}") -eq 2 ] && \ - strings $$b |grep -c "^public-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswHNiEB9Lxk$$" && \ - strings $$b |grep -c "^public-key-sha3-384: d-JcZF9nD9eBw7bwMnH61x-bklnQOhQud1Is6o_cn2wTj8EYDi9musrIT9z2MdAa$$"; \ - done; - # same for snap-repair - [ $$(strings _build/bin/snap-repair|grep -c -E "public-key-sha3-384: [a-zA-Z0-9_-]{64}") -eq 3 ] - # common with snapd - strings _build/bin/snap-repair|grep -c "^public-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswHNiEB9Lxk$$" - strings _build/bin/snap-repair|grep -c "^public-key-sha3-384: d-JcZF9nD9eBw7bwMnH61x-bklnQOhQud1Is6o_cn2wTj8EYDi9musrIT9z2MdAa$$" - # repair-root - strings _build/bin/snap-repair|grep -c "^public-key-sha3-384: nttW6NfBXI_E-00u38W-KH6eiksfQNXuI7IiumoV49_zkbhM0sYTzSnFlwZC-W4t$$" + $(MAKE) -f packaging/snapd.mk check-trusted-account-keys \ + SNAPD_DEFINES_DIR=$(CURDIR) endif ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) # run the snap-confine tests diff --git a/secboot/secboot_sb_test.go b/secboot/secboot_sb_test.go index 04d08730a0c..0ac80a80846 100644 --- a/secboot/secboot_sb_test.go +++ b/secboot/secboot_sb_test.go @@ -781,10 +781,10 @@ func (s *secbootSuite) TestUnlockVolumeUsingSealedKeyIfEncrypted(c *C) { if !tc.noKeyFile && !tc.errorReadKeyFile { if tc.oldKeyFormat { - keyPath = filepath.Join("test-data", "keyfile") + keyPath = filepath.Join("testdata", "keyfile") } else { - keyPath = filepath.Join("test-data", "keydata") - keyPath2 = filepath.Join("test-data", "keydata2") + keyPath = filepath.Join("testdata", "keydata") + keyPath2 = filepath.Join("testdata", "keydata2") } finfo, err := os.Lstat(keyPath) c.Assert(err, IsNil) @@ -1768,12 +1768,12 @@ func (s *secbootSuite) TestResealKeysWithTPM(c *C) { // To create full looking // mockSealedKeyObjects, although {},{} would // have been enough as well - mockSealedKeyFile := filepath.Join("test-data", "keyfile") + mockSealedKeyFile := filepath.Join("testdata", "keyfile") mockSealedKeyObject, err := sb_tpm2.ReadSealedKeyObjectFromFile(mockSealedKeyFile) c.Assert(err, IsNil) mockSealedKeyObjects = append(mockSealedKeyObjects, mockSealedKeyObject) } else { - mockSealedKeyFile := filepath.Join("test-data", "keydata") + mockSealedKeyFile := filepath.Join("testdata", "keydata") reader, err := sb.NewFileKeyDataReader(mockSealedKeyFile) c.Assert(err, IsNil) kd, err := sb.ReadKeyData(reader) @@ -4098,7 +4098,7 @@ func (s *secbootSuite) TestReadKeyFileKeyData(c *C) { func (s *secbootSuite) TestReadKeyFileSealedObject(c *C) { keyLoader := &secboot.DefaultKeyLoader{} const fdeHookHint = false - keyPath := filepath.Join("test-data", "keyfile") + keyPath := filepath.Join("testdata", "keyfile") readSealedKeyObjectFromFileCalls := 0 restore := secboot.MockSbReadSealedKeyObjectFromFile(func(path string) (*sb_tpm2.SealedKeyObject, error) { @@ -5395,7 +5395,7 @@ func (s *secbootSuite) TestResealKeyTPMKeyFileLegacy(c *C) { })() kd := &sb.KeyData{} - mockSealedKeyFile := filepath.Join("test-data", "keyfile") + mockSealedKeyFile := filepath.Join("testdata", "keyfile") mockSealedKeyObject, err := sb_tpm2.ReadSealedKeyObjectFromFile(mockSealedKeyFile) c.Assert(err, IsNil) diff --git a/secboot/test-data/keydata b/secboot/testdata/keydata similarity index 100% rename from secboot/test-data/keydata rename to secboot/testdata/keydata diff --git a/secboot/test-data/keydata2 b/secboot/testdata/keydata2 similarity index 100% rename from secboot/test-data/keydata2 rename to secboot/testdata/keydata2 diff --git a/secboot/test-data/keyfile b/secboot/testdata/keyfile similarity index 100% rename from secboot/test-data/keyfile rename to secboot/testdata/keyfile From 82e3264e18d4f7a5c14d90a30fb453ae542e2e0a Mon Sep 17 00:00:00 2001 From: Zeyad Yasser Date: Mon, 13 Apr 2026 07:57:45 +0200 Subject: [PATCH 09/44] tests: add /debug endpoint to fakestore to allow interrupting downloads (#16881) * tests: add /debug endpoint to fakestore to allow interrupting downloads Signed-off-by: Zeyad Gouda * fixup! tests: add /debug endpoint to fakestore to allow interrupting downloads * tests/lib/fakestore/store: add synchronization, add debug reset action Signed-off-by: Maciej Borzecki * fixup! tests: add /debug endpoint to fakestore to allow interrupting downloads * tests/lib/fakestore/store: close the connection after exceeding the limit Signed-off-by: Maciej Borzecki * fixup! tests/lib/fakestore/store: close the connection after exceeding the limit * tests/lib/fakestore/store: fix race in how the quota is counted Fix a race in consumign and trackign the left quota. Signed-off-by: Maciej Borzecki --------- Signed-off-by: Zeyad Gouda Signed-off-by: Maciej Borzecki Co-authored-by: Maciej Borzecki --- tests/lib/fakestore/store/store.go | 198 +++++++++++++++++++++++- tests/lib/fakestore/store/store_test.go | 139 +++++++++++++++++ 2 files changed, 336 insertions(+), 1 deletion(-) diff --git a/tests/lib/fakestore/store/store.go b/tests/lib/fakestore/store/store.go index 8d4cc707cdf..03d13111004 100644 --- a/tests/lib/fakestore/store/store.go +++ b/tests/lib/fakestore/store/store.go @@ -35,6 +35,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" "github.com/snapcore/snapd/asserts" @@ -67,6 +68,8 @@ type snapCachedInfo struct { // Store is our snappy software store implementation type Store struct { + lock sync.Mutex + url string blobDir string assertDir string @@ -79,6 +82,11 @@ type Store struct { channelRepository *ChannelRepository snapsCache map[string]snapCachedInfo + + // endpoint -> quota value, note this is stateful, i.e. the quota is counted + // for all requests to a given endpoint and after exceeding it, all + // subsequent requests will fail until it is reset through a request + killAfter map[string]int64 } // NewStore creates a new store server serving snaps from the given top directory and assertions from topDir/asserts. If assertFallback is true missing assertions are looked up in the main online store. @@ -105,13 +113,16 @@ func NewStore(topDir, addr string, assertFallback bool) *Store { rootDir: filepath.Join(topDir, "channels"), }, snapsCache: make(map[string]snapCachedInfo), + killAfter: make(map[string]int64), } mux.HandleFunc("/", rootEndpoint) mux.HandleFunc("/api/v1/snaps/search", store.searchEndpoint) mux.HandleFunc("/api/v1/snaps/details/", store.detailsEndpoint) mux.HandleFunc("/api/v1/snaps/metadata", store.bulkEndpoint) - mux.Handle("/download/", http.StripPrefix("/download/", http.FileServer(http.Dir(topDir)))) + + fileServer := http.StripPrefix("/download/", http.FileServer(http.Dir(topDir))) + mux.Handle("/download/", logRangeHeader(store.applyKillAfter(fileServer.ServeHTTP))) mux.HandleFunc("/api/v1/snaps/auth/nonces", store.nonceEndpoint) mux.HandleFunc("/api/v1/snaps/auth/sessions", store.sessionEndpoint) @@ -122,6 +133,8 @@ func NewStore(topDir, addr string, assertFallback bool) *Store { mux.HandleFunc("/v2/repairs/", store.repairsEndpoint) + mux.HandleFunc("/debug", store.debugEndpoint) + return store } @@ -354,6 +367,189 @@ type detailsReplyJSON struct { Base string `json:"base,omitempty"` } +type killAfterWriter struct { + http.ResponseWriter + path string + consumeQuota func(want int) int +} + +func (kaw *killAfterWriter) Write(p []byte) (int, error) { + toWrite := p + shouldKill := false + + got := kaw.consumeQuota(len(toWrite)) + if len(p) > got { + // write only up to the remaining quota + toWrite = p[:got] + shouldKill = true + } + + n, err := kaw.ResponseWriter.Write(toWrite) + + if shouldKill { + logger.Noticef("request to %s was force killed, quota exceeded", kaw.path) + kaw.hijackAndClose() + return n, fmt.Errorf("connection killed") + } + + return n, err +} + +func (kaw *killAfterWriter) hijackAndClose() { + // flush any buffered data before closing + if f, ok := kaw.ResponseWriter.(http.Flusher); ok { + f.Flush() + } + // and proceed to close + hj, ok := kaw.ResponseWriter.(http.Hijacker) + if ok { + conn, _, _ := hj.Hijack() + conn.Close() + } +} + +type debugRequestJSON struct { + Action string `json:"action"` + + KillPath string `json:"kill-path"` + KillAfter int64 `json:"kill-after"` +} + +type debugResultJSON struct { + KillAfter map[string]int64 `json:"kill-after"` +} + +func (s *Store) debugEndpoint(w http.ResponseWriter, req *http.Request) { + if req.Method == http.MethodGet { + out, err := func() ([]byte, error) { + s.lock.Lock() + defer s.lock.Unlock() + res := debugResultJSON{ + KillAfter: s.killAfter, + } + return json.Marshal(res) + }() + if err != nil { + http.Error(w, fmt.Sprintf("cannot marshal: %v", err), 500) + return + } + w.Write(out) + return + } + + if req.Method != http.MethodPost { + w.WriteHeader(405) // Method Not Allowed + return + } + + var debugReq *debugRequestJSON + decoder := json.NewDecoder(req.Body) + if err := decoder.Decode(&debugReq); err != nil { + http.Error(w, fmt.Sprintf("cannot decode request body: %v", err), 400) + return + } + + var err error + switch debugReq.Action { + case "kill-request": + err = s.debugActionKillDownload(debugReq) + case "reset": + s.debugActionReset(debugReq) + default: + err = fmt.Errorf("unexpected debug action %q", debugReq.Action) + } + if err != nil { + w.WriteHeader(400) + fmt.Fprint(w, err.Error()) + } +} + +func (s *Store) debugActionKillDownload(debugReq *debugRequestJSON) error { + if debugReq.KillPath == "" { + return fmt.Errorf("kill-path cannot be empty") + } + + if strings.HasPrefix(debugReq.KillPath, "/debug/") { + return fmt.Errorf("kill-path cannot be applied to /debug/ endpoints") + } + + s.lock.Lock() + defer s.lock.Unlock() + + if debugReq.KillAfter == 0 { + delete(s.killAfter, debugReq.KillPath) + } else { + s.killAfter[debugReq.KillPath] = debugReq.KillAfter + } + return nil +} + +func (s *Store) debugActionReset(debugReq *debugRequestJSON) { + s.lock.Lock() + defer s.lock.Unlock() + + s.killAfter = map[string]int64{} +} + +func logRangeHeader(handler http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + path := req.URL.Path + if len(req.Header["Range"]) > 0 { + logger.Noticef(`requested range for %s is %v`, path, req.Header["Range"]) + } + handler(w, req) + } +} + +func (s *Store) applyKillAfter(handler http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + path := req.URL.Path + + exists := func() bool { + s.lock.Lock() + defer s.lock.Unlock() + _, ok := s.killAfter[path] + return ok + }() + + if !exists { + handler(w, req) + return + } + + kaw := &killAfterWriter{ + ResponseWriter: w, + path: path, + consumeQuota: func(want int) int { + s.lock.Lock() + defer s.lock.Unlock() + + v, ok := s.killAfter[path] + if !ok { + // no quota set + return want + } + + left := int(v) + + var got int + if want > left { + got = left + left = 0 + } else { + got = want + left -= want + } + s.killAfter[path] = int64(left) + + return got + }, + } + handler(kaw, req) + + } +} + func (s *Store) searchEndpoint(w http.ResponseWriter, req *http.Request) { w.WriteHeader(501) fmt.Fprintf(w, "search not implemented") diff --git a/tests/lib/fakestore/store/store_test.go b/tests/lib/fakestore/store/store_test.go index 3534ba7bdda..05e082ca074 100644 --- a/tests/lib/fakestore/store/store_test.go +++ b/tests/lib/fakestore/store/store_test.go @@ -1303,3 +1303,142 @@ func (s *storeTestSuite) TestSnapActionEndpointUnknownSnapAutoRefresh(c *C) { }, }) } + +func (s *storeTestSuite) TestDebugEndpointKillAfter(c *C) { + snapFn := s.makeTestSnap(c, "name: foo\nversion: 1") + snapInfo, err := os.Stat(snapFn) + c.Assert(err, IsNil) + + downloadPath := "/download/foo_1_all.snap" + killAfter := int64(512) + c.Assert(snapInfo.Size() > killAfter, Equals, true, + Commentf("test snap must be larger than kill-after threshold")) + + // Set a rule + resp, err := s.StorePostJSON("/debug", []byte(fmt.Sprintf(`{ + "action": "kill-request", + "kill-path": "%s", + "kill-after": %d + }`, downloadPath, killAfter))) + c.Assert(err, IsNil) + resp.Body.Close() + + resp, err = s.StoreGet("/debug") + c.Assert(err, IsNil) + defer resp.Body.Close() + + c.Assert(resp.StatusCode, Equals, 200) + var body debugResultJSON + c.Assert(json.NewDecoder(resp.Body).Decode(&body), IsNil) + c.Check(body.KillAfter, DeepEquals, map[string]int64{ + downloadPath: killAfter, + }) + + // Download is interrupted, we get fewer bytes than the full snap + resp, err = s.StoreGet(downloadPath) + c.Assert(err, IsNil) + defer resp.Body.Close() + + got, _ := io.ReadAll(resp.Body) + // Connection forcefully closed mid-transfer, exactly killAfter bytes received + c.Check(int64(len(got)), Equals, killAfter) + + // Retry the request, which should be killed after receiving 0 bytes because + // the killAfter effect is stateful. + resp, err = s.StoreGet(downloadPath) + c.Assert(err, IsNil) + defer resp.Body.Close() + + got, _ = io.ReadAll(resp.Body) + // Connection forcefully closed mid-transfer, exactly killAfter bytes received + c.Check(int64(len(got)), Equals, int64(0)) + + // Clear it by setting kill-after to 0 + resp, err = s.StorePostJSON("/debug", []byte(fmt.Sprintf(`{ + "action": "kill-request", + "kill-path": "%s", + "kill-after": 0 + }`, downloadPath))) + c.Assert(err, IsNil) + resp.Body.Close() + + resp, err = s.StoreGet("/debug") + c.Assert(err, IsNil) + defer resp.Body.Close() + + var bodyAfterClear debugResultJSON + c.Assert(json.NewDecoder(resp.Body).Decode(&bodyAfterClear), IsNil) + c.Check(bodyAfterClear.KillAfter, HasLen, 0) + + // Download succeeds after clearing kill-after + resp, err = s.StoreGet(downloadPath) + c.Assert(err, IsNil) + defer resp.Body.Close() + + c.Assert(resp.StatusCode, Equals, 200) + got, err = io.ReadAll(resp.Body) + c.Assert(err, IsNil) + c.Check(int64(len(got)), Equals, snapInfo.Size()) +} + +func (s *storeTestSuite) TestDebugEndpointUnknownAction(c *C) { + resp, err := s.StorePostJSON("/debug", []byte(`{ + "action": "unknown-action" + }`)) + c.Assert(err, IsNil) + defer resp.Body.Close() + + c.Assert(resp.StatusCode, Equals, 400) + body, err := io.ReadAll(resp.Body) + c.Assert(err, IsNil) + c.Check(string(body), Equals, `unexpected debug action "unknown-action"`) +} + +func (s *storeTestSuite) TestDebugEndpointMethodNotAllowed(c *C) { + req, err := http.NewRequest(http.MethodPut, s.store.URL()+"/debug", nil) + c.Assert(err, IsNil) + resp, err := s.client.Do(req) + c.Assert(err, IsNil) + defer resp.Body.Close() + + c.Assert(resp.StatusCode, Equals, 405) +} + +func (s *storeTestSuite) TestDebugActionReset(c *C) { + // Set a rule for endpoint connection interrupt + resp, err := s.StorePostJSON("/debug", []byte(`{ + "action": "kill-request", + "kill-path": "/foo/bar", + "kill-after": 123 + }`)) + c.Assert(err, IsNil) + resp.Body.Close() + c.Assert(resp.StatusCode, Equals, 200) + + resp, err = s.StoreGet("/debug") + c.Assert(err, IsNil) + defer resp.Body.Close() + + var buf bytes.Buffer + c.Assert(resp.StatusCode, Equals, 200) + _, err = io.Copy(&buf, resp.Body) + c.Assert(err, IsNil) + c.Check(buf.String(), Equals, `{"kill-after":{"/foo/bar":123}}`) + + // Clear it by setting kill-after to 0 + resp, err = s.StorePostJSON("/debug", []byte(`{ + "action": "reset" + }`)) + c.Assert(err, IsNil) + resp.Body.Close() + c.Assert(resp.StatusCode, Equals, 200) + + resp, err = s.StoreGet("/debug") + c.Assert(err, IsNil) + defer resp.Body.Close() + + buf.Reset() + _, err = io.Copy(&buf, resp.Body) + c.Assert(err, IsNil) + c.Check(buf.String(), Equals, `{"kill-after":{}}`) +} From 38fd84db7b0e5acdf2632d3517d200be286bcd76 Mon Sep 17 00:00:00 2001 From: Katie May Date: Mon, 13 Apr 2026 08:58:53 +0200 Subject: [PATCH 10/44] github: fix spread-tests.yaml to allow for a large number of artifacts (#16896) * github: fix spread-tests.yaml to allow for a large number of artifacts * github: use total count instead of artifact number * github: use total count instead of artifact number also in spread-results-reporter and rerun --- .github/workflows/rerun.yaml | 2 +- .../workflows/spread-results-reporter.yaml | 4 ++-- .github/workflows/spread-tests.yaml | 22 ++++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rerun.yaml b/.github/workflows/rerun.yaml index 8d8c7a8fdb3..4fdbe72388d 100644 --- a/.github/workflows/rerun.yaml +++ b/.github/workflows/rerun.yaml @@ -55,7 +55,7 @@ jobs: }); allArtifacts = allArtifacts.concat(response.data.artifacts); page++; - } while (response.data.artifacts.length === per_page); + } while (allArtifacts.length < response.data.total_count); let matchArtifact = allArtifacts.filter((artifact) => { return artifact.name == "pr_number" })[0]; diff --git a/.github/workflows/spread-results-reporter.yaml b/.github/workflows/spread-results-reporter.yaml index 6877fb63f8b..7f343f0922a 100644 --- a/.github/workflows/spread-results-reporter.yaml +++ b/.github/workflows/spread-results-reporter.yaml @@ -36,7 +36,7 @@ jobs: }); allArtifacts = allArtifacts.concat(response.data.artifacts); page++; - } while (response.data.artifacts.length === per_page); + } while (allArtifacts.length < response.data.total_count); let matchArtifact = allArtifacts.filter((artifact) => { return artifact.name == "pr_number" })[0]; @@ -70,7 +70,7 @@ jobs: }); allArtifacts = allArtifacts.concat(response.data.artifacts); page++; - } while (response.data.artifacts.length === per_page); + } while (allArtifacts.length < response.data.total_count); let matchingArtifacts = allArtifacts.filter((artifact) => { return artifact.name.startsWith(`spread-results-${context.payload.workflow_run.id}-${context.payload.workflow_run.run_attempt}`); diff --git a/.github/workflows/spread-tests.yaml b/.github/workflows/spread-tests.yaml index 5354fc53421..143e6613fd2 100644 --- a/.github/workflows/spread-tests.yaml +++ b/.github/workflows/spread-tests.yaml @@ -381,11 +381,21 @@ jobs: let child_process = require('child_process'); let systems = process.env.SYSTEMS.split(' '); - let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - }); + let page = 1; + let per_page = 100; + let allArtifacts = []; + let response; + do { + response = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.runId, + per_page: per_page, + page: page + }); + allArtifacts = allArtifacts.concat(response.data.artifacts); + page++; + } while (allArtifacts.length < response.data.total_count); for (let system of systems) { let artifactName = `package-${system}`; @@ -395,7 +405,7 @@ jobs: const arch = match[2] || ''; artifactName = `package-ubuntu-${version}.04${arch}-64`; } - let artifact = allArtifacts.data.artifacts.find(a => a.name === artifactName); + let artifact = allArtifacts.find(a => a.name === artifactName); if (!artifact) { console.log(`Artifact not found for system: ${system}`); continue; From 1c1cc8e02ef5efdfbbd6dc17559c22ad78be6bb9 Mon Sep 17 00:00:00 2001 From: Katie May Date: Mon, 13 Apr 2026 11:17:34 +0200 Subject: [PATCH 11/44] tests: give nested/manual/seeding-failure more attempts when waiting for device initialization (#16903) --- tests/nested/manual/seeding-failure/task.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nested/manual/seeding-failure/task.yaml b/tests/nested/manual/seeding-failure/task.yaml index f3a94770761..e9a78e63313 100644 --- a/tests/nested/manual/seeding-failure/task.yaml +++ b/tests/nested/manual/seeding-failure/task.yaml @@ -104,7 +104,7 @@ execute: | # this waits for the next attempt at seeding to work tests.nested setup-vm - remote.wait-for device-initialized + remote.wait-for device-initialized --attempts 120 remote.exec "snap list failing-service" From 19520909653bdbdd34139d78638adcc7bf13d052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alfonso=20S=C3=A1nchez-Beato?= Date: Thu, 9 Apr 2026 17:08:54 -0400 Subject: [PATCH 12/44] tests: use cloud-init track for core26 nested tests when cloud-init is needed in the tests. --- tests/lib/nested.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/lib/nested.sh b/tests/lib/nested.sh index 931473c6686..2598acffe73 100755 --- a/tests/lib/nested.sh +++ b/tests/lib/nested.sh @@ -490,7 +490,7 @@ nested_cleanup_env() { rm -rf "$(nested_get_extra_snaps_path)" } -nested_get_core_channel() { +nested_get_image_channel() { if nested_is_core_26_system; then # TODO: Remove when it becomes available in the other channels echo "edge" @@ -515,7 +515,7 @@ nested_get_gadget_channel() { nested_get_image_name_base() { local TYPE="$1" local SOURCE - SOURCE="$(nested_get_core_channel)" + SOURCE="$(nested_get_image_channel)" local NAME="${NESTED_IMAGE_ID:-generic}" local VERSION @@ -1013,14 +1013,22 @@ nested_create_core_vm() { export SNAPPY_FORCE_SAS_URL UBUNTU_IMAGE_SNAP_CMD=/usr/bin/snap export UBUNTU_IMAGE_SNAP_CMD - local core_channel - core_channel="$(nested_get_core_channel)" - if [ -n "${core_channel}" ]; then - UBUNTU_IMAGE_CHANNEL_ARG="--channel ${core_channel}" - else + local image_channel + image_channel="$(nested_get_image_channel)" + if [ -n "${image_channel}" ]; then + UBUNTU_IMAGE_CHANNEL_ARG="--channel ${image_channel}" + else UBUNTU_IMAGE_CHANNEL_ARG="" fi + # Starting on core26 we have different tracks depending on whether + # cloud-init is included in the snap or not. This won't have any + # effect if using an unasserted snap. + local BASE_CHANNEL="" + if nested_is_core_26_system && [ "$NESTED_USE_CLOUD_INIT" = "true" ]; then + BASE_CHANNEL="--snap core26=$image_channel/cloud-init" + fi + declare -a UBUNTU_IMAGE_PRESEED_ARGS if [ -n "$NESTED_UBUNTU_IMAGE_PRESEED_KEY" ]; then # shellcheck disable=SC2191 @@ -1031,6 +1039,7 @@ nested_create_core_vm() { SNAPD_DEBUG=1 "$UBUNTU_IMAGE" snap --image-size 10G \ "$NESTED_MODEL" \ $UBUNTU_IMAGE_CHANNEL_ARG \ + $BASE_CHANNEL \ "${UBUNTU_IMAGE_PRESEED_ARGS[@]:-}" \ --output-dir "$NESTED_IMAGES_DIR" \ --sector-size "${NESTED_DISK_LOGICAL_BLOCK_SIZE}" \ From e9df9b56fec85b73258345febabd7fa22b48da96 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 14:05:52 +0000 Subject: [PATCH 13/44] github: bump actions/github-script in the actions-deps group (#16911) Bumps the actions-deps group with 1 update: [actions/github-script](https://github.com/actions/github-script). Updates `actions/github-script` from 8 to 9 - [Release notes](https://github.com/actions/github-script/releases) - [Commits](https://github.com/actions/github-script/compare/v8...v9) --- updated-dependencies: - dependency-name: actions/github-script dependency-version: '9' dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/rerun.yaml | 2 +- .github/workflows/spread-results-reporter.yaml | 4 ++-- .github/workflows/spread-tests.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rerun.yaml b/.github/workflows/rerun.yaml index 4fdbe72388d..d5c7ab18af6 100644 --- a/.github/workflows/rerun.yaml +++ b/.github/workflows/rerun.yaml @@ -38,7 +38,7 @@ jobs: uses: actions/checkout@v6 - name: Get PR number - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | let page = 1; diff --git a/.github/workflows/spread-results-reporter.yaml b/.github/workflows/spread-results-reporter.yaml index 7f343f0922a..4b338d4d0d2 100644 --- a/.github/workflows/spread-results-reporter.yaml +++ b/.github/workflows/spread-results-reporter.yaml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@v6 - name: Get PR number - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | let page = 1; @@ -53,7 +53,7 @@ jobs: run: unzip pr_number.zip - name: Get generated data - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | let page = 1; diff --git a/.github/workflows/spread-tests.yaml b/.github/workflows/spread-tests.yaml index 143e6613fd2..8df996b0b0b 100644 --- a/.github/workflows/spread-tests.yaml +++ b/.github/workflows/spread-tests.yaml @@ -372,7 +372,7 @@ jobs: if: ${{ env.SKIP_SPREAD_LABEL != 'true' && env.RUN_TESTS == 'true' }} env: SYSTEMS: ${{ inputs.systems }} - uses: actions/github-script@v8 + uses: actions/github-script@v9 with: script: | let fs = require('fs'); From fee5d7c9a24eb900c90a75e3dfe30d83510d7ebf Mon Sep 17 00:00:00 2001 From: Katie May Date: Mon, 13 Apr 2026 16:30:30 +0200 Subject: [PATCH 14/44] github: fail go channels job if no go channels were found (#16897) * github: fail go channels job if no go channels were found * github: write no go channels resolved error on stderr --- .github/workflows/ci-test.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml index 200402a6f2c..47a37cdaf66 100644 --- a/.github/workflows/ci-test.yaml +++ b/.github/workflows/ci-test.yaml @@ -26,6 +26,14 @@ jobs: include-snapd-build-fips-go-channel: true include-latest-go-channel: false + - name: Check go channels + run: | + echo "Resolved Go channels: ${{ steps.resolve-go-channels.outputs.go-channels }}" + if [ -z "${{ steps.resolve-go-channels.outputs.go-channels }}" ]; then + echo "Error: No Go channels resolved" >&2 + exit 1 + fi + snap-builds: uses: ./.github/workflows/snap-builds.yaml with: From 2f8c0d99a2322450db88320a0884f67a3ccc3323 Mon Sep 17 00:00:00 2001 From: Valentin David Date: Thu, 9 Apr 2026 15:27:14 +0200 Subject: [PATCH 15/44] tests/nested/manual/minimal-smoke: add more memory with secure boot Kernel stub fails to load initrd with error EFI_OUT_OF_RESOURCES. --- tests/nested/manual/minimal-smoke/task.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/nested/manual/minimal-smoke/task.yaml b/tests/nested/manual/minimal-smoke/task.yaml index 5d47b63bf4a..00f21221e0b 100644 --- a/tests/nested/manual/minimal-smoke/task.yaml +++ b/tests/nested/manual/minimal-smoke/task.yaml @@ -29,6 +29,9 @@ execute: | if [ "$VERSION" -ge 20 ]; then MINIMAL_MEM=512 fi + if [ "${NESTED_ENABLE_SECURE_BOOT}" = "true" ] && [ "$VERSION" -ge 26 ]; then + MINIMAL_MEM=768 + fi tests.nested create-vm core --param-mem "$MINIMAL_MEM" From 0ac9a53c5e672bd4a901d68e39ec715ad72f99e4 Mon Sep 17 00:00:00 2001 From: Ernest Lotter Date: Tue, 14 Apr 2026 08:55:28 +0200 Subject: [PATCH 16/44] secboot: update to rev 3f8b98c for TPM/FDE bug fixes (#16886) * secboot: update to rev 7557d93 for TPM/FDE bug fixes - Access to the HFSTS registers via the HECI is not possible on systems that use Intel's High Assurance Platform mode. The startup ACM mirrors some BootGuard policy settings to a MSR so this can be checked as a workaround. - Do not fail preinstall check due to lack of TPM_CAP_AUTH_POLICIES. * secboot: update to rev 3f8b98c for TPM/FDE bug fixes - Access to the HFSTS registers via the HECI is not possible on systems that use Intel's High Assurance Platform mode. The startup ACM mirrors some BootGuard policy settings to a MSR so this can be checked as a workaround. This is a partial fix. - Permit pre-OS application launches from SPI flash in PCR4. Fixes: - https://github.com/canonical/secboot/issues/509 - FR-12927 - Relax recovery key parsing. Rather than permitting each group of 5 digits be separated by an optional '-', just permit an arbitrary number of '-' or whitespace characters instead. Fixes: - FR-11924 - Do not fail preinstall check due to lack of TPM_CAP_AUTH_POLICIES. Fixes: - https://github.com/canonical/secboot/issues/408 - The PCR4 and PCR7 checks were relying on the BootCurrent EFI variable to identify the EV_EFI_BOOT_SERVICES_APPLICATION. Instead, assume that the first OS-present EV_EFI_BOOT_SERVICES_APPLICATION event that isn't Absolute is the initial OS loader. Fixes: - https://github.com/canonical/secboot/issues/517 - https://github.com/canonical/secboot/issues/519 * tests/nested: adapt error message to secboot change * tests/nested/manual: review improvements --- go.mod | 2 +- go.sum | 4 +-- .../hybrid-tpm-fde-preinstall-check/task.yaml | 28 ++++++++----------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index c78fb3aefff..ce7ed96d56b 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( // if below two libseccomp-golang lines are updated, one must also update packaging/ubuntu-14.04/rules github.com/mvo5/libseccomp-golang v0.9.1-0.20180308152521-f4de83b52afb // old trusty builds only github.com/seccomp/libseccomp-golang v0.9.2-0.20220502024300-f57e1d55ea18 - github.com/snapcore/secboot v0.0.0-20260320145120-26dce572077a + github.com/snapcore/secboot v0.0.0-20260410084611-3f8b98c2db70 golang.org/x/crypto v0.23.0 golang.org/x/net v0.21.0 // indirect golang.org/x/sync v0.8.0 diff --git a/go.sum b/go.sum index b3c99316b51..2a132ac8025 100644 --- a/go.sum +++ b/go.sum @@ -61,8 +61,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502024300-f57e1d55ea18 h1:A15 github.com/seccomp/libseccomp-golang v0.9.2-0.20220502024300-f57e1d55ea18/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/snapcore/maze.io-x-crypto v0.0.0-20190131090603-9b94c9afe066 h1:InG0EmriMOiI4YgtQNOo+6fNxzLCYioo3Q3BCVLdMCE= github.com/snapcore/maze.io-x-crypto v0.0.0-20190131090603-9b94c9afe066/go.mod h1:VuAdaITF1MrGzxPU+8GxagM1HW2vg7QhEFEeGHbmEMU= -github.com/snapcore/secboot v0.0.0-20260320145120-26dce572077a h1:eKljZN+SzPKM1ua7KG5E2vQdbw9JQn17tbbukHj6hvw= -github.com/snapcore/secboot v0.0.0-20260320145120-26dce572077a/go.mod h1:+qs2Juv0XZeTmQHJgFTtAd9520h7QUWRDyvSwbJ3xEU= +github.com/snapcore/secboot v0.0.0-20260410084611-3f8b98c2db70 h1:EewX3F1DSpJz8wPZxFxC3MdOnFiFVAazbOIv1OQNDME= +github.com/snapcore/secboot v0.0.0-20260410084611-3f8b98c2db70/go.mod h1:+qs2Juv0XZeTmQHJgFTtAd9520h7QUWRDyvSwbJ3xEU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= diff --git a/tests/nested/manual/hybrid-tpm-fde-preinstall-check/task.yaml b/tests/nested/manual/hybrid-tpm-fde-preinstall-check/task.yaml index 64081bf3132..0dedcc1df5f 100644 --- a/tests/nested/manual/hybrid-tpm-fde-preinstall-check/task.yaml +++ b/tests/nested/manual/hybrid-tpm-fde-preinstall-check/task.yaml @@ -132,22 +132,18 @@ execute: | api_get_v2_systems_classic | gojq '.result["storage-encryption"].support' | MATCH "unavailable" api_get_v2_systems_classic | gojq '.result["storage-encryption"]."availability-check-errors"' > availability-check-errors - cat < availability-check-errors.reference - [ - { - "actions": [ - "contact-oem" - ], - "args": { - "pcr": 7 - }, - "kind": "tpm-pcr-unusable", - "message": "error with secure boot policy (PCR7) measurements: OS initial boot loader was not verified by any X.509 certificate measured by any EV_EFI_VARIABLE_AUTHORITY event" - } - ] + cat <<'EOF' > availability-check-errors.reference.jq + length == 1 and + .[0].actions == ["contact-oem"] and + .[0].args == {"pcr": 7} and + .[0].kind == "tpm-pcr-unusable" and + ( + .[0].message + | test("^error with secure boot policy \\(PCR7\\) measurements: OS-present EV_EFI_BOOT_SERVICES_APPLICATION event for .*shim.*\\.efi is not associated with the initial boot loader image$") + ) EOF - diff availability-check-errors availability-check-errors.reference + gojq -e -f availability-check-errors.reference.jq availability-check-errors >/dev/null # Check that the basic availability check that do not consider shim, grub and kernel still works remote.exec "sudo cp /etc/os-release /etc/os-release.orig" @@ -219,10 +215,10 @@ execute: | api_get_v2_systems_classic | gojq '.result["storage-encryption"].support' | MATCH "unavailable" api_get_v2_systems_classic | gojq '.result["storage-encryption"]."availability-check-errors"' > availability-check-errors - diff availability-check-errors availability-check-errors.reference + gojq -e -f availability-check-errors.reference.jq availability-check-errors >/dev/null # - Then request an action echo '{"action": "fix-encryption-support", "fix-action": ""}' | api_post_v2_systems_classic | gojq '.result["storage-encryption"]."availability-check-errors"' > availability-check-errors - diff availability-check-errors availability-check-errors.reference + gojq -e -f availability-check-errors.reference.jq availability-check-errors >/dev/null From 4f657056a2389195c2fa7228a1e0b494dd91c734 Mon Sep 17 00:00:00 2001 From: Maciej Borzecki Date: Tue, 14 Apr 2026 10:02:16 +0200 Subject: [PATCH 17/44] release-tools/is-lp-fips-build: attempt to workaround LP FIPS build detection problem (#16913) * release-tools/is-lp-fips-build: attempt to workaround LP FIPS build detection problem Turns out that when the LP snap job publishes the snap to a store using the name provided in the snap configuration, the git repository is cloned to a directory using that same name. This with the snapd-fips job using 'snapd' store name, our detection of a FIPS build job on snapd no longer works. Attempt a workaround, where we check whether the OpenSSL FIPS provider module package - openssl-fips-module-3 is available. It will only be present when the FIPS PPA is added. Related: SNAPDENG-21236 Signed-off-by: Maciej Borzecki * fixup! release-tools/is-lp-fips-build: attempt to workaround LP FIPS build detection problem --------- Signed-off-by: Maciej Borzecki --- release-tools/is-lp-fips-build.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/release-tools/is-lp-fips-build.sh b/release-tools/is-lp-fips-build.sh index c53c0089452..bb2b2f6ea4a 100755 --- a/release-tools/is-lp-fips-build.sh +++ b/release-tools/is-lp-fips-build.sh @@ -7,20 +7,24 @@ set -e # recipe https://launchpad.net/~fips-cc-stig/fips-cc-stig/+snap/snapd-fips # # - git origin https://git.launchpad.net/~snappy-dev/snapd/+git/snapd -# - build path: /build/snapd-fips +# - openssl-fips-module-3 package is available (meaning FIPS PPA is present) +# +# TODO: we really need knobs on LP/snapcraft side to make this explicit if ! git remote get-url origin | grep "git.launchpad.net" >&2 ; then echo "false" exit 0 fi -# when building from https://launchpad.net/~fips-cc-stig/fips-cc-stig/+snap/snapd-fips -# recipe, the code is cloned at /build/snapd-fips/ -if ! echo "$PWD" | grep '^/build/snapd-fips/' >&2 ; then - echo "false" +if apt show openssl-fips-module-3 > /dev/null; then + echo ":: openssl-fips-module-3 package is available" >&2 + # when building with Pro FIPS Updates PPA + # https://launchpad.net/~ubuntu-advantage/+archive/ubuntu/pro-fips-updates, the + # openssl-fips-module-3 package will be available + echo "true" exit 0 fi # TODO check if a FIPS PPA is enabled? -echo "true" +echo "false" From 69bcf4f68fb0aff5be9905ccf28c57585a016e0c Mon Sep 17 00:00:00 2001 From: Philip Meulengracht Date: Tue, 14 Apr 2026 13:53:14 +0200 Subject: [PATCH 18/44] overlord: simplify cert-db updates on model-base refresh/installs (#16837) * overlord: refresh certificate-db on boot-base refreshes * overlord: simplify when generation happens, just do it on boot-base refreshes and installs, not on track-switch or cohort changes. This allows the snap.go orchestrator to just always inject it, and we dont need special case handling for remodel tests * tests/core: remove the filter * overlord: review feedback, handle remodelling case in addLinkNewBaseOrKernelTasks where existing installed snap may be the case, fixup tests, simplify a couple of things * tests/core/base-refresh-cert-db: add core26 --- overlord/certstate/certmgr.go | 20 +- overlord/devicestate/devicestate.go | 46 +--- .../devicestate/devicestate_remodel_test.go | 221 +++++------------- overlord/managers_test.go | 51 ++-- overlord/snapstate/export_test.go | 25 +- overlord/snapstate/snap.go | 23 ++ overlord/snapstate/snapstate.go | 17 +- overlord/snapstate/snapstate_test.go | 94 +++++++- overlord/snapstate/snapstate_update_test.go | 97 +++++++- tests/core/base-refresh-cert-db/task.yaml | 75 ++++++ .../core/model-base-refresh-cert-db/task.yaml | 58 +++++ 11 files changed, 467 insertions(+), 260 deletions(-) create mode 100644 tests/core/base-refresh-cert-db/task.yaml create mode 100644 tests/core/model-base-refresh-cert-db/task.yaml diff --git a/overlord/certstate/certmgr.go b/overlord/certstate/certmgr.go index 55d0dcd2f73..22a1d21fa0f 100644 --- a/overlord/certstate/certmgr.go +++ b/overlord/certstate/certmgr.go @@ -74,7 +74,7 @@ func (m *CertManager) Ensure() error { } // If the ssl certs directory is missing, nothing to do. - if exists, isDir, err := osutil.DirExists(dirs.SystemCertsDir); !exists || !isDir || err != nil { + if !hasSystemCertsDir() { logger.Debugf("/etc/ssl/certs is not available on this system, skipping ca-certificates generation") return nil } @@ -85,7 +85,16 @@ func (m *CertManager) Ensure() error { return GenerateCertificateDatabase() } -func (m *CertManager) doUpdateCertificateDatabase(_ *state.Task, _ *tomb.Tomb) error { +func (m *CertManager) doUpdateCertificateDatabase(t *state.Task, _ *tomb.Tomb) error { + st := t.State() + st.Lock() + defer st.Unlock() + + if !hasSystemCertsDir() { + t.Logf("/etc/ssl/certs is not available on this system, skipping certificate database update") + return nil + } + return GenerateCertificateDatabase() } @@ -101,3 +110,10 @@ func (m *CertManager) undoUpdateCertificateDatabase(_ *state.Task, _ *tomb.Tomb) } return nil } + +func hasSystemCertsDir() bool { + if exists, isDir, err := osutil.DirExists(dirs.SystemCertsDir); !exists || !isDir || err != nil { + return false + } + return true +} diff --git a/overlord/devicestate/devicestate.go b/overlord/devicestate/devicestate.go index f7a0ed8e3f6..cddaecef83a 100644 --- a/overlord/devicestate/devicestate.go +++ b/overlord/devicestate/devicestate.go @@ -986,7 +986,7 @@ func remodelEssentialSnapTasks( if ms.newModelSnap != nil && ms.newModelSnap.SnapType == "gadget" { return snapstate.SwitchToNewGadget(st, name, fromChange) } - return snapstate.LinkNewBaseOrKernel(st, name, fromChange) + return snapstate.LinkNewBaseOrKernel(st, name, fromChange, rm.deviceCtx) } // as a bit of a special case, we support adding the needed tasks that make @@ -1000,7 +1000,7 @@ func remodelEssentialSnapTasks( if ms.newModelSnap != nil && ms.newModelSnap.SnapType == "gadget" { return snapstate.AddGadgetAssetsTasks(st, tss[0]) } - return snapstate.AddLinkNewBaseOrKernel(st, tss[0]) + return snapstate.AddLinkNewBaseOrKernel(st, tss[0], rm.deviceCtx) } switch action { @@ -1113,38 +1113,6 @@ func sortNonEssentialRemodelTaskSetsBasesFirst(snaps []*asserts.ModelSnap) []*as return sorted } -func shouldRegenerateCertificateDatabase(current, new *asserts.Model) bool { - // If the boot-base is being changed, then we should regenerate the cert db - // as that carry system certificates. When the certificates are changed, - // the managed snapd database must be regenerated - - // When upgrading the base, and when the track is changed - if current.Base() != "" && new.Base() != "" { - // Non core16 models, if they are not matching, then we should regenerate the database - if current.Base() != new.Base() { - return true - } - } - - baseTrack := func(ms *asserts.ModelSnap) string { - if ms == nil { - return "" - } - if ms.PinnedTrack != "" { - return ms.PinnedTrack - } - ch, err := channel.ParseVerbatim(ms.DefaultChannel, "-") - if err != nil { - return "" - } - return ch.Track - } - if baseTrack(current.BaseSnap()) != baseTrack(new.BaseSnap()) { - return true - } - return false -} - func remodelTasks(ctx context.Context, st *state.State, current, new *asserts.Model, deviceCtx snapstate.DeviceContext, fromChange string, opts RemodelOptions) ([]*state.TaskSet, error) { @@ -1384,16 +1352,6 @@ func remodelTasks(ctx context.Context, st *state.State, current, new *asserts.Mo recoverySetupTaskID = createRecoveryTasks.Tasks()[0].ID() } - // When the base of the model is changing, refresh the certificate database managed by - // snapd as the new base will likely carry newer certificates. - if shouldRegenerateCertificateDatabase(current, new) { - updateCertDB := st.NewTask("update-cert-db", i18n.G("Update certificate database")) - for _, tsPrev := range tss { - updateCertDB.WaitAll(tsPrev) - } - tss = append(tss, state.NewTaskSet(updateCertDB)) - } - // Set the new model assertion - this *must* be the last thing done // by the change. setModel := st.NewTask("set-model", i18n.G("Set new model assertion")) diff --git a/overlord/devicestate/devicestate_remodel_test.go b/overlord/devicestate/devicestate_remodel_test.go index 99bbe17bd38..57c385771b8 100644 --- a/overlord/devicestate/devicestate_remodel_test.go +++ b/overlord/devicestate/devicestate_remodel_test.go @@ -2314,78 +2314,12 @@ func (s *deviceMgrSuite) TestRemodelSwitchBase(c *C) { tss, err := devicestate.RemodelTasks(context.Background(), s.state, current, new, testDeviceCtx, "99", devicestate.RemodelOptions{}) c.Assert(err, IsNil) - // 1 switch to a new base, 1 switch to new gadget, cert db update, plus set-model - c.Assert(tss, HasLen, 4) + // 1 switch to a new base, 1 switch to new gadget, plus set-model + c.Assert(tss, HasLen, 3) // API was hit c.Assert(snapstateInstallWithDeviceContextCalled, Equals, 2) } -func (s *deviceMgrRemodelSuite) TestRemodelSwitchBaseRunsUpdateCertDB(c *C) { - s.state.Lock() - s.state.Set("seeded", true) - s.state.Set("refresh-privacy-key", "some-privacy-key") - - s.mockTasksNopHandler("fake-download", "validate-snap", "fake-install", "set-model") - - var updateCertDBCalls int - s.o.TaskRunner().AddHandler("update-cert-db", func(task *state.Task, _ *tomb.Tomb) error { - updateCertDBCalls++ - return nil - }, nil) - - restore := devicestate.MockSnapstateInstallOne(func(ctx context.Context, st *state.State, goal snapstate.InstallGoal, opts snapstate.Options) (*snap.Info, *state.TaskSet, error) { - g := goal.(*storeInstallGoalRecorder) - name := g.snaps[0].InstanceName - - tDownload := s.state.NewTask("fake-download", fmt.Sprintf("Download %s", name)) - tValidate := s.state.NewTask("validate-snap", fmt.Sprintf("Validate %s", name)) - tValidate.WaitFor(tDownload) - tInstall := s.state.NewTask("fake-install", fmt.Sprintf("Install %s", name)) - tInstall.WaitFor(tValidate) - ts := state.NewTaskSet(tDownload, tValidate, tInstall) - ts.MarkEdge(tValidate, snapstate.LastBeforeLocalModificationsEdge) - return nil, ts, nil - }) - defer restore() - - current := s.brands.Model("canonical", "pc-model", map[string]any{ - "architecture": "amd64", - "kernel": "pc-kernel", - "gadget": "pc", - "base": "core18", - }) - err := assertstate.Add(s.state, current) - c.Assert(err, IsNil) - - s.makeSerialAssertionInState(c, "canonical", "pc-model", "serial") - devicestatetest.SetDevice(s.state, &auth.DeviceState{ - Brand: "canonical", - Model: "pc-model", - Serial: "serial", - }) - - new := s.brands.Model("canonical", "pc-model", map[string]any{ - "architecture": "amd64", - "kernel": "pc-kernel", - "gadget": "pc-20", - "base": "core20", - "revision": "1", - }) - - chg, err := devicestate.Remodel(s.state, new, devicestate.RemodelOptions{}) - c.Assert(err, IsNil) - s.state.Unlock() - - s.settle(c) - - s.state.Lock() - defer s.state.Unlock() - - c.Check(chg.IsReady(), Equals, true) - c.Check(chg.Err(), IsNil) - c.Check(updateCertDBCalls, Equals, 1) -} - func (s *deviceMgrRemodelSuite) TestRemodelUC20RequiredSnapsAndRecoverySystem(c *C) { s.state.Lock() defer s.state.Unlock() @@ -3167,8 +3101,8 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnaps(c *C) { c.Logf("%s: %s", t.Kind(), t.Summary()) } - // 3 snaps (2 tasks for each) + assets update + gadget (3 tasks) + recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 3*2+1+3+2+1+1) + // 3 snaps (2 tasks for each) + assets update + update cert db + gadget (3 tasks) + recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 3*2+1+1+3+2+1) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -3186,14 +3120,14 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnaps(c *C) { tLinkKernel := tl[2] tPrepareBase := tl[3] tLinkBase := tl[4] - tPrepareGadget := tl[5] - tUpdateAssets := tl[6] - tUpdateCmdline := tl[7] - tValidateApp := tl[8] - tInstallApp := tl[9] - tCreateRecovery := tl[10] - tFinalizeRecovery := tl[11] - tUpdateCertDB := tl[12] + tUpdateCertDB := tl[5] + tPrepareGadget := tl[6] + tUpdateAssets := tl[7] + tUpdateCmdline := tl[8] + tValidateApp := tl[9] + tInstallApp := tl[10] + tCreateRecovery := tl[11] + tFinalizeRecovery := tl[12] tSetModel := tl[13] // check the tasks @@ -3227,8 +3161,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnaps(c *C) { c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // check the ordering, prepare/link are part of download edge and come first @@ -3254,7 +3186,7 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnaps(c *C) { }) c.Assert(tUpdateAssets.WaitTasks(), DeepEquals, []*state.Task{ tPrepareGadget, - tLinkBase, + tUpdateCertDB, }) c.Assert(tUpdateCmdline.WaitTasks(), DeepEquals, []*state.Task{ tUpdateAssets, @@ -3270,21 +3202,14 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnaps(c *C) { // last snap of the download chain (see above) tValidateApp, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tPrepareKernel, tUpdateAssetsKernel, - tLinkKernel, tPrepareBase, tLinkBase, - tPrepareGadget, tUpdateAssets, tUpdateCmdline, - tValidateApp, tInstallApp, - tCreateRecovery, tFinalizeRecovery, - }) // setModel waits for everything in the change c.Assert(tSetModel.WaitTasks(), DeepEquals, []*state.Task{ tPrepareKernel, tUpdateAssetsKernel, tLinkKernel, tPrepareBase, tLinkBase, + tUpdateCertDB, tPrepareGadget, tUpdateAssets, tUpdateCmdline, tValidateApp, tInstallApp, tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) snapsups := []any{tPrepareKernel.ID(), tPrepareBase.ID(), tPrepareGadget.ID(), tValidateApp.ID()} @@ -3441,8 +3366,8 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnapsChannelSwitch c.Logf("%s: %s", t.Kind(), t.Summary()) } - // 3 snaps (2 tasks for each) + assets update from kernel + gadget (3 tasks) + recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 3*2+1+3+2+1+1) + // 3 snaps (2 tasks for each) + assets update from kernel + update cert db + gadget (3 tasks) + recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 3*2+1+1+3+2+1) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -3460,14 +3385,14 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnapsChannelSwitch tLinkKernel := tl[2] tPrepareBase := tl[3] tLinkBase := tl[4] - tSwitchGadget := tl[5] - tUpdateAssets := tl[6] - tUpdateCmdline := tl[7] - tValidateApp := tl[8] - tInstallApp := tl[9] - tCreateRecovery := tl[10] - tFinalizeRecovery := tl[11] - tUpdateCertDB := tl[12] + tUpdateCertDB := tl[5] + tSwitchGadget := tl[6] + tUpdateAssets := tl[7] + tUpdateCmdline := tl[8] + tValidateApp := tl[9] + tInstallApp := tl[10] + tCreateRecovery := tl[11] + tFinalizeRecovery := tl[12] tSetModel := tl[13] // check the tasks @@ -3501,8 +3426,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnapsChannelSwitch c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // check the ordering, prepare/link are part of download edge and come first @@ -3528,7 +3451,7 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnapsChannelSwitch }) c.Assert(tUpdateAssets.WaitTasks(), DeepEquals, []*state.Task{ tSwitchGadget, - tLinkBase, + tUpdateCertDB, }) c.Assert(tUpdateCmdline.WaitTasks(), DeepEquals, []*state.Task{ tUpdateAssets, @@ -3544,21 +3467,14 @@ func (s *deviceMgrRemodelSuite) TestRemodelOfflineUseInstalledSnapsChannelSwitch // last snap of the download chain (see above) tValidateApp, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tSwitchKernel, tUpdateAssetsKernel, - tLinkKernel, tPrepareBase, tLinkBase, - tSwitchGadget, tUpdateAssets, tUpdateCmdline, - tValidateApp, tInstallApp, - tCreateRecovery, tFinalizeRecovery, - }) // setModel waits for everything in the change c.Assert(tSetModel.WaitTasks(), DeepEquals, []*state.Task{ tSwitchKernel, tUpdateAssetsKernel, tLinkKernel, tPrepareBase, tLinkBase, + tUpdateCertDB, tSwitchGadget, tUpdateAssets, tUpdateCmdline, tValidateApp, tInstallApp, tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) snapsups := []any{tSwitchKernel.ID(), tPrepareBase.ID(), tSwitchGadget.ID(), tValidateApp.ID()} @@ -3711,8 +3627,8 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseGadgetSnapsInstal c.Assert(chg.Summary(), Equals, "Refresh model assertion from revision 0 to 1") tl := chg.Tasks() - // 2 snaps (2 tasks for each) + assets update and setup from kernel + gadget (3 tasks) + recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 2*2+1+3+2+1+1) + // 2 snaps (2 tasks for each) + assets update and setup from kernel + update cert db + gadget (3 tasks) + recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 2*2+1+1+3+2+1) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -3730,12 +3646,12 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseGadgetSnapsInstal tLinkKernel := tl[2] tPrepareBase := tl[3] tLinkBase := tl[4] - tPrepareGadget := tl[5] - tUpdateAssets := tl[6] - tUpdateCmdline := tl[7] - tCreateRecovery := tl[8] - tFinalizeRecovery := tl[9] - tUpdateCertDB := tl[10] + tUpdateCertDB := tl[5] + tPrepareGadget := tl[6] + tUpdateAssets := tl[7] + tUpdateCmdline := tl[8] + tCreateRecovery := tl[9] + tFinalizeRecovery := tl[10] tSetModel := tl[11] // check the tasks @@ -3765,8 +3681,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseGadgetSnapsInstal c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // check the ordering, prepare/link are part of download edge and come first @@ -3792,7 +3706,7 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseGadgetSnapsInstal }) c.Assert(tUpdateAssets.WaitTasks(), DeepEquals, []*state.Task{ tPrepareGadget, - tLinkBase, + tUpdateCertDB, }) c.Assert(tUpdateCmdline.WaitTasks(), DeepEquals, []*state.Task{ tUpdateAssets, @@ -3808,19 +3722,13 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseGadgetSnapsInstal // last snap of the download chain (see above) tPrepareGadget, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tPrepareKernel, tUpdateAssetsKernel, - tLinkKernel, tPrepareBase, tLinkBase, - tPrepareGadget, tUpdateAssets, tUpdateCmdline, - tCreateRecovery, tFinalizeRecovery, - }) // setModel waits for everything in the change c.Assert(tSetModel.WaitTasks(), DeepEquals, []*state.Task{ tPrepareKernel, tUpdateAssetsKernel, tLinkKernel, tPrepareBase, tLinkBase, + tUpdateCertDB, tPrepareGadget, tUpdateAssets, tUpdateCmdline, tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) // verify recovery system setup data on appropriate tasks var systemSetupData map[string]any @@ -4082,9 +3990,9 @@ func (s *deviceMgrRemodelSuite) testRemodelUC20SwitchKernelBaseGadgetSnapsInstal tl := chg.Tasks() // 2 snaps with (snap switch channel + link snap) + assets update for the - // kernel snap + gadget snap (switch channel, assets update, cmdline update) - // + recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 2*2+1+3+2+1+1) + // kernel snap + update cert db + gadget snap (switch channel, assets update, cmdline update) + // + recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 2*2+1+1+3+2+1) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -4102,12 +4010,12 @@ func (s *deviceMgrRemodelSuite) testRemodelUC20SwitchKernelBaseGadgetSnapsInstal tLinkKernel := tl[2] tSwitchChannelBase := tl[3] tLinkBase := tl[4] - tSwitchChannelGadget := tl[5] - tUpdateAssetsFromGadget := tl[6] - tUpdateCmdlineFromGadget := tl[7] - tCreateRecovery := tl[8] - tFinalizeRecovery := tl[9] - tUpdateCertDB := tl[10] + tUpdateCertDB := tl[5] + tSwitchChannelGadget := tl[6] + tUpdateAssetsFromGadget := tl[7] + tUpdateCmdlineFromGadget := tl[8] + tCreateRecovery := tl[9] + tFinalizeRecovery := tl[10] tSetModel := tl[11] // check the tasks @@ -4133,8 +4041,6 @@ func (s *deviceMgrRemodelSuite) testRemodelUC20SwitchKernelBaseGadgetSnapsInstal c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // check the ordering, prepare/link are part of download edge and come first @@ -4163,19 +4069,13 @@ func (s *deviceMgrRemodelSuite) testRemodelUC20SwitchKernelBaseGadgetSnapsInstal c.Assert(tLinkBase.WaitTasks(), DeepEquals, []*state.Task{ tSwitchChannelBase, tLinkKernel, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tSwitchChannelKernel, tUpdateAssetsFromKernel, - tLinkKernel, tSwitchChannelBase, tLinkBase, - tSwitchChannelGadget, tUpdateAssetsFromGadget, tUpdateCmdlineFromGadget, - tCreateRecovery, tFinalizeRecovery, - }) // setModel waits for everything in the change c.Assert(tSetModel.WaitTasks(), DeepEquals, []*state.Task{ tSwitchChannelKernel, tUpdateAssetsFromKernel, tLinkKernel, tSwitchChannelBase, tLinkBase, + tUpdateCertDB, tSwitchChannelGadget, tUpdateAssetsFromGadget, tUpdateCmdlineFromGadget, tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) // verify recovery system setup data on appropriate tasks var systemSetupData map[string]any @@ -4349,8 +4249,8 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseSnapsInstalledSna c.Assert(chg.Summary(), Equals, "Refresh model assertion from revision 0 to 1") tl := chg.Tasks() - // 2 snaps (3 tasks for each) + recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 2*3+2+1+1) + // 2 snaps (3 tasks for each) + recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 2*3+2+1) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -4371,8 +4271,7 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseSnapsInstalledSna tInstallBase := tl[5] tCreateRecovery := tl[6] tFinalizeRecovery := tl[7] - tUpdateCertDB := tl[8] - tSetModel := tl[9] + tSetModel := tl[8] // check the tasks expectedLabel := now.Format("20060102") @@ -4390,8 +4289,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseSnapsInstalledSna c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // check the ordering, prepare/link are part of download edge and come first @@ -4422,11 +4319,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseSnapsInstalledSna // last snap of the download chain (added later) tValidateBase, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tDownloadKernel, tValidateKernel, tInstallKernel, - tDownloadBase, tValidateBase, tInstallBase, - tCreateRecovery, tFinalizeRecovery, - }) c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") @@ -4435,7 +4327,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20SwitchKernelBaseSnapsInstalledSna tDownloadKernel, tValidateKernel, tInstallKernel, tDownloadBase, tValidateBase, tInstallBase, tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) // verify recovery system setup data on appropriate tasks @@ -4577,8 +4468,8 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20EssentialSnapsTrackingDifferentCh c.Assert(chg.Summary(), Equals, "Refresh model assertion from revision 0 to 1") tl := chg.Tasks() - // recovery system (2 tasks) + update-cert-db + set-model - c.Assert(tl, HasLen, 4) + // recovery system (2 tasks) + set-model + c.Assert(tl, HasLen, 3) deviceCtx, err := devicestate.DeviceCtx(s.state, tl[0], nil) c.Assert(err, IsNil) @@ -4593,8 +4484,7 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20EssentialSnapsTrackingDifferentCh // check the tasks tCreateRecovery := tl[0] tFinalizeRecovery := tl[1] - tUpdateCertDB := tl[2] - tSetModel := tl[3] + tSetModel := tl[2] // check the tasks expectedLabel := now.Format("20060102") @@ -4602,8 +4492,6 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20EssentialSnapsTrackingDifferentCh c.Assert(tCreateRecovery.Summary(), Equals, fmt.Sprintf("Create recovery system with label %q", expectedLabel)) c.Assert(tFinalizeRecovery.Kind(), Equals, "finalize-recovery-system") c.Assert(tFinalizeRecovery.Summary(), Equals, fmt.Sprintf("Finalize recovery system with label %q", expectedLabel)) - c.Assert(tUpdateCertDB.Kind(), Equals, "update-cert-db") - c.Assert(tUpdateCertDB.Summary(), Equals, "Update certificate database") c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") c.Assert(tCreateRecovery.WaitTasks(), HasLen, 0) @@ -4611,17 +4499,12 @@ func (s *deviceMgrRemodelSuite) TestRemodelUC20EssentialSnapsTrackingDifferentCh // recovery system being created tCreateRecovery, }) - c.Assert(tUpdateCertDB.WaitTasks(), DeepEquals, []*state.Task{ - tCreateRecovery, - tFinalizeRecovery, - }) c.Assert(tSetModel.Kind(), Equals, "set-model") c.Assert(tSetModel.Summary(), Equals, "Set new model assertion") // setModel waits for everything in the change c.Assert(tSetModel.WaitTasks(), DeepEquals, []*state.Task{ tCreateRecovery, tFinalizeRecovery, - tUpdateCertDB, }) // verify recovery system setup data on appropriate tasks diff --git a/overlord/managers_test.go b/overlord/managers_test.go index 67ff7d25435..ffb87c04d6d 100644 --- a/overlord/managers_test.go +++ b/overlord/managers_test.go @@ -5151,6 +5151,7 @@ const ( isGadget isKernel needsKernelSetup + isModelBase ) func validateInstallTasks(c *C, tasks []*state.Task, name, revno string, flags int) int { @@ -5197,6 +5198,10 @@ func validateInstallTasks(c *C, tasks []*state.Task, name, revno string, flags i c.Assert(tasks[i].Summary(), Equals, fmt.Sprintf(`Run default-configure hook of "%s" snap if present`, name)) i++ } + if flags&isModelBase != 0 { + c.Assert(tasks[i].Summary(), Equals, `Update certificate database`) + i++ + } c.Assert(tasks[i].Summary(), Equals, fmt.Sprintf(`Start snap "%s" (%s) services`, name, revno)) i++ if flags&noConfigure == 0 { @@ -5256,6 +5261,10 @@ func validateRefreshTasks(c *C, tasks []*state.Task, name, revno string, flags i i++ c.Assert(tasks[i].Summary(), Equals, fmt.Sprintf(`Run post-refresh hook of "%s" snap if present`, name)) i++ + if flags&isModelBase != 0 { + c.Assert(tasks[i].Summary(), Equals, `Update certificate database`) + i++ + } c.Assert(tasks[i].Summary(), Equals, fmt.Sprintf(`Start snap "%s" (%s) services`, name, revno)) i++ c.Assert(tasks[i].Summary(), Equals, fmt.Sprintf(`Clean up "%s" (%s) install`, name, revno)) @@ -5696,11 +5705,9 @@ version: 20.04` i += validateDownloadCheckTasks(c, tasks[i:], "foo", "1", "stable") // then all installs in sequential order - i += validateInstallTasks(c, tasks[i:], "core20", "2", noConfigure) + i += validateInstallTasks(c, tasks[i:], "core20", "2", noConfigure|isModelBase) i += validateInstallTasks(c, tasks[i:], "pc-20", "2", isGadget) i += validateInstallTasks(c, tasks[i:], "foo", "1", 0) - c.Assert(tasks[i].Summary(), Equals, `Update certificate database`) - i++ c.Assert(tasks[i].Summary(), Equals, `Set new model assertion`) i++ @@ -8633,7 +8640,7 @@ func (s *mgrsSuiteCore) TestRemodelUC20DifferentBaseChannel(c *C) { // then create recovery i += validateRecoverySystemTasks(c, tasks[i:], expectedLabel) // then all refreshes in sequential order (no configure hooks for bases though) - validateRefreshTasks(c, tasks[i:], "core20", "33", noConfigure) + validateRefreshTasks(c, tasks[i:], "core20", "33", noConfigure|isModelBase) } func (s *mgrsSuiteCore) TestRemodelUC20BackToPreviousGadget(c *C) { @@ -9514,7 +9521,9 @@ func (s *mgrsSuiteCore) TestRemodelRollbackValidationSets(c *C) { // gadget update for the seed partition has been applied c.Check(updater.updateCalls, Equals, 1) - c.Check(certDBUpdateCalls, Equals, 0) + + // after we booted the base, the cert-db should be updated as well + c.Check(certDBUpdateCalls, Equals, 1) dumpTasks(c, "after gadget install", chg.Tasks()) @@ -9553,9 +9562,6 @@ func (s *mgrsSuiteCore) TestRemodelRollbackValidationSets(c *C) { c.Assert(chg.Status(), Equals, state.ErrorStatus) - // update-cert-db should have been called prior to setting the new model - c.Check(certDBUpdateCalls, Equals, 1) - // list validation sets that are currently tracked currentSets, err := assertstate.TrackedEnforcedValidationSets(st) c.Assert(err, IsNil) @@ -10051,11 +10057,8 @@ func (s *mgrsSuiteCore) TestRemodelReplaceValidationSets(c *C) { i += validateRecoverySystemTasks(c, tasks[i:], expectedLabel) // then all refreshes and install in sequential order (no configure hooks for bases though) i += validateRefreshTasks(c, tasks[i:], "pc-kernel", "33", isKernel) - i += validateInstallTasks(c, tasks[i:], "core22", "1", noConfigure) + i += validateInstallTasks(c, tasks[i:], "core22", "1", noConfigure|isModelBase) i += validateRefreshTasks(c, tasks[i:], "pc", "34", isGadget) - // then update certificate database for the new base - c.Assert(tasks[i].Summary(), Equals, `Update certificate database`) - i++ // finally new model assertion c.Assert(tasks[i].Summary(), Equals, `Set new model assertion`) i++ @@ -10377,11 +10380,8 @@ func (s *mgrsSuiteCore) testRemodelUC20ToUC22(c *C, mockSnapdRefresh bool) { i += validateRecoverySystemTasks(c, tasks[i:], expectedLabel) // then all refreshes and install in sequential order (no configure hooks for bases though) i += validateRefreshTasks(c, tasks[i:], "pc-kernel", "33", isKernel) - i += validateInstallTasks(c, tasks[i:], "core22", "1", noConfigure) + i += validateInstallTasks(c, tasks[i:], "core22", "1", noConfigure|isModelBase) i += validateRefreshTasks(c, tasks[i:], "pc", "34", isGadget) - // then update certificate database for the new base - c.Assert(tasks[i].Summary(), Equals, `Update certificate database`) - i++ // finally new model assertion c.Assert(tasks[i].Summary(), Equals, `Set new model assertion`) i++ @@ -11825,9 +11825,28 @@ func (s *mgrsSuiteCore) testUpdateKernelBaseSingleRebootSetup(c *C) (*boottest.R p, _ = s.makeStoreTestSnap(c, snapYamlContent, "2") s.serveSnap(p, "2") + c.Assert(os.MkdirAll(dirs.SystemCertsDir, 0755), IsNil) + affected, tss, err := snapstate.UpdateMany(context.Background(), st, []string{"pc-kernel", "core20", "some-snap"}, nil, 0, nil) c.Assert(err, IsNil) c.Assert(affected, DeepEquals, []string{"core20", "pc-kernel", "some-snap"}) + + // Regular refresh path should include certificate DB refresh when the + // model boot-base (core20) is refreshed. + foundUpdateCertDB := false + for _, ts := range tss { + for _, t := range ts.Tasks() { + if t.Kind() == "update-cert-db" { + foundUpdateCertDB = true + break + } + } + if foundUpdateCertDB { + break + } + } + c.Assert(foundUpdateCertDB, Equals, true) + chg := st.NewChange("update-many", "...") for _, ts := range tss { chg.AddAll(ts) diff --git a/overlord/snapstate/export_test.go b/overlord/snapstate/export_test.go index 1afc6171d29..e7eaa75fdca 100644 --- a/overlord/snapstate/export_test.go +++ b/overlord/snapstate/export_test.go @@ -447,18 +447,19 @@ func MockSecurityProfilesDiscardLate(fn func(snapName string, rev snap.Revision, type HoldState = holdState var ( - HoldDurationLeft = holdDurationLeft - LastRefreshed = lastRefreshed - PruneRefreshCandidates = pruneRefreshCandidates - UpdateRefreshCandidates = updateRefreshCandidates - ResetGatingForRefreshed = resetGatingForRefreshed - PruneGating = pruneGating - PruneSnapsHold = pruneSnapsHold - CreateGateAutoRefreshHooks = createGateAutoRefreshHooks - AutoRefreshPhase1 = autoRefreshPhase1 - RefreshRetain = refreshRetain - RefreshCheck = refreshAppsCheck - AffectsRunningHooks = affectsRunningHooks + HoldDurationLeft = holdDurationLeft + LastRefreshed = lastRefreshed + PruneRefreshCandidates = pruneRefreshCandidates + UpdateRefreshCandidates = updateRefreshCandidates + ResetGatingForRefreshed = resetGatingForRefreshed + PruneGating = pruneGating + PruneSnapsHold = pruneSnapsHold + CreateGateAutoRefreshHooks = createGateAutoRefreshHooks + AutoRefreshPhase1 = autoRefreshPhase1 + RefreshRetain = refreshRetain + RefreshCheck = refreshAppsCheck + AffectsRunningHooks = affectsRunningHooks + ShouldScheduleUpdateCertDBForRefresh = shouldScheduleUpdateCertDBForRefresh ExcludeFromRefreshAppAwareness = excludeFromRefreshAppAwareness ) diff --git a/overlord/snapstate/snap.go b/overlord/snapstate/snap.go index 9a2dbdc2d61..d6a4546e71f 100644 --- a/overlord/snapstate/snap.go +++ b/overlord/snapstate/snap.go @@ -375,6 +375,21 @@ func removeExtraComponentsTasks(st *state.State, snapst *SnapState, targetRevisi return unlinkTasks, discardTasks, nil } +// shouldScheduleUpdateCertDBForRefresh reports whether a snap operation +// should inject an update-cert-db task. +func shouldScheduleUpdateCertDBForRefresh(instanceName string, snapType snap.Type, ctx DeviceContext) bool { + if snapType != snap.TypeBase { + return false + } + + model := ctx.Model() + if model.Classic() { + return false + } + + return instanceName == model.Base() +} + func (sc *snapInstallChoreographer) AfterLinkSnapAndPostReboot(st *state.State, s *taskChainSpan, ic installContext) ([]*state.Task, error) { if !sc.requiresKmodSetup() { // Let tasks know if they have to do something about restarts @@ -399,6 +414,14 @@ func (sc *snapInstallChoreographer) AfterLinkSnapAndPostReboot(st *state.State, } } + // Refreshing the model base may bring updated system certificates. + // Regenerate the managed certificate database as part of the post-reboot + // refresh stage for that base. + if shouldScheduleUpdateCertDBForRefresh(sc.snapsup.InstanceName(), sc.snapsup.Type, ic.DeviceCtx) { + updateCertDB := st.NewTask("update-cert-db", i18n.G("Update certificate database")) + s.Append(updateCertDB) + } + if sc.snapsup.QuotaGroupName != "" { quotaAddSnapTask, err := AddSnapToQuotaGroup(st, sc.snapsup.InstanceName(), sc.snapsup.QuotaGroupName) if err != nil { diff --git a/overlord/snapstate/snapstate.go b/overlord/snapstate/snapstate.go index c56e8c2b9ae..e0ebeb3db85 100644 --- a/overlord/snapstate/snapstate.go +++ b/overlord/snapstate/snapstate.go @@ -2680,7 +2680,7 @@ func MigrateHome(st *state.State, snaps []string) ([]*state.TaskSet, error) { // this was not needed. Since this function is only used if the snap is // installed already installed, then it is expected that the drivers tree is // present. Thus, the prepare-kernel-snap task would be redundant. -func LinkNewBaseOrKernel(st *state.State, name string, fromChange string) (*state.TaskSet, error) { +func LinkNewBaseOrKernel(st *state.State, name string, fromChange string, deviceCtx DeviceContext) (*state.TaskSet, error) { var snapst SnapState err := Get(st, name, &snapst) if errors.Is(err, state.ErrNoState) { @@ -2725,14 +2725,14 @@ func LinkNewBaseOrKernel(st *state.State, name string, fromChange string) (*stat ts.MarkEdge(prepareSnap, LastBeforeLocalModificationsEdge) ts.MarkEdge(prepareSnap, SnapSetupEdge) - if err := addLinkNewBaseOrKernelTasks(st, snapst, ts, prepareSnap); err != nil { + if err := addLinkNewBaseOrKernelTasks(st, snapst, ts, prepareSnap, deviceCtx); err != nil { return nil, err } return ts, nil } -func addLinkNewBaseOrKernelTasks(st *state.State, snapst SnapState, ts *state.TaskSet, snapsupTask *state.Task) error { +func addLinkNewBaseOrKernelTasks(st *state.State, snapst SnapState, ts *state.TaskSet, snapsupTask *state.Task, deviceCtx DeviceContext) error { tasks := ts.Tasks() if len(tasks) == 0 { return errors.New("internal error: task set must be seeded with at least one task") @@ -2786,6 +2786,13 @@ func addLinkNewBaseOrKernelTasks(st *state.State, snapst SnapState, ts *state.Ta snapsupTask.Set("component-setup-tasks", compsupTasks) + // Switching to a new model base may require regenerating the managed + // certificate database. + if shouldScheduleUpdateCertDBForRefresh(info.InstanceName(), info.Type(), deviceCtx) { + updateCertDB := st.NewTask("update-cert-db", i18n.G("Update certificate database")) + add(updateCertDB) + } + return nil } @@ -2820,7 +2827,7 @@ func findSnapSetupTask(tasks []*state.Task) (*state.Task, *SnapSetup, error) { // this was not needed. Since this function is only used if the snap is // installed already installed, then it is expected that the drivers tree is // present. Thus, the prepare-kernel-snap task would be redundant. -func AddLinkNewBaseOrKernel(st *state.State, ts *state.TaskSet) (*state.TaskSet, error) { +func AddLinkNewBaseOrKernel(st *state.State, ts *state.TaskSet, deviceCtx DeviceContext) (*state.TaskSet, error) { if ts.MaybeEdge(LastBeforeLocalModificationsEdge) != nil { return nil, errors.New("internal error: cannot add tasks to link new base or kernel to task set that introduces local modifications") } @@ -2848,7 +2855,7 @@ func AddLinkNewBaseOrKernel(st *state.State, ts *state.TaskSet) (*state.TaskSet, // tasks introduced here modify system state ts.MarkEdge(allTasks[len(allTasks)-1], LastBeforeLocalModificationsEdge) - if err := addLinkNewBaseOrKernelTasks(st, snapst, ts, snapSetupTask); err != nil { + if err := addLinkNewBaseOrKernelTasks(st, snapst, ts, snapSetupTask, deviceCtx); err != nil { return nil, err } diff --git a/overlord/snapstate/snapstate_test.go b/overlord/snapstate/snapstate_test.go index 3f75067409d..2e8512f30a4 100644 --- a/overlord/snapstate/snapstate_test.go +++ b/overlord/snapstate/snapstate_test.go @@ -260,6 +260,9 @@ func (s *snapmgrBaseTest) SetUpTest(c *C) { s.snapmgr, err = snapstate.Manager(s.state, s.o.TaskRunner()) c.Assert(err, IsNil) + s.o.TaskRunner().AddHandler("update-cert-db", func(_ *state.Task, _ *tomb.Tomb) error { + return nil + }, nil) AddForeignTaskHandlers(s.o.TaskRunner(), s.fakeBackend) @@ -8606,7 +8609,8 @@ func (s *snapmgrTestSuite) testRemodelLinkNewBaseOrKernelHappy(c *C, model *asse const withComponents = false s.addSnapsForRemodel(c, withComponents) - ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-kernel", "") + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: model} + ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-kernel", "", deviceCtx) c.Assert(err, IsNil) tasks := ts.Tasks() @@ -8627,7 +8631,7 @@ func (s *snapmgrTestSuite) testRemodelLinkNewBaseOrKernelHappy(c *C, model *asse c.Assert(tLink.WaitTasks(), DeepEquals, []*state.Task{tUpdateGadgetAssets}) c.Assert(ts.MaybeEdge(snapstate.MaybeRebootEdge), Equals, tLink) - ts, err = snapstate.LinkNewBaseOrKernel(s.state, "some-base", "") + ts, err = snapstate.LinkNewBaseOrKernel(s.state, "some-base", "", deviceCtx) c.Assert(err, IsNil) tasks = ts.Tasks() c.Check(taskKinds(tasks), DeepEquals, expectedDoInstallTasks(snap.TypeBase, 0, 0, 0, []string{"prepare-snap"}, nil, kindsToSet(nonReLinkKinds))) @@ -8642,6 +8646,41 @@ func (s *snapmgrTestSuite) testRemodelLinkNewBaseOrKernelHappy(c *C, model *asse c.Assert(ts.MaybeEdge(snapstate.MaybeRebootEdge), Equals, tLink) } +func (s *snapmgrTestSuite) TestRemodelLinkNewBaseUpdatesCertDB(c *C) { + restore := release.MockOnClassic(false) + defer restore() + + s.BaseTest.AddCleanup(snapstate.MockSnapReadInfo(snap.ReadInfo)) + s.state.Lock() + defer s.state.Unlock() + + // Use a model whose base matches the installed "some-base" snap, so + // that shouldScheduleUpdateCertDBForRefresh returns true. + model := MakeModel20("brand-gadget", map[string]any{"base": "some-base"}) + defer snapstatetest.MockDeviceModel(model)() + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: model} + + const withComponents = false + s.addSnapsForRemodel(c, withComponents) + + // Linking the model base injects an update-cert-db task. + ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-base", "", deviceCtx) + c.Assert(err, IsNil) + tasks := ts.Tasks() + c.Assert(tasks, HasLen, 3) + c.Assert(tasks[0].Kind(), Equals, "prepare-snap") + c.Assert(tasks[1].Kind(), Equals, "link-snap") + c.Assert(tasks[2].Kind(), Equals, "update-cert-db") + c.Assert(tasks[2].Summary(), Equals, "Update certificate database") + + // Linking a kernel does not inject update-cert-db. + ts, err = snapstate.LinkNewBaseOrKernel(s.state, "some-kernel", "", deviceCtx) + c.Assert(err, IsNil) + for _, t := range ts.Tasks() { + c.Assert(t.Kind(), Not(Equals), "update-cert-db") + } +} + func (s *snapmgrTestSuite) TestRemodelLinkNewBaseOrKernelWithComponent(c *C) { model := MakeModel20("brand-gadget", nil) @@ -8657,7 +8696,8 @@ func (s *snapmgrTestSuite) TestRemodelLinkNewBaseOrKernelWithComponent(c *C) { const withComponents = true s.addSnapsForRemodel(c, withComponents) - ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-kernel", "") + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: model} + ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-kernel", "", deviceCtx) c.Assert(err, IsNil) comps := []string{"comp-1", "comp-2"} @@ -8707,7 +8747,8 @@ func (s *snapmgrTestSuite) TestRemodelAddLinkNewBaseOrKernelWithComponent(c *C) ts := state.NewTaskSet(switchSnap) - ts, err := snapstate.AddLinkNewBaseOrKernel(s.state, ts) + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: model} + ts, err := snapstate.AddLinkNewBaseOrKernel(s.state, ts, deviceCtx) c.Assert(err, IsNil) comps := []string{"comp-1", "comp-2"} @@ -8747,11 +8788,12 @@ func (s *snapmgrTestSuite) TestRemodelLinkNewBaseOrKernelBadType(c *C) { Current: si.Revision, SnapType: "app", }) - ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-snap", "") + + ts, err := snapstate.LinkNewBaseOrKernel(s.state, "some-snap", "", nil) c.Assert(err, ErrorMatches, `internal error: cannot link type app`) c.Assert(ts, IsNil) - ts, err = snapstate.LinkNewBaseOrKernel(s.state, "some-gadget", "") + ts, err = snapstate.LinkNewBaseOrKernel(s.state, "some-gadget", "", nil) c.Assert(err, ErrorMatches, `internal error: cannot link type gadget`) c.Assert(ts, IsNil) } @@ -8771,7 +8813,8 @@ func (s *snapmgrTestSuite) TestRemodelLinkNewBaseOrKernelNoRemodelConflict(c *C) chg := s.state.NewChange("remodel", "remodel") chg.AddTask(tugc) - _, err := snapstate.LinkNewBaseOrKernel(s.state, "some-base", chg.ID()) + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: DefaultModel()} + _, err := snapstate.LinkNewBaseOrKernel(s.state, "some-base", chg.ID(), deviceCtx) c.Assert(err, IsNil) } @@ -8813,7 +8856,8 @@ func (s *snapmgrTestSuite) testRemodelAddLinkNewBaseOrKernel(c *C, model *assert testTask := s.state.NewTask("test-task", "test task") ts := state.NewTaskSet(tPrepare, testTask) - tsNew, err := snapstate.AddLinkNewBaseOrKernel(s.state, ts) + deviceCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: model} + tsNew, err := snapstate.AddLinkNewBaseOrKernel(s.state, ts, deviceCtx) c.Assert(err, IsNil) c.Assert(tsNew, NotNil) tasks := tsNew.Tasks() @@ -8845,7 +8889,7 @@ func (s *snapmgrTestSuite) testRemodelAddLinkNewBaseOrKernel(c *C, model *assert Type: "base", }) ts = state.NewTaskSet(tPrepare) - tsNew, err = snapstate.AddLinkNewBaseOrKernel(s.state, ts) + tsNew, err = snapstate.AddLinkNewBaseOrKernel(s.state, ts, deviceCtx) c.Assert(err, IsNil) c.Assert(tsNew, NotNil) tasks = tsNew.Tasks() @@ -8862,7 +8906,7 @@ func (s *snapmgrTestSuite) testRemodelAddLinkNewBaseOrKernel(c *C, model *assert // but bails when there is no task with snap setup ts = state.NewTaskSet() - tsNew, err = snapstate.AddLinkNewBaseOrKernel(s.state, ts) + tsNew, err = snapstate.AddLinkNewBaseOrKernel(s.state, ts, deviceCtx) c.Assert(err, ErrorMatches, `internal error: cannot identify task with snap-setup`) c.Assert(tsNew, IsNil) } @@ -10133,6 +10177,9 @@ func validateEnforcementOrder(c *C, st *state.State, tss []*state.TaskSet, class enforce = ts.Tasks()[0] continue } + if len(ts.Tasks()) == 1 && ts.Tasks()[0].Kind() == "update-cert-db" { + continue + } snapsup, err := snapstate.TaskSnapSetup(ts.Tasks()[0]) c.Assert(err, IsNil) @@ -12598,6 +12645,33 @@ func (s *snapStateSuite) TestEnsureLoopLogging(c *C) { testutil.CheckEnsureLoopLogging("snapmgr.go", c, true, "autorefresh.go", "catalogrefresh.go", "refreshhints.go") } +func (s *snapStateSuite) TestShouldScheduleUpdateCertDBForRefresh(c *C) { + modelBaseCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: ModelWithBase("core18")} + remodelCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: ModelWithBase("core18"), Remodeling: true} + classicCtx := &snapstatetest.TrivialDeviceContext{DeviceModel: MakeModelClassicWithModes("pc", nil)} + + tests := []struct { + name string + ctx snapstate.DeviceContext + snapType snap.Type + snapName string + expected bool + }{ + {name: "base-snap refresh", ctx: modelBaseCtx, snapType: snap.TypeBase, snapName: "core18", expected: true}, + {name: "remodel refresh path", ctx: remodelCtx, snapType: snap.TypeBase, snapName: "core18", expected: true}, + {name: "remodel install path", ctx: remodelCtx, snapType: snap.TypeBase, snapName: "core18", expected: true}, + {name: "non-base snap", ctx: modelBaseCtx, snapType: snap.TypeApp, snapName: "core18", expected: false}, + {name: "classic model", ctx: classicCtx, snapType: snap.TypeBase, snapName: "core22", expected: false}, + {name: "non-model base", ctx: modelBaseCtx, snapType: snap.TypeBase, snapName: "some-base", expected: false}, + {name: "model base", ctx: modelBaseCtx, snapType: snap.TypeBase, snapName: "core18", expected: true}, + } + + for _, tc := range tests { + c.Check(snapstate.ShouldScheduleUpdateCertDBForRefresh( + tc.snapName, tc.snapType, tc.ctx), Equals, tc.expected, Commentf(tc.name)) + } +} + func verifyDelayedEffectsTasks(c *C, ts *state.TaskSet, expectedLanes []int, expectedJoinLane int) { c.Assert(ts.Tasks(), HasLen, 1) c.Check(taskKinds(ts.Tasks()), DeepEquals, []string{"mock-process-delayed-security-backend-effects"}) diff --git a/overlord/snapstate/snapstate_update_test.go b/overlord/snapstate/snapstate_update_test.go index 9310ad40edc..3461815e7c3 100644 --- a/overlord/snapstate/snapstate_update_test.go +++ b/overlord/snapstate/snapstate_update_test.go @@ -5984,6 +5984,96 @@ func (s *snapmgrTestSuite) TestUpdateManyOneSwitchesChannel(c *C) { c.Check(switchTask.Kind(), Equals, "switch-snap-channel") } +func (s *snapmgrTestSuite) TestUpdateManyModelBaseSwitchesChannelUpdateCertDB(c *C) { + s.state.Lock() + defer s.state.Unlock() + + restore := snapstatetest.MockDeviceModel(ModelWithBase("core18")) + defer restore() + + si := snap.SideInfo{ + RealName: "core18", + Revision: snap.R(1), + SnapID: "core18-snap-id", + } + + snaptest.MockSnap(c, "name: core18\ntype: base", &si) + snapstate.Set(s.state, "core18", &snapstate.SnapState{ + Active: true, + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{&si}), + Current: si.Revision, + SnapType: "base", + TrackingChannel: "latest/stable", + }) + + goal := snapstate.StoreUpdateGoal(snapstate.StoreUpdate{ + InstanceName: "core18", + RevOpts: snapstate.RevisionOptions{ + Channel: "channel-for-1/stable", + }, + }) + + names, uts, err := snapstate.UpdateWithGoal(context.Background(), s.state, goal, nil, snapstate.Options{}) + c.Assert(err, IsNil) + c.Assert(names, DeepEquals, []string{"core18"}) + + updateCertDBTasks := 0 + for _, ts := range uts.Refresh { + for _, t := range ts.Tasks() { + if t.Kind() == "update-cert-db" { + updateCertDBTasks++ + } + } + } + c.Check(updateCertDBTasks, Equals, 1) +} + +func (s *snapmgrTestSuite) TestUpdateManyNonModelBaseRefreshNoUpdateCertDB(c *C) { + s.state.Lock() + defer s.state.Unlock() + + restore := snapstatetest.MockDeviceModel(ModelWithBase("core18")) + defer restore() + + siBootBase := snap.SideInfo{ + RealName: "core18", + Revision: snap.R(1), + SnapID: "core18-snap-id", + } + snaptest.MockSnap(c, "name: core18\ntype: base", &siBootBase) + snapstate.Set(s.state, "core18", &snapstate.SnapState{ + Active: true, + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{&siBootBase}), + Current: siBootBase.Revision, + SnapType: "base", + TrackingChannel: "latest/stable", + }) + + siOtherBase := snap.SideInfo{ + RealName: "some-base", + Revision: snap.R(1), + SnapID: "some-base-id", + } + snaptest.MockSnap(c, "name: some-base\ntype: base", &siOtherBase) + snapstate.Set(s.state, "some-base", &snapstate.SnapState{ + Active: true, + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{&siOtherBase}), + Current: siOtherBase.Revision, + SnapType: "base", + TrackingChannel: "latest/stable", + }) + + updates, uts, err := snapstate.UpdateMany(context.Background(), s.state, []string{"some-base"}, nil, 0, nil) + c.Assert(err, IsNil) + c.Assert(updates, DeepEquals, []string{"some-base"}) + + for _, ts := range uts { + for _, t := range ts.Tasks() { + c.Check(t.Kind(), Not(Equals), "update-cert-db") + } + } +} + func (s *snapmgrTestSuite) TestUpdateManyOneSwitchesChannelWithAutoAlias(c *C) { sideInfos := []snap.SideInfo{ { @@ -10784,7 +10874,7 @@ func (s *snapmgrTestSuite) TestUpdateBaseGadgetKernelSingleRebootUndone(c *C) { case state.DoneStatus: // following tasks don't have undo logic switch t.Kind() { - case "prerequisites", "validate-snap", "run-hook", "cleanup": + case "prerequisites", "validate-snap", "run-hook", "cleanup", "update-cert-db": break default: c.Errorf("unexpected done-status for %s task %s", name, t.Kind()) @@ -10922,12 +11012,15 @@ func (s *snapmgrTestSuite) testUpdateEssentialSnapsOrder(c *C, order []string) { return nil } + for _, ts := range tss { + chg.AddAll(ts) + } + // Map all relevant task-sets. tsByName := make(map[string]*state.TaskSet) for _, sn := range order { ts := findTs(sn) c.Assert(ts, NotNil) - chg.AddAll(ts) tsByName[sn] = ts } diff --git a/tests/core/base-refresh-cert-db/task.yaml b/tests/core/base-refresh-cert-db/task.yaml new file mode 100644 index 00000000000..0e1811f5c5e --- /dev/null +++ b/tests/core/base-refresh-cert-db/task.yaml @@ -0,0 +1,75 @@ +summary: Verify non-model base refresh does not inject certificate database update + +details: | + Ensure that refreshing a base snap that is not the current model base does + not inject the "Update certificate database" task. + +prepare: | + . /etc/os-release + core_snap="core${VERSION_ID}" + + case "$VERSION_ID" in + 18) + other_base="core" + ;; + 20) + other_base="core18" + ;; + 22) + other_base="core20" + ;; + 24) + other_base="core22" + ;; + 26) + other_base="core24" + ;; + *) + echo "unsupported VERSION_ID: $VERSION_ID" + exit 1 + ;; + esac + + readlink "/snap/$core_snap/current" > core.rev + test "$core_snap" != "$other_base" + + echo "$other_base" > other-base.name + + # ensure the other base is installed + if ! snap list "$other_base" > /dev/null 2>&1; then + snap install "$other_base" --edge + fi + +execute: | + . /etc/os-release + core_snap="core${VERSION_ID}" + other_base="$(cat other-base.name)" + + wait_and_verify_change() { + local change_id="$1" + snap watch "$change_id" || true + snap changes | MATCH "$change_id\\s+Done" + snap change "$change_id" > tasks.done + NOMATCH '^Error' < tasks.done + NOMATCH '^Undone' < tasks.done + NOMATCH '^Wait' < tasks.done + retry -n 30 sh -c 'systemctl is-system-running | MATCH "(running|degraded)"' + } + + # Refreshing a non-model base should not request update-cert-db. + snap download "$other_base" --edge --basename "$other_base" + + # Create two distinct unasserted payloads to force an update. + printf '\0' >> "${other_base}.snap" + + # Refresh the snap + snap install "${other_base}.snap" --dangerous --no-wait > install-change-id + change_id="$(cat install-change-id)" + test -n "$change_id" + wait_and_verify_change "$change_id" + + # Verify that refresh of non-model base does not inject cert DB update. + NOMATCH 'Update certificate database' < tasks.done + + # Ensure we did not change the model base. + test "$(readlink /snap/$core_snap/current)" = "$(cat core.rev)" diff --git a/tests/core/model-base-refresh-cert-db/task.yaml b/tests/core/model-base-refresh-cert-db/task.yaml new file mode 100644 index 00000000000..5e543641483 --- /dev/null +++ b/tests/core/model-base-refresh-cert-db/task.yaml @@ -0,0 +1,58 @@ +summary: Verify model base refresh injects certificate database update + +details: | + Ensure that refreshing the current model base injects exactly one + "Update certificate database" task in the refresh change. + +prepare: | + . /etc/os-release + core_snap="core${VERSION_ID}" + readlink "/snap/$core_snap/current" > core.rev + +execute: | + . /etc/os-release + core_snap="core${VERSION_ID}" + + wait_and_verify_change() { + local change_id="$1" + snap watch "$change_id" || true + snap changes | MATCH "$change_id\\s+Done" + snap change "$change_id" > tasks.done + NOMATCH '^Error' < tasks.done + NOMATCH '^Undone' < tasks.done + NOMATCH '^Wait' < tasks.done + retry -n 30 sh -c 'systemctl is-system-running | MATCH "(running|degraded)"' + } + + if [ "$SPREAD_REBOOT" -eq 0 ]; then + snap download "$core_snap" --edge + printf '\0' >> "$core_snap"_*.snap + + snap install "$(ls ${core_snap}_*.snap)" --dangerous --no-wait > refresh-change-id + change_id="$(cat refresh-change-id)" + test -n "$change_id" + + retry -n 50 --wait 1 sh -c 'journalctl -b -u snapd | MATCH "Waiting for system reboot"' + REBOOT + elif [ "$SPREAD_REBOOT" -eq 1 ]; then + change_id="$(cat refresh-change-id)" + wait_and_verify_change "$change_id" + + MATCH 'Update certificate database' < tasks.done + [ "$(grep -c 'Update certificate database' tasks.done)" -eq 1 ] + + snap revert "$core_snap" --no-wait > revert-change-id + retry -n 50 --wait 1 sh -c 'journalctl -b -u snapd | MATCH "Waiting for system reboot"' + REBOOT + elif [ "$SPREAD_REBOOT" -eq 2 ]; then + change_id="$(cat revert-change-id)" + wait_and_verify_change "$change_id" + + MATCH 'Update certificate database' < tasks.done + [ "$(grep -c 'Update certificate database' tasks.done)" -eq 1 ] + + test "$(readlink /snap/$core_snap/current)" = "$(cat core.rev)" + else + echo "unexpected reboot" + exit 1 + fi From b860983d8c4419d525ada88bbb8203bafde49eb7 Mon Sep 17 00:00:00 2001 From: Maciej Borzecki Date: Tue, 14 Apr 2026 16:14:02 +0200 Subject: [PATCH 19/44] overlord/snapstate: improve handling of failed downloads and cleanup of snap/component files (#16841) * overlord/snapstate: leave partial files on failed download Keep the partial files if the download failed. Related: SNAPDENG-36634 Signed-off-by: Maciej Borzecki * overlord/snapstate: improve downloads cleanup to handle components and partial files Improve downloads cleanup to keep component files for ones that are present in the state, or referenced by any pending changes. Extend the code to keep track of partially downloaded files (named *.snap.partial or *.comp.partial) and keep the ones that are part of pending changes. Related: SNAPDENG-36634 Signed-off-by: Maciej Borzecki * tests/main/proxy-no-core: run on 24.04 Signed-off-by: Maciej Borzecki * overlord/snapstate: make downloads cleanup run periodically, not just on startup Signed-off-by: Maciej Borzecki * fixup! overlord/snapstate: improve downloads cleanup to handle components and partial files * fixup! overlord/snapstate: improve downloads cleanup to handle components and partial files * tests: add test for resuming partial snap downloads Signed-off-by: Zeyad Gouda * fixup! tests: add test for resuming partial snap downloads * fixup! tests: add test for resuming partial snap downloads * fixup! tests: add test for resuming partial snap downloads --------- Signed-off-by: Maciej Borzecki Signed-off-by: Zeyad Gouda Co-authored-by: Zeyad Gouda --- overlord/snapstate/backend_test.go | 11 +- overlord/snapstate/export_test.go | 12 +- overlord/snapstate/handlers.go | 10 +- overlord/snapstate/handlers_components.go | 5 +- .../handlers_components_download_test.go | 8 +- overlord/snapstate/handlers_download_test.go | 8 +- overlord/snapstate/snapmgr.go | 21 +- overlord/snapstate/snapstate.go | 94 +++-- overlord/snapstate/snapstate_test.go | 334 ++++++++++++++---- overlord/snapstate/snapstate_update_test.go | 20 +- tests/main/proxy-no-core/task.yaml | 12 +- .../resume-partial-snap-downloads/task.yaml | 111 ++++++ 12 files changed, 517 insertions(+), 129 deletions(-) create mode 100644 tests/main/resume-partial-snap-downloads/task.yaml diff --git a/overlord/snapstate/backend_test.go b/overlord/snapstate/backend_test.go index 30c6b0edd84..eba9bcfbd76 100644 --- a/overlord/snapstate/backend_test.go +++ b/overlord/snapstate/backend_test.go @@ -190,6 +190,9 @@ type fakeStore struct { mu sync.Mutex + // download options are normalized to nil if they match the expected default value + expectedDefaultDownloadOpts *store.DownloadOptions + downloads []fakeDownload iconDownloads []fakeIconDownload refreshRevnos map[string]snap.Revision @@ -944,8 +947,12 @@ func (f *fakeStore) Download(ctx context.Context, name, targetFn string, snapInf if user != nil { macaroon = user.StoreMacaroon } - // only add the options if they contain anything interesting - if dlOpts != nil && *dlOpts == (store.DownloadOptions{}) { + // only add the options if they differ from the defaults + dflt := f.expectedDefaultDownloadOpts + if dflt == nil { + dflt = &store.DownloadOptions{} + } + if dlOpts != nil && *dlOpts == *dflt { dlOpts = nil } f.appendDownload(&fakeDownload{ diff --git a/overlord/snapstate/export_test.go b/overlord/snapstate/export_test.go index e7eaa75fdca..8eacf715d71 100644 --- a/overlord/snapstate/export_test.go +++ b/overlord/snapstate/export_test.go @@ -361,12 +361,12 @@ func MockEnsuredDesktopFilesUpdated(m *SnapManager, ensured bool) (restore func( } } -func MockEnsuredDownloadsCleaned(m *SnapManager, ensured bool) (restore func()) { - old := m.ensuredDownloadsCleaned - m.ensuredDownloadsCleaned = ensured - return func() { - m.ensuredDownloadsCleaned = old - } +func SetEnsuredDownloadsCleanedNext(m *SnapManager, next time.Time) { + m.ensuredDownloadsCleanedNext = next +} + +func GetEnsuredDownloadsCleanedNext(m *SnapManager) time.Time { + return m.ensuredDownloadsCleanedNext } func MockPidsOfSnap(f func(instanceName string) (map[string][]int, error)) func() { diff --git a/overlord/snapstate/handlers.go b/overlord/snapstate/handlers.go index 149f94b1137..0357fdfdea9 100644 --- a/overlord/snapstate/handlers.go +++ b/overlord/snapstate/handlers.go @@ -909,8 +909,9 @@ func (m *SnapManager) doDownloadSnap(t *state.Task, tomb *tomb.Tomb) error { iconURL := snapsup.Media.IconURL() dlOpts := &store.DownloadOptions{ - Scheduled: snapsup.IsAutoRefresh, - RateLimit: rate, + Scheduled: snapsup.IsAutoRefresh, + RateLimit: rate, + LeavePartialOnError: true, } if snapsup.DownloadInfo == nil { vsets, err := EnforcedValidationSets(st) @@ -1025,8 +1026,9 @@ func (m *SnapManager) doPreDownloadSnap(t *state.Task, tomb *tomb.Tomb) error { targetFn := snapsup.BlobPath() dlOpts := &store.DownloadOptions{ // pre-downloads are only triggered in auto-refreshes - Scheduled: true, - RateLimit: autoRefreshRateLimited(st), + Scheduled: true, + RateLimit: autoRefreshRateLimited(st), + LeavePartialOnError: true, } perfTimings := state.TimingsForTask(t) diff --git a/overlord/snapstate/handlers_components.go b/overlord/snapstate/handlers_components.go index 81c4ec82418..aaa67fbd5df 100644 --- a/overlord/snapstate/handlers_components.go +++ b/overlord/snapstate/handlers_components.go @@ -201,8 +201,9 @@ func (m *SnapManager) doDownloadComponent(t *state.Task, tomb *tomb.Tomb) error timings.Run(perf, "download", fmt.Sprintf("download component %q", compsup.ComponentName()), func(timings.Measurer) { compRef := compsup.CompSideInfo.Component.String() opts := &store.DownloadOptions{ - Scheduled: snapsup.IsAutoRefresh, - RateLimit: rate, + Scheduled: snapsup.IsAutoRefresh, + RateLimit: rate, + LeavePartialOnError: true, } err = sto.Download(tomb.Context(nil), compRef, target, compsup.DownloadInfo, meter, user, opts) diff --git a/overlord/snapstate/handlers_components_download_test.go b/overlord/snapstate/handlers_components_download_test.go index 945e9920190..acb5d2ca046 100644 --- a/overlord/snapstate/handlers_components_download_test.go +++ b/overlord/snapstate/handlers_components_download_test.go @@ -35,6 +35,9 @@ func (s *downloadComponentSuite) SetUpTest(c *C) { s.fakeStore = &fakeStore{ state: s.state, fakeBackend: s.fakeBackend, + expectedDefaultDownloadOpts: &store.DownloadOptions{ + LeavePartialOnError: true, + }, } s.state.Lock() @@ -139,8 +142,9 @@ func (s *downloadComponentSuite) testDoDownloadComponent(c *C, opts testDoDownlo var downloadOpts *store.DownloadOptions if opts.autoRefresh { downloadOpts = &store.DownloadOptions{ - RateLimit: 1234, - Scheduled: true, + RateLimit: 1234, + Scheduled: true, + LeavePartialOnError: true, } } diff --git a/overlord/snapstate/handlers_download_test.go b/overlord/snapstate/handlers_download_test.go index 966b1367843..5979279d292 100644 --- a/overlord/snapstate/handlers_download_test.go +++ b/overlord/snapstate/handlers_download_test.go @@ -53,6 +53,9 @@ func (s *downloadSnapSuite) SetUpTest(c *C) { s.fakeStore = &fakeStore{ state: s.state, fakeBackend: s.fakeBackend, + expectedDefaultDownloadOpts: &store.DownloadOptions{ + LeavePartialOnError: true, + }, } s.state.Lock() defer s.state.Unlock() @@ -514,8 +517,9 @@ func (s *downloadSnapSuite) TestDoDownloadRateLimitedIntegration(c *C) { name: "foo", target: filepath.Join(dirs.SnapBlobDir, "foo_11.snap"), opts: &store.DownloadOptions{ - RateLimit: 1234, - Scheduled: true, + RateLimit: 1234, + Scheduled: true, + LeavePartialOnError: true, }, }, }) diff --git a/overlord/snapstate/snapmgr.go b/overlord/snapstate/snapmgr.go index c2efdfc4218..c8383227a9f 100644 --- a/overlord/snapstate/snapmgr.go +++ b/overlord/snapstate/snapmgr.go @@ -87,10 +87,10 @@ type SnapManager struct { preseed bool - ensuredMountsUpdated bool - ensuredDesktopFilesUpdated bool - ensuredDownloadsCleaned bool - ensureStoreCacheCleanNext time.Time + ensuredMountsUpdated bool + ensuredDesktopFilesUpdated bool + ensuredDownloadsCleanedNext time.Time + ensureStoreCacheCleanNext time.Time changeCallbackID int } @@ -822,7 +822,6 @@ func Manager(st *state.State, runner *state.TaskRunner) (*SnapManager, error) { preseed: preseed, ensuredMountsUpdated: false, ensuredDesktopFilesUpdated: false, - ensuredDownloadsCleaned: false, } if preseed { m.backend = backend.NewForPreseedMode() @@ -1601,10 +1600,6 @@ func (m *SnapManager) ensureDownloadsCleaned() error { m.state.Lock() defer m.state.Unlock() - if m.ensuredDownloadsCleaned { - return nil - } - // only run after we are seeded var seeded bool err := m.state.Get("seeded", &seeded) @@ -1615,13 +1610,19 @@ func (m *SnapManager) ensureDownloadsCleaned() error { return nil } + now := timeNow() + + if !m.ensuredDownloadsCleanedNext.IsZero() && m.ensuredDownloadsCleanedNext.After(now) { + return nil + } + logger.Trace("ensure", "manager", "SnapManager", "func", "ensureDownloadsCleaned") if err := cleanDownloads(m.state); err != nil { return err } - m.ensuredDownloadsCleaned = true + m.ensuredDownloadsCleanedNext = now.Add(maxUnusedDownloadRetention / 4) return nil } diff --git a/overlord/snapstate/snapstate.go b/overlord/snapstate/snapstate.go index e0ebeb3db85..9842d330dde 100644 --- a/overlord/snapstate/snapstate.go +++ b/overlord/snapstate/snapstate.go @@ -28,7 +28,6 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "sort" "strings" "time" @@ -4104,23 +4103,33 @@ func downloadsToKeep(st *state.State) (map[string]bool, error) { } var downloadsToKeep map[string]bool - keep := func(name string, rev snap.Revision) { + + keepBlob := func(blobPath string) { + if blobPath == "" { + return + } + if downloadsToKeep == nil { downloadsToKeep = make(map[string]bool) } - downloadsToKeep[fmt.Sprintf("%s_%s.snap", name, rev)] = true + downloadsToKeep[filepath.Base(blobPath)] = true } // keep revisions in snap's sequence for snapName, snapst := range snapStates { - for _, si := range snapst.Sequence.SideInfos() { - keep(snapName, si.Revision) + for _, rss := range snapst.Sequence.Revisions { + keepBlob(snap.MountFile(snapName, rss.Snap.Revision)) + for _, comp := range rss.Components { + cpi := snap.MinimalComponentContainerPlaceInfo(comp.SideInfo.Component.ComponentName, + comp.SideInfo.Revision, snapName) + keepBlob(cpi.MountFile()) + } } } // keep revisions in refresh hints for snapName, hint := range refreshHints { - keep(snapName, hint.Revision()) + keepBlob(snap.MountFile(snapName, hint.Revision())) } // keep revisions pointed to by a download task in an ongoing change @@ -4129,14 +4138,30 @@ func downloadsToKeep(st *state.State) (map[string]bool, error) { continue } for _, t := range chg.Tasks() { - if t.Kind() != "download-snap" { - continue - } - snapsup, err := TaskSnapSetup(t) - if err != nil { - return nil, err + switch t.Kind() { + case "download-snap": + snapsup, err := TaskSnapSetup(t) + if err != nil { + return nil, err + } + + keepBlob(snapsup.BlobPath()) + case "download-component": + compsup, snapsup, err := TaskComponentSetup(t) + if err != nil { + return nil, err + } + keepBlob(snapsup.BlobPath()) + // component download sets CompPath at some point when the + // download task runs, which may, or may not have run already. + if compsup.CompPath == "" { + cpi := snap.MinimalComponentContainerPlaceInfo(compsup.ComponentName(), + compsup.Revision(), snapsup.InstanceName()) + keepBlob(cpi.MountFile()) + } else { + keepBlob(compsup.CompPath) + } } - keep(snapsup.InstanceName(), snapsup.Revision()) } } @@ -4169,14 +4194,30 @@ var cleanDownloads = func(st *state.State) error { return err } - matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*.snap")) - if err != nil { - return err + var blobs []string + for _, pattern := range []string{ + "*.snap", "*.snap.partial", // snaps + "*.comp", "*.comp.partial", // and their components + } { + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, pattern)) + if err != nil { + return err + } + blobs = append(blobs, matches...) } - for _, file := range matches { + + for _, file := range blobs { if keep[filepath.Base(file)] { continue } + + if targetFile, _, partial := strings.Cut(file, ".partial"); partial { + if keep[filepath.Base(targetFile)] && !osutil.FileExists(targetFile) { + // only keep the partial file if the target does not exist yet + continue + } + } + if rmErr := maybeRemoveSnapDownload(file); rmErr != nil { // continue deletion, report error in the end err = rmErr @@ -4195,19 +4236,26 @@ var cleanSnapDownloads = func(st *state.State, snapName string) error { return err } - regex := regexp.MustCompile(fmt.Sprintf("^%s_x?[0-9]+\\.snap$", snapName)) - matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, fmt.Sprintf("%s_*.snap", snapName))) if err != nil { return err } - for _, file := range matches { - if !regex.MatchString(filepath.Base(file)) { - continue - } + partial, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, fmt.Sprintf("%s_*.snap.partial", snapName))) + if err != nil { + return err + } + for _, file := range append(matches, partial...) { if keep[filepath.Base(file)] { continue } + + if targetFile, _, partial := strings.Cut(file, ".partial"); partial { + if keep[filepath.Base(targetFile)] && !osutil.FileExists(targetFile) { + // only keep the partial file if the target does not exist yet + continue + } + } + if rmErr := maybeRemoveSnapDownload(file); rmErr != nil { // continue deletion, report error in the end err = rmErr diff --git a/overlord/snapstate/snapstate_test.go b/overlord/snapstate/snapstate_test.go index 2e8512f30a4..aa600e32e54 100644 --- a/overlord/snapstate/snapstate_test.go +++ b/overlord/snapstate/snapstate_test.go @@ -204,6 +204,9 @@ func (s *snapmgrBaseTest) SetUpTest(c *C) { refreshRevnos: make(map[string]snap.Revision), idsToNames: make(map[string]string), namesToAssertedIDs: make(map[string]string), + expectedDefaultDownloadOpts: &store.DownloadOptions{ + LeavePartialOnError: true, + }, } // make tests work consistently also in containers @@ -4003,7 +4006,7 @@ func (s *snapmgrTestSuite) TestEsnureCleansOldSideloads(c *C) { } // prevent removing snap file - defer snapstate.MockEnsuredDownloadsCleaned(s.snapmgr, true)() + snapstate.SetEnsuredDownloadsCleanedNext(s.snapmgr, time.Now().Add(time.Hour)) defer snapstate.MockLocalInstallCleanupWait(200 * time.Millisecond)() c.Assert(os.MkdirAll(dirs.SnapBlobDir, 0700), IsNil) @@ -10756,11 +10759,8 @@ Exec=%s/test-snap } func (s *snapmgrTestSuite) TestEnsureSnapStateDownloadsCleanedBlockedOnSeeding(c *C) { - restore := snapstate.MockEnsuredDownloadsCleaned(s.snapmgr, false) - defer restore() - called := 0 - restore = snapstate.MockCleanDownloads(func(st *state.State) error { + restore := snapstate.MockCleanDownloads(func(st *state.State) error { called++ return nil }) @@ -10779,7 +10779,17 @@ func (s *snapmgrTestSuite) TestEnsureSnapStateDownloadsCleanedBlockedOnSeeding(c } func (s *snapmgrTestSuite) TestEnsureSnapStateDownloadsCleaned(c *C) { - restore := snapstate.MockEnsuredDownloadsCleaned(s.snapmgr, false) + start := snapstate.GetEnsuredDownloadsCleanedNext(s.snapmgr) + c.Check(start.IsZero(), Equals, true) + + now := time.Now() + restore := snapstate.MockTimeNow(func() time.Time { + return now + }) + defer restore() + + mockedRetention := 4 * time.Hour + restore = snapstate.MockMaxUnusedDownloadRetention(mockedRetention) defer restore() called := 0 @@ -10789,13 +10799,32 @@ func (s *snapmgrTestSuite) TestEnsureSnapStateDownloadsCleaned(c *C) { }) defer restore() + c.Check(s.snapmgr.Ensure(), Equals, nil) + + // called once + c.Check(called, Equals, 1) + // simulate ensure called many times for i := 0; i < 5; i++ { c.Check(s.snapmgr.Ensure(), Equals, nil) } - // system-wide snap downloads cleaning should only run once - c.Check(called, Equals, 1) + // check that next is set reasonably + next := snapstate.GetEnsuredDownloadsCleanedNext(s.snapmgr) + c.Check(next, Equals, now.Add(mockedRetention/4)) + + restore = snapstate.MockTimeNow(func() time.Time { + return next.Add(time.Second) + }) + defer restore() + + c.Check(s.snapmgr.Ensure(), Equals, nil) + + // called again + c.Check(called, Equals, 2) + + next2 := snapstate.GetEnsuredDownloadsCleanedNext(s.snapmgr) + c.Check(next2.Equal(next.Add(time.Second).Add(mockedRetention/4)), Equals, true) } func (s *snapmgrTestSuite) TestSaveRefreshCandidatesOnAutoRefresh(c *C) { @@ -11891,6 +11920,7 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsSequences(c *C) { defer restore() initSnapDownloads(c, []snap.Revision{snap.R(1), snap.R(1111), snap.R(2), snap.R(3)}) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_5.snap.partial"), nil, 0644), IsNil) snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ Active: true, @@ -11906,12 +11936,14 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsSequences(c *C) { err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // revision not in sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1111.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + // revisions not in sequence or partial downloads should be removed + c.Check(matches, DeepEquals, []string{ + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + }) } func (s *snapmgrTestSuite) TestCleanSnapDownloadsRefreshHint(c *C) { @@ -11922,6 +11954,9 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsRefreshHint(c *C) { defer restore() initSnapDownloads(c, []snap.Revision{snap.R(1), snap.R(11111), snap.R(2), snap.R(3), snap.R(4)}) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "other-snap_4.snap"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "other-snap_5.snap.partial"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "other-snap_6.snap.partial"), nil, 0644), IsNil) snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ Active: true, @@ -11933,6 +11968,16 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsRefreshHint(c *C) { Current: snap.R(3), SnapType: "app", }) + snapstate.Set(s.state, "other-snap", &snapstate.SnapState{ + Active: true, + Sequence: sequence.SnapSequence{ + Revisions: []*sequence.RevisionSideState{ + {Snap: &snap.SideInfo{RealName: "other-snap", SnapID: "other-snap-id", Revision: snap.R(4)}}, + }, + }, + Current: snap.R(4), + SnapType: "app", + }) refreshHints := map[string]*snapstate.RefreshCandidate{ "some-snap": { SnapSetup: snapstate.SnapSetup{ @@ -11943,19 +11988,52 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsRefreshHint(c *C) { }, }, }, + "other-snap": { + SnapSetup: snapstate.SnapSetup{ + Type: "app", + SideInfo: &snap.SideInfo{ + RealName: "other-snap", + Revision: snap.R(5), + }, + }, + }, } s.state.Set("refresh-candidates", refreshHints) + // check the first snap err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // revisions not in refresh hint or sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_11111.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) - // revisions in refresh hint should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_4.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + // revisions of some-snap not in refresh hint or sequence should be removed + c.Check(matches, DeepEquals, []string{ + // other snaps are untouched + filepath.Join(dirs.SnapBlobDir, "other-snap_4.snap"), + filepath.Join(dirs.SnapBlobDir, "other-snap_5.snap.partial"), + filepath.Join(dirs.SnapBlobDir, "other-snap_6.snap.partial"), + + // exists in sequence + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + // fully downloaded revision in refresh hint + filepath.Join(dirs.SnapBlobDir, "some-snap_4.snap"), + }) + + // and now the other snap + err = snapstate.CleanSnapDownloads(s.state, "other-snap") + c.Check(err, IsNil) + matches, err = filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + // revisions of other-snap not in refresh hint or sequence should be removed + c.Check(matches, DeepEquals, []string{ + // revisions in sequence should be kept + filepath.Join(dirs.SnapBlobDir, "other-snap_4.snap"), + // as well as partial downloads of snaps in refresh hints + filepath.Join(dirs.SnapBlobDir, "other-snap_5.snap.partial"), + + // other snaps are untouched + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_4.snap"), + }) } func (s *snapmgrTestSuite) TestCleanSnapDownloadsOngoingChange(c *C) { @@ -12001,14 +12079,15 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsOngoingChange(c *C) { err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // revisions in a finished change should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FileAbsent) - // revisions pointed to by a pre-download task don't count and should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FilePresent) - // revisions pointed to by a download task in an ongoing change should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_4.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + // revisions in a finished change, or pointed to by a pre-download task should be removed + c.Check(matches, DeepEquals, []string{ + // revisions in sequence should be kept + filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), + // revisions pointed to by a download task in an ongoing change should be kept + filepath.Join(dirs.SnapBlobDir, "some-snap_4.snap"), + }) } func (s *snapmgrTestSuite) TestCleanSnapDownloadsLocalRevisions(c *C) { @@ -12034,11 +12113,13 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsLocalRevisions(c *C) { err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // revision not in sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_x1.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_x2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_x3.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + // revisions in sequence should be kept + filepath.Join(dirs.SnapBlobDir, "some-snap_x2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_x3.snap"), + }) } func (s *snapmgrTestSuite) TestCleanSnapDownloadsParallelInstalls(c *C) { @@ -12051,7 +12132,6 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsParallelInstalls(c *C) { // parallel install downloads c.Assert(os.MkdirAll(dirs.SnapBlobDir, 0755), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_1_1.snap"), nil, 0644), IsNil) - c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_2_2.snap"), nil, 0644), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_3_x3.snap"), nil, 0644), IsNil) snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ @@ -12064,13 +12144,38 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsParallelInstalls(c *C) { Current: snap.R(4), SnapType: "app", }) + snapstate.Set(s.state, "some-snap_1", &snapstate.SnapState{ + Active: true, + Sequence: sequence.SnapSequence{ + Revisions: []*sequence.RevisionSideState{ + {Snap: &snap.SideInfo{RealName: "some-snap", SnapID: "some-snap-id", Revision: snap.R(1)}}, + }, + }, + Current: snap.R(1), + SnapType: "app", + InstanceKey: "1", + }) + snapstate.Set(s.state, "some-snap_3", &snapstate.SnapState{ + Active: true, + Sequence: sequence.SnapSequence{ + Revisions: []*sequence.RevisionSideState{ + {Snap: &snap.SideInfo{RealName: "some-snap", SnapID: "some-snap-id", Revision: snap.R(-3)}}, + }, + }, + Current: snap.R(-3), + SnapType: "app", + InstanceKey: "3", + }) err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // parallel installs should not be affected - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1_1.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3_x3.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + // parallel installs should not be affected + filepath.Join(dirs.SnapBlobDir, "some-snap_1_1.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3_x3.snap"), + }) } func (s *snapmgrTestSuite) TestCleanSnapDownloadsKeepsNewDownloads(c *C) { @@ -12097,10 +12202,14 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsKeepsNewDownloads(c *C) { err := snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) // all snaps will be kept because retention period is still going - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1111.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_1111.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + }) // simulate retention period passed restore = snapstate.MockMaxUnusedDownloadRetention(0) @@ -12108,12 +12217,13 @@ func (s *snapmgrTestSuite) TestCleanSnapDownloadsKeepsNewDownloads(c *C) { err = snapstate.CleanSnapDownloads(s.state, "some-snap") c.Check(err, IsNil) - // revision not in sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1111.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + matches, err = filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + // parallel installs should not be affected + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + }) } func (s *snapmgrTestSuite) TestCleanDownloads(c *C) { @@ -12125,11 +12235,17 @@ func (s *snapmgrTestSuite) TestCleanDownloads(c *C) { // check that we delete leftovers of non-existing snaps c.Assert(os.MkdirAll(dirs.SnapBlobDir, 0755), IsNil) + // revision not in sequence should be removed c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), nil, 0644), IsNil) + // both files will be kept as revisions are present in the state c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), nil, 0644), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), nil, 0644), IsNil) + // unlikely but a duplicate of a fully downloaded snap + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap.partial"), nil, 0644), IsNil) + // all of the rest goes away c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), nil, 0644), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-yet-another-snap_1.snap.partial"), nil, 0644), IsNil) snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ Active: true, @@ -12145,14 +12261,13 @@ func (s *snapmgrTestSuite) TestCleanDownloads(c *C) { err := snapstate.CleanDownloads(s.state) c.Check(err, IsNil) - // leftovers from non-existing snaps should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), testutil.FileAbsent) - // revision not in sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + }) } func (s *snapmgrTestSuite) TestCleanDownloadsKeepsNewDownloads(c *C) { @@ -12166,6 +12281,7 @@ func (s *snapmgrTestSuite) TestCleanDownloadsKeepsNewDownloads(c *C) { c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), nil, 0644), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), nil, 0644), IsNil) c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-yet-another-snap_1.snap.partial"), nil, 0644), IsNil) snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ Active: true, @@ -12185,11 +12301,16 @@ func (s *snapmgrTestSuite) TestCleanDownloadsKeepsNewDownloads(c *C) { err := snapstate.CleanDownloads(s.state) c.Check(err, IsNil) // all snaps will be kept because retention period is still going - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), + filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + filepath.Join(dirs.SnapBlobDir, "some-yet-another-snap_1.snap.partial"), + }) // simulate retention period passed restore = snapstate.MockMaxUnusedDownloadRetention(0) @@ -12197,14 +12318,85 @@ func (s *snapmgrTestSuite) TestCleanDownloadsKeepsNewDownloads(c *C) { err = snapstate.CleanDownloads(s.state) c.Check(err, IsNil) - // leftovers from non-existing snaps should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), testutil.FileAbsent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-other-other-snap_1.snap"), testutil.FileAbsent) - // revision not in sequence should be removed - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_1.snap"), testutil.FileAbsent) - // revisions in sequence should be kept - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), testutil.FilePresent) - c.Check(filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), testutil.FilePresent) + + matches, err = filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + // revisions in sequence should be kept + filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), + filepath.Join(dirs.SnapBlobDir, "some-snap_3.snap"), + }) +} + +func (s *snapmgrTestSuite) TestCleanDownloadsKeepsPendingDownloads(c *C) { + s.state.Lock() + defer s.state.Unlock() + + // check that we delete leftovers of non-existing snaps + c.Assert(os.MkdirAll(dirs.SnapBlobDir, 0755), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap_2.snap"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "other-snap_11.snap.partial"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-other-snap_1.snap"), nil, 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(dirs.SnapBlobDir, "some-snap+standard-component_3.comp.partial"), nil, 0644), IsNil) + + chgSnap := s.state.NewChange("install", "install a snap") + ts, err := snapstate.Install(context.Background(), s.state, "other-snap", nil, s.user.ID, snapstate.Flags{}) + c.Assert(err, IsNil) + chgSnap.AddAll(ts) + + snapRev := snap.R(1) + compName := "standard-component" + info := createTestSnapInfoForComponent(c, "some-snap", snapRev, compName) + + setStateWithOneSnap(s.state, "some-snap", snapRev) + s.fakeStore.snapResourcesFn = func(info *snap.Info) []store.SnapResourceResult { + return []store.SnapResourceResult{{ + DownloadInfo: snap.DownloadInfo{ + DownloadURL: "http://example.com/" + compName, + }, + Name: compName, + Revision: 3, + Type: "component/standard", + Version: "1.0", + CreatedAt: "2024-01-01T00:00:00Z", + }} + } + + chgComp := s.state.NewChange("install", "install a component") + tss, err := snapstate.InstallComponents(context.Background(), s.state, + []string{compName}, info, nil, snapstate.Options{UserID: s.user.ID}) + c.Assert(err, IsNil) + for _, ts := range tss { + chgComp.AddAll(ts) + } + + restore := snapstate.MockMaxUnusedDownloadRetention(0) + defer restore() + + err = snapstate.CleanDownloads(s.state) + c.Check(err, IsNil) + // the partial file is kept as we have a pending operation, other files are gone + matches, err := filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, DeepEquals, []string{ + filepath.Join(dirs.SnapBlobDir, "other-snap_11.snap.partial"), + filepath.Join(dirs.SnapBlobDir, "some-snap+standard-component_3.comp.partial"), + }) + + // abort both changes, making partial files no longer needed + chgSnap.Abort() + c.Check(chgSnap.IsReady(), Equals, true) + + chgComp.Abort() + c.Check(chgComp.IsReady(), Equals, true) + + // clean again, the partial file should be removed + err = snapstate.CleanDownloads(s.state) + c.Check(err, IsNil) + // all snaps will be kept because retention period is still going + matches, err = filepath.Glob(filepath.Join(dirs.SnapBlobDir, "*")) + c.Assert(err, IsNil) + c.Check(matches, HasLen, 0) } func (s *snapmgrTestSuite) TestRefreshInhibitProceedTime(c *C) { diff --git a/overlord/snapstate/snapstate_update_test.go b/overlord/snapstate/snapstate_update_test.go index 3461815e7c3..d4ba08847eb 100644 --- a/overlord/snapstate/snapstate_update_test.go +++ b/overlord/snapstate/snapstate_update_test.go @@ -12602,7 +12602,25 @@ func (s *snapmgrTestSuite) TestPreDownloadTaskContinuesAutoRefreshIfSoftCheckOk( // check that the auto-refresh was completed without asking the store for refresh info c.Assert(s.fakeBackend.ops.Count("storesvc-snap-action"), Equals, 0) - c.Assert(s.fakeStore.downloads, HasLen, 2) + c.Assert(s.fakeBackend.ops.Count("storesvc-download"), Equals, 2) + c.Assert(s.fakeStore.downloads, DeepEquals, []fakeDownload{ + { + name: "foo", + target: filepath.Join(dirs.SnapBlobDir, "foo_2.snap"), + opts: &store.DownloadOptions{ + Scheduled: true, + LeavePartialOnError: true, + }, + }, + { + name: "foo", + target: filepath.Join(dirs.SnapBlobDir, "foo_2.snap"), + opts: &store.DownloadOptions{ + Scheduled: true, + LeavePartialOnError: true, + }, + }, + }) } func findChange(st *state.State, kind string) *state.Change { diff --git a/tests/main/proxy-no-core/task.yaml b/tests/main/proxy-no-core/task.yaml index 05b4bf1590b..f9fdf145a06 100644 --- a/tests/main/proxy-no-core/task.yaml +++ b/tests/main/proxy-no-core/task.yaml @@ -8,7 +8,12 @@ details: | an installed core (to install core). # only needs a test on classic -systems: [ubuntu-16.04-64, ubuntu-18.04-64] +systems: [ubuntu-16.04-64, ubuntu-18.04-64, ubuntu-24.04-64] + +skip: + - reason: requires python3 + if: | + not command -v python3 prepare: | @@ -63,11 +68,6 @@ restore: | systemctl stop tinyproxy || true execute: | - if ! command -v python3; then - echo "SKIP: need python3" - exit 0 - fi - # We need the tiny proxy just when snapd does not connect to the store through a real proxy PROXY="$HTTPS_PROXY" if [ "${SNAPD_USE_PROXY:-}" != true ]; then diff --git a/tests/main/resume-partial-snap-downloads/task.yaml b/tests/main/resume-partial-snap-downloads/task.yaml new file mode 100644 index 00000000000..dc27911102b --- /dev/null +++ b/tests/main/resume-partial-snap-downloads/task.yaml @@ -0,0 +1,111 @@ +summary: Check that snap download resumes after interruption + +details: | + Verify that when a snap download is interrupted mid-way, the partial + download file is preserved and the subsequent download attempt resumes + from where it left off using an HTTP Range header. + + The test uses the fakestore debug endpoint to force-kill the TCP + connection after a configured number of bytes have been served, + simulating a network interruption during download. + +# ubuntu-14.04: systemd-run not supported +# ubuntu-core: needs curl +# TODO implement fakestore CLI for debug API calls +systems: [-ubuntu-14.04*, -ubuntu-core-*] + +kill-timeout: 5m + +environment: + BLOB_DIR: $(pwd)/fake-store-blobdir + +skip: + - reason: This test needs test keys to be trusted + if: | + [ "$TRUST_TEST_KEYS" = "false" ] + +restore: | + snap remove --purge test-snap-resume || true + "$TESTSTOOLS"/store-state teardown-fake-store "$BLOB_DIR" + rm -rf test-snap test-snap-resume_1.0_all.snap + rm -f /var/lib/snapd/snaps/test-snap-resume_*.snap.partial + +debug: | + ls -la /var/lib/snapd/snaps/test-snap-resume_* 2>/dev/null || true + "$TESTSTOOLS"/journal-state get-log -u fakestore | tail -50 || true + +execute: | + "$TESTSTOOLS"/store-state setup-fake-store "$BLOB_DIR" + + snap ack "$TESTSLIB/assertions/testrootorg-store.account-key" + + # Create a snap large enough for partial download testing. + # The snap needs to span multiple HTTP write chunks (~32KB each) + # so the fakestore kill-after mechanism can interrupt mid-transfer. + mkdir -p test-snap/meta + cat > test-snap/meta/snap.yaml << 'EOF' + name: test-snap-resume + version: 1.0 + summary: Test snap for download resume + description: A test snap with padding data for resume testing + EOF + # Using urandom data ensures squashfs cannot compress it away + dd if=/dev/urandom of=test-snap/data.bin bs=1M count=100 2>/dev/null + snap pack test-snap + + snap_file="test-snap-resume_1.0_all.snap" + snap_size=$(stat -c %s "$snap_file") + echo "Snap file size: $snap_size" + + "$TESTSTOOLS"/store-state make-snap-installable "$BLOB_DIR" "$snap_file" + + # Tell the fakestore to kill the download connection after 20MB. + # This ensures some successful write chunks are delivered before the + # connection is severed, leaving a meaningful partial file. + kill_after=20971520 + curl --fail -s -X POST http://localhost:11028/debug \ + -d "{\"action\":\"kill-request\",\"kill-path\":\"/download/$snap_file\",\"kill-after\":$kill_after}" + + echo "First install attempt: expect failure due to interrupted download" + not snap install test-snap-resume + + # Confirm the fakestore actually killed the connection + "$TESTSTOOLS"/journal-state get-log -u fakestore | MATCH "/download/$snap_file was force killed, quota exceeded" + + # Verify a partial download file was preserved. + # The fakestore assigns revision 1 by default. + partial_path="/var/lib/snapd/snaps/test-snap-resume_1.snap.partial" + test -f "$partial_path" + + partial_size=$(stat -c %s "$partial_path") + test "$partial_size" -gt 0 + test "$partial_size" -lt "$snap_size" + echo "Partial download preserved: $partial_size bytes of $snap_size total" + partial_inode="$(stat -c '%i' "$partial_path")" + mod_time="$(stat -c '%y' "$partial_path")" + + # Reset error injection + curl --fail -s -X POST http://localhost:11028/debug \ + -d "{\"action\":\"reset\"}" + + # Make sure the logs for the fakestore are not polluted by previous snapd download + # retries starting from last cursor. + cursor=$("$TESTSTOOLS"/journal-state get-last-cursor) + + echo "Second install attempt: expect success with resumed download" + snap install test-snap-resume + + # Verify the fakestore received a Range header on the resumed download, + # confirming that snapd resumed from the partial file offset + "$TESTSTOOLS"/journal-state get-log-from-cursor "$cursor" -u fakestore | MATCH "requested range for /download/$snap_file" + + # Verify snap is installed + snap list test-snap-resume + + complete_path="/var/lib/snapd/snaps/test-snap-resume_1.snap" + test -f "$complete_path" + complete_inode="$(stat -c '%i' "$complete_path")" + complete_mod_time="$(stat -c '%y' "$complete_path")" + # same inode proves, file was renamed, not removed an recreated + test "$partial_inode" = "$complete_inode" + test "$mod_time" != "$complete_mod_time" From 7407b7daae8daebb4eb17c4ff20180d3f1c7f62f Mon Sep 17 00:00:00 2001 From: Nathnael Bekele <135308096+natibek@users.noreply.github.com> Date: Tue, 14 Apr 2026 10:03:41 -0500 Subject: [PATCH 20/44] daemon, snap, o/snapstate: extend snap.AlreadyInstalledError to multiple snaps and components (#16869) * daemon, snap, o/snapstate: update snap.AlreadyInstalledError to multiple snaps and components * o/snapstate: fix formating * o/snapstate: improve test coverage for changes * daemon, snap, o/snapstate: update AlreadyInstalledError.Error and implement error.Is for AlreadyInstalledError * daemon, snap: make sliceEquals independent of order and sort map keys before iteration * snap: add tests for Error and Is methods on AlreadyInstalledError * o/snapstate, snap: fixups * o/snapstate, snap: fixups * o/snapstate, snap: make slicesEqual compatible with slices.Equal and sort comps/snaps before creating AlreadyInstalledError * snap: sort components after assembling them in AlreadyInstalledError.Error * o/snapstate: collect already installed snaps to include them all in AlreadyInstalledError * many: use helper functions to generate AlreadyInstalledError with sorted snaps and components * snap: check that length of Components is the same for AlreadyInstalledError.Is * daemon, snap: update tests to use constructors for AlreadyInstalledError * snap: fix typo * daemon: fix formatting * daemon, snap: return pointer to AlreadyInstalledError from constructors --- daemon/api_snaps_test.go | 2 +- daemon/errors.go | 7 +- daemon/errors_test.go | 11 +- overlord/snapstate/component.go | 7 +- overlord/snapstate/component_install_test.go | 15 +- overlord/snapstate/snapstate_install_test.go | 48 ++++++- overlord/snapstate/target.go | 10 +- snap/errors.go | 108 +++++++++++++- snap/errors_test.go | 142 +++++++++++++++++++ 9 files changed, 332 insertions(+), 18 deletions(-) diff --git a/daemon/api_snaps_test.go b/daemon/api_snaps_test.go index e1eae1f931b..6c432812753 100644 --- a/daemon/api_snaps_test.go +++ b/daemon/api_snaps_test.go @@ -3047,7 +3047,7 @@ func (s *snapsSuite) TestErrToResponseNoSnapsDoesNotPanic(c *check.C) { &store.RevisionNotAvailableError{}, store.ErrNoUpdateAvailable, store.ErrLocalSnap, - &snap.AlreadyInstalledError{Snap: "foo"}, + &snap.AlreadyInstalledError{Snaps: []string{"foo"}}, &snap.NotInstalledError{Snap: "foo"}, &snapstate.SnapNeedsDevModeError{Snap: "foo"}, &snapstate.SnapNeedsClassicError{Snap: "foo"}, diff --git a/daemon/errors.go b/daemon/errors.go index cae22bdfe80..3fa418d01fb 100644 --- a/daemon/errors.go +++ b/daemon/errors.go @@ -377,8 +377,13 @@ func errToResponse(err error, snaps []string, fallback errorResponder, format st return InternalError("store.RevisionNotAvailable with %d snaps", len(snaps)) } case *snap.AlreadyInstalledError: + // TODO: handle error for multiple snaps and components kind = client.ErrorKindSnapAlreadyInstalled - snapName = err.Snap + if len(err.Snaps) == 1 { + snapName = err.Snaps[0] + } else { + handled = false + } case *snap.NotInstalledError: kind = client.ErrorKindSnapNotInstalled snapName = err.Snap diff --git a/daemon/errors_test.go b/daemon/errors_test.go index feb1e3ed01a..161397c9d5f 100644 --- a/daemon/errors_test.go +++ b/daemon/errors_test.go @@ -120,7 +120,10 @@ func (e fakeNetError) Timeout() bool { return e.timeout } func (e fakeNetError) Temporary() bool { return e.temporary } func (s *errorsSuite) TestErrToResponse(c *C) { - aie := &snap.AlreadyInstalledError{Snap: "foo"} + aieSnap := snap.NewAlreadyInstalledSnapsError([]string{"foo"}) + aieSnaps := snap.NewAlreadyInstalledSnapsError([]string{"foo", "bar"}) + aieComps := snap.NewAlreadyInstalledComponentsError("foo", []string{"comp1", "comp2"}) + aieSnapsComps := snap.NewAlreadyInstalledError([]string{"foo", "bar"}, map[string][]string{"foo": {"comp1", "comp2"}}) nie := &snap.NotInstalledError{Snap: "foo"} scce := &snapstate.ChangeConflictError{Snap: "foo"} ndme := &snapstate.SnapNeedsDevModeError{Snap: "foo"} @@ -159,7 +162,11 @@ func (s *errorsSuite) TestErrToResponse(c *C) { {store.ErrSnapNotFound, daemon.SnapNotFound("foo", store.ErrSnapNotFound), false}, {store.ErrNoUpdateAvailable, makeErrorRsp(client.ErrorKindSnapNoUpdateAvailable, store.ErrNoUpdateAvailable, ""), false}, {store.ErrLocalSnap, makeErrorRsp(client.ErrorKindSnapLocal, store.ErrLocalSnap, ""), false}, - {aie, makeErrorRsp(client.ErrorKindSnapAlreadyInstalled, aie, "foo"), false}, + {aieSnap, makeErrorRsp(client.ErrorKindSnapAlreadyInstalled, aieSnap, "foo"), false}, + // TODO: these should not be generic BadRequest errors + {aieSnaps, daemon.BadRequest("ERR: snaps \"bar,foo\" are already installed"), false}, + {aieComps, daemon.BadRequest("ERR: components \"foo+comp1,foo+comp2\" are already installed"), false}, + {aieSnapsComps, daemon.BadRequest("ERR: snaps \"bar,foo\" and components \"foo+comp1,foo+comp2\" are already installed"), false}, {nie, daemon.SnapNotInstalled("foo", nie), false}, {ndme, makeErrorRsp(client.ErrorKindSnapNeedsDevMode, ndme, "foo"), false}, {nc, makeErrorRsp(client.ErrorKindSnapNotClassic, nc, "foo"), false}, diff --git a/overlord/snapstate/component.go b/overlord/snapstate/component.go index 78aff753dd9..a290ca61fed 100644 --- a/overlord/snapstate/component.go +++ b/overlord/snapstate/component.go @@ -68,11 +68,16 @@ func InstallComponents( // we only check for already installed components when no validation // sets are provided, since this will allow us to refresh and install // new components at the same time when resolving validation sets + var alreadyInstalled []string for _, comp := range names { if snapst.CurrentComponentSideInfo(naming.NewComponentRef(info.SnapName(), comp)) != nil { - return nil, snap.AlreadyInstalledComponentError{Component: comp} + alreadyInstalled = append(alreadyInstalled, comp) } } + + if len(alreadyInstalled) > 0 { + return nil, snap.NewAlreadyInstalledComponentsError(info.SnapName(), alreadyInstalled) + } } revOpts := RevisionOptions{ diff --git a/overlord/snapstate/component_install_test.go b/overlord/snapstate/component_install_test.go index 73d4ba384f2..a978a701d7c 100644 --- a/overlord/snapstate/component_install_test.go +++ b/overlord/snapstate/component_install_test.go @@ -1115,8 +1115,9 @@ func (s *snapmgrTestSuite) TestInstallComponentsAlreadyInstalledError(c *C) { snapRev := snap.R(1) compNamesToType := map[string]string{ - "one": "test", - "two": "test", + "one": "test", + "two": "test", + "three": "test", } info := createTestSnapInfoForComponents(c, snapName, snapRev, compNamesToType) @@ -1139,6 +1140,11 @@ func (s *snapmgrTestSuite) TestInstallComponentsAlreadyInstalledError(c *C) { Revision: snap.R(1), }, snap.StandardComponent)) + seq.AddComponentForRevision(snapRev, sequence.NewComponentState(&snap.ComponentSideInfo{ + Component: naming.NewComponentRef(snapName, "two"), + Revision: snap.R(1), + }, snap.StandardComponent)) + snapstate.Set(s.state, snapName, &snapstate.SnapState{ Active: true, Sequence: seq, @@ -1146,9 +1152,10 @@ func (s *snapmgrTestSuite) TestInstallComponentsAlreadyInstalledError(c *C) { TrackingChannel: "channel-for-components", }) - _, err := snapstate.InstallComponents(context.TODO(), s.state, []string{"one", "two"}, info, nil, snapstate.Options{}) + _, err := snapstate.InstallComponents(context.TODO(), s.state, []string{"one", "two", "three"}, info, nil, snapstate.Options{}) - c.Assert(err, testutil.ErrorIs, snap.AlreadyInstalledComponentError{Component: "one"}) + expectedErr := snap.AlreadyInstalledError{Components: map[string][]string{snapName: {"one", "two"}}} + c.Assert(err, testutil.ErrorIs, expectedErr) } func (s *snapmgrTestSuite) TestInstallComponentsInvalidFlagAndTransaction(c *C) { diff --git a/overlord/snapstate/snapstate_install_test.go b/overlord/snapstate/snapstate_install_test.go index 7b375b89361..c085e84be76 100644 --- a/overlord/snapstate/snapstate_install_test.go +++ b/overlord/snapstate/snapstate_install_test.go @@ -356,8 +356,52 @@ func (s *snapmgrTestSuite) TestInstallAlreadyInstalled(c *C) { opts := &snapstate.RevisionOptions{Channel: "some-channel"} _, err := snapstate.Install(context.Background(), s.state, "some-snap", opts, 0, snapstate.Flags{}) c.Assert(err, NotNil) - c.Check(err, ErrorMatches, `snap "some-snap" is already installed`) - c.Check(err, FitsTypeOf, &snap.AlreadyInstalledError{}) + expectedErr := snap.AlreadyInstalledError{Snaps: []string{"some-snap"}} + c.Assert(err, testutil.ErrorIs, expectedErr) +} + +func (s *snapmgrTestSuite) TestInstallAlreadyInstalledMany(c *C) { + s.state.Lock() + defer s.state.Unlock() + + snapstate.Set(s.state, "some-snap", &snapstate.SnapState{ + Active: true, + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{ + {RealName: "some-snap", Revision: snap.R(7)}, + }), + Current: snap.R(7), + SnapType: "app", + }) + + snapstate.Set(s.state, "other-snap", &snapstate.SnapState{ + Active: true, + Sequence: snapstatetest.NewSequenceFromSnapSideInfos([]*snap.SideInfo{ + {RealName: "other-snap", Revision: snap.R(8)}, + }), + Current: snap.R(8), + SnapType: "app", + }) + + names := []string{"some-snap", "other-snap", "some-other-snap"} + var snaps []snapstate.StoreSnap + for _, name := range names { + sn := snapstate.StoreSnap{InstanceName: name} + snaps = append(snaps, sn) + } + target := snapstate.StoreInstallGoal(snaps...) + infos, tts, err := snapstate.InstallWithGoal(context.Background(), s.state, target, snapstate.Options{}) + + c.Assert(infos, IsNil) + c.Assert(tts, IsNil) + c.Assert(err, NotNil) + expectedErr := snap.AlreadyInstalledError{Snaps: []string{"other-snap", "some-snap"}} + c.Assert(err, testutil.ErrorIs, expectedErr) + + // Check that already installed snaps are skipped correctly with InstallMany + affected, tss, err := snapstate.InstallMany(s.state, names, nil, s.user.ID, nil) + c.Assert(err, IsNil) + c.Assert(affected, DeepEquals, []string{"some-other-snap"}) + c.Check(tss, HasLen, 2) } func (s *snapmgrTestSuite) TestInstallInvalidOptions(c *C) { diff --git a/overlord/snapstate/target.go b/overlord/snapstate/target.go index 7a6383c81b4..dce8f8a8341 100644 --- a/overlord/snapstate/target.go +++ b/overlord/snapstate/target.go @@ -741,6 +741,7 @@ func invalidComponentRevisionError(action, snapName, componentName string, sets func (s *storeInstallGoal) validateAndPrune(st *state.State, installedSnaps map[string]*SnapState, opts Options) error { enforcedSetsFunc := cachedEnforcedValidationSets(st) uninstalled := s.snaps[:0] + var alreadyInstalled []string for _, sn := range s.snaps { if err := snap.ValidateInstanceName(sn.InstanceName); err != nil { return fmt.Errorf("invalid instance name: %v", err) @@ -753,11 +754,15 @@ func (s *storeInstallGoal) validateAndPrune(st *state.State, installedSnaps map[ snapst, ok := installedSnaps[sn.InstanceName] if ok && snapst.IsInstalled() { if !sn.SkipIfPresent { - return &snap.AlreadyInstalledError{Snap: sn.InstanceName} + alreadyInstalled = append(alreadyInstalled, sn.InstanceName) } continue } + if len(alreadyInstalled) > 0 { + continue + } + // only provide a default the channel if the revision is not set, since // we don't want to prevent the user from installing a specific revision // that doesn't happen to exist in the "stable" risk @@ -775,6 +780,9 @@ func (s *storeInstallGoal) validateAndPrune(st *state.State, installedSnaps map[ uninstalled = append(uninstalled, sn) } + if len(alreadyInstalled) > 0 { + return snap.NewAlreadyInstalledSnapsError(alreadyInstalled) + } s.snaps = uninstalled diff --git a/snap/errors.go b/snap/errors.go index 359aef19efe..5435860257a 100644 --- a/snap/errors.go +++ b/snap/errors.go @@ -21,22 +21,118 @@ package snap import ( "fmt" + "sort" + "strings" ) +// Should not construct this error directly. Use NewAlreadyInstalledSnapsError, +// NewAlreadyInstalledComponentsError, or NewAlreadyInstalledError. type AlreadyInstalledError struct { - Snap string + Snaps []string + Components map[string][]string } func (e AlreadyInstalledError) Error() string { - return fmt.Sprintf("snap %q is already installed", e.Snap) + var comps []string + for snap, components := range e.Components { + for _, comp := range components { + comps = append(comps, SnapComponentName(snap, comp)) + } + } + sort.Strings(comps) + + builder := strings.Builder{} + if len(e.Snaps) == 1 { + fmt.Fprintf(&builder, "snap %q ", e.Snaps[0]) + } else if len(e.Snaps) > 1 { + fmt.Fprintf(&builder, "snaps %q ", strings.Join(e.Snaps, ",")) + } + + if len(e.Snaps) > 0 && len(comps) > 0 { + fmt.Fprintf(&builder, "and ") + } + + if len(comps) == 1 { + fmt.Fprintf(&builder, "component %q ", comps[0]) + } else if len(comps) > 1 { + fmt.Fprintf(&builder, "components %q ", strings.Join(comps, ",")) + } + + if len(e.Snaps)+len(comps) > 1 { + fmt.Fprintf(&builder, "are already installed") + } else { + fmt.Fprintf(&builder, "is already installed") + } + + return builder.String() } -type AlreadyInstalledComponentError struct { - Component string +func (e AlreadyInstalledError) Is(err error) bool { + other, ok := err.(AlreadyInstalledError) + if !ok { + return false + } + + if !slicesEqual(e.Snaps, other.Snaps) { + return false + } + + if len(e.Components) != len(other.Components) { + return false + } + + for snap, comps := range e.Components { + otherComps, ok := other.Components[snap] + if !ok || !slicesEqual(comps, otherComps) { + return false + } + } + return true +} + +// slicesEqual is a helper function to compare two slices for equality. +// TODO:GOVERSION 1.21: replace with slices.Equal +func slicesEqual[S []E, E comparable](a, b S) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func NewAlreadyInstalledSnapsError(snaps []string) *AlreadyInstalledError { + return NewAlreadyInstalledError(snaps, nil) +} + +func NewAlreadyInstalledComponentsError(snapName string, comps []string) *AlreadyInstalledError { + return NewAlreadyInstalledError(nil, map[string][]string{ + snapName: comps, + }) } -func (e AlreadyInstalledComponentError) Error() string { - return fmt.Sprintf("component %q is already installed", e.Component) +func NewAlreadyInstalledError(snaps []string, comps map[string][]string) *AlreadyInstalledError { + // sort snaps for use with .Is() + if len(snaps) > 0 { + sort.Strings(snaps) + } + + if len(comps) > 0 { + // sort components for use with .Is() + for _, scomps := range comps { + if len(scomps) > 0 { + sort.Strings(scomps) + } + } + } + + return &AlreadyInstalledError{ + Snaps: snaps, + Components: comps, + } } type NotInstalledError struct { diff --git a/snap/errors_test.go b/snap/errors_test.go index 1aa2f361f03..33b46af3c41 100644 --- a/snap/errors_test.go +++ b/snap/errors_test.go @@ -32,6 +32,148 @@ type errorsSuite struct{} var _ = Suite(&errorsSuite{}) +func (s *errorsSuite) TestAlreadyInstalledError(c *C) { + + for _, testCase := range []struct { + snaps []string + components map[string][]string + expectedStr string + }{ + { + []string{"some-snap"}, + nil, + `snap "some-snap" is already installed`, + }, + { + nil, + map[string][]string{"some-snap": {"comp"}}, + `component "some-snap\+comp" is already installed`, + }, + // check that snap names are sorted + { + []string{"some-snap", "other-snap"}, + nil, + `snaps "other-snap,some-snap" are already installed`, + }, + { + nil, + map[string][]string{"some-snap": {"comp1", "comp2"}}, + `components "some-snap\+comp1,some-snap\+comp2" are already installed`, + }, + { + nil, + map[string][]string{"some-snap": {"comp1"}, "other-snap": {"comp2"}}, + `components "other-snap\+comp2,some-snap\+comp1" are already installed`, + }, + { + []string{"some-snap", "other-snap"}, + map[string][]string{"some-snap": {"comp"}}, + `snaps "other-snap,some-snap" and component "some-snap\+comp" are already installed`, + }, + { + []string{"some-snap"}, + map[string][]string{"other-snap": {"comp1", "comp2"}}, + `snap "some-snap" and components "other-snap\+comp1,other-snap\+comp2" are already installed`, + }, + // check that component names are sorted + { + []string{"some-snap"}, + map[string][]string{"other-snap": {"comp"}, "some-other-snap": {"comp"}}, + `snap "some-snap" and components "other-snap\+comp,some-other-snap\+comp" are already installed`, + }, + { + []string{"some-snap", "other-snap"}, + map[string][]string{"other-snap": {"comp"}, "some-other-snap": {"comp"}}, + `snaps "other-snap,some-snap" and components "other-snap\+comp,some-other-snap\+comp" are already installed`, + }, + } { + err := snap.NewAlreadyInstalledError(testCase.snaps, testCase.components) + c.Check(err, ErrorMatches, testCase.expectedStr) + } + + err := snap.AlreadyInstalledError{ + Snaps: []string{"foo", "bar"}, + Components: map[string][]string{"some-snap": {"comp1", "comp2"}}, + } + c.Check(errors.Is(err, err), Equals, true) + + // Different error type should not match + c.Check(errors.Is(err, errors.New("some other error")), Equals, false) + // nil - should not match + c.Check(errors.Is(err, nil), Equals, false) + + // different snap order should not match + otherErr := snap.AlreadyInstalledError{ + Snaps: []string{"bar", "foo"}, + Components: map[string][]string{"some-snap": {"comp1", "comp2"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // different component order should not match + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"foo", "bar"}, + Components: map[string][]string{"some-snap": {"comp2", "comp1"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // different snaps should not match + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"foo"}, + Components: map[string][]string{"some-snap": {"comp1", "comp2"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // different snap for component should not match + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"foo", "bar"}, + Components: map[string][]string{"other-snap": {"comp1", "comp2"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // different number of components should not match + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"foo", "bar"}, + Components: map[string][]string{"some-snap": {"comp1", "comp2"}, "other-snap": {"comp"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // different components should not match + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"bar", "foo"}, + Components: map[string][]string{"some-snap": {"comp1"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // check that the error generated with snap.NewAlreadyInstalledError matches + otherErr2 := snap.NewAlreadyInstalledError([]string{"foo", "bar"}, map[string][]string{"some-snap": {"comp1"}}) + c.Check(errors.Is(otherErr, *otherErr2), Equals, true) + + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"foo", "bar"}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // check that snap.NewAlreadyInstalledSnapsError sorts the snaps in the resulting error + c.Check(errors.Is(otherErr, snap.NewAlreadyInstalledSnapsError([]string{"foo", "bar"})), Equals, false) + otherErr = snap.AlreadyInstalledError{ + Snaps: []string{"bar", "foo"}, + } + c.Check(errors.Is(otherErr, *snap.NewAlreadyInstalledSnapsError([]string{"foo", "bar"})), Equals, true) + + otherErr = snap.AlreadyInstalledError{ + Components: map[string][]string{"other-snap": {"comp2", "comp1"}}, + } + c.Check(errors.Is(err, otherErr), Equals, false) + + // check that snap.NewAlreadyInstalledComponentsError sorts the snaps in the resulting error + c.Check(errors.Is(otherErr, snap.NewAlreadyInstalledComponentsError("other-snap", []string{"comp1", "comp2"})), Equals, false) + otherErr = snap.AlreadyInstalledError{ + Components: map[string][]string{"other-snap": {"comp1", "comp2"}}, + } + c.Check(errors.Is(otherErr, *snap.NewAlreadyInstalledComponentsError("other-snap", []string{"comp1", "comp2"})), Equals, true) + +} + func (s *errorsSuite) TestNotSnapErrorNoDetails(c *C) { err := snap.NotSnapError{Path: "some-path"} c.Check(err, ErrorMatches, `cannot process snap or snapdir "some-path"`) From 8134fde2b80c96fd4acbc006bb5fc611a272244e Mon Sep 17 00:00:00 2001 From: Oliver Calder Date: Tue, 14 Apr 2026 10:26:47 -0500 Subject: [PATCH 21/44] o/i/apparmorprompting: improve unit test reliability on slow systems (#16892) * o/i/apparmorprompting: improve unit test reliability on slow systems Signed-off-by: Oliver Calder * fixup! o/i/apparmorprompting: improve unit test reliability on slow systems Signed-off-by: Oliver Calder * o/i/apparmorprompting: use HostScaledTimeout for slow concurrent test Signed-off-by: Oliver Calder * fixup! o/i/apparmorprompting: use HostScaledTimeout for slow concurrent test Signed-off-by: Oliver Calder --------- Signed-off-by: Oliver Calder --- .../apparmorprompting/noticebackend_test.go | 4 +-- .../apparmorprompting/prompting_test.go | 26 +++++++++++++------ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/overlord/ifacestate/apparmorprompting/noticebackend_test.go b/overlord/ifacestate/apparmorprompting/noticebackend_test.go index 1fa76b666a9..d5808c3ea2b 100644 --- a/overlord/ifacestate/apparmorprompting/noticebackend_test.go +++ b/overlord/ifacestate/apparmorprompting/noticebackend_test.go @@ -1111,7 +1111,7 @@ func (s *noticebackendSuite) TestBackendWaitNoticesConcurrent(c *C) { wg.Add(1) go func(i int) { defer wg.Done() - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), testutil.HostScaledTimeout(10*time.Second)) defer cancel() key := prompting.IDType(i).String() notices, err := ruleBackend.BackendWaitNotices(ctx, &state.NoticeFilter{Keys: []string{key}}) @@ -1133,7 +1133,7 @@ func (s *noticebackendSuite) TestBackendWaitNoticesConcurrent(c *C) { close(done) }() select { - case <-time.After(time.Second): + case <-time.After(testutil.HostScaledTimeout(15 * time.Second)): c.Fatalf("timed out waiting for BackendWaitNotices goroutines to finish") case <-done: } diff --git a/overlord/ifacestate/apparmorprompting/prompting_test.go b/overlord/ifacestate/apparmorprompting/prompting_test.go index 193bd99a77d..825f2a75972 100644 --- a/overlord/ifacestate/apparmorprompting/prompting_test.go +++ b/overlord/ifacestate/apparmorprompting/prompting_test.go @@ -265,6 +265,10 @@ func (s *apparmorpromptingSuite) TestHandleRequestErrors(c *C) { Path: fmt.Sprintf("/home/test/%d", i), Interface: "home", Permissions: []string{"write"}, + Reply: func([]string) error { + c.Error("unexpectedly called request.Reply()") + return errors.New("unexpectedly called request.Reply()") + }, } reqChan <- req } @@ -342,7 +346,7 @@ func (s *apparmorpromptingSuite) simulateRequest(c *C, reqChan chan *prompting.R defer restore() // Simulate request from the kernel - s.fillInPartialRequest(req) + s.fillInPartialRequest(c, req) whenSent := time.Now() // push a request reqChan <- req @@ -397,7 +401,7 @@ func (s *apparmorpromptingSuite) simulateRequest(c *C, reqChan chan *prompting.R // fillInPartialRequest fills in any blank fields from the given request // with default non-empty values. -func (s *apparmorpromptingSuite) fillInPartialRequest(req *prompting.Request) { +func (s *apparmorpromptingSuite) fillInPartialRequest(c *C, req *prompting.Request) { if req.PID == 0 { req.PID = 1234 } @@ -419,6 +423,12 @@ func (s *apparmorpromptingSuite) fillInPartialRequest(req *prompting.Request) { if req.Permissions == nil { req.Permissions = []string{"read"} } + if req.Reply == nil { + req.Reply = func(allowedPerms []string) error { + c.Errorf("unexpectedly called request.Reply(%#v)", allowedPerms) + return fmt.Errorf("unexpectedly called request.Reply(%#v)", allowedPerms) + } + } } var errNoReply = errors.New("no reply received") @@ -850,7 +860,7 @@ func (s *apparmorpromptingSuite) TestExistingRuleAllowsNewPrompt(c *C) { req, replyChan := requestWithReplyChan(&prompting.Request{ Permissions: []string{"read", "write"}, }) - s.fillInPartialRequest(req) + s.fillInPartialRequest(c, req) whenSent := time.Now() reqChan <- req time.Sleep(10 * time.Millisecond) @@ -943,7 +953,7 @@ func (s *apparmorpromptingSuite) TestExistingRulePartiallyDeniesNewPrompt(c *C) req, replyChan := requestWithReplyChan(&prompting.Request{ Permissions: []string{"read", "write"}, }) - s.fillInPartialRequest(req) + s.fillInPartialRequest(c, req) whenSent := time.Now() reqChan <- req time.Sleep(10 * time.Millisecond) @@ -992,7 +1002,7 @@ func (s *apparmorpromptingSuite) TestExistingRulesMixedMatchNewPromptDenies(c *C req, replyChan := requestWithReplyChan(&prompting.Request{ Permissions: []string{"read", "write"}, }) - s.fillInPartialRequest(req) + s.fillInPartialRequest(c, req) whenSent := time.Now() reqChan <- req time.Sleep(10 * time.Millisecond) @@ -1313,14 +1323,14 @@ func (s *apparmorpromptingSuite) testReplyRuleHandlesFuturePrompts(c *C, outcome writeReq, writeReplyChan := requestWithReplyChan(&prompting.Request{ Permissions: []string{"write"}, }) - s.fillInPartialRequest(writeReq) + s.fillInPartialRequest(c, writeReq) reqChan <- writeReq // Add request for read and write rwReq, rwReplyChan := requestWithReplyChan(&prompting.Request{ Permissions: []string{"read", "write"}, }) - s.fillInPartialRequest(rwReq) + s.fillInPartialRequest(c, rwReq) reqChan <- rwReq // Check that kernel received replies @@ -1851,7 +1861,7 @@ func (s *apparmorpromptingSuite) TestListenerReadyNotCausesPromptsHandleReadying Key: "kernel:2", UID: 1000, } - s.fillInPartialRequest(req) + s.fillInPartialRequest(c, req) reqChan <- req // Check that the prompts are not ready yet From eb3e254de49874498deac4ba83a7c7b247a20faa Mon Sep 17 00:00:00 2001 From: Sergio Cazzolato Date: Tue, 14 Apr 2026 17:08:58 -0300 Subject: [PATCH 22/44] tests: new tests to validated resealing on real hardware (#16821) * tests: new releasing test on real hardware with secboot enabled * tests: new tests to validated resealing on real hardware This change adds a new test to validate resealing feature in real hardware with tpm and secure-boot enabled. So far it is being validated in uc20 and uc22 as there are not available uc24 in the lab with secboot and which allows provisioning a new image for the test. Is is also included a new env var which indicates the systems hasn't be reset to run a new test. This is to avoid issues when resetting and making the execution faster on slow devices. * make the test manual to avoid execuion on vms * Fix shellcheck * Adding a comment about the backend and systems used * Chack kernel and snapd versions after reboot * Updated the env var name used to skip resets * Just reboot when the systems is waiting for this * remove exit used for testing * Support early refreshes * add missing en var for fde backend * removing SNAPD_SKIP_STATE_RESET * Make sure the auto-refresh is cancelled * restore deleted reset --- spread.yaml | 6 ++ tests/lib/prepare-restore.sh | 7 ++ tests/lib/spread/backend.testflinger.fde.yaml | 19 ++++++ tests/lib/tools/snapd-state | 23 +++++++ tests/nightly/snapd-resealing/task.yaml | 64 +++++++++++++++++++ 5 files changed, 119 insertions(+) create mode 100644 tests/lib/spread/backend.testflinger.fde.yaml create mode 100644 tests/nightly/snapd-resealing/task.yaml diff --git a/spread.yaml b/spread.yaml index 3f4abf8d04b..3c60ee3595c 100644 --- a/spread.yaml +++ b/spread.yaml @@ -66,7 +66,10 @@ environment: https_proxy: '$(HOST: echo "$SPREAD_HTTPS_PROXY")' NO_PROXY: '$(HOST: echo "${SPREAD_NO_PROXY:-127.0.0.1}")' no_proxy: '$(HOST: echo "${SPREAD_NO_PROXY:-127.0.0.1}")' + # use NEW_CORE_CHANNEL to control the core channel used in refresh tests NEW_CORE_CHANNEL: '$(HOST: echo "$SPREAD_NEW_CORE_CHANNEL")' + # use NEW_SNAPD_CHANNEL to control the snapd channel used in refresh tests + NEW_SNAPD_CHANNEL: '$(HOST: echo "${SPREAD_NEW_SNAPD_CHANNEL:-edge}")' SRU_VALIDATION: '$(HOST: echo "${SPREAD_SRU_VALIDATION:-0}")' # use the ppa_validation_name to install snapd from a public ppa PPA_VALIDATION_NAME: '$(HOST: echo "${SPREAD_PPA_VALIDATION_NAME:-}")' @@ -137,6 +140,9 @@ environment: SNAPD_LOG_SNAP_TO_JOURNAL: '$(HOST: echo "${SPREAD_SNAPD_LOG_SNAP_TO_JOURNAL:-$SPREAD_TAG_FEATURES}")' SNAPD_TRACE: '$(HOST: echo "${SPREAD_SNAPD_TRACE:-$SPREAD_TAG_FEATURES}")' + # Whether snapd should skip early refreshes. + SNAPD_SKIP_EARLY_REFRESH: '$(HOST: echo "${SPREAD_SNAPD_SKIP_EARLY_REFRESH:-}")' + backends: google: key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")' diff --git a/tests/lib/prepare-restore.sh b/tests/lib/prepare-restore.sh index 4b892b9a41f..1b9ba31f0f9 100755 --- a/tests/lib/prepare-restore.sh +++ b/tests/lib/prepare-restore.sh @@ -227,6 +227,13 @@ install_dependencies_gce_bucket(){ ### prepare_project() { + if [ "$SNAPD_SKIP_EARLY_REFRESH" = true ] && command -v snap >/dev/null 2>&1; then + "$TESTSTOOLS"/snapd-state cancel-autorefresh + + # Set a far future date to prevent automatic refreshes during the test execution. + snap set system refresh.hold="2050-01-01T00:00:00Z" + fi + if os.query is-ubuntu && os.query is-classic; then apt-get remove --purge -y lxd lxcfs || true apt-get autoremove --purge -y diff --git a/tests/lib/spread/backend.testflinger.fde.yaml b/tests/lib/spread/backend.testflinger.fde.yaml new file mode 100644 index 00000000000..08c8ad49a2a --- /dev/null +++ b/tests/lib/spread/backend.testflinger.fde.yaml @@ -0,0 +1,19 @@ + #this backend is used for resealing validation with secure boot enabled + testflinger: + wait-timeout: 30m + kill-timeout: 40m + halt-timeout: 1h + environment: + SNAPD_SKIP_EARLY_REFRESH: true + TRUST_TEST_KEYS: "false" + systems: + - ubuntu-core-20-64-nuc: + queue: dawson-i-uc20-fde + image: core20-latest-stable + username: ubuntu + password: ubuntu + - ubuntu-core-22-64-fde: + queue: uc22-fde + image: core22-latest-stable + username: ubuntu + password: ubuntu \ No newline at end of file diff --git a/tests/lib/tools/snapd-state b/tests/lib/tools/snapd-state index 9e7ce1b76ba..c2aa15d8c3d 100755 --- a/tests/lib/tools/snapd-state +++ b/tests/lib/tools/snapd-state @@ -49,6 +49,29 @@ check_state() { fi } +cancel_autorefresh() { + CHANGE_ID="$(snap changes | grep -E '.*( Do | Doing ).*Auto-refresh' | awk '{print $1}')" + if [ -n "$CHANGE_ID" ]; then + snap abort "$CHANGE_ID" + echo "snapd-state: aborting auto-refresh with change id $CHANGE_ID" + #shellcheck disable=SC2016 + if retry -n 60 --wait 2 --env CHANGE_ID="$CHANGE_ID" sh -c 'snap changes | grep -E ".*$CHANGE_ID.*( Undone | Error ).*Auto-refresh"'; then + echo "snapd-state: auto-refresh aborted successfully" + exit 0 + else + if snap changes | grep -E '.*$CHANGE_ID.* Done .*Auto-refresh'; then + echo "snapd-state: auto-refresh already completed, cannot abort" + exit 1 + fi + echo "snapd-state: timeout while waiting for auto-refresh to be aborted" + exit 1 + fi + else + echo "snapd-state: no auto-refresh in progress, nothing to abort" + exit 0 + fi +} + change_snap_channel() { local SNAP="$1" local CHANNEL="$2" diff --git a/tests/nightly/snapd-resealing/task.yaml b/tests/nightly/snapd-resealing/task.yaml new file mode 100644 index 00000000000..2a2fb74ade3 --- /dev/null +++ b/tests/nightly/snapd-resealing/task.yaml @@ -0,0 +1,64 @@ +summary: Check snapd reseals properly after it is refreshed and system rebooted + +details: | + This test checks that snapd reseals properly after it is refreshed and system rebooted. + It is expected that snapd will print "resealing .* succeeded" in the logs after the reboot. + This test is intended to be executed in the testflinger backend on real hardware. + +# This test is intended to be executed in the testflinger backend on real hardware. +# It is not expected to work in a VM environment. +# backends: [testflinger] + +systems: [ubuntu-core-*] + +manual: true + +environment: + SNAPD_REFRESH_CHANNEL: ${NEW_SNAPD_CHANNEL:-edge} + KERNEL_REFRESH_CHANNEL: beta + +skip: + - reason: Secure boot is not enabled in the current system + if: journalctl -b | NOMATCH "secureboot.*enabled" + +prepare: | + if [ "$SPREAD_REBOOT" = 0 ]; then + # In uc20 the refresh is triggering a reboot because of the kernel command lines is updated. + # This is expected when running in images with older snapd versions. + snap refresh snapd --revision 25202 --no-wait # this is the revision 2.71 amd64 + if retry -n 50 --wait 2 sh -c 'journalctl -b -u snapd | MATCH "Waiting for system reboot"'; then + echo "Snapd is waiting for system reboot, rebooting now..." + REBOOT + else + echo "Snapd is not waiting for system reboot" + fi + fi + +execute: | + snap list pc-kernel --unicode=never | awk 'NR==2 {print $3}' > kernel_rev_"$SPREAD_REBOOT".txt + snap list snapd --unicode=never | awk 'NR==2 {print $3}' > snapd_rev_"$SPREAD_REBOOT".txt + if [ "$SPREAD_REBOOT" = 0 ]; then + VERSION=$(grep 'VERSION_ID=' /etc/os-release | cut -d '"' -f 2) + snap refresh snapd --channel="latest/$SNAPD_REFRESH_CHANNEL" + snap refresh pc-kernel --channel="$VERSION/$KERNEL_REFRESH_CHANNEL" + REBOOT + fi + + # Check the kernel revision has changed after the reboot + if diff -q kernel_rev_0.txt kernel_rev_1.txt; then + echo "Kernel revision has not changed after reboot, which is unexpected" + exit 1 + else + echo "Kernel revision has changed after reboot, as expected" + fi + + # Check snapd revision has changed after the reboot + if diff -q snapd_rev_0.txt snapd_rev_1.txt; then + echo "Snapd revision has not changed after reboot, which is unexpected" + exit 1 + else + echo "Snapd revision has changed after reboot, as expected" + fi + + # Check snapd resealed properly after the reboot + "$TESTSTOOLS"/journal-state get-log -u snapd | MATCH "resealing .* succeeded" From ad1057305f33e34917c9d2a7a0d1cdcd5bd91bd5 Mon Sep 17 00:00:00 2001 From: Mohit Chachada Date: Tue, 14 Apr 2026 23:29:53 +0200 Subject: [PATCH 23/44] tests: add spread tests for snap remove impacted by mounts (#16887) * tests: add spread tests for snap remove impacted by mounts * .woke: ignore test file due to slave mount propagation name * tests: parallelize all tests using variants * tests: make variant names more descriptive * tests: adapt test variant management * tests: fix static check error --- .woke.yaml | 1 + .../test-snapd-mount-control/meta/snap.yaml | 3 + .../main/remove-impacted-by-mounts/task.yaml | 169 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 tests/main/remove-impacted-by-mounts/task.yaml diff --git a/.woke.yaml b/.woke.yaml index bb4f7346ccd..7783ec21d9d 100644 --- a/.woke.yaml +++ b/.woke.yaml @@ -32,3 +32,4 @@ ignore_files: - packaging/ubuntu-16.04/changelog - tests/lib/snaps/store/test-snapd-ovmf/snapcraft.yaml - tests/lib/tools/tests.session + - tests/main/remove-impacted-by-mounts/task.yaml diff --git a/tests/lib/snaps/test-snapd-mount-control/meta/snap.yaml b/tests/lib/snaps/test-snapd-mount-control/meta/snap.yaml index 897fc9bf845..e3e0a7126b4 100644 --- a/tests/lib/snaps/test-snapd-mount-control/meta/snap.yaml +++ b/tests/lib/snaps/test-snapd-mount-control/meta/snap.yaml @@ -11,6 +11,9 @@ plugs: where: $SNAP_COMMON/target1 options: [rw, bind] persistent: true + - what: /var/tmp/** + where: $SNAP_DATA/target1 + options: [rw, bind] - what: /dev/sd* where: /media/** type: [ext2, ext3, ext4] diff --git a/tests/main/remove-impacted-by-mounts/task.yaml b/tests/main/remove-impacted-by-mounts/task.yaml new file mode 100644 index 00000000000..09baa60cb0e --- /dev/null +++ b/tests/main/remove-impacted-by-mounts/task.yaml @@ -0,0 +1,169 @@ +summary: Tests for snap remove impacted by mounts + +details: | + Snap remove is impacted by mounts under the snap's global (SNAP_COMMON, SNAP_DATA) + and user (SNAP_USER_COMMON, SNAP_USER_DATA) data directories. + These mounts can be created, for example, by a snap using the mount-control interface + or by a user on classic systems. + The test checks that + - snap remove succeeds if the mounts under snap's global data directories + are visible only in the snap's namespace (and not in the host's namespace) + - _currently_ snap remove fails if the mounts under snap's global/user data + directories are visible in the host's namespace. This behavior needs to be fixed. + +environment: + SNAP_NAME: test-snapd-mount-control + SNAP_COMMON: /var/snap/$SNAP_NAME/common + SNAP_DATA: /var/snap/$SNAP_NAME/x1 + SNAP_USER_COMMON: /root/snap/$SNAP_NAME/common + SNAP_USER_DATA: /root/snap/$SNAP_NAME/x1 + # what: /var/tmp/** | where: $SNAP_/target1 | options: [rw, bind] + # mount-control interface does not allow mounts under snap's + # user directories (SNAP_USER_COMMON, SNAP_USER_DATA) + MOUNT_SRC: /var/tmp/$SNAP_NAME + MOUNT_DEST_GLOBAL_COMMON: $SNAP_COMMON/target1 + MOUNT_DEST_GLOBAL_DATA: $SNAP_DATA/target1 + MOUNT_DEST_USER_COMMON: $SNAP_USER_COMMON/target1 + MOUNT_DEST_USER_DATA: $SNAP_USER_DATA/target1 + # each test variant will execute: test_$CASE $MOUNT_DEST + CASE/v1,v2: remove_succeeds_when_slave_mount_in_snap_ns_under + CASE/v3,v4: remove_currently_fails_when_snapctl_mount_in_host_ns_under + CASE/v5,v6: remove_currently_fails_when_user_mount_in_host_ns_under + MOUNT_DEST/v1,v3: $MOUNT_DEST_GLOBAL_COMMON + MOUNT_DEST/v2,v4: $MOUNT_DEST_GLOBAL_DATA + MOUNT_DEST/v5: $MOUNT_DEST_USER_COMMON + MOUNT_DEST/v6: $MOUNT_DEST_USER_DATA + +restore: | + rm -rf "$MOUNT_SRC" "$MOUNT_DEST" + +execute: | + setup_src_data() { + mkdir -p "$MOUNT_SRC/dir1" + echo "Something" > "$MOUNT_SRC/file1" + } + + setup_test_snap() { + echo "Install the test snap with mount-control interface" + "$TESTSTOOLS"/snaps-state install-local "${SNAP_NAME}" + echo "Connect its mount-control interface" + snap connect "${SNAP_NAME}":mntctl + } + + when_slave_mount_in_snap_ns() { + local mount_dest=$1 + echo "Create a mount under snap's $mount_dest data directory in the snap's namespace" + "${SNAP_NAME}".cmd mkdir -p "$mount_dest" + "${SNAP_NAME}".cmd mount -o bind,rw "$MOUNT_SRC" "$mount_dest" + echo "Verify that the mount has been performed" + "${SNAP_NAME}".cmd grep "$mount_dest" /proc/self/mountinfo + echo "and that it's only in the snap's namespace" + NOMATCH "$mount_dest" < /proc/self/mountinfo + echo "Ensure that the mounted files are visible" + "${SNAP_NAME}".cmd test -e "$mount_dest/file1" + } + + then_remove_succeeds_ns_removed_src_data_intact() { + echo "Verify that snap remove succeeds" + snap remove "${SNAP_NAME}" + echo "and the snap's mount namespace is cleaned up" + test ! -e /run/snapd/ns/"${SNAP_NAME}".mnt + echo "but the mount's source data is still there" + test -e "$MOUNT_SRC/file1" + } + + test_remove_succeeds_when_slave_mount_in_snap_ns_under() { + local mount_dest=$1 + setup_src_data + setup_test_snap + when_slave_mount_in_snap_ns "$mount_dest" + then_remove_succeeds_ns_removed_src_data_intact + echo + } + + when_snapctl_mount_in_host_ns() { + local mount_dest=$1 + echo "Create a mount under snap's $mount_dest data directory in the host's namespace" + mkdir -p "$mount_dest" + "${SNAP_NAME}".cmd snapctl mount -o bind,rw "$MOUNT_SRC" "$mount_dest" + echo "Verify that the mount has been performed in the host's namespace" + MATCH "$mount_dest" < /proc/self/mountinfo + echo "and that it's also visible in the snap's namespace" + "${SNAP_NAME}".cmd grep "$mount_dest" /proc/self/mountinfo + echo "Ensure that the mounted files are visible" + test -e "$mount_dest/file1" + } + + then_remove_currently_fails_mount_remains_src_data_lost () { + local mount_dest=$1 + echo "Currently snap remove will fail due to the mount being present in the snap's data directory" + echo "THIS NEEDS TO BE FIXED!" + not snap remove "${SNAP_NAME}" + echo "and the mount is still active" + echo "THIS NEEDS TO BE FIXED!" + MATCH "$mount_dest" < /proc/self/mountinfo + echo "and currently the mount's source data is unfortunately removed" + echo "THIS NEEDS TO BE FIXED!" + test ! -e "$MOUNT_SRC/file1" + } + + cleaning_up_snapctl_mount_should_make_remove_successful() { + local mount_dest=$1 + echo "Below steps can be removed once the above snap remove issues are fixed" + echo "Enable the snap (if required) after a failed remove to allow cleanup" + snap enable "${SNAP_NAME}" || true + echo "Unmount via snapctl" + "${SNAP_NAME}".cmd snapctl umount "$mount_dest" + echo "and unmount should have succeeded" + NOMATCH "$mount_dest" < /proc/self/mountinfo + echo "Now snap remove should succeed" + snap remove "${SNAP_NAME}" + } + + test_remove_currently_fails_when_snapctl_mount_in_host_ns_under() { + local mount_dest=$1 + setup_src_data + setup_test_snap + when_snapctl_mount_in_host_ns "$mount_dest" + then_remove_currently_fails_mount_remains_src_data_lost "$mount_dest" + cleaning_up_snapctl_mount_should_make_remove_successful "$mount_dest" + echo + } + + when_user_mount_in_host_ns() { + local mount_dest=$1 + echo "Create a mount under snap's $mount_dest data directory in the host's namespace" + mkdir -p "$mount_dest" + mount -o bind,rw "$MOUNT_SRC" "$mount_dest" + echo "Verify that the mount has been performed in the host's namespace" + MATCH "$mount_dest" < /proc/self/mountinfo + echo "and that it's also visible in the snap's namespace" + "${SNAP_NAME}".cmd grep "$mount_dest" /proc/self/mountinfo + echo "Ensure that the mounted files are visible" + test -e "$mount_dest/file1" + } + + cleaning_up_user_mount_should_make_remove_successful() { + local mount_dest=$1 + echo "Below steps can be removed once the above snap remove issues are fixed" + echo "Unmount the user created mount" + umount "$mount_dest" + echo "and unmount should have succeeded" + NOMATCH "$mount_dest" < /proc/self/mountinfo + echo "Now snap remove should succeed" + snap remove "${SNAP_NAME}" + } + + test_remove_currently_fails_when_user_mount_in_host_ns_under() { + local mount_dest=$1 + setup_src_data + setup_test_snap + when_user_mount_in_host_ns "$mount_dest" + then_remove_currently_fails_mount_remains_src_data_lost "$mount_dest" + cleaning_up_user_mount_should_make_remove_successful "$mount_dest" + echo + } + + echo + echo "Executing test_$CASE $MOUNT_DEST" + test_$CASE $MOUNT_DEST From 147bc3c1f5e81ed0bc2f3e37236e275a2f955bf6 Mon Sep 17 00:00:00 2001 From: Stephen Mwangi Date: Wed, 15 Apr 2026 09:40:44 +0300 Subject: [PATCH 24/44] o/devicemgmtstate: implement dispatch-mgmt-messages task (#16547) This PR implements the dispatch-mgmt-messages task. This task handles both sequenced & unsequenced tasks. Unsequenced messages get dispatched immediately. Sequenced messages are dispatched in order, starting from where the sequence left off, and subsequent messages are chained via task dependencies. Sequences are tracked in an LRU cache (max size 256). When capacity is exceeded, the least recently used sequence is evicted and its earliest pending message gets a rejection response queued. --- overlord/devicemgmtstate/devicemgmtmgr.go | 206 ++++- .../devicemgmtstate/devicemgmtmgr_test.go | 806 ++++++++++++++++-- overlord/devicemgmtstate/export_test.go | 18 +- 3 files changed, 919 insertions(+), 111 deletions(-) diff --git a/overlord/devicemgmtstate/devicemgmtmgr.go b/overlord/devicemgmtstate/devicemgmtmgr.go index c336f06f05a..8c7c781a36e 100644 --- a/overlord/devicemgmtstate/devicemgmtmgr.go +++ b/overlord/devicemgmtstate/devicemgmtmgr.go @@ -27,6 +27,7 @@ package devicemgmtstate import ( "errors" "fmt" + "sort" "time" "github.com/snapcore/snapd/asserts" @@ -50,6 +51,9 @@ const ( var ( timeNow = time.Now + maxSequences = 256 + maxBlockedMessagesPerSequence = 8 + deviceMgmtExchangeChangeKind = swfeats.RegisterChangeKind("device-management-exchange") ) @@ -86,6 +90,11 @@ type RequestMessage struct { Body string `json:"body"` ReceiveTime time.Time `json:"receive-time"` + + Dispatched bool `json:"dispatched"` + + Status asserts.MessageStatus `json:"status,omitempty"` + Error string `json:"error,omitempty"` } // ID returns the full message identifier `BaseID[-SeqNum]`. @@ -97,11 +106,24 @@ func (msg *RequestMessage) ID() string { return msg.BaseID } +// sequenceState holds the messages and progress for a single base ID, +// covering both sequenced & unsequenced messages. +type sequenceState struct { + // Messages holds request messages from receipt until their response is queued. + Messages []*RequestMessage `json:"messages"` + + // Applied is the highest sequence number successfully applied. A sequenced + // message can only be applied once its predecessor has been applied. + Applied int `json:"applied"` +} + // deviceMgmtState holds the persistent state for device management operations. type deviceMgmtState struct { - // PendingRequests maps message IDs to request messages being processed. - // A message stays here from receipt until its response is queued. - PendingRequests map[string]*RequestMessage `json:"pending-requests"` + // Sequences maps base IDs to their per-base-ID state. + Sequences map[string]*sequenceState `json:"sequences"` + + // SequenceLRU tracks sequenced base IDs in least-recently-used order for eviction. + SequenceLRU []string `json:"sequence-lru"` // LastReceivedToken is the token of the last message successfully stored locally, // sent in the "after" field of the next exchange to acknowledge receipt @@ -128,9 +150,28 @@ func (ms *deviceMgmtState) enqueueRequests(pollResp *store.MessageExchangeRespon continue } - _, exists := ms.PendingRequests[reqMsg.ID()] - if !exists { - ms.PendingRequests[reqMsg.ID()] = reqMsg + seq := ms.Sequences[reqMsg.BaseID] + if seq == nil { + seq = &sequenceState{} + ms.Sequences[reqMsg.BaseID] = seq + } + + // TODO:GOVERSION:1.21: replace with slices.BinarySearchFunc + i := sort.Search(len(seq.Messages), func(i int) bool { + return seq.Messages[i].SeqNum >= reqMsg.SeqNum + }) + if i < len(seq.Messages) && seq.Messages[i].SeqNum == reqMsg.SeqNum { + continue // duplicate + } + // TODO:GOVERSION:1.21: replace with slices.Insert(seq.Messages, i, reqMsg) + seq.Messages = append(seq.Messages, nil) + copy(seq.Messages[i+1:], seq.Messages[i:]) + seq.Messages[i] = reqMsg + + if reqMsg.SeqNum > 0 { + // Move to end of LRU to mark as recently used. + ms.removeSequenceFromLRU(reqMsg.BaseID) + ms.SequenceLRU = append(ms.SequenceLRU, reqMsg.BaseID) } } @@ -142,7 +183,16 @@ func (ms *deviceMgmtState) enqueueRequests(pollResp *store.MessageExchangeRespon } ms.ReadyResponses = make(map[string]store.Message) - ms.LastExchangeTime = timeNow() +} + +// removeSequenceFromLRU removes a sequence from the LRU list, if present. +func (ms *deviceMgmtState) removeSequenceFromLRU(baseID string) { + for i, id := range ms.SequenceLRU { + if id == baseID { + ms.SequenceLRU = append(ms.SequenceLRU[:i], ms.SequenceLRU[i+1:]...) + return + } + } } // DeviceMgmtManager handles device management operations. @@ -176,14 +226,22 @@ func (m *DeviceMgmtManager) getState() (*deviceMgmtState, error) { if err != nil { if errors.Is(err, state.ErrNoState) { return &deviceMgmtState{ - PendingRequests: make(map[string]*RequestMessage), - ReadyResponses: make(map[string]store.Message), + Sequences: make(map[string]*sequenceState), + ReadyResponses: make(map[string]store.Message), }, nil } return nil, err } + if ms.Sequences == nil { + ms.Sequences = make(map[string]*sequenceState) + } + + if ms.ReadyResponses == nil { + ms.ReadyResponses = map[string]store.Message{} + } + return &ms, nil } @@ -202,8 +260,7 @@ func (m *DeviceMgmtManager) Ensure() error { return err } - exchange := m.shouldExchangeMessages(ms) - if !exchange { + if !m.shouldExchangeMessages(ms) { return nil } @@ -228,8 +285,7 @@ func (m *DeviceMgmtManager) Ensure() error { return nil } -// isRemoteManagementEnabled checks whether the remote management feature is enabled. -// Caller must hold state lock. +// isRemoteDeviceManagementEnabled checks whether the remote device management feature is enabled. func (m *DeviceMgmtManager) isRemoteDeviceManagementEnabled() bool { tr := config.NewTransaction(m.state) enabled, err := features.Flag(tr, features.RemoteDeviceManagement) @@ -244,7 +300,6 @@ func (m *DeviceMgmtManager) isRemoteDeviceManagementEnabled() bool { } // shouldExchangeMessages checks whether a message exchange should happen now. -// Caller must hold state lock. func (m *DeviceMgmtManager) shouldExchangeMessages(ms *deviceMgmtState) bool { nextExchange := ms.LastExchangeTime.Add(defaultExchangeInterval) if timeNow().Before(nextExchange) { @@ -266,6 +321,11 @@ func (m *DeviceMgmtManager) doExchangeMessages(t *state.Task, tomb *tomb.Tomb) e return err } + defer func() { + ms.LastExchangeTime = timeNow() + m.setState(ms) + }() + deviceCtx, err := snapstate.DevicePastSeeding(m.state, nil) if err != nil { return err @@ -294,14 +354,128 @@ func (m *DeviceMgmtManager) doExchangeMessages(t *state.Task, tomb *tomb.Tomb) e } ms.enqueueRequests(pollResp) - m.setState(ms) return nil } // doDispatchMessages selects pending requests for processing and queues tasks for them. func (m *DeviceMgmtManager) doDispatchMessages(t *state.Task, _ *tomb.Tomb) error { - // TODO: implement this task, no-op for now. + m.state.Lock() + defer m.state.Unlock() + + ms, err := m.getState() + if err != nil { + return err + } + + chg := t.Change() + // Evict oldest sequences when the LRU exceeds capacity. + for len(ms.SequenceLRU) > maxSequences { + baseID := ms.SequenceLRU[0] + ms.SequenceLRU = ms.SequenceLRU[1:] + err = m.rejectSequence(ms, chg, baseID, "cannot process message: sequence evicted due to capacity limits") + if err != nil { + return err + } + } + + for baseID, seq := range ms.Sequences { + dispatched := m.dispatchSequence(t, seq) + // If nothing was dispatched, the sequence is stuck at a gap (one or more missing predecessors). + // Reject if too many messages have accumulated waiting on it. + if dispatched == 0 && len(seq.Messages) > maxBlockedMessagesPerSequence { + err = m.rejectSequence(ms, chg, baseID, "cannot process message: too many messages waiting on missing predecessors in sequence") + if err != nil { + return err + } + } + } + + m.setState(ms) + + return nil +} + +// dispatchSequence dispatches pending messages in a sequence starting from where +// it left off, chaining consecutive messages. Gaps in the sequence stop the chain. +// Messages are assumed to be sorted by SeqNum. Returns the number of messages dispatched. +func (m *DeviceMgmtManager) dispatchSequence(dispatchTask *state.Task, seq *sequenceState) int { + // Unsequenced messages have SeqNum 0. + expectedSeqNum := 0 + // Sequenced messages resume from where the sequence left off. + if len(seq.Messages) > 0 && seq.Messages[0].SeqNum != 0 { + expectedSeqNum = seq.Applied + 1 + } + + dispatched := 0 + awaitTask := dispatchTask + for _, msg := range seq.Messages { + // Skip messages already dispatched or that have reached a final status. + if msg.Dispatched || msg.Status != "" { + continue + } + + if msg.SeqNum != expectedSeqNum { + // Gap in sequence, stop chaining. + break + } + + awaitTask = m.dispatchMessage(awaitTask, msg) + expectedSeqNum++ + dispatched++ + } + + return dispatched +} + +// dispatchMessage creates the task chain for a single message and returns +// the final task so callers can chain subsequent messages after it. +func (m *DeviceMgmtManager) dispatchMessage(prevTask *state.Task, msg *RequestMessage) *state.Task { + chg := prevTask.Change() + // TODO: add tests verifying that a failure in one message's task chain does not + // affect other messages (lanes provide this isolation, but it needs test coverage). + lane := m.state.NewLane() + + addTask := func(kind, summary string) { + t := m.state.NewTask(kind, summary) + t.Set("message-id", msg.ID()) + t.WaitFor(prevTask) + t.JoinLane(lane) + chg.AddTask(t) + + prevTask = t + } + + addTask("validate-mgmt-message", fmt.Sprintf("Validate message with id %q", msg.ID())) + addTask("apply-mgmt-message", fmt.Sprintf("Apply message with id %q", msg.ID())) + addTask("queue-mgmt-response", fmt.Sprintf("Queue response for message with id %q", msg.ID())) + + msg.Dispatched = true + + return prevTask +} + +// rejectSequence rejects the earliest pending message in a sequence and discards +// the rest. It removes the sequence from the LRU and queues a rejection response. +func (m *DeviceMgmtManager) rejectSequence(ms *deviceMgmtState, chg *state.Change, baseID, reason string) error { + seq := ms.Sequences[baseID] + if seq == nil || len(seq.Messages) == 0 { + return fmt.Errorf("internal error: rejectSequence called for baseID %q with no pending messages", baseID) + } + + earliest := seq.Messages[0] + earliest.Status = asserts.MessageStatusRejected + earliest.Error = reason + seq.Messages = []*RequestMessage{earliest} + + ms.removeSequenceFromLRU(baseID) + + lane := m.state.NewLane() + queue := m.state.NewTask("queue-mgmt-response", fmt.Sprintf("Queue response for message with id %q", earliest.ID())) + queue.Set("message-id", earliest.ID()) + queue.JoinLane(lane) + chg.AddTask(queue) + return nil } diff --git a/overlord/devicemgmtstate/devicemgmtmgr_test.go b/overlord/devicemgmtstate/devicemgmtmgr_test.go index 9fa5ac44233..7e3be9b252a 100644 --- a/overlord/devicemgmtstate/devicemgmtmgr_test.go +++ b/overlord/devicemgmtstate/devicemgmtmgr_test.go @@ -23,6 +23,9 @@ import ( "bytes" "context" "fmt" + "sort" + "strconv" + "strings" "testing" "time" @@ -77,6 +80,8 @@ type deviceMgmtMgrSuite struct { var _ = Suite(&deviceMgmtMgrSuite{}) +var fixedTestTime = time.Date(2025, 6, 14, 12, 0, 0, 0, time.UTC) + func (s *deviceMgmtMgrSuite) SetUpTest(c *C) { s.BaseTest.SetUpTest(c) @@ -127,7 +132,46 @@ func (s *deviceMgmtMgrSuite) mockStore(exchangeMessages func(context.Context, *s snapstate.ReplaceStore(s.st, &mockStore{exchangeMessages: exchangeMessages}) } +func (s *deviceMgmtMgrSuite) makeStoreMessage(c *C, messageID, token string) store.MessageWithToken { + oneHourAgo := fixedTestTime.Add(-time.Hour) + tomorrow := oneHourAgo.Add(24 * time.Hour) + body := []byte(`{"action": "get", "account": "my-brand", "view": "network/wifi-state"}`) + as, err := s.storeStack.Sign( + asserts.RequestMessageType, + map[string]any{ + "authority-id": "my-brand", + "account-id": "my-brand", + "message-id": messageID, + "message-kind": "confdb", + "devices": []any{"serial-1.my-model.my-brand"}, + "valid-since": oneHourAgo.UTC().Format(time.RFC3339), + "valid-until": tomorrow.UTC().Format(time.RFC3339), + "timestamp": oneHourAgo.UTC().Format(time.RFC3339), + }, + body, "", + ) + c.Assert(err, IsNil) + + return store.MessageWithToken{ + Token: token, + Message: store.Message{ + Format: "assertion", + Data: string(asserts.Encode(as)), + }, + } +} + +func (s *deviceMgmtMgrSuite) settle(c *C) { + s.st.Unlock() + defer s.st.Lock() + + err := s.o.Settle(testutil.HostScaledTimeout(5 * time.Second)) + c.Assert(err, IsNil) +} + func (s *deviceMgmtMgrSuite) TestShouldExchangeMessages(c *C) { + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + type test struct { name string flag any @@ -136,12 +180,8 @@ func (s *deviceMgmtMgrSuite) TestShouldExchangeMessages(c *C) { expected bool } - wayback := time.Date(2025, 6, 14, 12, 0, 0, 0, time.UTC) - restoreTime := devicemgmtstate.MockTimeNow(wayback) - defer restoreTime() - - tooSoon := wayback.Add(-5 * time.Second) - enoughTimePassed := wayback.Add(-2 * devicemgmtstate.DefaultExchangeInterval) + tooSoon := fixedTestTime.Add(-5 * time.Second) + enoughTimePassed := fixedTestTime.Add(-2 * devicemgmtstate.DefaultExchangeInterval) tests := []test{ { @@ -201,8 +241,9 @@ func (s *deviceMgmtMgrSuite) TestShouldExchangeMessages(c *C) { cmt := Commentf("%s test", tt.name) ms := &devicemgmtstate.DeviceMgmtState{ - LastExchangeTime: tt.lastExchangeTime, + Sequences: make(map[string]*devicemgmtstate.SequenceState), ReadyResponses: tt.readyResponses, + LastExchangeTime: tt.lastExchangeTime, } setRemoteMgmtFeatureFlag(c, s.st, tt.flag) @@ -218,19 +259,21 @@ func (s *deviceMgmtMgrSuite) TestEnsureOK(c *C) { setRemoteMgmtFeatureFlag(c, s.st, true) - s.st.Unlock() - err := s.mgr.Ensure() - s.st.Lock() - c.Assert(err, IsNil) + s.mockModel() + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{}, nil + }) + + s.settle(c) - changes := s.st.Changes() + changes := changesOfKind(s.st.Changes(), "device-management-exchange") c.Assert(changes, HasLen, 1) + chg := changes[0] - c.Check(chg.Kind(), Equals, "device-management-exchange") c.Check(chg.Summary(), Equals, "Process device management messages") tasks := chg.Tasks() - c.Check(tasks, HasLen, 2) + c.Assert(tasks, HasLen, 2) exchange := tasks[0] c.Check(exchange.Kind(), Equals, "exchange-mgmt-messages") @@ -246,9 +289,11 @@ func (s *deviceMgmtMgrSuite) TestEnsureChangeAlreadyInFlight(c *C) { s.st.Lock() defer s.st.Unlock() + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + setRemoteMgmtFeatureFlag(c, s.st, true) - expired := time.Now().Add(-(devicemgmtstate.DefaultExchangeInterval + time.Minute)) + expired := fixedTestTime.Add(-2 * devicemgmtstate.DefaultExchangeInterval) ms := &devicemgmtstate.DeviceMgmtState{ LastExchangeTime: expired, } @@ -257,94 +302,93 @@ func (s *deviceMgmtMgrSuite) TestEnsureChangeAlreadyInFlight(c *C) { chg := s.st.NewChange("device-management-exchange", "Process device management messages") chg.SetStatus(state.DoingStatus) - s.st.Unlock() - err := s.mgr.Ensure() - s.st.Lock() - c.Assert(err, IsNil) + s.settle(c) - changes := s.st.Changes() + changes := changesOfKind(s.st.Changes(), "device-management-exchange") c.Assert(changes, HasLen, 1) c.Check(changes[0].ID(), Equals, chg.ID()) } func (s *deviceMgmtMgrSuite) TestEnsureFeatureDisabled(c *C) { - err := s.mgr.Ensure() - c.Assert(err, IsNil) - s.st.Lock() defer s.st.Unlock() - changes := s.st.Changes() + s.settle(c) + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") c.Assert(changes, HasLen, 0) } +func (s *deviceMgmtMgrSuite) TestEnsureFeatureDisabledWithReadyResponses(c *C) { + s.st.Lock() + defer s.st.Unlock() + + s.mockModel() + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + c.Check(req.Limit, Equals, 0) + c.Check(req.Messages, HasLen, 1) + + return &store.MessageExchangeResponse{}, nil + }) + + ms := &devicemgmtstate.DeviceMgmtState{ + Sequences: make(map[string]*devicemgmtstate.SequenceState), + ReadyResponses: map[string]store.Message{ + "someId": {Format: "assertion", Data: "response-data"}, + }, + } + s.mgr.SetState(ms) + + s.settle(c) + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 1) + c.Check(changes[0].Err(), IsNil) + + c.Assert(changes[0].Tasks(), HasLen, 2) + + ms, err := s.mgr.GetState() + c.Assert(err, IsNil) + c.Check(ms.LastReceivedToken, Equals, "") + c.Check(ms.ReadyResponses, HasLen, 0) + c.Check(ms.Sequences, HasLen, 0) +} + func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesFetchOK(c *C) { s.st.Lock() defer s.st.Unlock() + setRemoteMgmtFeatureFlag(c, s.st, true) + s.mockModel() s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { c.Check(req.After, Equals, "") c.Check(req.Limit, Equals, devicemgmtstate.DefaultExchangeLimit) c.Check(req.Messages, HasLen, 0) - oneHourAgo := time.Now().Add(-1 * time.Hour) - tomorrow := oneHourAgo.Add(24 * time.Hour) - - body := []byte(`{"action": "get", "account": "my-brand", "view": "network/access-wifi"}`) - as, err := s.storeStack.Sign( - asserts.RequestMessageType, - map[string]any{ - "authority-id": "my-brand", - "account-id": "my-brand", - "message-id": "someId", - "message-kind": "confdb", - "devices": []any{"serial-1.my-model.my-brand"}, - "valid-since": oneHourAgo.UTC().Format(time.RFC3339), - "valid-until": tomorrow.UTC().Format(time.RFC3339), - "timestamp": oneHourAgo.UTC().Format(time.RFC3339), - }, - body, "", - ) - c.Assert(err, IsNil) - return &store.MessageExchangeResponse{ - Messages: []store.MessageWithToken{ - { - Token: "token-123", - Message: store.Message{ - Format: "assertion", - Data: string(asserts.Encode(as)), - }, - }, - }, + Messages: []store.MessageWithToken{s.makeStoreMessage(c, "someId", "token-123")}, TotalPendingMessages: 0, }, nil }) - setRemoteMgmtFeatureFlag(c, s.st, true) - - t := s.st.NewTask("exchange-mgmt-messages", "test exchange-mgmt-messages task") - - s.st.Unlock() - err := s.mgr.DoExchangeMessages(t, &tomb.Tomb{}) - s.st.Lock() - c.Assert(err, IsNil) + s.settle(c) ms, err := s.mgr.GetState() c.Assert(err, IsNil) c.Check(ms.LastReceivedToken, Equals, "token-123") c.Check(ms.LastExchangeTime.IsZero(), Equals, false) - c.Assert(ms.PendingRequests, HasLen, 1) + c.Assert(ms.Sequences, HasLen, 1) + c.Assert(ms.Sequences["someId"].Messages, HasLen, 1) - msg := ms.PendingRequests["someId"] + msg := ms.Sequences["someId"].Messages[0] c.Check(msg.BaseID, Equals, "someId") c.Check(msg.SeqNum, Equals, 0) c.Check(msg.AccountID, Equals, "my-brand") c.Check(msg.AuthorityID, Equals, "my-brand") c.Check(msg.Kind, Equals, "confdb") c.Check(msg.Devices, DeepEquals, []string{"serial-1.my-model.my-brand"}) - c.Check(msg.Body, Equals, `{"action": "get", "account": "my-brand", "view": "network/access-wifi"}`) + c.Check(msg.Body, Equals, `{"action": "get", "account": "my-brand", "view": "network/wifi-state"}`) } func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesReplyOK(c *C) { @@ -364,31 +408,59 @@ func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesReplyOK(c *C) { }) ms := &devicemgmtstate.DeviceMgmtState{ + Sequences: make(map[string]*devicemgmtstate.SequenceState), + LastReceivedToken: "token-123", ReadyResponses: map[string]store.Message{ "someId": {Format: "assertion", Data: "response-data"}, }, - LastReceivedToken: "token-123", } s.mgr.SetState(ms) - t := s.st.NewTask("exchange-mgmt-messages", "test exchange-mgmt-messages task") + s.settle(c) - s.st.Unlock() - err := s.mgr.DoExchangeMessages(t, &tomb.Tomb{}) - s.st.Lock() - c.Assert(err, IsNil) - - ms, err = s.mgr.GetState() + ms, err := s.mgr.GetState() c.Assert(err, IsNil) c.Check(ms.LastReceivedToken, Equals, "") c.Check(ms.ReadyResponses, HasLen, 0) - c.Check(ms.PendingRequests, HasLen, 0) + c.Check(ms.Sequences, HasLen, 0) +} + +func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesSequenceLRU(c *C) { + s.st.Lock() + defer s.st.Unlock() + + setRemoteMgmtFeatureFlag(c, s.st, true) + + s.mockModel() + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{ + Messages: []store.MessageWithToken{ + s.makeStoreMessage(c, "seqA-1", "token-seqA-1"), + s.makeStoreMessage(c, "seqB-1", "token-seqB-1"), + s.makeStoreMessage(c, "uns7", "token-uns1"), + s.makeStoreMessage(c, "seqB-2", "token-seqB-2"), + s.makeStoreMessage(c, "seqC-1", "token-seqC-1"), + s.makeStoreMessage(c, "seqA-2", "token-seqA-2"), + s.makeStoreMessage(c, "uns8", "token-uns2"), + }, + }, nil + }) + + s.settle(c) + + ms, err := s.mgr.GetState() + c.Assert(err, IsNil) + + // seqA's second touch moves it after seqC, leaving seqB least recently used. + c.Check(ms.SequenceLRU, DeepEquals, []string{"seqB", "seqC", "seqA"}) } func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesInvalidMessage(c *C) { s.st.Lock() defer s.st.Unlock() + setRemoteMgmtFeatureFlag(c, s.st, true) + s.mockModel() s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { return &store.MessageExchangeResponse{ @@ -405,27 +477,48 @@ func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesInvalidMessage(c *C) { }, nil }) - setRemoteMgmtFeatureFlag(c, s.st, true) + s.settle(c) - t := s.st.NewTask("exchange-mgmt-messages", "test exchange-mgmt-messages task") + c.Check(s.logbuf.String(), testutil.Contains, "cannot parse request-message with token token-123") - s.st.Unlock() - err := s.mgr.DoExchangeMessages(t, &tomb.Tomb{}) - s.st.Lock() + ms, err := s.mgr.GetState() c.Assert(err, IsNil) + c.Check(ms.LastReceivedToken, Equals, "token-123") + c.Check(ms.Sequences, HasLen, 0) +} - c.Check(s.logbuf.String(), testutil.Contains, "cannot parse request-message with token token-123") +func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesDuplicateMessage(c *C) { + s.st.Lock() + defer s.st.Unlock() + + setRemoteMgmtFeatureFlag(c, s.st, true) + + s.mockModel() + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + msg := s.makeStoreMessage(c, "someId", "token-1") + return &store.MessageExchangeResponse{ + Messages: []store.MessageWithToken{ + msg, + {Token: "token-2", Message: msg.Message}, + }, + TotalPendingMessages: 0, + }, nil + }) + + s.settle(c) ms, err := s.mgr.GetState() c.Assert(err, IsNil) - c.Check(ms.LastReceivedToken, Equals, "token-123") - c.Check(ms.PendingRequests, HasLen, 0) + // The duplicate should have been dropped, leaving one message in the sequence. + c.Assert(ms.Sequences["someId"].Messages, HasLen, 1) } func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesDeviceNotSeeded(c *C) { s.st.Lock() defer s.st.Unlock() + setRemoteMgmtFeatureFlag(c, s.st, true) + s.AddCleanup(snapstatetest.MockDeviceContext(nil)) s.st.Set("seeded", false) @@ -436,34 +529,478 @@ func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesDeviceNotSeeded(c *C) { return nil, fmt.Errorf("call not expected") }) - t := s.st.NewTask("exchange-mgmt-messages", "test exchange-mgmt-messages task") + s.settle(c) - s.st.Unlock() - err := s.mgr.DoExchangeMessages(t, &tomb.Tomb{}) - s.st.Lock() + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 1) c.Assert( - err, ErrorMatches, - "too early for operation, device not yet seeded or device model not acknowledged", + changes[0].Err(), ErrorMatches, + "(?s).*too early for operation, device not yet seeded or device model not acknowledged.*", ) + c.Assert(changes[0].Tasks(), HasLen, 2) } func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesStoreError(c *C) { s.st.Lock() defer s.st.Unlock() + setRemoteMgmtFeatureFlag(c, s.st, true) + s.mockModel() s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { return nil, fmt.Errorf("network timeout") }) + s.settle(c) + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 1) + c.Assert(changes[0].Err(), ErrorMatches, "(?s).*network timeout.*") + c.Assert(changes[0].Tasks(), HasLen, 2) +} + +func (s *deviceMgmtMgrSuite) TestDoExchangeMessagesIdempotent(c *C) { + s.st.Lock() + defer s.st.Unlock() + + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + setRemoteMgmtFeatureFlag(c, s.st, true) - t := s.st.NewTask("exchange-mgmt-messages", "test exchange-mgmt-messages task") + s.mockModel() + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{ + Messages: []store.MessageWithToken{ + // deliver the same message twice + s.makeStoreMessage(c, "someId", "token-1"), + }, + }, nil + }) - s.st.Unlock() - err := s.mgr.DoExchangeMessages(t, &tomb.Tomb{}) + s.settle(c) + + ms, err := s.mgr.GetState() + c.Assert(err, IsNil) + c.Assert(ms.Sequences["someId"].Messages, HasLen, 1) + + // Advance time past the exchange interval to trigger a second exchange. + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime.Add(2 * devicemgmtstate.DefaultExchangeInterval))) + + s.settle(c) + + ms, err = s.mgr.GetState() + c.Assert(err, IsNil) + c.Assert(ms.Sequences["someId"].Messages, HasLen, 1) +} + +func (s *deviceMgmtMgrSuite) TestDoDispatchMessagesUnsequenced(c *C) { s.st.Lock() - c.Assert(err, ErrorMatches, "network timeout") + defer s.st.Unlock() + + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + + setRemoteMgmtFeatureFlag(c, s.st, true) + + s.mockModel() + + // Exchange 1: receive msg1 only so it gets dispatched. + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{ + Messages: []store.MessageWithToken{ + s.makeStoreMessage(c, "msg1", "token-1"), + }, + }, nil + }) + + s.settle(c) + + // Exchange 2: msg1 is dedup'd by exchange; msg2 and msg3 are new. + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime.Add(2 * devicemgmtstate.DefaultExchangeInterval))) + + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{ + Messages: []store.MessageWithToken{ + s.makeStoreMessage(c, "msg1", "token-1"), + s.makeStoreMessage(c, "msg2", "token-2"), + s.makeStoreMessage(c, "msg3", "token-3"), + }, + }, nil + }) + + s.settle(c) + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 2) + + ti := buildTaskIndex(changes[1]) + assertMessagesDispatched(c, ti, []string{"msg2", "msg3"}, "unsequenced") + assertMessagesNotDispatched(c, ti, []string{"msg1"}, "unsequenced") + + waitOn := map[string]string{"msg2": "", "msg3": ""} + assertMessagesWaitOn(c, ti, waitOn, "unsequenced") +} + +func (s *deviceMgmtMgrSuite) TestDoDispatchMessagesSequenced(c *C) { + s.st.Lock() + defer s.st.Unlock() + + makeRequestMessage := func(messageID, kind string, dispatched bool) *devicemgmtstate.RequestMessage { + baseID, seqStr, hasSeq := strings.Cut(messageID, "-") + seqNum := 0 + if hasSeq { + seqNum, _ = strconv.Atoi(seqStr) + } + + return &devicemgmtstate.RequestMessage{ + AccountID: "my-brand", + AuthorityID: "my-brand", + BaseID: baseID, + SeqNum: seqNum, + Kind: kind, + Devices: []string{"serial-1.my-model.my-brand"}, + ValidSince: fixedTestTime, + ValidUntil: fixedTestTime.Add(24 * time.Hour), + Body: `{"action": "get", "account": "my-brand", "view": "network/wifi-state"}`, + ReceiveTime: fixedTestTime.Add(6 * time.Hour), + Dispatched: dispatched, + } + } + + type test struct { + name string + sequences map[string]int // last applied message per sequence + pendingRequests []*devicemgmtstate.RequestMessage + expectedChain map[string]string + } + + tests := []test{ + { + name: "consecutive from start", + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-1", "confdb", false), + makeRequestMessage("seqA-2", "confdb", false), + makeRequestMessage("seqA-3", "confdb", false), + }, + expectedChain: map[string]string{ + "seqA-1": "", + "seqA-2": "seqA-1", + "seqA-3": "seqA-2", + }, + }, + { + name: "gap stops chaining", + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-1", "confdb", false), + makeRequestMessage("seqA-2", "confdb", false), + makeRequestMessage("seqA-4", "confdb", false), // 3 is missing + makeRequestMessage("seqA-5", "confdb", false), + }, + expectedChain: map[string]string{ + "seqA-1": "", + "seqA-2": "seqA-1", + }, + }, + { + name: "resume from last message applied", + sequences: map[string]int{"seqA": 2}, + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-3", "confdb", false), + makeRequestMessage("seqA-4", "confdb", false), + }, + expectedChain: map[string]string{ + "seqA-3": "", + "seqA-4": "seqA-3", + }, + }, + { + name: "no dispatchable messages", + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-5", "confdb", false), // can't start here + }, + }, + { + name: "already dispatched skipped", + sequences: map[string]int{"seqA": 1}, + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-1", "confdb", true), // already dispatched + makeRequestMessage("seqA-2", "confdb", false), + makeRequestMessage("seqA-3", "confdb", false), + }, + expectedChain: map[string]string{ + "seqA-2": "", + "seqA-3": "seqA-2", + }, + }, + { + name: "message with final status is skipped and blocks successor", + pendingRequests: []*devicemgmtstate.RequestMessage{ + func() *devicemgmtstate.RequestMessage { + msg := makeRequestMessage("seqA-1", "confdb", false) + msg.Status = asserts.MessageStatusRejected + return msg + }(), + makeRequestMessage("seqA-2", "confdb", false), + }, + expectedChain: map[string]string{}, + }, + { + name: "mixed sequenced and unsequenced", + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("uns1", "confdb", false), + makeRequestMessage("uns2", "confdb", false), + makeRequestMessage("seqA-1", "confdb", false), + makeRequestMessage("seqA-2", "confdb", false), + }, + expectedChain: map[string]string{ + "uns1": "", + "uns2": "", + "seqA-1": "", + "seqA-2": "seqA-1", + }, + }, + { + name: "multiple independent sequences", + pendingRequests: []*devicemgmtstate.RequestMessage{ + makeRequestMessage("seqA-1", "confdb", false), + makeRequestMessage("seqA-2", "confdb", false), + makeRequestMessage("seqB-1", "confdb", false), + makeRequestMessage("seqB-2", "confdb", false), + }, + expectedChain: map[string]string{ + "seqA-1": "", + "seqA-2": "seqA-1", + "seqB-1": "", + "seqB-2": "seqB-1", + }, + }, + } + + for _, tt := range tests { + cmt := Commentf("%s test", tt.name) + + sequences := make(map[string]*devicemgmtstate.SequenceState) + for _, msg := range tt.pendingRequests { + if sequences[msg.BaseID] == nil { + sequences[msg.BaseID] = &devicemgmtstate.SequenceState{} + } + + sequences[msg.BaseID].Messages = append(sequences[msg.BaseID].Messages, msg) + } + + sequenceLRU := make([]string, 0) + for seqID, lastApplied := range tt.sequences { + sequences[seqID].Applied = lastApplied + sequenceLRU = append(sequenceLRU, seqID) + } + + ms := &devicemgmtstate.DeviceMgmtState{ + Sequences: sequences, + SequenceLRU: sequenceLRU, + ReadyResponses: make(map[string]store.Message), + } + s.mgr.SetState(ms) + + chg := s.st.NewChange("test", "test change") + dispatchTask := s.st.NewTask("dispatch-mgmt-messages", "test dispatch-messages task") + chg.AddTask(dispatchTask) + + alreadyDispatched := make(map[string]bool) + for _, msg := range tt.pendingRequests { + if msg.Dispatched { + alreadyDispatched[msg.ID()] = true + } + } + + s.st.Unlock() + err := s.mgr.DoDispatchMessages(dispatchTask, &tomb.Tomb{}) + s.st.Lock() + c.Assert(err, IsNil, cmt) + + ms, err = s.mgr.GetState() + c.Assert(err, IsNil, cmt) + + var notDispatched []string + dispatched := make([]string, 0, len(tt.expectedChain)) + for _, seq := range ms.Sequences { + for _, msg := range seq.Messages { + _, inChain := tt.expectedChain[msg.ID()] + if inChain { + dispatched = append(dispatched, msg.ID()) + } else { + notDispatched = append(notDispatched, msg.ID()) + } + + c.Check(msg.Dispatched, Equals, alreadyDispatched[msg.ID()] || inChain, Commentf("%s: %s", tt.name, msg.ID())) + } + } + + ti := buildTaskIndex(chg) + assertMessagesDispatched(c, ti, dispatched, tt.name) + assertMessagesNotDispatched(c, ti, notDispatched, tt.name) + assertMessagesWaitOn(c, ti, tt.expectedChain, tt.name) + } +} + +func (s *deviceMgmtMgrSuite) TestDoDispatchMessagesEvictedSequenceRejected(c *C) { + s.st.Lock() + defer s.st.Unlock() + + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + + const maxSequences = 4 + s.AddCleanup(devicemgmtstate.MockMaxSequences(maxSequences)) + + setRemoteMgmtFeatureFlag(c, s.st, true) + + s.mockModel() + + // seqA and seqB are the 2 oldest in LRU and will be evicted; each gets 2 + // messages to verify the 2nd is dropped on eviction. + messages := []store.MessageWithToken{ + s.makeStoreMessage(c, "seqA-1", "token-seqA-1"), + s.makeStoreMessage(c, "seqA-2", "token-seqA-2"), + s.makeStoreMessage(c, "seqB-1", "token-seqB-1"), + s.makeStoreMessage(c, "seqB-2", "token-seqB-2"), + } + for i := 3; i <= maxSequences+2; i++ { + baseID := fmt.Sprintf("seq%c", rune('A'+i-1)) + messages = append(messages, + s.makeStoreMessage(c, fmt.Sprintf("%s-1", baseID), fmt.Sprintf("token-%s-1", baseID)), + ) + } + + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{Messages: messages}, nil + }) + + s.settle(c) + + ms, err := s.mgr.GetState() + c.Assert(err, IsNil) + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 1) + + // seqA evicted (oldest in LRU). + seqA := ms.Sequences["seqA"] + c.Assert(seqA.Messages, HasLen, 1, Commentf("the 2nd message in seqA should have been deleted")) + c.Check(seqA.Messages[0].Status, Equals, asserts.MessageStatusRejected) + c.Check(seqA.Messages[0].Error, Equals, "cannot process message: sequence evicted due to capacity limits") + + ti := buildTaskIndex(changes[0]) + c.Check(ti.validate["seqA-1"], IsNil) + c.Check(ti.apply["seqA-1"], IsNil) + c.Check(ti.queue["seqA-1"], NotNil) + + // seqB also evicted. + seqB := ms.Sequences["seqB"] + c.Assert(seqB.Messages, HasLen, 1, Commentf("the 2nd message in seqB should have been deleted")) + c.Check(seqB.Messages[0].Status, Equals, asserts.MessageStatusRejected) + + c.Check(ms.SequenceLRU, DeepEquals, []string{"seqC", "seqD", "seqE", "seqF"}) +} + +func (s *deviceMgmtMgrSuite) TestDoDispatchMessagesBlockedSequenceRejected(c *C) { + s.st.Lock() + defer s.st.Unlock() + + s.AddCleanup(devicemgmtstate.MockTimeNow(fixedTestTime)) + + const maxBlockedMessagesPerSequence = 4 + s.AddCleanup(devicemgmtstate.MockMaxBlockedMessagesPerSequence(maxBlockedMessagesPerSequence)) + + setRemoteMgmtFeatureFlag(c, s.st, true) + + s.mockModel() + + // Build a sequence stuck at a gap: seqNum 1 is missing, messages start at 2. + messages := make([]store.MessageWithToken, maxBlockedMessagesPerSequence+1) + for i := range messages { + seqNum := i + 2 + messages[i] = s.makeStoreMessage(c, fmt.Sprintf("seqA-%d", seqNum), fmt.Sprintf("token-seqA-%d", seqNum)) + } + + s.mockStore(func(ctx context.Context, req *store.MessageExchangeRequest) (*store.MessageExchangeResponse, error) { + return &store.MessageExchangeResponse{Messages: messages}, nil + }) + + s.settle(c) + + ms, err := s.mgr.GetState() + c.Assert(err, IsNil) + + seqA := ms.Sequences["seqA"] + c.Assert(seqA.Messages, HasLen, 1, Commentf("remaining messages should have been deleted")) + c.Check(seqA.Messages[0].Status, Equals, asserts.MessageStatusRejected) + c.Check(seqA.Messages[0].Error, Equals, "cannot process message: too many messages waiting on missing predecessors in sequence") + + changes := changesOfKind(s.st.Changes(), "device-management-exchange") + c.Assert(changes, HasLen, 1) + ti := buildTaskIndex(changes[0]) + c.Check(ti.queue["seqA-2"], NotNil) + c.Check(ti.validate["seqA-2"], IsNil) + c.Check(ti.apply["seqA-2"], IsNil) +} + +func (s *deviceMgmtMgrSuite) TestDoDispatchMessagesIdempotent(c *C) { + s.st.Lock() + defer s.st.Unlock() + + ms := &devicemgmtstate.DeviceMgmtState{ + Sequences: map[string]*devicemgmtstate.SequenceState{ + "msg1": { + Messages: []*devicemgmtstate.RequestMessage{ + { + AccountID: "my-brand", + AuthorityID: "my-brand", + BaseID: "msg1", + Kind: "confdb", + Devices: []string{"serial-1.my-model.my-brand"}, + ValidSince: fixedTestTime, + ValidUntil: fixedTestTime.Add(24 * time.Hour), + Body: `{"action": "get", "account": "my-brand", "view": "network/wifi-state"}`, + }, + }, + }, + "msg2": { + Messages: []*devicemgmtstate.RequestMessage{ + { + AccountID: "my-brand", + AuthorityID: "my-brand", + BaseID: "msg2", + Kind: "confdb", + Devices: []string{"serial-1.my-model.my-brand"}, + ValidSince: fixedTestTime, + ValidUntil: fixedTestTime.Add(24 * time.Hour), + Body: `{"action": "get", "account": "my-brand", "view": "network/wifi-state"}`, + }, + }, + }, + }, + } + s.mgr.SetState(ms) + + chg := s.st.NewChange("test", "test change") + for i := 1; i <= 3; i++ { + t := s.st.NewTask("dispatch-mgmt-messages", fmt.Sprintf("test dispatch %d", i)) + chg.AddTask(t) + } + + s.settle(c) + + c.Check(chg.Status(), Equals, state.DoneStatus) + + // Each message should have been dispatched exactly once: + // 3 dispatch tasks + 2 messages * 3 tasks each = 9 tasks. + c.Assert(chg.Tasks(), HasLen, 9) + + ti := buildTaskIndex(chg) + c.Check(ti.validate["msg1"], NotNil) + c.Check(ti.apply["msg1"], NotNil) + c.Check(ti.queue["msg1"], NotNil) + c.Check(ti.validate["msg2"], NotNil) + c.Check(ti.apply["msg2"], NotNil) + c.Check(ti.queue["msg2"], NotNil) } func (s *deviceMgmtMgrSuite) TestParseRequestMessageInvalid(c *C) { @@ -508,3 +1045,90 @@ func (s *deviceMgmtMgrSuite) TestParseRequestMessageInvalid(c *C) { c.Check(msg, IsNil, cmt) } } + +func changesOfKind(changes []*state.Change, kind string) []*state.Change { + var result []*state.Change + for _, chg := range changes { + if chg.Kind() == kind { + result = append(result, chg) + } + } + + sort.Slice(result, func(i, j int) bool { + idI, _ := strconv.Atoi(result[i].ID()) + idJ, _ := strconv.Atoi(result[j].ID()) + return idI < idJ + }) + + return result +} + +type taskIndex struct { + validate map[string]*state.Task + apply map[string]*state.Task + queue map[string]*state.Task +} + +func buildTaskIndex(chg *state.Change) *taskIndex { + ti := &taskIndex{ + validate: make(map[string]*state.Task), + apply: make(map[string]*state.Task), + queue: make(map[string]*state.Task), + } + for _, t := range chg.Tasks() { + var id string + err := t.Get("message-id", &id) + if err != nil { + continue + } + + switch t.Kind() { + case "validate-mgmt-message": + ti.validate[id] = t + case "apply-mgmt-message": + ti.apply[id] = t + case "queue-mgmt-response": + ti.queue[id] = t + } + } + + return ti +} + +func assertMessagesDispatched(c *C, ti *taskIndex, msgIDs []string, testName string) { + for _, id := range msgIDs { + cmt := Commentf("%s: expected %s to be dispatched", testName, id) + c.Assert(ti.validate[id], NotNil, cmt) + c.Assert(ti.apply[id], NotNil, cmt) + c.Assert(ti.queue[id], NotNil, cmt) + } +} + +func assertMessagesNotDispatched(c *C, ti *taskIndex, msgIDs []string, testName string) { + for _, id := range msgIDs { + cmt := Commentf("%s: expected %s to not be dispatched", testName, id) + c.Assert(ti.validate[id], IsNil, cmt) + c.Assert(ti.apply[id], IsNil, cmt) + c.Assert(ti.queue[id], IsNil, cmt) + } +} + +func assertMessagesWaitOn(c *C, ti *taskIndex, waitOn map[string]string, testName string) { + for msgID, prevID := range waitOn { + cmt := Commentf("%s: invalid wait chain for %s", testName, msgID) + + validate := ti.validate[msgID] + c.Assert(validate, NotNil, cmt) + + waitTasks := validate.WaitTasks() + c.Assert(waitTasks, HasLen, 1, cmt) + + if prevID == "" { + c.Assert(waitTasks[0].Kind(), Equals, "dispatch-mgmt-messages", cmt) + } else { + prevQueue := ti.queue[prevID] + c.Assert(prevQueue, NotNil, cmt) + c.Assert(waitTasks[0].ID(), Equals, prevQueue.ID(), cmt) + } + } +} diff --git a/overlord/devicemgmtstate/export_test.go b/overlord/devicemgmtstate/export_test.go index 4f63f8e99f2..5f77fc4f508 100644 --- a/overlord/devicemgmtstate/export_test.go +++ b/overlord/devicemgmtstate/export_test.go @@ -35,15 +35,25 @@ var ( DefaultExchangeInterval = defaultExchangeInterval ) -type DeviceMgmtState deviceMgmtState +func MockMaxSequences(n int) func() { + return testutil.Mock(&maxSequences, n) +} + +func MockMaxBlockedMessagesPerSequence(n int) func() { + return testutil.Mock(&maxBlockedMessagesPerSequence, n) +} + +type SequenceState = sequenceState + +type DeviceMgmtState = deviceMgmtState func (m *DeviceMgmtManager) GetState() (*DeviceMgmtState, error) { ms, err := m.getState() - return (*DeviceMgmtState)(ms), err + return ms, err } func (m *DeviceMgmtManager) SetState(ms *DeviceMgmtState) { - m.setState((*deviceMgmtState)(ms)) + m.setState(ms) } func (m *DeviceMgmtManager) MockHandler(kind string, handler MessageHandler) { @@ -55,7 +65,7 @@ func (m *DeviceMgmtManager) MockSigner(signer ResponseMessageSigner) { } func (m *DeviceMgmtManager) ShouldExchangeMessages(ms *DeviceMgmtState) bool { - return m.shouldExchangeMessages((*deviceMgmtState)(ms)) + return m.shouldExchangeMessages(ms) } func (m *DeviceMgmtManager) DoExchangeMessages(t *state.Task, tomb *tomb.Tomb) error { From 2b316311ca4966bada01501f74a6cc6ae9c457d1 Mon Sep 17 00:00:00 2001 From: Sergio Costas Date: Wed, 15 Apr 2026 09:42:07 +0200 Subject: [PATCH 25/44] ci: Execute rest_api_test tests in codeconv (#16653) * ci: Execute rest_api_test tests in codeconv The tests in the rest_api_test.go file require dbus-launcher, available in dbus-x11. If that binary isn't available, the tests are skipped. A clear proof of this is the Codeconv checks, which show that the code in rest_api.go isn't being checked, even when there are tests that cover it. This patch fixes this. * Add dbus-launch check in run-checks --- .../actions/download-install-debian-deps/action.yaml | 2 +- run-checks | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/download-install-debian-deps/action.yaml b/.github/actions/download-install-debian-deps/action.yaml index 2e5cc601a99..b25630bf9bd 100644 --- a/.github/actions/download-install-debian-deps/action.yaml +++ b/.github/actions/download-install-debian-deps/action.yaml @@ -27,6 +27,6 @@ runs: sudo apt update ln -s packaging/ubuntu-16.04 debian sudo apt build-dep -y "${{ inputs.snapd-src-dir }}" - sudo apt install -y clang + sudo apt install -y clang dbus-x11 sudo apt install -y gcovr lcov rm -rf ./debian-deps diff --git a/run-checks b/run-checks index c2fcb29ee31..19ffa85116d 100755 --- a/run-checks +++ b/run-checks @@ -24,6 +24,7 @@ tool_version() { golangci-lint) ;& # fallthrough python3) ;& # fallthrough pytest-3) ;& # fallthrough + dbus-launch) ;& # fallthrough clang-format) $tool --version ;; @@ -49,6 +50,7 @@ tool_install() { shellcheck) ;& # fallthrough python3) ;& # fallthrough pytest-3) ;& # fallthrough + dbus-launch) ;& # fallthrough clang-format) ;; *) return 1 @@ -79,6 +81,7 @@ tool_configure() { modernize) ;& # fallthrough python3) ;& # fallthrough pytest-3) ;& # fallthrough + dbus-launch) ;& # fallthrough clang-format) ;; *) return 1 @@ -374,6 +377,13 @@ if [ "$STATIC" = 1 ]; then exit 1 fi + if command -v dbus-launch>/dev/null && [ -z "${SKIP_DBUS_LAUNCH:-}" ]; then + echo ">> [Shell] Checking for dbus-launch" + dbus-launch --version + else + echo ">> [Shell] Skipping tests that require dbus-launch" + fi + echo ">> [Go] Checking cmd binaries for forbidden dependencies" forbidden_cmd_go_deps From a9979b11991c4856fd63b9ff2995cbc6ac66ac2e Mon Sep 17 00:00:00 2001 From: Katie May Date: Tue, 14 Apr 2026 12:40:13 +0200 Subject: [PATCH 26/44] tests: add checks in os.query if a version is latest LTS, interim, or devel --- .../lib/external/snapd-testing-tools/tools/os.query | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/lib/external/snapd-testing-tools/tools/os.query b/tests/lib/external/snapd-testing-tools/tools/os.query index 0d6aa690305..7d6b05483ac 100755 --- a/tests/lib/external/snapd-testing-tools/tools/os.query +++ b/tests/lib/external/snapd-testing-tools/tools/os.query @@ -113,6 +113,18 @@ is_resolute() { grep -qFx 'UBUNTU_CODENAME=resolute' /etc/os-release } +is_ubuntu_latest_lts() { + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --lts)" ]] +} + +is_ubuntu_interim() { + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --stable)" ]] +} + +is_ubuntu_devel() { + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --devel)" ]] +} + is_ubuntu() { VERSION=$1 if [ -z "$VERSION" ]; then From 7c92eff7fe28396f5a440b060ce166bf522337f8 Mon Sep 17 00:00:00 2001 From: Katie May Date: Tue, 14 Apr 2026 12:41:23 +0200 Subject: [PATCH 27/44] tests: add manual distro-upgrade test --- tests/upgrade/distro-upgrade/check-snapd.sh | 37 +++++++++++ tests/upgrade/distro-upgrade/task.yaml | 72 +++++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100755 tests/upgrade/distro-upgrade/check-snapd.sh create mode 100644 tests/upgrade/distro-upgrade/task.yaml diff --git a/tests/upgrade/distro-upgrade/check-snapd.sh b/tests/upgrade/distro-upgrade/check-snapd.sh new file mode 100755 index 00000000000..8838102da63 --- /dev/null +++ b/tests/upgrade/distro-upgrade/check-snapd.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euxo pipefail + +systemctl is-active snapd.service +systemctl is-active snapd.socket + +tests.invariant check crashed-snap-confine +tests.invariant check broken-snaps + +snap version | grep -q "$(cat snap-version.txt)" + +snap debug confinement | MATCH "strict" + +snap connections go-example-webserver > tmp-webserver-connectionts.txt +diff -u webserver-connections.txt tmp-webserver-connectionts.txt + +tests.systemd wait-for-service -n 30 --state active snap.go-example-webserver.webserver.service +curl --fail --silent --show-error -o /dev/null localhost:8081 + +snap list > tmp-snap-list.txt +diff -u snap-list.txt tmp-snap-list.txt + +test-snapd-sh.sh -c 'echo Hello' | MATCH "Hello" +test-snapd-sh.sh -c 'env' | MATCH "SNAP_NAME=test-snapd-sh" + +echo Hello > /var/tmp/myevil.txt +if test-snapd-sh.sh -c 'cat /var/tmp/myevil.txt'; then + exit 1 +fi + +test_snapd_wellknown1 | MATCH "ok wellknown 1" +test_snapd_wellknown2 | MATCH "ok wellknown 2" +snap aliases|MATCH "test-snapd-auto-aliases.wellknown1 +test_snapd_wellknown1 +-" +snap aliases|MATCH "test-snapd-auto-aliases.wellknown2 +test_snapd_wellknown2 +-" + +test-snapd-classic-confinement.recurse 5 diff --git a/tests/upgrade/distro-upgrade/task.yaml b/tests/upgrade/distro-upgrade/task.yaml new file mode 100644 index 00000000000..0874e143fa8 --- /dev/null +++ b/tests/upgrade/distro-upgrade/task.yaml @@ -0,0 +1,72 @@ +summary: Check that with the new snapd installed, we can upgrade the distro without issues. + +details: | + This test performs a series of distro upgrades until it reaches the latest devel verison. + After each upgrade, it checks that snapd is still working correctly and performs some simple smoke checks. + It expects to have done an upgrade to the latest LTS, the latest interim release and the latest devel release. + +# This test should only be run on the lowest supported version of ubuntu for a given release +systems: + - ubuntu-* + +manual: true + +prepare: | + snap install go-example-webserver + snap install test-snapd-sh + snap install test-snapd-auto-aliases + "$TESTSTOOLS"/snaps-state install-local test-snapd-classic-confinement --classic + + snap connections go-example-webserver > webserver-connections.txt + snap version | grep snap > snap-version.txt + snap list > snap-list.txt + sed -i 's/Prompt=never/Prompt=normal/' /etc/update-manager/release-upgrades + +restore: | + snap remove --purge go-example-webserver + snap remove --purge test-snapd-sh + snap remove --purge test-snapd-auto-aliases + snap remove --purge test-snapd-classic-confinement + sed -i 's/Prompt=normal/Prompt=never/' /etc/update-manager/release-upgrades + +execute: | + while ! os.query is-ubuntu-devel; do + # We can only upgrade the distro if we have all upgrades installed + apt update + apt upgrade -y + if [[ "$SPREAD_REBOOT" = 0 ]]; then + REBOOT + fi + + devel_option= + if os.query is-ubuntu-interim; then + devel_option="-d" + fi + + echo "Upgrading $(grep -oP 'CODENAME=\K.*' /etc/os-release | head -1)" + sudo do-release-upgrade "$devel_option" -f DistUpgradeViewNonInteractive + + ./check-snapd.sh + + if os.query is-ubuntu-latest-lts; then + touch lts-upgrade-done + elif os.query is-ubuntu-interim; then + touch interim-upgrade-done + elif os.query is-ubuntu-devel; then + touch devel-upgrade-done + fi + SPREAD_REBOOT=0 + done + + if ! [ -f lts-upgrade-done ]; then + echo "We should have done an LTS upgrade but didn't" + exit 1 + fi + if ! [ -f interim-upgrade-done ]; then + echo "We should have done an interim upgrade but didn't" + exit 1 + fi + if ! [ -f devel-upgrade-done ]; then + echo "We should have done a devel upgrade but didn't" + exit 1 + fi From 8d1d62d9c4f54de9eb9db90895644e5f5902bb38 Mon Sep 17 00:00:00 2001 From: Katie May Date: Tue, 14 Apr 2026 12:44:11 +0200 Subject: [PATCH 28/44] spread: add ability to skip suite restore steps on tests/upgrade The package retore logic fail due to distro upgrade. Since this test will be run in isolation, the logic is not needed anyway and can be skipped. --- spread.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/spread.yaml b/spread.yaml index 3c60ee3595c..9213b7795dc 100644 --- a/spread.yaml +++ b/spread.yaml @@ -143,6 +143,8 @@ environment: # Whether snapd should skip early refreshes. SNAPD_SKIP_EARLY_REFRESH: '$(HOST: echo "${SPREAD_SNAPD_SKIP_EARLY_REFRESH:-}")' + SKIP_RESTORE: '$(HOST: echo "${SPREAD_SKIP_RESTORE:-false}")' + backends: google: key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")' @@ -1596,9 +1598,13 @@ suites: prepare-each: | "$TESTSLIB"/prepare-restore.sh --prepare-suite-each restore-each: | - "$TESTSLIB"/prepare-restore.sh --restore-suite-each + if [ "$SKIP_RESTORE" = "false" ]; then + "$TESTSLIB"/prepare-restore.sh --restore-suite-each + fi restore: | - "$TESTSLIB"/prepare-restore.sh --restore-suite + if [ "$SKIP_RESTORE" = "false" ]; then + "$TESTSLIB"/prepare-restore.sh --restore-suite + fi tests/cross/: summary: Cross-compile tests systems: [ubuntu-24.04-64] From eb76bd2ce8a0d4eda84bb8e8b78efa86f190bba8 Mon Sep 17 00:00:00 2001 From: Katie May Date: Tue, 14 Apr 2026 17:16:18 +0200 Subject: [PATCH 29/44] tests: exclude ubuntu core from distro upgrade test --- tests/upgrade/distro-upgrade/task.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/upgrade/distro-upgrade/task.yaml b/tests/upgrade/distro-upgrade/task.yaml index 0874e143fa8..f678379e365 100644 --- a/tests/upgrade/distro-upgrade/task.yaml +++ b/tests/upgrade/distro-upgrade/task.yaml @@ -7,7 +7,8 @@ details: | # This test should only be run on the lowest supported version of ubuntu for a given release systems: - - ubuntu-* + - ubuntu-1* + - ubuntu-2* manual: true From 49bdae2bfb08bb80a3f5fd4724be0ef9d382003a Mon Sep 17 00:00:00 2001 From: Katie May Date: Tue, 14 Apr 2026 18:05:57 +0200 Subject: [PATCH 30/44] tests: use not instead of ! --- tests/upgrade/distro-upgrade/task.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/upgrade/distro-upgrade/task.yaml b/tests/upgrade/distro-upgrade/task.yaml index f678379e365..4605b5caa1a 100644 --- a/tests/upgrade/distro-upgrade/task.yaml +++ b/tests/upgrade/distro-upgrade/task.yaml @@ -59,15 +59,15 @@ execute: | SPREAD_REBOOT=0 done - if ! [ -f lts-upgrade-done ]; then + if not [ -f lts-upgrade-done ]; then echo "We should have done an LTS upgrade but didn't" exit 1 fi - if ! [ -f interim-upgrade-done ]; then + if not [ -f interim-upgrade-done ]; then echo "We should have done an interim upgrade but didn't" exit 1 fi - if ! [ -f devel-upgrade-done ]; then + if not [ -f devel-upgrade-done ]; then echo "We should have done a devel upgrade but didn't" exit 1 fi From 643935829128ffd929979d85c5543fb4a78f174c Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 15 Apr 2026 10:48:35 +0200 Subject: [PATCH 31/44] github: add workflow for distro upgrade/downgrade --- .github/workflows/ci-release.yaml | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 .github/workflows/ci-release.yaml diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml new file mode 100644 index 00000000000..a9b60e53676 --- /dev/null +++ b/.github/workflows/ci-release.yaml @@ -0,0 +1,98 @@ +name: Release Testing +on: + pull_request: + branches: [ "release/**" ] + +jobs: + run-gate: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip spread') }} + runs-on: ubuntu-latest + steps: + - run: echo "Running release testing." + + snap-builds: + uses: ./.github/workflows/snap-builds.yaml + needs: run-gate + with: + runs-on: '["ubuntu-latest"]' + toolchain: default + variant: test + + find-lowest-supported: + runs-on: ubuntu-latest + needs: run-gate + outputs: + version: ${{ steps.find-lowest-supported.outputs.version }} + system: ${{ steps.find-lowest-supported.outputs.system }} + steps: + - name: Find lowest supported Ubuntu version + id: find-lowest-supported + run: | + sudo apt update + version=$(ubuntu-distro-info --supported --release | head -1 | awk '{print $1}') + echo "Setting lowest supported version to $version and system to ubuntu-$version-64" + echo "system=ubuntu-$version-64" >> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT + + create-vendor-dir: + runs-on: ubuntu-latest + needs: run-gate + name: Create go vendor + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: vendor + run: | + sudo snap install --classic --channel 1.18/stable go + /snap/bin/go mod vendor + ( + cd c-vendor + ./vendor.sh + ) + mkdir /tmp/pkg + base_version="$(head -1 packaging/ubuntu-16.04/changelog | awk -F '[()]' '{print $2}')" + version="1337.$base_version" + ./packaging/pack-source -v "$version" -o /tmp/pkg + ./packaging/pack-source -v "$version" -o /tmp/pkg -s + echo "$version" > /tmp/pkg/version + + - name: Upload vendors + uses: actions/upload-artifact@v7 + with: + name: vendor-pkgs + path: /tmp/pkg + + package-snapd-deb: + needs: [find-lowest-supported, create-vendor-dir] + uses: ./.github/workflows/deb-builds.yaml + name: Package ${{ matrix.build-config.system }} + with: + runs-on: '["ubuntu-latest"]' + system: ${{ needs.find-lowest-supported.outputs.system }} + os: ubuntu + os-version: ${{ needs.find-lowest-supported.outputs.version }} + + test-snapd-deb: + needs: [package-snapd-deb, snap-builds, find-lowest-supported] + uses: ./.github/workflows/spread-tests.yaml + with: + runs-on: '["self-hosted", "spread-enabled"]' + group: deb-test + backend: openstack + systems: ${{ needs.find-lowest-supported.outputs.system }} + tasks: 'tests/upgrade/distro-upgrade' + rules: '' + spread-env: "SPREAD_SNAPD_DEB_FROM_REPO=false SPREAD_SNAP_REEXEC=0 SPREAD_SKIP_RESTORE=true" + + test-snapd-snap: + needs: [snap-builds, find-lowest-supported] + uses: ./.github/workflows/spread-tests.yaml + with: + runs-on: '["self-hosted", "spread-enabled"]' + group: snap-test + backend: openstack + systems: ${{ needs.find-lowest-supported.outputs.system }} + tasks: 'tests/upgrade/distro-upgrade' + rules: '' + spread-env: "SPREAD_SKIP_RESTORE=true" From 05bcfee162cfa5dcf3891bb8c97d05e3585d14d7 Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 15 Apr 2026 14:31:11 +0200 Subject: [PATCH 32/44] spread, github, tests: move distro-upgrade test to new test suite --- .github/workflows/ci-release.yaml | 7 +++---- spread.yaml | 18 ++++++++++-------- .../distro-upgrade/check-snapd.sh | 0 .../distro-upgrade/task.yaml | 0 4 files changed, 13 insertions(+), 12 deletions(-) rename tests/{upgrade => release}/distro-upgrade/check-snapd.sh (100%) rename tests/{upgrade => release}/distro-upgrade/task.yaml (100%) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index a9b60e53676..1a8df8bf0f4 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -81,9 +81,9 @@ jobs: group: deb-test backend: openstack systems: ${{ needs.find-lowest-supported.outputs.system }} - tasks: 'tests/upgrade/distro-upgrade' + tasks: 'tests/release/distro-upgrade' rules: '' - spread-env: "SPREAD_SNAPD_DEB_FROM_REPO=false SPREAD_SNAP_REEXEC=0 SPREAD_SKIP_RESTORE=true" + spread-env: "SPREAD_SNAPD_DEB_FROM_REPO=false SPREAD_SNAP_REEXEC=0" test-snapd-snap: needs: [snap-builds, find-lowest-supported] @@ -93,6 +93,5 @@ jobs: group: snap-test backend: openstack systems: ${{ needs.find-lowest-supported.outputs.system }} - tasks: 'tests/upgrade/distro-upgrade' + tasks: 'tests/release/distro-upgrade' rules: '' - spread-env: "SPREAD_SKIP_RESTORE=true" diff --git a/spread.yaml b/spread.yaml index 9213b7795dc..bba07a027d8 100644 --- a/spread.yaml +++ b/spread.yaml @@ -143,8 +143,6 @@ environment: # Whether snapd should skip early refreshes. SNAPD_SKIP_EARLY_REFRESH: '$(HOST: echo "${SPREAD_SNAPD_SKIP_EARLY_REFRESH:-}")' - SKIP_RESTORE: '$(HOST: echo "${SPREAD_SKIP_RESTORE:-false}")' - backends: google: key: '$(HOST: echo "$SPREAD_GOOGLE_KEY")' @@ -1587,6 +1585,14 @@ suites: restore: | "$TESTSLIB"/prepare-restore.sh --restore-suite + tests/release/: + summary: Tests for snapd during the release process + systems: [ubuntu-1*, ubuntu-2*] + prepare: | + "$TESTSLIB"/prepare-restore.sh --prepare-suite + prepare-each: | + "$TESTSLIB"/prepare-restore.sh --prepare-suite-each + tests/upgrade/: summary: Tests for snapd upgrade # Test cases are not yet ported to openSUSE that is why we keep @@ -1598,13 +1604,9 @@ suites: prepare-each: | "$TESTSLIB"/prepare-restore.sh --prepare-suite-each restore-each: | - if [ "$SKIP_RESTORE" = "false" ]; then - "$TESTSLIB"/prepare-restore.sh --restore-suite-each - fi + "$TESTSLIB"/prepare-restore.sh --restore-suite-each restore: | - if [ "$SKIP_RESTORE" = "false" ]; then - "$TESTSLIB"/prepare-restore.sh --restore-suite - fi + "$TESTSLIB"/prepare-restore.sh --restore-suite tests/cross/: summary: Cross-compile tests systems: [ubuntu-24.04-64] diff --git a/tests/upgrade/distro-upgrade/check-snapd.sh b/tests/release/distro-upgrade/check-snapd.sh similarity index 100% rename from tests/upgrade/distro-upgrade/check-snapd.sh rename to tests/release/distro-upgrade/check-snapd.sh diff --git a/tests/upgrade/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml similarity index 100% rename from tests/upgrade/distro-upgrade/task.yaml rename to tests/release/distro-upgrade/task.yaml From ba7f6009ebafa700fd46bb3238bacd49e227d468 Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 16 Apr 2026 14:41:31 +0200 Subject: [PATCH 33/44] tests: update os.query --- tests/lib/external/snapd-testing-tools/tools/os.query | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/lib/external/snapd-testing-tools/tools/os.query b/tests/lib/external/snapd-testing-tools/tools/os.query index 7d6b05483ac..8659dc09264 100755 --- a/tests/lib/external/snapd-testing-tools/tools/os.query +++ b/tests/lib/external/snapd-testing-tools/tools/os.query @@ -114,15 +114,15 @@ is_resolute() { } is_ubuntu_latest_lts() { - grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --lts)" ]] + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(grep -oP 'VERSION_CODENAME=\K.*' /etc/os-release)" = "$(ubuntu-distro-info --lts)" ]] } is_ubuntu_interim() { - grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --stable)" ]] + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(grep -oP 'VERSION_CODENAME=\K.*' /etc/os-release)" = "$(ubuntu-distro-info --stable)" ]] } is_ubuntu_devel() { - grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(lsb_release -c -s)" = "$(ubuntu-distro-info --devel)" ]] + grep -qFx 'ID=ubuntu' /etc/os-release && [[ "$(grep -oP 'VERSION_CODENAME=\K.*' /etc/os-release)" = "$(ubuntu-distro-info --devel)" ]] } is_ubuntu() { From a5b2fe714a3d77fa6c20251fff9b87a42d756fa9 Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 22 Apr 2026 08:17:05 +0200 Subject: [PATCH 34/44] tests: add snapd version check --- spread.yaml | 2 + .../distro-upgrade/check-snapd-version.sh | 23 +++++++++++ tests/release/distro-upgrade/check-snapd.sh | 8 ++-- tests/release/distro-upgrade/task.yaml | 39 ++++++++++++++++--- 4 files changed, 62 insertions(+), 10 deletions(-) create mode 100755 tests/release/distro-upgrade/check-snapd-version.sh diff --git a/spread.yaml b/spread.yaml index bba07a027d8..b9260253838 100644 --- a/spread.yaml +++ b/spread.yaml @@ -1588,6 +1588,8 @@ suites: tests/release/: summary: Tests for snapd during the release process systems: [ubuntu-1*, ubuntu-2*] + environment: + CORE_CHANNEL: stable prepare: | "$TESTSLIB"/prepare-restore.sh --prepare-suite prepare-each: | diff --git a/tests/release/distro-upgrade/check-snapd-version.sh b/tests/release/distro-upgrade/check-snapd-version.sh new file mode 100755 index 00000000000..ac23f9d3054 --- /dev/null +++ b/tests/release/distro-upgrade/check-snapd-version.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -euxo pipefail + +version="$1" + +if not tests.info is-reexec-in-use; then + snap version --verbose | MATCH "snapd-bin-from.*native-package$" + snap version --verbose | MATCH "snap-bin-from.*native-package$" +else + snap version --verbose | MATCH "snapd-bin-from.*snap$" + snap version --verbose | MATCH "snap-bin-from.*snap$" +fi + +if not tests.info is-reexec-in-use && tests.info is-snapd-from-archive; then + . /etc/os-release + version_id_re="${VERSION_ID//./\\.}" + snap version | grep -E '^snap[[:space:]]' | tee /dev/stderr | MATCH "^snap[[:space:]]+[0-9]+[.][0-9]+([.][0-9]+)?([.][0-9]+)?[+]ubuntu${version_id_re}([.][0-9]+)?$" + snap version | grep -E '^snapd[[:space:]]' | tee /dev/stderr | MATCH "^snapd[[:space:]]+[0-9]+[.][0-9]+([.][0-9]+)?([.][0-9]+)?[+]ubuntu${version_id_re}([.][0-9]+)?$" +fi + +snap version | grep -E '^snap[[:space:]]' | tee /dev/stderr | grep -qF "$version" +snap version | grep -E '^snapd[[:space:]]' | tee /dev/stderr | grep -qF "$version" \ No newline at end of file diff --git a/tests/release/distro-upgrade/check-snapd.sh b/tests/release/distro-upgrade/check-snapd.sh index 8838102da63..414f1551697 100755 --- a/tests/release/distro-upgrade/check-snapd.sh +++ b/tests/release/distro-upgrade/check-snapd.sh @@ -8,17 +8,15 @@ systemctl is-active snapd.socket tests.invariant check crashed-snap-confine tests.invariant check broken-snaps -snap version | grep -q "$(cat snap-version.txt)" - snap debug confinement | MATCH "strict" -snap connections go-example-webserver > tmp-webserver-connectionts.txt -diff -u webserver-connections.txt tmp-webserver-connectionts.txt +snap connections go-example-webserver > connections.txt +diff -u webserver-connections.txt connections.txt tests.systemd wait-for-service -n 30 --state active snap.go-example-webserver.webserver.service curl --fail --silent --show-error -o /dev/null localhost:8081 -snap list > tmp-snap-list.txt +snap list | grep -v "^snapd[[:space:]]" > tmp-snap-list.txt diff -u snap-list.txt tmp-snap-list.txt test-snapd-sh.sh -c 'echo Hello' | MATCH "Hello" diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index 4605b5caa1a..32f192cff3e 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -1,9 +1,15 @@ summary: Check that with the new snapd installed, we can upgrade the distro without issues. details: | - This test performs a series of distro upgrades until it reaches the latest devel verison. + This test performs a series of distro upgrades until it reaches the latest devel version. + It should be triggered from the earliest supported LTS version still in standard maintanence. + It can run against a snap, a spread-built deb, debs from a custom PPA, or from proposed. After each upgrade, it checks that snapd is still working correctly and performs some simple smoke checks. It expects to have done an upgrade to the latest LTS, the latest interim release and the latest devel release. + This spread tests has artifacts to verify snapd versions at each upgrade. Each file is named after its + distro's codename and contains the output of `snap version --verbose` on that distro. + Env for running on snappy-dev: SPREAD_SNAP_REEXEC=0 SPREAD_PPA_VALIDATION_NAME=ppa:snappy-dev/image + Env for running on proposed: SPREAD_SNAP_REEXEC=0 SPREAD_SRU_VALIDATION=1 # This test should only be run on the lowest supported version of ubuntu for a given release systems: @@ -19,9 +25,18 @@ prepare: | "$TESTSTOOLS"/snaps-state install-local test-snapd-classic-confinement --classic snap connections go-example-webserver > webserver-connections.txt - snap version | grep snap > snap-version.txt - snap list > snap-list.txt + snap list | grep -v "^snapd[[:space:]]" > snap-list.txt sed -i 's/Prompt=never/Prompt=normal/' /etc/update-manager/release-upgrades + if not tests.info is-reexec-in-use && tests.info is-snapd-from-archive; then + . /etc/os-release + version_id_re="${VERSION_ID//./\\.}" + snap version | grep -E '^snap[[:space:]]' | MATCH "[0-9]+[.][0-9]+([.][0-9]+)?([.][0-9]+)?[+]ubuntu${version_id_re}([.][0-9]+)?" + snap version | grep -E '^snapd[[:space:]]' | MATCH "[0-9]+[.][0-9]+([.][0-9]+)?([.][0-9]+)?[+]ubuntu${version_id_re}([.][0-9]+)?" + else + snap version | grep -E '^snap[[:space:]]' | MATCH "1337[.][0-9]+([.][0-9]+)?([.][0-9]+)?" + snap version | grep -E '^snapd[[:space:]]' | MATCH "1337[.][0-9]+([.][0-9]+)?([.][0-9]+)?" + fi + snap version | grep -E '^snapd[[:space:]]' | awk '{print $2}' | sed 's/+ubuntu.*//' > initial-snapd-version.txt restore: | snap remove --purge go-example-webserver @@ -31,14 +46,17 @@ restore: | sed -i 's/Prompt=normal/Prompt=never/' /etc/update-manager/release-upgrades execute: | - while ! os.query is-ubuntu-devel; do + # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro + # doesn't have the latest yet. + ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] + + while not os.query is-ubuntu-devel; do # We can only upgrade the distro if we have all upgrades installed apt update apt upgrade -y if [[ "$SPREAD_REBOOT" = 0 ]]; then REBOOT fi - devel_option= if os.query is-ubuntu-interim; then devel_option="-d" @@ -47,7 +65,18 @@ execute: | echo "Upgrading $(grep -oP 'CODENAME=\K.*' /etc/os-release | head -1)" sudo do-release-upgrade "$devel_option" -f DistUpgradeViewNonInteractive + if [ "$SRU_VALIDATION" = "1" ] || [ -n "$PPA_VALIDATION_NAME" ]; then + . "$TESTSLIB"/pkgdb.sh + distro_install_build_snapd + fi + + . "$TESTSLIB"/prepare.sh + prepare_each_classic + ./check-snapd.sh + # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro + # doesn't have the latest yet. + ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] if os.query is-ubuntu-latest-lts; then touch lts-upgrade-done From 05afad63964e301eb42783e42540a25b56c15d9f Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 22 Apr 2026 09:14:31 +0200 Subject: [PATCH 35/44] github: remove deb pre-build --- .github/workflows/ci-release.yaml | 45 ++----------------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 1a8df8bf0f4..4cd0dcf974e 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -22,7 +22,6 @@ jobs: runs-on: ubuntu-latest needs: run-gate outputs: - version: ${{ steps.find-lowest-supported.outputs.version }} system: ${{ steps.find-lowest-supported.outputs.system }} steps: - name: Find lowest supported Ubuntu version @@ -30,51 +29,11 @@ jobs: run: | sudo apt update version=$(ubuntu-distro-info --supported --release | head -1 | awk '{print $1}') - echo "Setting lowest supported version to $version and system to ubuntu-$version-64" + echo "Setting lowest supported system to ubuntu-$version-64" echo "system=ubuntu-$version-64" >> $GITHUB_OUTPUT - echo "version=$version" >> $GITHUB_OUTPUT - - create-vendor-dir: - runs-on: ubuntu-latest - needs: run-gate - name: Create go vendor - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: vendor - run: | - sudo snap install --classic --channel 1.18/stable go - /snap/bin/go mod vendor - ( - cd c-vendor - ./vendor.sh - ) - mkdir /tmp/pkg - base_version="$(head -1 packaging/ubuntu-16.04/changelog | awk -F '[()]' '{print $2}')" - version="1337.$base_version" - ./packaging/pack-source -v "$version" -o /tmp/pkg - ./packaging/pack-source -v "$version" -o /tmp/pkg -s - echo "$version" > /tmp/pkg/version - - - name: Upload vendors - uses: actions/upload-artifact@v7 - with: - name: vendor-pkgs - path: /tmp/pkg - - package-snapd-deb: - needs: [find-lowest-supported, create-vendor-dir] - uses: ./.github/workflows/deb-builds.yaml - name: Package ${{ matrix.build-config.system }} - with: - runs-on: '["ubuntu-latest"]' - system: ${{ needs.find-lowest-supported.outputs.system }} - os: ubuntu - os-version: ${{ needs.find-lowest-supported.outputs.version }} test-snapd-deb: - needs: [package-snapd-deb, snap-builds, find-lowest-supported] + needs: [snap-builds, find-lowest-supported] uses: ./.github/workflows/spread-tests.yaml with: runs-on: '["self-hosted", "spread-enabled"]' From aec1aced44d2eb0b377b5184b369fa1c026cc55a Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 22 Apr 2026 15:46:13 +0200 Subject: [PATCH 36/44] tests, github: add artifact collection to verify snapd versions during distro upgrade --- .github/workflows/ci-release.yaml | 2 ++ tests/release/distro-upgrade/task.yaml | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 4cd0dcf974e..a2790bf9ecd 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -43,6 +43,7 @@ jobs: tasks: 'tests/release/distro-upgrade' rules: '' spread-env: "SPREAD_SNAPD_DEB_FROM_REPO=false SPREAD_SNAP_REEXEC=0" + upload-artifacts: true test-snapd-snap: needs: [snap-builds, find-lowest-supported] @@ -54,3 +55,4 @@ jobs: systems: ${{ needs.find-lowest-supported.outputs.system }} tasks: 'tests/release/distro-upgrade' rules: '' + upload-artifacts: true diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index 32f192cff3e..bf19785cac3 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -18,6 +18,9 @@ systems: manual: true +artifacts: + - snapd-versions + prepare: | snap install go-example-webserver snap install test-snapd-sh @@ -38,6 +41,8 @@ prepare: | fi snap version | grep -E '^snapd[[:space:]]' | awk '{print $2}' | sed 's/+ubuntu.*//' > initial-snapd-version.txt + mkdir snapd-versions + restore: | snap remove --purge go-example-webserver snap remove --purge test-snapd-sh @@ -49,6 +54,7 @@ execute: | # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro # doesn't have the latest yet. ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] + snap version --verbose > snapd-versions/$(lsb_release -c -s) while not os.query is-ubuntu-devel; do # We can only upgrade the distro if we have all upgrades installed @@ -77,6 +83,7 @@ execute: | # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro # doesn't have the latest yet. ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] + snap version --verbose > snapd-versions/$(lsb_release -c -s) if os.query is-ubuntu-latest-lts; then touch lts-upgrade-done From 8c817090ed57a3e4d9a876e5cd660fe1cf3b4bdc Mon Sep 17 00:00:00 2001 From: Katie May Date: Wed, 22 Apr 2026 18:40:53 +0200 Subject: [PATCH 37/44] tests: fix static checks --- tests/release/distro-upgrade/task.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index bf19785cac3..b38ef731609 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -54,7 +54,7 @@ execute: | # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro # doesn't have the latest yet. ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] - snap version --verbose > snapd-versions/$(lsb_release -c -s) + snap version --verbose > snapd-versions/"$(lsb_release -c -s)" while not os.query is-ubuntu-devel; do # We can only upgrade the distro if we have all upgrades installed @@ -72,10 +72,12 @@ execute: | sudo do-release-upgrade "$devel_option" -f DistUpgradeViewNonInteractive if [ "$SRU_VALIDATION" = "1" ] || [ -n "$PPA_VALIDATION_NAME" ]; then + #shellcheck source=tests/lib/pkgdb.sh . "$TESTSLIB"/pkgdb.sh distro_install_build_snapd fi + #shellcheck source=tests/lib/prepare.sh . "$TESTSLIB"/prepare.sh prepare_each_classic @@ -83,7 +85,7 @@ execute: | # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro # doesn't have the latest yet. ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] - snap version --verbose > snapd-versions/$(lsb_release -c -s) + snap version --verbose > snapd-versions/"$(lsb_release -c -s)" if os.query is-ubuntu-latest-lts; then touch lts-upgrade-done From a67a4308f9da46aff3edc7ca5330a41d5ec415bd Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 08:45:49 +0200 Subject: [PATCH 38/44] spread, tests: update comments; add longer kill timeout --- spread.yaml | 3 +++ tests/release/distro-upgrade/task.yaml | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/spread.yaml b/spread.yaml index b9260253838..e76c342ad34 100644 --- a/spread.yaml +++ b/spread.yaml @@ -1594,6 +1594,9 @@ suites: "$TESTSLIB"/prepare-restore.sh --prepare-suite prepare-each: | "$TESTSLIB"/prepare-restore.sh --prepare-suite-each + # This suite does not have restore steps due to the destructive + # nature of its tests that will cause the vanila restore scripts to + # fail. The tests in this suite are all meant to be run in isolation. tests/upgrade/: summary: Tests for snapd upgrade diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index b38ef731609..aea072faff3 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -6,7 +6,7 @@ details: | It can run against a snap, a spread-built deb, debs from a custom PPA, or from proposed. After each upgrade, it checks that snapd is still working correctly and performs some simple smoke checks. It expects to have done an upgrade to the latest LTS, the latest interim release and the latest devel release. - This spread tests has artifacts to verify snapd versions at each upgrade. Each file is named after its + This spread test has artifacts to verify snapd versions at each upgrade. Each file is named after its distro's codename and contains the output of `snap version --verbose` on that distro. Env for running on snappy-dev: SPREAD_SNAP_REEXEC=0 SPREAD_PPA_VALIDATION_NAME=ppa:snappy-dev/image Env for running on proposed: SPREAD_SNAP_REEXEC=0 SPREAD_SRU_VALIDATION=1 @@ -18,6 +18,9 @@ systems: manual: true +warn-timeout: 80m +kill-timeout: 90m + artifacts: - snapd-versions From 1abc028bd0c519f7c08f500519305529d93b4bb1 Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 13:57:22 +0200 Subject: [PATCH 39/44] tests: remove support for ubuntu-1; slower warn-timeout --- tests/release/distro-upgrade/task.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index aea072faff3..d0a049be023 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -13,12 +13,11 @@ details: | # This test should only be run on the lowest supported version of ubuntu for a given release systems: - - ubuntu-1* - ubuntu-2* manual: true -warn-timeout: 80m +warn-timeout: 10m kill-timeout: 90m artifacts: From 94fdc1938ccd37ffa99bb7f035c0beb4ecb5d823 Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 15:56:38 +0200 Subject: [PATCH 40/44] github: echo snapd versions at end of workflow --- .github/workflows/ci-release.yaml | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index a2790bf9ecd..442fc785065 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -56,3 +56,59 @@ jobs: tasks: 'tests/release/distro-upgrade' rules: '' upload-artifacts: true + + report-results: + needs: [test-snapd-deb, test-snapd-snap] + runs-on: ubuntu-latest + steps: + - name: Download + uses: actions/github-script@v9 + with: + script: | + let page = 1; + let per_page = 100; + let allArtifacts = []; + let response; + do { + response = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + per_page: per_page, + page: page + }); + allArtifacts = allArtifacts.concat(response.data.artifacts); + page++; + } while (allArtifacts.length < response.data.total_count); + + let matchingArtifacts = allArtifacts.filter((artifact) => { + return artifact.name.startsWith(`spread-artifacts-`); + }); + + for (let artifact of matchingArtifacts) { + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifact.name}.zip`, Buffer.from(download.data)); + console.log(`Downloaded artifact: ${artifact.name}.zip`); + } + + - name: Unzip artifacts + run: | + find . -name "spread-artifacts-*.zip" | while read filename; do + dir="${filename%.zip}" + mkdir "$dir" + unzip "$filename" -d "$dir" + done + + - name: Report results + run: | + echo "Snap results" + find spread-artifacts-snap-test* -type f -exec sh -c "echo $1; cat $1" _ {} \; + echo "Deb results" + find spread-artifacts-deb-test* -type f -exec sh -c "echo $1; cat $1" _ {} \; + From 6513b60235d4639a598651e906ecd0bd36dddd52 Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 15:56:55 +0200 Subject: [PATCH 41/44] debug --- .github/workflows/ci-release.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 442fc785065..bab4c9f8cfc 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -36,9 +36,9 @@ jobs: needs: [snap-builds, find-lowest-supported] uses: ./.github/workflows/spread-tests.yaml with: - runs-on: '["self-hosted", "spread-enabled"]' + runs-on: '["ubuntu-latest"]' group: deb-test - backend: openstack + backend: garden systems: ${{ needs.find-lowest-supported.outputs.system }} tasks: 'tests/release/distro-upgrade' rules: '' @@ -49,9 +49,9 @@ jobs: needs: [snap-builds, find-lowest-supported] uses: ./.github/workflows/spread-tests.yaml with: - runs-on: '["self-hosted", "spread-enabled"]' + runs-on: '["ubuntu-latest"]' group: snap-test - backend: openstack + backend: garden systems: ${{ needs.find-lowest-supported.outputs.system }} tasks: 'tests/release/distro-upgrade' rules: '' From f3cf71c35e19e6903eaa59b3e02d85837379fc8d Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 16:01:35 +0200 Subject: [PATCH 42/44] debug --- tests/release/distro-upgrade/task.yaml | 109 +++++++++++++------------ 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/tests/release/distro-upgrade/task.yaml b/tests/release/distro-upgrade/task.yaml index d0a049be023..267d440143b 100644 --- a/tests/release/distro-upgrade/task.yaml +++ b/tests/release/distro-upgrade/task.yaml @@ -57,57 +57,58 @@ execute: | # doesn't have the latest yet. ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] snap version --verbose > snapd-versions/"$(lsb_release -c -s)" - - while not os.query is-ubuntu-devel; do - # We can only upgrade the distro if we have all upgrades installed - apt update - apt upgrade -y - if [[ "$SPREAD_REBOOT" = 0 ]]; then - REBOOT - fi - devel_option= - if os.query is-ubuntu-interim; then - devel_option="-d" - fi - - echo "Upgrading $(grep -oP 'CODENAME=\K.*' /etc/os-release | head -1)" - sudo do-release-upgrade "$devel_option" -f DistUpgradeViewNonInteractive - - if [ "$SRU_VALIDATION" = "1" ] || [ -n "$PPA_VALIDATION_NAME" ]; then - #shellcheck source=tests/lib/pkgdb.sh - . "$TESTSLIB"/pkgdb.sh - distro_install_build_snapd - fi - - #shellcheck source=tests/lib/prepare.sh - . "$TESTSLIB"/prepare.sh - prepare_each_classic - - ./check-snapd.sh - # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro - # doesn't have the latest yet. - ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] - snap version --verbose > snapd-versions/"$(lsb_release -c -s)" - - if os.query is-ubuntu-latest-lts; then - touch lts-upgrade-done - elif os.query is-ubuntu-interim; then - touch interim-upgrade-done - elif os.query is-ubuntu-devel; then - touch devel-upgrade-done - fi - SPREAD_REBOOT=0 - done - - if not [ -f lts-upgrade-done ]; then - echo "We should have done an LTS upgrade but didn't" - exit 1 - fi - if not [ -f interim-upgrade-done ]; then - echo "We should have done an interim upgrade but didn't" - exit 1 - fi - if not [ -f devel-upgrade-done ]; then - echo "We should have done a devel upgrade but didn't" - exit 1 - fi + snap version --verbose > snapd-versions/resolute + + # while not os.query is-ubuntu-devel; do + # # We can only upgrade the distro if we have all upgrades installed + # apt update + # apt upgrade -y + # if [[ "$SPREAD_REBOOT" = 0 ]]; then + # REBOOT + # fi + # devel_option= + # if os.query is-ubuntu-interim; then + # devel_option="-d" + # fi + + # echo "Upgrading $(grep -oP 'CODENAME=\K.*' /etc/os-release | head -1)" + # sudo do-release-upgrade "$devel_option" -f DistUpgradeViewNonInteractive + + # if [ "$SRU_VALIDATION" = "1" ] || [ -n "$PPA_VALIDATION_NAME" ]; then + # #shellcheck source=tests/lib/pkgdb.sh + # . "$TESTSLIB"/pkgdb.sh + # distro_install_build_snapd + # fi + + # #shellcheck source=tests/lib/prepare.sh + # . "$TESTSLIB"/prepare.sh + # prepare_each_classic + + # ./check-snapd.sh + # # Don't fail the test if the distro is missing the latest snapd version. Sometimes one distro + # # doesn't have the latest yet. + # ./check-snapd-version.sh "$(cat initial-snapd-version.txt)" || [ "$SRU_VALIDATION" = "1" ] + # snap version --verbose > snapd-versions/"$(lsb_release -c -s)" + + # if os.query is-ubuntu-latest-lts; then + # touch lts-upgrade-done + # elif os.query is-ubuntu-interim; then + # touch interim-upgrade-done + # elif os.query is-ubuntu-devel; then + # touch devel-upgrade-done + # fi + # SPREAD_REBOOT=0 + # done + + # if not [ -f lts-upgrade-done ]; then + # echo "We should have done an LTS upgrade but didn't" + # exit 1 + # fi + # if not [ -f interim-upgrade-done ]; then + # echo "We should have done an interim upgrade but didn't" + # exit 1 + # fi + # if not [ -f devel-upgrade-done ]; then + # echo "We should have done a devel upgrade but didn't" + # exit 1 + # fi From 29ddbe228aacbdc93b035bbf0f2185b2781f4cfe Mon Sep 17 00:00:00 2001 From: Katie May Date: Thu, 23 Apr 2026 16:37:22 +0200 Subject: [PATCH 43/44] correction --- .github/workflows/ci-release.yaml | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index bab4c9f8cfc..911cafcee3c 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -73,7 +73,7 @@ jobs: response = await github.rest.actions.listWorkflowRunArtifacts({ owner: context.repo.owner, repo: context.repo.repo, - run_id: context.payload.workflow_run.id, + run_id: context.runId, per_page: per_page, page: page }); @@ -103,12 +103,37 @@ jobs: dir="${filename%.zip}" mkdir "$dir" unzip "$filename" -d "$dir" + rm "$filename" done + - name: Untar artifacts + run: | + find spread-artifacts* -type f + find . -name "*.tar.gz" | while read filename; do + echo "----$filename----" + dir="${filename%.tar.gz}" + echo "dir: $dir" + mkdir -p "$dir" + tar -xzf "$filename" -C "$dir" + rm "$filename" + done + echo "All files" + find . -type f + echo "Delete .txt" + find . -name "*.txt" -type f -delete + echo "All files after delete" + find . -type f + - name: Report results run: | + echo "All files" + find spread-artifacts* -type f + echo "All snap files" + find spread-artifacts-snap-test* -type f + echo "All deb files" + find spread-artifacts-deb-test* -type f echo "Snap results" - find spread-artifacts-snap-test* -type f -exec sh -c "echo $1; cat $1" _ {} \; + find spread-artifacts-snap-test* -type f -exec sh -c 'echo $1; cat $1' _ {} \; echo "Deb results" - find spread-artifacts-deb-test* -type f -exec sh -c "echo $1; cat $1" _ {} \; + find spread-artifacts-deb-test* -type f -exec sh -c 'echo $1; cat $1' _ {} \; From fc44a2b89eb7ff2a291851a0f3779218b461833e Mon Sep 17 00:00:00 2001 From: Katie May Date: Fri, 24 Apr 2026 10:20:11 +0200 Subject: [PATCH 44/44] use-download --- .github/workflows/ci-release.yaml | 65 +++++-------------------------- 1 file changed, 9 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 911cafcee3c..3ee3c87f1c3 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -62,78 +62,31 @@ jobs: runs-on: ubuntu-latest steps: - name: Download - uses: actions/github-script@v9 + uses: actions/download-artifact@v8 with: - script: | - let page = 1; - let per_page = 100; - let allArtifacts = []; - let response; - do { - response = await github.rest.actions.listWorkflowRunArtifacts({ - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: per_page, - page: page - }); - allArtifacts = allArtifacts.concat(response.data.artifacts); - page++; - } while (allArtifacts.length < response.data.total_count); - - let matchingArtifacts = allArtifacts.filter((artifact) => { - return artifact.name.startsWith(`spread-artifacts-`); - }); - - for (let artifact of matchingArtifacts) { - let download = await github.rest.actions.downloadArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact.id, - archive_format: 'zip', - }); - let fs = require('fs'); - fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifact.name}.zip`, Buffer.from(download.data)); - console.log(`Downloaded artifact: ${artifact.name}.zip`); - } - - - name: Unzip artifacts - run: | - find . -name "spread-artifacts-*.zip" | while read filename; do - dir="${filename%.zip}" - mkdir "$dir" - unzip "$filename" -d "$dir" - rm "$filename" - done + pattern: spread-artifacts-* + merge-multiple: true - name: Untar artifacts run: | - find spread-artifacts* -type f find . -name "*.tar.gz" | while read filename; do - echo "----$filename----" dir="${filename%.tar.gz}" - echo "dir: $dir" mkdir -p "$dir" tar -xzf "$filename" -C "$dir" rm "$filename" done - echo "All files" - find . -type f - echo "Delete .txt" find . -name "*.txt" -type f -delete echo "All files after delete" find . -type f - name: Report results run: | - echo "All files" - find spread-artifacts* -type f - echo "All snap files" - find spread-artifacts-snap-test* -type f - echo "All deb files" - find spread-artifacts-deb-test* -type f - echo "Snap results" + echo "--------------------------------------------" + echo "----------------Snap results----------------" + echo "--------------------------------------------" find spread-artifacts-snap-test* -type f -exec sh -c 'echo $1; cat $1' _ {} \; - echo "Deb results" + echo "-------------------------------------------" + echo "----------------Deb results----------------" + echo "-------------------------------------------" find spread-artifacts-deb-test* -type f -exec sh -c 'echo $1; cat $1' _ {} \;