Skip to content

Commit 3a0d8bf

Browse files
authored
Merge pull request #38 from dmcgowan/fix-mount-handling
Fix mount handling causing errors in VM
2 parents 23a2a38 + f91e0ba commit 3a0d8bf

File tree

8 files changed

+283
-221
lines changed

8 files changed

+283
-221
lines changed

internal/mountutil/mount.go

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
//go:build linux
2+
3+
/*
4+
Copyright The containerd Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
// mountutil performs local mounts on Linux. This package should likely
20+
// be replaced with functions in the containerd mount code.
21+
package mountutil
22+
23+
import (
24+
"bytes"
25+
"context"
26+
"fmt"
27+
"os"
28+
"path/filepath"
29+
"strings"
30+
"text/template"
31+
"time"
32+
33+
types "github.com/containerd/containerd/api/types"
34+
"github.com/containerd/containerd/v2/core/mount"
35+
"github.com/containerd/log"
36+
)
37+
38+
func All(ctx context.Context, rootfs, mdir string, mounts []*types.Mount) (retErr error) {
39+
log.G(ctx).WithField("mounts", mounts).Debugf("mounting rootfs components")
40+
active := []mount.ActiveMount{}
41+
42+
// TODO: Use mount manager interface, mount temps to directory
43+
for i, m := range mounts {
44+
var target string
45+
if i < len(mounts)-1 {
46+
target = filepath.Join(mdir, fmt.Sprintf("%d", i))
47+
if err := os.MkdirAll(target, 0711); err != nil {
48+
return err
49+
}
50+
} else {
51+
target = rootfs
52+
}
53+
if t, ok := strings.CutPrefix(m.Type, "format/"); ok {
54+
m.Type = t
55+
for i, o := range m.Options {
56+
format := formatString(o)
57+
if format != nil {
58+
s, err := format(active)
59+
if err != nil {
60+
return fmt.Errorf("formatting mount option %q: %w", o, err)
61+
}
62+
m.Options[i] = s
63+
}
64+
}
65+
if format := formatString(m.Source); format != nil {
66+
s, err := format(active)
67+
if err != nil {
68+
return fmt.Errorf("formatting mount source %q: %w", m.Source, err)
69+
}
70+
m.Source = s
71+
}
72+
if format := formatString(m.Target); format != nil {
73+
s, err := format(active)
74+
if err != nil {
75+
return fmt.Errorf("formatting mount target %q: %w", m.Target, err)
76+
}
77+
m.Target = s
78+
}
79+
}
80+
if t, ok := strings.CutPrefix(m.Type, "mkdir/"); ok {
81+
m.Type = t
82+
var options []string
83+
for _, o := range m.Options {
84+
if strings.HasPrefix(o, "X-containerd.mkdir.") {
85+
prefix := "X-containerd.mkdir.path="
86+
if !strings.HasPrefix(o, prefix) {
87+
return fmt.Errorf("unknown mkdir mount option %q", o)
88+
}
89+
part := strings.SplitN(o[len(prefix):], ":", 4)
90+
switch len(part) {
91+
case 4:
92+
// TODO: Support setting uid/gid
93+
fallthrough
94+
case 3:
95+
fallthrough
96+
case 2:
97+
fallthrough
98+
case 1:
99+
dir := part[0]
100+
if !strings.HasPrefix(dir, mdir) {
101+
return fmt.Errorf("mkdir mount source %q must be under %q", dir, mdir)
102+
}
103+
if err := os.MkdirAll(dir, 0755); err != nil {
104+
return err
105+
}
106+
default:
107+
return fmt.Errorf("invalid mkdir mount option %q", o)
108+
}
109+
} else {
110+
options = append(options, o)
111+
}
112+
}
113+
m.Options = options
114+
115+
}
116+
t := time.Now()
117+
am := mount.ActiveMount{
118+
Mount: mount.Mount{
119+
Type: m.Type,
120+
Source: m.Source,
121+
Target: m.Target,
122+
Options: m.Options,
123+
},
124+
MountedAt: &t,
125+
MountPoint: target,
126+
}
127+
if err := am.Mount.Mount(target); err != nil {
128+
return err
129+
}
130+
active = append(active, am)
131+
132+
}
133+
defer func() {
134+
if retErr != nil {
135+
for i := len(active) - 1; i >= 0; i-- {
136+
// TODO: delegate custom types to handlers
137+
if active[i].Type == "mkdir" {
138+
continue
139+
}
140+
if err := mount.UnmountAll(active[i].MountPoint, 0); err != nil {
141+
log.G(ctx).WithError(err).WithField("mountpoint", active[i].MountPoint).Warn("failed to cleanup mount")
142+
}
143+
}
144+
}
145+
}()
146+
147+
return nil
148+
}
149+
150+
const formatCheck = "{{"
151+
152+
func formatString(s string) func([]mount.ActiveMount) (string, error) {
153+
if !strings.Contains(s, formatCheck) {
154+
return nil
155+
}
156+
157+
return func(a []mount.ActiveMount) (string, error) {
158+
fm := template.FuncMap{
159+
"source": func(i int) (string, error) {
160+
if i < 0 || i >= len(a) {
161+
return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
162+
}
163+
return a[i].Source, nil
164+
},
165+
"target": func(i int) (string, error) {
166+
if i < 0 || i >= len(a) {
167+
return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
168+
}
169+
return a[i].Target, nil
170+
},
171+
"mount": func(i int) (string, error) {
172+
if i < 0 || i >= len(a) {
173+
return "", fmt.Errorf("index out of bounds: %d, has %d active mounts", i, len(a))
174+
}
175+
return a[i].MountPoint, nil
176+
},
177+
"overlay": func(start, end int) (string, error) {
178+
var dirs []string
179+
if start > end {
180+
if start >= len(a) || end < 0 {
181+
return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a))
182+
}
183+
for i := start; i >= end; i-- {
184+
dirs = append(dirs, a[i].MountPoint)
185+
}
186+
} else {
187+
if start < 0 || end >= len(a) {
188+
return "", fmt.Errorf("invalid range: %d-%d, has %d active mounts", start, end, len(a))
189+
}
190+
for i := start; i <= end; i++ {
191+
dirs = append(dirs, a[i].MountPoint)
192+
}
193+
}
194+
return strings.Join(dirs, ":"), nil
195+
},
196+
}
197+
t, err := template.New("").Funcs(fm).Parse(s)
198+
if err != nil {
199+
return "", err
200+
}
201+
202+
buf := bytes.NewBuffer(nil)
203+
if err := t.Execute(buf, nil); err != nil {
204+
return "", err
205+
}
206+
return buf.String(), nil
207+
}
208+
}

internal/shim/manager/manager.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -333,7 +333,7 @@ func (m manager) Info(ctx context.Context, optionsR io.Reader) (*types.RuntimeIn
333333
//Revision: version.Revision,
334334
},
335335
Annotations: map[string]string{
336-
"containerd.io/runtime-allow-mounts": "mkdir/*,format/*",
336+
"containerd.io/runtime-allow-mounts": "mkdir/*,format/*,erofs",
337337
},
338338
}
339339
// TODO: Get features list from run_vminitd

internal/shim/task/mount.go

Lines changed: 42 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,29 @@ package task
1919
import (
2020
"context"
2121
"fmt"
22-
"path/filepath"
2322
"strings"
2423

2524
"github.com/containerd/containerd/api/types"
25+
"github.com/containerd/errdefs"
2626
"github.com/containerd/log"
2727

2828
"github.com/containerd/nerdbox/internal/vm"
2929
)
3030

31+
type diskOptions struct {
32+
name string
33+
source string
34+
readOnly bool
35+
}
36+
3137
// transformMounts does not perform any local mounts but transforms
3238
// the mounts to be used inside the VM via virtio
3339
func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount) ([]*types.Mount, error) {
3440
var (
35-
disks byte = 'a'
36-
am []*types.Mount
41+
disks byte = 'a'
42+
addDisks []diskOptions
43+
am []*types.Mount
44+
err error
3745
)
3846

3947
for _, m := range ms {
@@ -44,9 +52,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
4452
if len(disk) > 36 {
4553
disk = disk[:36]
4654
}
47-
if err := vmi.AddDisk(ctx, disk, m.Source, vm.WithReadOnly()); err != nil {
48-
return nil, err
49-
}
55+
addDisks = append(addDisks, diskOptions{
56+
name: disk,
57+
source: m.Source,
58+
readOnly: true,
59+
})
5060
am = append(am, &types.Mount{
5161
Type: "erofs",
5262
Source: fmt.Sprintf("/dev/vd%c", disks),
@@ -60,9 +70,12 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
6070
if len(disk) > 36 {
6171
disk = disk[:36]
6272
}
63-
if err := vmi.AddDisk(ctx, disk, m.Source); err != nil {
64-
return nil, err
65-
}
73+
// TODO: Check read only option
74+
addDisks = append(addDisks, diskOptions{
75+
name: disk,
76+
source: m.Source,
77+
readOnly: false,
78+
})
6679
am = append(am, &types.Mount{
6780
Type: "ext4",
6881
Source: fmt.Sprintf("/dev/vd%c", disks),
@@ -87,35 +100,11 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
87100
// TODO: Handle virtio for lowers?
88101
}
89102
if wdi > -1 && udi > -1 {
90-
udir, uname := filepath.Split(m.Options[udi][len("upperdir="):])
91-
92-
wdir, wname := filepath.Split(m.Options[wdi][len("workdir="):])
93-
if udir == wdir {
94-
tag := fmt.Sprintf("overlayfs-upper-%s", id)
95-
// virtiofs implementation has a limit of 36 characters for the tag
96-
if len(tag) > 36 {
97-
tag = tag[:36]
98-
}
99-
if err := vmi.AddFS(ctx, tag, udir); err != nil {
100-
return nil, err
101-
}
102-
m.Options[udi] = fmt.Sprintf("upperdir={{ mount %d }}/%s", len(am), uname)
103-
m.Options[wdi] = fmt.Sprintf("workdir={{ mount %d }}/%s", len(am), wname)
104-
am = append(am, &types.Mount{
105-
Type: "virtiofs",
106-
Source: tag,
107-
})
108-
log.G(ctx).WithFields(log.Fields{
109-
"workdir": m.Options[wdi],
110-
"upperdir": m.Options[udi],
111-
}).Warnf("transformed upper and work")
112-
} else {
113-
log.G(ctx).WithFields(log.Fields{
114-
"workdir": m.Options[wdi],
115-
"upperdir": m.Options[udi],
116-
}).Warnf("overlayfs workdir and upperdir should be in the same directory")
117-
}
118-
} else {
103+
// Having the upper as virtiofs may return invalid argument, avoid
104+
// transforming and attempt to perform the mounts on the host if
105+
// supported.
106+
return nil, fmt.Errorf("cannot use virtiofs for upper dir in overlay: %w", errdefs.ErrNotImplemented)
107+
} else if wdi == -1 || udi == -1 {
119108
log.G(ctx).WithField("options", m.Options).Warnf("overlayfs missing workdir or upperdir")
120109
}
121110

@@ -125,7 +114,21 @@ func transformMounts(ctx context.Context, vmi vm.Instance, id string, ms []*type
125114
}
126115
}
127116

128-
return am, nil
117+
if len(addDisks) > 10 {
118+
return nil, fmt.Errorf("exceeded maximum virtio disk count: %d > 10: %w", len(addDisks), errdefs.ErrNotImplemented)
119+
}
120+
121+
for _, do := range addDisks {
122+
var opts []vm.MountOpt
123+
if do.readOnly {
124+
opts = append(opts, vm.WithReadOnly())
125+
}
126+
if err := vmi.AddDisk(ctx, do.name, do.source, opts...); err != nil {
127+
return nil, err
128+
}
129+
}
130+
131+
return am, err
129132
}
130133

131134
func filterOptions(options []string) []string {

internal/shim/task/mount_darwin.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ import (
2424
"github.com/containerd/nerdbox/internal/vm"
2525
)
2626

27-
func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _ string) ([]*types.Mount, error) {
27+
func setupMounts(ctx context.Context, vmi vm.Instance, id string, ms []*types.Mount, _, _ string) ([]*types.Mount, error) {
2828
return transformMounts(ctx, vmi, id, ms)
2929
}

internal/shim/task/mount_linux.go

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@ import (
2222

2323
"github.com/containerd/containerd/api/types"
2424
"github.com/containerd/containerd/v2/core/mount"
25+
"github.com/containerd/errdefs"
2526

27+
"github.com/containerd/nerdbox/internal/mountutil"
2628
"github.com/containerd/nerdbox/internal/vm"
2729
)
2830

29-
func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs string) ([]*types.Mount, error) {
31+
func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mount, rootfs, lmounts string) ([]*types.Mount, error) {
3032
// Handle mounts
3133

3234
if len(m) == 1 && (m[0].Type == "overlay" || m[0].Type == "bind") {
@@ -66,5 +68,25 @@ func setupMounts(ctx context.Context, vmi vm.Instance, id string, m []*types.Mou
6668
Source: tag,
6769
}}, nil
6870
}
69-
return transformMounts(ctx, vmi, id, m)
71+
mounts, err := transformMounts(ctx, vmi, id, m)
72+
if err != nil && errdefs.IsNotImplemented(err) {
73+
if err := mountutil.All(ctx, rootfs, lmounts, m); err != nil {
74+
return nil, err
75+
}
76+
77+
// Fallback to original rootfs mount
78+
tag := fmt.Sprintf("rootfs-%s", id)
79+
// virtiofs implementation has a limit of 36 characters for the tag
80+
if len(tag) > 36 {
81+
tag = tag[:36]
82+
}
83+
if err := vmi.AddFS(ctx, tag, rootfs); err != nil {
84+
return nil, err
85+
}
86+
return []*types.Mount{{
87+
Type: "virtiofs",
88+
Source: tag,
89+
}}, nil
90+
}
91+
return mounts, err
7092
}

0 commit comments

Comments
 (0)