From 37c4cb0c104717fda4f8a7237efbba464c440654 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 3 Dec 2025 15:11:55 +0100 Subject: [PATCH 1/9] [no-relnote] Move tegra.tegraOptions to separate file Signed-off-by: Evan Lezar --- internal/platform-support/tegra/options.go | 101 +++++++++++++++++++++ internal/platform-support/tegra/tegra.go | 78 ---------------- 2 files changed, 101 insertions(+), 78 deletions(-) create mode 100644 internal/platform-support/tegra/options.go diff --git a/internal/platform-support/tegra/options.go b/internal/platform-support/tegra/options.go new file mode 100644 index 000000000..538a0cc22 --- /dev/null +++ b/internal/platform-support/tegra/options.go @@ -0,0 +1,101 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package tegra + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" + "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" + "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" +) + +type tegraOptions struct { + logger logger.Interface + csvFiles []string + driverRoot string + devRoot string + hookCreator discover.HookCreator + ldconfigPath string + librarySearchPaths []string + ignorePatterns ignoreMountSpecPatterns + + // The following can be overridden for testing + symlinkLocator lookup.Locator + symlinkChainLocator lookup.Locator + // TODO: This should be replaced by a regular mock + resolveSymlink func(string) (string, error) +} + +// Option defines a functional option for configuring a Tegra discoverer. +type Option func(*tegraOptions) + +// WithLogger sets the logger for the discoverer. +func WithLogger(logger logger.Interface) Option { + return func(o *tegraOptions) { + o.logger = logger + } +} + +// WithDriverRoot sets the driver root for the discoverer. +func WithDriverRoot(driverRoot string) Option { + return func(o *tegraOptions) { + o.driverRoot = driverRoot + } +} + +// WithDevRoot sets the /dev root. +// If this is unset, the driver root is assumed. +func WithDevRoot(devRoot string) Option { + return func(o *tegraOptions) { + o.devRoot = devRoot + } +} + +// WithCSVFiles sets the CSV files for the discoverer. +func WithCSVFiles(csvFiles []string) Option { + return func(o *tegraOptions) { + o.csvFiles = csvFiles + } +} + +// WithHookCreator sets the hook creator for the discoverer. +func WithHookCreator(hookCreator discover.HookCreator) Option { + return func(o *tegraOptions) { + o.hookCreator = hookCreator + } +} + +// WithLdconfigPath sets the path to the ldconfig program +func WithLdconfigPath(ldconfigPath string) Option { + return func(o *tegraOptions) { + o.ldconfigPath = ldconfigPath + } +} + +// WithLibrarySearchPaths sets the library search paths for the discoverer. +func WithLibrarySearchPaths(librarySearchPaths ...string) Option { + return func(o *tegraOptions) { + o.librarySearchPaths = librarySearchPaths + } +} + +// WithIngorePatterns sets patterns to ignore in the CSV files +func WithIngorePatterns(ignorePatterns ...string) Option { + return func(o *tegraOptions) { + o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns) + } +} diff --git a/internal/platform-support/tegra/tegra.go b/internal/platform-support/tegra/tegra.go index 6ad774b4e..26264f42c 100644 --- a/internal/platform-support/tegra/tegra.go +++ b/internal/platform-support/tegra/tegra.go @@ -20,31 +20,10 @@ import ( "fmt" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" - "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks" ) -type tegraOptions struct { - logger logger.Interface - csvFiles []string - driverRoot string - devRoot string - hookCreator discover.HookCreator - ldconfigPath string - librarySearchPaths []string - ignorePatterns ignoreMountSpecPatterns - - // The following can be overridden for testing - symlinkLocator lookup.Locator - symlinkChainLocator lookup.Locator - // TODO: This should be replaced by a regular mock - resolveSymlink func(string) (string, error) -} - -// Option defines a functional option for configuring a Tegra discoverer. -type Option func(*tegraOptions) - // New creates a new tegra discoverer using the supplied options. func New(opts ...Option) (discover.Discover, error) { o := &tegraOptions{} @@ -103,60 +82,3 @@ func New(opts ...Option) (discover.Discover, error) { return d, nil } - -// WithLogger sets the logger for the discoverer. -func WithLogger(logger logger.Interface) Option { - return func(o *tegraOptions) { - o.logger = logger - } -} - -// WithDriverRoot sets the driver root for the discoverer. -func WithDriverRoot(driverRoot string) Option { - return func(o *tegraOptions) { - o.driverRoot = driverRoot - } -} - -// WithDevRoot sets the /dev root. -// If this is unset, the driver root is assumed. -func WithDevRoot(devRoot string) Option { - return func(o *tegraOptions) { - o.devRoot = devRoot - } -} - -// WithCSVFiles sets the CSV files for the discoverer. -func WithCSVFiles(csvFiles []string) Option { - return func(o *tegraOptions) { - o.csvFiles = csvFiles - } -} - -// WithHookCreator sets the hook creator for the discoverer. -func WithHookCreator(hookCreator discover.HookCreator) Option { - return func(o *tegraOptions) { - o.hookCreator = hookCreator - } -} - -// WithLdconfigPath sets the path to the ldconfig program -func WithLdconfigPath(ldconfigPath string) Option { - return func(o *tegraOptions) { - o.ldconfigPath = ldconfigPath - } -} - -// WithLibrarySearchPaths sets the library search paths for the discoverer. -func WithLibrarySearchPaths(librarySearchPaths ...string) Option { - return func(o *tegraOptions) { - o.librarySearchPaths = librarySearchPaths - } -} - -// WithIngorePatterns sets patterns to ignore in the CSV files -func WithIngorePatterns(ignorePatterns ...string) Option { - return func(o *tegraOptions) { - o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns) - } -} From 881720899141c7836a3137e6017124e44331b28b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 3 Dec 2025 15:13:07 +0100 Subject: [PATCH 2/9] [no-relnote] Rename tegra.tegraOptions to tegra.options Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv.go | 2 +- internal/platform-support/tegra/csv_test.go | 2 +- internal/platform-support/tegra/options.go | 20 ++++++++++---------- internal/platform-support/tegra/symlinks.go | 2 +- internal/platform-support/tegra/tegra.go | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/platform-support/tegra/csv.go b/internal/platform-support/tegra/csv.go index edb7fdc48..75bb96e8c 100644 --- a/internal/platform-support/tegra/csv.go +++ b/internal/platform-support/tegra/csv.go @@ -28,7 +28,7 @@ import ( // newDiscovererFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied. // The constructed discoverer is comprised of a list, with each element in the list being associated with a // single CSV files. -func (o tegraOptions) newDiscovererFromCSVFiles() (discover.Discover, error) { +func (o options) newDiscovererFromCSVFiles() (discover.Discover, error) { if len(o.csvFiles) == 0 { o.logger.Warningf("No CSV files specified") return discover.None{}, nil diff --git a/internal/platform-support/tegra/csv_test.go b/internal/platform-support/tegra/csv_test.go index 1fcda971b..a144f7c7d 100644 --- a/internal/platform-support/tegra/csv_test.go +++ b/internal/platform-support/tegra/csv_test.go @@ -188,7 +188,7 @@ func TestDiscovererFromCSVFiles(t *testing.T) { t.Run(tc.description, func(t *testing.T) { defer setGetTargetsFromCSVFiles(tc.moutSpecs)() - o := tegraOptions{ + o := options{ logger: logger, hookCreator: hookCreator, csvFiles: []string{"dummy"}, diff --git a/internal/platform-support/tegra/options.go b/internal/platform-support/tegra/options.go index 538a0cc22..787cdf1c4 100644 --- a/internal/platform-support/tegra/options.go +++ b/internal/platform-support/tegra/options.go @@ -23,7 +23,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" ) -type tegraOptions struct { +type options struct { logger logger.Interface csvFiles []string driverRoot string @@ -41,18 +41,18 @@ type tegraOptions struct { } // Option defines a functional option for configuring a Tegra discoverer. -type Option func(*tegraOptions) +type Option func(*options) // WithLogger sets the logger for the discoverer. func WithLogger(logger logger.Interface) Option { - return func(o *tegraOptions) { + return func(o *options) { o.logger = logger } } // WithDriverRoot sets the driver root for the discoverer. func WithDriverRoot(driverRoot string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.driverRoot = driverRoot } } @@ -60,42 +60,42 @@ func WithDriverRoot(driverRoot string) Option { // WithDevRoot sets the /dev root. // If this is unset, the driver root is assumed. func WithDevRoot(devRoot string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.devRoot = devRoot } } // WithCSVFiles sets the CSV files for the discoverer. func WithCSVFiles(csvFiles []string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.csvFiles = csvFiles } } // WithHookCreator sets the hook creator for the discoverer. func WithHookCreator(hookCreator discover.HookCreator) Option { - return func(o *tegraOptions) { + return func(o *options) { o.hookCreator = hookCreator } } // WithLdconfigPath sets the path to the ldconfig program func WithLdconfigPath(ldconfigPath string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.ldconfigPath = ldconfigPath } } // WithLibrarySearchPaths sets the library search paths for the discoverer. func WithLibrarySearchPaths(librarySearchPaths ...string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.librarySearchPaths = librarySearchPaths } } // WithIngorePatterns sets patterns to ignore in the CSV files func WithIngorePatterns(ignorePatterns ...string) Option { - return func(o *tegraOptions) { + return func(o *options) { o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns) } } diff --git a/internal/platform-support/tegra/symlinks.go b/internal/platform-support/tegra/symlinks.go index 822d482fd..00e664a19 100644 --- a/internal/platform-support/tegra/symlinks.go +++ b/internal/platform-support/tegra/symlinks.go @@ -36,7 +36,7 @@ type symlinkHook struct { } // createCSVSymlinkHooks creates a discoverer for a hook that creates required symlinks in the container -func (o tegraOptions) createCSVSymlinkHooks(targets []string) discover.Discover { +func (o options) createCSVSymlinkHooks(targets []string) discover.Discover { return symlinkHook{ logger: o.logger, hookCreator: o.hookCreator, diff --git a/internal/platform-support/tegra/tegra.go b/internal/platform-support/tegra/tegra.go index 26264f42c..4ce0b9bba 100644 --- a/internal/platform-support/tegra/tegra.go +++ b/internal/platform-support/tegra/tegra.go @@ -26,7 +26,7 @@ import ( // New creates a new tegra discoverer using the supplied options. func New(opts ...Option) (discover.Discover, error) { - o := &tegraOptions{} + o := &options{} for _, opt := range opts { opt(o) } From 478488696a1fa29d7e378dd5a7b4da546573643f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 3 Dec 2025 15:19:33 +0100 Subject: [PATCH 3/9] [no-relnote] Introduce tegra.MountSpecPathsByType for refactoring Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv.go | 13 +++++++-- internal/platform-support/tegra/csv_test.go | 14 ++++------ .../platform-support/tegra/mount_specs.go | 28 +++++++++++++++++++ 3 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 internal/platform-support/tegra/mount_specs.go diff --git a/internal/platform-support/tegra/csv.go b/internal/platform-support/tegra/csv.go index 75bb96e8c..a6e4efdd9 100644 --- a/internal/platform-support/tegra/csv.go +++ b/internal/platform-support/tegra/csv.go @@ -35,6 +35,15 @@ func (o options) newDiscovererFromCSVFiles() (discover.Discover, error) { } targetsByType := getTargetsFromCSVFiles(o.logger, o.csvFiles) + return o.newDiscovererFromMountSpecs(targetsByType) +} + +// newDiscovererFromMountSpecs creates a discoverer for the specified mount specs. +func (o options) newDiscovererFromMountSpecs(targetsByType MountSpecPathsByType) (discover.Discover, error) { + if len(targetsByType) == 0 { + o.logger.Warningf("No mount specs specified") + return discover.None{}, nil + } devices := discover.NewCharDeviceDiscoverer( o.logger, @@ -89,8 +98,8 @@ func (o options) newDiscovererFromCSVFiles() (discover.Discover, error) { // These are aggregated by mount spec type. // TODO: We use a function variable here to allow this to be overridden for testing. // This should be properly mocked. -var getTargetsFromCSVFiles = func(logger logger.Interface, files []string) map[csv.MountSpecType][]string { - targetsByType := make(map[csv.MountSpecType][]string) +var getTargetsFromCSVFiles = func(logger logger.Interface, files []string) MountSpecPathsByType { + targetsByType := make(MountSpecPathsByType) for _, filename := range files { targets, err := loadCSVFile(logger, filename) if err != nil { diff --git a/internal/platform-support/tegra/csv_test.go b/internal/platform-support/tegra/csv_test.go index a144f7c7d..906f0afa5 100644 --- a/internal/platform-support/tegra/csv_test.go +++ b/internal/platform-support/tegra/csv_test.go @@ -26,15 +26,13 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" - - "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" ) func TestDiscovererFromCSVFiles(t *testing.T) { logger, _ := testlog.NewNullLogger() testCases := []struct { description string - moutSpecs map[csv.MountSpecType][]string + moutSpecs MountSpecPathsByType ignorePatterns []string symlinkLocator lookup.Locator symlinkChainLocator lookup.Locator @@ -49,7 +47,7 @@ func TestDiscovererFromCSVFiles(t *testing.T) { // TODO: This current resolves to two mounts that are the same. // These are deduplicated at a later stage. We could consider deduplicating earlier in the pipeline. description: "symlink is resolved to target; mounts and symlink are created", - moutSpecs: map[csv.MountSpecType][]string{ + moutSpecs: MountSpecPathsByType{ "lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, "sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"}, }, @@ -105,7 +103,7 @@ func TestDiscovererFromCSVFiles(t *testing.T) { // TODO: This current resolves to two mounts that are the same. // These are deduplicated at a later stage. We could consider deduplicating earlier in the pipeline. description: "single glob filter does not remove symlink mounts", - moutSpecs: map[csv.MountSpecType][]string{ + moutSpecs: MountSpecPathsByType{ "lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, "sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"}, }, @@ -160,7 +158,7 @@ func TestDiscovererFromCSVFiles(t *testing.T) { }, { description: "** filter removes symlink mounts", - moutSpecs: map[csv.MountSpecType][]string{ + moutSpecs: MountSpecPathsByType{ "lib": {"/usr/lib/aarch64-linux-gnu/tegra/libv4l2_nvargus.so"}, "sym": {"/usr/lib/aarch64-linux-gnu/libv4l/plugins/nv/libv4l2_nvargus.so"}, }, @@ -213,9 +211,9 @@ func TestDiscovererFromCSVFiles(t *testing.T) { } } -func setGetTargetsFromCSVFiles(override map[csv.MountSpecType][]string) func() { +func setGetTargetsFromCSVFiles(override MountSpecPathsByType) func() { original := getTargetsFromCSVFiles - getTargetsFromCSVFiles = func(logger logger.Interface, files []string) map[csv.MountSpecType][]string { + getTargetsFromCSVFiles = func(logger logger.Interface, files []string) MountSpecPathsByType { return override } diff --git a/internal/platform-support/tegra/mount_specs.go b/internal/platform-support/tegra/mount_specs.go new file mode 100644 index 000000000..9fec5fff7 --- /dev/null +++ b/internal/platform-support/tegra/mount_specs.go @@ -0,0 +1,28 @@ +/** +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package tegra + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" +) + +// MountSpecPathsByType define the per-type paths that define the entities +// (e.g. device nodes, directories, libraries, symlinks) that are required for +// gpu use on Tegra-based systems. +// These are typically populated from CSV files defined by the platform owner. +type MountSpecPathsByType map[csv.MountSpecType][]string From d810c8b580911d6355ada3772c916114c9347dc4 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 3 Dec 2025 15:45:21 +0100 Subject: [PATCH 4/9] [no-relnote] Specify mountSpecs instead of CSV files Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv.go | 23 ++++----------------- internal/platform-support/tegra/csv_test.go | 17 +-------------- internal/platform-support/tegra/options.go | 15 +++++++------- internal/platform-support/tegra/tegra.go | 10 ++++----- pkg/nvcdi/lib-csv.go | 2 +- 5 files changed, 18 insertions(+), 49 deletions(-) diff --git a/internal/platform-support/tegra/csv.go b/internal/platform-support/tegra/csv.go index a6e4efdd9..bbd83e54f 100644 --- a/internal/platform-support/tegra/csv.go +++ b/internal/platform-support/tegra/csv.go @@ -25,19 +25,6 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" ) -// newDiscovererFromCSVFiles creates a discoverer for the specified CSV files. A logger is also supplied. -// The constructed discoverer is comprised of a list, with each element in the list being associated with a -// single CSV files. -func (o options) newDiscovererFromCSVFiles() (discover.Discover, error) { - if len(o.csvFiles) == 0 { - o.logger.Warningf("No CSV files specified") - return discover.None{}, nil - } - - targetsByType := getTargetsFromCSVFiles(o.logger, o.csvFiles) - return o.newDiscovererFromMountSpecs(targetsByType) -} - // newDiscovererFromMountSpecs creates a discoverer for the specified mount specs. func (o options) newDiscovererFromMountSpecs(targetsByType MountSpecPathsByType) (discover.Discover, error) { if len(targetsByType) == 0 { @@ -94,13 +81,11 @@ func (o options) newDiscovererFromMountSpecs(targetsByType MountSpecPathsByType) return d, nil } -// getTargetsFromCSVFiles returns the list of mount specs from the specified CSV files. -// These are aggregated by mount spec type. -// TODO: We use a function variable here to allow this to be overridden for testing. -// This should be properly mocked. -var getTargetsFromCSVFiles = func(logger logger.Interface, files []string) MountSpecPathsByType { +// MountSpecsFromCSVFiles returns a MountSpecPathsByTyper for the specified list +// of CSV files. +func MountSpecsFromCSVFiles(logger logger.Interface, csvFilePaths ...string) MountSpecPathsByType { targetsByType := make(MountSpecPathsByType) - for _, filename := range files { + for _, filename := range csvFilePaths { targets, err := loadCSVFile(logger, filename) if err != nil { logger.Warningf("Skipping CSV file %v: %v", filename, err) diff --git a/internal/platform-support/tegra/csv_test.go b/internal/platform-support/tegra/csv_test.go index 906f0afa5..80e3314c7 100644 --- a/internal/platform-support/tegra/csv_test.go +++ b/internal/platform-support/tegra/csv_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/require" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" - "github.com/NVIDIA/nvidia-container-toolkit/internal/logger" "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup" ) @@ -184,19 +183,16 @@ func TestDiscovererFromCSVFiles(t *testing.T) { hookCreator := discover.NewHookCreator() for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - defer setGetTargetsFromCSVFiles(tc.moutSpecs)() - o := options{ logger: logger, hookCreator: hookCreator, - csvFiles: []string{"dummy"}, ignorePatterns: tc.ignorePatterns, symlinkLocator: tc.symlinkLocator, symlinkChainLocator: tc.symlinkChainLocator, resolveSymlink: tc.symlinkResolver, } - d, err := o.newDiscovererFromCSVFiles() + d, err := o.newDiscovererFromMountSpecs(tc.moutSpecs) require.ErrorIs(t, err, tc.expectedError) hooks, err := d.Hooks() @@ -210,14 +206,3 @@ func TestDiscovererFromCSVFiles(t *testing.T) { }) } } - -func setGetTargetsFromCSVFiles(override MountSpecPathsByType) func() { - original := getTargetsFromCSVFiles - getTargetsFromCSVFiles = func(logger logger.Interface, files []string) MountSpecPathsByType { - return override - } - - return func() { - getTargetsFromCSVFiles = original - } -} diff --git a/internal/platform-support/tegra/options.go b/internal/platform-support/tegra/options.go index 787cdf1c4..325945a83 100644 --- a/internal/platform-support/tegra/options.go +++ b/internal/platform-support/tegra/options.go @@ -25,7 +25,7 @@ import ( type options struct { logger logger.Interface - csvFiles []string + mountSpecs MountSpecPathsByType driverRoot string devRoot string hookCreator discover.HookCreator @@ -65,13 +65,6 @@ func WithDevRoot(devRoot string) Option { } } -// WithCSVFiles sets the CSV files for the discoverer. -func WithCSVFiles(csvFiles []string) Option { - return func(o *options) { - o.csvFiles = csvFiles - } -} - // WithHookCreator sets the hook creator for the discoverer. func WithHookCreator(hookCreator discover.HookCreator) Option { return func(o *options) { @@ -99,3 +92,9 @@ func WithIngorePatterns(ignorePatterns ...string) Option { o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns) } } + +func WithMountSpecs(mountSpecs MountSpecPathsByType) Option { + return func(o *options) { + o.mountSpecs = mountSpecs + } +} diff --git a/internal/platform-support/tegra/tegra.go b/internal/platform-support/tegra/tegra.go index 4ce0b9bba..680b5ffd2 100644 --- a/internal/platform-support/tegra/tegra.go +++ b/internal/platform-support/tegra/tegra.go @@ -24,7 +24,7 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/lookup/symlinks" ) -// New creates a new tegra discoverer using the supplied options. +// New creates a new tegra discoverer using the supplied functional options. func New(opts ...Option) (discover.Discover, error) { o := &options{} for _, opt := range opts { @@ -54,12 +54,12 @@ func New(opts ...Option) (discover.Discover, error) { o.resolveSymlink = symlinks.Resolve } - csvDiscoverer, err := o.newDiscovererFromCSVFiles() + mountSpecDiscoverer, err := o.newDiscovererFromMountSpecs(o.mountSpecs) if err != nil { - return nil, fmt.Errorf("failed to create CSV discoverer: %v", err) + return nil, fmt.Errorf("failed to create discoverer for mount specs: %w", err) } - ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(o.logger, csvDiscoverer, o.hookCreator, o.ldconfigPath) + ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(o.logger, mountSpecDiscoverer, o.hookCreator, o.ldconfigPath) if err != nil { return nil, fmt.Errorf("failed to create ldcach update hook discoverer: %v", err) } @@ -74,7 +74,7 @@ func New(opts ...Option) (discover.Discover, error) { ) d := discover.Merge( - csvDiscoverer, + mountSpecDiscoverer, // The ldcacheUpdateHook is added last to ensure that the created symlinks are included ldcacheUpdateHook, tegraSystemMounts, diff --git a/pkg/nvcdi/lib-csv.go b/pkg/nvcdi/lib-csv.go index 6380d79dc..263299d68 100644 --- a/pkg/nvcdi/lib-csv.go +++ b/pkg/nvcdi/lib-csv.go @@ -52,8 +52,8 @@ func (l *csvlib) GetDeviceSpecs() ([]specs.Device, error) { tegra.WithDevRoot(l.devRoot), tegra.WithHookCreator(l.hookCreator), tegra.WithLdconfigPath(l.ldconfigPath), - tegra.WithCSVFiles(l.csvFiles), tegra.WithLibrarySearchPaths(l.librarySearchPaths...), + tegra.WithMountSpecs(tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...)), tegra.WithIngorePatterns(l.csvIgnorePatterns...), ) if err != nil { From ddb320e3d6432d2b803bf7b3cffe52661a60b22a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 3 Dec 2025 16:10:26 +0100 Subject: [PATCH 5/9] [no-relnote] Remove ignored symlink patterns at construction Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv.go | 6 +-- internal/platform-support/tegra/csv_test.go | 8 +++- internal/platform-support/tegra/filter.go | 20 +++++++-- .../platform-support/tegra/filter_test.go | 2 +- .../platform-support/tegra/mount_specs.go | 41 +++++++++++++++++++ internal/platform-support/tegra/options.go | 13 ++---- internal/platform-support/tegra/tegra.go | 2 +- pkg/nvcdi/lib-csv.go | 8 +++- 8 files changed, 76 insertions(+), 24 deletions(-) diff --git a/internal/platform-support/tegra/csv.go b/internal/platform-support/tegra/csv.go index bbd83e54f..bd1a02f9f 100644 --- a/internal/platform-support/tegra/csv.go +++ b/internal/platform-support/tegra/csv.go @@ -60,15 +60,13 @@ func (o options) newDiscovererFromMountSpecs(targetsByType MountSpecPathsByType) ) // We process the explicitly requested symlinks. - symlinkTargets := o.ignorePatterns.Apply(targetsByType[csv.MountSpecSym]...) - o.logger.Debugf("Filtered symlink targets: %v", symlinkTargets) symlinks := discover.NewMounts( o.logger, o.symlinkLocator, o.driverRoot, - symlinkTargets, + targetsByType[csv.MountSpecSym], ) - createSymlinks := o.createCSVSymlinkHooks(symlinkTargets) + createSymlinks := o.createCSVSymlinkHooks(targetsByType[csv.MountSpecSym]) d := discover.Merge( devices, diff --git a/internal/platform-support/tegra/csv_test.go b/internal/platform-support/tegra/csv_test.go index 80e3314c7..2fae13103 100644 --- a/internal/platform-support/tegra/csv_test.go +++ b/internal/platform-support/tegra/csv_test.go @@ -186,13 +186,17 @@ func TestDiscovererFromCSVFiles(t *testing.T) { o := options{ logger: logger, hookCreator: hookCreator, - ignorePatterns: tc.ignorePatterns, symlinkLocator: tc.symlinkLocator, symlinkChainLocator: tc.symlinkChainLocator, resolveSymlink: tc.symlinkResolver, } - d, err := o.newDiscovererFromMountSpecs(tc.moutSpecs) + mountSpecs := Transform( + tc.moutSpecs, + IgnoreSymlinkMountSpecsByPattern(tc.ignorePatterns...), + ) + + d, err := o.newDiscovererFromMountSpecs(mountSpecs.MountSpecPathsByType()) require.ErrorIs(t, err, tc.expectedError) hooks, err := d.Hooks() diff --git a/internal/platform-support/tegra/filter.go b/internal/platform-support/tegra/filter.go index 03b18bf74..2a0ec16cd 100644 --- a/internal/platform-support/tegra/filter.go +++ b/internal/platform-support/tegra/filter.go @@ -19,11 +19,13 @@ package tegra import ( "path/filepath" "strings" + + "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" ) -type ignoreMountSpecPatterns []string +type ignoreSymlinkMountSpecPatterns []string -func (d ignoreMountSpecPatterns) Match(name string) bool { +func (d ignoreSymlinkMountSpecPatterns) match(name string) bool { for _, pattern := range d { target := name if strings.HasPrefix(pattern, "**/") { @@ -37,13 +39,23 @@ func (d ignoreMountSpecPatterns) Match(name string) bool { return false } -func (d ignoreMountSpecPatterns) Apply(input ...string) []string { +func (d ignoreSymlinkMountSpecPatterns) filter(input ...string) []string { var filtered []string for _, name := range input { - if d.Match(name) { + if d.match(name) { continue } filtered = append(filtered, name) } return filtered } + +func (d ignoreSymlinkMountSpecPatterns) Apply(input MountSpecPathsByTyper) MountSpecPathsByTyper { + ms := input.MountSpecPathsByType() + + if symlinks, ok := ms[csv.MountSpecSym]; ok { + ms[csv.MountSpecSym] = d.filter(symlinks...) + } + + return ms +} diff --git a/internal/platform-support/tegra/filter_test.go b/internal/platform-support/tegra/filter_test.go index a3b1a8f7c..4862cb1f4 100644 --- a/internal/platform-support/tegra/filter_test.go +++ b/internal/platform-support/tegra/filter_test.go @@ -50,7 +50,7 @@ func TestIgnorePatterns(t *testing.T) { for _, tc := range testCases { t.Run(tc.description, func(t *testing.T) { - filtered := ignoreMountSpecPatterns(tc.blockedFilter).Apply(tc.input...) + filtered := ignoreSymlinkMountSpecPatterns(tc.blockedFilter).filter(tc.input...) require.ElementsMatch(t, tc.expected, filtered) }) } diff --git a/internal/platform-support/tegra/mount_specs.go b/internal/platform-support/tegra/mount_specs.go index 9fec5fff7..0b2a9813b 100644 --- a/internal/platform-support/tegra/mount_specs.go +++ b/internal/platform-support/tegra/mount_specs.go @@ -21,8 +21,49 @@ import ( "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" ) +type MountSpecPathsByTyper interface { + MountSpecPathsByType() MountSpecPathsByType +} + // MountSpecPathsByType define the per-type paths that define the entities // (e.g. device nodes, directories, libraries, symlinks) that are required for // gpu use on Tegra-based systems. // These are typically populated from CSV files defined by the platform owner. type MountSpecPathsByType map[csv.MountSpecType][]string + +var _ MountSpecPathsByTyper = (MountSpecPathsByType)(nil) + +// MountSpecPathsByType for a variable of type MountSpecPathsByType returns the +// underlying data structure. +// This allows for using this type in functions such as Merge and Filter. +func (m MountSpecPathsByType) MountSpecPathsByType() MountSpecPathsByType { + return m +} + +// A Transformer modifies a specified set of mount specs by type. +// The output of a transformer is itself a set of mount specs by type. +type Transformer interface { + Apply(MountSpecPathsByTyper) MountSpecPathsByTyper +} + +// Transform applies the specified transforms to a set of mount specs by type. +// The result is itself a set of mount specs by type. +func Transform(input MountSpecPathsByTyper, t Transformer) MountSpecPathsByTyper { + return transformMountSpecByPathsByType{ + Transformer: t, + input: input, + } +} + +type transformMountSpecByPathsByType struct { + Transformer + input MountSpecPathsByTyper +} + +func (m transformMountSpecByPathsByType) MountSpecPathsByType() MountSpecPathsByType { + return m.Apply(m.input).MountSpecPathsByType() +} + +func IgnoreSymlinkMountSpecsByPattern(ignorePatterns ...string) Transformer { + return ignoreSymlinkMountSpecPatterns(ignorePatterns) +} diff --git a/internal/platform-support/tegra/options.go b/internal/platform-support/tegra/options.go index 325945a83..613437f92 100644 --- a/internal/platform-support/tegra/options.go +++ b/internal/platform-support/tegra/options.go @@ -25,19 +25,19 @@ import ( type options struct { logger logger.Interface - mountSpecs MountSpecPathsByType driverRoot string devRoot string hookCreator discover.HookCreator ldconfigPath string librarySearchPaths []string - ignorePatterns ignoreMountSpecPatterns // The following can be overridden for testing symlinkLocator lookup.Locator symlinkChainLocator lookup.Locator // TODO: This should be replaced by a regular mock resolveSymlink func(string) (string, error) + + mountSpecs MountSpecPathsByTyper } // Option defines a functional option for configuring a Tegra discoverer. @@ -86,14 +86,7 @@ func WithLibrarySearchPaths(librarySearchPaths ...string) Option { } } -// WithIngorePatterns sets patterns to ignore in the CSV files -func WithIngorePatterns(ignorePatterns ...string) Option { - return func(o *options) { - o.ignorePatterns = ignoreMountSpecPatterns(ignorePatterns) - } -} - -func WithMountSpecs(mountSpecs MountSpecPathsByType) Option { +func WithMountSpecs(mountSpecs MountSpecPathsByTyper) Option { return func(o *options) { o.mountSpecs = mountSpecs } diff --git a/internal/platform-support/tegra/tegra.go b/internal/platform-support/tegra/tegra.go index 680b5ffd2..0a44465bb 100644 --- a/internal/platform-support/tegra/tegra.go +++ b/internal/platform-support/tegra/tegra.go @@ -54,7 +54,7 @@ func New(opts ...Option) (discover.Discover, error) { o.resolveSymlink = symlinks.Resolve } - mountSpecDiscoverer, err := o.newDiscovererFromMountSpecs(o.mountSpecs) + mountSpecDiscoverer, err := o.newDiscovererFromMountSpecs(o.mountSpecs.MountSpecPathsByType()) if err != nil { return nil, fmt.Errorf("failed to create discoverer for mount specs: %w", err) } diff --git a/pkg/nvcdi/lib-csv.go b/pkg/nvcdi/lib-csv.go index 263299d68..cf11486e8 100644 --- a/pkg/nvcdi/lib-csv.go +++ b/pkg/nvcdi/lib-csv.go @@ -53,8 +53,12 @@ func (l *csvlib) GetDeviceSpecs() ([]specs.Device, error) { tegra.WithHookCreator(l.hookCreator), tegra.WithLdconfigPath(l.ldconfigPath), tegra.WithLibrarySearchPaths(l.librarySearchPaths...), - tegra.WithMountSpecs(tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...)), - tegra.WithIngorePatterns(l.csvIgnorePatterns...), + tegra.WithMountSpecs( + tegra.Transform( + tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...), + tegra.IgnoreSymlinkMountSpecsByPattern(l.csvIgnorePatterns...), + ), + ), ) if err != nil { return nil, fmt.Errorf("failed to create discoverer for CSV files: %v", err) From f015630ebb05155cf89e748569d08f3562fba49f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 4 Dec 2025 13:18:31 +0100 Subject: [PATCH 6/9] [no-relnote] Add collection of mountSpecs Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv.go | 7 +++++-- .../platform-support/tegra/mount_specs.go | 20 +++++++++++++++++++ internal/platform-support/tegra/options.go | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/platform-support/tegra/csv.go b/internal/platform-support/tegra/csv.go index bd1a02f9f..a6507e0c7 100644 --- a/internal/platform-support/tegra/csv.go +++ b/internal/platform-support/tegra/csv.go @@ -82,18 +82,21 @@ func (o options) newDiscovererFromMountSpecs(targetsByType MountSpecPathsByType) // MountSpecsFromCSVFiles returns a MountSpecPathsByTyper for the specified list // of CSV files. func MountSpecsFromCSVFiles(logger logger.Interface, csvFilePaths ...string) MountSpecPathsByType { - targetsByType := make(MountSpecPathsByType) + var mountSpecs mountSpecPathsByTypers + for _, filename := range csvFilePaths { targets, err := loadCSVFile(logger, filename) if err != nil { logger.Warningf("Skipping CSV file %v: %v", filename, err) continue } + targetsByType := make(MountSpecPathsByType) for _, t := range targets { targetsByType[t.Type] = append(targetsByType[t.Type], t.Path) } + mountSpecs = append(mountSpecs, targetsByType) } - return targetsByType + return mountSpecs.MountSpecPathsByType() } // loadCSVFile loads the specified CSV file and returns the list of mount specs diff --git a/internal/platform-support/tegra/mount_specs.go b/internal/platform-support/tegra/mount_specs.go index 0b2a9813b..378b50531 100644 --- a/internal/platform-support/tegra/mount_specs.go +++ b/internal/platform-support/tegra/mount_specs.go @@ -25,6 +25,9 @@ type MountSpecPathsByTyper interface { MountSpecPathsByType() MountSpecPathsByType } +// moutSpecPathsByTypers represents a collection of MountSpecPathsByTyper +type mountSpecPathsByTypers []MountSpecPathsByTyper + // MountSpecPathsByType define the per-type paths that define the entities // (e.g. device nodes, directories, libraries, symlinks) that are required for // gpu use on Tegra-based systems. @@ -32,6 +35,7 @@ type MountSpecPathsByTyper interface { type MountSpecPathsByType map[csv.MountSpecType][]string var _ MountSpecPathsByTyper = (MountSpecPathsByType)(nil) +var _ MountSpecPathsByTyper = (mountSpecPathsByTypers)(nil) // MountSpecPathsByType for a variable of type MountSpecPathsByType returns the // underlying data structure. @@ -40,6 +44,22 @@ func (m MountSpecPathsByType) MountSpecPathsByType() MountSpecPathsByType { return m } +// MountSpecPathsByType returns the combination of mount specs by type for the +// collection. +func (collection mountSpecPathsByTypers) MountSpecPathsByType() MountSpecPathsByType { + merged := make(MountSpecPathsByType) + for _, t := range collection { + if t == nil { + continue + } + for tType, targets := range t.MountSpecPathsByType() { + merged[tType] = append(merged[tType], targets...) + } + } + return merged + +} + // A Transformer modifies a specified set of mount specs by type. // The output of a transformer is itself a set of mount specs by type. type Transformer interface { diff --git a/internal/platform-support/tegra/options.go b/internal/platform-support/tegra/options.go index 613437f92..fdc8ee577 100644 --- a/internal/platform-support/tegra/options.go +++ b/internal/platform-support/tegra/options.go @@ -86,8 +86,8 @@ func WithLibrarySearchPaths(librarySearchPaths ...string) Option { } } -func WithMountSpecs(mountSpecs MountSpecPathsByTyper) Option { +func WithMountSpecs(mountSpecs ...MountSpecPathsByTyper) Option { return func(o *options) { - o.mountSpecs = mountSpecs + o.mountSpecs = mountSpecPathsByTypers(mountSpecs) } } From 9cc857700a4fc23312821b56af152760b1760613 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 14 Oct 2025 15:18:36 +0200 Subject: [PATCH 7/9] [no-relnote] Refactor creation of discoverer for tegra systems This change updates the way we construct a discoverer for tegra systems to be more flexible in terms of how the SOURCES of the mount specs can be specified. This allows for subsequent changes like adding (or removing) mount specs at the point of construction. Signed-off-by: Evan Lezar --- internal/platform-support/tegra/csv_test.go | 12 +-- internal/platform-support/tegra/filter.go | 86 +++++++++++++++++++ .../platform-support/tegra/filter_test.go | 2 +- .../platform-support/tegra/mount_specs.go | 66 ++++++++++++++ internal/platform-support/tegra/tegra.go | 5 +- 5 files changed, 162 insertions(+), 9 deletions(-) diff --git a/internal/platform-support/tegra/csv_test.go b/internal/platform-support/tegra/csv_test.go index 2fae13103..7f41227f1 100644 --- a/internal/platform-support/tegra/csv_test.go +++ b/internal/platform-support/tegra/csv_test.go @@ -189,14 +189,14 @@ func TestDiscovererFromCSVFiles(t *testing.T) { symlinkLocator: tc.symlinkLocator, symlinkChainLocator: tc.symlinkChainLocator, resolveSymlink: tc.symlinkResolver, - } - mountSpecs := Transform( - tc.moutSpecs, - IgnoreSymlinkMountSpecsByPattern(tc.ignorePatterns...), - ) + mountSpecs: Transform( + tc.moutSpecs, + IgnoreSymlinkMountSpecsByPattern(tc.ignorePatterns...), + ), + } - d, err := o.newDiscovererFromMountSpecs(mountSpecs.MountSpecPathsByType()) + d, err := o.newDiscovererFromMountSpecs(o.mountSpecs.MountSpecPathsByType()) require.ErrorIs(t, err, tc.expectedError) hooks, err := d.Hooks() diff --git a/internal/platform-support/tegra/filter.go b/internal/platform-support/tegra/filter.go index 2a0ec16cd..989906dc0 100644 --- a/internal/platform-support/tegra/filter.go +++ b/internal/platform-support/tegra/filter.go @@ -59,3 +59,89 @@ func (d ignoreSymlinkMountSpecPatterns) Apply(input MountSpecPathsByTyper) Mount return ms } + +// A filter removes elements from an input list and returns the remaining +// elements. +type filter interface { + apply(...string) []string +} + +// A stringMatcher implements the MatchString function. +type stringMatcher interface { + MatchString(string) bool +} + +// A matcherAsFilter is used to ensure that a string matcher can be used as a filter. +type matcherAsFilter struct { + stringMatcher +} + +type filterByMountSpecType map[csv.MountSpecType]filter + +type pathPatterns []string +type pathPattern string +type basenamePattern string + +// MatchString for a set of path patterns returns true if any of the patterns +// matches against the input string. +func (d pathPatterns) MatchString(input string) bool { + for _, pattern := range d { + if match := pathPattern(pattern).MatchString(input); match { + return true + } + } + return false +} + +// MatchString attempts to match a path pattern to the specified input string. +// If the pattern starts with `**/` the input is treated as a path and only +// the basenames are matched using regular glob rules. +func (d pathPattern) MatchString(input string) bool { + if strings.HasPrefix(string(d), "**/") { + return basenamePattern(d).MatchString(input) + } + match, _ := filepath.Match(string(d), input) + return match +} + +// MatchString for a basename pattern applies the specified pattern against the +// basename of the input. +// If the pattern starts with **/, this is stripped before attempting to match. +func (d basenamePattern) MatchString(input string) bool { + pattern := strings.TrimPrefix(string(d), "**/") + match, _ := filepath.Match(pattern, filepath.Base(input)) + return match +} + +// Apply the specified per-type filters to the input mount specs. +func (p filterByMountSpecType) Apply(input MountSpecPathsByTyper) MountSpecPathsByTyper { + ms := input.MountSpecPathsByType() + for t, filter := range p { + if len(ms[t]) == 0 { + continue + } + ms[t] = filter.apply(ms[t]...) + } + return ms +} + +// apply uses a matcher to filter an input string. +// Each element in the input that matches is skipped and the remaining elements +// are returned. +func (f *matcherAsFilter) apply(input ...string) []string { + var filtered []string + for _, path := range input { + if f.MatchString(path) { + continue + } + filtered = append(filtered, path) + } + return filtered +} + +// removeAll is a filter that will not return any inputs. +type removeAll struct{} + +func (a removeAll) apply(...string) []string { + return nil +} diff --git a/internal/platform-support/tegra/filter_test.go b/internal/platform-support/tegra/filter_test.go index 4862cb1f4..1e112938d 100644 --- a/internal/platform-support/tegra/filter_test.go +++ b/internal/platform-support/tegra/filter_test.go @@ -25,7 +25,7 @@ import ( func TestIgnorePatterns(t *testing.T) { testCases := []struct { description string - blockedFilter []string + blockedFilter pathPatterns input []string expected []string }{ diff --git a/internal/platform-support/tegra/mount_specs.go b/internal/platform-support/tegra/mount_specs.go index 378b50531..05c766830 100644 --- a/internal/platform-support/tegra/mount_specs.go +++ b/internal/platform-support/tegra/mount_specs.go @@ -18,6 +18,8 @@ package tegra import ( + "regexp" + "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv" ) @@ -80,6 +82,28 @@ type transformMountSpecByPathsByType struct { input MountSpecPathsByTyper } +type merge []MountSpecPathsByTyper + +// Merge combines the MountSpecPathsByType for the specified sources. +func Merge(sources ...MountSpecPathsByTyper) MountSpecPathsByTyper { + return merge(sources) +} + +// MountSpecPathsByType for a set of merged mount specs combines the list of +// paths per type. +func (ts merge) MountSpecPathsByType() MountSpecPathsByType { + targetsByType := make(MountSpecPathsByType) + for _, t := range ts { + if t == nil { + continue + } + for tType, targets := range t.MountSpecPathsByType() { + targetsByType[tType] = append(targetsByType[tType], targets...) + } + } + return targetsByType +} + func (m transformMountSpecByPathsByType) MountSpecPathsByType() MountSpecPathsByType { return m.Apply(m.input).MountSpecPathsByType() } @@ -87,3 +111,45 @@ func (m transformMountSpecByPathsByType) MountSpecPathsByType() MountSpecPathsBy func IgnoreSymlinkMountSpecsByPattern(ignorePatterns ...string) Transformer { return ignoreSymlinkMountSpecPatterns(ignorePatterns) } + +// OnlyDeviceNodes creates a transformer that will remove any input mounts specs +// that are not of the `MountSpecDev` type. +func OnlyDeviceNodes() Transformer { + return filterByMountSpecType{ + csv.MountSpecDir: removeAll{}, + csv.MountSpecLib: removeAll{}, + csv.MountSpecSym: removeAll{}, + } +} + +// WithoutDeviceNodes creates a transformer that will remove entries with type +// MountSpecDevice from the input. +func WithoutDeviceNodes() Transformer { + return filterByMountSpecType{ + csv.MountSpecDev: removeAll{}, + } +} + +// WithoutRegularDeviceNodes creates a transfomer which removes +// regular `/dev/nvidia[0-9]+` device nodes from the source. +func WithoutRegularDeviceNodes() Transformer { + return filterByMountSpecType{ + csv.MountSpecDev: &matcherAsFilter{regexp.MustCompile("^/dev/nvidia[0-9]+$")}, + } +} + +// DeviceNodes creates a set of MountSpecPaths for the specified device nodes. +// These have the MoutSpecDev type. +func DeviceNodes(dn ...string) MountSpecPathsByTyper { + return MountSpecPathsByType{ + csv.MountSpecDev: dn, + } +} + +// DeviceNodes creates a set of MountSpecPaths for the specified symlinks. +// These have the MountSpecSym type. +func Symlinks(s ...string) MountSpecPathsByTyper { + return MountSpecPathsByType{ + csv.MountSpecSym: s, + } +} diff --git a/internal/platform-support/tegra/tegra.go b/internal/platform-support/tegra/tegra.go index 0a44465bb..b31a98a3d 100644 --- a/internal/platform-support/tegra/tegra.go +++ b/internal/platform-support/tegra/tegra.go @@ -56,7 +56,7 @@ func New(opts ...Option) (discover.Discover, error) { mountSpecDiscoverer, err := o.newDiscovererFromMountSpecs(o.mountSpecs.MountSpecPathsByType()) if err != nil { - return nil, fmt.Errorf("failed to create discoverer for mount specs: %w", err) + return nil, fmt.Errorf("failed to create discoverer for mount specs: %v", err) } ldcacheUpdateHook, err := discover.NewLDCacheUpdateHook(o.logger, mountSpecDiscoverer, o.hookCreator, o.ldconfigPath) @@ -75,7 +75,8 @@ func New(opts ...Option) (discover.Discover, error) { d := discover.Merge( mountSpecDiscoverer, - // The ldcacheUpdateHook is added last to ensure that the created symlinks are included + // The ldcacheUpdateHook is added after the mount spec discoverer to + // ensure that the symlinks are included. ldcacheUpdateHook, tegraSystemMounts, ) From 39644f923da5f4aa3a2f722540a84d810c33f96f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 28 Nov 2025 16:48:42 +0100 Subject: [PATCH 8/9] Bump github.com/NVIDIA/go-nvlib from 0.8.1 to d0f42ba016dd Signed-off-by: Evan Lezar --- go.mod | 2 +- go.sum | 4 +- internal/info/auto_test.go | 5 +- .../NVIDIA/go-nvlib/pkg/nvlib/info/api.go | 6 +- .../pkg/nvlib/info/property-extractor.go | 36 ++-- .../pkg/nvlib/info/property-extractor_mock.go | 154 +++++------------- .../go-nvlib/pkg/nvlib/info/resolver.go | 6 +- vendor/modules.txt | 2 +- 8 files changed, 69 insertions(+), 146 deletions(-) diff --git a/go.mod b/go.mod index 2b87cc0c4..706ada957 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/NVIDIA/nvidia-container-toolkit go 1.25.0 require ( - github.com/NVIDIA/go-nvlib v0.9.0 + github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd github.com/NVIDIA/go-nvml v0.13.0-1 github.com/google/uuid v1.6.0 github.com/moby/sys/mountinfo v0.7.2 diff --git a/go.sum b/go.sum index faa60c2d3..93b161264 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= -github.com/NVIDIA/go-nvlib v0.9.0 h1:GKLIvLJ0uhCtTLLZp2Q8QIDRxOYH45MM4Y5OO3U5Rho= -github.com/NVIDIA/go-nvlib v0.9.0/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c= +github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd h1:MC1w/VYuo9Zt0se4SSx9BVid4a46ai+voN3knRvVWjE= +github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c= github.com/NVIDIA/go-nvml v0.13.0-1 h1:OLX8Jq3dONuPOQPC7rndB6+iDmDakw0XTYgzMxObkEw= github.com/NVIDIA/go-nvml v0.13.0-1/go.mod h1:+KNA7c7gIBH7SKSJ1ntlwkfN80zdx8ovl4hrK3LmPt4= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/internal/info/auto_test.go b/internal/info/auto_test.go index f6d99c7e5..9301514d6 100644 --- a/internal/info/auto_test.go +++ b/internal/info/auto_test.go @@ -215,13 +215,10 @@ func TestResolveAutoMode(t *testing.T) { HasDXCoreFunc: func() (bool, string) { return tc.info["dxcore"], "dxcore" }, - IsTegraSystemFunc: func() (bool, string) { - return tc.info["tegra"], "tegra" - }, HasTegraFilesFunc: func() (bool, string) { return tc.info["tegra"], "tegra" }, - HasOnlyIntegratedGPUsFunc: func() (bool, string) { + HasAnIntegratedGPUFunc: func() (bool, string) { return tc.info["nvgpu"], "nvgpu" }, } diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/api.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/api.go index bcc9cb69c..48abff5d6 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/api.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/api.go @@ -35,9 +35,5 @@ type PropertyExtractor interface { HasDXCore() (bool, string) HasNvml() (bool, string) HasTegraFiles() (bool, string) - // Deprecated: Use HasTegraFiles instead. - IsTegraSystem() (bool, string) - // Deprecated: Use HasOnlyIntegratedGPUs - UsesOnlyNVGPUModule() (bool, string) - HasOnlyIntegratedGPUs() (bool, string) + HasAnIntegratedGPU() (bool, string) } diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor.go index 9d8e6defd..3ef582dd0 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor.go @@ -90,25 +90,20 @@ func (i *propertyExtractor) HasTegraFiles() (bool, string) { return false, fmt.Sprintf("%v has no 'tegra' prefix", tegraFamilyFile) } -// UsesOnlyNVGPUModule checks whether the only the nvgpu module is used. -// -// Deprecated: UsesOnlyNVGPUModule is deprecated, use HasOnlyIntegratedGPUs instead. -func (i *propertyExtractor) UsesOnlyNVGPUModule() (uses bool, reason string) { - return i.HasOnlyIntegratedGPUs() -} - -// HasOnlyIntegratedGPUs checks whether all GPUs are iGPUs that use NVML. +// HasAnIntegratedGPU checks whether any of the GPUs reported by NVML is an +// integrated GPU. // // As of Orin-based systems iGPUs also support limited NVML queries. -// In the absence of a robust API, we rely on heuristics to make this decision. +// In the absence of a robust API, we rely on heuristics based on the device +// name to make this decision. // -// The following device names are checked: +// Devices with the following names are considered integrated GPUs: // // GPU 0: Orin (nvgpu) (UUID: 54d0709b-558d-5a59-9c65-0c5fc14a21a4) // GPU 0: NVIDIA Thor (UUID: 54d0709b-558d-5a59-9c65-0c5fc14a21a4) // -// This function returns true if ALL devices are detected as iGPUs. -func (i *propertyExtractor) HasOnlyIntegratedGPUs() (uses bool, reason string) { +// (Where this shows the nvidia-smi -L output on these systems). +func (i *propertyExtractor) HasAnIntegratedGPU() (uses bool, reason string) { // We ensure that this function never panics defer func() { if err := recover(); err != nil { @@ -144,14 +139,23 @@ func (i *propertyExtractor) HasOnlyIntegratedGPUs() (uses bool, reason string) { } for _, name := range names { - if !isIntegratedGPUName(name) { - return false, fmt.Sprintf("device %q does not use nvgpu module", name) + if IsIntegratedGPUName(name) { + return true, fmt.Sprintf("device %q is an integrated GPU", name) } } - return true, "all devices use nvgpu module" + return false, "no integrated GPUs found" } -func isIntegratedGPUName(name string) bool { +// IsIntegratedGPUName checks whether the specified device name is associated +// with a known integrated GPU. +// +// Devices with the following names are considered integrated GPUs: +// +// GPU 0: Orin (nvgpu) (UUID: 54d0709b-558d-5a59-9c65-0c5fc14a21a4) +// GPU 0: NVIDIA Thor (UUID: 54d0709b-558d-5a59-9c65-0c5fc14a21a4) +// +// (Where this shows the nvidia-smi -L output on these systems). +func IsIntegratedGPUName(name string) bool { if strings.Contains(name, "(nvgpu)") { return true } diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor_mock.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor_mock.go index bd7d41353..708c6e833 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor_mock.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/property-extractor_mock.go @@ -17,24 +17,18 @@ var _ PropertyExtractor = &PropertyExtractorMock{} // // // make and configure a mocked PropertyExtractor // mockedPropertyExtractor := &PropertyExtractorMock{ +// HasAnIntegratedGPUFunc: func() (bool, string) { +// panic("mock out the HasAnIntegratedGPU method") +// }, // HasDXCoreFunc: func() (bool, string) { // panic("mock out the HasDXCore method") // }, // HasNvmlFunc: func() (bool, string) { // panic("mock out the HasNvml method") // }, -// HasOnlyIntegratedGPUsFunc: func() (bool, string) { -// panic("mock out the HasOnlyIntegratedGPUs method") -// }, // HasTegraFilesFunc: func() (bool, string) { // panic("mock out the HasTegraFiles method") // }, -// IsTegraSystemFunc: func() (bool, string) { -// panic("mock out the IsTegraSystem method") -// }, -// UsesOnlyNVGPUModuleFunc: func() (bool, string) { -// panic("mock out the UsesOnlyNVGPUModule method") -// }, // } // // // use mockedPropertyExtractor in code that requires PropertyExtractor @@ -42,51 +36,64 @@ var _ PropertyExtractor = &PropertyExtractorMock{} // // } type PropertyExtractorMock struct { + // HasAnIntegratedGPUFunc mocks the HasAnIntegratedGPU method. + HasAnIntegratedGPUFunc func() (bool, string) + // HasDXCoreFunc mocks the HasDXCore method. HasDXCoreFunc func() (bool, string) // HasNvmlFunc mocks the HasNvml method. HasNvmlFunc func() (bool, string) - // HasOnlyIntegratedGPUsFunc mocks the HasOnlyIntegratedGPUs method. - HasOnlyIntegratedGPUsFunc func() (bool, string) - // HasTegraFilesFunc mocks the HasTegraFiles method. HasTegraFilesFunc func() (bool, string) - // IsTegraSystemFunc mocks the IsTegraSystem method. - IsTegraSystemFunc func() (bool, string) - - // UsesOnlyNVGPUModuleFunc mocks the UsesOnlyNVGPUModule method. - UsesOnlyNVGPUModuleFunc func() (bool, string) - // calls tracks calls to the methods. calls struct { + // HasAnIntegratedGPU holds details about calls to the HasAnIntegratedGPU method. + HasAnIntegratedGPU []struct { + } // HasDXCore holds details about calls to the HasDXCore method. HasDXCore []struct { } // HasNvml holds details about calls to the HasNvml method. HasNvml []struct { } - // HasOnlyIntegratedGPUs holds details about calls to the HasOnlyIntegratedGPUs method. - HasOnlyIntegratedGPUs []struct { - } // HasTegraFiles holds details about calls to the HasTegraFiles method. HasTegraFiles []struct { } - // IsTegraSystem holds details about calls to the IsTegraSystem method. - IsTegraSystem []struct { - } - // UsesOnlyNVGPUModule holds details about calls to the UsesOnlyNVGPUModule method. - UsesOnlyNVGPUModule []struct { - } } - lockHasDXCore sync.RWMutex - lockHasNvml sync.RWMutex - lockHasOnlyIntegratedGPUs sync.RWMutex - lockHasTegraFiles sync.RWMutex - lockIsTegraSystem sync.RWMutex - lockUsesOnlyNVGPUModule sync.RWMutex + lockHasAnIntegratedGPU sync.RWMutex + lockHasDXCore sync.RWMutex + lockHasNvml sync.RWMutex + lockHasTegraFiles sync.RWMutex +} + +// HasAnIntegratedGPU calls HasAnIntegratedGPUFunc. +func (mock *PropertyExtractorMock) HasAnIntegratedGPU() (bool, string) { + if mock.HasAnIntegratedGPUFunc == nil { + panic("PropertyExtractorMock.HasAnIntegratedGPUFunc: method is nil but PropertyExtractor.HasAnIntegratedGPU was just called") + } + callInfo := struct { + }{} + mock.lockHasAnIntegratedGPU.Lock() + mock.calls.HasAnIntegratedGPU = append(mock.calls.HasAnIntegratedGPU, callInfo) + mock.lockHasAnIntegratedGPU.Unlock() + return mock.HasAnIntegratedGPUFunc() +} + +// HasAnIntegratedGPUCalls gets all the calls that were made to HasAnIntegratedGPU. +// Check the length with: +// +// len(mockedPropertyExtractor.HasAnIntegratedGPUCalls()) +func (mock *PropertyExtractorMock) HasAnIntegratedGPUCalls() []struct { +} { + var calls []struct { + } + mock.lockHasAnIntegratedGPU.RLock() + calls = mock.calls.HasAnIntegratedGPU + mock.lockHasAnIntegratedGPU.RUnlock() + return calls } // HasDXCore calls HasDXCoreFunc. @@ -143,33 +150,6 @@ func (mock *PropertyExtractorMock) HasNvmlCalls() []struct { return calls } -// HasOnlyIntegratedGPUs calls HasOnlyIntegratedGPUsFunc. -func (mock *PropertyExtractorMock) HasOnlyIntegratedGPUs() (bool, string) { - if mock.HasOnlyIntegratedGPUsFunc == nil { - panic("PropertyExtractorMock.HasOnlyIntegratedGPUsFunc: method is nil but PropertyExtractor.HasOnlyIntegratedGPUs was just called") - } - callInfo := struct { - }{} - mock.lockHasOnlyIntegratedGPUs.Lock() - mock.calls.HasOnlyIntegratedGPUs = append(mock.calls.HasOnlyIntegratedGPUs, callInfo) - mock.lockHasOnlyIntegratedGPUs.Unlock() - return mock.HasOnlyIntegratedGPUsFunc() -} - -// HasOnlyIntegratedGPUsCalls gets all the calls that were made to HasOnlyIntegratedGPUs. -// Check the length with: -// -// len(mockedPropertyExtractor.HasOnlyIntegratedGPUsCalls()) -func (mock *PropertyExtractorMock) HasOnlyIntegratedGPUsCalls() []struct { -} { - var calls []struct { - } - mock.lockHasOnlyIntegratedGPUs.RLock() - calls = mock.calls.HasOnlyIntegratedGPUs - mock.lockHasOnlyIntegratedGPUs.RUnlock() - return calls -} - // HasTegraFiles calls HasTegraFilesFunc. func (mock *PropertyExtractorMock) HasTegraFiles() (bool, string) { if mock.HasTegraFilesFunc == nil { @@ -196,57 +176,3 @@ func (mock *PropertyExtractorMock) HasTegraFilesCalls() []struct { mock.lockHasTegraFiles.RUnlock() return calls } - -// IsTegraSystem calls IsTegraSystemFunc. -func (mock *PropertyExtractorMock) IsTegraSystem() (bool, string) { - if mock.IsTegraSystemFunc == nil { - panic("PropertyExtractorMock.IsTegraSystemFunc: method is nil but PropertyExtractor.IsTegraSystem was just called") - } - callInfo := struct { - }{} - mock.lockIsTegraSystem.Lock() - mock.calls.IsTegraSystem = append(mock.calls.IsTegraSystem, callInfo) - mock.lockIsTegraSystem.Unlock() - return mock.IsTegraSystemFunc() -} - -// IsTegraSystemCalls gets all the calls that were made to IsTegraSystem. -// Check the length with: -// -// len(mockedPropertyExtractor.IsTegraSystemCalls()) -func (mock *PropertyExtractorMock) IsTegraSystemCalls() []struct { -} { - var calls []struct { - } - mock.lockIsTegraSystem.RLock() - calls = mock.calls.IsTegraSystem - mock.lockIsTegraSystem.RUnlock() - return calls -} - -// UsesOnlyNVGPUModule calls UsesOnlyNVGPUModuleFunc. -func (mock *PropertyExtractorMock) UsesOnlyNVGPUModule() (bool, string) { - if mock.UsesOnlyNVGPUModuleFunc == nil { - panic("PropertyExtractorMock.UsesOnlyNVGPUModuleFunc: method is nil but PropertyExtractor.UsesOnlyNVGPUModule was just called") - } - callInfo := struct { - }{} - mock.lockUsesOnlyNVGPUModule.Lock() - mock.calls.UsesOnlyNVGPUModule = append(mock.calls.UsesOnlyNVGPUModule, callInfo) - mock.lockUsesOnlyNVGPUModule.Unlock() - return mock.UsesOnlyNVGPUModuleFunc() -} - -// UsesOnlyNVGPUModuleCalls gets all the calls that were made to UsesOnlyNVGPUModule. -// Check the length with: -// -// len(mockedPropertyExtractor.UsesOnlyNVGPUModuleCalls()) -func (mock *PropertyExtractorMock) UsesOnlyNVGPUModuleCalls() []struct { -} { - var calls []struct { - } - mock.lockUsesOnlyNVGPUModule.RLock() - calls = mock.calls.UsesOnlyNVGPUModule - mock.lockUsesOnlyNVGPUModule.RUnlock() - return calls -} diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/resolver.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/resolver.go index 0454d8a6b..824373823 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/resolver.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/info/resolver.go @@ -48,13 +48,13 @@ func (p platformResolver) ResolvePlatform() Platform { hasNVML, reason := p.propertyExtractor.HasNvml() p.logger.Debugf("Is NVML-based system? %v: %v", hasNVML, reason) - hasOnlyIntegratedGPUs, reason := p.propertyExtractor.HasOnlyIntegratedGPUs() - p.logger.Debugf("Has only integrated GPUs? %v: %v", hasOnlyIntegratedGPUs, reason) + hasAnIntegratedGPU, reason := p.propertyExtractor.HasAnIntegratedGPU() + p.logger.Debugf("Has an integrated GPU? %v: %v", hasAnIntegratedGPU, reason) switch { case hasDXCore: return PlatformWSL - case (hasTegraFiles && !hasNVML), hasOnlyIntegratedGPUs: + case (hasTegraFiles && !hasNVML), hasAnIntegratedGPU: return PlatformTegra case hasNVML: return PlatformNVML diff --git a/vendor/modules.txt b/vendor/modules.txt index 078e30511..89c9d7684 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,7 +4,7 @@ cyphar.com/go-pathrs cyphar.com/go-pathrs/internal/fdutils cyphar.com/go-pathrs/internal/libpathrs cyphar.com/go-pathrs/procfs -# github.com/NVIDIA/go-nvlib v0.9.0 +# github.com/NVIDIA/go-nvlib v0.9.1-0.20251202135446-d0f42ba016dd ## explicit; go 1.20 github.com/NVIDIA/go-nvlib/pkg/nvlib/device github.com/NVIDIA/go-nvlib/pkg/nvlib/info From 4eb4ca6810d385d078710ea30b9e67c1ca13fc42 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 14 Nov 2025 14:25:27 +0100 Subject: [PATCH 9/9] Handle multiple GPUs in CDI spec generation from CSV This change allows CDI specs to be generated for multiple devices when using CSV mode. This can be used in cases where a Tegra-based system consists of an iGPU and dGPU. This behavior can be opted out of using the disable-multiple-csv-devices feature flag. This can be specified by adding the --feaure-flags=disable-multiple-csv-devices command line option to the nvidia-ctk cdi generate command or to the automatic CDI spec generation by adding NVIDIA_CTK_CDI_GENERATE_FEATURE_FLAGS=disable-multiple-csv-devices to the /etc/nvidia-container-toolkit/nvidia-cdi-refresh.env file. Signed-off-by: Evan Lezar --- pkg/nvcdi/api.go | 4 + pkg/nvcdi/common-nvml.go | 24 ++-- pkg/nvcdi/full-gpu-nvml.go | 14 +- pkg/nvcdi/lib-csv.go | 273 ++++++++++++++++++++++++++++++++++--- pkg/nvcdi/namer.go | 6 - 5 files changed, 282 insertions(+), 39 deletions(-) diff --git a/pkg/nvcdi/api.go b/pkg/nvcdi/api.go index fce32bc88..14cbdb83f 100644 --- a/pkg/nvcdi/api.go +++ b/pkg/nvcdi/api.go @@ -88,4 +88,8 @@ const ( // FeatureEnableCoherentAnnotations enables the addition of annotations // coherent or non-coherent devices. FeatureEnableCoherentAnnotations = FeatureFlag("enable-coherent-annotations") + + // FeatureDisableMultipleCSVDevices disables the handling of multiple devices + // in CSV mode. + FeatureDisableMultipleCSVDevices = FeatureFlag("disable-multiple-csv-devices") ) diff --git a/pkg/nvcdi/common-nvml.go b/pkg/nvcdi/common-nvml.go index fbb5f01d1..b4ebc7dc4 100644 --- a/pkg/nvcdi/common-nvml.go +++ b/pkg/nvcdi/common-nvml.go @@ -25,16 +25,7 @@ import ( // newCommonNVMLDiscoverer returns a discoverer for entities that are not associated with a specific CDI device. // This includes driver libraries and meta devices, for example. func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) { - metaDevices := discover.NewCharDeviceDiscoverer( - l.logger, - l.devRoot, - []string{ - "/dev/nvidia-modeset", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-uvm", - "/dev/nvidiactl", - }, - ) + metaDevices := l.controlDeviceNodeDiscoverer() graphicsMounts, err := discover.NewGraphicsMountsDiscoverer(l.logger, l.driver, l.hookCreator) if err != nil { @@ -54,3 +45,16 @@ func (l *nvmllib) newCommonNVMLDiscoverer() (discover.Discover, error) { return d, nil } + +func (l *nvmllib) controlDeviceNodeDiscoverer() discover.Discover { + return discover.NewCharDeviceDiscoverer( + l.logger, + l.devRoot, + []string{ + "/dev/nvidia-modeset", + "/dev/nvidia-uvm-tools", + "/dev/nvidia-uvm", + "/dev/nvidiactl", + }, + ) +} diff --git a/pkg/nvcdi/full-gpu-nvml.go b/pkg/nvcdi/full-gpu-nvml.go index c52f44e47..0555b81c9 100644 --- a/pkg/nvcdi/full-gpu-nvml.go +++ b/pkg/nvcdi/full-gpu-nvml.go @@ -37,7 +37,8 @@ type fullGPUDeviceSpecGenerator struct { uuid string index int - featureFlags map[FeatureFlag]bool + featureFlags map[FeatureFlag]bool + additionalDiscoverers []discover.Discover } var _ DeviceSpecGenerator = (*fullGPUDeviceSpecGenerator)(nil) @@ -145,7 +146,6 @@ func (l *fullGPUDeviceSpecGenerator) getDeviceEdits() (*cdi.ContainerEdits, erro if err != nil { return nil, fmt.Errorf("failed to create device discoverer: %v", err) } - editsForDevice, err := edits.FromDiscoverer(deviceDiscoverer) if err != nil { return nil, fmt.Errorf("failed to create container edits for device: %v", err) @@ -177,10 +177,18 @@ func (l *fullGPUDeviceSpecGenerator) newFullGPUDiscoverer(d device.Device) (disc deviceNodes, ) - dd := discover.Merge( + var discoverers []discover.Discover + + discoverers = append(discoverers, deviceNodes, deviceFolderPermissionHooks, ) + discoverers = append(discoverers, l.additionalDiscoverers...) + + dd := discover.Merge( + discoverers..., + ) + return dd, nil } diff --git a/pkg/nvcdi/lib-csv.go b/pkg/nvcdi/lib-csv.go index cf11486e8..f6493bd9a 100644 --- a/pkg/nvcdi/lib-csv.go +++ b/pkg/nvcdi/lib-csv.go @@ -18,10 +18,17 @@ package nvcdi import ( "fmt" + "slices" + "strconv" "tags.cncf.io/container-device-interface/pkg/cdi" "tags.cncf.io/container-device-interface/specs-go" + "github.com/NVIDIA/go-nvlib/pkg/nvlib/device" + "github.com/NVIDIA/go-nvlib/pkg/nvlib/info" + "github.com/NVIDIA/go-nvml/pkg/nvml" + "github.com/google/uuid" + "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" "github.com/NVIDIA/nvidia-container-toolkit/internal/edits" "github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra" @@ -29,9 +36,31 @@ import ( type csvlib nvcdilib +type mixedcsvlib nvcdilib + var _ deviceSpecGeneratorFactory = (*csvlib)(nil) +// DeviceSpecGenerators creates a set of generators for the specified set of +// devices. +// If NVML is not available or the disable-multiple-csv-devices feature flag is +// enabled, a single device is assumed. func (l *csvlib) DeviceSpecGenerators(ids ...string) (DeviceSpecGenerator, error) { + if l.featureFlags[FeatureDisableMultipleCSVDevices] { + return l.purecsvDeviceSpecGenerators(ids...) + } + hasNVML, _ := l.infolib.HasNvml() + if !hasNVML { + return l.purecsvDeviceSpecGenerators(ids...) + } + mixed, err := l.mixedDeviceSpecGenerators(ids...) + if err != nil { + l.logger.Warningf("Failed to create mixed CSV spec generator; falling back to pure CSV implementation: %v", err) + return l.purecsvDeviceSpecGenerators(ids...) + } + return mixed, nil +} + +func (l *csvlib) purecsvDeviceSpecGenerators(ids ...string) (DeviceSpecGenerator, error) { for _, id := range ids { switch id { case "all": @@ -40,35 +69,42 @@ func (l *csvlib) DeviceSpecGenerators(ids ...string) (DeviceSpecGenerator, error return nil, fmt.Errorf("unsupported device id: %v", id) } } + g := &csvDeviceGenerator{ + csvlib: l, + index: 0, + uuid: "", + } + return g, nil +} + +func (l *csvlib) mixedDeviceSpecGenerators(ids ...string) (DeviceSpecGenerator, error) { + return (*mixedcsvlib)(l).DeviceSpecGenerators(ids...) +} - return l, nil +// A csvDeviceGenerator generates CDI specs for a device based on a set of +// platform-specific CSV files. +type csvDeviceGenerator struct { + *csvlib + index int + uuid string +} + +func (l *csvDeviceGenerator) GetUUID() (string, error) { + return l.uuid, nil } // GetDeviceSpecs returns the CDI device specs for a single device. -func (l *csvlib) GetDeviceSpecs() ([]specs.Device, error) { - d, err := tegra.New( - tegra.WithLogger(l.logger), - tegra.WithDriverRoot(l.driverRoot), - tegra.WithDevRoot(l.devRoot), - tegra.WithHookCreator(l.hookCreator), - tegra.WithLdconfigPath(l.ldconfigPath), - tegra.WithLibrarySearchPaths(l.librarySearchPaths...), - tegra.WithMountSpecs( - tegra.Transform( - tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...), - tegra.IgnoreSymlinkMountSpecsByPattern(l.csvIgnorePatterns...), - ), - ), - ) +func (l *csvDeviceGenerator) GetDeviceSpecs() ([]specs.Device, error) { + deviceNodeDiscoverer, err := l.deviceNodeDiscoverer() if err != nil { - return nil, fmt.Errorf("failed to create discoverer for CSV files: %v", err) + return nil, fmt.Errorf("failed to create discoverer for device nodes from CSV files: %w", err) } - e, err := edits.FromDiscoverer(d) + e, err := edits.FromDiscoverer(deviceNodeDiscoverer) if err != nil { return nil, fmt.Errorf("failed to create container edits for CSV files: %v", err) } - names, err := l.deviceNamers.GetDeviceNames(0, uuidIgnored{}) + names, err := l.deviceNamers.GetDeviceNames(l.index, l) if err != nil { return nil, fmt.Errorf("failed to get device name: %v", err) } @@ -84,7 +120,204 @@ func (l *csvlib) GetDeviceSpecs() ([]specs.Device, error) { return deviceSpecs, nil } +// deviceNodeDiscoverer creates a discoverer for the device nodes associated +// with the specified device. +// The CSV mount specs are used as the source for which device nodes are +// required with the following additions: +// +// - Any regular device nodes (i.e. /dev/nvidia[0-9]+) are removed from the +// input set. +// - The device node (i.e. /dev/nvidia{{ .index }}) associated with this +// particular device is added to the set of device nodes to be discovered. +func (l *csvDeviceGenerator) deviceNodeDiscoverer() (discover.Discover, error) { + mountSpecs := tegra.Transform( + tegra.Transform( + tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...), + // We remove non-device nodes. + tegra.OnlyDeviceNodes(), + ), + // We remove the regular (nvidia[0-9]+) device nodes. + tegra.WithoutRegularDeviceNodes(), + ) + return tegra.New( + tegra.WithLogger(l.logger), + tegra.WithDriverRoot(l.driverRoot), + tegra.WithDevRoot(l.devRoot), + tegra.WithHookCreator(l.hookCreator), + tegra.WithLdconfigPath(l.ldconfigPath), + tegra.WithLibrarySearchPaths(l.librarySearchPaths...), + tegra.WithMountSpecs( + mountSpecs, + // We add the specific device node for this device. + tegra.DeviceNodes(fmt.Sprintf("/dev/nvidia%d", l.index)), + ), + ) +} + // GetCommonEdits generates a CDI specification that can be used for ANY devices +// These explicitly do not include any device nodes. func (l *csvlib) GetCommonEdits() (*cdi.ContainerEdits, error) { - return edits.FromDiscoverer(discover.None{}) + mountSpecs := tegra.Transform( + tegra.Transform( + tegra.MountSpecsFromCSVFiles(l.logger, l.csvFiles...), + tegra.WithoutDeviceNodes(), + ), + tegra.IgnoreSymlinkMountSpecsByPattern(l.csvIgnorePatterns...), + ) + driverDiscoverer, err := tegra.New( + tegra.WithLogger(l.logger), + tegra.WithDriverRoot(l.driverRoot), + tegra.WithDevRoot(l.devRoot), + tegra.WithHookCreator(l.hookCreator), + tegra.WithLdconfigPath(l.ldconfigPath), + tegra.WithLibrarySearchPaths(l.librarySearchPaths...), + tegra.WithMountSpecs(mountSpecs), + ) + if err != nil { + return nil, fmt.Errorf("failed to create driver discoverer from CSV files: %w", err) + } + return edits.FromDiscoverer(driverDiscoverer) +} + +func (l *mixedcsvlib) DeviceSpecGenerators(ids ...string) (DeviceSpecGenerator, error) { + asNvmlLib := (*nvmllib)(l) + err := asNvmlLib.init() + if err != nil { + return nil, fmt.Errorf("failed to initialize nvml: %w", err) + } + defer asNvmlLib.tryShutdown() + + if slices.Contains(ids, "all") { + ids, err = l.getAllDeviceIndices() + if err != nil { + return nil, fmt.Errorf("failed to get device indices: %w", err) + } + } + + var DeviceSpecGenerators DeviceSpecGenerators + for _, id := range ids { + generator, err := l.deviceSpecGeneratorForId(device.Identifier(id)) + if err != nil { + return nil, fmt.Errorf("failed to create device spec generator for device %q: %w", id, err) + } + DeviceSpecGenerators = append(DeviceSpecGenerators, generator) + } + + return DeviceSpecGenerators, nil +} + +func (l *mixedcsvlib) getAllDeviceIndices() ([]string, error) { + numDevices, ret := l.nvmllib.DeviceGetCount() + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("faled to get device count: %v", ret) + } + + var allIndices []string + for index := range numDevices { + allIndices = append(allIndices, fmt.Sprintf("%d", index)) + } + return allIndices, nil +} + +func (l *mixedcsvlib) deviceSpecGeneratorForId(id device.Identifier) (DeviceSpecGenerator, error) { + switch { + case id.IsGpuUUID(), isIntegratedGPUID(id): + uuid := string(id) + device, ret := l.nvmllib.DeviceGetHandleByUUID(uuid) + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device handle from UUID %q: %v", uuid, ret) + } + index, ret := device.GetIndex() + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device index: %v", ret) + } + return l.csvDeviceSpecGenerator(index, uuid, device) + case id.IsGpuIndex(): + index, err := strconv.Atoi(string(id)) + if err != nil { + return nil, fmt.Errorf("failed to convert device index to an int: %w", err) + } + device, ret := l.nvmllib.DeviceGetHandleByIndex(index) + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get device handle from index: %v", ret) + } + uuid, ret := device.GetUUID() + if ret != nvml.SUCCESS { + return nil, fmt.Errorf("failed to get UUID: %v", ret) + } + return l.csvDeviceSpecGenerator(index, uuid, device) + case id.IsMigUUID(): + fallthrough + case id.IsMigIndex(): + return nil, fmt.Errorf("generating a CDI spec for MIG id %q is not supported in CSV mode", id) + } + return nil, fmt.Errorf("identifier is not a valid UUID or index: %q", id) +} + +func (l *mixedcsvlib) csvDeviceSpecGenerator(index int, uuid string, device nvml.Device) (DeviceSpecGenerator, error) { + isIntegrated, err := isIntegratedGPU(device) + if err != nil { + return nil, fmt.Errorf("is-integrated check failed for device (index=%v,uuid=%v)", index, uuid) + } + + g := &csvDeviceGenerator{ + csvlib: (*csvlib)(l), + index: index, + uuid: uuid, + } + + if !isIntegrated { + csvDeviceNodeDiscoverer, err := g.deviceNodeDiscoverer() + if err != nil { + return nil, fmt.Errorf("failed to create discoverer for devices nodes: %w", err) + } + + // If this is not an integrated GPU, we also create a spec generator for + // the full GPU. + dgpu := (*nvmllib)(l).withInit(&fullGPUDeviceSpecGenerator{ + nvmllib: (*nvmllib)(l), + uuid: uuid, + index: index, + // For the CSV case, we include the control device nodes at a + // device level. + additionalDiscoverers: []discover.Discover{ + (*nvmllib)(l).controlDeviceNodeDiscoverer(), + csvDeviceNodeDiscoverer, + }, + featureFlags: l.featureFlags, + }) + return dgpu, nil + } + + return g, nil +} + +func isIntegratedGPUID(id device.Identifier) bool { + _, err := uuid.Parse(string(id)) + return err == nil +} + +// isIntegratedGPU checks whether the specified device is an integrated GPU. +// As a proxy we check the PCI Bus if for thes +// TODO: This should be replaced by an explicit NVML call once available. +func isIntegratedGPU(d nvml.Device) (bool, error) { + pciInfo, ret := d.GetPciInfo() + if ret == nvml.ERROR_NOT_SUPPORTED { + name, ret := d.GetName() + if ret != nvml.SUCCESS { + return false, fmt.Errorf("failed to get device name: %v", ret) + } + return info.IsIntegratedGPUName(name), nil + } + if ret != nvml.SUCCESS { + return false, fmt.Errorf("failed to get PCI info: %v", ret) + } + + if pciInfo.Domain != 0 { + return false, nil + } + if pciInfo.Bus != 1 { + return false, nil + } + return pciInfo.Device == 0, nil } diff --git a/pkg/nvcdi/namer.go b/pkg/nvcdi/namer.go index 8019f699e..8ebdd33b4 100644 --- a/pkg/nvcdi/namer.go +++ b/pkg/nvcdi/namer.go @@ -105,12 +105,6 @@ type convert struct { nvmlUUIDer } -type uuidIgnored struct{} - -func (m uuidIgnored) GetUUID() (string, error) { - return "", nil -} - type uuidUnsupported struct{} func (m convert) GetUUID() (string, error) {