diff --git a/images/usb-modules/Makefile b/images/usb-modules/Makefile new file mode 100644 index 0000000000..4381f74a05 --- /dev/null +++ b/images/usb-modules/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 +# Out-of-tree build: no kernel .config needed. +# Use: make -C /lib/modules/$(uname -r)/build M=$(pwd) modules + +obj-m += usbip-core.o +usbip-core-y := usbip_common.o usbip_event.o + +obj-m += vhci-hcd.o +vhci-hcd-y := vhci_sysfs.o vhci_tx.o vhci_rx.o vhci_hcd.o + +obj-m += usbip-host.o +usbip-host-y := stub_dev.o stub_main.o stub_rx.o stub_tx.o diff --git a/images/usb-modules/apply-patches.sh b/images/usb-modules/apply-patches.sh new file mode 100755 index 0000000000..c6c8d7d735 --- /dev/null +++ b/images/usb-modules/apply-patches.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Apply patches from patches/ to each kernel version tree in .out/ +# Usage: ./apply-patches.sh [out-dir] +# Default out-dir is .out (relative to script directory) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${1:-.out}" +PATCHES_DIR="${SCRIPT_DIR}/patches" + +cd "$SCRIPT_DIR" +OUT_DIR="$(realpath "$OUT_DIR")" + +if [[ ! -d "$OUT_DIR" ]]; then + echo "Error: output directory '$OUT_DIR' does not exist" >&2 + exit 1 +fi + +if [[ ! -d "$PATCHES_DIR" ]]; then + echo "Error: patches directory '$PATCHES_DIR' does not exist" >&2 + exit 1 +fi + +for version_dir in "$OUT_DIR"/*/; do + [[ -d "$version_dir" ]] || continue + version="$(basename "$version_dir")" + for patch in "$PATCHES_DIR"/*.patch; do + [[ -f "$patch" ]] || continue + echo "Applying $(basename "$patch") to $version..." + if ! patch -d "$version_dir" -p0 --forward --silent < "$patch"; then + if patch -d "$version_dir" -p0 --reverse --check --silent < "$patch" 2>/dev/null; then + echo " (already applied, skipping)" + else + echo " FAILED" >&2 + exit 1 + fi + fi + done +done + +echo "Done." diff --git a/images/usb-modules/build-usbip-modules.sh b/images/usb-modules/build-usbip-modules.sh new file mode 100755 index 0000000000..3bbf2b7380 --- /dev/null +++ b/images/usb-modules/build-usbip-modules.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Build usbip-core, usbip-host, vhci-hcd for the running kernel and put .ko into a tmp dir. +# Needs: kernel headers (e.g. linux-headers-$(uname -r)), make, gcc. No full kernel source. +# +# Usage: +# ./build-usbip-modules.sh [OUTPUT_DIR] +# OUTPUT_DIR defaults to /tmp/usbip-modules-$(uname -r) +# USBIP_SRC defaults to the directory where this script lives (driver source). +# +# Env: +# KVER - kernel version to build for (default: uname -r) +# USBIP_SRC - path to driver source (must contain .c and Makefile.standalone) +# OUTPUT_DIR - output directory (overrides optional argument) +# +# Minimal deps: bash, make, gcc, kernel headers package (e.g. linux-headers-${KVER}). + +set -e + +KVER="${KVER:-$(uname -r)}" +for base in /lib/modules /usr/lib/modules; do + [[ -d "${base}/${KVER}/build" || -L "${base}/${KVER}/build" ]] || continue + KBUILD="${base}/${KVER}/build" + break +done +KBUILD="${KBUILD:-/lib/modules/${KVER}/build}" +USBIP_SRC="${USBIP_SRC:-$(cd "$(dirname "$0")" && pwd)}" +OUTPUT_DIR="${OUTPUT_DIR:-${1:-/tmp/usbip-modules-${KVER}}}" + +if [[ ! -d "$KBUILD" && ! -L "$KBUILD" ]]; then + echo "build-usbip-modules: kernel build dir not found for ${KVER}" >&2 + echo "Install kernel headers (kernel-devel or linux-headers-${KVER}) on the host." >&2 + exit 1 +fi + +# Resolve symlink so make sees the real path (must be visible in container) +if [[ -L "$KBUILD" ]]; then + KBUILD=$(readlink -f "$KBUILD") +fi +if [[ ! -d "$KBUILD" ]]; then + echo "build-usbip-modules: build is a symlink but its target is not visible in the container." >&2 + echo " resolved path: $KBUILD" >&2 + echo "Mount the kernel build tree from the host, e.g.:" >&2 + echo " -v /usr/src/kernels:/usr/src/kernels:ro" >&2 + exit 1 +fi + +if [[ ! -f "$USBIP_SRC/Makefile" ]]; then + echo "build-usbip-modules: $USBIP_SRC/Makefile not found" >&2 + exit 1 +fi + +make -C "$KBUILD" M="$USBIP_SRC" CC="${CC:-gcc}" modules + +mkdir -p "$OUTPUT_DIR" +cp -f "$USBIP_SRC"/usbip-core.ko "$USBIP_SRC"/usbip-host.ko "$USBIP_SRC"/vhci-hcd.ko "$OUTPUT_DIR/" + +echo "Built for ${KVER}; modules in: ${OUTPUT_DIR}" +ls -la "$OUTPUT_DIR"/*.ko diff --git a/images/usb-modules/build.sh b/images/usb-modules/build.sh new file mode 100755 index 0000000000..2be6fadd0a --- /dev/null +++ b/images/usb-modules/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Copyright 2026 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +KVER=$(uname -r | cut -d. -f1,2) +USBIP_SRC="/src/linux/$KVER/drivers/usb/usbip" ./build-usbip-modules.sh "$@" diff --git a/images/usb-modules/patches/001-vhci-increase-ports-and-controllers.patch b/images/usb-modules/patches/001-vhci-increase-ports-and-controllers.patch new file mode 100644 index 0000000000..9b5da03c97 --- /dev/null +++ b/images/usb-modules/patches/001-vhci-increase-ports-and-controllers.patch @@ -0,0 +1,24 @@ +--- drivers/usb/usbip/vhci.h ++++ drivers/usb/usbip/vhci.h +@@ -73,19 +73,11 @@ enum hub_speed { + }; + + /* Number of supported ports. Value has an upperbound of USB_MAXCHILDREN */ +-#ifdef CONFIG_USBIP_VHCI_HC_PORTS +-#define VHCI_HC_PORTS CONFIG_USBIP_VHCI_HC_PORTS +-#else +-#define VHCI_HC_PORTS 8 +-#endif ++#define VHCI_HC_PORTS 16 + + /* Each VHCI has 2 hubs (USB2 and USB3), each has VHCI_HC_PORTS ports */ + #define VHCI_PORTS (VHCI_HC_PORTS*2) + +-#ifdef CONFIG_USBIP_VHCI_NR_HCS +-#define VHCI_NR_HCS CONFIG_USBIP_VHCI_NR_HCS +-#else +-#define VHCI_NR_HCS 1 +-#endif ++#define VHCI_NR_HCS 4 + + #define MAX_STATUS_NAME 16 diff --git a/images/usb-modules/patches/README.md b/images/usb-modules/patches/README.md new file mode 100644 index 0000000000..bb65ec425a --- /dev/null +++ b/images/usb-modules/patches/README.md @@ -0,0 +1,34 @@ + +This directory contains patches used to build the following out-of-tree kernel modules: + +- `usbip-core` +- `usbip-host` +- `vhci-hcd` + +--- + +## 001-vhci-increase-ports-and-controllers.patch + +This patch modifies the default configuration of the `vhci-hcd` (Virtual Host Controller Interface) driver. + +### Changes + +- Sets the number of ports per virtual hub to **16** (hardcoded). +- Sets the number of virtual host controllers to **4** (hardcoded). +- Removes dependency on: + - `CONFIG_USBIP_VHCI_HC_PORTS` + - `CONFIG_USBIP_VHCI_NR_HCS` + +### Resulting Capacity + +Each VHCI controller provides: +- 2 hubs (USB 2.0 and USB 3.0) +- 16 ports per hub + +With 4 controllers total: + +4 controllers × 2 hubs × 16 ports = **128 ports** + +This allows up to **128 USB devices** to be attached simultaneously via USB/IP (subject to kernel and system limitations). + +> Note: The number of ports and controllers is now fixed at compile time and no longer configurable via kernel config options. diff --git a/images/usb-modules/tools/go.mod b/images/usb-modules/tools/go.mod new file mode 100644 index 0000000000..2c9dd8a11b --- /dev/null +++ b/images/usb-modules/tools/go.mod @@ -0,0 +1,10 @@ +module tools + +go 1.25.0 + +require ( + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.9 +) + +require github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/images/usb-modules/tools/go.sum b/images/usb-modules/tools/go.sum new file mode 100644 index 0000000000..a6ee3e0fb3 --- /dev/null +++ b/images/usb-modules/tools/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/images/usb-modules/tools/usbip-downloader/main.go b/images/usb-modules/tools/usbip-downloader/main.go new file mode 100644 index 0000000000..538e3499f9 --- /dev/null +++ b/images/usb-modules/tools/usbip-downloader/main.go @@ -0,0 +1,219 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "sync" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var kernel5Versions = []string{"5.0", "5.1", "5.2", "5.3", "5.4", "5.5", "5.6", "5.7", "5.8", "5.9", "5.10", "5.11", "5.12", "5.13", "5.14", "5.15", "5.16", "5.17", "5.18", "5.19"} +var kernel6Versions = []string{"6.0", "6.1", "6.2", "6.3", "6.4", "6.5", "6.6", "6.7", "6.8", "6.9", "6.10", "6.11", "6.12", "6.13", "6.14", "6.15", "6.16", "6.17", "6.18", "6.19"} + +var kernelVersions = append(kernel5Versions, kernel6Versions...) + +const gitLinuxUsbipContentUrlTmpl = "https://api.github.com/repos/torvalds/linux/contents/drivers/usb/usbip?ref=%s" + +func main() { + if err := NewDownloader().Execute(); err != nil { + log.Fatal(err) + } +} + +func NewDownloader() *cobra.Command { + o := options{} + cmd := &cobra.Command{ + Use: "usbip-downloader", + Short: "Downloads kernel modules for USBIP", + SilenceErrors: true, + SilenceUsage: true, + RunE: o.Run, + } + + o.AddFlags(cmd.Flags()) + + return cmd +} + +type options struct { + timeout time.Duration + maxIdleConns int + maxIdleConnsPerHost int + idleConnTimeout time.Duration + outputDir string +} + +func (o *options) AddFlags(fs *pflag.FlagSet) { + fs.DurationVar(&o.timeout, "timeout", 5*time.Minute, "timeout per download") + fs.IntVar(&o.maxIdleConns, "max-idle-conns", 20, "limit number of parallel downloads") + fs.IntVar(&o.maxIdleConnsPerHost, "max-idle-conns-per-host", 10, "limit number of parallel downloads") + fs.DurationVar(&o.idleConnTimeout, "idle-conn-timeout", 30*time.Second, "limit number of parallel downloads") + fs.StringVar(&o.outputDir, "out-dir", ".out", "output directory") + +} + +func (o *options) Run(_ *cobra.Command, _ []string) error { + client := &http.Client{ + Timeout: o.timeout, + Transport: &http.Transport{ + MaxIdleConns: o.maxIdleConns, + MaxIdleConnsPerHost: o.maxIdleConnsPerHost, + IdleConnTimeout: o.idleConnTimeout, + }, + } + + var wg sync.WaitGroup + errCh := make(chan error, len(kernelVersions)) + + for _, version := range kernelVersions { + dest := o.dest(version) + if _, err := os.Stat(dest); err == nil { + log.Printf("Skipping %s, already exists\n", version) + continue + } + + wg.Add(1) + go func() { + defer wg.Done() + + if err := o.download(client, version); err != nil { + errCh <- fmt.Errorf("%s: %w", version, err) + } + }() + } + + wg.Wait() + close(errCh) + + var resultErr error + for err := range errCh { + resultErr = errors.Join(resultErr, err) + } + + if resultErr != nil { + return fmt.Errorf("error downloading kernels: %w", resultErr) + } + + return o.checkDownloads() +} + +func (o *options) download(client *http.Client, version string) error { + content, err := o.getContent(client, version) + if err != nil { + return err + } + + for _, file := range content { + if err := o.downloadFile(client, file, version); err != nil { + return err + } + } + + log.Printf("Done %s\n", version) + return nil +} + +type fileInfo struct { + DownloadURL string `json:"download_url"` + Sha string `json:"sha"` + Path string `json:"path"` +} + +func (o *options) getContent(client *http.Client, version string) ([]fileInfo, error) { + url := fmt.Sprintf(gitLinuxUsbipContentUrlTmpl, tag(version)) + + resp, err := client.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s", resp.Status) + } + + var files []fileInfo + err = json.NewDecoder(resp.Body).Decode(&files) + if err != nil { + return nil, fmt.Errorf("error decoding json: %w", err) + } + + return files, nil +} + +func (o *options) downloadFile(client *http.Client, file fileInfo, version string) error { + dest := o.dest(version) + targetFile := filepath.Join(dest, file.Path) + + log.Printf("Downloading file. file: %s, downloadUrl: %s, sha: %s \n", targetFile, file.DownloadURL, file.Sha) + + resp, err := client.Get(file.DownloadURL) + if err != nil { + return fmt.Errorf("error downloading %s: %w", file.Path, err) + } + defer resp.Body.Close() + + dir := filepath.Dir(targetFile) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %w", dir, err) + } + + f, err := os.OpenFile(targetFile, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(0644)) + if err != nil { + return fmt.Errorf("create file failed: %w", err) + } + + if _, err := io.Copy(f, resp.Body); err != nil { + f.Close() + return fmt.Errorf("write file failed: %w", err) + } + + if err := f.Close(); err != nil { + return err + } + + return nil +} + +func (o *options) dest(version string) string { + return filepath.Join(o.outputDir, version) +} + +func (o *options) checkDownloads() error { + for _, version := range kernelVersions { + dest := o.dest(version) + if _, err := os.Stat(dest); err != nil { + return fmt.Errorf("missing download: %s: %w", version, err) + } + } + return nil +} + +func tag(version string) string { + return "v" + version +} diff --git a/images/usb-modules/werf.inc.yaml b/images/usb-modules/werf.inc.yaml new file mode 100644 index 0000000000..31b5efee00 --- /dev/null +++ b/images/usb-modules/werf.inc.yaml @@ -0,0 +1,100 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }}-tools +final: false +fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.25" "builder/golang-alt-svace-1.25.4" }} +git: + - add: {{ .ModuleDir }}/images/{{ .ImageName }}/tools + to: /src/images/usb-modules/tools + stageDependencies: + install: + - go.mod + - go.sum + setup: + - "**/*.go" +secrets: + - id: GOPROXY + value: {{ .GOPROXY }} +mount: + - fromPath: ~/go-pkg-cache + to: /go/pkg +shell: + install: + - export GOPROXY=$(cat /run/secrets/GOPROXY) + - cd /src/images/usb-modules/tools + - go mod download + setup: + - cd /src/images/usb-modules/tools + - mkdir /out + - export GOOS=linux + - export GOARCH=amd64 + - export CGO_ENABLED=0 + + - | + echo "Build usbip-downloader binary" + {{- $_ := set $ "ProjectName" (list $.ImageName "usbip-downloader" | join "/") }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/usbip-downloader ./usbip-downloader`) | nindent 6 }} + +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }}-downloader +fromImage: {{ "builder/debian-trixie-slim" }} +final: false +import: + - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-tools + add: /out/usbip-downloader + to: /usb/bin/usbip-downloader + before: setup +shell: + install: + - apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* + setup: + - mkdir -p /src/linux + - /usb/bin/usbip-downloader --out-dir=/src/linux + +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }} +# Need GCC 14+ for -fmin-function-alignment (used by Fedora kernel build) +fromImage: {{ "builder/debian-trixie-slim" }} +import: + - image: {{ .ModuleNamePrefix }}{{ .ImageName }}-downloader + add: /src/linux + to: /src/linux + after: install +git: + - add: {{ .ModuleDir }}/images/{{ .ImageName }} + to: /src/images/usb-modules + stageDependencies: + setup: + - "**/Makefile" + - "**/build-usbip-modules.sh" + - "**/build.sh" + - "**/apply-patches.sh" + - "**/patches/*" +shell: + install: + - | + apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + patch \ + make \ + gcc \ + libc-dev \ + libelf-dev \ + && rm -rf /var/lib/apt/lists/* + setup: + # copy Makefile to every kernel version + - | + for dir in src/linux/*/drivers/usb/usbip; do + cp /src/images/usb-modules/Makefile "$dir/Makefile" + done + - | + chmod +x /src/images/usb-modules/apply-patches.sh + /src/images/usb-modules/apply-patches.sh "/src/linux/" + + - chmod +x /src/images/usb-modules/build-usbip-modules.sh + - chmod +x /src/images/usb-modules/build.sh + +imageSpec: + config: + user: 64535 + workingDir: "/src/images/usb-modules" + entrypoint: ["./build.sh", "/out"] diff --git a/templates/virtualization-dra/_helper.tpl b/templates/virtualization-dra/_helper.tpl index 469b9d5379..19ab94ec10 100644 --- a/templates/virtualization-dra/_helper.tpl +++ b/templates/virtualization-dra/_helper.tpl @@ -1,7 +1,7 @@ {{- define "virtualization-dra.isEnabled" -}} {{- if eq (include "hasValidModuleConfig" .) "true" -}} {{- if semverCompare ">=1.34" .Values.global.discovery.kubernetesVersion -}} -false +true {{- end -}} {{- end -}} {{- end -}} diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml index 8ac7eb4e42..0d7132c61c 100644 --- a/templates/virtualization-dra/daemonset.yaml +++ b/templates/virtualization-dra/daemonset.yaml @@ -65,10 +65,30 @@ spec: nodeSelector: kubernetes.io/os: linux initContainers: + - name: init-build + command: + - bash + - -c + - "./build.sh /tmp || echo 'Failed to build modules'" + image: {{ include "helm_lib_module_image" (list . "usbModules") }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- include "virtualization-dra_init_resources" . | nindent 14 }} + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /usr/src + name: usr-src + readOnly: true + - mountPath: /tmp + name: tmp - name: init-load image: {{ include "helm_lib_module_image" (list . "virtualizationDraUsb") }} args: - init + - --try-custom-build-path=/tmp {{- include "helm_lib_module_container_security_context_read_only_root_filesystem_capabilities_drop_all_and_add" (list . (list "SYS_MODULE")) | nindent 10 }} resources: requests: @@ -148,6 +168,9 @@ spec: - name: lib-modules hostPath: path: /lib/modules + - name: usr-src + hostPath: + path: /usr/src - name: plugins-registry hostPath: path: /var/lib/kubelet/plugins_registry