Feat/ubuntu 24

Jenkinsfile

@@ -35,6 +35,7 @@ def preserveContainersOnFailure = env.preserve_containers_on_failure?.toBoolean(
 
 def buildCustomComponents = env.build_custom_components?.toBoolean() ?: false
 def openmpfCustomRepoCredId = env.openmpf_custom_repo_cred_id
+def openmpfProjectsRepoCredId = env.openmpf_projects_repo_cred_id
 def applyCustomConfig = env.apply_custom_config?.toBoolean() ?: false
 def mvnTestOptions = env.mvn_test_options ?: ''
 
@@ -59,6 +60,8 @@ def preDockerBuildScriptPath = env.pre_docker_build_script_path
 def runTrivyScans = env.run_trivy_scans?.toBoolean() ?: false
 def runTrivyInsecure = env.run_trivy_insecure?.toBoolean() ?: false
 def runTrivyMaxTasks = env.run_trivy_max_tasks?.toInteger() ?: 4
+def trivyImg = env.trivy_image ?: 'aquasec/trivy:0.69.3@sha256:bcc376de8d77cfe086a917230e818dc9f8528e3c852f7b1aff648949b6258d1c'
+
 def dependencyTrackCredId = env.dependency_track_cred_id
 def dependencyTrackUploadSbom = env.dependency_track_upload_sbom?.toBoolean() ?: false
 def skipIntegrationTests = env.skip_integration_tests?.toBoolean() ?: false
@@ -93,10 +96,9 @@ class Repo {
 }
 
 
-def openmpfProjectsRepo = new Repo('openmpf-projects', 'https://github.com/openmpf/openmpf-projects.git',
+def openmpfProjectsRepo = new Repo('openmpf-projects', env.openmpf_projects_url,
         env.openmpf_projects_branch ?: 'develop')
 
-
 def openmpfDockerRepo = Repo.projectsSubRepo('openmpf-docker', env.openmpf_docker_branch)
 
 def openmpfRepo = Repo.projectsSubRepo('openmpf', env.openmpf_branch)
@@ -182,9 +184,19 @@ try {
         // Directory may not exist. In that case the command doesn't do anything.
         sh "rm -rf $openmpfDockerRepo.path/test-reports/*"
 
-        if (!fileExists(openmpfProjectsRepo.path)) {
-            sh "git clone --recurse-submodules $openmpfProjectsRepo.url"
-        }
+        checkout(
+            $class: 'GitSCM',
+            userRemoteConfigs: [[url: openmpfProjectsRepo.url, credentialsId: openmpfCustomRepoCredId]],
+            extensions: [
+                [$class: 'CleanBeforeCheckout'],
+                [$class: 'GitLFSPull'],
+                [$class: 'RelativeTargetDirectory', relativeTargetDir: openmpfProjectsRepo.path],
+                [$class: 'SubmoduleOption',
+                    disableSubmodules: false,
+                    parentCredentials: true,
+                    recursiveSubmodules: true,
+                    trackingSubmodules: false]])
+
         dir(openmpfProjectsRepo.path) {
             sh 'git clean -ffd'
             sh 'git submodule foreach git clean -ffd'
@@ -254,8 +266,8 @@ try {
     stage('Build images') {
     timeout(time: buildTimeout, unit: 'HOURS') {
         // Make sure we are using most recent version of external images
-        for (externalImage in ['docker/dockerfile:1.2', 'postgres:17-alpine',
-                               'redis:alpine', 'ubuntu:20.04']) {
+        for (externalImage in ['docker/dockerfile:1.20', 'postgres:17-alpine',
+                               'redis:alpine', 'ubuntu:24.04']) {
             try {
                 sh "docker pull '$externalImage'"
             }
@@ -361,14 +373,14 @@ try {
                         "../../$customComponentsRepo.path/docker-compose.custom-components.yml"
                 componentComposeFiles += ":$customComponentsComposeFile"
 
-                def customGpuOnlyComponentsComposeFile =
-                            "../../$customComponentsRepo.path/docker-compose.custom-gpu-only-components.yml"
-                componentComposeFiles += ":$customGpuOnlyComponentsComposeFile"
+                // def customGpuOnlyComponentsComposeFile =
+                //             "../../$customComponentsRepo.path/docker-compose.custom-gpu-only-components.yml"
+                // componentComposeFiles += ":$customGpuOnlyComponentsComposeFile"
 
                 customComponentServices =
                         readYaml(text: shOutput("cat $customComponentsComposeFile")).services.keySet()
-                customComponentServices +=
-                        readYaml(text: shOutput("cat $customGpuOnlyComponentsComposeFile")).services.keySet()
+                // customComponentServices +=
+                //         readYaml(text: shOutput("cat $customGpuOnlyComponentsComposeFile")).services.keySet()
             }
 
             withEnv(["COMPOSE_FILE=$componentComposeFiles"]) {
@@ -468,7 +480,8 @@ try {
                 composeYaml = readYaml(text: shOutput('docker compose config'))
             }
         }
-        sh 'docker pull aquasec/trivy'
+
+        sh "docker pull $trivyImg"
         def trivyVolume = "trivy_$inProgressTag"
         sh "docker volume create $trivyVolume"
         try {
@@ -479,22 +492,22 @@ try {
             def taskQueue = []
 
             // download db files
-            def exitCode = shStatus("docker run --rm " +
-                            "-e TRIVY_INSECURE=${runTrivyInsecure} " +
-                            "-v /var/run/docker.sock:/var/run/docker.sock " +
-                            "-v $trivyVolume:/root/.cache/ " +
-                            "aquasec/trivy image --download-db-only")
-            if (exitCode != 0) {
+            def dbDownloadExitCode = shStatus(
+                "docker run --rm " +
+                "-e TRIVY_INSECURE=${runTrivyInsecure} " +
+                "-v $trivyVolume:/root/.cache/ " +
+                "$trivyImg image --download-db-only")
+            if (dbDownloadExitCode != 0) {
                 echo 'Trivy failed to download the db files'
             }
 
             // download the java db files
-            exitCode = shStatus("docker run --rm " +
-                            "-e TRIVY_INSECURE=${runTrivyInsecure} " +
-                            "-v /var/run/docker.sock:/var/run/docker.sock " +
-                            "-v $trivyVolume:/root/.cache/ " +
-                            "aquasec/trivy image --download-java-db-only")
-            if (exitCode != 0) {
+            dbDownloadExitCode = shStatus(
+                "docker run --rm " +
+                "-e TRIVY_INSECURE=${runTrivyInsecure} " +
+                "-v $trivyVolume:/root/.cache/ " +
+                "$trivyImg image --download-java-db-only")
+            if (dbDownloadExitCode != 0) {
                 echo 'Trivy failed to download the java db files'
             }
 
@@ -503,28 +516,29 @@ try {
                     // fetch service using the serviceName
                     def service = composeYaml.services[serviceName]
 
+                    def workDir = "${pwd()}/$openmpfDockerRepo.path"
+
                     // save the output to a file
-                    exitCode = shStatus("docker run --rm " +
+                    def exitCode = shStatus("docker run --rm " +
                             "-e TRIVY_INSECURE=${runTrivyInsecure} " +
-                            "-v /var/run/docker.sock:/var/run/docker.sock " +
                             "-v $trivyVolume:/root/.cache/ " +
-                            "-v '${pwd()}/$openmpfDockerRepo.path/trivyignore.txt:/.trivyignore' " +
-                            "-v '${pwd()}/$openmpfDockerRepo.path:/trivy' " +
-                            "aquasec/trivy image --format cyclonedx --output /trivy/${serviceName}_sbom.json " +
-                            "--timeout 30m $service.image")
+                            "-v '${workDir}/trivyignore.txt:/.trivyignore' " +
+                            "--mount type=image,source=${service.image},target=/data/rootfs " +
+                            "$trivyImg rootfs --format cyclonedx " +
+                            "--timeout 30m /data/rootfs > ${workDir}/${serviceName}_sbom.json")
+
                     if (exitCode != 0) {
                         failedImages << service.image
                     } else {
                         // print the vulnerabilities to a file
                         exitCode = shStatus("docker run --rm " +
                             "-e TRIVY_INSECURE=${runTrivyInsecure} " +
-                            "-v /var/run/docker.sock:/var/run/docker.sock " +
                             "-v $trivyVolume:/root/.cache/ " +
-                            "-v '${pwd()}/$openmpfDockerRepo.path/trivyignore.txt:/.trivyignore' " +
-                            "-v '${pwd()}/$openmpfDockerRepo.path:/trivy' " +
-                            "aquasec/trivy sbom --severity CRITICAL,HIGH --exit-code 1 " +
-                            "--format json --output /trivy/${serviceName}_trivy.json " +
-                            "--timeout 30m --scanners vuln /trivy/${serviceName}_sbom.json")
+                            "-v '${workDir}/trivyignore.txt:/.trivyignore' " +
+                            "-v ${workDir}/${serviceName}_sbom.json:/data/sbom.json " +
+                            "$trivyImg sbom --severity CRITICAL,HIGH --exit-code 1 " +
+                            "--format json --timeout 30m --scanners vuln " +
+                            "/data/sbom.json > ${workDir}/${serviceName}_trivy.json")
                         if (exitCode != 0) {
                             failedImages << service.image
                         }
@@ -535,20 +549,13 @@ try {
 
                         // upload to dependency track
                         if (dependencyTrackUploadSbom) {
-                            def (serviceImageName, serviceVersion) = service.image?.split(':')
-                            if (imageVersion) {
-                                projectVersion = imageVersion
-                            } else {
-                                if (service.version) {
-                                    projectVersion = service.version
-                                }
-                            }
+                            def (serviceImageName, serviceVersion) = service.image.split(':')
 
                             // publish using the dependency track plugin
                             dependencyTrackPublisher(
                                 artifact: "${openmpfDockerRepo.path}/${serviceName}_sbom.json",
                                 projectName: serviceImageName,
-                                projectVersion: projectVersion,
+                                projectVersion: imageVersion ?: serviceVersion,
                                 synchronous: false,
                                 dependencyTrackApiKey: dependencyTrackCredId,
                                 projectProperties: [
@@ -574,7 +581,7 @@ try {
 
                 // add tasks the map
                 for (int i = 0; i < taskCount; i++) {
-                    def taskName = "Task ${i}" 
+                    def taskName = "Task ${i}"
                     tasksToRun[taskName] = taskQueue.remove(0)
                 }
 

components/cli_runner/tests/test_cli_runner.py

@@ -410,7 +410,7 @@ def test_can_run_generic_job(self):
 
         self.assertEqual(1, len(tracks))
         track = tracks[0]
-        self.assertAlmostEqual(95, track['confidence'])
+        self.assertAlmostEqual(94, track['confidence'])
         expected_properties = {
             'MISSING_LANGUAGE_MODELS': '',
             'OSD_FALLBACK_OCCURRED': 'false',
@@ -431,7 +431,7 @@ def test_can_run_generic_job(self):
         self.assertEqual(0, detection['y'])
         self.assertEqual(0, detection['width'])
         self.assertEqual(0, detection['height'])
-        self.assertAlmostEqual(95, detection['confidence'])
+        self.assertAlmostEqual(94, detection['confidence'])
         self.assertEqual(detection, track['exemplar'])
         self.assertEqual(expected_properties, detection['detectionProperties'])
 

components/cpp_component_build/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.4
+# syntax=docker/dockerfile:1.20
 
 #############################################################################
 # NOTICE                                                                    #
@@ -28,39 +28,41 @@
 
 ARG BUILD_REGISTRY
 ARG BUILD_TAG=latest
-FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} as openmpf_build
+FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} AS openmpf_build
 
 
-FROM ubuntu:20.04
+FROM ubuntu:24.04
 
 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
+ENV LANG=C.UTF-8
+ENV LC_ALL=C.UTF-8
 
 RUN --mount=type=tmpfs,target=/var/cache/apt \
     --mount=type=tmpfs,target=/var/lib/apt/lists  \
-    --mount=type=tmpfs,target=/tmp \
-    apt-get update; \
-    apt-get upgrade -y; \
+    --mount=type=tmpfs,target=/tmp <<eot
+    apt-get update
+    apt-get upgrade -y
     apt-get install --no-install-recommends -y \
         wget ca-certificates gnupg2 cmake make g++ libgtest-dev  liblog4cxx-dev libboost-dev \
-        libavcodec58 libavformat58 libswscale5 libopenblas0-pthread liblapacke; \
-    # Can't set up cuda repo at the beginning because it requires wget ca-certificates gnupg2
-    wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub \
-            | apt-key add -; \
-    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \
-        > /etc/apt/sources.list.d/cuda.list; \
-    apt-get update; \
+        libavcodec60 libavformat60 libswscale7 libopenblas0-pthread liblapacke
+
+    # Cannot set up cuda repo at the beginning because it requires wget ca-certificates gnupg2
+    wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \
+        | apt-key add -
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /" \
+        > /etc/apt/sources.list.d/cuda.list
+    apt-get update
     apt-get install -y --no-install-recommends \
-        cuda-minimal-build-11-4 libcudnn8-dev=8.2.4.15-1+cuda11.4 libcudnn8=8.2.4.15-1+cuda11.4 \
-        libcublas-dev-11-4; \
-    # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large.
-    rm /usr/local/cuda/lib64/libcublasLt_static.a /usr/local/cuda/lib64/libcublas_static.a \
-        /usr/lib/x86_64-linux-gnu/libcudnn*.a; \
-    ln --symbolic /usr/bin/cmake /usr/bin/cmake3;
+        cuda-minimal-build-12-9 libcudnn9-dev-cuda-12 libcublas-dev-12-9
 
-COPY --from=openmpf_build /opt/opencv-4.9.0 /opt/opencv-4.9.0
+    # OpenCV does not use the statically compiled CUDA libraries except for libcudart and they are relatively large.
+    find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete
+    ln --symbolic /usr/bin/cmake /usr/bin/cmake3
+eot
+
+
+COPY --from=openmpf_build /opt/opencv-4.13.0 /opt/opencv-4.13.0
 
 COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/
 
@@ -71,19 +73,21 @@ RUN mkdir /home/mpf; echo 'find_package(mpfCMakeHelpers REQUIRED)' > /home/mpf/C
 COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/include /home/mpf/mpf-sdk-install/include
 COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/lib /home/mpf/mpf-sdk-install/lib
 
-RUN echo '/opt/opencv-4.9.0/lib' > /etc/ld.so.conf.d/opencv.conf; \
-    echo '/home/mpf/mpf-sdk-install/lib' > /etc/ld.so.conf.d/mpf.conf; \
+RUN <<eot
+    echo '/opt/opencv-4.13.0/lib' > /etc/ld.so.conf.d/opencv.conf
+    echo '/home/mpf/mpf-sdk-install/lib' > /etc/ld.so.conf.d/mpf.conf
     ldconfig
+eot
 
 COPY cpp_component_build/scripts /scripts
 
-ENV PATH /scripts:$PATH
+ENV PATH=/scripts:$PATH
 
-ENV CMAKE_PREFIX_PATH /home/mpf/mpf-sdk-install/lib/cmake
+ENV CMAKE_PREFIX_PATH=/home/mpf/mpf-sdk-install/lib/cmake
 
-ENV BUILD_DIR /home/mpf/component_build
+ENV BUILD_DIR=/home/mpf/component_build
 
-ENV SRC_DIR /home/mpf/component_src
+ENV SRC_DIR=/home/mpf/component_src
 
 WORKDIR $SRC_DIR
 

components/cpp_executor/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:1.4
+# syntax=docker/dockerfile:1.20
 
 #############################################################################
 # NOTICE                                                                    #
@@ -31,7 +31,7 @@ ARG BUILD_TAG=latest
 FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} AS openmpf_build
 
 
-FROM ubuntu:20.04 AS common
+FROM ubuntu:24.04 AS common
 
 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 
@@ -40,35 +40,18 @@ ENV LC_ALL=C.UTF-8
 
 RUN --mount=type=tmpfs,target=/var/cache/apt \
     --mount=type=tmpfs,target=/var/lib/apt/lists  \
-    --mount=type=tmpfs,target=/tmp \
-    apt-get update; \
-    apt-get upgrade -y; \
-    apt-get install --no-install-recommends -y wget ca-certificates gnupg2; \
-    # Can't set up cuda repo at the beginning because it requires wget ca-certificates gnupg2
-    wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub \
-            | apt-key add -; \
-    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \
-        > /etc/apt/sources.list.d/cuda.list; \
-    apt-get update; \
-    apt-get install -y --no-install-recommends cuda-cudart-11-4;
-
-
-COPY --from=openmpf_build \
-    /usr/local/bin/python3.12 /usr/local/bin/python3.12-config \
-    /usr/local/bin/pip3.12 /usr/local/bin/pip3 \
-    /usr/local/bin/
-
-COPY --from=openmpf_build /usr/local/lib/python3.12 /usr/local/lib/python3.12
-
-COPY --from=openmpf_build /usr/local/lib/libpython3.12.so.1.0 /usr/local/lib/
-
-RUN <<eot
-    cd /usr/local/bin
-    ln --symbolic python3.12 python3
-    ln --symbolic python3.12-config python3-config
-    cd /usr/local/lib
-    ln --symbolic libpython3.12.so.1.0 libpython3.12.so
-    ln --symbolic /usr/local/lib/libpython3.12.so.1.0 /lib/x86_64-linux-gnu/libpython3.12.so.1.0
+    --mount=type=tmpfs,target=/tmp <<eot
+    apt-get update
+    apt-get upgrade -y
+    apt-get install --no-install-recommends -y wget ca-certificates gnupg2 python3 libpython3.12t64
+
+    # Can not set up cuda repo at the beginning because it requires wget ca-certificates gnupg2
+    wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \
+        | apt-key add -
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /" \
+        > /etc/apt/sources.list.d/cuda.list
+    apt-get update
+    apt-get install -y --no-install-recommends cuda-cudart-12-9
 eot
 
 COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/
@@ -103,7 +86,7 @@ COPY --from=openmpf_build /build-artifacts/install/bin/amq_detection_component \
 COPY --from=openmpf_build /build-artifacts/install/config/Log4cxxConfig.xml $MPF_HOME/config/
 
 COPY --from=openmpf_build /build-artifacts/install/lib/mpf_cpp_sdk.cpython-312-x86_64-linux-gnu.so \
-                          /usr/local/lib/python3.12/lib-dynload/
+                          /usr/lib/python3.12/lib-dynload
 
 COPY --from=openmpf_build /scripts/* /scripts/
 
@@ -115,7 +98,6 @@ ENV BUILD_DIR=/home/mpf/component_build
 
 ENV PLUGINS_DIR=$MPF_HOME/plugins
 
-
 COPY docker-entrypoint.sh component-executor.py /scripts/
 
 COPY cli_runner/*.py cli_runner/Log4cxxConfig.xml /scripts/cli_runner/