diff --git a/Jenkinsfile b/Jenkinsfile index 372279d9..fb048db6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -35,6 +35,7 @@ def preserveContainersOnFailure = env.preserve_containers_on_failure?.toBoolean( def buildCustomComponents = env.build_custom_components?.toBoolean() ?: false def openmpfCustomRepoCredId = env.openmpf_custom_repo_cred_id +def openmpfProjectsRepoCredId = env.openmpf_projects_repo_cred_id def applyCustomConfig = env.apply_custom_config?.toBoolean() ?: false def mvnTestOptions = env.mvn_test_options ?: '' @@ -59,6 +60,8 @@ def preDockerBuildScriptPath = env.pre_docker_build_script_path def runTrivyScans = env.run_trivy_scans?.toBoolean() ?: false def runTrivyInsecure = env.run_trivy_insecure?.toBoolean() ?: false def runTrivyMaxTasks = env.run_trivy_max_tasks?.toInteger() ?: 4 +def trivyImg = env.trivy_image ?: 'aquasec/trivy:0.69.3@sha256:bcc376de8d77cfe086a917230e818dc9f8528e3c852f7b1aff648949b6258d1c' + def dependencyTrackCredId = env.dependency_track_cred_id def dependencyTrackUploadSbom = env.dependency_track_upload_sbom?.toBoolean() ?: false def skipIntegrationTests = env.skip_integration_tests?.toBoolean() ?: false @@ -93,10 +96,9 @@ class Repo { } -def openmpfProjectsRepo = new Repo('openmpf-projects', 'https://github.com/openmpf/openmpf-projects.git', +def openmpfProjectsRepo = new Repo('openmpf-projects', env.openmpf_projects_url, env.openmpf_projects_branch ?: 'develop') - def openmpfDockerRepo = Repo.projectsSubRepo('openmpf-docker', env.openmpf_docker_branch) def openmpfRepo = Repo.projectsSubRepo('openmpf', env.openmpf_branch) @@ -182,9 +184,19 @@ try { // Directory may not exist. In that case the command doesn't do anything. sh "rm -rf $openmpfDockerRepo.path/test-reports/*" - if (!fileExists(openmpfProjectsRepo.path)) { - sh "git clone --recurse-submodules $openmpfProjectsRepo.url" - } + checkout( + $class: 'GitSCM', + userRemoteConfigs: [[url: openmpfProjectsRepo.url, credentialsId: openmpfCustomRepoCredId]], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'GitLFSPull'], + [$class: 'RelativeTargetDirectory', relativeTargetDir: openmpfProjectsRepo.path], + [$class: 'SubmoduleOption', + disableSubmodules: false, + parentCredentials: true, + recursiveSubmodules: true, + trackingSubmodules: false]]) + dir(openmpfProjectsRepo.path) { sh 'git clean -ffd' sh 'git submodule foreach git clean -ffd' @@ -254,8 +266,8 @@ try { stage('Build images') { timeout(time: buildTimeout, unit: 'HOURS') { // Make sure we are using most recent version of external images - for (externalImage in ['docker/dockerfile:1.2', 'postgres:17-alpine', - 'redis:alpine', 'ubuntu:20.04']) { + for (externalImage in ['docker/dockerfile:1.20', 'postgres:17-alpine', + 'redis:alpine', 'ubuntu:24.04']) { try { sh "docker pull '$externalImage'" } @@ -361,14 +373,14 @@ try { "../../$customComponentsRepo.path/docker-compose.custom-components.yml" componentComposeFiles += ":$customComponentsComposeFile" - def customGpuOnlyComponentsComposeFile = - "../../$customComponentsRepo.path/docker-compose.custom-gpu-only-components.yml" - componentComposeFiles += ":$customGpuOnlyComponentsComposeFile" + // def customGpuOnlyComponentsComposeFile = + // "../../$customComponentsRepo.path/docker-compose.custom-gpu-only-components.yml" + // componentComposeFiles += ":$customGpuOnlyComponentsComposeFile" customComponentServices = readYaml(text: shOutput("cat $customComponentsComposeFile")).services.keySet() - customComponentServices += - readYaml(text: shOutput("cat $customGpuOnlyComponentsComposeFile")).services.keySet() + // customComponentServices += + // readYaml(text: shOutput("cat $customGpuOnlyComponentsComposeFile")).services.keySet() } withEnv(["COMPOSE_FILE=$componentComposeFiles"]) { @@ -468,7 +480,8 @@ try { composeYaml = readYaml(text: shOutput('docker compose config')) } } - sh 'docker pull aquasec/trivy' + + sh "docker pull $trivyImg" def trivyVolume = "trivy_$inProgressTag" sh "docker volume create $trivyVolume" try { @@ -479,22 +492,22 @@ try { def taskQueue = [] // download db files - def exitCode = shStatus("docker run --rm " + - "-e TRIVY_INSECURE=${runTrivyInsecure} " + - "-v /var/run/docker.sock:/var/run/docker.sock " + - "-v $trivyVolume:/root/.cache/ " + - "aquasec/trivy image --download-db-only") - if (exitCode != 0) { + def dbDownloadExitCode = shStatus( + "docker run --rm " + + "-e TRIVY_INSECURE=${runTrivyInsecure} " + + "-v $trivyVolume:/root/.cache/ " + + "$trivyImg image --download-db-only") + if (dbDownloadExitCode != 0) { echo 'Trivy failed to download the db files' } // download the java db files - exitCode = shStatus("docker run --rm " + - "-e TRIVY_INSECURE=${runTrivyInsecure} " + - "-v /var/run/docker.sock:/var/run/docker.sock " + - "-v $trivyVolume:/root/.cache/ " + - "aquasec/trivy image --download-java-db-only") - if (exitCode != 0) { + dbDownloadExitCode = shStatus( + "docker run --rm " + + "-e TRIVY_INSECURE=${runTrivyInsecure} " + + "-v $trivyVolume:/root/.cache/ " + + "$trivyImg image --download-java-db-only") + if (dbDownloadExitCode != 0) { echo 'Trivy failed to download the java db files' } @@ -503,28 +516,29 @@ try { // fetch service using the serviceName def service = composeYaml.services[serviceName] + def workDir = "${pwd()}/$openmpfDockerRepo.path" + // save the output to a file - exitCode = shStatus("docker run --rm " + + def exitCode = shStatus("docker run --rm " + "-e TRIVY_INSECURE=${runTrivyInsecure} " + - "-v /var/run/docker.sock:/var/run/docker.sock " + "-v $trivyVolume:/root/.cache/ " + - "-v '${pwd()}/$openmpfDockerRepo.path/trivyignore.txt:/.trivyignore' " + - "-v '${pwd()}/$openmpfDockerRepo.path:/trivy' " + - "aquasec/trivy image --format cyclonedx --output /trivy/${serviceName}_sbom.json " + - "--timeout 30m $service.image") + "-v '${workDir}/trivyignore.txt:/.trivyignore' " + + "--mount type=image,source=${service.image},target=/data/rootfs " + + "$trivyImg rootfs --format cyclonedx " + + "--timeout 30m /data/rootfs > ${workDir}/${serviceName}_sbom.json") + if (exitCode != 0) { failedImages << service.image } else { // print the vulnerabilities to a file exitCode = shStatus("docker run --rm " + "-e TRIVY_INSECURE=${runTrivyInsecure} " + - "-v /var/run/docker.sock:/var/run/docker.sock " + "-v $trivyVolume:/root/.cache/ " + - "-v '${pwd()}/$openmpfDockerRepo.path/trivyignore.txt:/.trivyignore' " + - "-v '${pwd()}/$openmpfDockerRepo.path:/trivy' " + - "aquasec/trivy sbom --severity CRITICAL,HIGH --exit-code 1 " + - "--format json --output /trivy/${serviceName}_trivy.json " + - "--timeout 30m --scanners vuln /trivy/${serviceName}_sbom.json") + "-v '${workDir}/trivyignore.txt:/.trivyignore' " + + "-v ${workDir}/${serviceName}_sbom.json:/data/sbom.json " + + "$trivyImg sbom --severity CRITICAL,HIGH --exit-code 1 " + + "--format json --timeout 30m --scanners vuln " + + "/data/sbom.json > ${workDir}/${serviceName}_trivy.json") if (exitCode != 0) { failedImages << service.image } @@ -535,20 +549,13 @@ try { // upload to dependency track if (dependencyTrackUploadSbom) { - def (serviceImageName, serviceVersion) = service.image?.split(':') - if (imageVersion) { - projectVersion = imageVersion - } else { - if (service.version) { - projectVersion = service.version - } - } + def (serviceImageName, serviceVersion) = service.image.split(':') // publish using the dependency track plugin dependencyTrackPublisher( artifact: "${openmpfDockerRepo.path}/${serviceName}_sbom.json", projectName: serviceImageName, - projectVersion: projectVersion, + projectVersion: imageVersion ?: serviceVersion, synchronous: false, dependencyTrackApiKey: dependencyTrackCredId, projectProperties: [ @@ -574,7 +581,7 @@ try { // add tasks the map for (int i = 0; i < taskCount; i++) { - def taskName = "Task ${i}" + def taskName = "Task ${i}" tasksToRun[taskName] = taskQueue.remove(0) } diff --git a/components/cli_runner/tests/test_cli_runner.py b/components/cli_runner/tests/test_cli_runner.py index 891b8ffb..456761b6 100644 --- a/components/cli_runner/tests/test_cli_runner.py +++ b/components/cli_runner/tests/test_cli_runner.py @@ -410,7 +410,7 @@ def test_can_run_generic_job(self): self.assertEqual(1, len(tracks)) track = tracks[0] - self.assertAlmostEqual(95, track['confidence']) + self.assertAlmostEqual(94, track['confidence']) expected_properties = { 'MISSING_LANGUAGE_MODELS': '', 'OSD_FALLBACK_OCCURRED': 'false', @@ -431,7 +431,7 @@ def test_can_run_generic_job(self): self.assertEqual(0, detection['y']) self.assertEqual(0, detection['width']) self.assertEqual(0, detection['height']) - self.assertAlmostEqual(95, detection['confidence']) + self.assertAlmostEqual(94, detection['confidence']) self.assertEqual(detection, track['exemplar']) self.assertEqual(expected_properties, detection['detectionProperties']) diff --git a/components/cpp_component_build/Dockerfile b/components/cpp_component_build/Dockerfile index 76995f79..e819818d 100644 --- a/components/cpp_component_build/Dockerfile +++ b/components/cpp_component_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.4 +# syntax=docker/dockerfile:1.20 ############################################################################# # NOTICE # @@ -28,39 +28,41 @@ ARG BUILD_REGISTRY ARG BUILD_TAG=latest -FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} as openmpf_build +FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} AS openmpf_build -FROM ubuntu:20.04 +FROM ubuntu:24.04 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 +ENV LANG=C.UTF-8 +ENV LC_ALL=C.UTF-8 RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/var/lib/apt/lists \ - --mount=type=tmpfs,target=/tmp \ - apt-get update; \ - apt-get upgrade -y; \ + --mount=type=tmpfs,target=/tmp < /etc/apt/sources.list.d/cuda.list; \ - apt-get update; \ + libavcodec60 libavformat60 libswscale7 libopenblas0-pthread liblapacke + + # Cannot set up cuda repo at the beginning because it requires wget ca-certificates gnupg2 + wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \ + | apt-key add - + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /" \ + > /etc/apt/sources.list.d/cuda.list + apt-get update apt-get install -y --no-install-recommends \ - cuda-minimal-build-11-4 libcudnn8-dev=8.2.4.15-1+cuda11.4 libcudnn8=8.2.4.15-1+cuda11.4 \ - libcublas-dev-11-4; \ - # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large. - rm /usr/local/cuda/lib64/libcublasLt_static.a /usr/local/cuda/lib64/libcublas_static.a \ - /usr/lib/x86_64-linux-gnu/libcudnn*.a; \ - ln --symbolic /usr/bin/cmake /usr/bin/cmake3; + cuda-minimal-build-12-9 libcudnn9-dev-cuda-12 libcublas-dev-12-9 -COPY --from=openmpf_build /opt/opencv-4.9.0 /opt/opencv-4.9.0 + # OpenCV does not use the statically compiled CUDA libraries except for libcudart and they are relatively large. + find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete + ln --symbolic /usr/bin/cmake /usr/bin/cmake3 +eot + + +COPY --from=openmpf_build /opt/opencv-4.13.0 /opt/opencv-4.13.0 COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ @@ -71,19 +73,21 @@ RUN mkdir /home/mpf; echo 'find_package(mpfCMakeHelpers REQUIRED)' > /home/mpf/C COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/include /home/mpf/mpf-sdk-install/include COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/lib /home/mpf/mpf-sdk-install/lib -RUN echo '/opt/opencv-4.9.0/lib' > /etc/ld.so.conf.d/opencv.conf; \ - echo '/home/mpf/mpf-sdk-install/lib' > /etc/ld.so.conf.d/mpf.conf; \ +RUN < /etc/ld.so.conf.d/opencv.conf + echo '/home/mpf/mpf-sdk-install/lib' > /etc/ld.so.conf.d/mpf.conf ldconfig +eot COPY cpp_component_build/scripts /scripts -ENV PATH /scripts:$PATH +ENV PATH=/scripts:$PATH -ENV CMAKE_PREFIX_PATH /home/mpf/mpf-sdk-install/lib/cmake +ENV CMAKE_PREFIX_PATH=/home/mpf/mpf-sdk-install/lib/cmake -ENV BUILD_DIR /home/mpf/component_build +ENV BUILD_DIR=/home/mpf/component_build -ENV SRC_DIR /home/mpf/component_src +ENV SRC_DIR=/home/mpf/component_src WORKDIR $SRC_DIR diff --git a/components/cpp_executor/Dockerfile b/components/cpp_executor/Dockerfile index a65a3d90..d6c27a25 100644 --- a/components/cpp_executor/Dockerfile +++ b/components/cpp_executor/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.4 +# syntax=docker/dockerfile:1.20 ############################################################################# # NOTICE # @@ -31,7 +31,7 @@ ARG BUILD_TAG=latest FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} AS openmpf_build -FROM ubuntu:20.04 AS common +FROM ubuntu:24.04 AS common SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] @@ -40,35 +40,18 @@ ENV LC_ALL=C.UTF-8 RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/var/lib/apt/lists \ - --mount=type=tmpfs,target=/tmp \ - apt-get update; \ - apt-get upgrade -y; \ - apt-get install --no-install-recommends -y wget ca-certificates gnupg2; \ - # Can't set up cuda repo at the beginning because it requires wget ca-certificates gnupg2 - wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub \ - | apt-key add -; \ - echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64 /" \ - > /etc/apt/sources.list.d/cuda.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends cuda-cudart-11-4; - - -COPY --from=openmpf_build \ - /usr/local/bin/python3.12 /usr/local/bin/python3.12-config \ - /usr/local/bin/pip3.12 /usr/local/bin/pip3 \ - /usr/local/bin/ - -COPY --from=openmpf_build /usr/local/lib/python3.12 /usr/local/lib/python3.12 - -COPY --from=openmpf_build /usr/local/lib/libpython3.12.so.1.0 /usr/local/lib/ - -RUN < /etc/apt/sources.list.d/cuda.list + apt-get update + apt-get install -y --no-install-recommends cuda-cudart-12-9 eot COPY --from=openmpf_build /usr/local/bin/ffmpeg /usr/local/bin/ffprobe /usr/local/bin/ @@ -103,7 +86,7 @@ COPY --from=openmpf_build /build-artifacts/install/bin/amq_detection_component \ COPY --from=openmpf_build /build-artifacts/install/config/Log4cxxConfig.xml $MPF_HOME/config/ COPY --from=openmpf_build /build-artifacts/install/lib/mpf_cpp_sdk.cpython-312-x86_64-linux-gnu.so \ - /usr/local/lib/python3.12/lib-dynload/ + /usr/lib/python3.12/lib-dynload COPY --from=openmpf_build /scripts/* /scripts/ @@ -115,7 +98,6 @@ ENV BUILD_DIR=/home/mpf/component_build ENV PLUGINS_DIR=$MPF_HOME/plugins - COPY docker-entrypoint.sh component-executor.py /scripts/ COPY cli_runner/*.py cli_runner/Log4cxxConfig.xml /scripts/cli_runner/ diff --git a/components/java_component_build/Dockerfile b/components/java_component_build/Dockerfile index 52cf3143..e94c8e5b 100644 --- a/components/java_component_build/Dockerfile +++ b/components/java_component_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.4 +# syntax=docker/dockerfile:1.20 ############################################################################# # NOTICE # @@ -31,7 +31,7 @@ ARG BUILD_TAG=latest FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} AS openmpf_build -FROM ubuntu:20.04 +FROM ubuntu:24.04 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] @@ -40,11 +40,12 @@ ENV LC_ALL=C.UTF-8 RUN --mount=type=tmpfs,target=/var/cache/apt \ --mount=type=tmpfs,target=/var/lib/apt/lists \ - --mount=type=tmpfs,target=/tmp \ - apt-get update; \ - apt-get upgrade -y; \ + --mount=type=tmpfs,target=/tmp < /etc/apt/sources.list.d/cuda.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends cuda-minimal-build-11-4 libcufft-dev-11-4 \ - libnpp-dev-11-4 libcudnn8-dev=8.2.4.15-1+cuda11.4 libcudnn8=8.2.4.15-1+cuda11.4 \ - libcublas-dev-11-4; \ - # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large. - find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete; \ - rm /usr/lib/x86_64-linux-gnu/libcudnn*.a; \ - ln --symbolic /usr/include/x86_64-linux-gnu/openblas-pthread/cblas.h /usr/include/cblas.h; \ - ln --symbolic /usr/bin/cmake /usr/bin/cmake3; + libavutil-dev libswscale-dev libharfbuzz-dev libfreetype-dev \ + libde265-dev git nasm ninja-build pkg-config + + # Cannot set up cuda repo at the beginning because it requires wget ca-certificates gnupg2 + wget -O- https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/3bf863cc.pub \ + | apt-key add - + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64 /" \ + > /etc/apt/sources.list.d/cuda.list + apt-get update + apt-get install -y --no-install-recommends cuda-minimal-build-12-9 libcufft-dev-12-9 \ + libnpp-dev-12-9 libcublas-dev-12-9 libcudnn9-dev-cuda-12 + + # OpenCV does not use the statically compiled CUDA libraries except for libcudart and they are relatively large. + find /usr/local/cuda/lib64/ -name '*.a' -not -name libcudart_static.a -delete + ln --symbolic /usr/include/x86_64-linux-gnu/openblas-pthread/cblas.h /usr/include/cblas.h + ln --symbolic /usr/bin/cmake /usr/bin/cmake3 +eot FROM apt_install AS ocv_install -RUN --mount=type=tmpfs,target=/tmp \ - mkdir /tmp/opencv-contrib; \ - wget -O- 'https://github.com/opencv/opencv_contrib/archive/4.9.0.tar.gz' \ - | tar --extract --gzip --directory /tmp/opencv-contrib; \ - mkdir /tmp/opencv; \ - cd /tmp/opencv; \ - wget -O- 'https://github.com/opencv/opencv/archive/4.9.0.tar.gz' \ - | tar --extract --gzip; \ - cd opencv-4.9.0; \ - mkdir build; \ - cd build; \ - export OpenBLAS_HOME=/usr/lib/x86_64-linux-gnu/openblas-pthread; \ - cmake -DCMAKE_INSTALL_PREFIX:PATH='/opt/opencv-4.9.0' \ +RUN --mount=type=tmpfs,target=/tmp <