From 758e68581ca9e77a0fdb47ed19acbca397d2316d Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 30 Dec 2025 07:36:55 -0800 Subject: [PATCH 01/18] Initial Docker commit --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..52913a4d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,2 @@ +FROM ubuntu:questing + From 30b3a3a674bd77f0f8a7d44a1f64902b51af6f4c Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 30 Dec 2025 07:37:53 -0800 Subject: [PATCH 02/18] Setup initial dependencies --- .dockerignore | 2 ++ Dockerfile | 11 +++++++++++ 2 files changed, 13 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b1d8a0f6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.idea +external_dependencies \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 52913a4d..47ecefa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,13 @@ FROM ubuntu:questing +ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC + +RUN apt-get update && \ + apt-get install -y --no-install-recommends make wget tzdata git \ + python3 python3-pip python3-venv \ + apt-get clean + +COPY README.md install_gridpack.sh install_gridpack_deps.sh /app/ +COPY docs /app/docs +COPY python /app/python +COPY src /app/src From afa4f28c451622dd9faf3017e5d295b038c6283d Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 30 Dec 2025 07:38:28 -0800 Subject: [PATCH 03/18] Install OpenMPI --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 47ecefa7..b9c689a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC RUN apt-get update && \ apt-get install -y --no-install-recommends make wget tzdata git \ python3 python3-pip python3-venv \ + build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean COPY README.md install_gridpack.sh install_gridpack_deps.sh /app/ From 91aad9694fb43d41cc8b285932c563fc1d5d6864 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 30 Dec 2025 07:38:45 -0800 Subject: [PATCH 04/18] Compile and install Boost --- Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Dockerfile b/Dockerfile index b9c689a0..a995d9dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:questing +ARG boost_version=1.81.0 ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC RUN apt-get update && \ @@ -12,3 +13,14 @@ COPY README.md install_gridpack.sh install_gridpack_deps.sh /app/ COPY docs /app/docs COPY python /app/python COPY src /app/src + +# Compile/Install Boost +WORKDIR /deps +RUN wget "https://github.com/boostorg/boost/releases/download/boost-${boost_version}/boost-${boost_version}.tar.gz" +RUN tar -xf "boost-${boost_version}.tar.gz" +WORKDIR "/deps/boost-${boost_version}" +RUN ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system +RUN echo 'using mpi : mpicxx ; ' >> project-config.jam +RUN ./b2 -a -d+2 link="shared" stage +RUN ./b2 -a -d+2 link="shared" install + From fa15ff772a73d3c7010c9f5cd7f2e3df54ca7ad0 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 30 Dec 2025 07:38:58 -0800 Subject: [PATCH 05/18] Compile and install GA --- Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Dockerfile b/Dockerfile index a995d9dd..06975079 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:questing ARG boost_version=1.81.0 +ARG ga_version=5.9.1 ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC RUN apt-get update && \ @@ -24,3 +25,18 @@ RUN echo 'using mpi : mpicxx ; ' >> project-config.jam RUN ./b2 -a -d+2 link="shared" stage RUN ./b2 -a -d+2 link="shared" install +# Compile/Install GA (Global Arrays) +WORKDIR /deps +RUN wget "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" +RUN tar -xf "ga-${ga_version}.tar.gz" +WORKDIR "/deps/ga-${ga_version}" +RUN ./configure --with-mpi-ts --disable-f77 \ + --without-blas --without-lapack --without-scalapack \ + --enable-cxx --enable-i4 \ + --prefix=${PWD}/install_for_gridpack \ + CFLAGS="-Wno-implicit-function-declaration -Wno-incompatible-pointer-types -Wno-old-style-definition" \ + CXXFLAGS="-Wno-incompatible-pointer-types" \ + --enable-shared=yes --enable-static=no +RUN make -j 10 install +WORKDIR /deps + From 9aaa640f974e2d8b6d26c20e37a08fd196ff6656 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Wed, 31 Dec 2025 14:47:59 -0800 Subject: [PATCH 06/18] Compile and install PETSc --- Dockerfile | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06975079..550fcb47 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,14 @@ FROM ubuntu:questing ARG boost_version=1.81.0 ARG ga_version=5.9.1 -ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC +ARG petsc_version=3.24.2 + +ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC GNUMAKEFLAGS=--no-print-directory +ENV PETSC_DIR=/deps/petsc PETSC_ARCH=build-dir +ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 RUN apt-get update && \ - apt-get install -y --no-install-recommends make wget tzdata git \ + apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran \ python3 python3-pip python3-venv \ build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean @@ -40,3 +44,25 @@ RUN ./configure --with-mpi-ts --disable-f77 \ RUN make -j 10 install WORKDIR /deps +# Compile/Install PETSc +RUN git clone https://gitlab.com/petsc/petsc.git +WORKDIR /deps/petsc +RUN git checkout "tags/v${petsc_version}" -b "v${petsc_version}" +RUN ./configure \ + --prefix=${PWD}/install_for_gridpack \ + --scalar-type=real \ + --with-fortran-bindings=0 \ + --download-superlu_dist \ + --download-metis \ + --download-parmetis \ + --download-suitesparse \ + --download-f2cblaslapack \ + --download-scalapack \ + --download-mumps \ + --download-cmake=0 \ + --with-sowing=0 \ + --with-debugging=0 \ + --with-shared-libraries=1 +RUN make all +RUN make install +RUN make PETSC_DIR=/deps/petsc/install_for_gridpack PETSC_ARCH="" check From 483659dfed49cd479d4207c7d29aa1d62c82b5a4 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Wed, 31 Dec 2025 14:55:58 -0800 Subject: [PATCH 07/18] Finish dependency installation by adding additional variables and setting the workdir back to GRIDPACK root --- Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 550fcb47..12e98e57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,19 @@ ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC GNUMAKEFLAGS=--no-print-directory ENV PETSC_DIR=/deps/petsc PETSC_ARCH=build-dir ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 +# GridPACK dependency environment variables +ENV GRIDPACK_ROOT_DIR=/app +ENV GP_EXT_DEPS=/deps +ENV LD_LIBRARY_PATH=/deps/boost-${boost_version}/install_for_gridpack/lib:/deps/ga-${ga_version}/install_for_gridpack/lib:/deps/petsc/install_for_gridpack/lib +ENV DYLD_LIBRARY_PATH=/deps/boost-${boost_version}/install_for_gridpack/lib:/deps/ga-${ga_version}/install_for_gridpack/lib:/deps/petsc/install_for_gridpack/lib + RUN apt-get update && \ apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran \ python3 python3-pip python3-venv \ build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean -COPY README.md install_gridpack.sh install_gridpack_deps.sh /app/ +COPY README.md install_gridpack.sh /app/ COPY docs /app/docs COPY python /app/python COPY src /app/src @@ -66,3 +72,5 @@ RUN ./configure \ RUN make all RUN make install RUN make PETSC_DIR=/deps/petsc/install_for_gridpack PETSC_ARCH="" check + +WORKDIR /app \ No newline at end of file From a9124862086c648de3eb3c83b1d49e671bbbf9a8 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Fri, 2 Jan 2026 12:11:23 -0800 Subject: [PATCH 08/18] Update env vars and their usage --- Dockerfile | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 12e98e57..9a5d1996 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,14 +5,25 @@ ARG ga_version=5.9.1 ARG petsc_version=3.24.2 ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC GNUMAKEFLAGS=--no-print-directory -ENV PETSC_DIR=/deps/petsc PETSC_ARCH=build-dir ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 # GridPACK dependency environment variables -ENV GRIDPACK_ROOT_DIR=/app -ENV GP_EXT_DEPS=/deps -ENV LD_LIBRARY_PATH=/deps/boost-${boost_version}/install_for_gridpack/lib:/deps/ga-${ga_version}/install_for_gridpack/lib:/deps/petsc/install_for_gridpack/lib -ENV DYLD_LIBRARY_PATH=/deps/boost-${boost_version}/install_for_gridpack/lib:/deps/ga-${ga_version}/install_for_gridpack/lib:/deps/petsc/install_for_gridpack/lib +ENV GRIDPACK_ROOT_DIR=/app GP_EXT_DEPS=/deps +ENV GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build +ENV GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} + +ENV boost_dir=${GP_EXT_DEPS}/boost-${boost_version} \ + ga_dir=${GP_EXT_DEPS}/ga-${ga_version} \ + petsc_dir=${GP_EXT_DEPS}/petsc + +ENV boost_gp_dir=${boost_dir}/install_for_gridpack \ + ga_gp_dir=${ga_dir}/install_for_gridpack \ + petsc_gp_dir=${petsc_dir}/install_for_gridpack + +ENV PETSC_DIR=${petsc_dir} PETSC_ARCH=build-dir + +ENV LD_LIBRARY_PATH=${boost_gp_dir}/lib:${ga_gp_dir}/lib:${petsc_gp_dir}/lib +ENV DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH} RUN apt-get update && \ apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran \ @@ -20,39 +31,39 @@ RUN apt-get update && \ build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean -COPY README.md install_gridpack.sh /app/ -COPY docs /app/docs -COPY python /app/python -COPY src /app/src +COPY README.md install_gridpack.sh ${GRIDPACK_ROOT_DIR}/ +COPY docs ${GRIDPACK_ROOT_DIR}/docs +COPY python ${GRIDPACK_ROOT_DIR}/python +COPY src ${GRIDPACK_ROOT_DIR}/src # Compile/Install Boost -WORKDIR /deps +WORKDIR ${GP_EXT_DEPS} RUN wget "https://github.com/boostorg/boost/releases/download/boost-${boost_version}/boost-${boost_version}.tar.gz" RUN tar -xf "boost-${boost_version}.tar.gz" -WORKDIR "/deps/boost-${boost_version}" +WORKDIR ${boost_dir} RUN ./bootstrap.sh --prefix=install_for_gridpack --with-libraries=mpi,serialization,random,filesystem,system RUN echo 'using mpi : mpicxx ; ' >> project-config.jam RUN ./b2 -a -d+2 link="shared" stage RUN ./b2 -a -d+2 link="shared" install # Compile/Install GA (Global Arrays) -WORKDIR /deps +WORKDIR ${GP_EXT_DEPS} RUN wget "https://github.com/GlobalArrays/ga/releases/download/v${ga_version}/ga-${ga_version}.tar.gz" RUN tar -xf "ga-${ga_version}.tar.gz" -WORKDIR "/deps/ga-${ga_version}" +WORKDIR ${ga_dir} RUN ./configure --with-mpi-ts --disable-f77 \ --without-blas --without-lapack --without-scalapack \ --enable-cxx --enable-i4 \ - --prefix=${PWD}/install_for_gridpack \ + --prefix=${ga_gp_dir} \ CFLAGS="-Wno-implicit-function-declaration -Wno-incompatible-pointer-types -Wno-old-style-definition" \ CXXFLAGS="-Wno-incompatible-pointer-types" \ --enable-shared=yes --enable-static=no RUN make -j 10 install -WORKDIR /deps +WORKDIR ${GP_EXT_DEPS} # Compile/Install PETSc RUN git clone https://gitlab.com/petsc/petsc.git -WORKDIR /deps/petsc +WORKDIR ${petsc_dir} RUN git checkout "tags/v${petsc_version}" -b "v${petsc_version}" RUN ./configure \ --prefix=${PWD}/install_for_gridpack \ @@ -71,6 +82,6 @@ RUN ./configure \ --with-shared-libraries=1 RUN make all RUN make install -RUN make PETSC_DIR=/deps/petsc/install_for_gridpack PETSC_ARCH="" check +RUN make PETSC_DIR=${petsc_gp_dir} PETSC_ARCH="" check WORKDIR /app \ No newline at end of file From 44b6532f2845e90a29d58668d8b558888e3f2be4 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Fri, 2 Jan 2026 12:12:36 -0800 Subject: [PATCH 09/18] Set `python` to be same as `python3` --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a5d1996..45c8c95f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,6 @@ ARG petsc_version=3.24.2 ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC GNUMAKEFLAGS=--no-print-directory ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 -# GridPACK dependency environment variables ENV GRIDPACK_ROOT_DIR=/app GP_EXT_DEPS=/deps ENV GRIDPACK_INSTALL_DIR=${GRIDPACK_ROOT_DIR}/src/install GRIDPACK_BUILD_DIR=${GRIDPACK_ROOT_DIR}/src/build ENV GRIDPACK_DIR=${GRIDPACK_INSTALL_DIR} @@ -27,8 +26,8 @@ ENV DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH} RUN apt-get update && \ apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran \ - python3 python3-pip python3-venv \ build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ + python3 python3-pip python3-venv python-is-python3 \ apt-get clean COPY README.md install_gridpack.sh ${GRIDPACK_ROOT_DIR}/ From 7a53fc780922fad51f57d2004a3935fb6fedc46f Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Fri, 2 Jan 2026 12:49:31 -0800 Subject: [PATCH 10/18] Initial steps toward gridpack installation --- .dockerignore | 5 ++++- Dockerfile | 30 ++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index b1d8a0f6..6138cbcd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,5 @@ .idea -external_dependencies \ No newline at end of file +external_dependencies +/python/build +/python/dist +/python/gridpack_hadrec.egg-info \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 45c8c95f..667ef381 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,12 +25,13 @@ ENV LD_LIBRARY_PATH=${boost_gp_dir}/lib:${ga_gp_dir}/lib:${petsc_gp_dir}/lib ENV DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH} RUN apt-get update && \ - apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran \ - build-essential openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ + apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran build-essential pkg-config \ python3 python3-pip python3-venv python-is-python3 \ + openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean -COPY README.md install_gridpack.sh ${GRIDPACK_ROOT_DIR}/ +COPY README.md .gitignore .gitmodules ${GRIDPACK_ROOT_DIR}/ +COPY .git ${GRIDPACK_ROOT_DIR}/.git COPY docs ${GRIDPACK_ROOT_DIR}/docs COPY python ${GRIDPACK_ROOT_DIR}/python COPY src ${GRIDPACK_ROOT_DIR}/src @@ -83,4 +84,25 @@ RUN make all RUN make install RUN make PETSC_DIR=${petsc_gp_dir} PETSC_ARCH="" check -WORKDIR /app \ No newline at end of file +WORKDIR ${GRIDPACK_BUILD_DIR} +RUN cmake -Wdev -D GA_DIR:STRING=${ga_gp_dir} \ + -D Boost_ROOT:STRING=${boost_gp_dir} \ + -D Boost_DIR:string=${boost_gp_dir}/lib/cmake/Boost-${boost_version} \ + -D PETSC_DIR:PATH=${petsc_gp_dir} \ + -D MPI_CXX_COMPILER:STRING='mpicxx' \ + -D MPI_C_COMPILER:STRING='mpicc' \ + -D MPIEXEC:STRING='mpiexec' \ + -D MPIEXEC_MAX_NUMPROCS:STRING=2 \ + -D GRIDPACK_TEST_TIMEOUT:STRING=120 \ + -D ENABLE_ENVIRONMENT_FROM_COMM:BOOL=YES \ + -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ + -D CMAKE_BUILD_TYPE:STRING=Debug \ + -D BUILD_SHARED_LIBS=true \ + .. +RUN make install + +WORKDIR ${GRIDPACK_ROOT_DIR} +RUN git submodule update --init +WORKDIR ${GRIDPACK_ROOT_DIR}/python +#RUN pip install --no-deps --upgrade --prefix=${GRIDPACK_INSTALL_DIR} . +#ENV \ No newline at end of file From 01ac145d455099556741f02a865392ecc2e347a5 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 6 Jan 2026 11:24:09 -0800 Subject: [PATCH 11/18] Update PyBind11 to fix issues in Dockerfile --- .gitmodules | 1 + python/pybind11 | 2 +- python/pyproject.toml | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 python/pyproject.toml diff --git a/.gitmodules b/.gitmodules index 90489018..728215eb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,4 @@ [submodule "python/pybind11"] path = python/pybind11 url = https://github.com/pybind/pybind11.git + branch = master diff --git a/python/pybind11 b/python/pybind11 index 80d45248..c761608a 160000 --- a/python/pybind11 +++ b/python/pybind11 @@ -1 +1 @@ -Subproject commit 80d452484c5409444b0ec19383faa84bb7a4d351 +Subproject commit c761608a22b956e51218f4360661b0135dca3b7f diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..c66b9378 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,4 @@ +[build-system] +requires = ["setuptools>=40.8.0", "wheel", "mpi4py"] +build-backend = "setuptools.build_meta" + From 20f8658ae28716d8a8b31a05931bb5b3c915e780 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 6 Jan 2026 13:08:21 -0800 Subject: [PATCH 12/18] Finish compiling and installing GridPACK --- Dockerfile | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 667ef381..6497e421 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ FROM ubuntu:questing +# Configure dependency versions ARG boost_version=1.81.0 ARG ga_version=5.9.1 ARG petsc_version=3.24.2 +# Setup environment variables used throughout installation ENV DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC GNUMAKEFLAGS=--no-print-directory ENV OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 @@ -24,18 +26,13 @@ ENV PETSC_DIR=${petsc_dir} PETSC_ARCH=build-dir ENV LD_LIBRARY_PATH=${boost_gp_dir}/lib:${ga_gp_dir}/lib:${petsc_gp_dir}/lib ENV DYLD_LIBRARY_PATH=${LD_LIBRARY_PATH} +# Install required system packages RUN apt-get update && \ apt-get install -y --no-install-recommends cmake make wget tzdata git gfortran build-essential pkg-config \ - python3 python3-pip python3-venv python-is-python3 \ + python3 python3-pip python3-venv python3-dev python-is-python3 \ openmpi-bin openmpi-common openmpi-doc libopenmpi-dev && \ apt-get clean -COPY README.md .gitignore .gitmodules ${GRIDPACK_ROOT_DIR}/ -COPY .git ${GRIDPACK_ROOT_DIR}/.git -COPY docs ${GRIDPACK_ROOT_DIR}/docs -COPY python ${GRIDPACK_ROOT_DIR}/python -COPY src ${GRIDPACK_ROOT_DIR}/src - # Compile/Install Boost WORKDIR ${GP_EXT_DEPS} RUN wget "https://github.com/boostorg/boost/releases/download/boost-${boost_version}/boost-${boost_version}.tar.gz" @@ -84,6 +81,14 @@ RUN make all RUN make install RUN make PETSC_DIR=${petsc_gp_dir} PETSC_ARCH="" check +# Copy in GridPACK source code from repository +COPY README.md .gitignore .gitmodules ${GRIDPACK_ROOT_DIR}/ +COPY .git ${GRIDPACK_ROOT_DIR}/.git +COPY docs ${GRIDPACK_ROOT_DIR}/docs +COPY python ${GRIDPACK_ROOT_DIR}/python +COPY src ${GRIDPACK_ROOT_DIR}/src + +# Build GridPACK WORKDIR ${GRIDPACK_BUILD_DIR} RUN cmake -Wdev -D GA_DIR:STRING=${ga_gp_dir} \ -D Boost_ROOT:STRING=${boost_gp_dir} \ @@ -101,8 +106,19 @@ RUN cmake -Wdev -D GA_DIR:STRING=${ga_gp_dir} \ .. RUN make install +# Install Python bindings WORKDIR ${GRIDPACK_ROOT_DIR} RUN git submodule update --init +RUN pip config --global set global.break-system-packages true WORKDIR ${GRIDPACK_ROOT_DIR}/python -#RUN pip install --no-deps --upgrade --prefix=${GRIDPACK_INSTALL_DIR} . -#ENV \ No newline at end of file +RUN pip install --upgrade --prefix=${GRIDPACK_INSTALL_DIR} . + +# Configure Python module search path using .pth file (no environment variables needed) +# Python automatically reads .pth files from its site-packages directories +RUN pyvnum=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') && \ + system_site_packages=$(python3 -c 'import site; print(site.getsitepackages()[0])') && \ + echo "${GRIDPACK_INSTALL_DIR}/lib/python${pyvnum}/site-packages" > ${system_site_packages}/gridpack.pth && \ + echo "${GRIDPACK_INSTALL_DIR}/local/lib/python${pyvnum}/dist-packages" >> ${system_site_packages}/gridpack.pth && \ + echo "Configured Python ${pyvnum} module search path via ${system_site_packages}/gridpack.pth" + +WORKDIR ${GRIDPACK_ROOT_DIR} From 8ea4afd2b942d54bb738bf6db14d3add9817b778 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Thu, 8 Jan 2026 10:27:56 -0800 Subject: [PATCH 13/18] Fix tests by disabling GLIBC assertions --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 6497e421..9b2b1fd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,6 +103,7 @@ RUN cmake -Wdev -D GA_DIR:STRING=${ga_gp_dir} \ -D CMAKE_INSTALL_PREFIX:PATH=${GRIDPACK_INSTALL_DIR} \ -D CMAKE_BUILD_TYPE:STRING=Debug \ -D BUILD_SHARED_LIBS=true \ + -D CMAKE_CXX_FLAGS_DEBUG:STRING="-D_GLIBCXX_NO_ASSERTIONS" \ .. RUN make install From 7b008448352d853d3448657a22031126aa71808d Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Thu, 8 Jan 2026 10:58:53 -0800 Subject: [PATCH 14/18] Add the `install/bin` and `install/local/bin` folders to the $PATH --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 9b2b1fd4..4c21ed9f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,3 +123,4 @@ RUN pyvnum=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.versi echo "Configured Python ${pyvnum} module search path via ${system_site_packages}/gridpack.pth" WORKDIR ${GRIDPACK_ROOT_DIR} +ENV PATH=${GRIDPACK_INSTALL_DIR}/bin:${GRIDPACK_INSTALL_DIR}/local/bin:${PATH} From aba9d5b5df0f7299b729e0c5798bf915117cf17c Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Wed, 28 Jan 2026 13:56:18 -0800 Subject: [PATCH 15/18] Don't run `make check` for PETSc as this has limitations that cause problems on emulated hardware --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 4c21ed9f..97d5ac32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ RUN ./configure \ --with-shared-libraries=1 RUN make all RUN make install -RUN make PETSC_DIR=${petsc_gp_dir} PETSC_ARCH="" check +#RUN make PETSC_DIR=${petsc_gp_dir} PETSC_ARCH="" check # Copy in GridPACK source code from repository COPY README.md .gitignore .gitmodules ${GRIDPACK_ROOT_DIR}/ From 627b40376e2a71e31bd21a56b87661ff8d10c7f9 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Mon, 2 Feb 2026 12:47:12 -0800 Subject: [PATCH 16/18] Change the default work dir to `/app/workspace` --- Dockerfile | 2 +- Dockerfile.test | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) delete mode 100644 Dockerfile.test diff --git a/Dockerfile b/Dockerfile index 97d5ac32..bb29282d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -122,5 +122,5 @@ RUN pyvnum=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.versi echo "${GRIDPACK_INSTALL_DIR}/local/lib/python${pyvnum}/dist-packages" >> ${system_site_packages}/gridpack.pth && \ echo "Configured Python ${pyvnum} module search path via ${system_site_packages}/gridpack.pth" -WORKDIR ${GRIDPACK_ROOT_DIR} +WORKDIR ${GRIDPACK_ROOT_DIR}/workspace ENV PATH=${GRIDPACK_INSTALL_DIR}/bin:${GRIDPACK_INSTALL_DIR}/local/bin:${PATH} diff --git a/Dockerfile.test b/Dockerfile.test deleted file mode 100644 index b577046e..00000000 --- a/Dockerfile.test +++ /dev/null @@ -1,10 +0,0 @@ -FROM ubuntu:24.04 - -# Install minimal dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl ca-certificates && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Simple verification command -CMD ["echo", "Multi-arch Docker build pipeline test successful"] From d54ce4c0a55ff8644a9a1dcd35046d5125033aca Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 3 Feb 2026 13:18:55 -0800 Subject: [PATCH 17/18] Update documentation to include Docker --- README.md | 20 ++ docs/markdown/CI-CD.md | 203 ++++++++++++++++++ ...n9-Examples.rst => Section10-Examples.rst} | 0 docs/user_manual/sphinx/Section2-Docker.rst | 175 +++++++++++++++ ...eBuild.rst => Section3-ConfigureBuild.rst} | 0 ...ons.rst => Section4-BuildApplications.rst} | 0 ...s.rst => Section5-FrameworkComponents.rst} | 0 ...st => Section6-DevelopingApplications.rst} | 0 ...rst => Section7-AdvancedFunctionality.rst} | 0 ...> Section8-GeneralizedMatVecInterface.rst} | 0 ...es.rst => Section9-ApplicationModules.rst} | 0 docs/user_manual/sphinx/index.rst | 17 +- 12 files changed, 407 insertions(+), 8 deletions(-) create mode 100644 docs/markdown/CI-CD.md rename docs/user_manual/sphinx/{Section9-Examples.rst => Section10-Examples.rst} (100%) create mode 100644 docs/user_manual/sphinx/Section2-Docker.rst rename docs/user_manual/sphinx/{Section2-ConfigureBuild.rst => Section3-ConfigureBuild.rst} (100%) rename docs/user_manual/sphinx/{Section3-BuildApplications.rst => Section4-BuildApplications.rst} (100%) rename docs/user_manual/sphinx/{Section4-FrameworkComponents.rst => Section5-FrameworkComponents.rst} (100%) rename docs/user_manual/sphinx/{Section5-DevelopingApplications.rst => Section6-DevelopingApplications.rst} (100%) rename docs/user_manual/sphinx/{Section6-AdvancedFunctionality.rst => Section7-AdvancedFunctionality.rst} (100%) rename docs/user_manual/sphinx/{Section7-GeneralizedMatVecInterface.rst => Section8-GeneralizedMatVecInterface.rst} (100%) rename docs/user_manual/sphinx/{Section8-ApplicationModules.rst => Section9-ApplicationModules.rst} (100%) diff --git a/README.md b/README.md index 6eaf4368..c39e4955 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,32 @@ In addition, GridPACK is also a framework to simplify the development of new app ## Installation See the [instructions](docs/markdown/BASIC_INSTALL.md) for installing GridPACK, prerequisite software, and installation notes for different platforms. Formal releases are available [here](https://github.com/GridOPTICS/GridPACK/releases). +### Using Docker (Recommended) + +GridPACK is available as a multi-architecture Docker image with all dependencies pre-installed: + +```bash +# Pull the image +docker pull pnnl/gridpack:latest + +# Run with your files (container starts in /app/workspace) +docker run -it --rm -v $(pwd):/app/workspace pnnl/gridpack:latest bash +``` + +The Docker image supports both AMD64 and ARM64 architectures. See the [Docker usage guide](https://gridpack.readthedocs.io/en/latest/Section2-Docker.html) for more examples. + +### Building from Source + +See the [installation instructions](docs/markdown/BASIC_INSTALL.md) for building GridPACK from source, prerequisite software, and platform-specific installation notes. Formal releases are available [here](https://github.com/GridOPTICS/GridPACK/releases). + ## Usage See the [user manual](https://gridpack.readthedocs.io/en/latest/index.html) for a deep dive on GridPACK internals and/or refer to the [tutorials](docs/markdown/TUTORIALS.md) for more info. - Quick Guide (To do) ## Documentation +- [Docker Usage Guide](https://gridpack.readthedocs.io/en/latest/Section2-Docker.html) +- [CI/CD Pipeline](docs/markdown/CI-CD.md) - [User manual](https://gridpack.readthedocs.io/en/latest/index.html) - [Tutorials](docs/markdown/TUTORIALS.md) - [FAQS](docs/markdown/FAQS.md) diff --git a/docs/markdown/CI-CD.md b/docs/markdown/CI-CD.md new file mode 100644 index 00000000..5536a349 --- /dev/null +++ b/docs/markdown/CI-CD.md @@ -0,0 +1,203 @@ +# CI/CD Pipeline Documentation + +## Overview + +GridPACK uses GitHub Actions to build multi-architecture Docker images that are automatically published to Docker Hub at `pnnl/gridpack`. + +## Pipeline Architecture + +The pipeline uses native ARM64 and AMD64 runners for optimal build performance: + +``` +prepare-metadata → build-native (amd64 + arm64) → merge-manifest → test-native (amd64 + arm64) + ↓ parallel ↓ ↓ ↓ parallel ↓ +``` + +### Jobs + +1. **prepare-metadata**: Generates Docker tags and labels +2. **build-native**: Builds images natively on AMD64 and ARM64 runners in parallel +3. **merge-manifest**: Stitches architecture-specific images into a multi-arch manifest +4. **test-native**: Runs ctest smoke tests on both architectures in parallel + +### Performance + +- **Native ARM64 builds**: 3-10x faster than QEMU emulation +- **Parallel execution**: Both architectures build simultaneously +- **Total build time**: ~20-35 minutes (vs 45-90 minutes with QEMU) + +## Triggers + +### Automatic: Release Published + +When a GitHub release is created, the pipeline automatically builds and publishes: + +```bash +gh release create v3.3.0 --title "Release 3.3.0" --notes "Release notes" +``` + +Generated tags: +- `pnnl/gridpack:3.3.0` +- `pnnl/gridpack:3.3` +- `pnnl/gridpack:latest` + +### Manual: Workflow Dispatch + +Trigger builds manually from the GitHub Actions UI or CLI: + +**Via GitHub UI:** +1. Go to: Actions → Docker Multi-Architecture Build and Publish +2. Click "Run workflow" +3. Configure options: + - **tag**: Custom Docker tag (default: `dev`) + - **skip_tests**: Skip ctest smoke tests (default: `false`) + - **dockerfile**: Dockerfile path (default: `./Dockerfile`) +4. Click "Run workflow" + +**Via GitHub CLI:** + +```bash +# Basic run with defaults +gh workflow run docker-build.yml + +# With custom tag +gh workflow run docker-build.yml -f tag=test-build + +# Skip tests for faster iteration +gh workflow run docker-build.yml -f skip_tests=true + +# Use custom Dockerfile +gh workflow run docker-build.yml -f dockerfile=./Dockerfile.custom +``` + +## Docker Image + +### Repository +- **Location**: `pnnl/gridpack` +- **Registry**: Docker Hub + +### Supported Architectures +- `linux/amd64` (Intel/AMD) +- `linux/arm64` (ARM64/Apple Silicon/AWS Graviton) + +### Usage + +See [Docker Usage Guide](./DOCKER.md) for detailed examples. + +## Testing + +The pipeline runs ctest smoke tests on both architectures after building. Key points: + +- **Non-blocking**: Test failures do NOT prevent image publication (`continue-on-error: true`) +- **Expected pass rate**: ~94% +- **Can be skipped**: Use `skip_tests=true` for faster iteration + +## Secrets Required + +The pipeline requires two GitHub repository secrets: + +| Secret | Description | +|--------|-------------| +| `DOCKERHUB_USERNAME` | Docker Hub username for authentication | +| `DOCKERHUB_TOKEN` | Docker Hub access token (not password!) | + +### Setting Up Secrets + +1. **Generate Docker Hub Access Token:** + - Go to: https://hub.docker.com → Account Settings → Security + - Create token: Name: `gridpack-ci`, Permissions: Read, Write, Delete + - Copy the token (shown only once) + +2. **Add to GitHub:** + - Go to: Repository Settings → Secrets and variables → Actions + - Add both secrets with the values above + +## Monitoring + +### View Workflow Runs + +```bash +# List recent runs +gh run list --workflow=docker-build.yml --limit 5 + +# Watch a specific run +gh run watch + +# View detailed logs +gh run view --log +``` + +### Inspect Published Image + +```bash +# View manifest with both architectures +docker buildx imagetools inspect pnnl/gridpack:latest + +# Expected output shows: +# - linux/amd64 manifest +# - linux/arm64 manifest +``` + +## Caching + +The pipeline uses GitHub Actions cache to speed up builds: + +- Separate cache per architecture (`build-amd64`, `build-arm64`) +- Automatically managed by GitHub +- Improves rebuild times significantly + +## Troubleshooting + +### Build Fails on One Architecture + +The pipeline uses `fail-fast: false`, so if one architecture fails, the other continues. Check job logs for architecture-specific issues. + +### Manifest Merge Fails + +This usually indicates both architecture builds failed. Check that: +- Docker Hub credentials are correct +- Both build jobs completed successfully +- Digest artifacts were uploaded + +### Authentication Issues + +Verify secrets are configured correctly: +```bash +gh secret list +``` + +Should show: +- `DOCKERHUB_USERNAME` +- `DOCKERHUB_TOKEN` + +## Technical Details + +### Push-by-Digest Workflow + +The pipeline uses a digest-based approach: + +1. Each architecture builds and pushes: `pnnl/gridpack@sha256:abc123...` +2. These are content-addressed (by digest), not tagged yet +3. Manifest merge creates tags that reference both digests +4. Docker automatically selects the right image based on client platform + +### Why Native Runners? + +- **Performance**: 3-10x faster ARM64 builds without QEMU +- **Reliability**: No emulation issues or crashes +- **Testing**: Tests run on actual target platforms +- **Cost**: Same GitHub Actions pricing as AMD64 + +## Workflow File + +Location: `.github/workflows/docker-build.yml` + +The workflow definition must be in the default branch to be accessible. Changes to the workflow require merging to the default branch. + +## Future Enhancements + +Potential improvements: +- Additional architectures (RISC-V when runners available) +- Multi-registry support (GitHub Container Registry) +- Build time metrics/reporting +- Automated security scanning per architecture diff --git a/docs/user_manual/sphinx/Section9-Examples.rst b/docs/user_manual/sphinx/Section10-Examples.rst similarity index 100% rename from docs/user_manual/sphinx/Section9-Examples.rst rename to docs/user_manual/sphinx/Section10-Examples.rst diff --git a/docs/user_manual/sphinx/Section2-Docker.rst b/docs/user_manual/sphinx/Section2-Docker.rst new file mode 100644 index 00000000..edc38100 --- /dev/null +++ b/docs/user_manual/sphinx/Section2-Docker.rst @@ -0,0 +1,175 @@ +Using GridPACK with Docker +=========================== + +GridPACK is available as a Docker image with all dependencies pre-installed and ready to use. + +Quick Start +----------- + +Pull the Image +~~~~~~~~~~~~~~ + +.. code-block:: bash + + docker pull pnnl/gridpack:latest + +Run with Your Files +~~~~~~~~~~~~~~~~~~~ + +The container starts in ``/app/workspace`` by default. Mount your current directory to work with your files: + +.. code-block:: bash + + docker run -it --rm -v $(pwd):/app/workspace pnnl/gridpack:latest + +Usage Examples +-------------- + +Interactive Shell +~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Mount your working directory and start an interactive session + docker run -it --rm -v $(pwd):/app/workspace pnnl/gridpack:latest bash + +Run a Specific Command +~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Run a GridPACK application directly + docker run --rm -v $(pwd):/app/workspace pnnl/gridpack:latest \ + powerflow.x input.xml + +Python Scripts +~~~~~~~~~~~~~~ + +.. code-block:: bash + + # Execute a Python script using GridPACK Python bindings + docker run --rm -v $(pwd):/app/workspace pnnl/gridpack:latest \ + python3 my_gridpack_script.py + +Run Tests +~~~~~~~~~ + +.. code-block:: bash + + # Run the GridPACK test suite + docker run --rm pnnl/gridpack:latest bash -c "cd /app/src/build && ctest" + +Working Directory +----------------- + +The container is configured with: + +- **Working directory**: ``/app/workspace`` +- **GridPACK installation**: ``/app/src/install`` +- **Build directory**: ``/app/src/build`` + +Mount your data/scripts to ``/app/workspace`` to access them from within the container. + +Environment Variables +--------------------- + +The following environment variables are pre-configured: + +.. code-block:: bash + + GRIDPACK_DIR=/app/src/install + PATH=/app/src/install/bin:$PATH + LD_LIBRARY_PATH=/deps/boost-1.81.0/install_for_gridpack/lib:/deps/ga-5.9.1/install_for_gridpack/lib:/deps/petsc/install_for_gridpack/lib + +Multi-Architecture Support +--------------------------- + +The Docker image supports both AMD64 and ARM64 architectures. Docker automatically pulls the correct image for your platform. + +To explicitly specify an architecture: + +.. code-block:: bash + + # For AMD64 (Intel/AMD processors) + docker pull --platform linux/amd64 pnnl/gridpack:latest + + # For ARM64 (Apple Silicon, AWS Graviton) + docker pull --platform linux/arm64 pnnl/gridpack:latest + +Available Tags +-------------- + +- ``latest`` - Latest stable release +- ``dev`` - Development builds +- ``vX.Y.Z`` - Specific version releases + +Tips +---- + +- Add ``-it`` for interactive terminal access +- Add ``--rm`` to automatically remove the container after exit +- Use ``-v`` to mount local directories into the container +- Increase memory limits for large simulations: ``--memory=8g`` +- Use multiple cores: Set ``OMPI_NUM_PROCS`` or pass to MPI commands + +Example Workflow +---------------- + +.. code-block:: bash + + # 1. Create a project directory + mkdir my-gridpack-project + cd my-gridpack-project + + # 2. Create your input files + cat > input.xml << EOF + + + + + EOF + + # 3. Run GridPACK in the container + docker run -it --rm -v $(pwd):/app/workspace pnnl/gridpack:latest bash + + # Inside container: + # powerflow.x input.xml + # exit + +Troubleshooting +--------------- + +Permission Issues +~~~~~~~~~~~~~~~~~ + +If you encounter permission issues with mounted volumes: + +.. code-block:: bash + + # Run with your user ID + docker run -it --rm -u $(id -u):$(id -g) -v $(pwd):/app/workspace pnnl/gridpack:latest + +MPI Warnings +~~~~~~~~~~~~ + +The container sets ``OMPI_ALLOW_RUN_AS_ROOT=1`` to allow running MPI as root. This is safe for containerized environments. + +Running Applications with MPI +------------------------------ + +GridPACK applications can be run with MPI inside the Docker container. For example: + +.. code-block:: bash + + # Run with 4 MPI processes + docker run --rm -v $(pwd):/app/workspace pnnl/gridpack:latest \ + mpirun -n 4 powerflow.x input.xml + +For detailed information on MPI execution, application arguments, and input file conventions, +see :doc:`Section3-ConfigureBuild` (Running GridPACK Applications section). + +Building from Source +-------------------- + +If you need to build GridPACK from source instead of using Docker, see :doc:`Section3-ConfigureBuild` +for detailed instructions on using build scripts or manual compilation. diff --git a/docs/user_manual/sphinx/Section2-ConfigureBuild.rst b/docs/user_manual/sphinx/Section3-ConfigureBuild.rst similarity index 100% rename from docs/user_manual/sphinx/Section2-ConfigureBuild.rst rename to docs/user_manual/sphinx/Section3-ConfigureBuild.rst diff --git a/docs/user_manual/sphinx/Section3-BuildApplications.rst b/docs/user_manual/sphinx/Section4-BuildApplications.rst similarity index 100% rename from docs/user_manual/sphinx/Section3-BuildApplications.rst rename to docs/user_manual/sphinx/Section4-BuildApplications.rst diff --git a/docs/user_manual/sphinx/Section4-FrameworkComponents.rst b/docs/user_manual/sphinx/Section5-FrameworkComponents.rst similarity index 100% rename from docs/user_manual/sphinx/Section4-FrameworkComponents.rst rename to docs/user_manual/sphinx/Section5-FrameworkComponents.rst diff --git a/docs/user_manual/sphinx/Section5-DevelopingApplications.rst b/docs/user_manual/sphinx/Section6-DevelopingApplications.rst similarity index 100% rename from docs/user_manual/sphinx/Section5-DevelopingApplications.rst rename to docs/user_manual/sphinx/Section6-DevelopingApplications.rst diff --git a/docs/user_manual/sphinx/Section6-AdvancedFunctionality.rst b/docs/user_manual/sphinx/Section7-AdvancedFunctionality.rst similarity index 100% rename from docs/user_manual/sphinx/Section6-AdvancedFunctionality.rst rename to docs/user_manual/sphinx/Section7-AdvancedFunctionality.rst diff --git a/docs/user_manual/sphinx/Section7-GeneralizedMatVecInterface.rst b/docs/user_manual/sphinx/Section8-GeneralizedMatVecInterface.rst similarity index 100% rename from docs/user_manual/sphinx/Section7-GeneralizedMatVecInterface.rst rename to docs/user_manual/sphinx/Section8-GeneralizedMatVecInterface.rst diff --git a/docs/user_manual/sphinx/Section8-ApplicationModules.rst b/docs/user_manual/sphinx/Section9-ApplicationModules.rst similarity index 100% rename from docs/user_manual/sphinx/Section8-ApplicationModules.rst rename to docs/user_manual/sphinx/Section9-ApplicationModules.rst diff --git a/docs/user_manual/sphinx/index.rst b/docs/user_manual/sphinx/index.rst index bf4cd604..cdd328bf 100644 --- a/docs/user_manual/sphinx/index.rst +++ b/docs/user_manual/sphinx/index.rst @@ -8,14 +8,15 @@ GridPACK documentation Section0-GridPACK Section1-Introduction - Section2-ConfigureBuild - Section3-BuildApplications - Section4-FrameworkComponents - Section5-DevelopingApplications - Section6-AdvancedFunctionality - Section7-GeneralizedMatVecInterface - Section8-ApplicationModules - Section9-Examples + Section2-Docker + Section3-ConfigureBuild + Section4-BuildApplications + Section5-FrameworkComponents + Section6-DevelopingApplications + Section7-AdvancedFunctionality + Section8-GeneralizedMatVecInterface + Section9-ApplicationModules + Section10-Examples .. toctree:: :maxdepth: 1 From ea88ac47ef9d35cfd88040bc6a07ede454af6ff7 Mon Sep 17 00:00:00 2001 From: Thom Williams Date: Tue, 3 Feb 2026 13:23:09 -0800 Subject: [PATCH 18/18] Update pipeline to build on native and deploy to pnnl/gridpack --- .github/workflows/docker-build.yml | 180 +++++++++++++++++++++++------ 1 file changed, 145 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index fb89021d..8938fbde 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -17,43 +17,31 @@ on: required: false type: boolean default: false - platforms: - description: 'Target platforms (comma-separated)' - required: false - default: 'linux/amd64,linux/arm64' dockerfile: description: 'Dockerfile to use' required: false default: './Dockerfile' jobs: - build-and-push: - name: Build and Push Multi-Architecture Docker Images + # Generate metadata once to be shared across all jobs + prepare-metadata: + name: Prepare Build Metadata runs-on: ubuntu-latest + outputs: + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + json: ${{ steps.meta.outputs.json }} + primary-tag: ${{ steps.primary.outputs.tag }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up QEMU for ARM64 emulation - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Extract metadata (tags, labels) id: meta uses: docker/metadata-action@v5 with: - images: ${{ secrets.DOCKERHUB_USERNAME }}/gp + images: pnnl/gridpack tags: | type=ref,event=branch type=semver,pattern={{version}} @@ -61,38 +49,160 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=raw,value=${{ inputs.tag || 'dev' }},enable=${{ github.event_name == 'workflow_dispatch' }} - - name: Build and push Docker images + - name: Extract primary tag + id: primary + run: | + PRIMARY_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n 1) + echo "tag=${PRIMARY_TAG}" >> $GITHUB_OUTPUT + echo "Primary tag: ${PRIMARY_TAG}" + + # Build for each architecture using native runners + build-native: + name: Build ${{ matrix.platform }} on native runner + needs: prepare-metadata + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm64 + arch: arm64 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push by digest + id: build uses: docker/build-push-action@v5 with: context: . file: ${{ inputs.dockerfile || './Dockerfile' }} - platforms: ${{ inputs.platforms || 'linux/amd64,linux/arm64' }} + platforms: ${{ matrix.platform }} push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + labels: ${{ needs.prepare-metadata.outputs.labels }} + outputs: type=image,name=pnnl/gridpack,push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=build-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=build-${{ matrix.arch }} build-args: | boost_version=1.81.0 ga_version=5.9.1 petsc_version=3.24.2 + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + echo "Digest for ${{ matrix.arch }}: ${digest}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Merge the architecture-specific images into a multi-arch manifest + merge-manifest: + name: Create Multi-Architecture Manifest + runs-on: ubuntu-latest + needs: + - prepare-metadata + - build-native + + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-* + path: /tmp/digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + # Parse all tags from metadata + TAGS="${{ needs.prepare-metadata.outputs.tags }}" + + # Build docker buildx imagetools create command with all tags + CMD="docker buildx imagetools create" + for tag in ${TAGS}; do + CMD="${CMD} -t ${tag}" + done + + # Add all digests + for digest in $(ls -1); do + CMD="${CMD} pnnl/gridpack@sha256:${digest}" + done + + echo "Creating manifest with command:" + echo "${CMD}" + eval "${CMD}" + - name: Inspect multi-arch manifest run: | echo "Inspecting multi-architecture manifest..." - # Get the first tag from the tags output - TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n 1) + TAG="${{ needs.prepare-metadata.outputs.primary-tag }}" docker buildx imagetools inspect ${TAG} - - name: Run ctest smoke tests - if: ${{ !inputs.skip_tests }} - continue-on-error: true + # Test both architectures in parallel + test-native: + name: Test ${{ matrix.platform }} image + needs: + - prepare-metadata + - merge-manifest + runs-on: ${{ matrix.runner }} + if: ${{ !inputs.skip_tests }} + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm64 + arch: arm64 + + steps: + - name: Log in to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Run ctest smoke tests on ${{ matrix.arch }} run: | - echo "Running ctest smoke tests..." + echo "Running ctest smoke tests on ${{ matrix.arch }}..." echo "Note: Test failures will NOT cause the workflow to fail (continue-on-error: true)" echo "Typical pass rate: ~94%" echo "" - # Get the first tag from the tags output - TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n 1) + TAG="${{ needs.prepare-metadata.outputs.primary-tag }}" + docker pull ${TAG} docker run --rm ${TAG} bash -c "cd /app/src/build && ctest -j"