Add NCCL backend

.clang-format

@@ -27,7 +27,6 @@ BreakBeforeBraces: Allman
 # ConstructorInitializerAllOnOneLineOrOnePerLine: false
 BreakConstructorInitializers: BeforeComma
 ConstructorInitializerIndentWidth: 0
-BreakInheritanceList: BeforeComma
 #AllowShortBlocksOnASingleLine: Always
 AllowShortBlocksOnASingleLine: true
 AllowShortCaseLabelsOnASingleLine: false

.cscs-ci/container/build.Containerfile

@@ -0,0 +1,20 @@
+ARG DEPS_IMAGE
+FROM $DEPS_IMAGE
+
+COPY . /oomph
+WORKDIR /oomph
+
+ARG BACKEND
+ARG NUM_PROCS
+RUN spack -e ci build-env oomph -- \
+        cmake -G Ninja -B build \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DOOMPH_WITH_TESTING=ON \
+            -DOOMPH_WITH_$(echo $BACKEND | tr '[:lower:]' '[:upper:]')=ON \
+            -DOOMPH_USE_BUNDLED_LIBS=ON \
+            -DOOMPH_USE_BUNDLED_HWMALLOC=OFF \
+            -DMPIEXEC_EXECUTABLE="" \
+            -DMPIEXEC_NUMPROC_FLAG="" \
+            -DMPIEXEC_PREFLAGS="" \
+            -DMPIEXEC_POSTFLAGS="" && \
+    spack -e ci build-env oomph -- cmake --build build -j$NUM_PROCS

.cscs-ci/container/deps.Containerfile

@@ -0,0 +1,24 @@
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+ARG SPACK_SHA
+RUN mkdir -p /opt/spack && \
+    curl -fLsS "https://api.github.com/repos/spack/spack/tarball/$SPACK_SHA" | tar --strip-components=1 -xz -C /opt/spack
+
+ENV PATH="/opt/spack/bin:$PATH"
+
+ARG SPACK_PACKAGES_SHA
+RUN mkdir -p /opt/spack-packages && \
+    curl -fLsS "https://api.github.com/repos/spack/spack-packages/tarball/$SPACK_PACKAGES_SHA" | tar --strip-components=1 -xz -C /opt/spack-packages
+
+RUN spack repo remove --scope defaults:base builtin && \
+    spack repo add --scope site /opt/spack-packages/repos/spack_repo/builtin
+
+ARG SPACK_ENV_FILE
+COPY $SPACK_ENV_FILE /spack_environment/spack.yaml
+
+ARG NUM_PROCS
+RUN spack external find --all && \
+    spack env create ci /spack_environment/spack.yaml && \
+    spack -e ci concretize -f && \
+    spack -e ci install --jobs $NUM_PROCS --fail-fast --only=dependencies

.cscs-ci/default.yaml

@@ -0,0 +1,192 @@
+include:
+  - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'
+
+variables:
+  BASE_IMAGE: jfrog.svc.cscs.ch/docker-group-csstaff/alps-images/ngc-pytorch:26.01-py3-alps4-dev
+  SPACK_SHA: v1.1.1
+  SPACK_PACKAGES_SHA: 5f24787b5cd3c2356d9a8188b989ceb5307299c6 # https://github.com/msimberg/spack-packages/tree/oomph-nccl
+  FF_TIMESTAMPS: true
+
+.build_deps_template:
+  timeout: 1 hour
+  before_script:
+    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
+    - export DOCKERFILE_SHA=`sha256sum .cscs-ci/container/deps.Containerfile | head -c 16`
+    - export ENV_FILE_SHA=`sha256sum ${SPACK_ENV_FILE} | head -c 16`
+    - export CONFIG_TAG=`echo $DOCKERFILE_SHA-$BASE_IMAGE-$SPACK_SHA-$SPACK_PACKAGES_SHA-$ENV_FILE_SHA | sha256sum - | head -c 16`
+    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-spack-deps-$BACKEND:$CONFIG_TAG
+    - echo -e "CONFIG_TAG=$CONFIG_TAG" >> base-${BACKEND}.env
+    - echo -e "DEPS_IMAGE=$PERSIST_IMAGE_NAME" >> base-${BACKEND}.env
+  variables:
+    DOCKERFILE: .cscs-ci/container/deps.Containerfile
+    DOCKER_BUILD_ARGS: '["BASE_IMAGE", "SPACK_SHA", "SPACK_PACKAGES_SHA", "SPACK_ENV_FILE"]'
+    SPACK_ENV_FILE: .cscs-ci/spack/$BACKEND.yaml
+  artifacts:
+    reports:
+      dotenv: base-${BACKEND}.env
+
+build_deps_nccl:
+  variables:
+    BACKEND: nccl
+  extends:
+    - .container-builder-cscs-gh200
+    - .build_deps_template
+
+build_deps_mpi:
+  variables:
+    BACKEND: mpi
+  extends:
+    - .container-builder-cscs-gh200
+    - .build_deps_template
+
+build_deps_ucx:
+  variables:
+    BACKEND: ucx
+  extends:
+    - .container-builder-cscs-gh200
+    - .build_deps_template
+
+# TODO: Libfabric tests are currently failing on Alps and need to be fixed.
+# build_deps_libfabric:
+#   variables:
+#     BACKEND: libfabric
+#   extends:
+#     - .container-builder-cscs-gh200
+#     - .build_deps_template
+
+.build_template:
+  extends: .container-builder-cscs-gh200
+  timeout: 15 minutes
+  before_script:
+    - echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin || true
+    - export PERSIST_IMAGE_NAME=$CSCS_REGISTRY_PATH/public/oomph-build-$BACKEND:$CI_COMMIT_SHA
+    - echo -e "BUILD_IMAGE=$PERSIST_IMAGE_NAME" >> build-${BACKEND}.env
+  variables:
+    DOCKERFILE: .cscs-ci/container/build.Containerfile
+    DOCKER_BUILD_ARGS: '["DEPS_IMAGE", "BACKEND"]'
+  artifacts:
+    reports:
+      dotenv: build-${BACKEND}.env
+
+build_nccl:
+  variables:
+    BACKEND: nccl
+  extends: .build_template
+  needs:
+    - job: build_deps_nccl
+      artifacts: true
+
+build_mpi:
+  variables:
+    BACKEND: mpi
+  extends: .build_template
+  needs:
+    - job: build_deps_mpi
+      artifacts: true
+
+build_ucx:
+  variables:
+    BACKEND: ucx
+  extends: .build_template
+  needs:
+    - job: build_deps_ucx
+      artifacts: true
+
+# TODO: Libfabric tests are currently failing on Alps and need to be fixed.
+# build_libfabric:
+#   variables:
+#     BACKEND: libfabric
+#   extends: .build_template
+#   needs:
+#     - job: build_deps_libfabric
+#       artifacts: true
+
+.test_template_base:
+  extends: .container-runner-clariden-gh200
+  variables:
+    SLURM_JOB_NUM_NODES: 1
+    SLURM_GPUS_PER_TASK: 1
+    SLURM_TIMELIMIT: '5:00'
+    SLURM_PARTITION: normal
+    SLURM_MPI_TYPE: pmix
+    SLURM_NETWORK: disable_rdzv_get
+    SLURM_LABELIO: 1
+    SLURM_UNBUFFEREDIO: 1
+    PMIX_MCA_psec: native
+    PMIX_MCA_gds: "^shmem2"
+    USE_MPI: NO
+
+.test_serial_template:
+  extends: .test_template_base
+  variables:
+    SLURM_NTASKS: 1
+  script:
+    - ctest --test-dir /oomph/build -L "serial" --output-on-failure --timeout 60 --parallel 8
+
+.test_parallel_template:
+  extends: .test_template_base
+  variables:
+    SLURM_NTASKS: 4
+  script:
+    # All ranks write to ctest files in Testing, but this can deadlock when
+    # writing inside the container.
+    - if [[ "${SLURM_PROCID}" == 0 ]]; then rm -rf /oomph/build/Testing; mkdir /tmp/Testing; ln -s /tmp/Testing /oomph/build/Testing; fi
+    - sleep 1
+    - ctest --test-dir /oomph/build -L "parallel-ranks-4" --output-on-failure --timeout 60
+
+test_serial_nccl:
+  extends: .test_serial_template
+  needs:
+    - job: build_nccl
+      artifacts: true
+  image: $BUILD_IMAGE
+
+test_parallel_nccl:
+  extends: .test_parallel_template
+  needs:
+    - job: build_nccl
+      artifacts: true
+  image: $BUILD_IMAGE
+
+test_serial_mpi:
+  extends: .test_serial_template
+  needs:
+    - job: build_mpi
+      artifacts: true
+  image: $BUILD_IMAGE
+
+test_parallel_mpi:
+  extends: .test_parallel_template
+  needs:
+    - job: build_mpi
+      artifacts: true
+  image: $BUILD_IMAGE
+
+test_serial_ucx:
+  extends: .test_serial_template
+  needs:
+    - job: build_ucx
+      artifacts: true
+  image: $BUILD_IMAGE
+
+test_parallel_ucx:
+  extends: .test_parallel_template
+  needs:
+    - job: build_ucx
+      artifacts: true
+  image: $BUILD_IMAGE
+
+# TODO: Libfabric tests are currently failing on Alps and need to be fixed.
+# test_serial_libfabric:
+#   extends: .test_serial_template
+#   needs:
+#     - job: build_libfabric
+#       artifacts: true
+#   image: $BUILD_IMAGE
+
+# test_parallel_libfabric:
+#   extends: .test_parallel_template
+#   needs:
+#     - job: build_libfabric
+#       artifacts: true
+#   image: $BUILD_IMAGE

.cscs-ci/spack/libfabric.yaml

@@ -0,0 +1,6 @@
+spack:
+  specs:
+  - oomph@main backend=libfabric +cuda
+  view: false
+  concretizer:
+    unify: true

.cscs-ci/spack/mpi.yaml

@@ -0,0 +1,6 @@
+spack:
+  specs:
+  - oomph@main backend=mpi +cuda
+  view: false
+  concretizer:
+    unify: true

.cscs-ci/spack/nccl.yaml

@@ -0,0 +1,6 @@
+spack:
+  specs:
+  - oomph@main backend=nccl +cuda
+  view: false
+  concretizer:
+    unify: true

.cscs-ci/spack/ucx.yaml

@@ -0,0 +1,6 @@
+spack:
+  specs:
+  - oomph@main backend=ucx +cuda
+  view: false
+  concretizer:
+    unify: true

CMakeLists.txt

@@ -80,6 +80,11 @@ include(oomph_ucx)
 # ---------------------------------------------------------------------
 include(oomph_libfabric)
 
+# ---------------------------------------------------------------------
+# oomph NCCL variant
+# ---------------------------------------------------------------------
+include(oomph_nccl)
+
 # ---------------------------------------------------------------------
 # main src subdir
 # ---------------------------------------------------------------------
@@ -142,6 +147,7 @@ install(
         ${CMAKE_CURRENT_BINARY_DIR}/oomphConfig.cmake
         ${CMAKE_CURRENT_BINARY_DIR}/oomphConfigVersion.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindLibfabric.cmake
+        ${CMAKE_CURRENT_LIST_DIR}/cmake/FindNCCL.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindUCX.cmake
         ${CMAKE_CURRENT_LIST_DIR}/cmake/FindPMIx.cmake
     DESTINATION

README.md

@@ -4,9 +4,9 @@
 # OOMPH
 
 **Oomph** is a library for enabling high performance point-to-point, asynchronous communication over
-different fabrics.  It leverages the ubiquitos MPI library as well as UCX and Libfabric.  Both
-device and host memory are supported. Under the hood it uses
-[hwmalloc](https://github.com/ghex-org/hwmalloc) for memory registration.
+different fabrics. It leverages the ubiquitous MPI library as well as UCX and Libfabric. Both
+device and host memory are supported. A subset of functionality is also supported with NCCL. Under
+the hood it uses [hwmalloc](https://github.com/ghex-org/hwmalloc) for memory registration.
 
 **selling points**
 - lightweight, fast
@@ -136,6 +136,65 @@ comm.progress();
 // or progress until some event is triggered
 while(!completed) { comm.progress(); }
 ```
+
+### Groups
+
+Communicators expose group functionality as provided by NCCL (with
+ncclGroupStart and ncclGroupEnd). For non-NCCL backends the group functionality
+is a no-op. For NCCL using the group functionality can be a both a requirement
+to avoid deadlocks (communication within a group can make progress
+independently, while outside of a group communication is ordered) and for
+performance (a single device kernel is submitted for a NCCL group). 
+
+Groups are created by explicitly starting and ending the group:
+
+```cpp
+comm.start_group();
+oomph::send_request sreq = comm.send(smsg, 1, 0);
+oomph::recv_request rreq = comm.recv(rmsg, 1, 0);
+comm.end_group();
+
+// With NCCL, no progress will be made until after the group ends
+sreq.wait();
+rreq.wait();
+```
+
+### Stream awareness
+
+Some backend implementations can schedule communication on a GPU stream.
+Currently only the NCCL backend makes use of this. All other backends ignore
+the stream argument. To query if a backend is stream-aware use the
+`is_stream_aware` member query on a communicator. The stream can be passed as
+an optional last parameter to `send` or `recv`:
+
+```cpp
+if (comm.is_stream_aware()) {
+    # Schedule communication on the default CUDA stream if the backend is
+    # stream aware
+    cudaStream_t stream = 0;
+    oomph::send_request comm.send(msg, 1, 0, stream);
+}
+```
+
+### NCCL restrictions
+
+NCCL has significantly different semantics from MPI, libfabric, and UCX which
+is reflected in a number of restrictions on how the NCCL communicator can be
+used:
+
+- Tags are not supported by NCCL and ignored by the backend. Communication
+  order on different ranks must match (except within NCCL groups where there is
+  some flexibility). This also means that e.g. recv should not be called before
+  send unless within a NCCL group.
+- The `thread_safe` option for the NCCL communicator is not supported because
+  of the above ordering restrictions.
+- Cancellation is not supported.
+- `wait` and `progress` are disallowed when a NCCL group is active as no
+  progress can be made until a NCCL group is ended and submitted.
+
+The NCCL backend is primarily designed for use in GHEX where these differences
+can be hidden from the user.
+
 ## Acknowledgments
 This work was financially supported by the PRACE project funded in part by the EU's Horizon 2020
 Research and Innovation programme (2014-2020) under grant agreement 823767.