From c93164220868c0189b93adb75b5916f5df9c0ea8 Mon Sep 17 00:00:00 2001 From: Jinsun Yoo Date: Tue, 24 Sep 2024 07:04:23 +0000 Subject: [PATCH 1/3] Dockerfile dependencies for ns3 & Startup script --- .dockerfile/hoti2024/Dockerfile | 7 ++++--- hoti2024/clone_astra_sim_ns3.sh | 18 ++++++++++++++++++ hoti2024/clone_collectiveapi.sh | 13 +++++++++++++ hoti2024/compile_astra_sim.sh | 2 +- hoti2024/compile_astra_sim_ns3.sh | 11 +++++++++++ hoti2024/install_msccl_tools.sh | 12 ++++++++++++ 6 files changed, 59 insertions(+), 4 deletions(-) create mode 100755 hoti2024/clone_astra_sim_ns3.sh create mode 100755 hoti2024/clone_collectiveapi.sh create mode 100755 hoti2024/compile_astra_sim_ns3.sh create mode 100755 hoti2024/install_msccl_tools.sh diff --git a/.dockerfile/hoti2024/Dockerfile b/.dockerfile/hoti2024/Dockerfile index 6d56012..50a1bd4 100644 --- a/.dockerfile/hoti2024/Dockerfile +++ b/.dockerfile/hoti2024/Dockerfile @@ -1,5 +1,5 @@ ## Use Ubuntu -FROM ubuntu:24.04 +FROM ubuntu:22.04 LABEL maintainer="Will Won " @@ -13,8 +13,9 @@ RUN apt -y install \ gcc g++ clang-format \ make cmake \ libboost-dev libboost-program-options-dev \ - python3.12 python3-pip python3-venv \ - graphviz + python3.11 python3-pip python3-venv \ + graphviz \ + openmpi-bin openmpi-doc libopenmpi-dev ## Create Python venv: Required for Python 3.12 RUN python3 -m venv /opt/venv/astra-sim diff --git a/hoti2024/clone_astra_sim_ns3.sh b/hoti2024/clone_astra_sim_ns3.sh new file mode 100755 index 0000000..4b7005c --- /dev/null +++ b/hoti2024/clone_astra_sim_ns3.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") + +# Clone ASTRA-sim +( +git clone git@github.com:astra-sim/astra-sim.git +cd ${SCRIPT_DIR}/astra-sim/ +git checkout tags/tutorial-hoti2024-ns3 +git submodule update --init --recursive +) + +# Create Chakra symlink for easy access +( +ln -s astra-sim/extern/graph_frontend/chakra . +) diff --git a/hoti2024/clone_collectiveapi.sh b/hoti2024/clone_collectiveapi.sh new file mode 100755 index 0000000..de990a9 --- /dev/null +++ b/hoti2024/clone_collectiveapi.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") + +# Clone MSCCLang-tools +( +git clone git@github.com:astra-sim/collectiveapi.git +cd ${SCRIPT_DIR}/collectiveapi +git submodule update --init +) + diff --git a/hoti2024/compile_astra_sim.sh b/hoti2024/compile_astra_sim.sh index 6669f0f..9c918c5 100755 --- a/hoti2024/compile_astra_sim.sh +++ b/hoti2024/compile_astra_sim.sh @@ -4,7 +4,7 @@ set -e # Path SCRIPT_DIR=$(dirname "$(realpath $0)") -# Install Chakra +# Compile ASTRA-sim with analytical model ( cd ${SCRIPT_DIR}/astra-sim/build/astra_analytical ./build.sh diff --git a/hoti2024/compile_astra_sim_ns3.sh b/hoti2024/compile_astra_sim_ns3.sh new file mode 100755 index 0000000..8370f4c --- /dev/null +++ b/hoti2024/compile_astra_sim_ns3.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") + +# Compile ASTRA-sim with ns3 backend model +( +cd ${SCRIPT_DIR}/astra-sim +bash ./build/astra_ns3/build.sh -c +) diff --git a/hoti2024/install_msccl_tools.sh b/hoti2024/install_msccl_tools.sh new file mode 100755 index 0000000..0f1b951 --- /dev/null +++ b/hoti2024/install_msccl_tools.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") + +# Install MSCCL-tools +( + +cd ${SCRIPT_DIR}/collectiveapi/msccl-tools +pip install . +) From b6ca7d98d5d9304879dd57f19d7bcbe2e9292c9c Mon Sep 17 00:00:00 2001 From: Jinsun Yoo Date: Sun, 29 Sep 2024 18:26:45 +0000 Subject: [PATCH 2/3] Demo4 --- hoti2024/compile_astra_sim_ns3.sh | 10 +++- .../demo4/generate_all_reduce_128nodes.py | 47 +++++++++++++++++++ hoti2024/demo4/inputs/RemoteMemory.json | 3 ++ hoti2024/demo4/inputs/Ring_8.yml | 4 ++ hoti2024/demo4/inputs/Ring_Ring_Ring.yml | 4 ++ hoti2024/demo4/inputs/Ring_Switch.yml | 4 ++ hoti2024/demo4/inputs/Ring_Switch_sys.json | 13 +++++ hoti2024/demo4/inputs/Ring_sys.json | 13 +++++ .../demo4/inputs/logical_128nodes_1D.json | 3 ++ .../demo4/inputs/logical_128nodes_2D.json | 3 ++ hoti2024/demo4/inputs/logical_8nodes_1D.json | 3 ++ hoti2024/demo4/run_demo4-1.sh | 15 ++++++ hoti2024/demo4/run_demo4-2.sh | 33 +++++++++++++ hoti2024/demo4/run_demo4-3.sh | 33 +++++++++++++ 14 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 hoti2024/demo4/generate_all_reduce_128nodes.py create mode 100644 hoti2024/demo4/inputs/RemoteMemory.json create mode 100644 hoti2024/demo4/inputs/Ring_8.yml create mode 100644 hoti2024/demo4/inputs/Ring_Ring_Ring.yml create mode 100644 hoti2024/demo4/inputs/Ring_Switch.yml create mode 100644 hoti2024/demo4/inputs/Ring_Switch_sys.json create mode 100644 hoti2024/demo4/inputs/Ring_sys.json create mode 100644 hoti2024/demo4/inputs/logical_128nodes_1D.json create mode 100644 hoti2024/demo4/inputs/logical_128nodes_2D.json create mode 100644 hoti2024/demo4/inputs/logical_8nodes_1D.json create mode 100755 hoti2024/demo4/run_demo4-1.sh create mode 100755 hoti2024/demo4/run_demo4-2.sh create mode 100755 hoti2024/demo4/run_demo4-3.sh diff --git a/hoti2024/compile_astra_sim_ns3.sh b/hoti2024/compile_astra_sim_ns3.sh index 8370f4c..f56bea4 100755 --- a/hoti2024/compile_astra_sim_ns3.sh +++ b/hoti2024/compile_astra_sim_ns3.sh @@ -7,5 +7,13 @@ SCRIPT_DIR=$(dirname "$(realpath $0)") # Compile ASTRA-sim with ns3 backend model ( cd ${SCRIPT_DIR}/astra-sim -bash ./build/astra_ns3/build.sh -c +bash ./build/astra_ns3/build.sh ) + +# Create Symlinks to configuration files +# Symbolic links are only to make it easier to examine. +# Inputs to execution scripts, etc. need to use the actual files. +ln -s ${SCRIPT_DIR}/astra-sim/extern/network_backend/ns-3/scratch/config/config.txt ./demo4/inputs/ns3_config_switch.txt +ln -s ${SCRIPT_DIR}/astra-sim/extern/network_backend/ns-3/scratch/config/config_clos.txt ./demo4/inputs/ns3_config_clos.txt +ln -s ${SCRIPT_DIR}/astra-sim/extern/network_backend/ns-3/scratch/topology/8_nodes_1_switch_topology.txt ./demo4/inputs/8_nodes_1_switch.txt +ln -s ${SCRIPT_DIR}/astra-sim/extern/network_backend/ns-3/scratch/topology/128_nodes_16_switch_topology.txt ./demo4/inputs/128_nodes_16_switch_clos.txt diff --git a/hoti2024/demo4/generate_all_reduce_128nodes.py b/hoti2024/demo4/generate_all_reduce_128nodes.py new file mode 100644 index 0000000..4de61b4 --- /dev/null +++ b/hoti2024/demo4/generate_all_reduce_128nodes.py @@ -0,0 +1,47 @@ +import os + +from chakra.third_party.utils.protolib import encodeMessage as encode_message +from chakra.et_def.et_def_pb2 import ( + Node as ChakraNode, + BoolList, + GlobalMetadata, + AttributeProto as ChakraAttr, + COMM_COLL_NODE, + ALL_REDUCE, +) + + +def main() -> None: + # create directories + if not os.path.exists("./allreduce_128"): + os.makedirs("./allreduce_128") + + + # metadata + npus_count = 128 # 8 NPUs + coll_size = 1_048_576 # 1 MB + + for npu_id in range(npus_count): + output_filename = f"allreduce_128/allreduce.{npu_id}.et" + with open(output_filename, "wb") as et: + # Chakra Metadata + encode_message(et, GlobalMetadata(version="0.0.4")) + + # create Chakra Node + node = ChakraNode() + node.id = 1 + node.name = "All-Reduce" + node.type = COMM_COLL_NODE + + # assign attributes + node.attr.append(ChakraAttr(name="is_cpu_op", bool_val=False)) + node.attr.append(ChakraAttr(name="comm_type", int64_val=ALL_REDUCE)) + node.attr.append(ChakraAttr(name="comm_size", uint64_val=coll_size)) + node.attr.append(ChakraAttr(name="involved_dim", bool_list=BoolList(values=[True]))) + + # store Chakra ET file + encode_message(et, node) + + +if __name__ == "__main__": + main() diff --git a/hoti2024/demo4/inputs/RemoteMemory.json b/hoti2024/demo4/inputs/RemoteMemory.json new file mode 100644 index 0000000..84ed7d9 --- /dev/null +++ b/hoti2024/demo4/inputs/RemoteMemory.json @@ -0,0 +1,3 @@ +{ + "memory-type": "NO_MEMORY_EXPANSION" +} diff --git a/hoti2024/demo4/inputs/Ring_8.yml b/hoti2024/demo4/inputs/Ring_8.yml new file mode 100644 index 0000000..562d758 --- /dev/null +++ b/hoti2024/demo4/inputs/Ring_8.yml @@ -0,0 +1,4 @@ +topology: [ Ring ] +npus_count: [ 8 ] +bandwidth: [ 50.0 ] # GB/s +latency: [ 500.0 ] # ns diff --git a/hoti2024/demo4/inputs/Ring_Ring_Ring.yml b/hoti2024/demo4/inputs/Ring_Ring_Ring.yml new file mode 100644 index 0000000..ef1038c --- /dev/null +++ b/hoti2024/demo4/inputs/Ring_Ring_Ring.yml @@ -0,0 +1,4 @@ +topology: [ Ring, Ring, Ring ] +npus_count: [ 4, 2, 2 ] +bandwidth: [ 50.0, 50.0, 50.0 ] # GB/s +latency: [ 500.0, 500.0, 500.0 ] # ns diff --git a/hoti2024/demo4/inputs/Ring_Switch.yml b/hoti2024/demo4/inputs/Ring_Switch.yml new file mode 100644 index 0000000..3055dea --- /dev/null +++ b/hoti2024/demo4/inputs/Ring_Switch.yml @@ -0,0 +1,4 @@ +topology: [ Ring, Switch ] +npus_count: [ 4, 2 ] +bandwidth: [ 50.0, 10.0 ] # GB/s +latency: [ 500.0, 2500.0 ] # ns diff --git a/hoti2024/demo4/inputs/Ring_Switch_sys.json b/hoti2024/demo4/inputs/Ring_Switch_sys.json new file mode 100644 index 0000000..120ca2d --- /dev/null +++ b/hoti2024/demo4/inputs/Ring_Switch_sys.json @@ -0,0 +1,13 @@ +{ + "scheduling-policy": "LIFO", + "endpoint-delay": 10, + "active-chunks-per-dimension": 1, + "preferred-dataset-splits": 4, + "all-reduce-implementation": ["ring","ring"], + "all-gather-implementation": ["ring","ring"], + "reduce-scatter-implementation": ["ring","ring"], + "all-to-all-implementation": ["ring","ring"], + "collective-optimization": "localBWAware", + "local-mem-bw": 50, + "boost-mode": 0 +} diff --git a/hoti2024/demo4/inputs/Ring_sys.json b/hoti2024/demo4/inputs/Ring_sys.json new file mode 100644 index 0000000..4c33135 --- /dev/null +++ b/hoti2024/demo4/inputs/Ring_sys.json @@ -0,0 +1,13 @@ +{ + "scheduling-policy": "LIFO", + "endpoint-delay": 10, + "active-chunks-per-dimension": 1, + "preferred-dataset-splits": 4, + "all-reduce-implementation": ["ring"], + "all-gather-implementation": ["ring"], + "reduce-scatter-implementation": ["ring"], + "all-to-all-implementation": ["ring"], + "collective-optimization": "localBWAware", + "local-mem-bw": 50, + "boost-mode": 0 +} diff --git a/hoti2024/demo4/inputs/logical_128nodes_1D.json b/hoti2024/demo4/inputs/logical_128nodes_1D.json new file mode 100644 index 0000000..33abae2 --- /dev/null +++ b/hoti2024/demo4/inputs/logical_128nodes_1D.json @@ -0,0 +1,3 @@ +{ + "logical-dims": ["128"] +} diff --git a/hoti2024/demo4/inputs/logical_128nodes_2D.json b/hoti2024/demo4/inputs/logical_128nodes_2D.json new file mode 100644 index 0000000..38eccca --- /dev/null +++ b/hoti2024/demo4/inputs/logical_128nodes_2D.json @@ -0,0 +1,3 @@ +{ + "logical-dims": ["16","8"] +} diff --git a/hoti2024/demo4/inputs/logical_8nodes_1D.json b/hoti2024/demo4/inputs/logical_8nodes_1D.json new file mode 100644 index 0000000..8dfe304 --- /dev/null +++ b/hoti2024/demo4/inputs/logical_8nodes_1D.json @@ -0,0 +1,3 @@ +{ + "logical-dims": ["8"] +} diff --git a/hoti2024/demo4/run_demo4-1.sh b/hoti2024/demo4/run_demo4-1.sh new file mode 100755 index 0000000..71b75fc --- /dev/null +++ b/hoti2024/demo4/run_demo4-1.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") +ASTRA_SIM=${SCRIPT_DIR}/../astra-sim/build/astra_analytical/build/bin/AstraSim_Analytical_Congestion_Unaware + +# Run ASTRA-sim +( +${ASTRA_SIM} \ + --workload-configuration=${SCRIPT_DIR}/../demo1/allreduce/allreduce \ + --system-configuration=${SCRIPT_DIR}/inputs/Ring_Switch_sys.json \ + --network-configuration=${SCRIPT_DIR}/inputs/Ring_Switch.yml \ + --remote-memory-configuration=${SCRIPT_DIR}/inputs/RemoteMemory.json +) diff --git a/hoti2024/demo4/run_demo4-2.sh b/hoti2024/demo4/run_demo4-2.sh new file mode 100755 index 0000000..d006436 --- /dev/null +++ b/hoti2024/demo4/run_demo4-2.sh @@ -0,0 +1,33 @@ +# #!/bin/bash +# # Absolue path to this script +# SCRIPT_DIR=$(dirname "$(realpath $0)") +# # Absolute paths to useful directories +# ASTRA_SIM_DIR="${SCRIPT_DIR:?}"/../astra-sim +# NS3_DIR="${ASTRA_SIM_DIR:?}"/extern/network_backend/ns-3 +# # Inputs - change as necessary. +# WORKLOAD="${SCRIPT_DIR:?}"/../../extern/graph_frontend/chakra/one_comm_coll_node_allgather +# SYSTEM="${SCRIPT_DIR:?}"/inputs/Ring.json +# MEMORY="${SCRIPT_DIR:?}"/inputs/RemoteMemory.json +# LOGICAL_TOPOLOGY="${ASTRA_SIM_DIR:?}"/inputs/network/ns3/sample_8nodes_1D.json +# # Note that ONLY this file is relative to NS3_DIR/simulation +# NETWORK="../../../ns-3/scratch/config/config.txt" + +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") +ASTRA_SIM_BUILD_DIR=${SCRIPT_DIR}/../astra-sim/extern/network_backend/ns-3/build/scratch/ +ASTRA_SIM=./ns3.42-AstraSimNetwork-default + +# Run ASTRA-sim +( +cd ${ASTRA_SIM_BUILD_DIR} +${ASTRA_SIM} \ + --workload-configuration=${SCRIPT_DIR}/../demo1/allreduce/allreduce \ + --system-configuration=${SCRIPT_DIR}/inputs/Ring_sys.json \ + --remote-memory-configuration=${SCRIPT_DIR}/inputs/RemoteMemory.json \ + --logical-topology-configuration=${SCRIPT_DIR}/inputs/logical_8nodes_1D.json \ + --network-configuration=../../../ns-3/scratch/config/config.txt \ + --comm-group-configuration=\"empty\" +) diff --git a/hoti2024/demo4/run_demo4-3.sh b/hoti2024/demo4/run_demo4-3.sh new file mode 100755 index 0000000..652e87e --- /dev/null +++ b/hoti2024/demo4/run_demo4-3.sh @@ -0,0 +1,33 @@ +# #!/bin/bash +# # Absolue path to this script +# SCRIPT_DIR=$(dirname "$(realpath $0)") +# # Absolute paths to useful directories +# ASTRA_SIM_DIR="${SCRIPT_DIR:?}"/../astra-sim +# NS3_DIR="${ASTRA_SIM_DIR:?}"/extern/network_backend/ns-3 +# # Inputs - change as necessary. +# WORKLOAD="${SCRIPT_DIR:?}"/../../extern/graph_frontend/chakra/one_comm_coll_node_allgather +# SYSTEM="${SCRIPT_DIR:?}"/inputs/Ring.json +# MEMORY="${SCRIPT_DIR:?}"/inputs/RemoteMemory.json +# LOGICAL_TOPOLOGY="${ASTRA_SIM_DIR:?}"/inputs/network/ns3/sample_8nodes_1D.json +# # Note that ONLY this file is relative to NS3_DIR/simulation +# NETWORK="../../../ns-3/scratch/config/config.txt" + +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") +ASTRA_SIM_BUILD_DIR=${SCRIPT_DIR}/../astra-sim/extern/network_backend/ns-3/build/scratch/ +ASTRA_SIM=./ns3.42-AstraSimNetwork-default + +# Run ASTRA-sim +( +cd ${ASTRA_SIM_BUILD_DIR} +${ASTRA_SIM} \ + --workload-configuration=${SCRIPT_DIR}/allreduce_128/allreduce \ + --system-configuration=${SCRIPT_DIR}/inputs/Ring_sys.json \ + --remote-memory-configuration=${SCRIPT_DIR}/inputs/RemoteMemory.json \ + --logical-topology-configuration=${SCRIPT_DIR}/inputs/logical_128nodes_1D.json \ + --network-configuration=../../../ns-3/scratch/config/config_clos.txt \ + --comm-group-configuration=\"empty\" +) From 338777e0faf57796eae099a7db747d6ea33846a4 Mon Sep 17 00:00:00 2001 From: Jinsun Yoo Date: Tue, 1 Oct 2024 03:46:32 +0000 Subject: [PATCH 3/3] Demo5 --- hoti2024/clone_collectiveapi.sh | 2 +- hoti2024/demo5/inputs/RemoteMemory.json | 3 +++ hoti2024/demo5/inputs/Ring_8.yml | 4 ++++ hoti2024/demo5/inputs/Ring_chakra.json | 10 ++++++++++ hoti2024/demo5/mscclang_to_chakra.sh | 15 +++++++++++++++ hoti2024/demo5/run_demo5.sh | 15 +++++++++++++++ 6 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 hoti2024/demo5/inputs/RemoteMemory.json create mode 100644 hoti2024/demo5/inputs/Ring_8.yml create mode 100644 hoti2024/demo5/inputs/Ring_chakra.json create mode 100644 hoti2024/demo5/mscclang_to_chakra.sh create mode 100755 hoti2024/demo5/run_demo5.sh diff --git a/hoti2024/clone_collectiveapi.sh b/hoti2024/clone_collectiveapi.sh index de990a9..53b7e3c 100755 --- a/hoti2024/clone_collectiveapi.sh +++ b/hoti2024/clone_collectiveapi.sh @@ -8,6 +8,6 @@ SCRIPT_DIR=$(dirname "$(realpath $0)") ( git clone git@github.com:astra-sim/collectiveapi.git cd ${SCRIPT_DIR}/collectiveapi -git submodule update --init +git submodule update --init --recursive ) diff --git a/hoti2024/demo5/inputs/RemoteMemory.json b/hoti2024/demo5/inputs/RemoteMemory.json new file mode 100644 index 0000000..84ed7d9 --- /dev/null +++ b/hoti2024/demo5/inputs/RemoteMemory.json @@ -0,0 +1,3 @@ +{ + "memory-type": "NO_MEMORY_EXPANSION" +} diff --git a/hoti2024/demo5/inputs/Ring_8.yml b/hoti2024/demo5/inputs/Ring_8.yml new file mode 100644 index 0000000..562d758 --- /dev/null +++ b/hoti2024/demo5/inputs/Ring_8.yml @@ -0,0 +1,4 @@ +topology: [ Ring ] +npus_count: [ 8 ] +bandwidth: [ 50.0 ] # GB/s +latency: [ 500.0 ] # ns diff --git a/hoti2024/demo5/inputs/Ring_chakra.json b/hoti2024/demo5/inputs/Ring_chakra.json new file mode 100644 index 0000000..37b8f6c --- /dev/null +++ b/hoti2024/demo5/inputs/Ring_chakra.json @@ -0,0 +1,10 @@ +{ + "scheduling-policy": "LIFO", + "endpoint-delay": 10, + "active-chunks-per-dimension": 1, + "preferred-dataset-splits": 4, + "all-reduce-implementation-chakra": ["/app/hoti2024/demo5/inputs/custom_ring"], + "collective-optimization": "localBWAware", + "local-mem-bw": 50, + "boost-mode": 0 +} diff --git a/hoti2024/demo5/mscclang_to_chakra.sh b/hoti2024/demo5/mscclang_to_chakra.sh new file mode 100644 index 0000000..b243322 --- /dev/null +++ b/hoti2024/demo5/mscclang_to_chakra.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") + +# MSCCLang -> MSCCL-IR (XML) +python3 ${SCRIPT_DIR}/../collectiveapi/msccl-tools/examples/mscclang/allreduce_a100_ring.py 8 1 1 > inputs/custom_allreduce_ring.xml \ + +# MSCCL-IR -> Chakra +cd ${SCRIPT_DIR}/../collectiveapi/chakra_converter/ +python3 et_converter.py \ + --input_type msccl \ + --input_filename ../../demo5/inputs/custom_allreduce_ring.xml \ + --output_filename ../../demo5/inputs/custom_ring diff --git a/hoti2024/demo5/run_demo5.sh b/hoti2024/demo5/run_demo5.sh new file mode 100755 index 0000000..d4e3b39 --- /dev/null +++ b/hoti2024/demo5/run_demo5.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Path +SCRIPT_DIR=$(dirname "$(realpath $0)") +ASTRA_SIM=${SCRIPT_DIR}/../astra-sim/build/astra_analytical/build/bin/AstraSim_Analytical_Congestion_Aware + +# Run ASTRA-sim +( +${ASTRA_SIM} \ + --workload-configuration=${SCRIPT_DIR}/../demo1/allreduce/allreduce \ + --system-configuration=${SCRIPT_DIR}/inputs/Ring_chakra.json \ + --network-configuration=${SCRIPT_DIR}/inputs/Ring_8.yml \ + --remote-memory-configuration=${SCRIPT_DIR}/inputs/RemoteMemory.json +)