Add emulated multi-host T3K testing via single-host, multi-process (tenstorrent#25285)

### Ticket
Link to Github Issue

### Problem description
Add basic sanity regression testing for emulating distributed multi-host
`MeshDevice`

### What's changed
- Added some basic sanity tests to T3K unit tests for big-mesh
- Remove some hardcoding in ControlPlane for
`t3k_dual_host_mesh_graph_descriptor.yaml` handling

### Checklist
- [x] [All post
commit](https://github.com/tenstorrent/tt-metal/actions/workflows/all-post-commit-workflows.yaml)
CI passes:
https://github.com/tenstorrent/tt-metal/actions/runs/16360533162
(Pending)
- [x] [T3K Unit Tests]:
https://github.com/tenstorrent/tt-metal/actions/runs/16361825747

Author: cfjchu

.github/workflows/t3000-unit-tests-impl.yaml

@@ -25,6 +25,7 @@ jobs:
         test-group: [
           { name: "t3k ttmetal tests", arch: wormhole_b0, cmd: run_t3000_ttmetal_tests, timeout: 35, owner_id: ULMEPM2MA}, #Sean Nijjar
           { name: "t3k ttnn tests", arch: wormhole_b0, cmd: run_t3000_ttnn_tests, timeout: 45, owner_id: UBHPP2NDP}, #Joseph Chu
+          { name: "t3k big-mesh multiprocess tests", arch: wormhole_b0, cmd: run_t3000_dual_rank_big_mesh_tests, timeout: 20, owner_id: UBHPP2NDP}, #Joseph Chu
           { name: "t3k tt_metal multiprocess tests", arch: wormhole_b0, cmd: run_t3000_tt_metal_multiprocess_tests, timeout: 5, owner_id: U03NG0A5ND7}, #Aditya Saigal
           { name: "t3k falcon7b tests", arch: wormhole_b0, cmd: run_t3000_falcon7b_tests, timeout: 30, owner_id: UBHPP2NDP}, #Joseph Chu
           { name: "t3k falcon40b tests", arch: wormhole_b0, cmd: run_t3000_falcon40b_tests, timeout: 30, owner_id: U053W15B6JF}, #Djordje Ivanovic

tests/scripts/t3000/run_t3000_unit_tests.sh

@@ -43,6 +43,10 @@ run_t3000_ttmetal_tests() {
   fi
 }
 
+run_t3000_dual_rank_big_mesh_tests() {
+  tt-run --rank-binding tests/tt_metal/distributed/config/2x4_multiprocess_rank_bindings.yaml --mpi-args "--allow-run-as-root --tag-output" build/test/tt_metal/distributed/multiprocess/distributed_multiprocess_tests --gtest_filter="*BigMeshDualRankTestT3K*"
+}
+
 run_t3000_ttfabric_tests() {
   # Record the start time
   fail=0

tests/tt_metal/distributed/multiprocess/CMakeLists.txt

@@ -4,6 +4,7 @@ target_sources(
     PRIVATE
         main.cpp
         test_visible_devices_mp.cpp
+        test_sanity.cpp
 )
 set_target_properties(
     distributed_multiprocess_tests

tests/tt_metal/distributed/multiprocess/run_visible_devices_mp_tests.sh

@@ -17,7 +17,7 @@ for config in "${DEVICE_CONFIGS[@]}"; do
     echo "------------------------------------------------"
 
     # Run with mpirun, setting the environment variable
-    TT_METAL_VISIBLE_DEVICES="$config" mpirun --allow-run-as-root -np 1 ./build/test/tt_metal/distributed/multiprocess/distributed_multiprocess_tests
+    TT_METAL_VISIBLE_DEVICES="$config" mpirun --allow-run-as-root -np 1 ./build/test/tt_metal/distributed/multiprocess/distributed_multiprocess_tests --gtest_filter="*VisibleDevicesMPTest*"
 
     if [ $? -eq 0 ]; then
         echo "✓ [distributed tests] Test passed for configuration: $config"

tests/tt_metal/distributed/multiprocess/test_sanity.cpp

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: © 2025 Tenstorrent AI ULC
+//
+// SPDX-License-Identifier: Apache-2.0
+
+#include <fmt/base.h>
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include <impl/context/metal_context.hpp>
+#include <tt-metalium/control_plane.hpp>
+#include <tt-metalium/distributed_context.hpp>
+#include <tt-metalium/system_mesh.hpp>
+#include <tt-metalium/mesh_coord.hpp>
+#include <tt-metalium/distributed_host_buffer.hpp>
+#include <tt-metalium/fabric_types.hpp>
+#include <tt-metalium/host_buffer.hpp>
+
+#include <tt-metalium/tt_metal.hpp>
+
+namespace tt::tt_metal::distributed {
+
+using tt_fabric::HostRankId;
+using tt_fabric::MeshId;
+using tt_fabric::MeshScope;
+
+TEST(BigMeshDualRankTestT3K, DistributedContext) {
+    auto& dctx = MetalContext::instance().get_distributed_context();
+    auto world_size = dctx.size();
+    EXPECT_EQ(*world_size, 2);
+}
+
+TEST(BigMeshDualRankTestT3K, LocalRankBinding) {
+    auto& dctx = MetalContext::instance().get_distributed_context();
+    auto& control_plane = MetalContext::instance().get_control_plane();
+
+    tt_fabric::HostRankId local_rank_binding = control_plane.get_local_host_rank_id_binding();
+    if (*dctx.rank() == 0) {
+        EXPECT_EQ(*local_rank_binding, 0);
+    } else {
+        EXPECT_EQ(*local_rank_binding, 1);
+    }
+}
+
+TEST(BigMeshDualRankTestT3K, SystemMeshValidation) {
+    EXPECT_NO_THROW({
+        const auto& system_mesh = SystemMesh::instance();
+        EXPECT_EQ(system_mesh.shape(), MeshShape(2,4));
+        EXPECT_EQ(system_mesh.local_shape(), MeshShape(2,2));
+    });
+}
+
+TEST(BigMeshDualRankTestT3K, MeshDevice2x4Validation) {
+    auto mesh_device = MeshDevice::create(MeshDeviceConfig(MeshShape(2,4)), DEFAULT_L1_SMALL_SIZE, DEFAULT_TRACE_REGION_SIZE, 1, tt::tt_metal::DispatchCoreType::WORKER);
+    EXPECT_EQ(mesh_device->shape(), MeshShape(2,4));
+}
+
+TEST(BigMeshDualRankTestT3K, SystemMeshShape) {
+    const auto& system_mesh = SystemMesh::instance();
+    EXPECT_EQ(system_mesh.local_shape(), MeshShape(2, 2));
+
+    auto& control_plane = MetalContext::instance().get_control_plane();
+    auto rank = control_plane.get_local_host_rank_id_binding();
+
+    if (rank == HostRankId{0}) {
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(0, 0)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(0, 1)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(1, 0)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(1, 1)));
+    } else {
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(0, 2)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(0, 3)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(1, 2)));
+        EXPECT_NO_THROW(system_mesh.get_physical_device_id(MeshCoordinate(1, 3)));
+    }
+}
+
+}  // namespace tt::tt_metal::distributed

tt_metal/api/tt-metalium/mesh_coord.hpp

@@ -408,13 +408,18 @@ class DistributedMeshContainer : public MeshContainer<MaybeRemote<T>> {
     template <typename CoordSystem>
     void populate_local_region(const CoordSystem& coord_sys, const std::vector<T>& local_values) {
         TT_FATAL(
-            local_values.size() == this->shape().mesh_size(),
+            local_values.size() <= this->shape().mesh_size(),
             "Number of local values {} does not match mesh size {}",
             local_values.size(),
             this->shape().mesh_size());
         size_t idx = 0;
-        for (const auto& local_coord : this->coord_range()) {
-            this->at(local_coord) = MaybeRemote<T>::local(local_values[idx++]);
+        for (const auto& coord : this->coord_range()) {
+            // If local_values size equals mesh size, treat all coordinates as local
+            // Otherwise, only populate coordinates that are actually local
+            // TODO: Does not support reshaping. Implementation generalized to support this in PR #25190.
+            if (local_values.size() == this->shape().mesh_size() || coord_sys.is_local(coord)) {
+                this->at(coord) = MaybeRemote<T>::local(local_values[idx++]);
+            }
         }
     }
 

tt_metal/distributed/coordinate_translation.hpp

@@ -7,6 +7,7 @@
 #include <mesh_coord.hpp>
 #include <tt-metalium/fabric_types.hpp>
 #include <stdint.h>
+#include <tuple>
 
 namespace tt {
 namespace tt_metal {
@@ -31,6 +32,10 @@ class PhysicalMeshCoordinate {
     MeshId mesh_id() const { return mesh_id_; }
     chip_id_t chip_id() const { return chip_id_; }
 
+    // Needed for reflect / fmt
+    static constexpr auto attribute_names = std::forward_as_tuple("mesh_id", "chip_id");
+    auto attribute_values() const { return std::forward_as_tuple(mesh_id_, chip_id_); }
+
 private:
     MeshId mesh_id_{0};
     chip_id_t chip_id_{0};

tt_metal/distributed/system_mesh.cpp

@@ -52,7 +52,7 @@ MaybeRemoteDeviceId SystemMesh::Impl::get_maybe_remote_device_id(const MeshCoord
     return physical_coordinates_.at(coord).when(
         [&](const auto& physical_coord) {
             auto physical_device_id = get_physical_device_id(coord);
-            log_debug(LogDistributed, "Mesh coordinate: {}, Physical device ID: {}", coord, physical_device_id);
+            log_debug(LogDistributed, "Mesh coordinate: {} is local, Physical device ID: {}", coord, physical_device_id);
             return MaybeRemoteDeviceId::local(physical_device_id);
         },
         [&]() {
@@ -80,6 +80,8 @@ SystemMesh::Impl::Impl() :
     }
 
     // Use populate_local_region to set up the distributed container
+    log_debug(LogDistributed, "SystemMesh: Global shape: {}, Local shape: {}, Local offset: {}", global_shape_, local_coordinates.shape(), local_offset_);
+    log_debug(LogDistributed, "SystemMesh: Populating local region with physical coordinates: {}", ordered_physical_coords);
     physical_coordinates_.populate_local_region(coord_system, ordered_physical_coords);
 }
 

tt_metal/fabric/control_plane.cpp

@@ -509,35 +509,6 @@ std::map<FabricNodeId, chip_id_t> ControlPlane::get_logical_chip_to_physical_chi
             logical_mesh_chip_id_to_physical_chip_id_mapping.insert({FabricNodeId(MeshId{4}, i), physical_chip_ids[i]});
         }
         // This case can be depreciated once we have multi-host testing and validate it working
-    } else if (mesh_graph_desc_filename == "t3k_dual_host_mesh_graph_descriptor.yaml") {
-        // TODO(#24230): This path will soon be deprecated once we generalize logical mesh_chip_id to physical chip_id
-        // mapping
-        auto& cluster = tt::tt_metal::MetalContext::instance().get_cluster();
-        auto chip_eth_coords = cluster.get_user_chip_ethernet_coordinates();
-        std::vector<eth_coord_t> eth_coords;
-        eth_coords.reserve(chip_eth_coords.size());
-        for (const auto& [_, eth_coord] : chip_eth_coords) {
-            eth_coords.push_back(eth_coord);
-        }
-        std::sort(eth_coords.begin(), eth_coords.end(), EthCoordComparator());
-
-        auto mesh_ids = this->get_local_mesh_id_bindings();
-        auto mesh_id = mesh_ids.at(0);  // Use the first mesh ID
-        auto host_rank_id = this->get_local_host_rank_id_binding();
-        auto fabric_chip_ids = this->routing_table_generator_->mesh_graph->get_chip_ids(mesh_id, host_rank_id).values();
-
-        TT_FATAL(
-            fabric_chip_ids.size() == eth_coords.size(),
-            "Number of fabric chip ids {} does not match number of eth coords {}",
-            fabric_chip_ids.size(),
-            eth_coords.size());
-        for (std::uint32_t idx = 0; idx < fabric_chip_ids.size(); idx++) {
-            auto fabric_chip_id = fabric_chip_ids.at(idx);
-            auto eth_coord = eth_coords.at(idx);
-            logical_mesh_chip_id_to_physical_chip_id_mapping.insert(
-                {tt_fabric::FabricNodeId(mesh_id, fabric_chip_id),
-                 cluster.get_physical_chip_id_from_eth_coord(eth_coord)});
-        }
     } else {
         // Iterate over every mesh defined in the mesh-graph descriptor and embed it on top of
         // the physical cluster using the generic helper.