Synchronize sim success between chiptops

generators/chipyard/src/main/scala/config/ChipletConfigs.scala

@@ -144,12 +144,14 @@ class MultiSimLLCChipletRocketConfig extends Config(
 )
 
 class CTCRocketConfig extends Config(
+  new testchipip.soc.WithChipIdPin ++
   new chipyard.harness.WithCTCLoopback ++
   new testchipip.ctc.WithCTC(Seq(new testchipip.ctc.CTCParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1), noPhy=true))) ++ 
   new RocketConfig
 )
 
 class DoubleCTCRocketConfig extends Config(
+  new testchipip.soc.WithChipIdPin ++
   new chipyard.harness.WithCTCLoopback ++
   new testchipip.ctc.WithCTC(Seq(
     new testchipip.ctc.CTCParams(onchipAddr = 0x1000000000L, offchipAddr = 0x0L, size = ((1L << 32) - 1), noPhy=false),
@@ -160,13 +162,15 @@ class DoubleCTCRocketConfig extends Config(
 
 class MultiCTCRocketConfig extends Config(
   new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++
+  new chipyard.harness.WithANDSuccessFn ++
   new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=0, chip1portId=0) ++ // connect CTC port 0 of chip 0 and CTC port 0 of chip 1
   new chipyard.harness.WithMultiChip(0, new CTCRocketConfig) ++
   new chipyard.harness.WithMultiChip(1, new CTCRocketConfig)
 )
 
 class MultiDoubleCTCRocketConfig extends Config(
   new chipyard.harness.WithAbsoluteFreqHarnessClockInstantiator ++
+  new chipyard.harness.WithANDSuccessFn ++
   new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=0, chip1portId=0) ++ // connect CTC port 0 of chip 0 and CTC port 0 of chip 1
   new chipyard.harness.WithMultiChipCTC(chip0=0, chip1=1, chip0portId=1, chip1portId=1) ++ // connect CTC port 1 of chip 0 and CTC port 1 of chip 1
   new chipyard.harness.WithMultiChip(0, new DoubleCTCRocketConfig) ++

generators/chipyard/src/main/scala/harness/HarnessBinders.scala

@@ -181,7 +181,7 @@ class WithTieOffL2FBusAXI extends HarnessBinder({
 class WithSimJTAGDebug extends HarnessBinder({
   case (th: HasHarnessInstantiators, port: JTAGPort, chipId: Int) => {
     val dtm_success = WireInit(false.B)
-    when (dtm_success) { th.success := true.B }
+    when (dtm_success) { th.chiptopSuccess(chipId) := true.B }
     val jtag_wire = Wire(new JTAGIO)
     jtag_wire.TDO.data := port.io.TDO
     jtag_wire.TDO.driven := true.B
@@ -197,7 +197,7 @@ class WithSimJTAGDebug extends HarnessBinder({
 class WithSimDMI extends HarnessBinder({
   case (th: HasHarnessInstantiators, port: DMIPort, chipId: Int) => {
     val dtm_success = WireInit(false.B)
-    when (dtm_success) { th.success := true.B }
+    when (dtm_success) { th.chiptopSuccess(chipId) := true.B }
     val dtm = Module(new TestchipSimDTM()(Parameters.empty)).connect(th.harnessBinderClock, th.harnessBinderReset.asBool, port.io, dtm_success)
   }
 })
@@ -264,7 +264,7 @@ class WithSimTSIOverSerialTL extends HarnessBinder({
           io.in <> ram.io.ser.out
 
           val success = SimTSI.connect(ram.io.tsi, clock, th.harnessBinderReset, chipId)
-          when (success) { th.success := true.B }
+          when (success) { th.chiptopSuccess(chipId) := true.B }
         }
       }
     }
@@ -293,7 +293,7 @@ class WithSimTSIToUARTTSI extends HarnessBinder({
     val uart_to_serial = Module(new UARTToSerial(freq, port.io.uart.c))
     val serial_width_adapter = Module(new SerialWidthAdapter(8, TSI.WIDTH))
     val success = SimTSI.connect(Some(TSIIO(serial_width_adapter.io.wide)), th.harnessBinderClock, th.harnessBinderReset)
-    when (success) { th.success := true.B }
+    when (success) { th.chiptopSuccess(chipId) := true.B }
     assert(!uart_to_serial.io.dropped)
     serial_width_adapter.io.narrow.flipConnect(uart_to_serial.io.serial)
     uart_to_serial.io.uart.rxd := port.io.uart.txd
@@ -303,7 +303,7 @@ class WithSimTSIToUARTTSI extends HarnessBinder({
 
 class WithTraceGenSuccess extends HarnessBinder({
   case (th: HasHarnessInstantiators, port: SuccessPort, chipId: Int) => {
-    when (port.io) { th.success := true.B }
+    when (port.io) { th.chiptopSuccess(chipId) := true.B }
   }
 })
 

generators/chipyard/src/main/scala/harness/HasHarnessInstantiators.scala

@@ -25,6 +25,7 @@ case object HarnessClockInstantiatorKey extends Field[() => HarnessClockInstanti
 case object HarnessBinderClockFrequencyKey extends Field[Double](100.0) // MHz
 case object MultiChipIdx extends Field[Int](0)
 case object DontTouchChipTopPorts extends Field[Boolean](true)
+case object SuccessFn extends Field[Vec[Bool] => Bool]((cs: Vec[Bool]) => { cs.reduce(_ || _) })
 
 class WithMultiChip(id: Int, p: Parameters) extends Config((site, here, up) => {
   case MultiChipParameters(`id`) => p
@@ -44,6 +45,14 @@ class WithDontTouchChipTopPorts(b: Boolean = true) extends Config((site, here, u
   case DontTouchChipTopPorts => b
 })
 
+class WithCustomSuccessFn(fn: Vec[Bool] => Bool) extends Config((site, here, up) => {
+  case SuccessFn => fn
+})
+
+class WithANDSuccessFn extends Config((site, here, up) => {
+  case SuccessFn => (cs: Vec[Bool]) => { cs.reduce(_ && _) }
+})
+
 // A TestHarness mixing this in will
 // - use the HarnessClockInstantiator clock provide
 trait HasHarnessInstantiators {
@@ -76,6 +85,10 @@ trait HasHarnessInstantiators {
     case None => Seq(p)
   }
 
+  // Used to synchronize between chiptops
+  val chiptopSuccess: Vec[Bool] = WireInit(VecInit(Seq.fill(chipParameters.size)(false.B)))
+  val successFn: Vec[Bool] => Bool = p(SuccessFn)
+
   // This shold be called last to build the ChipTops
   def instantiateChipTops(): Seq[LazyModule] = {
     require(p(MultiChipNChips).isEmpty || supportsMultiChip,

generators/chipyard/src/main/scala/harness/TestHarness.scala

@@ -20,7 +20,7 @@ class TestHarness(implicit val p: Parameters) extends Module with HasHarnessInst
   val io = IO(new Bundle {
     val success = Output(Bool())
   })
-  val success = WireInit(false.B)
+  val success = successFn(chiptopSuccess)
   io.success := success
 
   override val supportsMultiChip = true

tests/multi-ctc-test.c

@@ -4,9 +4,11 @@
 #include <stdlib.h>
 #include <riscv-pk/encoding.h>
 #include "marchid.h"
+#include "mmio.h"
 
 #define CTC0_OFFSET (0x10L << 32)
 #define CTC1_OFFSET (0x20L << 32)
+#define CHIP_ID_ADDR 0x2000L
 
 uint32_t src[10] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9};
 uint32_t dest[10];
@@ -62,11 +64,18 @@ int rw_mem(uint64_t offset) {
 
 int main(void) {
 
-  printf("Testing CTC Port with NO PHY\n");
+  int chip_id = reg_read64(CHIP_ID_ADDR);
+  printf("Got chip id: %d\n", chip_id);
+
+  printf("Testing CTC Port on chip %d with NO PHY\n", chip_id);
   rw_mem(CTC1_OFFSET);
 
-  printf("Testing CTC Port with PHY\n");
+  printf("Testing CTC Port on chip %d with PHY\n", chip_id);
   rw_mem(CTC0_OFFSET);
 
+  if (chip_id == 0) {
+    printf("Chip 0 has an extra print to make the simulation run longer!\n");
+  }
+
   return 0;
 }