fix: improve test suite reliability for CI

.github/workflows/ltb.yaml

@@ -32,8 +32,7 @@ jobs:
       fail-fast: false
       max-parallel: 4
       matrix:
-        #os: [ubuntu-latest, windows-latest, macos-latest]
-        os: [ubuntu-latest, macos-latest]
+        os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 10
     steps:

_test/helpers.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/carlmjohnson/be"
 	"github.com/nats-io/nats-server/v2/server"
@@ -63,6 +64,9 @@ func StartNatsServer(t testing.TB, workDir string) *server.Server {
 	})
 
 	server.Start()
+	if !server.ReadyForConnections(5 * time.Second) {
+		t.Fatal("nats server failed to start")
+	}
 
 	return server
 }
@@ -131,10 +135,22 @@ func StartNexus(t testing.TB, ctx context.Context, natsUrl string, size int, sta
 		be.NilErr(t, err)
 		be.NilErr(t, node.Start())
 
-		for !node.IsReady() {
-		}
+		be.NilErr(t, node.IsReady(30*time.Second))
 
 		ret[i] = node
 	}
 	return ret
 }
+
+// WaitFor polls condition every 25ms until it returns true or timeout is reached.
+func WaitFor(t testing.TB, timeout time.Duration, condition func() bool, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if condition() {
+			return
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting: %s", msg)
+}

_test/nexlet_inmem/inmem_test.go

@@ -115,6 +115,9 @@ func TestFunctionStartWorkloadAndTrigger(t *testing.T) {
 		StoreDir:  t.TempDir(),
 	})
 	server.Start()
+	if !server.ReadyForConnections(5 * time.Second) {
+		t.Fatal("nats server failed to start")
+	}
 	defer server.Shutdown()
 
 	nc, err := nats.Connect(server.ClientURL())
@@ -142,17 +145,24 @@ func TestFunctionStartWorkloadAndTrigger(t *testing.T) {
 	)
 	be.NilErr(t, err)
 	be.NilErr(t, nn.Start())
-	for !nn.IsReady() {
-	}
+	be.NilErr(t, nn.IsReady(10*time.Second))
 
-	ctx, cancel := context.WithTimeout(t.Context(), time.Second*3)
+	ctx, cancel := context.WithTimeout(t.Context(), time.Second*30)
 	defer cancel()
 
 	nClient, err := client.NewClient(ctx, nc, models.SystemNamespace)
 	be.NilErr(t, err)
 	be.Nonzero(t, nClient)
 
-	aR, err := nClient.Auction("inmem", nil)
+	var aR []*models.AuctionResponse
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		aR, err = nClient.Auction("inmem", nil)
+		if err == nil && len(aR) == 1 {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
 	be.NilErr(t, err)
 	be.Equal(t, 1, len(aR))
 

agents/native/state_test.go

@@ -17,6 +17,18 @@ import (
 	"github.com/synadia-io/nex/models"
 )
 
+func waitFor(t testing.TB, timeout time.Duration, condition func() bool, msg string) {
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if condition() {
+			return
+		}
+		time.Sleep(25 * time.Millisecond)
+	}
+	t.Fatalf("timed out waiting: %s", msg)
+}
+
 func MockRunner(t testing.TB) (*agent.Runner, error) {
 	t.Helper()
 
@@ -116,7 +128,9 @@ func TestNexletState(t *testing.T) {
 			err = ns.RemoveWorkload("derp", "abc123")
 			be.NilErr(t, err)
 
-			time.Sleep(300 * time.Millisecond)
+			waitFor(t, 5*time.Second, func() bool {
+				return ns.WorkloadCount() == 0
+			}, "waiting for workload removal")
 
 			be.Equal(t, 1, ns.NamespaceCount())
 			be.Equal(t, 0, ns.WorkloadCount())
@@ -131,7 +145,9 @@ func TestNexletState(t *testing.T) {
 			err = ns.SetLameduckMode(time.Second)
 			be.NilErr(t, err)
 
-			time.Sleep(2000 * time.Millisecond)
+			waitFor(t, 10*time.Second, func() bool {
+				return ns.NamespaceCount() == 0 && ns.WorkloadCount() == 0
+			}, "waiting for lameduck cleanup")
 			be.Equal(t, 0, ns.NamespaceCount())
 			be.Equal(t, 0, ns.WorkloadCount())
 		})