record-tester: implement randomization matrix

[iameli]

The problem: we have too many things to test! For example:

• Stream into Node A, playback from Node B, for every node A + B — this is O(nodes^2)
• We want to test both long duration and short duration streams
• Recording enabled vs not
• Playback policy enabled vs not
• Multistream enabled vs not

Doubling the number of continuous automated tests every time we want to test a new feature is infeasible. Instead, we should implement some "fuzzing" — randomize the test parameters every time we want to boot up a new test. This will ensure we have coverage of all possible feature combinations without going over all of them exhaustively.

[iameli]

@hjpotter92 and I just outlined two proposals for how these tests could look, here they are:

Golang version:

package tester

import (
	"context"
	"sync"

	"golang.org/x/sync/errgroup"
)

func replicationTest(t *streamtester.T) {
	stream := t.makeStudioStream()

	nodes := t.getNodes(nodeSelector{
		Region: "sin",
		Count:  3,
	})

	var wg sync.WaitGroup

	ctx, done := context.WithCancel(t.ctx)
	g, gctx := errgroup.WithContext(ctx)
	g.Go(func() error {
		_, err := t.Stream(StreamParams{
			StudioStream: stream,
			Node:         nodes[0],
		})
		return err
	})

	g.Go(func() error {
		_, err := player.Play(PlayParams{
			StudioStream: stream,
			Node:         nodes[1],
		})
		return err
	})

	g.Go(func() error {
		_, err := player.Play(PlayParams{
			StudioStream: stream,
			Node:         nodes[2],
		})
		return err
	})

	err := g.Wait()
}

Golang Load Test example:

func LoadTest(t *streamtester.T) {
	ctx, done := context.WithCancel(t.ctx)
	g, gctx := errgroup.WithContext(ctx)

	nodes := t.getNodes(nodeSelector{
		Region: "sin",
		Count:  10,
	})

	for i, node := range nodes {
		stream := t.makeStudioStream()
		g.Go(func() error {
			_, err := t.Stream(StreamParams{
				StudioStream: stream,
				Node:         node,
			})
			return err
		})

		for i := 0; i < 100; i += 1 {
			g.Go(func() error {
				_, err := player.Play(PlayParams{
					StudioStream: stream,
					// Send each player to a different node
					Node:         nodes[i % len(nodes)],
				})
				return err
			})
		}
	}

	err := g.Wait()
}

Our first attempt, YAML syntax:

version: 1

tests:
  # Replication test
  - task: livestream
    nodes:
      count: 3
      tags:
        region: sin
    repeat: 5
    tasks:
    - task: livestream
      streamKey: abc123
      ingest: [0]
      playback: [1,2]
      duration: 300
    - task: livestream
      streamKey: abc123
      ingest: [1]
      playback: [0,2]
      duration: 300
    - task: livestream
      streamKey: abc123
      ingest: [2]
      playback: [0,1]
      duration: 300

  # record-tester
  - nodes:
      count: 1
      tags:
        region: sin
    repeat: 0
    apiKey: abc-123
    apiUrl: https://livepeer.monster/api
    tasks:
    - task: livestream
      ingest: [0]
      playback: [0]
    - task: wait
      duration: 300
    - task: play-vod


# playback load test
  - nodes:
      count: 20
    repeat: 1
    tasks:
    - task: livestream