Skip to content

Commit a36b600

Browse files
tellet-qpre-commit-ci[bot]
tellet-q
and
pre-commit-ci[bot]
authored
Add cascade workflow (#256)
* Add cascade workflow * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
1 parent 55efb07 commit a36b600

File tree

2 files changed

+273
-0
lines changed

2 files changed

+273
-0
lines changed

‎.github/workflows/manual-benchmarks-cascade.yaml‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
name: Automated Cascade of Benchmarks
2+
3+
on:
4+
repository_dispatch:
5+
types: [run-benchmark, run-benchmark-cascade]
6+
workflow_dispatch:
7+
inputs:
8+
qdrant_version:
9+
description: "Version of qdrant to benchmark (tags/v1.6.1, <commit-id>, my-branch, docker/v1.5.1, ghcr/dev)"
10+
default: ghcr/dev
11+
dataset:
12+
description: "Dataset to benchmark"
13+
default: laion-small-clip
14+
engine_config:
15+
description: "Engine config to benchmark"
16+
default: qdrant-continuous-benchmark
17+
feature_flags_all:
18+
type: boolean
19+
description: "Enable all feature flags (false by default)"
20+
default: false
21+
# Inputs to control cascade behavior
22+
benchmark_set:
23+
description: "Benchmark set to run (from benchmark-configs.json)"
24+
default: "smoke_test"
25+
current_index:
26+
description: "Current benchmark index in cascade (internal use)"
27+
default: "0"
28+
29+
jobs:
30+
runBenchmark:
31+
name: benchmark - ${{ github.event.client_payload.qdrant_version || inputs.qdrant_version }} - ${{ github.event.client_payload.dataset || inputs.dataset }}
32+
runs-on: ubuntu-latest
33+
steps:
34+
- uses: actions/checkout@v3
35+
- uses: webfactory/ssh-agent@v0.8.0
36+
with:
37+
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
38+
39+
# Load benchmark configurations from file and set current config
40+
- name: Set benchmark configuration
41+
id: config
42+
run: |
43+
CONFIG_FILE="benchmark-configs.json"
44+
45+
if [ "${{ github.event_name }}" = "repository_dispatch" ]; then
46+
# Repository dispatch - get values from payload
47+
BENCHMARK_SET="${{ github.event.client_payload.benchmark_set }}"
48+
CURRENT_INDEX=${{ github.event.client_payload.current_index || 0 }}
49+
50+
echo "qdrant_version=${{ github.event.client_payload.qdrant_version }}" >> $GITHUB_OUTPUT
51+
echo "dataset=${{ github.event.client_payload.dataset }}" >> $GITHUB_OUTPUT
52+
echo "engine_config=${{ github.event.client_payload.engine_config }}" >> $GITHUB_OUTPUT
53+
echo "feature_flags_all=${{ github.event.client_payload.feature_flags_all }}" >> $GITHUB_OUTPUT
54+
echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT
55+
echo "current_index=$CURRENT_INDEX" >> $GITHUB_OUTPUT
56+
else
57+
# Workflow dispatch - get benchmark set from input and read from file
58+
BENCHMARK_SET="${{ inputs.benchmark_set }}"
59+
CURRENT_INDEX=${{ inputs.current_index || 0 }}
60+
61+
if [ "$BENCHMARK_SET" = "single" ]; then
62+
# Single benchmark - use the input values directly
63+
echo "qdrant_version=${{ inputs.qdrant_version }}" >> $GITHUB_OUTPUT
64+
echo "dataset=${{ inputs.dataset }}" >> $GITHUB_OUTPUT
65+
echo "engine_config=${{ inputs.engine_config }}" >> $GITHUB_OUTPUT
66+
echo "feature_flags_all=${{ inputs.feature_flags_all }}" >> $GITHUB_OUTPUT
67+
echo "benchmark_set=single" >> $GITHUB_OUTPUT
68+
echo "current_index=0" >> $GITHUB_OUTPUT
69+
else
70+
# Benchmark set - read from config file
71+
if [ ! -f "$CONFIG_FILE" ]; then
72+
echo "Configuration file $CONFIG_FILE not found!"
73+
exit 1
74+
fi
75+
76+
if ! jq empty "$CONFIG_FILE" 2>/dev/null; then
77+
echo "Invalid JSON in $CONFIG_FILE"
78+
exit 1
79+
fi
80+
81+
BENCHMARK_CONFIGS=$(jq -r ".benchmark_sets.\"$BENCHMARK_SET\"" "$CONFIG_FILE")
82+
83+
if [ "$BENCHMARK_CONFIGS" = "null" ]; then
84+
echo "Benchmark set '$BENCHMARK_SET' not found in configuration file"
85+
echo "Available sets:"
86+
jq -r '.benchmark_sets | keys[]' "$CONFIG_FILE"
87+
exit 1
88+
fi
89+
90+
CURRENT_CONFIG=$(echo "$BENCHMARK_CONFIGS" | jq -r ".[$CURRENT_INDEX]")
91+
92+
if [ "$CURRENT_CONFIG" = "null" ]; then
93+
echo "Invalid index $CURRENT_INDEX for benchmark set '$BENCHMARK_SET'"
94+
exit 1
95+
fi
96+
97+
QDRANT_VERSION=$(echo "$CURRENT_CONFIG" | jq -r '.qdrant_version')
98+
DATASET=$(echo "$CURRENT_CONFIG" | jq -r '.dataset')
99+
ENGINE_CONFIG=$(echo "$CURRENT_CONFIG" | jq -r '.engine_config')
100+
FEATURE_FLAGS=$(echo "$CURRENT_CONFIG" | jq -r '.feature_flags_all')
101+
102+
echo "qdrant_version=$QDRANT_VERSION" >> $GITHUB_OUTPUT
103+
echo "dataset=$DATASET" >> $GITHUB_OUTPUT
104+
echo "engine_config=$ENGINE_CONFIG" >> $GITHUB_OUTPUT
105+
echo "feature_flags_all=$FEATURE_FLAGS" >> $GITHUB_OUTPUT
106+
echo "benchmark_set=$BENCHMARK_SET" >> $GITHUB_OUTPUT
107+
echo "current_index=$CURRENT_INDEX" >> $GITHUB_OUTPUT
108+
fi
109+
fi
110+
111+
- name: Run benchmark
112+
run: |
113+
export HCLOUD_TOKEN=${{ secrets.HCLOUD_TOKEN }}
114+
export POSTGRES_PASSWORD=${{ secrets.POSTGRES_PASSWORD }}
115+
export POSTGRES_HOST=${{ secrets.POSTGRES_HOST }}
116+
export QDRANT_VERSION=${{ steps.config.outputs.qdrant_version }}
117+
export DATASETS=${{ steps.config.outputs.dataset }}
118+
export ENGINE_NAME=${{ steps.config.outputs.engine_config }}
119+
export POSTGRES_TABLE=benchmark_manual
120+
export QDRANT__FEATURE_FLAGS__ALL=${{ steps.config.outputs.feature_flags_all }}
121+
bash -x tools/setup_ci.sh
122+
# bash -x tools/run_ci.sh
123+
124+
# Trigger next benchmark in cascade if there are more to run
125+
- name: Trigger next benchmark
126+
if: steps.config.outputs.benchmark_set != 'single'
127+
run: |
128+
CONFIG_FILE="benchmark-configs.json"
129+
BENCHMARK_SET="${{ steps.config.outputs.benchmark_set }}"
130+
CURRENT_INDEX=${{ steps.config.outputs.current_index }}
131+
NEXT_INDEX=$((CURRENT_INDEX + 1))
132+
133+
# Get the benchmark configurations for this set
134+
BENCHMARK_CONFIGS=$(jq -r ".benchmark_sets.\"$BENCHMARK_SET\"" "$CONFIG_FILE")
135+
TOTAL_CONFIGS=$(echo "$BENCHMARK_CONFIGS" | jq length)
136+
137+
echo "Benchmark set: $BENCHMARK_SET"
138+
echo "Current index: $CURRENT_INDEX"
139+
echo "Next index: $NEXT_INDEX"
140+
echo "Total configs: $TOTAL_CONFIGS"
141+
142+
if [ $NEXT_INDEX -lt $TOTAL_CONFIGS ]; then
143+
# Get next configuration
144+
NEXT_CONFIG=$(echo "$BENCHMARK_CONFIGS" | jq -r ".[$NEXT_INDEX]")
145+
NEXT_QDRANT_VERSION=$(echo "$NEXT_CONFIG" | jq -r '.qdrant_version')
146+
NEXT_DATASET=$(echo "$NEXT_CONFIG" | jq -r '.dataset')
147+
NEXT_ENGINE_CONFIG=$(echo "$NEXT_CONFIG" | jq -r '.engine_config')
148+
NEXT_FEATURE_FLAGS=$(echo "$NEXT_CONFIG" | jq -r '.feature_flags_all')
149+
150+
echo "🚀 Triggering next benchmark ($NEXT_INDEX/$((TOTAL_CONFIGS-1))):"
151+
echo " - qdrant_version: $NEXT_QDRANT_VERSION"
152+
echo " - dataset: $NEXT_DATASET"
153+
echo " - engine_config: $NEXT_ENGINE_CONFIG"
154+
echo " - feature_flags_all: $NEXT_FEATURE_FLAGS"
155+
156+
# Trigger next benchmark via repository dispatch
157+
curl -X POST \
158+
-H "Accept: application/vnd.github.v3+json" \
159+
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
160+
https://api.github.com/repos/${{ github.repository }}/dispatches \
161+
-d "{
162+
\"event_type\": \"run-benchmark-cascade\",
163+
\"client_payload\": {
164+
\"qdrant_version\": \"$NEXT_QDRANT_VERSION\",
165+
\"dataset\": \"$NEXT_DATASET\",
166+
\"engine_config\": \"$NEXT_ENGINE_CONFIG\",
167+
\"feature_flags_all\": $NEXT_FEATURE_FLAGS,
168+
\"benchmark_set\": \"$BENCHMARK_SET\",
169+
\"current_index\": $NEXT_INDEX
170+
}
171+
}"
172+
else
173+
echo "🎉 All benchmarks in set '$BENCHMARK_SET' completed! ($TOTAL_CONFIGS total)"
174+
fi
175+
env:
176+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

‎benchmark-configs.json‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
{
2+
"_comment": "Benchmark configuration sets for automated cascading",
3+
"_usage": "Select a benchmark_set when triggering the workflow. Use 'single' for one-off runs with manual inputs.",
4+
5+
"benchmark_sets": {
6+
"full_regression": [
7+
{
8+
"qdrant_version": "ghcr/dev",
9+
"dataset": "dbpedia-openai-1M-1536-angular",
10+
"engine_config": "qdrant-continuous-benchmark",
11+
"feature_flags_all": false
12+
},
13+
{
14+
"qdrant_version": "ghcr/dev",
15+
"dataset": "deep-image-96-angular",
16+
"engine_config": "qdrant-continuous-benchmark",
17+
"feature_flags_all": true
18+
},
19+
{
20+
"qdrant_version": "ghcr/dev",
21+
"dataset": "gist-960-euclidean",
22+
"engine_config": "qdrant-continuous-benchmark",
23+
"feature_flags_all": false
24+
},
25+
{
26+
"qdrant_version": "ghcr/dev",
27+
"dataset": "glove-100-angular",
28+
"engine_config": "qdrant-continuous-benchmark",
29+
"feature_flags_all": false
30+
},
31+
{
32+
"qdrant_version": "docker/master",
33+
"dataset": "dbpedia-openai-1M-1536-angular",
34+
"engine_config": "qdrant-continuous-benchmark",
35+
"feature_flags_all": false
36+
},
37+
{
38+
"qdrant_version": "docker/master",
39+
"dataset": "deep-image-96-angular",
40+
"engine_config": "qdrant-continuous-benchmark",
41+
"feature_flags_all": true
42+
},
43+
{
44+
"qdrant_version": "docker/master",
45+
"dataset": "gist-960-euclidean",
46+
"engine_config": "qdrant-continuous-benchmark",
47+
"feature_flags_all": false
48+
},
49+
{
50+
"qdrant_version": "docker/master",
51+
"dataset": "glove-100-angular",
52+
"engine_config": "qdrant-continuous-benchmark",
53+
"feature_flags_all": false
54+
}
55+
],
56+
"smoke_test": [
57+
{
58+
"qdrant_version": "ghcr/dev",
59+
"dataset": "laion-small-clip",
60+
"engine_config": "qdrant-continuous-benchmark",
61+
"feature_flags_all": false
62+
},
63+
{
64+
"qdrant_version": "ghcr/dev",
65+
"dataset": "laion-small-clip",
66+
"engine_config": "qdrant-continuous-benchmark",
67+
"feature_flags_all": true
68+
}
69+
],
70+
"dataset_comparison": [
71+
{
72+
"qdrant_version": "ghcr/dev",
73+
"dataset": "dbpedia-openai-1M-1536-angular",
74+
"engine_config": "qdrant-continuous-benchmark",
75+
"feature_flags_all": false
76+
},
77+
{
78+
"qdrant_version": "ghcr/dev",
79+
"dataset": "deep-image-96-angular",
80+
"engine_config": "qdrant-continuous-benchmark",
81+
"feature_flags_all": true
82+
},
83+
{
84+
"qdrant_version": "ghcr/dev",
85+
"dataset": "gist-960-euclidean",
86+
"engine_config": "qdrant-continuous-benchmark",
87+
"feature_flags_all": false
88+
},
89+
{
90+
"qdrant_version": "ghcr/dev",
91+
"dataset": "glove-100-angular",
92+
"engine_config": "qdrant-continuous-benchmark",
93+
"feature_flags_all": false
94+
}
95+
]
96+
}
97+
}

0 commit comments

Comments
 (0)