Merge pull request #262 from qdrant/new-bq-cascade

cascade bench config for new BQ

Author: generall

.github/workflows/actions/create-server-with-retry/action.yml

@@ -7,6 +7,10 @@ inputs:
   server_type:
     description: 'Type of server to create'
     required: true
+  region:
+    description: 'Region to create the server in'
+    required: false
+    default: 'fsn1'
   max_retries:
     description: 'Maximum number of retry attempts'
     required: false
@@ -25,12 +29,13 @@ runs:
         SERVER_NAME="${{ inputs.server_name }}"
         SERVER_TYPE="${{ inputs.server_type }}"
         MAX_RETRIES="${{ inputs.max_retries }}"
+        SERVER_LOCATION="${{ inputs.region }}"
         RETRY_COUNT=0
 
         while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
           echo "Creating server $SERVER_NAME (attempt $((RETRY_COUNT + 1))/$MAX_RETRIES)..."
 
-          if SERVER_NAME=$SERVER_NAME SERVER_TYPE=$SERVER_TYPE bash -x "tools/hetzner/create_and_install.sh"; then
+          if SERVER_NAME=$SERVER_NAME SERVER_TYPE=$SERVER_TYPE SERVER_LOCATION=$SERVER_LOCATION bash -x "tools/hetzner/create_and_install.sh"; then
             echo "Successfully created server $SERVER_NAME"
             echo "success=true" >> $GITHUB_OUTPUT
             exit 0

.github/workflows/manual-benchmarks-cascade.yaml

@@ -34,6 +34,15 @@ on:
         machines_info:
             description: "Internal use, JSON array of machine pairs to use when machines_per_bench is false, e.g. '[{\"server_name\":\"server-0\",\"client_name\":\"client-0\"},{\"server_name\":\"server-1\",\"client_name\":\"client-1\"}]'"
             default: "[]"
+        region:
+            description: "Hetzner region to run the benchmarks in"
+            default: "fsn1"
+        server_machine_type:
+            description: "Hetzner server machine type to run the benchmarks in"
+            default: "cpx41"
+        client_machine_type:
+            description: "Hetzner server machine type to run the benchmarks in"
+            default: "cpx31"
 
 env:
   # Common environment variables
@@ -223,15 +232,17 @@ jobs:
         uses: ./.github/workflows/actions/create-server-with-retry
         with:
           server_name: ${{ steps.extract_names.outputs.server_name }}
-          server_type: cpx51
+          server_type: ${{ inputs.server_machine_type }}
+          region: ${{ inputs.region }}
           max_retries: 5
 
       - name: Create Client
         if: ${{ inputs.machines_per_bench == true || inputs.current_batch == 0 }}
         uses: ./.github/workflows/actions/create-server-with-retry
         with:
           server_name: ${{ steps.extract_names.outputs.client_name }}
-          server_type: cpx41
+          server_type: ${{ inputs.client_machine_type }}
+          region: ${{ inputs.region }}
           max_retries: 5
   runBenchmarks:
     name: Run Benchmark ${{ matrix.config.index }}

experiments/configurations/qdrant-quantizations.json

@@ -6,14 +6,14 @@ authors = ["Qdrant Team <info@qdrant.tech>"]
 package-mode = false
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.13"
-qdrant-client = "^1.11.0"
+python = ">=3.9,<3.13"
+qdrant-client = { git = "https://github.com/qdrant/qdrant-client.git", branch = "dev" }
 typer = "^0.6.1"
 jsons = "^1.6.3"
 h5py = "^3.7.0"
 weaviate-client = ">= 4.5, < 4.7"
 elasticsearch = "^8.10.0"
-pymilvus = "^2.4.1"
+pymilvus = "^2.5.0"
 redis = "^5.0.1"
 ipdb = "^0.13.9"
 stopit = "^1.1.2"

poetry.lock

@@ -11,7 +11,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2022-08-05T10:03:50.900734Z",
@@ -32,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2022-08-05T10:03:50.982398Z",
@@ -42,15 +42,27 @@
      "name": "#%%\n"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(PosixPath('/home/generall/projects/vector_search/vector-db-benchmark/results'),\n",
+       " 'qdrant-low-m-8-laion-small-clip-search-0-2025-06-15-13-37-22.json')"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "DATA_DIR = Path().resolve().parent / \"results\"\n",
     "DATA_DIR, list(DATA_DIR.glob(\"*.json\"))[0].name"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2022-08-05T10:03:51.482299Z",
@@ -75,7 +87,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {
     "ExecuteTime": {
      "end_time": "2022-08-05T10:03:54.150582Z",
@@ -85,7 +97,18 @@
      "name": "#%%\n"
     }
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(0, 0)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "upload_results, search_results = [], []\n",
     "\n",
@@ -126,9 +149,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "IndexError",
+     "evalue": "list index out of range",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m upload_results, \u001b[43msearch_results\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n",
+      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
+     ]
+    }
+   ],
    "source": [
     "upload_results, search_results[0]"
    ]
@@ -258,7 +293,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "venv",
    "language": "python",
    "name": "python3"
   },
@@ -272,7 +307,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.18"
+   "version": "3.11.9"
   }
  },
  "nbformat": 4,

pyproject.toml

@@ -16,7 +16,7 @@ SERVER_TYPE=${SERVER_TYPE:-cx41}
 
 SERVER_IMAGE=${SERVER_IMAGE:-ubuntu-22.04}
 
-SERVER_LOCATION=${SERVER_LOCATION:-nbg1}
+SERVER_LOCATION=${SERVER_LOCATION:-fsn1}
 
 SERVER_SSH_KEY=${SERVER_SSH_KEY:-'benchmark@qdrant.tech'}