Store additional data for hnsw benchmarks (#255)

* Store raw precision values

* Store logs for docker container

* Small improvement

* Update scheduled time

Author: tellet-q

.github/workflows/continuous-benchmark-hnsw.yaml

@@ -3,8 +3,8 @@ name: Continuous Benchmark Hnsw Indexing
 on:
   workflow_dispatch:
   schedule:
-    # Run every day at 5am
-    - cron: "0 5 * * *"
+    # Run every day at 3am
+    - cron: "0 3 * * *"
 
 # Restrict to only running this workflow one at a time.
 # Any new runs will be queued until the previous run is complete.

ansible/playbooks/files/hnsw-indexing/get_score.py

@@ -30,11 +30,17 @@ def main():
         precision_after_iteration = given_output.get("precision_after_iteration", 1.0)
         score = round(precision_before_iteration / precision_after_iteration, 4)
         indexing_time = given_output.get("indexing_total_time_s", 0.0)
-        results[label] = {"score": score, "indexing_time": indexing_time}
+        results[label] = {
+            "indexing_time": indexing_time,
+            "precision_before_iteration": precision_before_iteration,
+            "precision_after_iteration": precision_after_iteration,
+            "precision_score": score,
+        }
 
     result = ""
     for label, data in results.items():
-        result += f"{label}_score={data['score']},{label}_indexing_time={data['indexing_time']},"
+        for key, value in data.items():
+            result += f"{label}_{key}={value},"
     print(result)
     return result
 

ansible/playbooks/playbook-hnsw-index.yml

@@ -59,7 +59,9 @@
       ansible.builtin.shell: |
         result="{{ hostvars['benchmark-machine']['precision_score'] }}"
         engine="{{ server.name }}-{{ server.version }}"
-        score=$(echo "$result" | grep -oP "${engine}_score=\K[^,]+")
+        precision_before_iteration=$(echo "$result" | grep -oP "${engine}_precision_before_iteration=\K[^,]+")
+        precision_after_iteration=$(echo "$result" | grep -oP "${engine}_precision_after_iteration=\K[^,]+")
+        precision_score=$(echo "$result" | grep -oP "${engine}_precision_score=\K[^,]+")
         indexing_time_s=$(echo "$result" | grep -oP "${engine}_indexing_time=\K[^,]+")
 
         MEASURE_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
@@ -70,6 +72,8 @@
             dataset,
             measure_timestamp,
             bench_name,
+            precision_before_iteration,
+            precision_after_iteration,
             precision_score,
             indexing_time_s
         ) VALUES (
@@ -78,7 +82,9 @@
             '{{ dataset_name }}',
             '${MEASURE_TIMESTAMP}',
             '{{ bench }}',
-            '${score}',
+            '${precision_before_iteration}',
+            '${precision_after_iteration}',
+            '${precision_score}',
             '${indexing_time_s}'
         );"
         docker exec -it qdrant-postgres psql -U qdrant -d postgres -c "${pg_query}"

ansible/playbooks/roles/run-hnsw-indexing-common/tasks/main.yml

@@ -43,6 +43,11 @@
   register: bench_result
   ignore_errors: yes
 
+- name: Save Docker container logs
+  ansible.builtin.shell: |
+    docker logs qdrant-continuous > "{{ working_dir }}/log-qdrant-container-{{ server_name }}-{{ server_version }}-{{ bench }}.log" 2>&1
+  ignore_errors: yes
+
 - name: Show script output if failed
   ansible.builtin.shell: cat "{{ working_dir }}/log-{{ server_name }}-{{ server_version }}-{{ bench }}.log"
   when: bench_result.rc != 0