From 0ebc47866889eb6ca27be0486b32c8fcbd60b4da Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Fri, 18 Oct 2024 12:06:12 -0600 Subject: [PATCH 1/6] feat: added webhook for test completion --- OpenBench/utils.py | 80 +++++++++++++++++++++++++++++++++++++++++++++- webhook | 1 + 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 webhook diff --git a/OpenBench/utils.py b/OpenBench/utils.py index 7a9eed44..96452a8a 100644 --- a/OpenBench/utils.py +++ b/OpenBench/utils.py @@ -18,6 +18,8 @@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +import sys + import datetime import hashlib import json @@ -420,6 +422,11 @@ def update_test(request, machine): test = Test.objects.select_for_update().get(id=test_id) + # Send update to webhook, if it exists + if test.finished and os.path.exists('webhook'): + print("---------------------------------------------------------Notifying webhook!", file=sys.stderr) + notify_webhook(request, test, test_id) + if test.finished or test.deleted: return { 'stop' : True } @@ -502,5 +509,76 @@ def update_test(request, machine): Machine.objects.filter(id=machine_id).update( updated=timezone.now() ) - + return [{}, { 'stop' : True }][test.finished] + +def notify_webhook(request, test, test_id): + + with open('webhook') as webhook_file: + webhook = webhook_file.readlines()[0] + + lower, elo, upper = OpenBench.stats.ELO(test.wins, test.losses, test.draws) + error = max(upper - elo, elo - lower) + elo = OpenBench.templatetags.mytags.twoDigitPrecision(elo) + error = OpenBench.templatetags.mytags.twoDigitPrecision(error) + h0 = OpenBench.templatetags.mytags.twoDigitPrecision(test.elolower) + h1 = OpenBench.templatetags.mytags.twoDigitPrecision(test.eloupper) + tokens = test.devoptions.split(' ') + threads = tokens[0].split('=')[1] + hash = tokens[1].split('=')[1] + outcome = 'passed' if test.passed else 'failed' + if test.test_mode == 'GAMES': + mode_string = f'{test.max_games} games' + else: + mode_string = f'SPRT [{h0}, {h1}]' + if test.passed: + color = 0x37F769 + elif test.wins < test.losses: + color = 0xFA4E4E + else: + color = 0xFEFF58 + + + msg = { + 'username': test.engine, + 'embeds': [{ + 'title': f'Test `{test.dev.name}` vs `{test.base.name}` {outcome}', + 'url': request.build_absolute_uri(f'/test/{test_id}'), + 'color': color, + 'author': { "name": test.author }, + 'fields': [ + { + 'name': 'Configuration', + 'value': f'{test.timecontrol}s Threads={threads} Hash={hash}MB', + }, + { + 'name': 'Mode', + 'value': mode_string, + }, + { + 'name': 'Wins', + 'value': f'{test.wins}', + 'inline': True, + }, + { + 'name': 'Losses', + 'value': f'{test.losses}', + 'inline': True, + }, + { + 'name': 'Draws', + 'value': f'{test.draws}', + 'inline': True, + }, + { + 'name': 'Elo', + 'value': f'{elo} ± {error} (95%)', + }, + ] + }] + } + print("--------------------------------------------------------------------------------------------------------------------------------", file=sys.stderr) + print(msg, file=sys.stderr) + print("--------------------------------------------------------------------------------------------------------------------------------", file=sys.stderr) + print(f"-----------------------------------------------Sending msg to {webhook}", file=sys.stderr) + requests.post(webhook, json=msg) diff --git a/webhook b/webhook new file mode 100644 index 00000000..70cbda81 --- /dev/null +++ b/webhook @@ -0,0 +1 @@ +https://discord.com/api/webhooks/1296891056419508327/nERyVqOUw3slLz5gUok1MNIUWzPaJ67vsVRm6-1buE8Nhr5Vg-2mXJMoBz1MjCXnesL4 \ No newline at end of file From 60bd61829043322fc552b2079d0133f0bd4fcbd2 Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Mon, 21 Oct 2024 16:35:35 -0600 Subject: [PATCH 2/6] fix: fixed invalid webhook format --- Engines/Toad.json | 4 +-- OpenBench/utils.py | 74 +++++++++++++++------------------------------- webhook | 2 +- 3 files changed, 27 insertions(+), 53 deletions(-) diff --git a/Engines/Toad.json b/Engines/Toad.json index 49d2cf1d..a243d0fc 100644 --- a/Engines/Toad.json +++ b/Engines/Toad.json @@ -12,8 +12,8 @@ "test_presets": { "default": { - "base_branch": "main", - "dev_branch": "dev", + "base_branch": "random-move", + "dev_branch": "main", "book_name": "Pohl.epd", "test_bounds": "[0.00, 10.00]", "test_confidence": "[0.05, 0.05]", diff --git a/OpenBench/utils.py b/OpenBench/utils.py index 96452a8a..268172ed 100644 --- a/OpenBench/utils.py +++ b/OpenBench/utils.py @@ -43,6 +43,7 @@ from OpenBench.config import OPENBENCH_CONFIG from OpenBench.models import * from OpenBench.stats import TrinomialSPRT, PentanomialSPRT +from OpenBench.templatetags.mytags import longStatBlock import OpenBench.views @@ -422,11 +423,6 @@ def update_test(request, machine): test = Test.objects.select_for_update().get(id=test_id) - # Send update to webhook, if it exists - if test.finished and os.path.exists('webhook'): - print("---------------------------------------------------------Notifying webhook!", file=sys.stderr) - notify_webhook(request, test, test_id) - if test.finished or test.deleted: return { 'stop' : True } @@ -510,6 +506,16 @@ def update_test(request, machine): updated=timezone.now() ) + # Send update to webhook, if it exists + if test.finished and os.path.exists('webhook'): + response = notify_webhook(request, test, test_id) + + # print(f'Text: {response.text}') + # print(f'JSON: {response.json()}') + # print(f'Status Code: {response.status_code}') + # print(f'Reason: {response.reason}') + # print(f'Url: {response.url}') + return [{}, { 'stop' : True }][test.finished] def notify_webhook(request, test, test_id): @@ -517,68 +523,36 @@ def notify_webhook(request, test, test_id): with open('webhook') as webhook_file: webhook = webhook_file.readlines()[0] - lower, elo, upper = OpenBench.stats.ELO(test.wins, test.losses, test.draws) + lower, elo, upper = OpenBench.stats.Elo(test.results()) error = max(upper - elo, elo - lower) elo = OpenBench.templatetags.mytags.twoDigitPrecision(elo) error = OpenBench.templatetags.mytags.twoDigitPrecision(error) - h0 = OpenBench.templatetags.mytags.twoDigitPrecision(test.elolower) - h1 = OpenBench.templatetags.mytags.twoDigitPrecision(test.eloupper) - tokens = test.devoptions.split(' ') - threads = tokens[0].split('=')[1] - hash = tokens[1].split('=')[1] outcome = 'passed' if test.passed else 'failed' - if test.test_mode == 'GAMES': - mode_string = f'{test.max_games} games' - else: - mode_string = f'SPRT [{h0}, {h1}]' + + color = 0xFEFF58 if test.passed: color = 0x37F769 elif test.wins < test.losses: color = 0xFA4E4E - else: - color = 0xFEFF58 - - msg = { - 'username': test.engine, + return requests.post(webhook, json={ + 'username': test.dev_engine, 'embeds': [{ + 'author': { 'name': test.author }, 'title': f'Test `{test.dev.name}` vs `{test.base.name}` {outcome}', 'url': request.build_absolute_uri(f'/test/{test_id}'), 'color': color, - 'author': { "name": test.author }, + 'description': f'```\n{longStatBlock(test)}\n```', 'fields': [ { - 'name': 'Configuration', - 'value': f'{test.timecontrol}s Threads={threads} Hash={hash}MB', - }, - { - 'name': 'Mode', - 'value': mode_string, - }, - { - 'name': 'Wins', - 'value': f'{test.wins}', - 'inline': True, - }, - { - 'name': 'Losses', - 'value': f'{test.losses}', - 'inline': True, + 'name': 'Stats', + 'value': f'```\n{longStatBlock(test)}\n```', }, { - 'name': 'Draws', - 'value': f'{test.draws}', + 'name': 'User', + 'value': test.author, 'inline': True, - }, - { - 'name': 'Elo', - 'value': f'{elo} ± {error} (95%)', - }, + } ] }] - } - print("--------------------------------------------------------------------------------------------------------------------------------", file=sys.stderr) - print(msg, file=sys.stderr) - print("--------------------------------------------------------------------------------------------------------------------------------", file=sys.stderr) - print(f"-----------------------------------------------Sending msg to {webhook}", file=sys.stderr) - requests.post(webhook, json=msg) + }) diff --git a/webhook b/webhook index 70cbda81..2a857a68 100644 --- a/webhook +++ b/webhook @@ -1 +1 @@ -https://discord.com/api/webhooks/1296891056419508327/nERyVqOUw3slLz5gUok1MNIUWzPaJ67vsVRm6-1buE8Nhr5Vg-2mXJMoBz1MjCXnesL4 \ No newline at end of file +https://discord.com/api/webhooks/1298046962657923094/AxHqlm4MHA2LcApY-hMOGsmlR83adwhA2abjyvtYkywpCl6DUVubhEe1Uokhnzv7DTgh From 848923f009de854023f61267fe5450ee1a65b239 Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Mon, 21 Oct 2024 17:19:32 -0600 Subject: [PATCH 3/6] feat: added code to notify webhook on test completion --- OpenBench/utils.py | 79 +++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 50 deletions(-) diff --git a/OpenBench/utils.py b/OpenBench/utils.py index 268172ed..db39fe19 100644 --- a/OpenBench/utils.py +++ b/OpenBench/utils.py @@ -18,14 +18,10 @@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -import sys - import datetime import hashlib -import json import math import os -import random import re import requests @@ -399,6 +395,34 @@ def network_edit(request, engine, network): return OpenBench.views.redirect(request, '/networks/%s' % (network.engine), status='Applied changes') +def notify_webhook(request, test_id): + webhook = open('webhook').read().strip() # Remove trailing whitespace/newline, if present + test = Test.objects.get(id=test_id) + + # Compute stats + lower, elo, upper = OpenBench.stats.Elo(test.results()) + error = max(upper - elo, elo - lower) + elo = OpenBench.templatetags.mytags.twoDigitPrecision(elo) + error = OpenBench.templatetags.mytags.twoDigitPrecision(error) + outcome = 'passed' if test.passed else 'failed' + + # Green if passing, red if failing. + color = 0xFEFF58 + if test.passed: + color = 0x37F769 + elif test.wins < test.losses: + color = 0xFA4E4E + + return requests.post(webhook, json={ + 'username': test.dev_engine, + 'embeds': [{ + 'author': { 'name': test.author }, + 'title': f'Test `{test.dev.name}` vs `{test.base.name}` {outcome}', + 'url': request.build_absolute_uri(f'/test/{test_id}'), + 'color': color, + 'description': f'```\n{longStatBlock(test)}\n```', + }] + }) def update_test(request, machine): @@ -508,51 +532,6 @@ def update_test(request, machine): # Send update to webhook, if it exists if test.finished and os.path.exists('webhook'): - response = notify_webhook(request, test, test_id) - - # print(f'Text: {response.text}') - # print(f'JSON: {response.json()}') - # print(f'Status Code: {response.status_code}') - # print(f'Reason: {response.reason}') - # print(f'Url: {response.url}') + notify_webhook(request, test_id) return [{}, { 'stop' : True }][test.finished] - -def notify_webhook(request, test, test_id): - - with open('webhook') as webhook_file: - webhook = webhook_file.readlines()[0] - - lower, elo, upper = OpenBench.stats.Elo(test.results()) - error = max(upper - elo, elo - lower) - elo = OpenBench.templatetags.mytags.twoDigitPrecision(elo) - error = OpenBench.templatetags.mytags.twoDigitPrecision(error) - outcome = 'passed' if test.passed else 'failed' - - color = 0xFEFF58 - if test.passed: - color = 0x37F769 - elif test.wins < test.losses: - color = 0xFA4E4E - - return requests.post(webhook, json={ - 'username': test.dev_engine, - 'embeds': [{ - 'author': { 'name': test.author }, - 'title': f'Test `{test.dev.name}` vs `{test.base.name}` {outcome}', - 'url': request.build_absolute_uri(f'/test/{test_id}'), - 'color': color, - 'description': f'```\n{longStatBlock(test)}\n```', - 'fields': [ - { - 'name': 'Stats', - 'value': f'```\n{longStatBlock(test)}\n```', - }, - { - 'name': 'User', - 'value': test.author, - 'inline': True, - } - ] - }] - }) From b6e9bf6a87de7d079211b1f7d3e5296807c834c0 Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Mon, 21 Oct 2024 18:09:17 -0600 Subject: [PATCH 4/6] feat: allow webhook to be stored in WEBHOOK_URL env var --- OpenBench/utils.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/OpenBench/utils.py b/OpenBench/utils.py index db39fe19..b1f5f705 100644 --- a/OpenBench/utils.py +++ b/OpenBench/utils.py @@ -395,8 +395,7 @@ def network_edit(request, engine, network): return OpenBench.views.redirect(request, '/networks/%s' % (network.engine), status='Applied changes') -def notify_webhook(request, test_id): - webhook = open('webhook').read().strip() # Remove trailing whitespace/newline, if present +def notify_webhook(request, webhook, test_id): test = Test.objects.get(id=test_id) # Compute stats @@ -424,6 +423,16 @@ def notify_webhook(request, test_id): }] }) +def get_webhook(): + # Fetch the webhook from the environment variable, if it exists + webhook = os.getenv('WEBHOOK_URL') + + # If the env var doesn't exist, but the `webhook` file does, read from that + if not webhook and os.path.exists('webhook'): + webhook = open('webhook').read().strip() # Removing trailing whitespace/newline, if present + + return webhook + def update_test(request, machine): # Extract error information @@ -531,7 +540,8 @@ def update_test(request, machine): ) # Send update to webhook, if it exists - if test.finished and os.path.exists('webhook'): - notify_webhook(request, test_id) + webhook = get_webhook() + if test.finished and webhook: + notify_webhook(request, webhook, test_id) return [{}, { 'stop' : True }][test.finished] From b353b5b74474e2f691c00989c4160b6a7fdf9b5c Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Mon, 21 Oct 2024 19:02:23 -0600 Subject: [PATCH 5/6] chore: removed ability to use webhook env var --- OpenBench/utils.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/OpenBench/utils.py b/OpenBench/utils.py index b1f5f705..997a204c 100644 --- a/OpenBench/utils.py +++ b/OpenBench/utils.py @@ -395,7 +395,8 @@ def network_edit(request, engine, network): return OpenBench.views.redirect(request, '/networks/%s' % (network.engine), status='Applied changes') -def notify_webhook(request, webhook, test_id): +def notify_webhook(request, test_id): + webhook = open('webhook').read().strip() # Removing trailing whitespace/newline, if present test = Test.objects.get(id=test_id) # Compute stats @@ -423,16 +424,6 @@ def notify_webhook(request, webhook, test_id): }] }) -def get_webhook(): - # Fetch the webhook from the environment variable, if it exists - webhook = os.getenv('WEBHOOK_URL') - - # If the env var doesn't exist, but the `webhook` file does, read from that - if not webhook and os.path.exists('webhook'): - webhook = open('webhook').read().strip() # Removing trailing whitespace/newline, if present - - return webhook - def update_test(request, machine): # Extract error information @@ -540,8 +531,7 @@ def update_test(request, machine): ) # Send update to webhook, if it exists - webhook = get_webhook() - if test.finished and webhook: - notify_webhook(request, webhook, test_id) + if test.finished and os.path.exists('webhook'): + notify_webhook(request, test_id) return [{}, { 'stop' : True }][test.finished] From 7a1d0d8e4d22ad2d3169689bb965bd320038f355 Mon Sep 17 00:00:00 2001 From: Danny Hammer Date: Mon, 21 Oct 2024 19:28:45 -0600 Subject: [PATCH 6/6] chore: updated gitignore to ignore webhook --- .gitignore | 2 ++ Engines/Toad.json | 4 ++-- webhook | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 webhook diff --git a/.gitignore b/.gitignore index d79145a3..19e638d7 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ Media/* Media/PGNs/* Media/Networks/* Media/Events/* + +webhook \ No newline at end of file diff --git a/Engines/Toad.json b/Engines/Toad.json index a243d0fc..49d2cf1d 100644 --- a/Engines/Toad.json +++ b/Engines/Toad.json @@ -12,8 +12,8 @@ "test_presets": { "default": { - "base_branch": "random-move", - "dev_branch": "main", + "base_branch": "main", + "dev_branch": "dev", "book_name": "Pohl.epd", "test_bounds": "[0.00, 10.00]", "test_confidence": "[0.05, 0.05]", diff --git a/webhook b/webhook deleted file mode 100644 index 2a857a68..00000000 --- a/webhook +++ /dev/null @@ -1 +0,0 @@ -https://discord.com/api/webhooks/1298046962657923094/AxHqlm4MHA2LcApY-hMOGsmlR83adwhA2abjyvtYkywpCl6DUVubhEe1Uokhnzv7DTgh