From d94c6da7ba2b957f7a23ae748a904bd5bb9c62cd Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 11:03:52 +0530 Subject: [PATCH 01/42] Create vlm_metrics.py --- benchmark-scripts/vlm_metrics.py | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 benchmark-scripts/vlm_metrics.py diff --git a/benchmark-scripts/vlm_metrics.py b/benchmark-scripts/vlm_metrics.py new file mode 100644 index 0000000..55b6960 --- /dev/null +++ b/benchmark-scripts/vlm_metrics.py @@ -0,0 +1,49 @@ +import datetime + +def log_vlm_metrics(vlm_result): + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + filename = "vlm_performance_metrics.txt" + with open(filename, "a") as f: + # if hasattr(vlm_result, 'perf_metrics'): + # # Log format: key=value pairs separated by spaces + # metrics = [ + # f'Timestamp="{timestamp}"', + # f'Load_Time={vlm_result.perf_metrics.get_load_time()}', + # f'Generated_Tokens={vlm_result.perf_metrics.get_num_generated_tokens()}', + # f'Input_Tokens={vlm_result.perf_metrics.get_num_input_tokens()}', + # f'TTFT_Mean={vlm_result.perf_metrics.get_ttft().mean}', + # f'TPOT_Mean={vlm_result.perf_metrics.get_tpot().mean}', + # f'Throughput_Mean={vlm_result.perf_metrics.get_throughput().mean}', + # f'Generate_Duration_Mean={vlm_result.perf_metrics.get_generate_duration().mean}', + # f'Tokenization_Duration_Mean={vlm_result.perf_metrics.get_tokenization_duration().mean}', + # f'Detokenization_Duration_Mean={vlm_result.perf_metrics.get_detokenization_duration().mean}', + # f'Grammar_Compile_Max={vlm_result.perf_metrics.get_grammar_compile_time().max}', + # f'Grammar_Compile_Min={vlm_result.perf_metrics.get_grammar_compile_time().min}', + # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', + # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' + # ] + f.write(" ".join("Timestamp="20251105_101317" Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0 +") + "\n") + f.close() + +def get_vlm_call_average_duration(): + filename = "vlm_performance_metrics.txt" + total_duration = 0.0 + count = 0 + with open(filename, "r") as f: + for line in f: + parts = line.strip().split() + for part in parts: + if part.startswith("Generate_Duration_Mean="): + duration_str = part.split("=")[1] + try: + duration = float(duration_str) + total_duration += duration + count += 1 + except ValueError: + continue + if count == 0: + return 0, 0.0 + + average_duration = total_duration / count + return count, average_duration From b61e36a9cbe5da933fb35f0d38a4bae18a33d710 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 11:37:51 +0530 Subject: [PATCH 02/42] Rename benchmark-scripts/vlm_metrics.py to benchmark-scripts/vlm/vlm_metrics.py --- benchmark-scripts/{ => vlm}/vlm_metrics.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmark-scripts/{ => vlm}/vlm_metrics.py (100%) diff --git a/benchmark-scripts/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py similarity index 100% rename from benchmark-scripts/vlm_metrics.py rename to benchmark-scripts/vlm/vlm_metrics.py From 917a02b4d30f8af6d16494bab7b95f1ea0c7a438 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 11:38:51 +0530 Subject: [PATCH 03/42] Create setup.py --- benchmark-scripts/vlm/setup.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 benchmark-scripts/vlm/setup.py diff --git a/benchmark-scripts/vlm/setup.py b/benchmark-scripts/vlm/setup.py new file mode 100644 index 0000000..104265e --- /dev/null +++ b/benchmark-scripts/vlm/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='vlm_metrics', + version='0.1', + py_modules=['vlm_metrics'], +) From c96820e232bac0298bd5059fdfd25f5b7d988438 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 11:55:23 +0530 Subject: [PATCH 04/42] Rename benchmark-scripts/vlm/setup.py to benchmark-scripts/setup.py --- benchmark-scripts/{vlm => }/setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmark-scripts/{vlm => }/setup.py (100%) diff --git a/benchmark-scripts/vlm/setup.py b/benchmark-scripts/setup.py similarity index 100% rename from benchmark-scripts/vlm/setup.py rename to benchmark-scripts/setup.py From f4a63fc409bc72f52ad12180d16618c6cfe30347 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 11:55:49 +0530 Subject: [PATCH 05/42] Rename benchmark-scripts/vlm/vlm_metrics.py to benchmark-scripts/vlm_metrics.py --- benchmark-scripts/{vlm => }/vlm_metrics.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmark-scripts/{vlm => }/vlm_metrics.py (100%) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm_metrics.py similarity index 100% rename from benchmark-scripts/vlm/vlm_metrics.py rename to benchmark-scripts/vlm_metrics.py From 6d0fda82cbc2c0211d93b1099bfc860c5d11c05d Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 12:09:36 +0530 Subject: [PATCH 06/42] Rename benchmark-scripts/setup.py to benchmark-scripts/vlm/setup.py --- benchmark-scripts/{ => vlm}/setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmark-scripts/{ => vlm}/setup.py (100%) diff --git a/benchmark-scripts/setup.py b/benchmark-scripts/vlm/setup.py similarity index 100% rename from benchmark-scripts/setup.py rename to benchmark-scripts/vlm/setup.py From 725e2228a3c29ffe280e623df02842b8e6c33023 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 12:09:55 +0530 Subject: [PATCH 07/42] Rename benchmark-scripts/vlm_metrics.py to benchmark-scripts/vlm/vlm_metrics.py --- benchmark-scripts/{ => vlm}/vlm_metrics.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename benchmark-scripts/{ => vlm}/vlm_metrics.py (100%) diff --git a/benchmark-scripts/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py similarity index 100% rename from benchmark-scripts/vlm_metrics.py rename to benchmark-scripts/vlm/vlm_metrics.py From 3670903066b7f331107468e4afaa29f8e418d605 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 12:28:56 +0530 Subject: [PATCH 08/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 55b6960..b4a9729 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -3,6 +3,9 @@ def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" + results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "results")) + os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists + filepath = os.path.join(results_dir, filename) with open(filename, "a") as f: # if hasattr(vlm_result, 'perf_metrics'): # # Log format: key=value pairs separated by spaces From 0ffe27740b30a6e5f69745af44471ee923fb1968 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 12:51:59 +0530 Subject: [PATCH 09/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index b4a9729..1c6a426 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,8 +25,7 @@ def log_vlm_metrics(vlm_result): # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' # ] - f.write(" ".join("Timestamp="20251105_101317" Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0 -") + "\n") + f.write(" ".join("Timestamp="20251105_101317" Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") f.close() def get_vlm_call_average_duration(): From 6fa88962c0d60664ff32ac6b7d9c92873fd7ebb7 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 13:13:03 +0530 Subject: [PATCH 10/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 1c6a426..d9ff853 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,7 +25,7 @@ def log_vlm_metrics(vlm_result): # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' # ] - f.write(" ".join("Timestamp="20251105_101317" Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") + f.write(" ".join("Timestamp="20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") f.close() def get_vlm_call_average_duration(): From 21705c2027b77295e6caef06ed29d551a56e460d Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 13:19:25 +0530 Subject: [PATCH 11/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index d9ff853..629f012 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,7 +25,7 @@ def log_vlm_metrics(vlm_result): # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' # ] - f.write(" ".join("Timestamp="20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") + f.write(" ".join("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") f.close() def get_vlm_call_average_duration(): From e098161dff343a0ef530ad656b2c95604261c013 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 13:28:12 +0530 Subject: [PATCH 12/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 629f012..ef7345a 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,7 +25,7 @@ def log_vlm_metrics(vlm_result): # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' # ] - f.write(" ".join("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") + f.write(" ".join("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n")) f.close() def get_vlm_call_average_duration(): From 865dffd6aa579bc9f49e12444269c19d641256ac Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 13:35:09 +0530 Subject: [PATCH 13/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index ef7345a..cb981f1 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -1,5 +1,5 @@ import datetime - +import os def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" From c142a0e1a1d9cc49470befd766a0e92b0469656e Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 14:13:23 +0530 Subject: [PATCH 14/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index cb981f1..8119b9b 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -6,7 +6,7 @@ def log_vlm_metrics(vlm_result): results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "results")) os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) - with open(filename, "a") as f: + with open(filepath, "a") as f: # if hasattr(vlm_result, 'perf_metrics'): # # Log format: key=value pairs separated by spaces # metrics = [ From 6c55c7ba5c6913f0d6c899b2b6c5cda2755b3087 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 14:23:27 +0530 Subject: [PATCH 15/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 8119b9b..9da787c 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -6,6 +6,7 @@ def log_vlm_metrics(vlm_result): results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "results")) os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) + print("The file path is: ", filepath) with open(filepath, "a") as f: # if hasattr(vlm_result, 'perf_metrics'): # # Log format: key=value pairs separated by spaces From 9158475d520b716c4471bd22a9b7ff8b5e204198 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 16:29:18 +0530 Subject: [PATCH 16/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 9da787c..d3ef97a 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -3,7 +3,7 @@ def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" - results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "results")) + results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(sys.modules['__main__'].__file__)), "..", "results")) os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) print("The file path is: ", filepath) From 2f2305bad9553e410ac8ef20f5fe82b64993982f Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 16:37:03 +0530 Subject: [PATCH 17/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index d3ef97a..3681d48 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -1,5 +1,6 @@ import datetime import os +import os def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" From 1f8e21e4d1ea0f17c7f1f0ad44cf246809406ad2 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 16:40:33 +0530 Subject: [PATCH 18/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 3681d48..d6d08a0 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -1,6 +1,6 @@ import datetime import os -import os +import sys def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" From 51a6ac2a78dcd411ea122a620e0e1ab72e0a28a7 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 10 Nov 2025 17:25:58 +0530 Subject: [PATCH 19/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index d6d08a0..3a7b288 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -4,7 +4,7 @@ def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" - results_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(sys.modules['__main__'].__file__)), "..", "results")) + results_dir = os.getenv("RESULTS_DIR") os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) print("The file path is: ", filepath) From d57ceb3ff215ee0873ee4730e88d7f34766361be Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 11 Nov 2025 11:21:40 +0530 Subject: [PATCH 20/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 3a7b288..2fee6f7 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -4,7 +4,7 @@ def log_vlm_metrics(vlm_result): timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") filename = "vlm_performance_metrics.txt" - results_dir = os.getenv("RESULTS_DIR") + results_dir = os.getenv("CONTAINER_RESULTS_PATH") os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) print("The file path is: ", filepath) From b09cdd1cfa93242103573fceba2ea0607fb46085 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 11 Nov 2025 11:40:49 +0530 Subject: [PATCH 21/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 2fee6f7..6367e0a 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -27,14 +27,16 @@ def log_vlm_metrics(vlm_result): # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' # ] - f.write(" ".join("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n")) + f.write("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") f.close() def get_vlm_call_average_duration(): filename = "vlm_performance_metrics.txt" + results_dir = os.getenv("RESULTS_DIR") + filepath = os.path.join(results_dir, filename) total_duration = 0.0 count = 0 - with open(filename, "r") as f: + with open(filepath, "r") as f: for line in f: parts = line.strip().split() for part in parts: From f3549d28c060cae013560499d6a7b1652bb44c84 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 11 Nov 2025 18:52:14 +0530 Subject: [PATCH 22/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 6367e0a..4e19bf3 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -1,9 +1,11 @@ import datetime import os import sys +import socket def log_vlm_metrics(vlm_result): - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - filename = "vlm_performance_metrics.txt" + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + container_id = socket.gethostname() + filename = "vlm_performance_metrics_" + container_id + ".txt" results_dir = os.getenv("CONTAINER_RESULTS_PATH") os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) From 7492a08881f053e5dc91a2a2714ce657008f7e1f Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 09:24:51 +0530 Subject: [PATCH 23/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 4e19bf3..7cfece2 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -1,11 +1,25 @@ import datetime import os import sys -import socket +import uuid + +_unique_file_name = None + +def get_unique_file_name(): + global _unique_file_name + if _unique_file_name is None: + # First call: create a new unique name + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + unique_id = uuid.uuid4().hex[:6] + _unique_file_name = f"vlm_performance_metrics_{timestamp}_{unique_id}.txt" + else: + # Subsequent calls reuse it + print(f"Reusing existing file: {_unique_file_name}") + return _unique_file_name + def log_vlm_metrics(vlm_result): - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - container_id = socket.gethostname() - filename = "vlm_performance_metrics_" + container_id + ".txt" + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + filename = get_unique_file_name() results_dir = os.getenv("CONTAINER_RESULTS_PATH") os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists filepath = os.path.join(results_dir, filename) From bc5462364149bf2c84094e8d17d543c4ff52efbc Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 09:43:42 +0530 Subject: [PATCH 24/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 7cfece2..940f719 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -9,7 +9,7 @@ def get_unique_file_name(): global _unique_file_name if _unique_file_name is None: # First call: create a new unique name - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f") unique_id = uuid.uuid4().hex[:6] _unique_file_name = f"vlm_performance_metrics_{timestamp}_{unique_id}.txt" else: From aa34f068c5bedb1b64cbef5c5e45ff294ad682eb Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 10:48:52 +0530 Subject: [PATCH 25/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 940f719..0020e38 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,25 +25,25 @@ def log_vlm_metrics(vlm_result): filepath = os.path.join(results_dir, filename) print("The file path is: ", filepath) with open(filepath, "a") as f: - # if hasattr(vlm_result, 'perf_metrics'): - # # Log format: key=value pairs separated by spaces - # metrics = [ - # f'Timestamp="{timestamp}"', - # f'Load_Time={vlm_result.perf_metrics.get_load_time()}', - # f'Generated_Tokens={vlm_result.perf_metrics.get_num_generated_tokens()}', - # f'Input_Tokens={vlm_result.perf_metrics.get_num_input_tokens()}', - # f'TTFT_Mean={vlm_result.perf_metrics.get_ttft().mean}', - # f'TPOT_Mean={vlm_result.perf_metrics.get_tpot().mean}', - # f'Throughput_Mean={vlm_result.perf_metrics.get_throughput().mean}', - # f'Generate_Duration_Mean={vlm_result.perf_metrics.get_generate_duration().mean}', - # f'Tokenization_Duration_Mean={vlm_result.perf_metrics.get_tokenization_duration().mean}', - # f'Detokenization_Duration_Mean={vlm_result.perf_metrics.get_detokenization_duration().mean}', - # f'Grammar_Compile_Max={vlm_result.perf_metrics.get_grammar_compile_time().max}', - # f'Grammar_Compile_Min={vlm_result.perf_metrics.get_grammar_compile_time().min}', - # f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', - # f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' - # ] - f.write("Timestamp=20251105_101317 Load_Time=1777.0 Generated_Tokens=50 Input_Tokens=103 TTFT_Mean=1792.8023681640625 TPOT_Mean=109.8032455444336 Throughput_Mean=9.107198715209961 Generate_Duration_Mean=7175.43212890625 Tokenization_Duration_Mean=3.36899995803833 Detokenization_Duration_Mean=0.44200000166893005 Grammar_Compile_Max=-1.0 Grammar_Compile_Min=-1.0 Grammar_Compile_Std=-1.0 Grammar_Compile_Mean=-1.0" + "\n") + if hasattr(vlm_result, 'perf_metrics'): + # Log format: key=value pairs separated by spaces + metrics = [ + f'Timestamp="{timestamp}"', + f'Load_Time={vlm_result.perf_metrics.get_load_time()}', + f'Generated_Tokens={vlm_result.perf_metrics.get_num_generated_tokens()}', + f'Input_Tokens={vlm_result.perf_metrics.get_num_input_tokens()}', + f'TTFT_Mean={vlm_result.perf_metrics.get_ttft().mean}', + f'TPOT_Mean={vlm_result.perf_metrics.get_tpot().mean}', + f'Throughput_Mean={vlm_result.perf_metrics.get_throughput().mean}', + f'Generate_Duration_Mean={vlm_result.perf_metrics.get_generate_duration().mean}', + f'Tokenization_Duration_Mean={vlm_result.perf_metrics.get_tokenization_duration().mean}', + f'Detokenization_Duration_Mean={vlm_result.perf_metrics.get_detokenization_duration().mean}', + f'Grammar_Compile_Max={vlm_result.perf_metrics.get_grammar_compile_time().max}', + f'Grammar_Compile_Min={vlm_result.perf_metrics.get_grammar_compile_time().min}', + f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', + f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' + ] + f.write(" ".join(metrics) + "\n") f.close() def get_vlm_call_average_duration(): From c63d4d52f20a927f92741a2a0f5a21c572860914 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 11:14:09 +0530 Subject: [PATCH 26/42] Update consolidate_multiple_run_of_metrics.py --- benchmark-scripts/consolidate_multiple_run_of_metrics.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 5bb157a..0e79976 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -570,7 +570,8 @@ def return_blank(self): "power_usage.log":PowerUsageExtractor, "pcm.csv":PCMExtractor, r"(?:^xpum).*\.json$": XPUMUsageExtractor, - r"^qmassa.*parsed.*\.json$": QMASSAGPUUsageExtractor, } + r"^qmassa.*parsed.*\.json$": QMASSAGPUUsageExtractor, + r"^vlm_performance_metrics.*\.txt$": VLMPerformanceMetricsExtractor,} def add_parser(): parser = argparse.ArgumentParser(description='Consolidate data') From 9c5d2460b8106e60b060b8f576906c65d45c878c Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 15:58:16 +0530 Subject: [PATCH 27/42] Update vlm_metrics.py --- benchmark-scripts/vlm/vlm_metrics.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py index 0020e38..eeccbfd 100644 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ b/benchmark-scripts/vlm/vlm_metrics.py @@ -25,9 +25,9 @@ def log_vlm_metrics(vlm_result): filepath = os.path.join(results_dir, filename) print("The file path is: ", filepath) with open(filepath, "a") as f: - if hasattr(vlm_result, 'perf_metrics'): - # Log format: key=value pairs separated by spaces - metrics = [ + if hasattr(vlm_result, 'perf_metrics'): + # Log format: key=value pairs separated by spaces + metrics = [ f'Timestamp="{timestamp}"', f'Load_Time={vlm_result.perf_metrics.get_load_time()}', f'Generated_Tokens={vlm_result.perf_metrics.get_num_generated_tokens()}', @@ -42,9 +42,10 @@ def log_vlm_metrics(vlm_result): f'Grammar_Compile_Min={vlm_result.perf_metrics.get_grammar_compile_time().min}', f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' - ] + ] f.write(" ".join(metrics) + "\n") f.close() + def get_vlm_call_average_duration(): filename = "vlm_performance_metrics.txt" From 1918a798902aa223f3280eabbe1d49b78adb52b2 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Thu, 13 Nov 2025 16:02:36 +0530 Subject: [PATCH 28/42] Update consolidate_multiple_run_of_metrics.py --- .../consolidate_multiple_run_of_metrics.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 0e79976..b7e65b2 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -87,6 +87,35 @@ def extract_data(self, log_file_path): def return_blank(self): return {AVG_NPU_USAGE_CONSTANT: "NA"} +class VLMPerformanceMetricsExtractor(KPIExtractor): + #overriding abstract method + def extract_data(self, log_file_path): + + print("parsing VLM latency") + latency = {} + total_duration = 0.0 + count = 0 + with open(log_file_path, "r") as f: + for line in f: + parts = line.strip().split() + for part in parts: + if part.startswith("Generate_Duration_Mean="): + duration_str = part.split("=")[1] + try: + duration = float(duration_str) + total_duration += duration + count += 1 + except ValueError: + continue + + latency['VLM_TOTAL_CALLS'] = count + latency['VLM_AVERAGE_CALL_DURATION'] = total_duration / count + + return latency + + def return_blank(self): + return {"VLM_TOTAL_CALLS": "NA", 'VLM_AVERAGE_CALL_DURATION': "NA"} + class GPUUsageExtractor(KPIExtractor): #overriding abstract method def extract_data(self, log_file_path): From f951f634c6f749f05d43058c5d1d3411210253b9 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 1 Dec 2025 17:07:09 +0530 Subject: [PATCH 29/42] Create vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 216 ++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 benchmark-scripts/vlm_metrics_logger.py diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py new file mode 100644 index 0000000..c13cba8 --- /dev/null +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -0,0 +1,216 @@ +import logging +import os +import time +import uuid +from datetime import datetime +from logging.handlers import RotatingFileHandler + + +class VLMMetricsLogger: + + def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_count=5): + self.log_dir = log_dir or os.getenv('CONTAINER_RESULTS_PATH') + timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f") + unique_id = uuid.uuid4().hex[:6] + if log_file is None: + self.log_file = f"vlm_application_metrics_{timestamp}_{unique_id}.txt" + else: + self.log_file = log_file + + # Performance metrics file + self.performance_log_file = f"vlm_performance_metrics_{timestamp}_{unique_id}.txt" + + self.logger = None + self.performance_logger = None + self._setup_logger(max_bytes, backup_count) + + def _setup_logger(self, max_bytes, backup_count): + """Setup the logger with file rotation""" + # Create logs directory if it doesn't exist + os.makedirs(self.log_dir, exist_ok=True) + + # Create main logger + self.logger = logging.getLogger('vlm_metrics_logger') + self.logger.setLevel(logging.INFO) + + # Create performance logger + self.performance_logger = logging.getLogger('vlm_performance_logger') + self.performance_logger.setLevel(logging.INFO) + + # Avoid duplicate handlers + if not self.logger.handlers: + # Create file handler for main logs + log_path = os.path.join(self.log_dir, self.log_file) + file_handler = RotatingFileHandler( + log_path, + maxBytes=max_bytes, + backupCount=backup_count + ) + + # Create formatter + formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + # Set formatter + file_handler.setFormatter(formatter) + + # Add handler to main logger + self.logger.addHandler(file_handler) + + # Setup performance logger + if not self.performance_logger.handlers: + # Create file handler for performance logs + performance_log_path = os.path.join(self.log_dir, self.performance_log_file) + performance_file_handler = RotatingFileHandler( + performance_log_path, + maxBytes=max_bytes, + backupCount=backup_count + ) + + # Create formatter for performance logs + performance_formatter = logging.Formatter( + '%(asctime)s - %(levelname)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S' + ) + + performance_file_handler.setFormatter(performance_formatter) + + # Add handler to performance logger (no console output for performance) + self.performance_logger.addHandler(performance_file_handler) + + def log_start_time(self, application_name): + """ + Log application start time + + Args: + application_name (str): Name of the application + """ + timestamp_ms = int(time.time() * 1000) + + log_data = { + 'application': application_name, + 'event': 'start', + 'timestamp_ms': timestamp_ms + } + + # Format log message + message_parts = [f"{key}={value}" for key, value in log_data.items()] + message = " ".join(message_parts) + + self.logger.info(message) + return timestamp_ms + + def log_end_time(self, application_name): + """ + Log application end time + + Args: + application_name (str): Name of the application + """ + timestamp_ms = int(time.time() * 1000) + + log_data = { + 'application': application_name, + 'event': 'end', + 'timestamp_ms': timestamp_ms + } + + # Format log message + message_parts = [f"{key}={value}" for key, value in log_data.items()] + message = " ".join(message_parts) + + self.logger.info(message) + return timestamp_ms + + def log_custom_event(self, event_type, application_name, **kwargs): + """ + Log custom event with additional parameters + + Args: + event_type (str): Type of event (e.g., 'error', 'warning', 'info') + application_name (str): Name of the application + **kwargs: Additional key-value pairs to log + """ + timestamp_ms = int(time.time() * 1000) + + log_data = { + 'application': application_name, + 'event': event_type, + 'timestamp_ms': timestamp_ms + } + + # Add custom parameters + log_data.update(kwargs) + + # Format log message + message_parts = [f"{key}={value}" for key, value in log_data.items()] + message = " ".join(message_parts) + + # Log at appropriate level + if event_type.lower() == 'error': + self.logger.error(message) + elif event_type.lower() == 'warning': + self.logger.warning(message) + else: + self.logger.info(message) + + return timestamp_ms + + def log_performance_metrics(self, vlm_metrics_result_object): + + timestamp_ms = int(time.time() * 1000) + vlm_metrics_result_object + log_data = { + 'application': 'vlm-performance', + 'timestamp_ms': timestamp_ms, + 'Load_Time' : vlm_metrics_result_object.perf_metrics.get_load_time(), + 'Generated_Tokens':vlm_metrics_result_object.perf_metrics.get_num_generated_tokens(), + 'Input_Tokens':vlm_metrics_result_object.perf_metrics.get_num_input_tokens(), + 'TTFT_Mean':vlm_metrics_result_object.perf_metrics.get_ttft().mean, + 'TPOT_Mean':vlm_metrics_result_object.perf_metrics.get_tpot().mean, + 'Throughput_Mean':vlm_metrics_result_object.perf_metrics.get_throughput().mean, + 'Generate_Duration_Mean':vlm_metrics_result_object.perf_metrics.get_generate_duration().mean, + 'Tokenization_Duration_Mean':vlm_metrics_result_object.perf_metrics.get_tokenization_duration().mean, + 'Detokenization_Duration_Mean':vlm_metrics_result_object.perf_metrics.get_detokenization_duration().mean, + 'Grammar_Compile_Max':vlm_metrics_result_object.perf_metrics.get_grammar_compile_time().max, + 'Grammar_Compile_Min':vlm_metrics_result_object.perf_metrics.get_grammar_compile_time().min, + 'Grammar_Compile_Std':vlm_metrics_result_object.perf_metrics.get_grammar_compile_time().std, + 'Grammar_Compile_Mean':vlm_metrics_result_object.perf_metrics.get_grammar_compile_time().mean + } + + + # Format log message + message_parts = [f"{key}={value}" for key, value in log_data.items()] + message = " ".join(message_parts) + + self.performance_logger.info(message) + return timestamp_ms + + +# Global logger instance (singleton pattern) +_vlm_metrics_logger = None + +def get_logger(): + """Get the global logger instance""" + global _vlm_metrics_logger + if _vlm_metrics_logger is None: + _vlm_metrics_logger = VLMMetricsLogger() + return _vlm_metrics_logger + +def log_start_time(application_name): + """Convenience function for logging start time""" + return get_logger().log_start_time(application_name) + +def log_end_time(application_name): + """Convenience function for logging end time""" + return get_logger().log_end_time(application_name) + +def log_custom_event(event_type, application_name, **kwargs): + """Convenience function for logging custom events""" + return get_logger().log_custom_event(event_type, application_name, **kwargs) + +def log_performance_metric(metrics): + """Convenience function for logging performance metrics""" + return get_logger().log_performance_metrics(metrics) From 3e4481ca94aba5cc280fc316017dacda651d979f Mon Sep 17 00:00:00 2001 From: sumanaga Date: Mon, 1 Dec 2025 17:28:34 +0530 Subject: [PATCH 30/42] Create setup.py --- benchmark-scripts/setup.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 benchmark-scripts/setup.py diff --git a/benchmark-scripts/setup.py b/benchmark-scripts/setup.py new file mode 100644 index 0000000..062d26d --- /dev/null +++ b/benchmark-scripts/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name='vlm_metrics', + version='0.1', + py_modules=['vlm_metrics_logger'], +) From 08291b5d8f596c925d68134227782b76062dfafc Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 2 Dec 2025 12:58:08 +0530 Subject: [PATCH 31/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index c13cba8..cc4ef1b 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -28,6 +28,16 @@ def _setup_logger(self, max_bytes, backup_count): """Setup the logger with file rotation""" # Create logs directory if it doesn't exist os.makedirs(self.log_dir, exist_ok=True) + + # Delete existing VLM metrics files if they exist + if self.log_dir: + for filename in os.listdir(self.log_dir): + if filename.startswith('vlm_application_metrics') or filename.startswith('vlm_performance_metrics'): + file_path = os.path.join(self.log_dir, filename) + try: + os.remove(file_path) + except OSError: + pass # Ignore errors if file can't be deleted # Create main logger self.logger = logging.getLogger('vlm_metrics_logger') From 022db3410e9e3a8d700c7a505f5b4acb5d851f46 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 2 Dec 2025 16:32:04 +0530 Subject: [PATCH 32/42] Update consolidate_multiple_run_of_metrics.py --- .../consolidate_multiple_run_of_metrics.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 334f54c..39a1805 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -87,6 +87,61 @@ def extract_data(self, log_file_path): def return_blank(self): return {AVG_NPU_USAGE_CONSTANT: "NA"} + +class VLMAppMetricsExtractor: + def extract_data(self, log_file_path): + print("parsing application performance metrics") + timing_data = defaultdict(list) + with open(log_file_path, 'r') as file: + for line in file: + line = line.strip() + if 'application=' in line and 'timestamp_ms=' in line: + pattern = r'(\w+)=([^\s]+)' + matches = re.findall(pattern, line) + data = dict(matches) + if data: + app_name = data.get('application') + event = data.get('event') + timestamp_ms = data.get('timestamp_ms') + + if app_name and event in ['start', 'end'] and timestamp_ms: + timing_data[app_name].append({ + 'event': event, + 'timestamp_ms': int(timestamp_ms) + }) + + # Calculate durations for each application + app_durations = defaultdict(list) + apps_to_analyze = timing_data.keys() + + for app_name in apps_to_analyze: + events = sorted(timing_data[app_name], key=lambda x: x['timestamp_ms']) + start_time = None + + for event in events: + if event['event'] == 'start': + start_time = event['timestamp_ms'] + elif event['event'] == 'end' and start_time is not None: + duration = event['timestamp_ms'] - start_time + app_durations[app_name].append(duration) + start_time = None + + durations = dict(app_durations) + + # Generate statistics + statistics = {} + for app_name, duration_list in durations.items(): + if duration_list: + statistics[f"{app_name} Total Duration (ms)"] = sum(duration_list) + + else: + statistics[f"{app_name} Total Duration (ms)"] = 0 + + return statistics + + def return_blank(self): + return {"Application Performance Metrics": "NA"} + class VLMPerformanceMetricsExtractor(KPIExtractor): #overriding abstract method def extract_data(self, log_file_path): @@ -600,6 +655,7 @@ def return_blank(self): "pcm.csv":PCMExtractor, r"(?:^xpum).*\.json$": XPUMUsageExtractor, r"^qmassa.*parsed.*\.json$": QMASSAGPUUsageExtractor, + r"^vlm_application_metrics.*\.txt$": VLMAppMetricsExtractor,} r"^vlm_performance_metrics.*\.txt$": VLMPerformanceMetricsExtractor,} def add_parser(): From 4a066327e98b2e6d2c57df3fb93cce44aabb000b Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 2 Dec 2025 16:33:01 +0530 Subject: [PATCH 33/42] Delete benchmark-scripts/vlm directory --- benchmark-scripts/vlm/setup.py | 7 --- benchmark-scripts/vlm/vlm_metrics.py | 72 ---------------------------- 2 files changed, 79 deletions(-) delete mode 100644 benchmark-scripts/vlm/setup.py delete mode 100644 benchmark-scripts/vlm/vlm_metrics.py diff --git a/benchmark-scripts/vlm/setup.py b/benchmark-scripts/vlm/setup.py deleted file mode 100644 index 104265e..0000000 --- a/benchmark-scripts/vlm/setup.py +++ /dev/null @@ -1,7 +0,0 @@ -from setuptools import setup - -setup( - name='vlm_metrics', - version='0.1', - py_modules=['vlm_metrics'], -) diff --git a/benchmark-scripts/vlm/vlm_metrics.py b/benchmark-scripts/vlm/vlm_metrics.py deleted file mode 100644 index eeccbfd..0000000 --- a/benchmark-scripts/vlm/vlm_metrics.py +++ /dev/null @@ -1,72 +0,0 @@ -import datetime -import os -import sys -import uuid - -_unique_file_name = None - -def get_unique_file_name(): - global _unique_file_name - if _unique_file_name is None: - # First call: create a new unique name - timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S%f") - unique_id = uuid.uuid4().hex[:6] - _unique_file_name = f"vlm_performance_metrics_{timestamp}_{unique_id}.txt" - else: - # Subsequent calls reuse it - print(f"Reusing existing file: {_unique_file_name}") - return _unique_file_name - -def log_vlm_metrics(vlm_result): - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - filename = get_unique_file_name() - results_dir = os.getenv("CONTAINER_RESULTS_PATH") - os.makedirs(results_dir, exist_ok=True) # <--- Ensure directory exists - filepath = os.path.join(results_dir, filename) - print("The file path is: ", filepath) - with open(filepath, "a") as f: - if hasattr(vlm_result, 'perf_metrics'): - # Log format: key=value pairs separated by spaces - metrics = [ - f'Timestamp="{timestamp}"', - f'Load_Time={vlm_result.perf_metrics.get_load_time()}', - f'Generated_Tokens={vlm_result.perf_metrics.get_num_generated_tokens()}', - f'Input_Tokens={vlm_result.perf_metrics.get_num_input_tokens()}', - f'TTFT_Mean={vlm_result.perf_metrics.get_ttft().mean}', - f'TPOT_Mean={vlm_result.perf_metrics.get_tpot().mean}', - f'Throughput_Mean={vlm_result.perf_metrics.get_throughput().mean}', - f'Generate_Duration_Mean={vlm_result.perf_metrics.get_generate_duration().mean}', - f'Tokenization_Duration_Mean={vlm_result.perf_metrics.get_tokenization_duration().mean}', - f'Detokenization_Duration_Mean={vlm_result.perf_metrics.get_detokenization_duration().mean}', - f'Grammar_Compile_Max={vlm_result.perf_metrics.get_grammar_compile_time().max}', - f'Grammar_Compile_Min={vlm_result.perf_metrics.get_grammar_compile_time().min}', - f'Grammar_Compile_Std={vlm_result.perf_metrics.get_grammar_compile_time().std}', - f'Grammar_Compile_Mean={vlm_result.perf_metrics.get_grammar_compile_time().mean}' - ] - f.write(" ".join(metrics) + "\n") - f.close() - - -def get_vlm_call_average_duration(): - filename = "vlm_performance_metrics.txt" - results_dir = os.getenv("RESULTS_DIR") - filepath = os.path.join(results_dir, filename) - total_duration = 0.0 - count = 0 - with open(filepath, "r") as f: - for line in f: - parts = line.strip().split() - for part in parts: - if part.startswith("Generate_Duration_Mean="): - duration_str = part.split("=")[1] - try: - duration = float(duration_str) - total_duration += duration - count += 1 - except ValueError: - continue - if count == 0: - return 0, 0.0 - - average_duration = total_duration / count - return count, average_duration From ce18890b51fc5c7338737d11931f805562b135ad Mon Sep 17 00:00:00 2001 From: sumanaga Date: Wed, 3 Dec 2025 08:33:02 +0530 Subject: [PATCH 34/42] Update trivyimagescan.yaml --- .github/workflows/trivyimagescan.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/trivyimagescan.yaml b/.github/workflows/trivyimagescan.yaml index c29cab9..be1811d 100644 --- a/.github/workflows/trivyimagescan.yaml +++ b/.github/workflows/trivyimagescan.yaml @@ -37,12 +37,12 @@ jobs: uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 with: image-ref: 'benchmark:latest' - format: 'template' - template: '@/contrib/sarif.tpl' - output: 'trivy-results.sarif' + format: 'table' severity: 'CRITICAL,HIGH' + output: 'trivy-performance-tools-results.txt' - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@1b549b9259bda1cb5ddde3b41741a82a2d15a841 + - name: Upload Trivy scan results + uses: actions/upload-artifact@v4 with: - sarif_file: 'trivy-results.sarif' \ No newline at end of file + name: trivy-performance-tools-html-report + path: trivy-performance-tools-results.txt From 476a09731b63c098a8770ceeadf184b90476a04d Mon Sep 17 00:00:00 2001 From: Avinash Reddy Palleti Date: Wed, 3 Dec 2025 09:32:29 +0530 Subject: [PATCH 35/42] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- benchmark-scripts/consolidate_multiple_run_of_metrics.py | 2 +- benchmark-scripts/vlm_metrics_logger.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 39a1805..58ea6db 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -164,7 +164,7 @@ def extract_data(self, log_file_path): continue latency['VLM_TOTAL_CALLS'] = count - latency['VLM_AVERAGE_CALL_DURATION'] = total_duration / count + latency['VLM_AVERAGE_CALL_DURATION'] = total_duration / count if count > 0 else 0 return latency diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index cc4ef1b..5d79b02 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -171,7 +171,6 @@ def log_custom_event(self, event_type, application_name, **kwargs): def log_performance_metrics(self, vlm_metrics_result_object): timestamp_ms = int(time.time() * 1000) - vlm_metrics_result_object log_data = { 'application': 'vlm-performance', 'timestamp_ms': timestamp_ms, From d0caa64d125e247ec356194c2f62a7ad70b72266 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Jan 2026 10:23:57 +0530 Subject: [PATCH 36/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 41 ++++++++----------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index 5d79b02..0003242 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -4,7 +4,7 @@ import uuid from datetime import datetime from logging.handlers import RotatingFileHandler - +from dotenv import load_dotenv, dotenv_values class VLMMetricsLogger: @@ -23,6 +23,7 @@ def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_c self.logger = None self.performance_logger = None self._setup_logger(max_bytes, backup_count) + load_dotenv("../../.env") def _setup_logger(self, max_bytes, backup_count): """Setup the logger with file rotation""" @@ -90,17 +91,12 @@ def _setup_logger(self, max_bytes, backup_count): # Add handler to performance logger (no console output for performance) self.performance_logger.addHandler(performance_file_handler) - def log_start_time(self, application_name): - """ - Log application start time + def log_start_time(self, usecase_name): - Args: - application_name (str): Name of the application - """ timestamp_ms = int(time.time() * 1000) log_data = { - 'application': application_name, + 'application': os.getenv(usecase_name), 'event': 'start', 'timestamp_ms': timestamp_ms } @@ -112,17 +108,12 @@ def log_start_time(self, application_name): self.logger.info(message) return timestamp_ms - def log_end_time(self, application_name): - """ - Log application end time + def log_end_time(self, usecase_name): - Args: - application_name (str): Name of the application - """ timestamp_ms = int(time.time() * 1000) log_data = { - 'application': application_name, + 'application': os.getenv(usecase_name), 'event': 'end', 'timestamp_ms': timestamp_ms } @@ -134,19 +125,11 @@ def log_end_time(self, application_name): self.logger.info(message) return timestamp_ms - def log_custom_event(self, event_type, application_name, **kwargs): - """ - Log custom event with additional parameters - - Args: - event_type (str): Type of event (e.g., 'error', 'warning', 'info') - application_name (str): Name of the application - **kwargs: Additional key-value pairs to log - """ + def log_custom_event(self, event_type, usecase_name, **kwargs): timestamp_ms = int(time.time() * 1000) log_data = { - 'application': application_name, + 'application': os.getenv(usecase_name), 'event': event_type, 'timestamp_ms': timestamp_ms } @@ -168,11 +151,11 @@ def log_custom_event(self, event_type, application_name, **kwargs): return timestamp_ms - def log_performance_metrics(self, vlm_metrics_result_object): + def log_performance_metrics(self, usecase_name, vlm_metrics_result_object): timestamp_ms = int(time.time() * 1000) log_data = { - 'application': 'vlm-performance', + 'application': os.getenv(usecase_name), 'timestamp_ms': timestamp_ms, 'Load_Time' : vlm_metrics_result_object.perf_metrics.get_load_time(), 'Generated_Tokens':vlm_metrics_result_object.perf_metrics.get_num_generated_tokens(), @@ -220,6 +203,6 @@ def log_custom_event(event_type, application_name, **kwargs): """Convenience function for logging custom events""" return get_logger().log_custom_event(event_type, application_name, **kwargs) -def log_performance_metric(metrics): +def log_performance_metric(application_name,metrics): """Convenience function for logging performance metrics""" - return get_logger().log_performance_metrics(metrics) + return get_logger().log_performance_metrics(application_name,metrics) From 872de8227cab83aed9a7ab9c65fb62bc80471319 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Jan 2026 10:24:28 +0530 Subject: [PATCH 37/42] Update consolidate_multiple_run_of_metrics.py --- benchmark-scripts/consolidate_multiple_run_of_metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 58ea6db..8958d44 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -655,8 +655,8 @@ def return_blank(self): "pcm.csv":PCMExtractor, r"(?:^xpum).*\.json$": XPUMUsageExtractor, r"^qmassa.*parsed.*\.json$": QMASSAGPUUsageExtractor, - r"^vlm_application_metrics.*\.txt$": VLMAppMetricsExtractor,} - r"^vlm_performance_metrics.*\.txt$": VLMPerformanceMetricsExtractor,} + r"^vlm_application_metrics.*\.txt$": VLMAppMetricsExtractor, + r"^vlm_performance_metrics.*\.txt$": VLMPerformanceMetricsExtractor} def add_parser(): parser = argparse.ArgumentParser(description='Consolidate data') From aee5fd9a45a2bb385bc8bed99f4db5b80098e64c Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Jan 2026 12:37:06 +0530 Subject: [PATCH 38/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index 0003242..c0daa35 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -23,6 +23,8 @@ def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_c self.logger = None self.performance_logger = None self._setup_logger(max_bytes, backup_count) + cwd = os.getcwd() + print("Current working directory:", cwd) load_dotenv("../../.env") def _setup_logger(self, max_bytes, backup_count): From ff4c076dc335c1825f340cb75bdbf7d92ff54b89 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Jan 2026 12:44:16 +0530 Subject: [PATCH 39/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index c0daa35..49427a2 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -22,10 +22,8 @@ def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_c self.logger = None self.performance_logger = None - self._setup_logger(max_bytes, backup_count) - cwd = os.getcwd() - print("Current working directory:", cwd) - load_dotenv("../../.env") + self._setup_logger(max_bytes, backup_count) + load_dotenv() def _setup_logger(self, max_bytes, backup_count): """Setup the logger with file rotation""" From 4c9d290b290d2717bde48f5c53ba4f19738cd4cc Mon Sep 17 00:00:00 2001 From: sumanaga Date: Tue, 6 Jan 2026 13:28:00 +0530 Subject: [PATCH 40/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index 49427a2..2b820dc 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -23,7 +23,7 @@ def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_c self.logger = None self.performance_logger = None self._setup_logger(max_bytes, backup_count) - load_dotenv() + #load_dotenv() def _setup_logger(self, max_bytes, backup_count): """Setup the logger with file rotation""" From 280645401510219e80bd986797d8ec0d4d479656 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Wed, 7 Jan 2026 14:17:30 +0530 Subject: [PATCH 41/42] Update consolidate_multiple_run_of_metrics.py --- benchmark-scripts/consolidate_multiple_run_of_metrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/benchmark-scripts/consolidate_multiple_run_of_metrics.py b/benchmark-scripts/consolidate_multiple_run_of_metrics.py index 8958d44..ae517a6 100644 --- a/benchmark-scripts/consolidate_multiple_run_of_metrics.py +++ b/benchmark-scripts/consolidate_multiple_run_of_metrics.py @@ -96,7 +96,7 @@ def extract_data(self, log_file_path): for line in file: line = line.strip() if 'application=' in line and 'timestamp_ms=' in line: - pattern = r'(\w+)=([^\s]+)' + pattern = r'(\w+)=((?:[^\s=]+(?:\s(?!\w+=)[^\s=]+)*)?)' matches = re.findall(pattern, line) data = dict(matches) if data: @@ -131,11 +131,13 @@ def extract_data(self, log_file_path): # Generate statistics statistics = {} for app_name, duration_list in durations.items(): + count = 0 if duration_list: - statistics[f"{app_name} Total Duration (ms)"] = sum(duration_list) + count = len(duration_list) + statistics[f"Based on {count} total calls, the average {app_name} in (ms)"] = (sum(duration_list)/count) else: - statistics[f"{app_name} Total Duration (ms)"] = 0 + statistics[f"Based on {count} total calls, the average {app_name} in (ms)"] = 0 return statistics From 62327f12b816fe92e7d2f504df114960f4380ec5 Mon Sep 17 00:00:00 2001 From: sumanaga Date: Wed, 7 Jan 2026 14:18:02 +0530 Subject: [PATCH 42/42] Update vlm_metrics_logger.py --- benchmark-scripts/vlm_metrics_logger.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/benchmark-scripts/vlm_metrics_logger.py b/benchmark-scripts/vlm_metrics_logger.py index 2b820dc..f4ba047 100644 --- a/benchmark-scripts/vlm_metrics_logger.py +++ b/benchmark-scripts/vlm_metrics_logger.py @@ -4,7 +4,6 @@ import uuid from datetime import datetime from logging.handlers import RotatingFileHandler -from dotenv import load_dotenv, dotenv_values class VLMMetricsLogger: @@ -22,8 +21,7 @@ def __init__(self, log_dir=None, log_file=None, max_bytes=10*1024*1024, backup_c self.logger = None self.performance_logger = None - self._setup_logger(max_bytes, backup_count) - #load_dotenv() + self._setup_logger(max_bytes, backup_count) def _setup_logger(self, max_bytes, backup_count): """Setup the logger with file rotation"""