From 77a8da4c6b10dbc253bceabeea464162d731bad3 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 14 Nov 2025 14:11:30 +0100 Subject: [PATCH 1/7] first commit --- .../bert_large/run_mlperf.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py b/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py index 284cd47f..4f555ab4 100644 --- a/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py +++ b/natural_language_processing/extractive_question_answering/bert_large/run_mlperf.py @@ -43,6 +43,8 @@ def parse_args(): parser.add_argument("--squad_path", type=str, help="path to directory with ImageNet validation images") + parser.add_argument("--fixed_input_size", type=int, + help='size of the input') parser.add_argument("--disable_jit_freeze", action='store_true', help="if true model will be run not in jit freeze mode") return parser.parse_args() @@ -93,7 +95,7 @@ def run_tf_fp16(model_path, batch_size, num_runs, timeout, squad_path, **kwargs) return run_tf_fp(model_path, batch_size, num_runs, timeout, squad_path) -def run_pytorch_fp(model_path, batch_size, num_runs, timeout, squad_path, disable_jit_freeze=False): +def run_pytorch_fp(model_path, batch_size, num_runs, timeout, squad_path, fixed_input_size, disable_jit_freeze=False): from utils.benchmark import run_model from utils.nlp.squad import Squad_v1_1 from transformers import AutoTokenizer, BertConfig, BertForQuestionAnswering @@ -117,7 +119,11 @@ def run_single_pass(pytorch_runner, squad): padding=True, truncation=True, model_max_length=512) def tokenize(question, text): - return tokenizer(question, text, padding=True, truncation=True, return_tensors="pt") + if fixed_input_size is not None: + return tokenizer(question, text, padding="max_length", truncation=True, + max_length=fixed_input_size, return_tensors="pt") + else: + return tokenizer(question, text, padding=True, truncation=True, return_tensors="pt") def detokenize(answer): return tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(answer)) @@ -199,8 +205,9 @@ def detokenize(answer): return run_model(run_single_pass, runner, dataset, batch_size, num_runs, timeout) -def run_pytorch_fp32(model_path, batch_size, num_runs, timeout, squad_path, disable_jit_freeze, **kwargs): - return run_pytorch_fp(model_path, batch_size, num_runs, timeout, squad_path, disable_jit_freeze) +def run_pytorch_fp32(model_path, batch_size, num_runs, timeout, squad_path, fixed_input_size, disable_jit_freeze, + **kwargs): + return run_pytorch_fp(model_path, batch_size, num_runs, timeout, squad_path, fixed_input_size, disable_jit_freeze) def main(): From 0b4ac30f1111e0717468a01d535ae516a8d9bbd6 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 14 Nov 2025 14:28:04 +0100 Subject: [PATCH 2/7] wip --- tests/test_pytorch_models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index cdfda02a..2551790d 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -222,7 +222,8 @@ def wrapper(**kwargs): exact_match_ref, f1_ref = 0.750, 0.817 acc = run_process(wrapper, {"model_path": self.model_path, "squad_path": self.dataset_path, - "batch_size": 1, "num_runs": 24, "timeout": None, "disable_jit_freeze": False}) + "batch_size": 1, "num_runs": 24, "timeout": None, + "fixed_input_size": None, "disable_jit_freeze": False}) self.assertTrue(acc["exact_match"] / exact_match_ref > 0.95) self.assertTrue(acc["f1"] / f1_ref > 0.95) From 021ba509b5c9b5eca42bf87ddeeb75062bb94326 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Tue, 18 Nov 2025 21:10:17 +0100 Subject: [PATCH 3/7] wip --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 36695a38..82a3a86d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,8 +100,8 @@ jobs: wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort - wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1 - tar -xf vgg16.tar.gz > /dev/null + wget -q https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz + tar -xf vgg16.tar.gz > /dev/null 2>&1 IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort test_arm64: From 3b54e1ab33a2bf73c4e1af548812db423ebd869b Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 19 Nov 2025 14:03:36 +0100 Subject: [PATCH 4/7] wip --- .github/workflows/test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 82a3a86d..01337cfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -100,8 +100,17 @@ jobs: wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort - wget -q https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz - tar -xf vgg16.tar.gz > /dev/null 2>&1 + rm -f vgg16.tar.gz + rm -rf vgg16 + df -h + df -h + quota -s 2>/dev/null || echo "No quota command / no quotas" + + wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz + tar -tzf vgg16.tar.gz | head + + tar -xzf vgg16.tar.gz + IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort test_arm64: From 72092f98971e2877750e173503f9f31d2f429605 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 19 Nov 2025 14:47:12 +0100 Subject: [PATCH 5/7] wip --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01337cfb..f92e6d88 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,7 +106,7 @@ jobs: df -h quota -s 2>/dev/null || echo "No quota command / no quotas" - wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz + wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1 tar -tzf vgg16.tar.gz | head tar -xzf vgg16.tar.gz From 46c1f29cf20ca7cf5b1ee50f7f437fd90d85e13d Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 19 Nov 2025 15:32:29 +0100 Subject: [PATCH 6/7] wip --- .github/workflows/test.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f92e6d88..84af734f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,21 +97,21 @@ jobs: wget $S3_URL_SSD_INCEPTION_V2_TF_FP32 > /dev/null 2>&1 IGNORE_DATASET_LIMITS=1 python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -p fp32 --timeout=60 - wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 - IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort + # wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 + # IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort - rm -f vgg16.tar.gz - rm -rf vgg16 - df -h - df -h - quota -s 2>/dev/null || echo "No quota command / no quotas" + # rm -f vgg16.tar.gz + # rm -rf vgg16 + # df -h + # df -h + # quota -s 2>/dev/null || echo "No quota command / no quotas" - wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1 - tar -tzf vgg16.tar.gz | head + # wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1 + # tar -tzf vgg16.tar.gz | head - tar -xzf vgg16.tar.gz + # tar -xzf vgg16.tar.gz - IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort + # IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort test_arm64: runs-on: self-hosted From cbc5ebf4e5705c1d59fcae2730a3f43b8cd42e39 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Wed, 19 Nov 2025 16:34:54 +0100 Subject: [PATCH 7/7] wip --- .github/workflows/test.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 84af734f..f086b591 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -97,20 +97,11 @@ jobs: wget $S3_URL_SSD_INCEPTION_V2_TF_FP32 > /dev/null 2>&1 IGNORE_DATASET_LIMITS=1 python3 computer_vision/object_detection/ssd_inception_v2/run.py -m ssd_inception_v2_tf_fp32.pb -p fp32 --timeout=60 - # wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 - # IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort - - # rm -f vgg16.tar.gz - # rm -rf vgg16 - # df -h - # df -h - # quota -s 2>/dev/null || echo "No quota command / no quotas" + wget https://zenodo.org/records/4735647/files/resnet50_v1.onnx > /dev/null 2>&1 + IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/resnet_50_v1/run.py -m resnet50_v1.onnx -p fp32 -f ort # wget https://s3.amazonaws.com/onnx-model-zoo/vgg/vgg16/vgg16.tar.gz > /dev/null 2>&1 - # tar -tzf vgg16.tar.gz | head - - # tar -xzf vgg16.tar.gz - + # tar -xf vgg16.tar.gz > /dev/null # IGNORE_DATASET_LIMITS=1 python3 computer_vision/classification/vgg_16/run.py -m vgg16/vgg16.onnx -p fp32 -f ort test_arm64: