From b24d8bdf4b041460643228eff1fe8ae94874a565 Mon Sep 17 00:00:00 2001 From: Pavel Holica Date: Tue, 29 Aug 2023 09:37:06 +0200 Subject: [PATCH 1/5] Add --showarch option to report command This option adds new Arch column to the report table. --- CHANGELOG.md | 1 + README.md | 3 ++- src/dispatch/__init__.py | 5 +++++ src/report/__main__.py | 14 +++++++++++--- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f810a5..43fb027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Calendar Versioning](https://calver.org).
### Added - Return code for report command - Report command option for downloading logs only for non-passed tests +- `--showarch` option in `report` command displaying architecture in the table ### Changed - `-c/--compose` to `-t/--target` diff --git a/README.md b/README.md index 6aa0d19..ed221f9 100644 --- a/README.md +++ b/README.md @@ -206,12 +206,13 @@ Corresponding return code is set based on the results with following logic: ```shell ❯ tesar report --help -usage: tesar report [-h] [-l2] [-s] [-stn SPLIT_TESTNAME] [-spn SPLIT_PLANNAME] [-w] [-d] [--skip-pass] [-lt | -f FILE | -c CMD] +usage: tesar report [-h] [--showarch] [-l2] [-s] [-stn SPLIT_TESTNAME] [-spn SPLIT_PLANNAME] [-w] [-d] [--skip-pass] [-lt | -f FILE | -c CMD] Parses task IDs, Testing Farm artifact URLs or Testing Farm API request URLs from multiple sources. options: -h, --help show this help message and exit + --showarch Display architecture. By default the architecture is not shown. -l2, --level2 Display test view detail. By default the report shows only plan view. -s, --short Display short test and plan names. -stn SPLIT_TESTNAME, --split-testname SPLIT_TESTNAME diff --git a/src/dispatch/__init__.py b/src/dispatch/__init__.py index 5c00050..4092572 100644 --- a/src/dispatch/__init__.py +++ b/src/dispatch/__init__.py @@ -254,6 +254,11 @@ def get_arguments(args=None): description="Parses task IDs, Testing Farm artifact URLs or Testing Farm API request URLs from multiple sources.", ) + report.add_argument( + "--showarch", + action="store_true", + help="""Display architecture. By default the architecture is not shown.""", + ) report.add_argument( "-l2", "--level2", diff --git a/src/report/__main__.py b/src/report/__main__.py index e462099..196fb25 100644 --- a/src/report/__main__.py +++ b/src/report/__main__.py @@ -123,6 +123,7 @@ def parse_request_xunit(request_url_list=None, tasks_source=None, skip_pass=Fals request_state = request.json()["state"].upper() request_uuid = request.json()["id"] request_target = request.json()["environments_requested"][0]["os"]["compose"] + request_arch = request.json()["environments_requested"][0]['arch'] request_datetime_created = request.json()["created"] request_datetime_parsed = request_datetime_created.split(".")[0] @@ -209,6 +210,7 @@ def parse_request_xunit(request_url_list=None, tasks_source=None, skip_pass=Fals if request_uuid not in parsed_dict: parsed_dict[request_uuid] = { "target_name": request_target, + "arch": request_arch, "testsuites": [], } @@ -277,7 +279,10 @@ def build_table(): result_table = PrettyTable() # prepare field names fields = [] - fields += ["UUID", "Target", "Test Plan"] + fields += ["UUID", "Target"] + if ARGS.showarch: + fields += ["Arch"] + fields += ["Test Plan"] if ARGS.level2: fields += ["Test Case"] fields += ["Result"] @@ -298,11 +303,14 @@ def build_table(): if ARGS.split_planname: planname_split_index = ARGS.split_planname - def _gen_row(uuid="", target="", testplan="", testcase="", result=""): + + def _gen_row(uuid="", target="", arch="", testplan="", testcase="", result=""): if "UUID" in fields: yield uuid if "Target" in fields: yield target + if "Arch" in fields: + yield arch if "Test Plan" in fields: yield testplan if "Test Case" in fields: @@ -314,7 +322,7 @@ def add_row(*args, **kwargs): result_table.add_row(tuple(_gen_row(*args, **kwargs))) for task_uuid, data in parsed_dict.items(): - add_row(task_uuid, data["target_name"]) + add_row(task_uuid, data["target_name"], data["arch"]) for testsuite_data in data["testsuites"]: testsuite_name_raw = testsuite_data["testsuite_name"].split("/") testsuite_name_raw.remove("") From 78fe86785a46f9099140eea94782dc4fe11e920f Mon Sep 17 00:00:00 2001 From: Pavel Holica Date: Thu, 27 Jul 2023 14:25:59 +0200 Subject: [PATCH 2/5] WIP: add stats option to aggregate results --- src/dispatch/__init__.py | 5 +++++ src/report/__main__.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/dispatch/__init__.py b/src/dispatch/__init__.py index 4092572..4b3871a 100644 --- a/src/dispatch/__init__.py +++ b/src/dispatch/__init__.py @@ -283,6 +283,11 @@ def get_arguments(args=None): type=int, help="""Specify an index from which will the plan name be shown. Passed to planname.split('/')[index:]""", ) + report.add_argument( + '--stats', + action='store_true', + help='Provide statistics how many times the same test plan and the same tests passed, failed or errored out.', + ) report.add_argument( "-w", "--wait", diff --git a/src/report/__main__.py b/src/report/__main__.py index 196fb25..0630dd3 100644 --- a/src/report/__main__.py +++ b/src/report/__main__.py @@ -279,7 +279,9 @@ def build_table(): result_table = PrettyTable() # prepare field names fields = [] - fields += ["UUID", "Target"] + if not ARGS.stats: + fields += ["UUID"] + fields += ["Target"] if ARGS.showarch: fields += ["Arch"] fields += ["Test Plan"] From 75809c936e6ae215350ea986d2a5d07233972b9a Mon Sep 17 00:00:00 2001 From: Pavel Holica Date: Fri, 25 Aug 2023 10:30:19 +0200 Subject: [PATCH 3/5] WIP: Make stats! Always! --- src/report/__main__.py | 52 +++++++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 8 deletions(-) diff --git a/src/report/__main__.py b/src/report/__main__.py index 0630dd3..b51c8ad 100644 --- a/src/report/__main__.py +++ b/src/report/__main__.py @@ -273,8 +273,24 @@ def parse_request_xunit(request_url_list=None, tasks_source=None, skip_pass=Fals return parsed_dict +def make_stats(parsed_dict): + testsuite_stats = {} # [target_name, testsuite_name] = [result, ...] + testcase_stats = {} # [target_name, testsuite_name, testcase_name] = [result, ...] + for run in parsed_dict.values(): + target_name = run['target_name'] + for testsuite in run['testsuites']: + testsuite_name = testsuite['testsuite_name'] + testsuite_stats.setdefault((target_name, testsuite_name), []) + testsuite_stats[(target_name, testsuite_name)].append(testsuite['testsuite_result']) + for testcase in testsuite['testcases']: + testcase_name = testcase['testcase_name'] + testcase_stats.setdefault((target_name, testsuite_name, testcase_name), []) + testcase_stats[(target_name, testsuite_name, testcase_name)].append(testcase['testcase_result']) + return {'testsuite' : testsuite_stats, 'testcase' : testcase_stats} + def build_table(): parsed_dict = parse_request_xunit(skip_pass=ARGS.skip_pass) + stats = make_stats(parsed_dict) result_table = PrettyTable() # prepare field names @@ -323,6 +339,12 @@ def _gen_row(uuid="", target="", arch="", testplan="", testcase="", result=""): def add_row(*args, **kwargs): result_table.add_row(tuple(_gen_row(*args, **kwargs))) + def stat_result(target_name, testsuite_name, testcase_name=None): + if testcase_name: + return stats['testcase'][target_name, testsuite_name, testcase_name] + return stats['testsuite'][target_name, testsuite_name] + + shown = set() for task_uuid, data in parsed_dict.items(): add_row(task_uuid, data["target_name"], data["arch"]) for testsuite_data in data["testsuites"]: @@ -330,20 +352,34 @@ def add_row(*args, **kwargs): testsuite_name_raw.remove("") testsuite_name = "/".join(testsuite_name_raw[planname_split_index:]) testsuite_result = testsuite_data["testsuite_result"] - add_row( - testplan=colorize(testsuite_result, testsuite_name), - result=colorize(testsuite_result), - ) + stats_desc = data['target_name'], testsuite_data["testsuite_name"] + if stats_desc not in shown: + add_row( + testplan=colorize(testsuite_result, testsuite_name), + result=stat_result(*stats_desc) + ) + shown.add(stats_desc) + #add_row( + # testplan=colorize(testsuite_result, testsuite_name), + # result=colorize(testsuite_result) + #) if "Test Case" in fields: for testcase in testsuite_data["testcases"]: testcase_name_raw = testcase["testcase_name"].split("/") testcase_name_raw.remove("") testcase_name = "/".join(testcase_name_raw[testname_split_index:]) testcase_result = testcase["testcase_result"] - add_row( - testcase=colorize(testcase_result, testcase_name), - result=colorize(testcase_result), - ) + stats_desc = data['target_name'], testsuite_data["testsuite_name"], testcase["testcase_name"] + if stats_desc not in shown: + add_row( + testcase=colorize(testcase_result, testcase_name), + result=stat_result(*stats_desc) + ) + shown.add(stats_desc) + #add_row( + # testcase=colorize(testcase_result, testcase_name), + # result=colorize(testcase_result) + #) result_table.align = "l" From b6feaaa68e111d3f38ce3ecd7bde3d3cb93191a2 Mon Sep 17 00:00:00 2001 From: Pavel Holica Date: Wed, 30 Aug 2023 10:04:34 +0200 Subject: [PATCH 4/5] Better formatting of stats --- src/report/__main__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/report/__main__.py b/src/report/__main__.py index b51c8ad..8df2f70 100644 --- a/src/report/__main__.py +++ b/src/report/__main__.py @@ -341,8 +341,15 @@ def add_row(*args, **kwargs): def stat_result(target_name, testsuite_name, testcase_name=None): if testcase_name: - return stats['testcase'][target_name, testsuite_name, testcase_name] - return stats['testsuite'][target_name, testsuite_name] + result = stats['testcase'][target_name, testsuite_name, testcase_name] + else: + result = stats['testsuite'][target_name, testsuite_name] + return ( + colorize('PASSED', f"{result.count('PASSED')}") + "," + + colorize('FAILED', f"{result.count('FAILED')}") + "," + + colorize('ERROR', f"{result.count('ERROR')}") + + f"/{len(result)}" + ) shown = set() for task_uuid, data in parsed_dict.items(): From 564d94ddbd6fb773ef573d87f256ad2ca5aad1da Mon Sep 17 00:00:00 2001 From: Pavel Holica Date: Wed, 30 Aug 2023 14:18:56 +0200 Subject: [PATCH 5/5] Drop colors from plans and tests --- src/report/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/report/__main__.py b/src/report/__main__.py index 8df2f70..0e53308 100644 --- a/src/report/__main__.py +++ b/src/report/__main__.py @@ -362,7 +362,7 @@ def stat_result(target_name, testsuite_name, testcase_name=None): stats_desc = data['target_name'], testsuite_data["testsuite_name"] if stats_desc not in shown: add_row( - testplan=colorize(testsuite_result, testsuite_name), + testplan=testsuite_name, result=stat_result(*stats_desc) ) shown.add(stats_desc) @@ -379,7 +379,7 @@ def stat_result(target_name, testsuite_name, testcase_name=None): stats_desc = data['target_name'], testsuite_data["testsuite_name"], testcase["testcase_name"] if stats_desc not in shown: add_row( - testcase=colorize(testcase_result, testcase_name), + testcase=testcase_name, result=stat_result(*stats_desc) ) shown.add(stats_desc)