diff --git a/daily_tests/daily_nightly_tests_report.py b/daily_tests/daily_nightly_tests_report.py index 29277a8..e27e3d0 100755 --- a/daily_tests/daily_nightly_tests_report.py +++ b/daily_tests/daily_nightly_tests_report.py @@ -3,8 +3,8 @@ import sys import smtplib import argparse +import subprocess -from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from pathlib import Path @@ -103,6 +103,46 @@ SCLORG_DIR = "/var/tmp/daily_scl_tests" +def run_command( + cmd, + return_output: bool = True, + ignore_error: bool = False, + shell: bool = True, + **kwargs, +): + """ + Run provided command on host system using the same user as invoked this code. + Raises subprocess.CalledProcessError if it fails. + :param cmd: list or str + :param return_output: bool, return output of the command + :param ignore_error: bool, do not fail in case nonzero return code + :param shell: bool, run command in shell + :param debug: bool, print command in shell, default is suppressed + :return: None or str + """ + print(f"command: {cmd}") + try: + if return_output: + return subprocess.check_output( + cmd, + stderr=subprocess.STDOUT, + universal_newlines=True, + shell=shell, + **kwargs, + ) + else: + return subprocess.check_call(cmd, shell=shell, **kwargs) + except subprocess.CalledProcessError as cpe: + if ignore_error: + if return_output: + return cpe.output + else: + return cpe.returncode + else: + print(f"failed with code {cpe.returncode} and output:\n{cpe.output}") + raise cpe + + class NightlyTestsReport(object): def __init__(self): self.tmp_path_dir: Path @@ -156,6 +196,16 @@ def prepare(self) -> bool: self.log_dir = self.args.log_dir return True + def send_file_to_pastebin(self, log_path, log_name: str): + if not os.path.exists(log_name): + return + send_paste_bin = os.getenv("HOME") + "/ci-scripts/send_to_paste_bin.sh" + cmd = f'{send_paste_bin} "{log_path}" "{log_name}"' + try: + run_command(cmd) + except subprocess.CalledProcessError: + pass + def get_pastebin_url(self, log_name: str) -> str: with open(log_name, "r") as f: lines = f.read() @@ -183,33 +233,29 @@ def collect_data(self): path_dir = Path(RESULTS_DIR) / test_case if not path_dir.is_dir(): print(f"The test case {path_dir} does not exists that is weird") - self.data_dict["tmt"]["logs"].append( - f"Nightly build tests for {test_case} is not finished or did not run. " - f"Check nightly build machine for logs." + continue + # It looks like TMT is still running for long time + if (path_dir / "tmt_running").exists(): + self.data_dict["tmt"]["msg"].append( + f"tmt tests for case {test_case} is still running." + f"Look at log in attachment called '{test_case}-log.txt'." ) + self.data_dict["tmt"]["tmt_running"].append(test_case) for sclorg in ["S2I", "NOS2I"]: name = f"{test_case}-{sclorg}" + self.send_file_to_pastebin( + log_path=Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", + log_name=f"{RESULTS_DIR}/{name}.txt", + ) self.data_dict["tmt"]["logs"].append( ( name, Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", - f"{name}.txt", + Path(RESULTS_DIR) / f"{name}.txt", ) ) failed_tests = True continue - # It looks like TMT is still running for long time - if (path_dir / "tmt_running").exists(): - self.data_dict["tmt"]["msg"].append( - f"tmt tests for case {test_case} is still running." - f"Look at log in attachment called '{test_case}-log.txt'." - ) - self.data_dict["tmt"]["tmt_running"].append(test_case) - self.data_dict["tmt"]["logs"].append( - (test_case, path_dir / "log.txt", "log.txt") - ) - failed_tests = True - continue # TMT command failed for some reason. Look at logs for given namespace # /var/tmp/daily_scl_tests//log.txt file if (path_dir / "tmt_failed").exists(): @@ -218,9 +264,19 @@ def collect_data(self): f"Look at log in attachment called '{test_case}-log.txt'." ) self.data_dict["tmt"]["tmt_failed"].append(test_case) - self.data_dict["tmt"]["logs"].append( - (test_case, path_dir / "log.txt", "log.txt") - ) + for sclorg in ["S2I", "NOS2I"]: + name = f"{test_case}-{sclorg}" + self.send_file_to_pastebin( + log_path=Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", + log_name=f"{RESULTS_DIR}/{name}.txt", + ) + self.data_dict["tmt"]["logs"].append( + ( + name, + Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", + Path(RESULTS_DIR) / f"{name}.txt", + ) + ) failed_tests = True continue data_dir = path_dir / "plans" / plan / "data" @@ -229,9 +285,19 @@ def collect_data(self): f"Data dir for test case {test_case} does not exist." f"Look at log in attachment called '{test_case}-log.txt'." ) - self.data_dict["tmt"]["logs"].append( - (test_case, path_dir / "log.txt", "log.txt") - ) + for sclorg in ["S2I", "NOS2I"]: + name = f"{test_case}-{sclorg}" + self.send_file_to_pastebin( + log_path=Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", + log_name=f"{RESULTS_DIR}/{name}.txt", + ) + self.data_dict["tmt"]["logs"].append( + ( + name, + Path(SCLORG_DIR) / f"{test_case}-{sclorg}" / "log.txt", + Path(RESULTS_DIR) / f"{name}.txt", + ) + ) failed_tests = True continue results_dir = data_dir / "results" @@ -266,7 +332,7 @@ def generate_email_body(self): body_failure = "Nightly builds Testing Farm failures:
" body_success = "These nightly builds were completely successful:
" # Function for generation mail body - if self.data_dict["tmt"]["logs"]: + if self.data_dict["tmt"]["msg"]: self.body += ( f"{body_failure}\n" f"Tests were not successful because Testing Farm failures. " @@ -305,22 +371,27 @@ def generate_failed_containers(self): def generate_success_containers(self): for test_case, cont_path, log_name in self.data_dict["SUCCESS_DATA"]: - mime_name = f"{test_case}-{log_name}" - if os.path.exists(cont_path): - attach = MIMEApplication(open(cont_path, "r").read(), Name=mime_name) - attach.add_header( - "Content-Disposition", 'attachment; filename="{}"'.format(mime_name) + if os.path.exists(log_name): + self.body += ( + f" See logs" + f"
" ) - self.mime_msg.attach(attach) def generate_tmt_logs_containers(self): for test_case, cont_path, log_name in self.data_dict["tmt"]["logs"]: - print(cont_path, log_name) - if os.path.exists(cont_path): + print(test_case, cont_path, log_name) + if os.path.exists(log_name): + self.body += ( + f"{test_case} " + f"See logs" + f"
" + ) + else: self.body += ( - f"{test_case}" - f"

" + f"{test_case} No logs available. " + f"Check nightly build machine
" ) + self.body += "
" def generate_emails(self): for test_case, plan, _ in self.available_test_case: diff --git a/send_to_paste_bin.sh b/send_to_paste_bin.sh new file mode 100755 index 0000000..196f888 --- /dev/null +++ b/send_to_paste_bin.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -x + +/usr/local/bin/pbincli send --server https://privatebin.corp.redhat.com --expire 1week --no-insecure-warning --no-check-certificate --format plaintext < $1 > $2 2>&1