CIV: Report Portal Integration with AMI Validation

README.md

@@ -82,6 +82,8 @@ options:
   --tags TAGS           Use this option to add tags to created cloud resources and modify CIV behaviour.
                         This tags should be passed in json format as in this example:
                         --tags '{"key1": "value1", "key2": "value2"}'
+  -rp, --report-portal  Use this option to upload the JUnit XML test results to your Report Portal project.
+                        Make sure to set the correct environment variables as explained in the README doc.
 ```
 
 Example of running CIV locally:

cloud-image-val.py

@@ -3,9 +3,10 @@
 from pprint import pprint
 from argparse import ArgumentParser, RawTextHelpFormatter
 from main.cloud_image_validator import CloudImageValidator
-from lib.config_lib import CIVConfig
+from lib.config_lib import CIVConfig, PytestConfig
 from lib import console_lib
 
+
 parser = ArgumentParser(formatter_class=RawTextHelpFormatter)
 
 parser.add_argument('-r', '--resources-file',
@@ -54,6 +55,11 @@
                          'This tags should be passed in json format as in this example:\n'
                          '--tags \'{"key1": "value1", "key2": "value2"}\'',
                     default=None)
+parser.add_argument('-rp', '--report-portal',
+                    help='Use this option to upload the JUnit XML test results to your Report Portal project.\n'
+                         'Make sure to set the correct environment variables as explained in the README doc.',
+                    action='store_true',
+                    default=None)
 
 if __name__ == '__main__':
     args = parser.parse_args()
@@ -64,6 +70,9 @@
     os.environ['PYTHONPATH'] = ':'.join(
         [f'{os.path.dirname(__file__)}', os.environ['PYTHONPATH']])
 
+    rp_config = PytestConfig()
+    rp_config.set_report_portal_config()
+
     config_manager = CIVConfig(args)
 
     config_manager.update_config()

lib/config_lib.py

@@ -1,6 +1,24 @@
 import os.path
 
 import yaml
+import configparser
+
+
+class PytestConfig:
+    def set_report_portal_config(self):
+        # Get the configuration from the env
+        rp_api_key = os.getenv('RP_API_KEY')
+        rp_endpoint = os.getenv('RP_ENDPOINT')
+        rp_project = os.getenv('RP_PROJECT')
+
+        config = configparser.ConfigParser()
+        config.read('pytest.ini')
+        config.set('pytest', 'rp_api_key', rp_api_key)
+        config.set('pytest', 'rp_endpoint', rp_endpoint)
+        config.set('pytest', 'rp_project', rp_project)
+
+        with open('pytest.ini', 'w') as configfile:
+            config.write(configfile)
 
 
 class CIVConfig:
@@ -87,6 +105,7 @@ def get_default_config(self):
             'parallel': False,
             'stop_cleanup': None,
             'test_filter': None,
+            'report_portal': None
         }
 
         return config_defaults

main/cloud_image_validator.py

@@ -114,7 +114,8 @@ def run_tests_in_all_instances(self, instances):
 
         return runner.run_tests(self.config['output_file'],
                                 self.config['test_filter'],
-                                self.config['include_markers'])
+                                self.config['include_markers'],
+                                self.config['report_portal'])
 
     def cleanup(self):
         self.infra_controller.destroy_infra()

pytest.ini

@@ -1,8 +1,10 @@
 [pytest]
 render_collapsed = True
-markers =
-    pub: Tests that have to be run on images that are already published into cloud providers marketplaces.
-    wait: Number of seconds the test should wait before it starts.
-    run_on: List of distro-version (or only distro) combinations where the test should be executed.
-    exclude_on: List of distro-version (or only distro) combinations where the test should NOT be executed.
-    jira_skip: List of Jira ticker ids. If any of these tickets are not closed, the test will be skipped.
+markers = 
+	pub: Tests that have to be run on images that are already published into cloud providers marketplaces.
+	wait: Number of seconds the test should wait before it starts.
+	run_on: List of distro-version (or only distro) combinations where the test should be executed.
+	exclude_on: List of distro-version (or only distro) combinations where the test should NOT be executed.
+	jira_skip: List of Jira ticker ids. If any of these tickets are not closed, the test will be skipped.
+rp_verify_ssl = False
+rp_is_skipped_an_issue = False

requirements.txt

@@ -16,3 +16,4 @@ paramiko==3.1.0
 awscli==1.27.119
 requests==2.28.1
 packaging==23.2
+pytest-reportportal==5.3.0

test/test_cloud_image_validator.py

@@ -15,7 +15,8 @@ class TestCloudImageValidator:
                    'parallel': True,
                    'debug': True,
                    'stop_cleanup': False,
-                   'config_file': '/tmp/test_config_file.yml',
+                   'report_portal': False,
+                   'config_file': '/tmp/test_config_file.yml'
                    }
     test_instances = {
         'instance-1': {'public_dns': 'value_1', 'username': 'value_2'},
@@ -134,7 +135,8 @@ def test_run_tests_in_all_instances(self, mocker, validator):
         validator.run_tests_in_all_instances(self.test_instances)
 
         mock_run_tests.assert_called_once_with(
-            validator.config["output_file"], self.test_config["test_filter"], self.test_config["include_markers"])
+            validator.config["output_file"], self.test_config["test_filter"], self.test_config["include_markers"],
+            self.test_config["report_portal"])
 
     def test_destroy_infrastructure(self, mocker, validator):
         mock_destroy_infra = mocker.patch.object(

test/test_suite_runner.py

@@ -11,6 +11,7 @@ class TestSuiteRunner:
     test_filter = 'test_test_name'
     test_marker = 'pub'
     test_connection = 'paramiko'
+    test_report_portal = ''
 
     @pytest.fixture
     def suite_runner(self):
@@ -31,7 +32,7 @@ def test_run_tests(self, mocker, suite_runner):
         mock_os_system = mocker.patch('os.system')
 
         # Act
-        suite_runner.run_tests(test_output_filepath, self.test_filter, self.test_marker)
+        suite_runner.run_tests(test_output_filepath, self.test_filter, self.test_marker, self.test_report_portal)
 
         # Assert
         mock_os_path_exists.assert_called_once_with(test_output_filepath)
@@ -40,18 +41,19 @@ def test_run_tests(self, mocker, suite_runner):
         mock_os_system.assert_called_once_with(test_composed_command)
         mock_compose_testinfra_command.assert_called_once_with(test_output_filepath,
                                                                self.test_filter,
-                                                               self.test_marker)
+                                                               self.test_marker,
+                                                               self.test_report_portal)
 
     @pytest.mark.parametrize(
-        'test_filter, test_marker, test_debug, test_parallel, expected_command_string',
-        [(None, None, False, False,
+        'test_filter, test_marker, test_debug, test_parallel, test_report_portal, expected_command_string',
+        [(None, None, False, False, False,
           'pytest path1 path2 --hosts=user1@host1,user2@host2 '
           f'--connection={test_connection} '
           f'--ssh-config {test_ssh_config} --junit-xml {test_output_filepath} '
           f'--html {test_output_filepath.replace("xml", "html")} '
           f'--self-contained-html '
           f'--json-report --json-report-file={test_output_filepath.replace("xml", "json")}'),
-         (test_filter, test_marker, False, False,
+         (test_filter, test_marker, False, False, False,
           'pytest path1 path2 --hosts=user1@host1,user2@host2 '
           f'--connection={test_connection} '
           f'--ssh-config {test_ssh_config} --junit-xml {test_output_filepath} '
@@ -60,7 +62,7 @@ def test_run_tests(self, mocker, suite_runner):
           f'--json-report --json-report-file={test_output_filepath.replace("xml", "json")} '
           f'-k "{test_filter}" '
           f'-m "{test_marker}"'),
-         (None, None, False, True,
+         (None, None, False, True, False,
           'pytest path1 path2 --hosts=user1@host1,user2@host2 '
           f'--connection={test_connection} '
           f'--ssh-config {test_ssh_config} --junit-xml {test_output_filepath} '
@@ -71,7 +73,7 @@ def test_run_tests(self, mocker, suite_runner):
           '--only-rerun="socket.timeout|refused|ConnectionResetError|TimeoutError|SSHException|NoValidConnectionsError'
           '|Error while installing Development tools group" '
           '--reruns 3 --reruns-delay 5'),
-         (None, None, True, True,
+         (None, None, True, True, False,
           'pytest path1 path2 --hosts=user1@host1,user2@host2 '
           f'--connection={test_connection} '
           f'--ssh-config {test_ssh_config} --junit-xml {test_output_filepath} '
@@ -91,6 +93,7 @@ def test_compose_testinfra_command(self,
                                        test_marker,
                                        test_debug,
                                        test_parallel,
+                                       test_report_portal,
                                        expected_command_string):
         # Arrange
         test_hosts = 'user1@host1,user2@host2'
@@ -108,7 +111,8 @@ def test_compose_testinfra_command(self,
         # Act, Assert
         assert suite_runner.compose_testinfra_command(self.test_output_filepath,
                                                       test_filter,
-                                                      test_marker) == expected_command_string
+                                                      test_marker,
+                                                      test_report_portal) == expected_command_string
 
         mock_get_all_instances_hosts_with_users.assert_called_once()
 

test_suite/conftest.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 import re
 import time
@@ -8,12 +9,21 @@
 from packaging import version
 from py.xml import html
 from pytest_html import extras
+from reportportal_client import RPLogger
 from requests.adapters import HTTPAdapter
 from requests.packages.urllib3.util.retry import Retry
 
 from lib import test_lib
 
 
+@pytest.fixture(scope="session")
+def rp_logger():
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    logging.setLoggerClass(RPLogger)
+    return logger
+
+
 def __get_host_info(host):
     host_info = {}
     host_info['distro'] = host.system_info.distribution

test_suite/suite_runner.py

@@ -34,18 +34,21 @@ def __init__(self,
     def run_tests(self,
                   output_filepath,
                   test_filter=None,
-                  include_markers=None):
+                  include_markers=None,
+                  report_portal=None):
         if os.path.exists(output_filepath):
             os.remove(output_filepath)
 
         return os.system(self.compose_testinfra_command(output_filepath,
                                                         test_filter,
-                                                        include_markers))
+                                                        include_markers,
+                                                        report_portal))
 
     def compose_testinfra_command(self,
                                   output_filepath,
                                   test_filter,
-                                  include_markers):
+                                  include_markers,
+                                  report_portal):
         all_hosts = self.get_all_instances_hosts_with_users()
 
         command_with_args = [
@@ -77,6 +80,9 @@ def compose_testinfra_command(self,
         if self.debug:
             command_with_args.append('-v')
 
+        if report_portal:
+            command_with_args.append('--reportportal')
+
         return ' '.join(command_with_args)
 
     def get_test_suite_paths(self):