From 0cf2c658eee9371a4a54151c9d1c01303e106adb Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 13 Aug 2025 18:40:57 +0200 Subject: [PATCH 1/4] Remote: Automatically manage known hosts keys --- pysqa/base/remote.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pysqa/base/remote.py b/pysqa/base/remote.py index b0db8ca..11dd48c 100644 --- a/pysqa/base/remote.py +++ b/pysqa/base/remote.py @@ -93,9 +93,12 @@ def __init__( ) self._ssh_host = config["ssh_host"] self._ssh_username = config["ssh_username"] - self._ssh_known_hosts = os.path.abspath( - os.path.expanduser(config["known_hosts"]) - ) + if "known_hosts" in config: + self._ssh_known_hosts = os.path.abspath( + os.path.expanduser(config["known_hosts"]) + ) + else: + self._ssh_known_hosts = None self._ssh_ask_for_password: Union[bool, str] = False self._ssh_key = ( os.path.abspath(os.path.expanduser(config["ssh_key"])) @@ -346,7 +349,10 @@ def _open_ssh_connection(self) -> paramiko.SSHClient: paramiko.SSHClient: The SSH connection object. """ ssh = paramiko.SSHClient() - ssh.load_host_keys(self._ssh_known_hosts) + if self._ssh_known_hosts is not None: + ssh.load_host_keys(self._ssh_known_hosts) + else: + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if ( self._ssh_key is not None and self._ssh_key_passphrase is not None From 41a42a01f82081962dbcc399d198ea1fac05ed0d Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 13 Aug 2025 18:45:24 +0200 Subject: [PATCH 2/4] work with strings --- pysqa/base/remote.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysqa/base/remote.py b/pysqa/base/remote.py index 11dd48c..1d7e42c 100644 --- a/pysqa/base/remote.py +++ b/pysqa/base/remote.py @@ -98,7 +98,7 @@ def __init__( os.path.expanduser(config["known_hosts"]) ) else: - self._ssh_known_hosts = None + self._ssh_known_hosts = "" self._ssh_ask_for_password: Union[bool, str] = False self._ssh_key = ( os.path.abspath(os.path.expanduser(config["ssh_key"])) @@ -349,7 +349,7 @@ def _open_ssh_connection(self) -> paramiko.SSHClient: paramiko.SSHClient: The SSH connection object. """ ssh = paramiko.SSHClient() - if self._ssh_known_hosts is not None: + if len(self._ssh_known_hosts) > 0: ssh.load_host_keys(self._ssh_known_hosts) else: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) From 8868a61849ca1af96edef653251e65c094406c6a Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 13 Aug 2025 18:51:18 +0200 Subject: [PATCH 3/4] add more tests --- tests/config/remote_rebex_hosts/queue.yaml | 14 +++++++++++ tests/test_remote.py | 27 ++++++++++++++++++---- tests/test_remote_auth.py | 6 ++--- 3 files changed, 39 insertions(+), 8 deletions(-) create mode 100644 tests/config/remote_rebex_hosts/queue.yaml diff --git a/tests/config/remote_rebex_hosts/queue.yaml b/tests/config/remote_rebex_hosts/queue.yaml new file mode 100644 index 0000000..8fd3923 --- /dev/null +++ b/tests/config/remote_rebex_hosts/queue.yaml @@ -0,0 +1,14 @@ +queue_type: REMOTE +queue_primary: remote +ssh_host: test.rebex.net +ssh_username: demo +known_hosts: ~/.ssh/known_hosts +ssh_password: password +ssh_remote_config_dir: / +ssh_remote_path: / +ssh_local_path: /home/localuser/projects/ +ssh_continous_connection: False +ssh_port: 22 +python_executable: python3 +queues: + remote: {cores_max: 100, cores_min: 10, run_time_max: 259200} diff --git a/tests/test_remote.py b/tests/test_remote.py index 7f3a000..be399b9 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -151,36 +151,53 @@ def test_remote_command_continous_connection(self): output = remote._adapter._execute_remote_command(command="pwd") self.assertEqual(output, "/\n") + def test_remote_command_individual_connections_hosts(self): + path = os.path.dirname(os.path.abspath(__file__)) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) + remote._adapter._ssh_remote_path = path + remote._adapter._open_ssh_connection() + output = remote._adapter._execute_remote_command(command="pwd") + self.assertEqual(output, "/\n") + + def test_remote_command_continous_connection_hosts(self): + path = os.path.dirname(os.path.abspath(__file__)) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) + remote._adapter._ssh_remote_path = path + remote._adapter._ssh_continous_connection = True + remote._adapter._open_ssh_connection() + output = remote._adapter._execute_remote_command(command="pwd") + self.assertEqual(output, "/\n") + def test_submit_job(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_remote_path = path output = remote._adapter.submit_job(working_directory=os.path.join(path, "config/empty"), command="echo 1") self.assertEqual(output, 1) def test_transferfile_individual_connections(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_remote_path = path self.assertIsNone(remote._adapter.transfer_file(file="readme.txt", transfer_back=True)) def test_transferfile_continous_connection(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_remote_path = path remote._adapter._ssh_continous_connection = True self.assertIsNone(remote._adapter.transfer_file(file="readme.txt", transfer_back=True)) def test_get_transport(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) self.assertIsNotNone(get_transport(remote._adapter._open_ssh_connection())) with self.assertRaises(ValueError): get_transport(ssh=FakeSSH()) def test_get_job_from_remote(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_remote_path = path remote._adapter._ssh_local_path = path remote._adapter._ssh_delete_file_on_remote = True diff --git a/tests/test_remote_auth.py b/tests/test_remote_auth.py index 2446940..0a7f266 100644 --- a/tests/test_remote_auth.py +++ b/tests/test_remote_auth.py @@ -19,21 +19,21 @@ class TestRemoteQueueAdapterAuth(unittest.TestCase): def test_password_auth(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_ask_for_password = False remote._adapter._ssh_key = None self.assertIsNotNone(remote._adapter._open_ssh_connection()) def test_key_auth(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_password = None with self.assertRaises(ValueError): remote._adapter._open_ssh_connection() def test_two_factor_auth(self): path = os.path.dirname(os.path.abspath(__file__)) - remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex")) + remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts")) remote._adapter._ssh_two_factor_authentication = True with self.assertRaises(paramiko.ssh_exception.AuthenticationException): remote._adapter._open_ssh_connection() From 6dd12d05e2959f6b6d3e56c5be037be3908df192 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 13 Aug 2025 18:58:32 +0200 Subject: [PATCH 4/4] fixes --- tests/config/remote_rebex/queue.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/config/remote_rebex/queue.yaml b/tests/config/remote_rebex/queue.yaml index 8fd3923..99590df 100644 --- a/tests/config/remote_rebex/queue.yaml +++ b/tests/config/remote_rebex/queue.yaml @@ -2,7 +2,6 @@ queue_type: REMOTE queue_primary: remote ssh_host: test.rebex.net ssh_username: demo -known_hosts: ~/.ssh/known_hosts ssh_password: password ssh_remote_config_dir: / ssh_remote_path: /