Remote: Automatically manage known hosts keys

pysqa/base/remote.py

@@ -93,9 +93,12 @@ def __init__(
         )
         self._ssh_host = config["ssh_host"]
         self._ssh_username = config["ssh_username"]
-        self._ssh_known_hosts = os.path.abspath(
-            os.path.expanduser(config["known_hosts"])
-        )
+        if "known_hosts" in config:
+            self._ssh_known_hosts = os.path.abspath(
+                os.path.expanduser(config["known_hosts"])
+            )
+        else:
+            self._ssh_known_hosts = ""
         self._ssh_ask_for_password: Union[bool, str] = False
         self._ssh_key = (
             os.path.abspath(os.path.expanduser(config["ssh_key"]))
@@ -346,7 +349,10 @@ def _open_ssh_connection(self) -> paramiko.SSHClient:
             paramiko.SSHClient: The SSH connection object.
         """
         ssh = paramiko.SSHClient()
-        ssh.load_host_keys(self._ssh_known_hosts)
+        if len(self._ssh_known_hosts) > 0:
+            ssh.load_host_keys(self._ssh_known_hosts)
+        else:
+            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
         if (
             self._ssh_key is not None
             and self._ssh_key_passphrase is not None

tests/config/remote_rebex_hosts/queue.yaml

@@ -0,0 +1,14 @@
+queue_type: REMOTE
+queue_primary: remote
+ssh_host: test.rebex.net
+ssh_username: demo
+known_hosts: ~/.ssh/known_hosts
+ssh_password: password
+ssh_remote_config_dir: /
+ssh_remote_path: /
+ssh_local_path: /home/localuser/projects/
+ssh_continous_connection: False
+ssh_port: 22
+python_executable: python3
+queues:
+  remote: {cores_max: 100, cores_min: 10, run_time_max: 259200}

tests/test_remote.py

@@ -151,36 +151,53 @@ def test_remote_command_continous_connection(self):
         output = remote._adapter._execute_remote_command(command="pwd")
         self.assertEqual(output, "/\n")
 
+    def test_remote_command_individual_connections_hosts(self):
+        path = os.path.dirname(os.path.abspath(__file__))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
+        remote._adapter._ssh_remote_path = path
+        remote._adapter._open_ssh_connection()
+        output = remote._adapter._execute_remote_command(command="pwd")
+        self.assertEqual(output, "/\n")
+
+    def test_remote_command_continous_connection_hosts(self):
+        path = os.path.dirname(os.path.abspath(__file__))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
+        remote._adapter._ssh_remote_path = path
+        remote._adapter._ssh_continous_connection = True
+        remote._adapter._open_ssh_connection()
+        output = remote._adapter._execute_remote_command(command="pwd")
+        self.assertEqual(output, "/\n")
+
     def test_submit_job(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_remote_path = path
         output = remote._adapter.submit_job(working_directory=os.path.join(path, "config/empty"), command="echo 1")
         self.assertEqual(output, 1)
 
     def test_transferfile_individual_connections(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_remote_path = path
         self.assertIsNone(remote._adapter.transfer_file(file="readme.txt", transfer_back=True))
 
     def test_transferfile_continous_connection(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_remote_path = path
         remote._adapter._ssh_continous_connection = True
         self.assertIsNone(remote._adapter.transfer_file(file="readme.txt", transfer_back=True))
 
     def test_get_transport(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         self.assertIsNotNone(get_transport(remote._adapter._open_ssh_connection()))
         with self.assertRaises(ValueError):
             get_transport(ssh=FakeSSH())
 
     def test_get_job_from_remote(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_remote_path = path
         remote._adapter._ssh_local_path = path
         remote._adapter._ssh_delete_file_on_remote = True

tests/test_remote_auth.py

@@ -19,21 +19,21 @@
 class TestRemoteQueueAdapterAuth(unittest.TestCase):
     def test_password_auth(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_ask_for_password = False
         remote._adapter._ssh_key = None
         self.assertIsNotNone(remote._adapter._open_ssh_connection())
 
     def test_key_auth(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_password = None
         with self.assertRaises(ValueError):
             remote._adapter._open_ssh_connection()
 
     def test_two_factor_auth(self):
         path = os.path.dirname(os.path.abspath(__file__))
-        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex"))
+        remote = QueueAdapter(directory=os.path.join(path, "config/remote_rebex_hosts"))
         remote._adapter._ssh_two_factor_authentication = True
         with self.assertRaises(paramiko.ssh_exception.AuthenticationException):
             remote._adapter._open_ssh_connection()