Scp send copy args (#232)

* Added copy args functionality to scp_send. 
* Updated changelog. 
* Added large file scp_send test to multiple remote servers.

Author: pkittenis

Changelog.rst

@@ -27,7 +27,7 @@ See `Upgrading to API 2.0 <https://parallel-ssh.readthedocs.io/en/latest/api_upg
 * Removed ``run_command(greenlet_timeout=<..>)`` argument - now uses global timeout setting.
 * Renamed ``run_command`` ``timeout`` to ``read_timeout=<seconds>)`` for setting output read timeout individually - defaults to global timeout setting.
 * Removed ``pssh.native`` package and native code.
-* No native code means package architecture has changed to ``none-any``.
+* ``ParallelSSHClient.scp_send`` now supports ``copy_args`` keyword argument for providing per-host file name arguments like rest of ``scp_*`` and ``copy_*`` functionality.
 
 
 Fixes
@@ -38,6 +38,12 @@ Fixes
 * Clients could raise ``Timeout`` early when timeout settings were used with many hosts.
 
 
+Packaging
+---------
+
+* Package architecture has changed to ``none-any``.
+
+
 1.13.0
 ++++++
 

doc/advanced.rst

@@ -512,6 +512,8 @@ Configurable per host Filenames
 
 File name arguments, for both local and remote files and for copying to and from remote hosts, can be configured on a per-host basis similarly to `host arguments <#per-host-cmds>`_ in ``run_command``.
 
+Example shown applies to all file copy functionality, all of ``scp_send``, ``scp_recv``, ``copy_file`` and ``copy_remote_file``.
+
 For example, to copy the local files ``['local_file_1', 'local_file_2']`` as remote files ``['remote_file_1', 'remote_file_2']`` on the two hosts ``['host1', 'host2']``
 
 .. code-block:: python
@@ -532,9 +534,9 @@ For example, to copy the local files ``['local_file_1', 'local_file_2']`` as rem
 
 The client will copy ``local_file_1`` to ``host1`` as ``remote_file_1`` and ``local_file_2`` to ``host2`` as ``remote_file_2``.
 
-Items in ``copy_args`` list may be tuples or dictionaries as shown above. Number of ``copy_args`` must match length of ``client.hosts`` if provided.
+Each item in ``copy_args`` list should be a dictionary as shown above. Number of ``copy_args`` must match length of ``client.hosts`` if provided or exception will be raised.
 
-``copy_remote_file`` may be used in the same manner to configure remote and local file names per host.
+``copy_remote_file``, ``scp_send`` and ``scp_recv`` may all be used in the same manner to configure remote and local file names per host.
 
 .. seealso::
 

pssh/clients/native/parallel.py

@@ -437,7 +437,7 @@ def _scp_recv(self, host_i, host, remote_file, local_file, recurse=False):
             self._host_clients[(host_i, host)].scp_recv, host,
             remote_file, local_file, recurse=recurse)
 
-    def scp_send(self, local_file, remote_file, recurse=False):
+    def scp_send(self, local_file, remote_file, recurse=False, copy_args=None):
         """Copy local file to remote file in parallel via SCP.
 
         This function returns a list of greenlets which can be
@@ -469,9 +469,22 @@ def scp_send(self, local_file, remote_file, recurse=False):
         :raises: :py:class:`pss.exceptions.SCPError` on errors copying file.
         :raises: :py:class:`OSError` on local OS errors like permission denied.
         """
-        return [self.pool.spawn(self._scp_send, host_i, host, local_file,
-                                remote_file, recurse=recurse)
-                for host_i, host in enumerate(self.hosts)]
+        copy_args = [{'local_file': local_file,
+                      'remote_file': remote_file}
+                     for i, host in enumerate(self.hosts)] \
+            if copy_args is None else copy_args
+        local_file = "%(local_file)s"
+        remote_file = "%(remote_file)s"
+        try:
+            return [self.pool.spawn(self._scp_send, host_i, host,
+                                    local_file % copy_args[host_i],
+                                    remote_file % copy_args[host_i],
+                                    recurse=recurse)
+                    for host_i, host in enumerate(self.hosts)]
+        except IndexError:
+            raise HostArgumentException(
+                "Number of per-host copy arguments provided does not match "
+                "number of hosts")
 
     def scp_recv(self, remote_file, local_file, recurse=False, copy_args=None,
                  suffix_separator='_'):

tests/native/test_parallel_client.py

@@ -26,10 +26,10 @@
 import sys
 import string
 import random
+from hashlib import sha256
 from datetime import datetime
 from platform import python_version
 
-from pytest import mark
 from gevent import joinall, spawn, socket, Greenlet, sleep
 from pssh.config import HostConfig
 from pssh.clients.native import ParallelSSHClient
@@ -1354,29 +1354,81 @@ def test_scp_send_dir_recurse(self):
                 pass
 
     def test_scp_send(self):
-        test_file_data = 'test'
+        server2_host = '127.0.0.11'
+        server3_host = '127.0.0.12'
+        server2 = OpenSSHServer(server2_host, port=self.port)
+        server3 = OpenSSHServer(server3_host, port=self.port)
+        for server in (server2, server3):
+            server.start_server()
+        hosts = [self.host, server2_host, server3_host]
+        client = ParallelSSHClient(hosts, port=self.port, pkey=self.user_key, num_retries=1)
         local_filename = 'test_file'
-        remote_test_dir, remote_filepath = 'remote_test_dir', 'test_file_copy'
-        with open(local_filename, 'w') as file_h:
-            file_h.writelines([test_file_data + os.linesep])
-        remote_file_abspath = os.path.expanduser('~/' + remote_filepath)
-        cmds = self.client.scp_send(local_filename, remote_filepath)
+        remote_filepath = 'file_copy'
+        copy_args = [{
+            'local_file': local_filename,
+            'remote_file': 'host_%s_%s' % (n, remote_filepath)}
+                     for n in range(len(hosts))]
+        remote_file_names = [arg['remote_file'] for arg in copy_args]
+        sha = sha256()
+        with open(local_filename, 'wb') as file_h:
+            for _ in range(10000):
+                data = os.urandom(1024)
+                file_h.write(data)
+                sha.update(data)
+        source_file_sha = sha.hexdigest()
+        sha = sha256()
+        cmds = client.scp_send('%(local_file)s', '%(remote_file)s', copy_args=copy_args)
         try:
             joinall(cmds, raise_error=True)
         except Exception:
             raise
         else:
-            self.assertTrue(os.path.isfile(remote_file_abspath))
-            remote_contents = open(remote_file_abspath, 'rb').read()
-            local_contents = open(local_filename, 'rb').read()
-            self.assertEqual(local_contents, remote_contents)
+            sleep(.2)
+            for remote_file_name in remote_file_names:
+                remote_file_abspath = os.path.expanduser('~/' + remote_file_name)
+                self.assertTrue(os.path.isfile(remote_file_abspath))
+                with open(remote_file_abspath, 'rb') as remote_fh:
+                    for data in remote_fh:
+                        sha.update(data)
+                remote_file_sha = sha.hexdigest()
+                sha = sha256()
+                self.assertEqual(source_file_sha, remote_file_sha)
         finally:
             try:
                 os.unlink(local_filename)
-                os.unlink(remote_file_abspath)
+                for remote_file_name in remote_file_names:
+                    remote_file_abspath = os.path.expanduser('~/' + remote_file_name)
+                    os.unlink(remote_file_abspath)
             except OSError:
                 pass
 
+    def test_scp_send_bad_copy_args(self):
+        client = ParallelSSHClient([self.host, self.host])
+        copy_args = [{'local_file': 'test', 'remote_file': 'test'}]
+        self.assertRaises(HostArgumentException,
+                          client.scp_send, '%(local_file)s', '%(remote_file)s',
+                          copy_args=copy_args)
+
+    def test_scp_send_exc(self):
+        client = ParallelSSHClient([self.host], pkey=self.user_key, num_retries=1)
+        def _scp_send(*args):
+            raise Exception
+        def _client_send(*args):
+            return client._handle_greenlet_exc(_scp_send, 'fake')
+        client._scp_send = _client_send
+        cmds = client.scp_send('local_file', 'remote_file')
+        self.assertRaises(Exception, joinall, cmds, raise_error=True)
+
+    def test_scp_recv_exc(self):
+        client = ParallelSSHClient([self.host], pkey=self.user_key, num_retries=1)
+        def _scp_recv(*args):
+            raise Exception
+        def _client_recv(*args):
+            return client._handle_greenlet_exc(_scp_recv, 'fake')
+        client._scp_recv = _client_recv
+        cmds = client.scp_recv('remote_file', 'local_file')
+        self.assertRaises(Exception, joinall, cmds, raise_error=True)
+
     def test_scp_recv_failure(self):
         cmds = self.client.scp_recv(
             'fakey fakey fake fake', 'equally fake')