fixes #699

Author: jph00

fastcore/_modidx.py

@@ -652,12 +652,15 @@
                                 'fastcore.xtras.repo_details': ('xtras.html#repo_details', 'fastcore/xtras.py'),
                                 'fastcore.xtras.repr_dict': ('xtras.html#repr_dict', 'fastcore/xtras.py'),
                                 'fastcore.xtras.round_multiple': ('xtras.html#round_multiple', 'fastcore/xtras.py'),
+                                'fastcore.xtras.rsync_multi': ('xtras.html#rsync_multi', 'fastcore/xtras.py'),
                                 'fastcore.xtras.run': ('xtras.html#run', 'fastcore/xtras.py'),
                                 'fastcore.xtras.save_iter': ('xtras.html#save_iter', 'fastcore/xtras.py'),
                                 'fastcore.xtras.save_pickle': ('xtras.html#save_pickle', 'fastcore/xtras.py'),
                                 'fastcore.xtras.set_num_threads': ('xtras.html#set_num_threads', 'fastcore/xtras.py'),
+                                'fastcore.xtras.shell': ('xtras.html#shell', 'fastcore/xtras.py'),
                                 'fastcore.xtras.shufflish': ('xtras.html#shufflish', 'fastcore/xtras.py'),
                                 'fastcore.xtras.sparkline': ('xtras.html#sparkline', 'fastcore/xtras.py'),
+                                'fastcore.xtras.ssh': ('xtras.html#ssh', 'fastcore/xtras.py'),
                                 'fastcore.xtras.stringfmt_names': ('xtras.html#stringfmt_names', 'fastcore/xtras.py'),
                                 'fastcore.xtras.time_policy': ('xtras.html#time_policy', 'fastcore/xtras.py'),
                                 'fastcore.xtras.timed_cache': ('xtras.html#timed_cache', 'fastcore/xtras.py'),

fastcore/xtras.py

@@ -7,9 +7,9 @@
 
 # %% auto 0
 __all__ = ['spark_chars', 'UNSET', 'walk', 'globtastic', 'maybe_open', 'mkdir', 'image_size', 'bunzip', 'loads', 'loads_multi',
-           'dumps', 'untar_dir', 'repo_details', 'run', 'open_file', 'save_pickle', 'load_pickle', 'parse_env',
-           'expand_wildcards', 'dict2obj', 'obj2dict', 'repr_dict', 'is_listy', 'mapped', 'IterLen',
-           'ReindexCollection', 'SaveReturn', 'trim_wraps', 'save_iter', 'asave_iter', 'friendly_name',
+           'dumps', 'untar_dir', 'repo_details', 'shell', 'ssh', 'rsync_multi', 'run', 'open_file', 'save_pickle',
+           'load_pickle', 'parse_env', 'expand_wildcards', 'dict2obj', 'obj2dict', 'repr_dict', 'is_listy', 'mapped',
+           'IterLen', 'ReindexCollection', 'SaveReturn', 'trim_wraps', 'save_iter', 'asave_iter', 'friendly_name',
            'n_friendly_names', 'exec_eval', 'get_source_link', 'truncstr', 'sparkline', 'modify_exception',
            'round_multiple', 'set_num_threads', 'join_path_file', 'autostart', 'EventTimer', 'stringfmt_names',
            'PartialFormatter', 'partial_format', 'utc2local', 'local2utc', 'trace', 'modified_env', 'ContextManagers',
@@ -198,6 +198,25 @@ def repo_details(url):
     res = res.split(':')[-1]
     return res.split('/')[-2:]
 
+# %% ../nbs/03_xtras.ipynb
+def shell(*args, **kwargs):
+    "Shortcut for `subprocess.run(shell=True)`"
+    import subprocess
+    return subprocess.run(*args, shell=True, **kwargs)
+
+# %% ../nbs/03_xtras.ipynb
+def ssh(host, args='', user='ubuntu', sock=None):
+    "Run SSH command with given arguments"
+    sock_opts = f'-S {sock}' if sock else ''
+    return shell(f'ssh {sock_opts} {args} {user}@{host}')
+
+# %% ../nbs/03_xtras.ipynb
+def rsync_multi(ip, files, user='ubuntu', persist='5m'):
+    "Transfer multiple files with rename using persistent SSH connection"
+    sock = f'/tmp/ssh-{ip}-{user}'
+    ssh(ip, f'-o ControlMaster=auto -o ControlPersist={persist} -N -f', user, sock)
+    for src,dst in files: shell(f'rsync -az -e "ssh -S {sock}" {src} {user}@{ip}:{dst}')
+
 # %% ../nbs/03_xtras.ipynb
 def run(cmd, *rest, same_in_win=False, ignore_ex=False, as_bytes=False, stderr=False):
     "Pass `cmd` (splitting with `shlex` if string) to `subprocess.run`; return `stdout`; raise `IOError` if fails"

nbs/03_xtras.ipynb

@@ -665,6 +665,46 @@
     "test_eq(repo_details('[email protected]:fastai/nbdev.git\\n'), ['fastai', 'nbdev'])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def shell(*args, **kwargs):\n",
+    "    \"Shortcut for `subprocess.run(shell=True)`\"\n",
+    "    import subprocess\n",
+    "    return subprocess.run(*args, shell=True, **kwargs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def ssh(host, args='', user='ubuntu', sock=None):\n",
+    "    \"Run SSH command with given arguments\"\n",
+    "    sock_opts = f'-S {sock}' if sock else ''\n",
+    "    return shell(f'ssh {sock_opts} {args} {user}@{host}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#|export\n",
+    "def rsync_multi(ip, files, user='ubuntu', persist='5m'):\n",
+    "    \"Transfer multiple files with rename using persistent SSH connection\"\n",
+    "    sock = f'/tmp/ssh-{ip}-{user}'\n",
+    "    ssh(ip, f'-o ControlMaster=auto -o ControlPersist={persist} -N -f', user, sock)\n",
+    "    for src,dst in files: shell(f'rsync -az -e \"ssh -S {sock}\" {src} {user}@{ip}:{dst}')"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,