{"cells": [{"cell_type": "markdown", "metadata": {}, "source": ["# Communication with a remote Linux machine through SSH\n", "\n", "In this notebook, we show how to communicate with a remote machine. This machine was set up using Ubuntu 12 (and with [Azure](http://azure.microsoft.com/) : [Create a Virtual Machine Running Linux](http://azure.microsoft.com/en-us/documentation/articles/virtual-machines-linux-tutorial/)). The following code is used to get the credentials without printing them on the notebook."]}, {"cell_type": "code", "execution_count": 1, "metadata": {"collapsed": true}, "outputs": [], "source": ["%load_ext pyensae\n", "%load_ext pyenbc"]}, {"cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [{"data": {"text/html": ["
credentials\n", "
password \n", "
server \n", "
username \n", "
\n", ""], "text/plain": [""]}, "execution_count": 3, "metadata": {}, "output_type": "execute_result"}], "source": ["import pyquickhelper.ipythonhelper as ipy\n", "params={\"server\":\"\", \"username\":\"\", \"password\":\"\"}\n", "ipy.open_html_form(params=params,title=\"credentials\",key_save=\"ssh_remote\")"]}, {"cell_type": "code", "execution_count": 3, "metadata": {"collapsed": true}, "outputs": [], "source": ["password = ssh_remote[\"password\"]\n", "server = ssh_remote[\"server\"]\n", "username = ssh_remote[\"username\"]"]}, {"cell_type": "markdown", "metadata": {}, "source": ["We are going to use the following magic commands [MagicRemote](http://www.xavierdupre.fr/app/pyenbc/helpsphinx/pyenbc/remote/magic_remote.html#pyenbc.remote.magic_remote.MagicRemote):"]}, {"cell_type": "code", "execution_count": 4, "metadata": {"scrolled": false}, "outputs": [{"name": "stdout", "output_type": "stream", "text": ["Help on class ASSHClient in module pyenbc.remote.ssh_remote_connection:\n", "\n", "class ASSHClient(builtins.object)\n", " | A simple class to access to remote machine through SSH.\n", " | It requires modules\n", " | `paramiko `_,\n", " | `pycrypto `_,\n", " | `ecdsa `_.\n", " | \n", " | This class is used in magic command @see me remote_open.\n", " | On Windows, the installation of pycrypto can be tricky.\n", " | See `Pycrypto on Windows `_.\n", " | Those modules are part of the `Anaconda `_ distribution.\n", " | \n", " | Methods defined here:\n", " | \n", " | __init__(self, server, username, password)\n", " | constructor\n", " | \n", " | @param server server\n", " | @param username username\n", " | @param password password\n", " | \n", " | __str__(self)\n", " | usual\n", " | \n", " | close(self)\n", " | close the connection\n", " | \n", " | close_session(self)\n", " | close a session\n", " | \n", " | connect(self)\n", " | connect\n", " | \n", " | dfs_exists(self, path)\n", " | tells if a file exists on the cluster\n", " | \n", " | @param path path\n", " | @return boolean\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | dfs_ls(self, path)\n", " | return the content of a folder on the cluster as a DataFrame\n", " | \n", " | @param path path on the cluster\n", " | @return DataFrame\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | dfs_mkdir(self, path)\n", " | creates a directory on the cluster\n", " | \n", " | @param path path\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | dfs_rm(self, path, recursive=False)\n", " | removes a file on the cluster\n", " | \n", " | @param path path\n", " | @param recursive boolean\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | download(self, remotepath, localpath)\n", " | download a file from the remote machine (not on the cluster)\n", " | @param localpath local file\n", " | @param remotepath remote file (it can be a list, localpath is a folder in that case)\n", " | \n", " | .. versionchanged:: 1.1\n", " | remotepath can be a list of paths\n", " | \n", " | download_cluster(self, remotepath, localpath, merge=False)\n", " | download a file directly from the cluster to the local machine\n", " | @param localpath local file\n", " | @param remotepath remote file (it can be a list, localpath is a folder in that case)\n", " | @param merge True to use getmerge instead of get\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | execute_command(self, command, no_exception=False, fill_stdin=None)\n", " | execute a command line, it raises an error\n", " | if there is an error\n", " | \n", " | @param command command\n", " | @param no_exception if True, do not raise any exception\n", " | @param fill_stdin data to send on the stdin input\n", " | @return stdout, stderr\n", " | \n", " | Example of commands::\n", " | \n", " | ssh.execute_command(\"ls\")\n", " | ssh.execute_command(\"hdfs dfs -ls\")\n", " | \n", " | exists(self, path)\n", " | tells if a file exists on the bridge\n", " | \n", " | @param path path\n", " | @return boolean\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | ls(self, path)\n", " | return the content of a folder on the bridge as a DataFrame\n", " | \n", " | @param path path on the bridge\n", " | @return DataFrame\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | open_session(self, no_exception=False, timeout=1.0, add_eol=True, prompts=('~$', '>>>'), out_format=None)\n", " | open a session with method `invoke_shell `_\n", " | \n", " | @param no_exception if True, do not raise any exception in case of error\n", " | @param timeout timeout in s\n", " | @param add_eol if True, the function will add a EOL to the sent command if it does not have one\n", " | @param prompts if function terminates if the output ends by one of those strings.\n", " | @param out_format None, plain, html\n", " | \n", " | @example(How to open a remote shell?)\n", " | @code\n", " | ssh = ASSHClient( \"\",\n", " | \"\",\n", " | \"\")\n", " | ssh.connect()\n", " | out = ssh.send_recv_session(\"ls\")\n", " | print( ssh.send_recv_session(\"python\") )\n", " | print( ssh.send_recv_session(\"print('3')\") )\n", " | print( ssh.send_recv_session(\"import sys\\nsys.executable\") )\n", " | print( ssh.send_recv_session(\"sys.exit()\") )\n", " | print( ssh.send_recv_session(None) )\n", " | ssh.close_session()\n", " | ssh.close()\n", " | @endcode\n", " | \n", " | The notebook :ref:`exampleofsshclientcommunicationrst` illustrates\n", " | the output of these instructions.\n", " | \n", " | @endexample\n", " | \n", " | pig_submit(self, pig_file, dependencies=None, params=None, redirection='redirection', local=False, stop_on_failure=False, check=False, no_exception=True, fLOG=)\n", " | submits a PIG script, it first upload the script\n", " | to the default folder and submit it\n", " | \n", " | @param pig_file pig script (local)\n", " | @param dependencies others files to upload (still in the default folder)\n", " | @param params parameters to send to the job\n", " | @param redirection string empty or not\n", " | @param local local run or not (option `-x local `_) (in that case, redirection will be empty)\n", " | @param stop_on_failure if True, add option ``-stop_on_failure`` on the command line\n", " | @param check if True, add option ``-check`` (in that case, redirection will be empty)\n", " | @param no_exception sent to @see me execute_command\n", " | @param fLOG logging function\n", " | @return out, err from @see me execute_command\n", " | \n", " | If *redirection* is not empty, the job is submitted but\n", " | the function returns after the standard output and error were\n", " | redirected to ``redirection.out`` and ``redirection.err``.\n", " | \n", " | The first file will contain the results of commands\n", " | `DESCRIBE `_\n", " | `DUMP `_,\n", " | `EXPLAIN `_.\n", " | The standard error receives logs and exceptions.\n", " | \n", " | The function executes the command line::\n", " | \n", " | pig -execute -f \n", " | \n", " | With redirection::\n", " | \n", " | pig -execute -f 2> redirection.err 1> redirection.out &\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | send_recv_session(self, fillin)\n", " | Send something through a session,\n", " | the function is supposed to return when the execute of the given command is done,\n", " | but this is quite difficult to detect without knowing what exactly was send.\n", " | \n", " | So we add a timeout just to tell the function it has to return even if nothing\n", " | tells the command has finished. It fillin is None, the function will just\n", " | listen to the output.\n", " | \n", " | @param fillin sent to stdin\n", " | @return stdout\n", " | \n", " | The output contains\n", " | `escape codes `_.\n", " | They can be converted to plain text or HTML\n", " | by using the module `ansiconv `_\n", " | and `ansi2html `_.\n", " | This can be specified when opening the session.\n", " | \n", " | upload(self, localpath, remotepath)\n", " | upload a file to the remote machine (not on the cluster)\n", " | \n", " | @param localpath local file (or a list of files)\n", " | @param remotepath remote file\n", " | \n", " | .. versionchanged:: 1.1\n", " | it can upload multiple files if localpath is a list\n", " | \n", " | upload_cluster(self, localpath, remotepath)\n", " | the function directly uploads the file to the cluster, it first goes\n", " | to the bridge, uploads it to the cluster and deletes it from the bridge\n", " | \n", " | @param localpath local filename (or list of files)\n", " | @param remotepath path to the cluster\n", " | @return filename\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | ----------------------------------------------------------------------\n", " | Static methods defined here:\n", " | \n", " | build_command_line_parameters(params)\n", " | builds a string for ``pig`` based on the parameters in params\n", " | \n", " | @param params dictionary\n", " | @return string\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | parse_lsout(out, local_schema=True)\n", " | parses the output of a command ls\n", " | \n", " | @param out output\n", " | @param local_schema schema for the bridge or the cluster (False)\n", " | @return DataFrame\n", " | \n", " | .. versionadded:: 1.1\n", " | \n", " | ----------------------------------------------------------------------\n", " | Data descriptors defined here:\n", " | \n", " | __dict__\n", " | dictionary for instance variables (if defined)\n", " | \n", " | __weakref__\n", " | list of weak references to the object (if defined)\n", "\n"]}], "source": ["from pyense.remote import ASSHClient\n", "help(ASSHClient)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["We open the connection using the variables stored two cells above:"]}, {"cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [{"data": {"text/plain": [""]}, "execution_count": 6, "metadata": {}, "output_type": "execute_result"}], "source": ["%remote_open"]}, {"cell_type": "markdown", "metadata": {}, "source": ["We try a simple command:"]}, {"cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [{"data": {"text/html": ["
\n", "centrer_reduire.pig\n", "ConfLongDemo_JSI.txt\n", "diff_cluster\n", "dummy\n", "filter_example.pig\n", "filter_example.redirect.err\n", "filter_example.redirect.out\n", "init_random.pig\n", "iteration_complete.pig\n", "nb_obervations.pig\n", "pig_1436911046432.log\n", "pig_1436913856496.log\n", "pig_1436997076356.log\n", "post_traitement.pig\n", "pystream.pig\n", "pystream.py\n", "redirection.err\n", "redirection.out\n", "Skin_NonSkin.txt\n", "\n", "
"], "text/plain": [""]}, "execution_count": 7, "metadata": {}, "output_type": "execute_result"}], "source": ["%remote_cmd ls ."]}, {"cell_type": "markdown", "metadata": {}, "source": ["We then execute a python program on the remote machine, we first same the following code as a program:"]}, {"cell_type": "code", "execution_count": 7, "metadata": {"collapsed": true}, "outputs": [], "source": ["%%PYTHON exemple.py\n", "\n", "import sys\n", "print(\"path to python\", sys.executable)\n", "print(\"version \", sys.version_info)"]}, {"cell_type": "markdown", "metadata": {}, "source": ["The next command stores it as a file, uploads it to the remote machine and then executes it:"]}, {"cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [{"data": {"text/html": ["
\n", "('path to python', '/usr/bin/python')\n", "('version ', (2, 6, 6, 'final', 0))\n", "\n", "
"], "text/plain": [""]}, "execution_count": 9, "metadata": {}, "output_type": "execute_result"}], "source": ["%remote_py exemple.py"]}, {"cell_type": "markdown", "metadata": {}, "source": ["We check it is different from the local version:"]}, {"cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [{"data": {"text/plain": ["('c:\\\\python34_x64\\\\python.exe',\n", " sys.version_info(major=3, minor=4, micro=3, releaselevel='final', serial=0))"]}, "execution_count": 10, "metadata": {}, "output_type": "execute_result"}], "source": ["import sys\n", "sys.executable, sys.version_info"]}, {"cell_type": "markdown", "metadata": {}, "source": ["If you want to use a different version of the interpreter, you can try (not available here):"]}, {"cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [{"data": {"text/html": ["ERR:
\n", "bash: anaconda3/bin/python3.4: No such file or directory\n", "\n", "
OUT:
\n", "\n", "
"], "text/plain": [""]}, "execution_count": 11, "metadata": {}, "output_type": "execute_result"}], "source": ["%remote_py -i=anaconda3/bin/python3.4 exemple.py"]}, {"cell_type": "markdown", "metadata": {}, "source": ["Sometimes, you need an interactive shell such as [Putty](http://www.putty.org/). Let's see how it works:"]}, {"cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [{"data": {"text/plain": ["True"]}, "execution_count": 12, "metadata": {}, "output_type": "execute_result"}], "source": ["%open_remote_shell"]}, {"cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [{"data": {"text/html": ["\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
\n", "Last login: Fri Jan 30 11:37:57 2015 from aputeaux-551-1-113-111.w92-132.abo.wanadoo.fr\n", "[xavierdupre@ws09-en1.tl: ~]$ ls\n", "centrer_reduire.pig   diff_cluster  exemple.py          filter_example.redirect.err  init_random.pig         nb_obervations.pig     pig_1436913856496.log  post_traitement.pig  pystream.py      redirection.out\n", "ConfLongDemo_JSI.txt  dummy         filter_example.pig  filter_example.redirect.out  iteration_complete.pig  pig_1436911046432.log  pig_1436997076356.log  pystream.pig         redirection.err  Skin_NonSkin.txt\n", "[xavierdupre@ws09-en1.tl: ~]$ \n", "
\n", "\n", "\n", "\n"], "text/plain": [""]}, "execution_count": 13, "metadata": {}, "output_type": "execute_result"}], "source": ["%%shell_remote\n", "ls"]}, {"cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [{"data": {"text/html": ["\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
\n", "python\n", "Python 2.6.6 (r266:84292, Jan 22 2014, 09:42:36) \n", "[GCC 4.4.7 20120313 (Red Hat 4.4.7-4)] on linux2\n", "Type \"help\", \"copyright\", \"credits\" or \"license\" for more information.\n", ">>> \n", "
\n", "\n", "\n", "\n"], "text/plain": [""]}, "execution_count": 14, "metadata": {}, "output_type": "execute_result"}], "source": ["%shell_remote python"]}, {"cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [{"data": {"text/html": ["\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
\n", "\n", ">>> import sys\n", ">>> sys.executable\n", "'/usr/bin/python'\n", ">>> \n", "
\n", "\n", "\n", "\n"], "text/plain": [""]}, "execution_count": 15, "metadata": {}, "output_type": "execute_result"}], "source": ["%%shell_remote\n", "\n", "import sys\n", "sys.executable"]}, {"cell_type": "markdown", "metadata": {}, "source": ["To close the shell, we can just type:"]}, {"cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [{"data": {"text/plain": ["True"]}, "execution_count": 16, "metadata": {}, "output_type": "execute_result"}], "source": ["%close_remote_shell"]}, {"cell_type": "markdown", "metadata": {}, "source": ["And to close the connection:"]}, {"cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [{"data": {"text/plain": ["True"]}, "execution_count": 17, "metadata": {}, "output_type": "execute_result"}], "source": ["%remote_close"]}, {"cell_type": "markdown", "metadata": {}, "source": ["**END**"]}, {"cell_type": "code", "execution_count": 17, "metadata": {"collapsed": true}, "outputs": [], "source": []}], "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"codemirror_mode": {"name": "ipython", "version": 3}, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.1"}}, "nbformat": 4, "nbformat_minor": 2}