Coverage for src/pyenbc/remote/magic_remote_ssh.py: 0%
367 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-20 05:47 +0200
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-20 05:47 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Magic command to communicate with an Hadoop cluster (Linux).
5"""
6import os
8from IPython.core.magic import magics_class, line_magic, cell_magic
9from IPython.core.magic import line_cell_magic
10from IPython.core.display import HTML
11from pyquickhelper.ipythonhelper import MagicClassWithHelpers, MagicCommandParser
12from .ssh_remote_connection import ASSHClient
15@magics_class
16class MagicRemoteSSH(MagicClassWithHelpers):
18 """
19 Defines commands to access a remote machine (bridge) through SSH,
20 for the time being, all the command do not accept another parameters
21 such as a SSH client which means only one connection
22 can be opened at the same time.
24 The notebooks :ref:`pythonhadooppigrst` and
25 :ref:`exampleofsshclientcommunicationrst` show how these commands can be used.
26 """
28 def _replace_params(self, cell):
29 """
30 replaces parameter such ``__PASSWORD__`` by variable in the notebook environment
32 @param cell string
33 @return modified string
34 """
35 if "__PASSWORD__" in cell and self.shell is not None and "password" in self.shell.user_ns:
36 cell = cell.replace("__PASSWORD__", self.shell.user_ns["password"])
37 return cell
39 def get_connection(self):
40 """
41 returns the connection stored in the workspace
42 """
43 if self.shell is None:
44 raise Exception("No detected workspace.")
46 if "remote_ssh" not in self.shell.user_ns:
47 raise KeyError("No opened SSH connection.")
49 return self.shell.user_ns["remote_ssh"]
51 @staticmethod
52 def PIG_parser():
53 """
54 defines the way to parse the magic command ``%%PIG``
55 """
56 parser = MagicCommandParser(prog="PIG",
57 description='The command store the content of the cell as a local file.')
58 parser.add_argument(
59 'file',
60 type=str,
61 help='file name')
62 return parser
64 @cell_magic
65 def PIG(self, line, cell=None):
66 """
67 defines command ``%%PIG``
69 .. nbref::
70 :tag: Hadoop
71 :title: PIG
73 The code for magic command ``%PIG`` is equivalent to::
75 with open(filename, "w", encoding="utf8") as f:
76 f.write(script)
79 """
80 parser = self.get_parser(MagicRemoteSSH.PIG_parser, "PIG")
81 args = self.get_args(line, parser)
83 if args is not None:
84 filename = args.file
85 with open(filename, "w", encoding="utf8") as f:
86 f.write(cell.replace("\r", ""))
88 @staticmethod
89 def HIVE_parser():
90 """
91 defines the way to parse the magic command ``%%HIVE``
92 """
93 parser = MagicCommandParser(prog="HIVE",
94 description='The command store the content of the cell as a local file.')
95 parser.add_argument(
96 'file',
97 type=str,
98 help='file name')
99 return parser
101 @cell_magic
102 def HIVE(self, line, cell=None):
103 """
104 defines command ``%%HIVE``
106 .. nbref::
107 :tag: Hadoop
108 :title: HIVE
110 The code for magic command ``%HIVE`` is equivalent to::
112 with open(filename, "w", encoding="utf8") as f:
113 f.write(script)
116 """
117 parser = self.get_parser(MagicRemoteSSH.HIVE_parser, "HIVE")
118 args = self.get_args(line, parser)
120 if args is not None:
121 filename = args.file
122 with open(filename, "w", encoding="utf8") as f:
123 f.write(cell.replace("\r", ""))
125 @staticmethod
126 def pig_submit_parser():
127 """
128 defines the way to parse the magic command ``%pig_submit``
129 """
130 parser = MagicCommandParser(prog="pig_submit",
131 description='Submits a job to the cluster, the job is local, the job ' +
132 'is first uploaded to the cluster. The magic command populates ' +
133 'the local variable last_job with the submitted job id.')
134 parser.add_argument(
135 'file',
136 type=str,
137 help='file name')
138 parser.add_argument(
139 '-d',
140 '--dependency',
141 nargs="*",
142 type=str,
143 help='dependency of the job, the python script')
144 parser.add_argument(
145 '-l',
146 '--local',
147 action='store_true',
148 default=False,
149 help='run locally on the bridge or on the cluster (default)')
150 parser.add_argument(
151 '-r',
152 '--redirection',
153 type=str,
154 default="redirection.pig",
155 help='list of options for the job')
156 parser.add_argument(
157 '--raw-output',
158 default=False,
159 action='store_true',
160 help='display raw text instead of HTML')
161 parser.add_argument(
162 '-s',
163 '--stop_on_failure',
164 action='store_true',
165 default=False,
166 help='if true, the job stops on failure right away')
167 parser.add_argument(
168 '-o',
169 '--option',
170 nargs='*',
171 type=str,
172 help='list of options for the job')
173 return parser
175 @line_magic
176 def pig_submit(self, line):
177 """
178 defines command ``%pig_submit``
180 .. nbref::
181 :tag: Hadoop
182 :title: pig_submit
184 The code for magic command ``%pig_submit`` is equivalent to::
186 ssh = ASSHClient(server, username, password)
187 ssh.connect()
188 out, err = ssh.pig_submit(
189 pig, dependencies=dependencies, redirection=redirection, local=local, stop_on_failure=stop_on_failure)
190 ssh.close()
191 """
192 parser = self.get_parser(
193 MagicRemoteSSH.pig_submit_parser, "pig_submit")
194 args = self.get_args(line, parser)
196 if args is not None:
197 pig = args.file
198 pys = [_ for _ in args.dependency if _.endswith(
199 ".py")] if args.dependency is not None else []
200 redirection = None if args.redirection in [
201 None, "None", "", "-"] else args.redirection
203 ssh = self.get_connection()
204 out, err = ssh.pig_submit(
205 pig, dependencies=pys, redirection=redirection, local=args.local, stop_on_failure=args.stop_on_failure)
207 if args.raw_output:
208 if len(err) > 0 and (
209 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
210 return err
211 else:
212 return out
213 else:
214 if len(err) > 0 and (
215 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
216 return HTML("<pre>\n%s\n</pre>" % err)
217 else:
218 return HTML("<pre>\n%s\n</pre>" % out)
219 return None
221 @staticmethod
222 def hive_submit_parser():
223 """
224 defines the way to parse the magic command ``%hive_submit``
225 """
226 parser = MagicCommandParser(prog="hive_submit",
227 description='Submits a job to the cluster, the job is local, ' +
228 'the job is first uploaded to the cluster. The magic ' +
229 'command populates the local variable last_job with the submitted job id.')
230 parser.add_argument(
231 'file',
232 type=str,
233 help='file name')
234 parser.add_argument(
235 '-r',
236 '--redirection',
237 type=str,
238 default="redirection",
239 help='list of options for the job')
240 parser.add_argument(
241 '--raw-output',
242 default=False,
243 action='store_true',
244 help='display raw text instead of HTML')
245 return parser
247 @line_magic
248 def hive_submit(self, line):
249 """
250 defines command ``%hive_submit``
252 .. nbref::
253 :tag: Hadoop
254 :title: hive_submit
256 The code for magic command ``%hive_submit`` is equivalent to::
258 ssh = ASSHClient(server, username, password)
259 ssh.connect()
260 out, err = ssh.hive_submit(
261 pig, redirection=redirection, local=local)
262 ssh.close()
263 """
264 parser = self.get_parser(
265 MagicRemoteSSH.hive_submit_parser, "hive_submit")
266 args = self.get_args(line, parser)
268 if args is not None:
269 pig = args.file
270 ssh = self.get_connection()
271 out, err = ssh.hive_submit(
272 pig, redirection=args.redirection, local=args.local)
274 if args.raw_output:
275 if len(err) > 0 and (
276 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
277 return err
278 else:
279 return out
280 else:
281 if len(err) > 0 and (
282 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
283 return HTML("<pre>\n%s\n</pre>" % err)
284 else:
285 return HTML("<pre>\n%s\n</pre>" % out)
286 return None
288 @staticmethod
289 def remote_py_parser():
290 """
291 defines the way to parse the magic command ``%remote_py``
292 """
293 parser = MagicCommandParser(prog="remote_py",
294 description='run a python script on the bridge')
295 parser.add_argument(
296 'file',
297 type=str,
298 help='file name')
299 parser.add_argument(
300 'args',
301 nargs='*',
302 type=str,
303 help='list of options for the job')
304 parser.add_argument(
305 '-i',
306 '--interpreter',
307 type=str,
308 default='python',
309 help='change the interpreter, python by default')
310 parser.add_argument(
311 '--raw-output',
312 default=False,
313 action='store_true',
314 help='display raw text instead of HTML')
315 return parser
317 @line_magic
318 def remote_py(self, line):
319 """
320 defines command ``%remote_py``
322 .. nbref::
323 :tag: Hadoop
324 :title: remote_py
326 The code for magic command ``%remote_py`` is equivalent to::
328 ssh = ASSHClient(server, username, password)
329 ssh.connect()
330 ssh.upload(filename, dest)
331 args = " ".join('"{}"'.format(_)
332 for _ in args.args) if args.args is not None else ""
333 out, err = ssh.execute_command(exe + " " + dest + " " + args, no_exception=True)
334 ssh.close()
335 """
336 parser = self.get_parser(
337 MagicRemoteSSH.remote_py_parser, "remote_py")
338 args = self.get_args(line, parser)
340 if args is not None:
341 filename = args.file
342 exe = args.interpreter
343 args = " ".join('"{}"'.format(_)
344 for _ in args.args) if args.args is not None else ""
346 dest = os.path.split(filename)[-1]
347 ssh = self.get_connection()
348 ssh.upload(filename, dest)
350 cmd = exe + " " + dest + " " + args
352 out, err = ssh.execute_command(cmd, no_exception=True)
353 if args.raw_output: # pylint: disable=E1101
354 if len(err) > 0:
355 return err
356 else:
357 return out
358 else:
359 if len(err) > 0:
360 return HTML(
361 "<b>ERR:</b><br /><pre>\n%s\n</pre><b>OUT:</b><br /><pre>\n%s\n</pre>" % (err, out))
362 else:
363 return HTML("<pre>\n%s\n</pre>" % out)
364 return None
366 @staticmethod
367 def job_syntax_parser():
368 """
369 defines the way to parse the magic command ``%job_syntax``
370 """
371 parser = MagicCommandParser(prog="remote_py",
372 description='check syntax of a pig job')
373 parser.add_argument(
374 'file',
375 type=str,
376 help='file name')
377 parser.add_argument(
378 '--raw-output',
379 default=False,
380 action='store_true',
381 help='display raw text instead of HTML')
382 return parser
384 @line_magic
385 def job_syntax(self, line):
386 """
387 defines command ``%job_syntax``
388 """
389 parser = self.get_parser(
390 MagicRemoteSSH.job_syntax_parser, "job_syntax")
391 args = self.get_args(line, parser)
393 if args is not None:
394 filename = args.file
395 if not os.path.exists(filename):
396 raise FileNotFoundError(filename)
398 ssh = self.get_connection()
399 out, err = ssh.pig_submit(filename, check=True, no_exception=True)
400 if args.raw_output:
401 if len(err) > 0 and (
402 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
403 return err
404 else:
405 return out
406 else:
407 if len(err) > 0 and (
408 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
409 return HTML("<pre>\n%s\n</pre>" % err)
410 else:
411 return HTML("<pre>\n%s\n</pre>" % out)
412 return None
414 @staticmethod
415 def remote_open_parser():
416 """
417 defines the way to parse the magic command ``%remote_open``
418 """
419 parser = MagicCommandParser(prog="remote_open",
420 description='open a remote SSH connection tp the bridge')
421 parser.add_argument(
422 '-s',
423 '--server',
424 type=str,
425 default='server',
426 help='server name')
427 parser.add_argument(
428 '-u',
429 '--username',
430 type=str,
431 default='username',
432 help='username')
433 parser.add_argument(
434 '-p',
435 '--password',
436 type=str,
437 default='password',
438 help='password')
439 return parser
441 @line_magic
442 def remote_open(self, line):
443 """
444 open a SSH connection and store the connection
445 into the notebook workspace
447 .. nbref::
448 :tag: Hadoop
449 :title: remote_open
451 The code for magic command ``%remote_open`` is equivalent to::
453 ssh = ASSHClient(server, username, password)
454 ssh.connect()
455 """
456 parser = self.get_parser(
457 MagicRemoteSSH.remote_open_parser, "remote_open")
458 args = self.get_args(line, parser)
460 if args is not None:
461 if self.shell is None:
462 raise Exception("No detected workspace.")
464 ssh = ASSHClient(args.server, args.username, args.password)
465 ssh.connect()
467 self.shell.user_ns["remote_ssh"] = ssh
468 return ssh
469 return None
471 @line_magic
472 def remote_close(self, line):
473 """
474 close a SSH connection and store the connection
475 into the notebook workspace
477 .. nbref::
478 :tag: Hadoop
479 :title: remote_close
481 The code for magic command ``%remote_close`` is equivalent to::
483 ssh = ASSHClient(server, username, password)
484 # ... ssh.connect()
485 ssh.close()
488 """
489 self.get_connection().close()
490 return True
492 @line_cell_magic
493 def remote_cmd(self, line, cell=None):
494 """
495 run a command on the remote machine
497 Example::
499 %remote_cmd ls
501 Or::
503 %%remote_cmd <something>
504 anything going to stdin
506 In the second case, if __PASSWORD__ is found, it will be replaced by the password stored in
507 workspace.
509 .. nbref::
510 :tag: Hadoop
511 :title: remote_cmd
513 The code for magic command ``%remote_cmd`` is equivalent to::
515 ssh = ASSHClient(server, username, password)
516 ssh.connect()
517 out, err = ssh.execute_command(
518 line, no_exception=True, fill_stdin=cell)
519 ssh.close()
520 """
521 if "--help" in line:
522 print("Usage: %remote_cmd <cmd>")
523 else:
524 ssh = self.get_connection()
526 if isinstance(cell, str):
527 cell = self._replace_params(cell)
529 out, err = ssh.execute_command(
530 line, no_exception=True, fill_stdin=cell)
531 if len(err) > 0 and (
532 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
533 return HTML("<pre>\n%s\n</pre>" % err)
534 else:
535 return HTML("<pre>\n%s\n</pre>" % out)
536 return None
538 @line_cell_magic
539 def remote_cmd_text(self, line, cell=None):
540 """
541 run a command on the remote machine and returns raw text (not HTML)
543 Example::
545 %remote_cmd_text ls
547 Or::
549 %%remote_cmd_text <something>
550 anything going to stdin
552 In the second case, if __PASSWORD__ is found, it will be replaced by the password stored in
553 workspace.
554 """
555 if "--help" in line:
556 print("Usage: %remote_cmd_text <cmd>")
557 else:
558 ssh = self.get_connection()
560 if isinstance(cell, str):
561 cell = self._replace_params(cell)
563 out, err = ssh.execute_command(
564 line, no_exception=True, fill_stdin=cell)
565 if len(err) > 0 and (
566 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err):
567 return err
568 else:
569 return out
570 return None
572 @staticmethod
573 def remote_up_parser():
574 """
575 defines the way to parse the magic command ``%remote_up``
576 """
577 parser = MagicCommandParser(prog="remote_up",
578 description='upload a file to the remote machine')
579 parser.add_argument(
580 'localfile',
581 type=str,
582 help='local file to upload')
583 parser.add_argument(
584 'remotepath',
585 type=str,
586 help='remote path of the uploaded file')
587 return parser
589 @line_magic
590 def remote_up(self, line):
591 """
592 upload a file to the remote machine,
594 Example::
596 %remote_up localfile remotepath
598 the command does not allow spaces in files
600 .. nbref::
601 :tag: Hadoop
602 :title: remote_up
604 The code for magic command ``%remote_up`` is equivalent to::
606 ssh = ASSHClient(server, username, password)
607 ssh.connect()
608 ssh.upload(localfile, remotepath)
609 ssh.close()
610 """
611 parser = self.get_parser(MagicRemoteSSH.remote_up_parser, "remote_up")
612 args = self.get_args(line, parser)
614 if args is not None:
615 localfile, remotepath = args.localfile, args.remotepath
616 if not os.path.exists(localfile):
617 raise FileNotFoundError(localfile)
618 ssh = self.get_connection()
619 ssh.upload(localfile, remotepath)
620 return remotepath
621 return None
623 @staticmethod
624 def remote_up_cluster_parser():
625 """
626 defines the way to parse the magic command ``%remote_up_cluster``
627 """
628 parser = MagicCommandParser(prog="remote_up_cluster",
629 description='upload a file to the remote machine and then to the cluster')
630 parser.add_argument(
631 'localfile',
632 type=str,
633 help='local file to upload')
634 parser.add_argument(
635 'remotepath',
636 type=str,
637 help='remote path (HDFS) of the uploaded file')
638 return parser
640 @line_magic
641 def remote_up_cluster(self, line):
642 """
643 upload a file to the remote machine and then to the remote cluster,
645 Example::
647 %remote_up_cluster localfile remotepath
649 the command does not allow spaces in files
651 .. nbref::
652 :tag: Hadoop
653 :title: remote_up_cluster
655 The code for magic command ``%remote_up_cluster`` is equivalent to::
657 ssh = ASSHClient(server, username, password)
658 ssh.connect()
659 ssh.upload_cluster(localfile, remotepath)
660 ssh.close()
662 .. versionadded:: 1.1
663 """
664 parser = self.get_parser(
665 MagicRemoteSSH.remote_up_cluster_parser, "remote_up_cluster")
666 args = self.get_args(line, parser)
668 if args is not None:
669 localfile, remotepath = args.localfile, args.remotepath
670 if not os.path.exists(localfile):
671 raise FileNotFoundError(localfile)
672 ssh = self.get_connection()
673 ssh.upload_cluster(localfile, remotepath)
674 return remotepath
675 return None
677 @staticmethod
678 def remote_down_parser():
679 """
680 defines the way to parse the magic command ``%remote_down``
681 """
682 parser = MagicCommandParser(prog="remote_down",
683 description='download a file from the remote machine')
684 parser.add_argument(
685 'remotepath',
686 type=str,
687 help='remote path of the uploaded file')
688 parser.add_argument(
689 'localfile',
690 type=str,
691 help='local file to upload')
692 parser.add_argument(
693 '-o',
694 '--overwrite',
695 action='store_true',
696 default=False,
697 help='overwrite the local file')
698 return parser
700 @line_magic
701 def remote_down(self, line):
702 """
703 download a file from the remote machine,
705 Example::
707 %remote_down remotepath localfile
709 the command does not allow spaces in files
711 .. nbref::
712 :tag: Hadoop
713 :title: remote_down
715 The code for magic command ``%remote_down`` is equivalent to::
717 ssh = ASSHClient(server, username, password)
718 ssh.connect()
719 ssh.download(remotepath, localfile)
720 ssh.close()
721 """
722 parser = self.get_parser(
723 MagicRemoteSSH.remote_down_parser, "remote_down")
724 args = self.get_args(line, parser)
726 if args is not None:
727 localfile, remotepath = args.localfile, args.remotepath
728 ssh = self.get_connection()
729 if os.path.exists(localfile):
730 if args.overwrite:
731 os.remove(localfile)
732 else:
733 raise Exception(
734 "file {0} cannot be overwritten".format(localfile))
735 ssh.download(remotepath, localfile)
736 return localfile
737 return None
739 @staticmethod
740 def remote_down_cluster_parser():
741 """
742 defines the way to parse the magic command ``%remote_down_cluster``
743 """
744 parser = MagicCommandParser(prog="remote_down_cluster",
745 description='download a file from the cluster to the remote machine and then to your local machine')
746 parser.add_argument(
747 'remotepath',
748 type=str,
749 help='remote path (HDFS) of the uploaded file')
750 parser.add_argument(
751 'localfile',
752 type=str,
753 help='local file to upload')
754 parser.add_argument(
755 '-o',
756 '--overwrite',
757 action='store_true',
758 default=False,
759 help='overwrite the local file')
760 parser.add_argument(
761 '-m',
762 '--merge',
763 action='store_true',
764 default=False,
765 help='merges files in folder in a single file')
766 return parser
768 @line_magic
769 def remote_down_cluster(self, line):
770 """
771 download a file from the cluster to the local machine through the bridge
773 Example::
775 %remote_down_cluster remotepath localfile
777 the command does not allow spaces in files
779 .. nbref::
780 :tag: Hadoop
781 :title: remote_down_cluster
783 The code for magic command ``%remote_down_cluster`` is equivalent to::
785 ssh = ASSHClient(server, username, password)
786 ssh.connect()
787 ssh.download_cluster(remotepath, localfile, merge=merge)
788 ssh.close()
790 .. versionadded:: 1.1
791 """
792 parser = self.get_parser(
793 MagicRemoteSSH.remote_down_cluster_parser, "remote_down_cluster")
794 args = self.get_args(line, parser)
796 if args is not None:
797 localfile, remotepath = args.localfile, args.remotepath
798 if os.path.exists(localfile):
799 if args.overwrite:
800 os.remove(localfile)
801 else:
802 raise Exception(
803 "file {0} cannot be overwritten".format(localfile))
804 ssh = self.get_connection()
805 ssh.download_cluster(remotepath, localfile, merge=args.merge)
806 return localfile
807 return None
809 @staticmethod
810 def open_remote_shell_parser():
811 """
812 defines the way to parse the magic command ``%open_remote_shell``
813 """
814 parser = MagicCommandParser(prog="open_remote_shell",
815 description='command will execute as if they were in a shell')
816 parser.add_argument(
817 '-f',
818 '--format',
819 type=str,
820 default='html',
821 help='formart of this output, html or plain')
822 return parser
824 @line_magic
825 def open_remote_shell(self, line):
826 """
827 Defines ``%open_remote_shell``
829 .. nbref::
830 :tag: Hadoop
831 :title: open_remote_shell
833 The code for magic command ``%open_remote_shell`` is equivalent to::
835 ssh = ASSHClient(server, username, password)
836 ssh.connect()
837 ssh.open_session(out_format=format)
838 ssh.close()
842 """
843 parser = self.get_parser(
844 MagicRemoteSSH.open_remote_shell_parser, "open_remote_shell")
845 args = self.get_args(line, parser)
847 if args is not None:
848 ssh = self.get_connection()
849 ssh.open_session(out_format=args.format)
850 return True
851 return None
853 @line_magic
854 def close_remote_shell(self, line):
855 """
856 Defines ``%close_remote_shell``
857 """
858 ssh = self.get_connection()
859 ssh.close_session()
860 return True
862 @line_cell_magic
863 def shell_remote(self, line, cell=None):
864 """
865 Defines ``%shell_remote`` and ``%%shell_remote``
866 """
867 return HTML(self.shell_remote_text(line, cell))
869 @line_cell_magic
870 def shell_remote_text(self, line, cell=None):
871 """
872 Defines ``%shell_remote_text`` and ``%%shell_remote_text``
873 """
874 ssh = self.get_connection()
875 if cell is None:
876 out = ssh.send_recv_session(line)
877 else:
878 out = ssh.send_recv_session(cell)
880 return out
882 @staticmethod
883 def remote_ls_parser():
884 """
885 defines the way to parse the magic command ``%remote_ls``
886 """
887 parser = MagicCommandParser(prog="remote_ls",
888 description='returns the content of a folder as a dataframe')
889 parser.add_argument(
890 'path',
891 type=str,
892 help='path to look into')
893 return parser
895 @line_magic
896 def remote_ls(self, line):
897 """
898 returns the content of a folder on the remote machine as a dataframe
900 Example::
902 %remote_ls .
904 .. nbref::
905 :tag: Hadoop
906 :title: remote_ls
908 The code for magic command ``%remote_ls`` is equivalent to::
910 ssh = ASSHClient(server, username, password)
911 ssh.connect()
912 df = ssh.ls(path)
913 ssh.close()
917 .. versionadded:: 1.1
918 """
919 parser = self.get_parser(MagicRemoteSSH.remote_ls_parser, "remote_ls")
920 args = self.get_args(line, parser)
922 if args is not None:
923 ssh = self.get_connection()
924 df = ssh.ls(args.path)
925 return df
926 return None
928 @staticmethod
929 def dfs_ls_parser():
930 """
931 defines the way to parse the magic command ``%dfs_ls``
932 """
933 parser = MagicCommandParser(prog="dfs_ls",
934 description='returns the content of a folder from the cluster as a dataframe')
935 parser.add_argument(
936 'path',
937 type=str,
938 help='path to look into')
939 return parser
941 @line_magic
942 def dfs_ls(self, line):
943 """
944 returns the content of a folder on the cluster as a dataframe
946 Example::
948 %dfs_ls .
950 .. nbref::
951 :tag: Hadoop
952 :title: dfs_ls
954 The code for magic command ``%dfs_ls`` is equivalent to::
956 ssh = ASSHClient(server, username, password)
957 ssh.connect()
958 df = ssh.dfs_ls(args.path)
959 ssh.close()
963 .. versionadded:: 1.1
964 """
965 parser = self.get_parser(MagicRemoteSSH.dfs_ls_parser, "dfs_ls")
966 args = self.get_args(line, parser)
968 if args is not None:
969 ssh = self.get_connection()
970 df = ssh.dfs_ls(args.path)
971 return df
972 return None
974 @staticmethod
975 def dfs_rm_parser():
976 """
977 defines the way to parse the magic command ``%dfs_rm``
978 """
979 parser = MagicCommandParser(prog="dfs_rm",
980 description='remove a file on the cluster')
981 parser.add_argument(
982 'path',
983 type=str,
984 help='path to remove')
985 parser.add_argument(
986 '-r',
987 '--recursive',
988 action='store_true',
989 default=False,
990 help='to remove subfolders too')
991 return parser
993 @line_magic
994 def dfs_rm(self, line):
995 """
996 remove a file on the cluster
998 Example::
1000 %dfs_rm .
1002 .. nbref::
1003 :tag: Hadoop
1004 :title: dfs_rm
1006 The code for magic command ``%dfs_rm`` is equivalent to::
1008 ssh = ASSHClient(server, username, password)
1009 ssh.connect()
1010 df = ssh.dfs_rm(path, recursive=recursive)
1011 ssh.close()
1015 .. versionadded:: 1.1
1016 """
1017 parser = self.get_parser(MagicRemoteSSH.dfs_rm_parser, "dfs_rm")
1018 args = self.get_args(line, parser)
1020 if args is not None:
1021 ssh = self.get_connection()
1022 df = ssh.dfs_rm(args.path, recursive=args.recursive)
1023 return df
1024 return None
1026 @staticmethod
1027 def dfs_mkdir_parser():
1028 """
1029 defines the way to parse the magic command ``%dfs_mkdir``
1030 """
1031 parser = MagicCommandParser(prog="dfs_mkdir",
1032 description='create a folder')
1033 parser.add_argument(
1034 'path',
1035 type=str,
1036 help='path to remove')
1037 return parser
1039 @line_magic
1040 def dfs_mkdir(self, line):
1041 """
1042 Creates a folder on the cluster.
1044 Example::
1046 %dfs_mkdir afolder
1048 .. nbref::
1049 :tag: Hadoop
1050 :title: dfs_mkdir
1052 The code for magic command ``%dfs_mkdir`` is equivalent to::
1054 ssh = ASSHClient(server, username, password)
1055 ssh.connect()
1056 df = ssh.dfs_mkdir(path)
1057 ssh.close()
1061 .. versionadded:: 1.1
1062 """
1063 parser = self.get_parser(MagicRemoteSSH.dfs_mkdir_parser, "dfs_mkdir")
1064 args = self.get_args(line, parser)
1066 if args is not None:
1067 ssh = self.get_connection()
1068 df = ssh.dfs_mkdir(args.path)
1069 return df
1070 return None
1073def register_magics_ssh(ip=None):
1074 """
1075 register magics function, can be called from a notebook
1077 @param ip from ``get_ipython()``
1078 """
1079 if ip is None:
1080 from IPython import get_ipython
1081 ip = get_ipython()
1082 ip.register_magics(MagicRemoteSSH)