Coverage for src/pyenbc/remote/magic_remote_ssh.py: 0%

367 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-20 05:47 +0200

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Magic command to communicate with an Hadoop cluster (Linux). 

5""" 

6import os 

7 

8from IPython.core.magic import magics_class, line_magic, cell_magic 

9from IPython.core.magic import line_cell_magic 

10from IPython.core.display import HTML 

11from pyquickhelper.ipythonhelper import MagicClassWithHelpers, MagicCommandParser 

12from .ssh_remote_connection import ASSHClient 

13 

14 

15@magics_class 

16class MagicRemoteSSH(MagicClassWithHelpers): 

17 

18 """ 

19 Defines commands to access a remote machine (bridge) through SSH, 

20 for the time being, all the command do not accept another parameters 

21 such as a SSH client which means only one connection 

22 can be opened at the same time. 

23 

24 The notebooks :ref:`pythonhadooppigrst` and 

25 :ref:`exampleofsshclientcommunicationrst` show how these commands can be used. 

26 """ 

27 

28 def _replace_params(self, cell): 

29 """ 

30 replaces parameter such ``__PASSWORD__`` by variable in the notebook environment 

31 

32 @param cell string 

33 @return modified string 

34 """ 

35 if "__PASSWORD__" in cell and self.shell is not None and "password" in self.shell.user_ns: 

36 cell = cell.replace("__PASSWORD__", self.shell.user_ns["password"]) 

37 return cell 

38 

39 def get_connection(self): 

40 """ 

41 returns the connection stored in the workspace 

42 """ 

43 if self.shell is None: 

44 raise Exception("No detected workspace.") 

45 

46 if "remote_ssh" not in self.shell.user_ns: 

47 raise KeyError("No opened SSH connection.") 

48 

49 return self.shell.user_ns["remote_ssh"] 

50 

51 @staticmethod 

52 def PIG_parser(): 

53 """ 

54 defines the way to parse the magic command ``%%PIG`` 

55 """ 

56 parser = MagicCommandParser(prog="PIG", 

57 description='The command store the content of the cell as a local file.') 

58 parser.add_argument( 

59 'file', 

60 type=str, 

61 help='file name') 

62 return parser 

63 

64 @cell_magic 

65 def PIG(self, line, cell=None): 

66 """ 

67 defines command ``%%PIG`` 

68 

69 .. nbref:: 

70 :tag: Hadoop 

71 :title: PIG 

72 

73 The code for magic command ``%PIG`` is equivalent to:: 

74 

75 with open(filename, "w", encoding="utf8") as f: 

76 f.write(script) 

77 

78 

79 """ 

80 parser = self.get_parser(MagicRemoteSSH.PIG_parser, "PIG") 

81 args = self.get_args(line, parser) 

82 

83 if args is not None: 

84 filename = args.file 

85 with open(filename, "w", encoding="utf8") as f: 

86 f.write(cell.replace("\r", "")) 

87 

88 @staticmethod 

89 def HIVE_parser(): 

90 """ 

91 defines the way to parse the magic command ``%%HIVE`` 

92 """ 

93 parser = MagicCommandParser(prog="HIVE", 

94 description='The command store the content of the cell as a local file.') 

95 parser.add_argument( 

96 'file', 

97 type=str, 

98 help='file name') 

99 return parser 

100 

101 @cell_magic 

102 def HIVE(self, line, cell=None): 

103 """ 

104 defines command ``%%HIVE`` 

105 

106 .. nbref:: 

107 :tag: Hadoop 

108 :title: HIVE 

109 

110 The code for magic command ``%HIVE`` is equivalent to:: 

111 

112 with open(filename, "w", encoding="utf8") as f: 

113 f.write(script) 

114 

115 

116 """ 

117 parser = self.get_parser(MagicRemoteSSH.HIVE_parser, "HIVE") 

118 args = self.get_args(line, parser) 

119 

120 if args is not None: 

121 filename = args.file 

122 with open(filename, "w", encoding="utf8") as f: 

123 f.write(cell.replace("\r", "")) 

124 

125 @staticmethod 

126 def pig_submit_parser(): 

127 """ 

128 defines the way to parse the magic command ``%pig_submit`` 

129 """ 

130 parser = MagicCommandParser(prog="pig_submit", 

131 description='Submits a job to the cluster, the job is local, the job ' + 

132 'is first uploaded to the cluster. The magic command populates ' + 

133 'the local variable last_job with the submitted job id.') 

134 parser.add_argument( 

135 'file', 

136 type=str, 

137 help='file name') 

138 parser.add_argument( 

139 '-d', 

140 '--dependency', 

141 nargs="*", 

142 type=str, 

143 help='dependency of the job, the python script') 

144 parser.add_argument( 

145 '-l', 

146 '--local', 

147 action='store_true', 

148 default=False, 

149 help='run locally on the bridge or on the cluster (default)') 

150 parser.add_argument( 

151 '-r', 

152 '--redirection', 

153 type=str, 

154 default="redirection.pig", 

155 help='list of options for the job') 

156 parser.add_argument( 

157 '--raw-output', 

158 default=False, 

159 action='store_true', 

160 help='display raw text instead of HTML') 

161 parser.add_argument( 

162 '-s', 

163 '--stop_on_failure', 

164 action='store_true', 

165 default=False, 

166 help='if true, the job stops on failure right away') 

167 parser.add_argument( 

168 '-o', 

169 '--option', 

170 nargs='*', 

171 type=str, 

172 help='list of options for the job') 

173 return parser 

174 

175 @line_magic 

176 def pig_submit(self, line): 

177 """ 

178 defines command ``%pig_submit`` 

179 

180 .. nbref:: 

181 :tag: Hadoop 

182 :title: pig_submit 

183 

184 The code for magic command ``%pig_submit`` is equivalent to:: 

185 

186 ssh = ASSHClient(server, username, password) 

187 ssh.connect() 

188 out, err = ssh.pig_submit( 

189 pig, dependencies=dependencies, redirection=redirection, local=local, stop_on_failure=stop_on_failure) 

190 ssh.close() 

191 """ 

192 parser = self.get_parser( 

193 MagicRemoteSSH.pig_submit_parser, "pig_submit") 

194 args = self.get_args(line, parser) 

195 

196 if args is not None: 

197 pig = args.file 

198 pys = [_ for _ in args.dependency if _.endswith( 

199 ".py")] if args.dependency is not None else [] 

200 redirection = None if args.redirection in [ 

201 None, "None", "", "-"] else args.redirection 

202 

203 ssh = self.get_connection() 

204 out, err = ssh.pig_submit( 

205 pig, dependencies=pys, redirection=redirection, local=args.local, stop_on_failure=args.stop_on_failure) 

206 

207 if args.raw_output: 

208 if len(err) > 0 and ( 

209 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

210 return err 

211 else: 

212 return out 

213 else: 

214 if len(err) > 0 and ( 

215 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

216 return HTML("<pre>\n%s\n</pre>" % err) 

217 else: 

218 return HTML("<pre>\n%s\n</pre>" % out) 

219 return None 

220 

221 @staticmethod 

222 def hive_submit_parser(): 

223 """ 

224 defines the way to parse the magic command ``%hive_submit`` 

225 """ 

226 parser = MagicCommandParser(prog="hive_submit", 

227 description='Submits a job to the cluster, the job is local, ' + 

228 'the job is first uploaded to the cluster. The magic ' + 

229 'command populates the local variable last_job with the submitted job id.') 

230 parser.add_argument( 

231 'file', 

232 type=str, 

233 help='file name') 

234 parser.add_argument( 

235 '-r', 

236 '--redirection', 

237 type=str, 

238 default="redirection", 

239 help='list of options for the job') 

240 parser.add_argument( 

241 '--raw-output', 

242 default=False, 

243 action='store_true', 

244 help='display raw text instead of HTML') 

245 return parser 

246 

247 @line_magic 

248 def hive_submit(self, line): 

249 """ 

250 defines command ``%hive_submit`` 

251 

252 .. nbref:: 

253 :tag: Hadoop 

254 :title: hive_submit 

255 

256 The code for magic command ``%hive_submit`` is equivalent to:: 

257 

258 ssh = ASSHClient(server, username, password) 

259 ssh.connect() 

260 out, err = ssh.hive_submit( 

261 pig, redirection=redirection, local=local) 

262 ssh.close() 

263 """ 

264 parser = self.get_parser( 

265 MagicRemoteSSH.hive_submit_parser, "hive_submit") 

266 args = self.get_args(line, parser) 

267 

268 if args is not None: 

269 pig = args.file 

270 ssh = self.get_connection() 

271 out, err = ssh.hive_submit( 

272 pig, redirection=args.redirection, local=args.local) 

273 

274 if args.raw_output: 

275 if len(err) > 0 and ( 

276 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

277 return err 

278 else: 

279 return out 

280 else: 

281 if len(err) > 0 and ( 

282 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

283 return HTML("<pre>\n%s\n</pre>" % err) 

284 else: 

285 return HTML("<pre>\n%s\n</pre>" % out) 

286 return None 

287 

288 @staticmethod 

289 def remote_py_parser(): 

290 """ 

291 defines the way to parse the magic command ``%remote_py`` 

292 """ 

293 parser = MagicCommandParser(prog="remote_py", 

294 description='run a python script on the bridge') 

295 parser.add_argument( 

296 'file', 

297 type=str, 

298 help='file name') 

299 parser.add_argument( 

300 'args', 

301 nargs='*', 

302 type=str, 

303 help='list of options for the job') 

304 parser.add_argument( 

305 '-i', 

306 '--interpreter', 

307 type=str, 

308 default='python', 

309 help='change the interpreter, python by default') 

310 parser.add_argument( 

311 '--raw-output', 

312 default=False, 

313 action='store_true', 

314 help='display raw text instead of HTML') 

315 return parser 

316 

317 @line_magic 

318 def remote_py(self, line): 

319 """ 

320 defines command ``%remote_py`` 

321 

322 .. nbref:: 

323 :tag: Hadoop 

324 :title: remote_py 

325 

326 The code for magic command ``%remote_py`` is equivalent to:: 

327 

328 ssh = ASSHClient(server, username, password) 

329 ssh.connect() 

330 ssh.upload(filename, dest) 

331 args = " ".join('"{}"'.format(_) 

332 for _ in args.args) if args.args is not None else "" 

333 out, err = ssh.execute_command(exe + " " + dest + " " + args, no_exception=True) 

334 ssh.close() 

335 """ 

336 parser = self.get_parser( 

337 MagicRemoteSSH.remote_py_parser, "remote_py") 

338 args = self.get_args(line, parser) 

339 

340 if args is not None: 

341 filename = args.file 

342 exe = args.interpreter 

343 args = " ".join('"{}"'.format(_) 

344 for _ in args.args) if args.args is not None else "" 

345 

346 dest = os.path.split(filename)[-1] 

347 ssh = self.get_connection() 

348 ssh.upload(filename, dest) 

349 

350 cmd = exe + " " + dest + " " + args 

351 

352 out, err = ssh.execute_command(cmd, no_exception=True) 

353 if args.raw_output: # pylint: disable=E1101 

354 if len(err) > 0: 

355 return err 

356 else: 

357 return out 

358 else: 

359 if len(err) > 0: 

360 return HTML( 

361 "<b>ERR:</b><br /><pre>\n%s\n</pre><b>OUT:</b><br /><pre>\n%s\n</pre>" % (err, out)) 

362 else: 

363 return HTML("<pre>\n%s\n</pre>" % out) 

364 return None 

365 

366 @staticmethod 

367 def job_syntax_parser(): 

368 """ 

369 defines the way to parse the magic command ``%job_syntax`` 

370 """ 

371 parser = MagicCommandParser(prog="remote_py", 

372 description='check syntax of a pig job') 

373 parser.add_argument( 

374 'file', 

375 type=str, 

376 help='file name') 

377 parser.add_argument( 

378 '--raw-output', 

379 default=False, 

380 action='store_true', 

381 help='display raw text instead of HTML') 

382 return parser 

383 

384 @line_magic 

385 def job_syntax(self, line): 

386 """ 

387 defines command ``%job_syntax`` 

388 """ 

389 parser = self.get_parser( 

390 MagicRemoteSSH.job_syntax_parser, "job_syntax") 

391 args = self.get_args(line, parser) 

392 

393 if args is not None: 

394 filename = args.file 

395 if not os.path.exists(filename): 

396 raise FileNotFoundError(filename) 

397 

398 ssh = self.get_connection() 

399 out, err = ssh.pig_submit(filename, check=True, no_exception=True) 

400 if args.raw_output: 

401 if len(err) > 0 and ( 

402 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

403 return err 

404 else: 

405 return out 

406 else: 

407 if len(err) > 0 and ( 

408 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

409 return HTML("<pre>\n%s\n</pre>" % err) 

410 else: 

411 return HTML("<pre>\n%s\n</pre>" % out) 

412 return None 

413 

414 @staticmethod 

415 def remote_open_parser(): 

416 """ 

417 defines the way to parse the magic command ``%remote_open`` 

418 """ 

419 parser = MagicCommandParser(prog="remote_open", 

420 description='open a remote SSH connection tp the bridge') 

421 parser.add_argument( 

422 '-s', 

423 '--server', 

424 type=str, 

425 default='server', 

426 help='server name') 

427 parser.add_argument( 

428 '-u', 

429 '--username', 

430 type=str, 

431 default='username', 

432 help='username') 

433 parser.add_argument( 

434 '-p', 

435 '--password', 

436 type=str, 

437 default='password', 

438 help='password') 

439 return parser 

440 

441 @line_magic 

442 def remote_open(self, line): 

443 """ 

444 open a SSH connection and store the connection 

445 into the notebook workspace 

446 

447 .. nbref:: 

448 :tag: Hadoop 

449 :title: remote_open 

450 

451 The code for magic command ``%remote_open`` is equivalent to:: 

452 

453 ssh = ASSHClient(server, username, password) 

454 ssh.connect() 

455 """ 

456 parser = self.get_parser( 

457 MagicRemoteSSH.remote_open_parser, "remote_open") 

458 args = self.get_args(line, parser) 

459 

460 if args is not None: 

461 if self.shell is None: 

462 raise Exception("No detected workspace.") 

463 

464 ssh = ASSHClient(args.server, args.username, args.password) 

465 ssh.connect() 

466 

467 self.shell.user_ns["remote_ssh"] = ssh 

468 return ssh 

469 return None 

470 

471 @line_magic 

472 def remote_close(self, line): 

473 """ 

474 close a SSH connection and store the connection 

475 into the notebook workspace 

476 

477 .. nbref:: 

478 :tag: Hadoop 

479 :title: remote_close 

480 

481 The code for magic command ``%remote_close`` is equivalent to:: 

482 

483 ssh = ASSHClient(server, username, password) 

484 # ... ssh.connect() 

485 ssh.close() 

486 

487 

488 """ 

489 self.get_connection().close() 

490 return True 

491 

492 @line_cell_magic 

493 def remote_cmd(self, line, cell=None): 

494 """ 

495 run a command on the remote machine 

496 

497 Example:: 

498 

499 %remote_cmd ls 

500 

501 Or:: 

502 

503 %%remote_cmd <something> 

504 anything going to stdin 

505 

506 In the second case, if __PASSWORD__ is found, it will be replaced by the password stored in 

507 workspace. 

508 

509 .. nbref:: 

510 :tag: Hadoop 

511 :title: remote_cmd 

512 

513 The code for magic command ``%remote_cmd`` is equivalent to:: 

514 

515 ssh = ASSHClient(server, username, password) 

516 ssh.connect() 

517 out, err = ssh.execute_command( 

518 line, no_exception=True, fill_stdin=cell) 

519 ssh.close() 

520 """ 

521 if "--help" in line: 

522 print("Usage: %remote_cmd <cmd>") 

523 else: 

524 ssh = self.get_connection() 

525 

526 if isinstance(cell, str): 

527 cell = self._replace_params(cell) 

528 

529 out, err = ssh.execute_command( 

530 line, no_exception=True, fill_stdin=cell) 

531 if len(err) > 0 and ( 

532 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

533 return HTML("<pre>\n%s\n</pre>" % err) 

534 else: 

535 return HTML("<pre>\n%s\n</pre>" % out) 

536 return None 

537 

538 @line_cell_magic 

539 def remote_cmd_text(self, line, cell=None): 

540 """ 

541 run a command on the remote machine and returns raw text (not HTML) 

542 

543 Example:: 

544 

545 %remote_cmd_text ls 

546 

547 Or:: 

548 

549 %%remote_cmd_text <something> 

550 anything going to stdin 

551 

552 In the second case, if __PASSWORD__ is found, it will be replaced by the password stored in 

553 workspace. 

554 """ 

555 if "--help" in line: 

556 print("Usage: %remote_cmd_text <cmd>") 

557 else: 

558 ssh = self.get_connection() 

559 

560 if isinstance(cell, str): 

561 cell = self._replace_params(cell) 

562 

563 out, err = ssh.execute_command( 

564 line, no_exception=True, fill_stdin=cell) 

565 if len(err) > 0 and ( 

566 len(out) == 0 or "ERROR" in err or "FATAL" in err or "Exception" in err): 

567 return err 

568 else: 

569 return out 

570 return None 

571 

572 @staticmethod 

573 def remote_up_parser(): 

574 """ 

575 defines the way to parse the magic command ``%remote_up`` 

576 """ 

577 parser = MagicCommandParser(prog="remote_up", 

578 description='upload a file to the remote machine') 

579 parser.add_argument( 

580 'localfile', 

581 type=str, 

582 help='local file to upload') 

583 parser.add_argument( 

584 'remotepath', 

585 type=str, 

586 help='remote path of the uploaded file') 

587 return parser 

588 

589 @line_magic 

590 def remote_up(self, line): 

591 """ 

592 upload a file to the remote machine, 

593 

594 Example:: 

595 

596 %remote_up localfile remotepath 

597 

598 the command does not allow spaces in files 

599 

600 .. nbref:: 

601 :tag: Hadoop 

602 :title: remote_up 

603 

604 The code for magic command ``%remote_up`` is equivalent to:: 

605 

606 ssh = ASSHClient(server, username, password) 

607 ssh.connect() 

608 ssh.upload(localfile, remotepath) 

609 ssh.close() 

610 """ 

611 parser = self.get_parser(MagicRemoteSSH.remote_up_parser, "remote_up") 

612 args = self.get_args(line, parser) 

613 

614 if args is not None: 

615 localfile, remotepath = args.localfile, args.remotepath 

616 if not os.path.exists(localfile): 

617 raise FileNotFoundError(localfile) 

618 ssh = self.get_connection() 

619 ssh.upload(localfile, remotepath) 

620 return remotepath 

621 return None 

622 

623 @staticmethod 

624 def remote_up_cluster_parser(): 

625 """ 

626 defines the way to parse the magic command ``%remote_up_cluster`` 

627 """ 

628 parser = MagicCommandParser(prog="remote_up_cluster", 

629 description='upload a file to the remote machine and then to the cluster') 

630 parser.add_argument( 

631 'localfile', 

632 type=str, 

633 help='local file to upload') 

634 parser.add_argument( 

635 'remotepath', 

636 type=str, 

637 help='remote path (HDFS) of the uploaded file') 

638 return parser 

639 

640 @line_magic 

641 def remote_up_cluster(self, line): 

642 """ 

643 upload a file to the remote machine and then to the remote cluster, 

644 

645 Example:: 

646 

647 %remote_up_cluster localfile remotepath 

648 

649 the command does not allow spaces in files 

650 

651 .. nbref:: 

652 :tag: Hadoop 

653 :title: remote_up_cluster 

654 

655 The code for magic command ``%remote_up_cluster`` is equivalent to:: 

656 

657 ssh = ASSHClient(server, username, password) 

658 ssh.connect() 

659 ssh.upload_cluster(localfile, remotepath) 

660 ssh.close() 

661 

662 .. versionadded:: 1.1 

663 """ 

664 parser = self.get_parser( 

665 MagicRemoteSSH.remote_up_cluster_parser, "remote_up_cluster") 

666 args = self.get_args(line, parser) 

667 

668 if args is not None: 

669 localfile, remotepath = args.localfile, args.remotepath 

670 if not os.path.exists(localfile): 

671 raise FileNotFoundError(localfile) 

672 ssh = self.get_connection() 

673 ssh.upload_cluster(localfile, remotepath) 

674 return remotepath 

675 return None 

676 

677 @staticmethod 

678 def remote_down_parser(): 

679 """ 

680 defines the way to parse the magic command ``%remote_down`` 

681 """ 

682 parser = MagicCommandParser(prog="remote_down", 

683 description='download a file from the remote machine') 

684 parser.add_argument( 

685 'remotepath', 

686 type=str, 

687 help='remote path of the uploaded file') 

688 parser.add_argument( 

689 'localfile', 

690 type=str, 

691 help='local file to upload') 

692 parser.add_argument( 

693 '-o', 

694 '--overwrite', 

695 action='store_true', 

696 default=False, 

697 help='overwrite the local file') 

698 return parser 

699 

700 @line_magic 

701 def remote_down(self, line): 

702 """ 

703 download a file from the remote machine, 

704 

705 Example:: 

706 

707 %remote_down remotepath localfile 

708 

709 the command does not allow spaces in files 

710 

711 .. nbref:: 

712 :tag: Hadoop 

713 :title: remote_down 

714 

715 The code for magic command ``%remote_down`` is equivalent to:: 

716 

717 ssh = ASSHClient(server, username, password) 

718 ssh.connect() 

719 ssh.download(remotepath, localfile) 

720 ssh.close() 

721 """ 

722 parser = self.get_parser( 

723 MagicRemoteSSH.remote_down_parser, "remote_down") 

724 args = self.get_args(line, parser) 

725 

726 if args is not None: 

727 localfile, remotepath = args.localfile, args.remotepath 

728 ssh = self.get_connection() 

729 if os.path.exists(localfile): 

730 if args.overwrite: 

731 os.remove(localfile) 

732 else: 

733 raise Exception( 

734 "file {0} cannot be overwritten".format(localfile)) 

735 ssh.download(remotepath, localfile) 

736 return localfile 

737 return None 

738 

739 @staticmethod 

740 def remote_down_cluster_parser(): 

741 """ 

742 defines the way to parse the magic command ``%remote_down_cluster`` 

743 """ 

744 parser = MagicCommandParser(prog="remote_down_cluster", 

745 description='download a file from the cluster to the remote machine and then to your local machine') 

746 parser.add_argument( 

747 'remotepath', 

748 type=str, 

749 help='remote path (HDFS) of the uploaded file') 

750 parser.add_argument( 

751 'localfile', 

752 type=str, 

753 help='local file to upload') 

754 parser.add_argument( 

755 '-o', 

756 '--overwrite', 

757 action='store_true', 

758 default=False, 

759 help='overwrite the local file') 

760 parser.add_argument( 

761 '-m', 

762 '--merge', 

763 action='store_true', 

764 default=False, 

765 help='merges files in folder in a single file') 

766 return parser 

767 

768 @line_magic 

769 def remote_down_cluster(self, line): 

770 """ 

771 download a file from the cluster to the local machine through the bridge 

772 

773 Example:: 

774 

775 %remote_down_cluster remotepath localfile 

776 

777 the command does not allow spaces in files 

778 

779 .. nbref:: 

780 :tag: Hadoop 

781 :title: remote_down_cluster 

782 

783 The code for magic command ``%remote_down_cluster`` is equivalent to:: 

784 

785 ssh = ASSHClient(server, username, password) 

786 ssh.connect() 

787 ssh.download_cluster(remotepath, localfile, merge=merge) 

788 ssh.close() 

789 

790 .. versionadded:: 1.1 

791 """ 

792 parser = self.get_parser( 

793 MagicRemoteSSH.remote_down_cluster_parser, "remote_down_cluster") 

794 args = self.get_args(line, parser) 

795 

796 if args is not None: 

797 localfile, remotepath = args.localfile, args.remotepath 

798 if os.path.exists(localfile): 

799 if args.overwrite: 

800 os.remove(localfile) 

801 else: 

802 raise Exception( 

803 "file {0} cannot be overwritten".format(localfile)) 

804 ssh = self.get_connection() 

805 ssh.download_cluster(remotepath, localfile, merge=args.merge) 

806 return localfile 

807 return None 

808 

809 @staticmethod 

810 def open_remote_shell_parser(): 

811 """ 

812 defines the way to parse the magic command ``%open_remote_shell`` 

813 """ 

814 parser = MagicCommandParser(prog="open_remote_shell", 

815 description='command will execute as if they were in a shell') 

816 parser.add_argument( 

817 '-f', 

818 '--format', 

819 type=str, 

820 default='html', 

821 help='formart of this output, html or plain') 

822 return parser 

823 

824 @line_magic 

825 def open_remote_shell(self, line): 

826 """ 

827 Defines ``%open_remote_shell`` 

828 

829 .. nbref:: 

830 :tag: Hadoop 

831 :title: open_remote_shell 

832 

833 The code for magic command ``%open_remote_shell`` is equivalent to:: 

834 

835 ssh = ASSHClient(server, username, password) 

836 ssh.connect() 

837 ssh.open_session(out_format=format) 

838 ssh.close() 

839 

840 

841 

842 """ 

843 parser = self.get_parser( 

844 MagicRemoteSSH.open_remote_shell_parser, "open_remote_shell") 

845 args = self.get_args(line, parser) 

846 

847 if args is not None: 

848 ssh = self.get_connection() 

849 ssh.open_session(out_format=args.format) 

850 return True 

851 return None 

852 

853 @line_magic 

854 def close_remote_shell(self, line): 

855 """ 

856 Defines ``%close_remote_shell`` 

857 """ 

858 ssh = self.get_connection() 

859 ssh.close_session() 

860 return True 

861 

862 @line_cell_magic 

863 def shell_remote(self, line, cell=None): 

864 """ 

865 Defines ``%shell_remote`` and ``%%shell_remote`` 

866 """ 

867 return HTML(self.shell_remote_text(line, cell)) 

868 

869 @line_cell_magic 

870 def shell_remote_text(self, line, cell=None): 

871 """ 

872 Defines ``%shell_remote_text`` and ``%%shell_remote_text`` 

873 """ 

874 ssh = self.get_connection() 

875 if cell is None: 

876 out = ssh.send_recv_session(line) 

877 else: 

878 out = ssh.send_recv_session(cell) 

879 

880 return out 

881 

882 @staticmethod 

883 def remote_ls_parser(): 

884 """ 

885 defines the way to parse the magic command ``%remote_ls`` 

886 """ 

887 parser = MagicCommandParser(prog="remote_ls", 

888 description='returns the content of a folder as a dataframe') 

889 parser.add_argument( 

890 'path', 

891 type=str, 

892 help='path to look into') 

893 return parser 

894 

895 @line_magic 

896 def remote_ls(self, line): 

897 """ 

898 returns the content of a folder on the remote machine as a dataframe 

899 

900 Example:: 

901 

902 %remote_ls . 

903 

904 .. nbref:: 

905 :tag: Hadoop 

906 :title: remote_ls 

907 

908 The code for magic command ``%remote_ls`` is equivalent to:: 

909 

910 ssh = ASSHClient(server, username, password) 

911 ssh.connect() 

912 df = ssh.ls(path) 

913 ssh.close() 

914 

915 

916 

917 .. versionadded:: 1.1 

918 """ 

919 parser = self.get_parser(MagicRemoteSSH.remote_ls_parser, "remote_ls") 

920 args = self.get_args(line, parser) 

921 

922 if args is not None: 

923 ssh = self.get_connection() 

924 df = ssh.ls(args.path) 

925 return df 

926 return None 

927 

928 @staticmethod 

929 def dfs_ls_parser(): 

930 """ 

931 defines the way to parse the magic command ``%dfs_ls`` 

932 """ 

933 parser = MagicCommandParser(prog="dfs_ls", 

934 description='returns the content of a folder from the cluster as a dataframe') 

935 parser.add_argument( 

936 'path', 

937 type=str, 

938 help='path to look into') 

939 return parser 

940 

941 @line_magic 

942 def dfs_ls(self, line): 

943 """ 

944 returns the content of a folder on the cluster as a dataframe 

945 

946 Example:: 

947 

948 %dfs_ls . 

949 

950 .. nbref:: 

951 :tag: Hadoop 

952 :title: dfs_ls 

953 

954 The code for magic command ``%dfs_ls`` is equivalent to:: 

955 

956 ssh = ASSHClient(server, username, password) 

957 ssh.connect() 

958 df = ssh.dfs_ls(args.path) 

959 ssh.close() 

960 

961 

962 

963 .. versionadded:: 1.1 

964 """ 

965 parser = self.get_parser(MagicRemoteSSH.dfs_ls_parser, "dfs_ls") 

966 args = self.get_args(line, parser) 

967 

968 if args is not None: 

969 ssh = self.get_connection() 

970 df = ssh.dfs_ls(args.path) 

971 return df 

972 return None 

973 

974 @staticmethod 

975 def dfs_rm_parser(): 

976 """ 

977 defines the way to parse the magic command ``%dfs_rm`` 

978 """ 

979 parser = MagicCommandParser(prog="dfs_rm", 

980 description='remove a file on the cluster') 

981 parser.add_argument( 

982 'path', 

983 type=str, 

984 help='path to remove') 

985 parser.add_argument( 

986 '-r', 

987 '--recursive', 

988 action='store_true', 

989 default=False, 

990 help='to remove subfolders too') 

991 return parser 

992 

993 @line_magic 

994 def dfs_rm(self, line): 

995 """ 

996 remove a file on the cluster 

997 

998 Example:: 

999 

1000 %dfs_rm . 

1001 

1002 .. nbref:: 

1003 :tag: Hadoop 

1004 :title: dfs_rm 

1005 

1006 The code for magic command ``%dfs_rm`` is equivalent to:: 

1007 

1008 ssh = ASSHClient(server, username, password) 

1009 ssh.connect() 

1010 df = ssh.dfs_rm(path, recursive=recursive) 

1011 ssh.close() 

1012 

1013 

1014 

1015 .. versionadded:: 1.1 

1016 """ 

1017 parser = self.get_parser(MagicRemoteSSH.dfs_rm_parser, "dfs_rm") 

1018 args = self.get_args(line, parser) 

1019 

1020 if args is not None: 

1021 ssh = self.get_connection() 

1022 df = ssh.dfs_rm(args.path, recursive=args.recursive) 

1023 return df 

1024 return None 

1025 

1026 @staticmethod 

1027 def dfs_mkdir_parser(): 

1028 """ 

1029 defines the way to parse the magic command ``%dfs_mkdir`` 

1030 """ 

1031 parser = MagicCommandParser(prog="dfs_mkdir", 

1032 description='create a folder') 

1033 parser.add_argument( 

1034 'path', 

1035 type=str, 

1036 help='path to remove') 

1037 return parser 

1038 

1039 @line_magic 

1040 def dfs_mkdir(self, line): 

1041 """ 

1042 Creates a folder on the cluster. 

1043 

1044 Example:: 

1045 

1046 %dfs_mkdir afolder 

1047 

1048 .. nbref:: 

1049 :tag: Hadoop 

1050 :title: dfs_mkdir 

1051 

1052 The code for magic command ``%dfs_mkdir`` is equivalent to:: 

1053 

1054 ssh = ASSHClient(server, username, password) 

1055 ssh.connect() 

1056 df = ssh.dfs_mkdir(path) 

1057 ssh.close() 

1058 

1059 

1060 

1061 .. versionadded:: 1.1 

1062 """ 

1063 parser = self.get_parser(MagicRemoteSSH.dfs_mkdir_parser, "dfs_mkdir") 

1064 args = self.get_args(line, parser) 

1065 

1066 if args is not None: 

1067 ssh = self.get_connection() 

1068 df = ssh.dfs_mkdir(args.path) 

1069 return df 

1070 return None 

1071 

1072 

1073def register_magics_ssh(ip=None): 

1074 """ 

1075 register magics function, can be called from a notebook 

1076 

1077 @param ip from ``get_ipython()`` 

1078 """ 

1079 if ip is None: 

1080 from IPython import get_ipython 

1081 ip = get_ipython() 

1082 ip.register_magics(MagicRemoteSSH)