Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Magic command to handle files 

5""" 

6import sys 

7import os 

8import pandas 

9 

10from IPython.core.magic import magics_class, line_magic, cell_magic 

11from IPython.core.display import HTML 

12 

13from pyquickhelper.filehelper.synchelper import explore_folder_iterfile, explore_folder_iterfile_repo 

14from pyquickhelper.loghelper import run_cmd 

15from pyquickhelper.ipythonhelper import MagicCommandParser, MagicClassWithHelpers 

16from pyquickhelper.helpgen import docstring2html 

17from .format_helper import format_file_size, format_file_mtime 

18from .content_helper import file_head, file_tail, enumerate_grep, file_encoding 

19 

20 

21@magics_class 

22class MagicFile(MagicClassWithHelpers): 

23 

24 """ 

25 Defines magic commands to help with files 

26 

27 .. faqref:: 

28 :title: Magic command not found 

29 

30 Magic commands are automatically added when importing 

31 the module:: 

32 

33 import pyensae 

34 

35 This instruction does not raise any exception. 

36 To understand what went wrong, the following instruction must be run:: 

37 

38 %load_ext pyensae 

39 

40 Usually, one necessary module is missing. 

41 

42 .. versionadded:: 1.1 

43 """ 

44 

45 @staticmethod 

46 def head_parser(): 

47 """ 

48 defines the way to parse the magic command ``%head`` 

49 """ 

50 parser = MagicCommandParser(prog="head", 

51 description='display the first lines of a text file') 

52 parser.add_argument('f', type=str, help='filename') 

53 parser.add_argument( 

54 '-n', 

55 '--n', 

56 type=int, 

57 default=10, 

58 help='number of lines to display') 

59 parser.add_argument( 

60 '-r', 

61 '--raw', 

62 default=False, 

63 action='store_true', 

64 help='display raw text instead of HTML') 

65 parser.add_argument( 

66 '-e', 

67 '--encoding', 

68 default="utf8", 

69 help='file encoding') 

70 parser.add_argument( 

71 '-s', 

72 '--errors', 

73 default="", 

74 help='What about errors: "", strict, replace, surrogateescape, ' 

75 'xmlcharrefreplace, backslashreplace, namereplace') 

76 return parser 

77 

78 @line_magic 

79 def head(self, line): 

80 """ 

81 Defines ``%head`` 

82 which displays the first lines of a file. 

83 

84 .. nbref:: 

85 :tag: file 

86 :title: head 

87 

88 The magic command ``%head`` is equivalent to:: 

89 

90 from pyensae.file_helper import file_head 

91 file_head(<filename>, <n>, <encoding>) 

92 """ 

93 parser = self.get_parser(MagicFile.head_parser, "head") 

94 args = self.get_args(line, parser) 

95 

96 if args is not None: 

97 errors = args.errors 

98 if errors is not None and len(errors) < 2: 

99 errors = None 

100 encoding = args.encoding 

101 if encoding is not None and len(encoding) < 2: 

102 encoding = None 

103 rows = file_head(args.f, args.n, encoding, errors=errors) 

104 if args.raw: 

105 return "".join(rows) 

106 else: 

107 return HTML("<pre>\n{0}\n</pre>".format("".join(rows))) 

108 return None 

109 

110 @staticmethod 

111 def encoding_parser(): 

112 """ 

113 Defines the way to parse the magic command ``%encoding``. 

114 """ 

115 parser = MagicCommandParser(prog="encoding", 

116 description='guess the encoding of a file') 

117 parser.add_argument('f', type=str, help='filename') 

118 parser.add_argument( 

119 '-n', 

120 '--n', 

121 type=int, 

122 default=2**20, 

123 help='maximum number of lines to use to guess the encoding') 

124 return parser 

125 

126 @line_magic 

127 def encoding(self, line): 

128 """ 

129 Defines ``%encoding`` 

130 which guesses the encoding. 

131 

132 .. nbref:: 

133 :tag: file 

134 :title: encoding 

135 

136 The magic command ``%encoding`` is equivalent to:: 

137 

138 from pyensae.file_helper import file_head 

139 file_head(<filename>, <n>, <encoding>) 

140 """ 

141 parser = self.get_parser(MagicFile.encoding_parser, "encoding") 

142 args = self.get_args(line, parser) 

143 

144 if args is not None: 

145 res = file_encoding(args.f, args.n) 

146 return str(res) 

147 return None 

148 

149 @staticmethod 

150 def grep_parser(): 

151 """ 

152 defines the way to parse the magic command ``%grep`` 

153 """ 

154 parser = MagicCommandParser(prog="grep", 

155 description='display the first lines of a text file') 

156 parser.add_argument('f', type=str, help='filename') 

157 parser.add_argument('regex', type=str, help='regular expression') 

158 parser.add_argument( 

159 '-n', 

160 '--n', 

161 type=int, 

162 default=-1, 

163 help='number of lines to display, -1 for all') 

164 parser.add_argument( 

165 '-r', 

166 '--raw', 

167 default=False, 

168 action='store_true', 

169 help='display raw text instead of HTML') 

170 parser.add_argument( 

171 '-e', 

172 '--encoding', 

173 default="utf8", 

174 help='file encoding') 

175 parser.add_argument( 

176 '-s', 

177 '--errors', 

178 default="", 

179 help='What about errors: "", strict, replace, surrogateescape, ' 

180 'xmlcharrefreplace, backslashreplace, namereplace') 

181 return parser 

182 

183 @line_magic 

184 def grep(self, line): 

185 """ 

186 defines ``%grep`` 

187 which displays the first lines of a file 

188 

189 .. nbref:: 

190 :tag: file 

191 :title: grep 

192 

193 The magic command ``%grep`` is equivalent to:: 

194 

195 from pyensae.file_helper import enumerate_grep 

196 list(enumerate_grep(<filename>, <regex>, <encoding>)) 

197 """ 

198 parser = self.get_parser(MagicFile.grep_parser, "grep") 

199 args = self.get_args(line, parser) 

200 

201 if args is not None: 

202 errors = args.errors 

203 if errors is not None and len(errors) < 2: 

204 errors = None 

205 encoding = args.encoding 

206 if encoding is not None and len(encoding) < 2: 

207 encoding = None 

208 iter = enumerate_grep(args.f, args.regex, 

209 encoding, errors=errors) 

210 if args.n != -1: 

211 rows = [] 

212 for r in iter: 

213 if len(rows) >= args.n: 

214 break 

215 rows.append(r) 

216 else: 

217 rows = list(iter) 

218 

219 if args.raw: 

220 return "".join(rows) 

221 else: 

222 return HTML("<pre>\n{0}\n</pre>".format("".join(rows))) 

223 return None 

224 

225 @staticmethod 

226 def tail_parser(): 

227 """ 

228 defines the way to parse the magic command ``%tail`` 

229 """ 

230 parser = MagicCommandParser(prog="tail", 

231 description='display the last lines of a text file') 

232 parser.add_argument('f', type=str, help='filename') 

233 parser.add_argument( 

234 '-n', 

235 '--n', 

236 type=int, 

237 default=10, 

238 help='number of lines to display') 

239 parser.add_argument( 

240 '-r', 

241 '--raw', 

242 default=False, 

243 action='store_true', 

244 help='display raw text instead of HTML') 

245 parser.add_argument( 

246 '-e', 

247 '--encoding', 

248 default="utf8", 

249 help='file encoding') 

250 parser.add_argument( 

251 '-s', 

252 '--errors', 

253 default="", 

254 help='What about errors: "", strict, replace, surrogateescape, ' 

255 'xmlcharrefreplace, backslashreplace, namereplace') 

256 return parser 

257 

258 @line_magic 

259 def tail(self, line): 

260 """ 

261 defines ``%tail`` 

262 which displays the last lines of a file 

263 

264 .. nbref:: 

265 :tag: file 

266 :title: tail 

267 

268 The magic command ``%tail`` is equivalent to:: 

269 

270 from pyensae.file_helper import file_tail 

271 file_tail(<filename>, <n>, <encoding>) 

272 """ 

273 parser = self.get_parser(MagicFile.tail_parser, "tail") 

274 args = self.get_args(line, parser) 

275 

276 if args is not None: 

277 errors = args.errors 

278 if errors is not None and len(errors) < 2: 

279 errors = None 

280 encoding = args.encoding 

281 if encoding is not None and len(encoding) < 2: 

282 encoding = None 

283 rows = file_tail(args.f, args.n, encoding, errors=errors) 

284 if args.raw: 

285 return "".join(rows) 

286 else: 

287 return HTML("<pre>\n{0}\n</pre>".format("".join(rows))) 

288 return None 

289 

290 @staticmethod 

291 def lsr_parser(): 

292 """ 

293 defines the way to parse the magic command ``%lsr`` 

294 """ 

295 parser = MagicCommandParser(prog="lsr", 

296 description='display the content of a folder as a dataframe') 

297 parser.add_argument( 

298 'path', 

299 type=str, 

300 nargs="?", 

301 help='path', 

302 default=".") 

303 parser.add_argument( 

304 '-f', 

305 '--filter', 

306 type=str, 

307 default=".*", 

308 help='filter, same syntax as a regular expression') 

309 return parser 

310 

311 @line_magic 

312 def lsr(self, line): 

313 """ 

314 Defines ``%lsr`` which returns the content of a folder, 

315 the method stops after around 10000 files 

316 --> you should precise the filter. 

317 

318 .. nbref:: 

319 :tag: file 

320 :title: lsr 

321 

322 The magic command ``%lsr`` is almost equivalent to:: 

323 

324 from pyquickhelper.file_helper import explore_folder_iterfile 

325 res = explore_folder_iterfile(<filename>, <pattern>) 

326 for f in res: 

327 print(f) 

328 """ 

329 parser = self.get_parser(MagicFile.lsr_parser, "lsr") 

330 args = self.get_args(line, parser) 

331 

332 if args is not None: 

333 if args.path is None or len(args.path) == 0: 

334 filename = "." 

335 else: 

336 filename = args.path 

337 pattern = args.filter 

338 

339 if "*" in filename: 

340 pattern = filename 

341 filename = "." 

342 

343 iter = explore_folder_iterfile(filename, pattern) 

344 rows = [] 

345 for r in iter: 

346 d = os.path.isfile(r) 

347 if d: 

348 st = os.stat(r) 

349 r = {"name": r, 

350 "size": format_file_size(st.st_size), 

351 "last_modified": format_file_mtime(st.st_mtime), 

352 "directory": False} 

353 else: 

354 r = {"name": r, "directory": True} 

355 rows.append(r) 

356 return pandas.DataFrame(rows) 

357 return None 

358 

359 @staticmethod 

360 def PYTHON_parser(): 

361 """ 

362 Defines the way to parse the magic command ``%%PYTHON``. 

363 """ 

364 parser = MagicCommandParser(prog="PYTHON", 

365 description='the command stores the content of the cell as a local file.') 

366 parser.add_argument( 

367 'file', 

368 type=str, 

369 help='filename') 

370 return parser 

371 

372 @cell_magic 

373 def PYTHON(self, line, cell=None): 

374 """ 

375 Defines command ``%%PYTHON``. 

376 

377 .. nbref:: 

378 :title: PYTHON 

379 

380 The magic command ``%%PYTHON`` is almost to: 

381 

382 :: 

383 

384 with open(<filename>, "w", encoding="utf8") as f: 

385 f.write("# -*- coding: utf8 -*-\\n") 

386 f.write(cell.replace("\\r", "")) 

387 """ 

388 parser = self.get_parser(MagicFile.PYTHON_parser, "PYTHON") 

389 args = self.get_args(line, parser) 

390 

391 if args is not None: 

392 filename = args.file 

393 with open(filename, "w", encoding="utf8") as f: 

394 f.write("# -*- coding: utf8 -*-\n") 

395 f.write(cell.replace("\r", "")) 

396 

397 @staticmethod 

398 def runpy_parser(): 

399 """ 

400 Defines the way to parse the magic command ``%%runpy``. 

401 """ 

402 parser = MagicCommandParser(prog="runpy", 

403 description='run a python script which accepts standards input and ' + 

404 'produces standard outputs, a timeout is set up at 10s') 

405 parser.add_argument( 

406 'file', 

407 type=str, 

408 help='python file') 

409 parser.add_argument( 

410 'args', 

411 type=str, 

412 nargs="*", 

413 help='arguments for the scripts', 

414 default=".") 

415 return parser 

416 

417 @cell_magic 

418 def runpy(self, line, cell=None): 

419 """ 

420 Defines command ``%%runpy``. 

421 

422 .. nbref:: 

423 :title: runpy 

424 

425 ``%%runpy`` runs a python script which accepts 

426 standards input and produces standard outputs, 

427 a timeout is set up at 10s. It is almost equivalent to:: 

428 

429 from pyquickhelper.loghelper import run_cmd 

430 import sys 

431 cmd = sys.executable.replace( 

432 "pythonw", 

433 "python") + " " + filename + " " + args 

434 out, err = run_cmd( 

435 cmd, wait=True, sin=cell, communicate=True, timeout=10, shell=False) 

436 

437 .. versionadded:: 1.1 

438 """ 

439 parser = self.get_parser(MagicFile.runpy_parser, "runpy") 

440 args = self.get_args(line, parser) 

441 

442 if args is not None: 

443 filename = args.file 

444 if len(filename) == 0: 

445 self.runpy("") 

446 else: 

447 args = args.args 

448 cmd = sys.executable.replace("pythonw", "python") 

449 cmd += " " + filename + " " 

450 cmd += " ".join('"{0}"'.format(_) 

451 for _ in args) if isinstance(args, list) else args 

452 tosend = cell 

453 out, err = run_cmd( 

454 cmd, wait=True, sin=tosend, communicate=True, timeout=10, shell=False) 

455 if len(err) > 0: 

456 return HTML( 

457 '<font color="#DD0000">Error</font><br /><pre>\n%s\n</pre>' % err) 

458 else: 

459 return HTML('<pre>\n%s\n</pre>' % out) 

460 return None 

461 

462 @staticmethod 

463 def lsrepo_parser(): 

464 """ 

465 Defines the way to parse the magic command ``%lsrepo``. 

466 """ 

467 parser = MagicCommandParser(prog="lsrepo", 

468 description='display the content of a repository (GIT or SVN)') 

469 parser.add_argument('path', type=str, nargs="?", 

470 help='path', default=".") 

471 return parser 

472 

473 @line_magic 

474 def lsrepo(self, line): 

475 """ 

476 Defines ``%lsrepo``. 

477 

478 .. nbref:: 

479 :tag: file 

480 :title: lsrepo 

481 

482 The method returns the files present in a repository 

483 (:epkg:`GIT` or :pkg:`SVN`). The code is equivalent to:: 

484 

485 from pyquickhelper.filehelper import explore_folder_iterfile_repo 

486 res = explore_folder_iterfile_repo(<filename>, <pattern>) 

487 for f in res: 

488 print(f) 

489 

490 .. versionadded:: 1.1 

491 """ 

492 parser = self.get_parser(MagicFile.lsrepo_parser, "lsrepo") 

493 args = self.get_args(line, parser) 

494 

495 if args is not None: 

496 if args.path is None or len(args.path) == 0: 

497 filename = "." 

498 else: 

499 filename = args.path 

500 

501 iter = explore_folder_iterfile_repo(filename) 

502 rows = [] 

503 for r in iter: 

504 d = os.path.isfile(r) 

505 if d: 

506 st = os.stat(r) 

507 r = {"name": r, 

508 "size": format_file_size(st.st_size), 

509 "last_modified": format_file_mtime(st.st_mtime), 

510 "directory": False} 

511 else: 

512 r = {"name": r, "directory": True} 

513 rows.append(r) 

514 return pandas.DataFrame(rows) 

515 return None 

516 

517 @staticmethod 

518 def hhelp_parser(): 

519 """ 

520 Defines the way to parse the magic command ``%hhelp``. 

521 

522 .. versionadded:: 1.1 

523 """ 

524 parser = MagicCommandParser(prog="hhelp", 

525 description='display help for an object in HTML format') 

526 parser.add_argument( 

527 'obj', 

528 type=str, 

529 help='a python object') 

530 parser.add_argument( 

531 '-f', 

532 '--format', 

533 type=str, 

534 default="html", 

535 help='format', 

536 choices=['text', 'html', 'rst', 'rawhtml']) 

537 parser.add_argument( 

538 '-np', 

539 '--no-print', 

540 action='store_true', 

541 help='by default, the magic command outputs everything on the standard output, ' 

542 'if specified, it returns a string') 

543 return parser 

544 

545 @line_magic 

546 def hhelp(self, line): 

547 """ 

548 Define ``%hhelp``, it displays the help for an object in :epkg:`HTML`. 

549 

550 .. nbref:: 

551 :title: hhelp 

552 

553 The magic command is equivalent to:: 

554 

555 from pyquickhelper import docstring2html 

556 docstring2html(obj, format=format) 

557 

558 .. versionadded:: 1.1 

559 """ 

560 parser = self.get_parser(MagicFile.hhelp_parser, "hhelp") 

561 args = self.get_args(line, parser) 

562 

563 if args is not None: 

564 obj = args.obj 

565 format = args.format 

566 nop = args.no_print 

567 if nop or format == "html": 

568 return docstring2html(obj, format=format) 

569 else: 

570 print(docstring2html(obj, format=format)) 

571 return None 

572 

573 

574def register_file_magics(ip=None): 

575 """ 

576 register magics function, can be called from a notebook 

577 

578 @param ip from ``get_ipython()`` 

579 """ 

580 if ip is None: 

581 from IPython import get_ipython 

582 ip = get_ipython() 

583 ip.register_magics(MagicFile)