Coverage for src/pyensae/filehelper/magic_file.py: 72%
226 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-03 02:16 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-03 02:16 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Magic command to handle files
5"""
6import sys
7import os
8import pandas
10from IPython.core.magic import magics_class, line_magic, cell_magic
11from IPython.core.display import HTML
13from pyquickhelper.filehelper.synchelper import explore_folder_iterfile, explore_folder_iterfile_repo
14from pyquickhelper.loghelper import run_cmd
15from pyquickhelper.ipythonhelper import MagicCommandParser, MagicClassWithHelpers
16from pyquickhelper.helpgen import docstring2html
17from .format_helper import format_file_size, format_file_mtime
18from .content_helper import file_head, file_tail, enumerate_grep, file_encoding
21@magics_class
22class MagicFile(MagicClassWithHelpers):
24 """
25 Defines magic commands to help with files
27 .. faqref::
28 :title: Magic command not found
30 Magic commands are automatically added when importing
31 the module::
33 import pyensae
35 This instruction does not raise any exception.
36 To understand what went wrong, the following instruction must be run::
38 %load_ext pyensae
40 Usually, one necessary module is missing.
42 .. versionadded:: 1.1
43 """
45 @staticmethod
46 def head_parser():
47 """
48 defines the way to parse the magic command ``%head``
49 """
50 parser = MagicCommandParser(prog="head",
51 description='display the first lines of a text file')
52 parser.add_argument('f', type=str, help='filename')
53 parser.add_argument(
54 '-n',
55 '--n',
56 type=int,
57 default=10,
58 help='number of lines to display')
59 parser.add_argument(
60 '-r',
61 '--raw',
62 default=False,
63 action='store_true',
64 help='display raw text instead of HTML')
65 parser.add_argument(
66 '-e',
67 '--encoding',
68 default="utf8",
69 help='file encoding')
70 parser.add_argument(
71 '-s',
72 '--errors',
73 default="",
74 help='What about errors: "", strict, replace, surrogateescape, '
75 'xmlcharrefreplace, backslashreplace, namereplace')
76 return parser
78 @line_magic
79 def head(self, line):
80 """
81 Defines ``%head``
82 which displays the first lines of a file.
84 .. nbref::
85 :tag: file
86 :title: head
88 The magic command ``%head`` is equivalent to::
90 from pyensae.file_helper import file_head
91 file_head(<filename>, <n>, <encoding>)
92 """
93 parser = self.get_parser(MagicFile.head_parser, "head")
94 args = self.get_args(line, parser)
96 if args is not None:
97 errors = args.errors
98 if errors is not None and len(errors) < 2:
99 errors = None
100 encoding = args.encoding
101 if encoding is not None and len(encoding) < 2:
102 encoding = None
103 rows = file_head(args.f, args.n, encoding, errors=errors)
104 if args.raw:
105 return "".join(rows)
106 else:
107 return HTML("<pre>\n{0}\n</pre>".format("".join(rows)))
108 return None
110 @staticmethod
111 def encoding_parser():
112 """
113 Defines the way to parse the magic command ``%encoding``.
114 """
115 parser = MagicCommandParser(prog="encoding",
116 description='guess the encoding of a file')
117 parser.add_argument('f', type=str, help='filename')
118 parser.add_argument(
119 '-n',
120 '--n',
121 type=int,
122 default=2**20,
123 help='maximum number of lines to use to guess the encoding')
124 return parser
126 @line_magic
127 def encoding(self, line):
128 """
129 Defines ``%encoding``
130 which guesses the encoding.
132 .. nbref::
133 :tag: file
134 :title: encoding
136 The magic command ``%encoding`` is equivalent to::
138 from pyensae.file_helper import file_head
139 file_head(<filename>, <n>, <encoding>)
140 """
141 parser = self.get_parser(MagicFile.encoding_parser, "encoding")
142 args = self.get_args(line, parser)
144 if args is not None:
145 res = file_encoding(args.f, args.n)
146 return str(res)
147 return None
149 @staticmethod
150 def grep_parser():
151 """
152 defines the way to parse the magic command ``%grep``
153 """
154 parser = MagicCommandParser(prog="grep",
155 description='display the first lines of a text file')
156 parser.add_argument('f', type=str, help='filename')
157 parser.add_argument('regex', type=str, help='regular expression')
158 parser.add_argument(
159 '-n',
160 '--n',
161 type=int,
162 default=-1,
163 help='number of lines to display, -1 for all')
164 parser.add_argument(
165 '-r',
166 '--raw',
167 default=False,
168 action='store_true',
169 help='display raw text instead of HTML')
170 parser.add_argument(
171 '-e',
172 '--encoding',
173 default="utf8",
174 help='file encoding')
175 parser.add_argument(
176 '-s',
177 '--errors',
178 default="",
179 help='What about errors: "", strict, replace, surrogateescape, '
180 'xmlcharrefreplace, backslashreplace, namereplace')
181 return parser
183 @line_magic
184 def grep(self, line):
185 """
186 defines ``%grep``
187 which displays the first lines of a file
189 .. nbref::
190 :tag: file
191 :title: grep
193 The magic command ``%grep`` is equivalent to::
195 from pyensae.file_helper import enumerate_grep
196 list(enumerate_grep(<filename>, <regex>, <encoding>))
197 """
198 parser = self.get_parser(MagicFile.grep_parser, "grep")
199 args = self.get_args(line, parser)
201 if args is not None:
202 errors = args.errors
203 if errors is not None and len(errors) < 2:
204 errors = None
205 encoding = args.encoding
206 if encoding is not None and len(encoding) < 2:
207 encoding = None
208 iter = enumerate_grep(args.f, args.regex,
209 encoding, errors=errors)
210 if args.n != -1:
211 rows = []
212 for r in iter:
213 if len(rows) >= args.n:
214 break
215 rows.append(r)
216 else:
217 rows = list(iter)
219 if args.raw:
220 return "".join(rows)
221 else:
222 return HTML("<pre>\n{0}\n</pre>".format("".join(rows)))
223 return None
225 @staticmethod
226 def tail_parser():
227 """
228 defines the way to parse the magic command ``%tail``
229 """
230 parser = MagicCommandParser(prog="tail",
231 description='display the last lines of a text file')
232 parser.add_argument('f', type=str, help='filename')
233 parser.add_argument(
234 '-n',
235 '--n',
236 type=int,
237 default=10,
238 help='number of lines to display')
239 parser.add_argument(
240 '-r',
241 '--raw',
242 default=False,
243 action='store_true',
244 help='display raw text instead of HTML')
245 parser.add_argument(
246 '-e',
247 '--encoding',
248 default="utf8",
249 help='file encoding')
250 parser.add_argument(
251 '-s',
252 '--errors',
253 default="",
254 help='What about errors: "", strict, replace, surrogateescape, '
255 'xmlcharrefreplace, backslashreplace, namereplace')
256 return parser
258 @line_magic
259 def tail(self, line):
260 """
261 defines ``%tail``
262 which displays the last lines of a file
264 .. nbref::
265 :tag: file
266 :title: tail
268 The magic command ``%tail`` is equivalent to::
270 from pyensae.file_helper import file_tail
271 file_tail(<filename>, <n>, <encoding>)
272 """
273 parser = self.get_parser(MagicFile.tail_parser, "tail")
274 args = self.get_args(line, parser)
276 if args is not None:
277 errors = args.errors
278 if errors is not None and len(errors) < 2:
279 errors = None
280 encoding = args.encoding
281 if encoding is not None and len(encoding) < 2:
282 encoding = None
283 rows = file_tail(args.f, args.n, encoding, errors=errors)
284 if args.raw:
285 return "".join(rows)
286 else:
287 return HTML("<pre>\n{0}\n</pre>".format("".join(rows)))
288 return None
290 @staticmethod
291 def lsr_parser():
292 """
293 defines the way to parse the magic command ``%lsr``
294 """
295 parser = MagicCommandParser(prog="lsr",
296 description='display the content of a folder as a dataframe')
297 parser.add_argument(
298 'path',
299 type=str,
300 nargs="?",
301 help='path',
302 default=".")
303 parser.add_argument(
304 '-f',
305 '--filter',
306 type=str,
307 default=".*",
308 help='filter, same syntax as a regular expression')
309 return parser
311 @line_magic
312 def lsr(self, line):
313 """
314 Defines ``%lsr`` which returns the content of a folder,
315 the method stops after around 10000 files
316 --> you should precise the filter.
318 .. nbref::
319 :tag: file
320 :title: lsr
322 The magic command ``%lsr`` is almost equivalent to::
324 from pyquickhelper.file_helper import explore_folder_iterfile
325 res = explore_folder_iterfile(<filename>, <pattern>)
326 for f in res:
327 print(f)
328 """
329 parser = self.get_parser(MagicFile.lsr_parser, "lsr")
330 args = self.get_args(line, parser)
332 if args is not None:
333 if args.path is None or len(args.path) == 0:
334 filename = "."
335 else:
336 filename = args.path
337 pattern = args.filter
339 if "*" in filename:
340 pattern = filename
341 filename = "."
343 iter = explore_folder_iterfile(filename, pattern)
344 rows = []
345 for r in iter:
346 d = os.path.isfile(r)
347 if d:
348 st = os.stat(r)
349 r = {"name": r,
350 "size": format_file_size(st.st_size),
351 "last_modified": format_file_mtime(st.st_mtime),
352 "directory": False}
353 else:
354 r = {"name": r, "directory": True}
355 rows.append(r)
356 return pandas.DataFrame(rows)
357 return None
359 @staticmethod
360 def PYTHON_parser():
361 """
362 Defines the way to parse the magic command ``%%PYTHON``.
363 """
364 parser = MagicCommandParser(prog="PYTHON",
365 description='the command stores the content of the cell as a local file.')
366 parser.add_argument(
367 'file',
368 type=str,
369 help='filename')
370 return parser
372 @cell_magic
373 def PYTHON(self, line, cell=None):
374 """
375 Defines command ``%%PYTHON``.
377 .. nbref::
378 :title: PYTHON
380 The magic command ``%%PYTHON`` is almost to:
382 ::
384 with open(<filename>, "w", encoding="utf8") as f:
385 f.write("# -*- coding: utf8 -*-\\n")
386 f.write(cell.replace("\\r", ""))
387 """
388 parser = self.get_parser(MagicFile.PYTHON_parser, "PYTHON")
389 args = self.get_args(line, parser)
391 if args is not None:
392 filename = args.file
393 with open(filename, "w", encoding="utf8") as f:
394 f.write("# -*- coding: utf8 -*-\n")
395 f.write(cell.replace("\r", ""))
397 @staticmethod
398 def runpy_parser():
399 """
400 Defines the way to parse the magic command ``%%runpy``.
401 """
402 parser = MagicCommandParser(prog="runpy",
403 description='run a python script which accepts standards input and ' +
404 'produces standard outputs, a timeout is set up at 10s')
405 parser.add_argument(
406 'file',
407 type=str,
408 help='python file')
409 parser.add_argument(
410 'args',
411 type=str,
412 nargs="*",
413 help='arguments for the scripts',
414 default=".")
415 return parser
417 @cell_magic
418 def runpy(self, line, cell=None):
419 """
420 Defines command ``%%runpy``.
422 .. nbref::
423 :title: runpy
425 ``%%runpy`` runs a python script which accepts
426 standards input and produces standard outputs,
427 a timeout is set up at 10s. It is almost equivalent to::
429 from pyquickhelper.loghelper import run_cmd
430 import sys
431 cmd = sys.executable.replace(
432 "pythonw",
433 "python") + " " + filename + " " + args
434 out, err = run_cmd(
435 cmd, wait=True, sin=cell, communicate=True, timeout=10, shell=False)
437 .. versionadded:: 1.1
438 """
439 parser = self.get_parser(MagicFile.runpy_parser, "runpy")
440 args = self.get_args(line, parser)
442 if args is not None:
443 filename = args.file
444 if len(filename) == 0:
445 self.runpy("")
446 else:
447 args = args.args
448 cmd = sys.executable.replace("pythonw", "python")
449 cmd += " " + filename + " "
450 cmd += " ".join('"{0}"'.format(_)
451 for _ in args) if isinstance(args, list) else args
452 tosend = cell
453 out, err = run_cmd(
454 cmd, wait=True, sin=tosend, communicate=True, timeout=10, shell=False)
455 if len(err) > 0:
456 return HTML(
457 '<font color="#DD0000">Error</font><br /><pre>\n%s\n</pre>' % err)
458 else:
459 return HTML('<pre>\n%s\n</pre>' % out)
460 return None
462 @staticmethod
463 def lsrepo_parser():
464 """
465 Defines the way to parse the magic command ``%lsrepo``.
466 """
467 parser = MagicCommandParser(prog="lsrepo",
468 description='display the content of a repository (GIT or SVN)')
469 parser.add_argument('path', type=str, nargs="?",
470 help='path', default=".")
471 return parser
473 @line_magic
474 def lsrepo(self, line):
475 """
476 Defines ``%lsrepo``.
478 .. nbref::
479 :tag: file
480 :title: lsrepo
482 The method returns the files present in a repository
483 (:epkg:`GIT` or :pkg:`SVN`). The code is equivalent to::
485 from pyquickhelper.filehelper import explore_folder_iterfile_repo
486 res = explore_folder_iterfile_repo(<filename>, <pattern>)
487 for f in res:
488 print(f)
490 .. versionadded:: 1.1
491 """
492 parser = self.get_parser(MagicFile.lsrepo_parser, "lsrepo")
493 args = self.get_args(line, parser)
495 if args is not None:
496 if args.path is None or len(args.path) == 0:
497 filename = "."
498 else:
499 filename = args.path
501 iter = explore_folder_iterfile_repo(filename)
502 rows = []
503 for r in iter:
504 d = os.path.isfile(r)
505 if d:
506 st = os.stat(r)
507 r = {"name": r,
508 "size": format_file_size(st.st_size),
509 "last_modified": format_file_mtime(st.st_mtime),
510 "directory": False}
511 else:
512 r = {"name": r, "directory": True}
513 rows.append(r)
514 return pandas.DataFrame(rows)
515 return None
517 @staticmethod
518 def hhelp_parser():
519 """
520 Defines the way to parse the magic command ``%hhelp``.
522 .. versionadded:: 1.1
523 """
524 parser = MagicCommandParser(prog="hhelp",
525 description='display help for an object in HTML format')
526 parser.add_argument(
527 'obj',
528 type=str,
529 help='a python object')
530 parser.add_argument(
531 '-f',
532 '--format',
533 type=str,
534 default="html",
535 help='format',
536 choices=['text', 'html', 'rst', 'rawhtml'])
537 parser.add_argument(
538 '-np',
539 '--no-print',
540 action='store_true',
541 help='by default, the magic command outputs everything on the standard output, '
542 'if specified, it returns a string')
543 return parser
545 @line_magic
546 def hhelp(self, line):
547 """
548 Define ``%hhelp``, it displays the help for an object in :epkg:`HTML`.
550 .. nbref::
551 :title: hhelp
553 The magic command is equivalent to::
555 from pyquickhelper import docstring2html
556 docstring2html(obj, format=format)
558 .. versionadded:: 1.1
559 """
560 parser = self.get_parser(MagicFile.hhelp_parser, "hhelp")
561 args = self.get_args(line, parser)
563 if args is not None:
564 obj = args.obj
565 format = args.format
566 nop = args.no_print
567 if nop or format == "html":
568 return docstring2html(obj, format=format)
569 else:
570 print(docstring2html(obj, format=format))
571 return None
574def register_file_magics(ip=None):
575 """
576 register magics function, can be called from a notebook
578 @param ip from ``get_ipython()``
579 """
580 if ip is None:
581 from IPython import get_ipython
582 ip = get_ipython()
583 ip.register_magics(MagicFile)