Coverage for pyquickhelper/helpgen/process_notebooks.py: 91%
577 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains the main function to generate the documentation
5for a module designed the same way as this one, @see fn generate_help_sphinx.
7"""
8import datetime
9import json
10import os
11import sys
12import shutil
13import warnings
14from io import StringIO
15from nbconvert.exporters.base import ExporterNameError
17from .utils_sphinx_doc_helpers import HelpGenException
18from .conf_path_tools import find_latex_path, find_pandoc_path
19from .post_process import (
20 post_process_latex_output, post_process_latex_output_any,
21 post_process_rst_output, post_process_html_output,
22 post_process_slides_output, post_process_python_output)
23from .helpgen_exceptions import NotebookConvertError
24from .install_js_dep import install_javascript_tools
25from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE
26from .process_notebook_api import nb2rst
27from ..loghelper.flog import run_cmd, fLOG, noLOG
28from ..pandashelper import df2rst
29from ..filehelper.synchelper import has_been_updated, explore_folder
32template_examples = """
34List of programs
35++++++++++++++++
37.. toctree::
38 :maxdepth: 2
40.. autosummary:: __init__.py
41 :toctree: %s/
42 :template: modules.rst
44Another list
45++++++++++++
47"""
50def find_pdflatex(latex_path):
51 """
52 Returns the executable for latex.
54 @param latex_path path to look (only on Windows)
55 @return executable
56 """
57 if sys.platform.startswith("win"): # pragma: no cover
58 lat = os.path.join(latex_path, "xelatex.exe")
59 if os.path.exists(lat):
60 return lat
61 lat = os.path.join(latex_path, "pdflatex.exe")
62 if os.path.exists(lat):
63 return lat
64 raise FileNotFoundError(
65 f"Unable to find pdflatex or xelatex in '{latex_path}'")
66 if sys.platform.startswith("darwin"): # pragma: no cover
67 try:
68 err = run_cmd("/Library/TeX/texbin/xelatex --help", wait=True)[1]
69 if len(err) == 0:
70 return "/Library/TeX/texbin/xelatex"
71 raise FileNotFoundError( # pragma: no cover
72 f"Unable to run xelatex\n{err}")
73 except Exception:
74 return "/Library/TeX/texbin/pdflatex"
75 try:
76 err = run_cmd("xelatex --help", wait=True)[1]
77 if len(err) == 0:
78 return "xelatex"
79 else:
80 raise FileNotFoundError(
81 f"Unable to run xelatex\n{err}")
82 except Exception: # pragma: no cover
83 return "pdflatex"
86def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=None,
87 formats="ipynb,html,python,rst,slides,pdf,github",
88 fLOG=fLOG, exc=True, remove_unicode_latex=False, nblinks=None,
89 notebook_replacements=None):
90 """
91 Converts notebooks into :epkg:`html`, :epkg:`rst`, :epkg:`latex`,
92 :epkg:`pdf`, :epkg:`python`, :epkg:`docx` using
93 :epkg:`nbconvert`.
95 @param notebooks list of notebooks or comma separated values
96 @param outfold folder which will contains the outputs
97 @param build temporary folder which contains all produced files
98 @param pandoc_path path to pandoc
99 @param formats list of formats to convert into (pdf format means latex
100 then compilation), or comma separated values
101 @param latex_path path to the latex compiler
102 @param fLOG logging function
103 @param exc raises an exception (True) or a warning (False) if an error happens
104 @param nblinks dictionary ``{ref: url}`` or a string in :epkg:`json`
105 format
106 @param remove_unicode_latex remove unicode characters for latex (to avoid failing)
107 @param notebook_replacements string replacement in a notebook before conversion
108 or a string in :epkg:`json` format
109 @return list of tuple *[(file, created or skipped)]*
111 This function relies on :epkg:`pandoc`.
112 It also needs modules :epkg:`pywin32`,
113 :epkg:`pygments`.
115 :epkg:`pywin32` might have some issues
116 to find its DLL, look @see fn import_pywin32.
118 The latex compilation uses :epkg:`MiKTeX`.
119 The conversion into Word document directly uses pandoc.
120 It still has an issue with table.
122 Some latex templates (for nbconvert) uses ``[commandchars=\\\\\\{\\}]{\\|}`` which allows commands ``\\\\`` and it does not compile.
123 The one used here is ``report``.
124 Some others bugs can be found at: `schlichtanders/latex_test.html <https://gist.github.com/schlichtanders/e108ed0be80108178af2>`_.
125 For example, you must not let spaces between symbol ``$`` and the
126 formulas it indicates.
128 If *pandoc_path* is None, uses @see fn find_pandoc_path to guess it.
129 If *latex_path* is None, uses @see fn find_latex_path to guess it.
131 .. exref::
132 :title: Convert a notebook into multiple formats
134 ::
136 from pyquickhelper.ipythonhelper import process_notebooks
137 process_notebooks("td1a_correction_session7.ipynb",
138 "dest_folder", "dest_folder",
139 formats=("ipynb", "html", "python", "rst", "slides", "pdf",
140 "docx", "github")])
142 For latex and pdf, a custom processor was added to handle raw data
143 and add ``\\begin{verbatim}`` and ``\\end{verbatim}``.
144 Format *github* adds a link to file on :epkg:`github`.
146 .. todoext::
147 :title: check differences between _process_notebooks_in_private and _process_notebooks_in_private_cmd
148 :tag: bug
150 For :epkg:`latex` and :epkg:`pdf`,
151 the custom preprocessor is not taken into account.
152 by function @see fn _process_notebooks_in_private.
153 """
154 if isinstance(notebooks, str):
155 notebooks = notebooks.split(',')
156 if isinstance(formats, str):
157 formats = formats.split(',')
158 if isinstance(notebook_replacements, str):
159 notebook_replacements = json.loads(notebook_replacements)
160 if isinstance(nblinks, str):
161 nblinks = json.loads(nblinks)
162 if build is None:
163 raise ValueError( # pragma: no cover
164 "build cannot be None")
166 res = _process_notebooks_in(notebooks=notebooks, outfold=outfold, build=build,
167 latex_path=latex_path, pandoc_path=pandoc_path,
168 formats=formats, fLOG=fLOG, exc=exc, nblinks=nblinks,
169 remove_unicode_latex=remove_unicode_latex,
170 notebook_replacements=notebook_replacements)
171 if "slides" in formats:
172 # we copy javascript dependencies, reveal.js
173 reveal = os.path.join(outfold, "reveal.js")
174 if not os.path.exists(reveal):
175 install_javascript_tools(None, dest=outfold)
176 reveal = os.path.join(build, "reveal.js")
177 if not os.path.exists(reveal):
178 install_javascript_tools(None, dest=build)
179 return res
182def _process_notebooks_in_private(fnbcexe, list_args, options_args):
183 """
184 This function fails in nbconvert 6.0 when the conversion
185 is called more than once. The conversion probably changes the
186 initial state.
187 """
188 out = StringIO()
189 err = StringIO()
190 memo_out = sys.stdout
191 memo_err = sys.stderr
192 sys.stdout = out
193 sys.stderr = err
194 try:
195 if list_args:
196 fnbcexe(argv=list_args, **options_args)
197 else:
198 fnbcexe(**options_args)
199 exc = None
200 except SystemExit as e: # pragma: no cover
201 exc = e
202 except IndentationError as e: # pragma: no cover
203 # This is change in IPython 6.0.0.
204 # The conversion fails on IndentationError.
205 # We switch to another one.
206 from ..ipythonhelper import read_nb
207 i = list_args.index("--template")
208 format = list_args[i + 1]
209 if format == "python":
210 i = list_args.index("--output")
211 dest = list_args[i + 1]
212 if not dest.endswith(".py"):
213 dest += ".py"
214 src = list_args[-1]
215 nb = read_nb(src)
216 code = nb.to_python()
217 with open(dest, "w", encoding="utf-8") as f:
218 f.write(code)
219 exc = None
220 else:
221 # We do nothing in this case.
222 exc = e
223 except (AttributeError, FileNotFoundError, ValueError) as e:
224 exc = e
225 except ExporterNameError as e: # pragma: no cover
226 exc = e
227 sys.stdout = memo_out
228 sys.stderr = memo_err
229 out = out.getvalue()
230 err = err.getvalue()
231 if exc:
232 if "Unsupported mimetype 'text/html'" in str(exc):
233 from nbconvert.nbconvertapp import main
234 main(argv=list_args, **options_args)
235 return "", ""
236 env = "\n".join(f"{k}={v}"
237 for k, v in sorted(os.environ.items()))
238 raise RuntimeError( # pragma: no cover
239 "Notebook conversion failed.\nfnbcexe\n{}\noptions_args\n{}"
240 "\n--ARGS--\n{}\n--OUT--\n{}\n--ERR--\n{}\n--ENVIRON--\n{}"
241 "".format(fnbcexe, options_args, list_args, out, err,
242 env)) from exc
243 return out, err
246def _process_notebooks_in_private_cmd(fnbcexe, list_args, options_args, fLOG):
247 this = os.path.join(os.path.dirname(
248 os.path.abspath(__file__)), "process_notebooks_cmd.py")
249 res = []
250 for c in list_args:
251 if c[0] == '"' or c[-1] == '"' or ' ' not in c:
252 res.append(c)
253 else:
254 res.append(f'"{c}"')
255 sargs = " ".join(res)
256 cmd = f"\"{sys.executable.replace('w.exe', '.exe')}\" \"{this}\" {sargs}"
257 fLOG("[_process_notebooks_in_private_cmd]", cmd)
258 return run_cmd(cmd, wait=True, fLOG=fLOG)
261def _preprocess_notebook(notebook_content):
262 """
263 Preprocesses the content of a notebook.
265 @param notebook_content notebook content
266 @return modified content
267 """
268 def walk_through(field):
269 if isinstance(field, list):
270 for f in field:
271 walk_through(f)
272 elif isinstance(field, dict):
273 if (field.get('version_major', -1) == 2 and
274 field.get('version_minor', -1) == 0):
275 field['version_minor'] = 2
276 elif (field.get('nbformat', -1) == 4 and
277 field.get('nbformat_minor', -1) in (0, 1)):
278 field['nbformat_minor'] = 2
279 for _, v in field.items():
280 walk_through(v)
282 content = json.loads(notebook_content)
283 walk_through(content)
284 new_content = json.dumps(content)
285 return new_content
288def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_path=None,
289 formats=("ipynb", "html", "python", "rst",
290 "slides", "pdf", "github"),
291 fLOG=fLOG, exc=True, nblinks=None, remove_unicode_latex=False,
292 notebook_replacements=None):
293 """
294 The notebook conversion does not handle images from url
295 for :epkg:`pdf` and :epkg:`docx`. They could be downloaded first
296 and replaced by local files.
298 .. note::
300 :epkg:`nbconvert` introduced a commit which breaks
301 the conversion of notebooks in latex if they have
302 a cell outputting *svg*
303 (see `PR 910 <https://github.com/jupyter/nbconvert/pull/910>`_).
305 Use `xelatex <https://doc.ubuntu-fr.org/xelatex>`_ if possible.
306 """
307 from nbconvert.nbconvertapp import main as nbconvert_main
308 if pandoc_path is None:
309 pandoc_path = find_pandoc_path()
311 if latex_path is None:
312 latex_path = find_latex_path()
314 if isinstance(notebooks, str):
315 notebooks = [notebooks]
317 if "PANDOCPY" in os.environ and sys.platform.startswith("win"): # pragma: no cover
318 exe = os.environ["PANDOCPY"]
319 exe = exe.rstrip("\\/")
320 if exe.endswith("\\Scripts"):
321 exe = exe[:len(exe) - len("Scripts") - 1]
322 if not os.path.exists(exe):
323 raise FileNotFoundError(exe)
324 fLOG("[_process_notebooks_in] ** using PANDOCPY", exe)
325 else:
326 if sys.platform.startswith("win"): # pragma: no cover
327 from .utils_pywin32 import import_pywin32
328 try:
329 import_pywin32()
330 except ModuleNotFoundError as e:
331 warnings.warn(e)
332 exe = os.path.split(sys.executable)[0]
334 extensions = {"ipynb": ".ipynb", "latex": ".tex", "elatex": ".tex", "pdf": ".pdf",
335 "html": ".html", "rst": ".rst", "python": ".py", "docx": ".docx",
336 "word": ".docx", "slides": ".slides.html"}
338 files = []
339 skipped = []
341 # main(argv=None, **kwargs)
342 fnbc = nbconvert_main
344 if "slides" in formats:
345 build_slide = os.path.join(build, "bslides")
346 if not os.path.exists(build_slide):
347 os.mkdir(build_slide)
349 copied_images = dict()
351 for notebook_in in notebooks:
352 thisfiles = []
354 # we copy available images (only notebook folder)
355 # in case they are used in latex
356 currentdir = os.path.abspath(os.path.dirname(notebook_in))
357 for curfile in os.listdir(currentdir):
358 ext = os.path.splitext(curfile)[1]
359 if ext in {'.png', '.jpg', '.bmp', '.gif', '.jpeg', '.svg', '.mp4'}:
360 src = os.path.join(currentdir, curfile)
361 if src not in copied_images:
362 dest = os.path.join(build, curfile)
363 shutil.copy(src, build)
364 fLOG(f"[_process_notebooks_in] copy '{src}' to '{build}'.")
365 copied_images[src] = dest
367 # copy of the notebook into the build folder
368 # and changes the source
369 _name = os.path.splitext(os.path.split(notebook_in)[-1])[0]
370 _name += '.ipynb'
371 notebook = os.path.join(build, _name)
372 fLOG("[_process_notebooks_in] -- copy notebook '{}' to '{}'.".format(
373 notebook_in, notebook))
374 with open(notebook_in, "r", encoding="utf-8") as _f:
375 content = _f.read()
376 content = _preprocess_notebook(content)
377 with open(notebook, "w", encoding="utf-8") as _f:
378 _f.write(content)
380 # next
381 nbout = os.path.split(notebook)[-1]
382 if " " in nbout:
383 raise HelpGenException( # pragma: no cover
384 f"spaces are not allowed in notebooks file names: {notebook}")
385 nbout = os.path.splitext(nbout)[0]
386 for format in formats:
388 if format == "github":
389 # we add a link on the rst page in that case
390 continue
392 if format not in extensions:
393 raise NotebookConvertError( # pragma: no cover
394 "Unable to find format: '{}' in {}".format(
395 format, ", ".join(extensions.keys())))
397 # output
398 format_ = format
399 outputfile_noext = os.path.join(build, nbout)
400 if format == 'html':
401 outputfile = outputfile_noext + '2html' + extensions[format]
402 outputfile_noext_fixed = outputfile_noext + '2html'
403 else:
404 outputfile = outputfile_noext + extensions[format]
405 outputfile_noext_fixed = outputfile_noext
406 trueoutputfile = outputfile
407 pandoco = "docx" if format in ("word", "docx") else None
409 # The function checks it was not done before.
410 if os.path.exists(trueoutputfile):
411 dto = os.stat(trueoutputfile).st_mtime
412 dtnb = os.stat(notebook).st_mtime
413 if dtnb < dto: # pragma: no cover
414 fLOG("[_process_notebooks_in] -- skipping notebook", format,
415 notebook, "(", trueoutputfile, ")")
416 if trueoutputfile not in thisfiles:
417 thisfiles.append(trueoutputfile)
418 if pandoco is None:
419 skipped.append(trueoutputfile)
420 continue
421 out2 = os.path.splitext(
422 trueoutputfile)[0] + "." + pandoco
423 if os.path.exists(out2):
424 skipped.append(trueoutputfile)
425 continue
427 # if the format is slides, we update the metadata
428 options_args = {}
429 if format == "slides":
430 nb_slide = add_tag_for_slideshow(notebook, build_slide)
431 fnbcexe = fnbc
432 else:
433 nb_slide = None
434 fnbcexe = fnbc
436 # compilation
437 list_args = []
438 custom_config = os.path.join(os.path.abspath(
439 os.path.dirname(__file__)), "_nbconvert_config.py")
440 if format == "pdf":
441 if not os.path.exists(custom_config):
442 raise FileNotFoundError( # pragma: no cover
443 custom_config)
444 # title = os.path.splitext(
445 # os.path.split(notebook)[-1])[0].replace("_", " ")
446 list_args.extend(['--config', f'"{custom_config}"'])
447 format = "latex"
448 compilation = True
449 thisfiles.append(os.path.splitext(outputfile)[0] + ".tex")
450 elif format in ("latex", "elatex"):
451 if not os.path.exists(custom_config):
452 raise FileNotFoundError( # pragma: no cover
453 custom_config)
454 list_args.extend(['--config', f'"{custom_config}"'])
455 compilation = False
456 format = "latex"
457 elif format in ("word", "docx"):
458 format = "html"
459 compilation = False
460 elif format in ("slides", ):
461 list_args.extend(["--reveal-prefix", "reveal.js"])
462 compilation = False
463 else:
464 compilation = False
466 # output
467 # set templates to None to avoid error
468 # No template sub-directory with name 'article' found in the following paths:
469 templ = {'html': None, 'latex': None,
470 'elatex': None}.get(format, format)
471 fLOG("[_process_notebooks_in] ### convert into '{}' (done: {}): '{}' -> '{}'".format(
472 format_, os.path.exists(outputfile), notebook, outputfile))
474 list_args.extend(["--output", outputfile_noext_fixed])
475 if templ is not None and format != "slides":
476 list_args.extend(["--template", templ])
478 # execution
479 if format not in ("ipynb", ):
480 # nbconvert is messing up with static variables in sphinx or
481 # docutils if format is slides, not sure about the others
482 if format in ('rst', ):
483 fLOG("[_process_notebooks_in] NBcn:", format, options_args)
484 nb2rst(notebook, outputfile, post_process=False)
485 err = ""
486 c = ""
487 elif nbconvert_main != fnbcexe or format not in (
488 "slides", "elatex", "latex", "pdf", "html"):
489 if options_args:
490 fLOG("[_process_notebooks_in] NBp*:",
491 format, options_args)
492 else:
493 list_args.extend(["--to", format,
494 notebook if nb_slide is None else nb_slide])
495 fLOG(
496 f"[_process_notebooks_in] NBc* format='{format}' args={list_args}")
497 fLOG(f"[_process_notebooks_in] cwd='{os.getcwd()}'")
499 c = " ".join(list_args)
500 out, err = _process_notebooks_in_private(
501 fnbcexe, list_args, options_args)
502 else:
503 # conversion into slides alter Jinja2 environment
504 # jinja2.exceptions.TemplateNotFound: rst
505 if options_args:
506 fLOG("[_process_notebooks_in] NBp+:",
507 format, options_args)
508 else:
509 list_args.extend(["--to", format,
510 notebook if nb_slide is None else nb_slide])
511 fLOG("[_process_notebooks_in] NBc+:", format, list_args)
512 fLOG("[_process_notebooks_in]", os.getcwd())
514 c = " ".join(list_args)
515 out, err = _process_notebooks_in_private_cmd(
516 fnbcexe, list_args, options_args, fLOG)
518 if "raise ImportError" in err or "Unknown exporter" in err:
519 raise ImportError(
520 f"cmd: {fnbcexe} {list_args}\n--ERR--\n{err}")
521 if len(err) > 0:
522 if format in ("elatex", "latex"):
523 # There might be some errors because the latex script needs to be post-processed
524 # sometimes (wrong characters such as " or formulas not
525 # captured as formulas).
526 if err and "usage: process_notebooks_cmd.py" in err:
527 raise RuntimeError( # pragma: no cover
528 "Unable to convert a notebook\n----\n{}----\n{}\n"
529 "---ERR---\n{}\n---OUT---\n{}".format(
530 fnbcexe, list_args, err, out))
531 fLOG("[_process_notebooks_in] LATEX --ERR--\n" + err)
532 fLOG("[_process_notebooks_in] LATEX --OUT--\n" + out)
533 else:
534 err = err.lower()
535 if "critical" in err or "bad config" in err:
536 raise HelpGenException( # pragma: no cover
537 f"CMD:\n{list_args}\n[nberror]\n{err}")
538 else:
539 # format ipynb
540 # we do nothing
541 pass
543 format = extensions[format].strip(".")
545 # we add the file to the list of generated files
546 if outputfile not in thisfiles:
547 thisfiles.append(outputfile)
549 fLOG("[_process_notebooks_in] -",
550 format, compilation, outputfile)
552 if compilation:
553 # compilation latex
554 if not sys.platform.startswith("win") or os.path.exists(latex_path):
555 lat = find_pdflatex(latex_path)
557 tex = set(_ for _ in thisfiles if os.path.splitext(
558 _)[-1] == ".tex")
559 if len(tex) != 1:
560 raise FileNotFoundError( # pragma: no cover
561 "No latex file was generated or more than one (={0}), nb={1}\nthisfile=\n{2}".format(
562 len(tex), notebook, "\n".join(thisfiles)))
563 tex = list(tex)[0]
564 try:
565 post_process_latex_output_any(
566 tex, custom_latex_processing=None, nblinks=nblinks,
567 remove_unicode=remove_unicode_latex, fLOG=fLOG)
568 except FileNotFoundError as e: # pragma: no cover
569 mes = ("[_process_notebooks_in-ERROR] Unable to convert into latex"
570 "notebook %r due to %r.") % (tex, e)
571 warnings.warn(mes, RuntimeWarning)
572 fLOG(mes)
573 continue
575 # -interaction=batchmode
576 c = '"{0}" "{1}" -max-print-line=900 -output-directory="{2}"'.format(
577 lat, tex, os.path.split(tex)[0])
578 fLOG("[_process_notebooks_in] ** LATEX compilation (b)", c)
579 if not sys.platform.startswith("win"):
580 c = c.replace('"', '')
581 if sys.platform.startswith("win"):
582 change_path = None
583 else:
584 # On Linux the parameter --output-directory is sometimes ignored.
585 # And it only works from the current directory.
586 change_path = os.path.split(tex)[0]
587 out, err = run_cmd(
588 c, wait=True, log_error=False, shell=sys.platform.startswith("win"),
589 catch_exit=True, prefix_log="[latex] ", change_path=change_path)
590 if out is not None and ("Output written" in out or 'bytes written' in out):
591 # The output was produced. We ignore the return code.
592 fLOG("[_process_notebooks_in] WARNINGS: "
593 "Latex compilation had warnings:", c)
594 out += "\n--ERR--\n" + err
595 err = ""
596 if len(err) > 0:
597 raise HelpGenException( # pragma: no cover
598 f"CMD:\n{c}\n[nberror]\n{err}\nOUT:\n{out}------")
599 f = os.path.join(build, nbout + ".pdf")
600 if not os.path.exists(f): # pragma: no cover
601 # On Linux the parameter --output-directory is sometimes ignored.
602 # And it only works from the current directory.
603 # We check again.
604 loc = os.path.split(f)[-1]
605 if os.path.exists(loc):
606 # We move the file.
607 moved = True
608 shutil.move(loc, f)
609 else:
610 moved = False
611 if not os.path.exists(f):
612 files = "\n".join(os.listdir(build))
613 msg = "Content of '{0}':\n{1}\n----\n'{2}' moved? {3}\nCMD:\n{4}".format(
614 build, files, loc, moved, c)
615 raise HelpGenException(
616 f"Missing file: '{f}'\nCMD\n{c}nOUT:\n{out}\n[nberror]\n{err}\n-----\n{msg}")
617 thisfiles.append(f)
618 else:
619 fLOG("[_process_notebooks_in] unable to find latex in", latex_path)
621 elif pandoco is not None: # pragma: no cover
622 # compilation pandoc
623 fLOG("[_process_notebooks_in] ** pandoc compilation (b)", pandoco)
624 inputfile = os.path.splitext(outputfile)[0] + ".html"
625 outfilep = os.path.splitext(outputfile)[0] + "." + pandoco
627 # for some files, the following error might appear:
628 # Stack space overflow: current size 33692 bytes.
629 # Use `+RTS -Ksize -RTS' to increase it.
630 # it usually means there is something wrong (circular
631 # reference, ...)
632 if sys.platform.startswith("win"):
633 c = '"{0}\\pandoc.exe" +RTS -K32m -RTS -f html -t {1} "{2}" -o "{3}"'.format(
634 pandoc_path, pandoco, inputfile, outfilep)
635 else:
636 c = 'pandoc +RTS -K32m -RTS -f html -t {0} "{1}" -o "{2}"'.format(
637 pandoco, outputfile, outfilep)
639 if not sys.platform.startswith("win"):
640 c = c.replace('"', '')
641 out, err = run_cmd(
642 c, wait=True, log_error=False, shell=sys.platform.startswith("win"))
643 if len(err) > 0:
644 lines = err.strip("\r\n").split("\n")
645 # we filter out the message
646 # pandoc.exe: Could not find image `https://
647 left = [
648 _ for _ in lines if _ and "Could not find image `http" not in _]
649 if len(left) > 0:
650 raise HelpGenException(
651 f"issue with cmd: {c}\n[nberror]\n{err}")
652 for _ in lines:
653 fLOG("[_process_notebooks_in] w, pandoc issue: {0}".format(
654 _.strip("\n\r")))
655 outputfile = outfilep
656 format = "docx"
658 nb_replacements = notebook_replacements.get(
659 format, None) if notebook_replacements else None
661 if format == "html":
662 # we add a link to the notebook
663 if not os.path.exists(outputfile):
664 raise FileNotFoundError( # pragma: no cover
665 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join(
666 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + c)
667 thisfiles += add_link_to_notebook(outputfile, notebook, "pdf" in formats, False,
668 "python" in formats, "slides" in formats,
669 exc=exc, nblinks=nblinks, fLOG=fLOG,
670 notebook_replacements=nb_replacements)
672 elif format == "slides.html":
673 # we add a link to the notebook
674 if not os.path.exists(outputfile):
675 raise FileNotFoundError( # pragma: no cover
676 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join(
677 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + str(list_args))
678 thisfiles += add_link_to_notebook(outputfile, notebook,
679 "pdf" in formats, False, "python" in formats,
680 "slides" in formats, exc=exc,
681 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
683 elif format == "ipynb":
684 # we just copy the notebook
685 thisfiles += add_link_to_notebook(outputfile, notebook,
686 "ipynb" in formats, False, "python" in formats,
687 "slides" in formats, exc=exc,
688 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
690 elif format == "rst":
691 # It adds a link to the notebook.
692 thisfiles += add_link_to_notebook(
693 outputfile, notebook, "pdf" in formats, "html" in formats, "python" in formats,
694 "slides" in formats, exc=exc, github="github" in formats,
695 notebook=notebook, nblinks=nblinks, fLOG=fLOG)
697 elif format in ("tex", "elatex", "latex", "pdf"):
698 thisfiles += add_link_to_notebook(outputfile, notebook, False, False,
699 False, False, exc=exc, nblinks=nblinks,
700 fLOG=fLOG, notebook_replacements=nb_replacements)
702 elif format in ("py", "python"):
703 post_process_python_output(
704 outputfile, True, nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
706 elif format in ["docx", "word"]:
707 pass
709 else:
710 raise HelpGenException( # pragma: no cover
711 "unexpected format " + format)
713 files.extend(thisfiles)
714 fLOG("[_process_notebooks_in] ### conversion into '{}' done into '{}'.".format(
715 format_, outputfile))
717 copy = []
718 for f in files:
719 dest = os.path.join(outfold, os.path.split(f)[-1])
720 if not f.endswith(".tex"):
722 if sys.version_info >= (3, 4):
723 try:
724 shutil.copy(f, outfold)
725 fLOG("[_process_notebooks_in] copy ",
726 f, " to ", outfold, "[", dest, "]")
727 except shutil.SameFileError:
728 fLOG("[_process_notebooks_in] w,file ",
729 dest, "already exists")
730 else: # pragma: no cover
731 try:
732 shutil.copy(f, outfold)
733 fLOG("[_process_notebooks_in] copy ",
734 f, " to ", outfold, "[", dest, "]")
735 except shutil.Error as e:
736 if "are the same file" in str(e):
737 fLOG("[_process_notebooks_in] w,file ",
738 dest, "already exists")
739 else:
740 raise e
742 if not os.path.exists(dest):
743 raise FileNotFoundError(dest) # pragma: no cover
744 copy.append((dest, True))
746 # image
747 for image in os.listdir(build):
748 if (image.endswith(".png") or image.endswith(".html") or
749 image.endswith(".pdf") or image.endswith(".svg") or
750 image.endswith(".jpg") or image.endswith(".gif") or
751 image.endswith(".xml") or image.endswith(".jpeg")):
752 image = os.path.join(build, image)
753 dest = os.path.join(outfold, os.path.split(image)[-1])
755 try:
756 shutil.copy(image, outfold)
757 fLOG("[_process_notebooks_in] copy ",
758 image, " to ", outfold, "[", dest, "]")
759 except shutil.SameFileError:
760 fLOG("[_process_notebooks_in] w,file ",
761 dest, "already exists")
763 if not os.path.exists(dest):
764 raise FileNotFoundError(dest) # pragma: no cover
765 copy.append((dest, True))
767 return copy + [(_, False) for _ in skipped]
770def add_link_to_notebook(file, nb, pdf, html, python, slides, exc=True,
771 github=False, notebook=None, nblinks=None, fLOG=None,
772 notebook_replacements=None):
773 """
774 Adds a link to the notebook in :epkg:`HTML` format and does a little bit of cleaning
775 for various format.
777 @param file notebook.html
778 @param nb notebook (.ipynb)
779 @param pdf if True, add a link to the PDF, assuming it will exists at the same location
780 @param html if True, add a link to the HTML conversion
781 @param python if True, add a link to the Python conversion
782 @param slides if True, add a link to the HTML slides
783 @param exc raises an exception (True) or a warning (False)
784 @param github add a link to the notebook on github
785 @param notebook location of the notebook (file might be a copy)
786 @param nblinks dictionary ``{ref: url}``
787 @param notebook_replacements stirng replacement in notebooks
788 @param fLOG logging function
789 @return list of generated files
791 The function does some cleaning too in the files.
792 """
793 core, ext = os.path.splitext(file)
794 if core.endswith(".slides"):
795 ext = ".slides" + ext
796 fLOG("[add_link_to_notebook] add_link_to_notebook", ext, " file ", file)
798 fold = os.path.split(file)[0]
799 res = [os.path.join(fold, os.path.split(nb)[-1])]
800 newr = has_been_updated(nb, res[-1])[0]
801 if newr:
802 shutil.copy(nb, fold)
804 if ext == ".ipynb":
805 return res
806 if ext == ".pdf":
807 return res
808 if ext == ".html":
809 post_process_html_output(
810 file, pdf, python, slides, exc=exc, nblinks=nblinks,
811 fLOG=fLOG, notebook_replacements=notebook_replacements)
812 return res
813 if ext == ".slides.html":
814 post_process_slides_output(
815 file, pdf, python, slides, exc=exc, nblinks=nblinks,
816 fLOG=fLOG, notebook_replacements=notebook_replacements)
817 return res
818 if ext == ".slides2p.html":
819 post_process_slides_output(
820 file, pdf, python, slides, exc=exc, nblinks=nblinks,
821 fLOG=fLOG, notebook_replacements=notebook_replacements)
822 return res
823 if ext == ".tex":
824 post_process_latex_output(
825 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG,
826 notebook_replacements=notebook_replacements)
827 return res
828 if ext == ".py":
829 post_process_python_output(
830 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG,
831 notebook_replacements=notebook_replacements)
832 return res
833 if ext == ".rst":
834 post_process_rst_output(
835 file, html, pdf, python, slides, is_notebook=True, exc=exc,
836 github=github, notebook=notebook, nblinks=nblinks, fLOG=fLOG,
837 notebook_replacements=notebook_replacements)
838 return res
839 raise HelpGenException( # pragma: no cover
840 f"Unable to add a link to this extension: {ext!r}")
843def build_thumbail_in_gallery(nbfile, folder_snippet, relative, rst_link, layout, snippet_folder=None, fLOG=None):
844 """
845 Returns :epkg:`rst` code for a notebook.
847 @param nbfile notebook file
848 @param folder_snippet where to store the snippet
849 @param relative the path to the snippet will be relative to this folder
850 @param rst_link rst link
851 @param layout ``'classic'`` or ``'table'``
852 @param snippet_folder folder where to find custom snippet for notebooks,
853 the snippet should have the same name as the notebook
854 itself, snippet must have extension ``.png``
855 @return RST
857 Modifies the function to bypass the generation of a snippet
858 if a custom one was found. Parameter *snippet_folder* was added.
859 """
860 from ..ipythonhelper import read_nb
861 nb = read_nb(nbfile)
862 _, desc = nb.get_description()
864 if snippet_folder is not None and os.path.exists(snippet_folder):
865 custom_snippet = os.path.join(snippet_folder, os.path.splitext(
866 os.path.split(nbfile)[-1])[0] + '.png')
867 else:
868 custom_snippet = None
870 if custom_snippet is not None and os.path.exists(custom_snippet):
871 # reading a custom snippet
872 if fLOG:
873 fLOG(
874 f"[build_thumbail_in_gallery] custom snippet '{custom_snippet}'")
875 try:
876 from PIL import Image
877 except ImportError: # pragma: no cover
878 import Image
879 image = Image.open(custom_snippet)
880 else:
881 # generating an image
882 if fLOG:
883 fLOG(
884 f"[build_thumbail_in_gallery] build snippet from '{nbfile}'")
885 image = nb.get_thumbnail()
887 if image is None:
888 image = nb.get_thumbnail(use_default=True)
890 if image is None:
891 raise ValueError( # pragma: no cover
892 f"The snippet cannot be null, notebook='{nbfile}'.")
893 name = os.path.splitext(os.path.split(nbfile)[-1])[0]
894 name += ".thumb"
895 full = os.path.join(folder_snippet, name)
897 dirname = os.path.dirname(full)
898 if not os.path.exists(dirname):
899 raise FileNotFoundError( # pragma: no cover
900 "Unable to find folder '{0}'\nfolder_snippet='{1}'\nrelative='{2}'\nnbfile='{3}'".format(
901 dirname, folder_snippet, relative, nbfile))
903 if isinstance(image, str):
904 # SVG
905 full += ".svg"
906 name += ".svg"
907 with open(full, "w", encoding="utf-8") as f:
908 f.write(image)
909 else:
910 # Image
911 full += ".png"
912 name += ".png"
913 image.save(full)
915 rel = os.path.relpath(full, start=relative).replace("\\", "/")
916 nb_name = rel.replace(".thumb.png", ".html")
917 if layout == "classic":
918 rst = THUMBNAIL_TEMPLATE.format(
919 snippet=desc, thumbnail=rel, ref_name=rst_link)
920 elif layout == "table":
921 rst = THUMBNAIL_TEMPLATE_TABLE.format(
922 snippet=desc, thumbnail=rel, ref_name=rst_link, nb_name=nb_name)
923 else:
924 raise ValueError(
925 "layout must be 'classic' or 'table'") # pragma: no cover
926 return rst
929def add_tag_for_slideshow(ipy, folder, encoding="utf8"):
930 """
931 Modifies a notebook to add tag for a slideshow.
933 @param ipy notebook file
934 @param folder where to write the new notebook
935 @param encoding encoding
936 @return written file
937 """
938 from ..ipythonhelper import read_nb
939 filename = os.path.split(ipy)[-1]
940 output = os.path.join(folder, filename)
941 nb = read_nb(ipy, encoding=encoding, kernel=False)
942 nb.add_tag_slide()
943 nb.to_json(output)
944 return output
947def build_notebooks_gallery(nbs, fileout, layout="classic", neg_pattern=None,
948 snippet_folder=None, fLOG=noLOG):
949 """
950 Creates a :epkg:`rst` page (gallery) with links to all notebooks.
951 For each notebook, it creates a snippet.
953 @param nbs list of notebooks to consider or tuple(full path, rst),
954 @param fileout file to create
955 @param layout ``'classic'`` or ``'table'``
956 @param neg_pattern do not consider notebooks matching this regular expression
957 @param snippet_folder folder where to find custom snippet for notebooks,
958 the snippet should have the same name as the notebook
959 itself, snippet must have extension ``.png``
960 @param fLOG logging function
961 @return created file name
963 Example for parameter *nbs*:
965 ::
967 ('challenges\\city_tour\\city_tour_1.ipynb',
968 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1.ipynb')
969 ('challenges\\city_tour\\city_tour_1_solution.ipynb',
970 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1_solution.ipynb')
971 ('challenges\\city_tour\\city_tour_data_preparation.ipynb',
972 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_data_preparation.ipynb')
973 ('challenges\\city_tour\\city_tour_long.ipynb',
974 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_long.ipynb')
975 ('cheat_sheets\\chsh_files.ipynb',
976 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_files.ipynb')
977 ('cheat_sheets\\chsh_geo.ipynb',
978 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_geo.ipynb')
980 *nbs* can be a folder, in that case, the function will build
981 the list of all notebooks in that folder.
982 *nbs* can be a list of tuple.
983 the function adds a thumbnail, organizes the list of notebook
984 as a galley, it adds a link on notebook coverage.
985 The function bypasses the generation of a snippet
986 if a custom one was found.
987 """
988 from ..ipythonhelper import read_nb
989 if not isinstance(nbs, list):
990 fold = nbs
991 nbs = explore_folder(
992 fold, ".*[.]ipynb", neg_pattern=neg_pattern, fullname=True)[1]
993 if len(nbs) == 0:
994 raise FileNotFoundError( # pragma: no cover
995 f"Unable to find notebooks in folder '{nbs}'.")
996 nbs = [(os.path.relpath(n, fold), n) for n in nbs]
998 # Go through the list of notebooks.
999 fLOG("[build_notebooks_gallery]", len(nbs), "notebooks")
1000 hier = set()
1001 rst = []
1002 containers = {}
1003 for tu in nbs:
1004 if isinstance(tu, (tuple, list)):
1005 if tu[0] is None or ("/" not in tu[0] and "\\" not in tu[0]):
1006 rst.append((tuple(), tu[1]))
1007 else:
1008 way = tuple(tu[0].replace("\\", "/").split("/")[:-1])
1009 hier.add(way)
1010 rst.append((way, tu[1]))
1011 else:
1012 rst.append((tuple(), tu))
1013 name = rst[-1][1]
1014 ext = os.path.splitext(name)[-1]
1015 if ext != ".ipynb":
1016 raise ValueError( # pragma: no cover
1017 f"One file is not a notebook: {rst[-1][1]}")
1018 dirname, na = os.path.split(name)
1019 if dirname not in containers:
1020 containers[dirname] = []
1021 containers[dirname].append(na)
1022 rst.sort()
1024 folder_index = os.path.dirname(os.path.normpath(fileout))
1025 folder = os.path.join(folder_index, "notebooks")
1026 if not os.path.exists(folder):
1027 os.mkdir(folder)
1029 # reordering based on titles
1030 titles = {}
1031 reord = []
1032 for hi, nbf in rst:
1033 nb = read_nb(nbf)
1034 title = nb.get_description()[0]
1035 titles[nbf] = title
1036 reord.append((hi, title, nbf))
1037 reord.sort()
1038 rst = [_[:1] + _[-1:] for _ in reord]
1040 # containers
1041 containers = list(sorted((k, v) for k, v in containers.items()))
1043 # find root
1044 hi, rs = rst[0]
1045 if len(hi) == 0:
1046 root = os.path.dirname(rs)
1047 else:
1048 spl = rs.replace("\\", "/").split("/")
1049 ro = spl[:-len(hi) - 1]
1050 root = "/".join(ro)
1052 # look for README.txt
1053 fLOG("[build_notebooks_gallery] root", root)
1054 rows = ["", ":orphan:", ""]
1055 exp = os.path.join(root, "README.txt")
1056 if os.path.exists(exp):
1057 fLOG("[build_notebooks_gallery] found", exp)
1058 with open(exp, "r", encoding="utf-8") as f:
1059 try:
1060 rows.extend(["", ".. _l-notebooks:", "", f.read(), ""])
1061 except UnicodeDecodeError as e: # pragma: no cover
1062 raise ValueError(f"Issue with file '{exp}'") from e
1063 else:
1064 fLOG("[build_notebooks_gallery] not found", exp)
1065 rows.extend(["", ".. _l-notebooks:", "", "", "Notebooks Gallery",
1066 "=================", ""])
1068 rows.extend(["", ":ref:`l-notebooks-coverage`", "",
1069 "", ".. contents::", " :depth: 1",
1070 " :local:", ""])
1072 # produces the final files
1073 if len(hier) == 0:
1074 # case where there is no hierarchy
1075 fLOG("[build_notebooks_gallery] no hierarchy")
1076 rows.append(".. toctree::")
1077 rows.append(" :maxdepth: 1")
1078 if layout == "table":
1079 rows.append(" :hidden:")
1080 rows.append("")
1081 for hi, file in rst:
1082 rs = os.path.splitext(os.path.split(file)[-1])[0]
1083 fLOG("[build_notebooks_gallery] adding",
1084 rs, " title ", titles.get(file, None))
1085 rows.append(f" notebooks/{rs}")
1086 if layout == "table" and len(rst) > 0:
1087 rows.extend(["", "", ".. list-table::",
1088 " :header-rows: 0", " :widths: 3 5 15", ""])
1090 for _, file in rst:
1091 link = os.path.splitext(os.path.split(file)[-1])[0]
1092 link = link.replace("_", "") + "rst"
1093 if not os.path.exists(file):
1094 raise FileNotFoundError( # pragma: no cover
1095 "Unable to find: '{0}'\nRST=\n{1}".format(
1096 file, "\n".join(str(_) for _ in rst)))
1097 r = build_thumbail_in_gallery(
1098 file, folder, folder_index, link, layout,
1099 snippet_folder=snippet_folder, fLOG=fLOG)
1100 rows.append(r)
1101 else:
1102 # case where there are subfolders
1103 fLOG("[build_notebooks_gallery] subfolders")
1104 already = "\n".join(rows)
1105 level = "-+^"
1106 rows.append("")
1107 if ".. contents::" not in already:
1108 rows.append(".. contents::")
1109 rows.append(" :local:")
1110 rows.append(" :depth: 2")
1111 rows.append("")
1112 stack_file = []
1113 last = None
1114 for hi, r in rst:
1115 rs0 = os.path.splitext(os.path.split(r)[-1])[0]
1116 r0 = r
1117 if hi != last:
1118 fLOG("[build_notebooks_gallery] new level", hi)
1119 # It adds the thumbnail.
1120 if layout == "table" and len(stack_file) > 0:
1121 rows.extend(
1122 ["", "", ".. list-table::", " :header-rows: 0", " :widths: 3 5 15", ""])
1124 for nbf in stack_file:
1125 fLOG("[build_notebooks_gallery] ", nbf)
1126 rs = os.path.splitext(os.path.split(nbf)[-1])[0]
1127 link = rs.replace("_", "") + "rst"
1128 r = build_thumbail_in_gallery(
1129 nbf, folder, folder_index, link, layout)
1130 rows.append(r)
1131 fLOG(f"[build_notebooks_gallery] saw {len(stack_file)} files")
1132 stack_file = []
1134 # It switches to the next gallery.
1135 if layout == "classic":
1136 rows.append(".. raw:: html")
1137 rows.append("")
1138 rows.append(" <div style='clear:both'></div>")
1139 rows.append("")
1141 # It adds menus and subfolders.
1142 lastk = 0
1143 for k in range(0, len(hi)):
1144 lastk = k
1145 if last is None or k >= len(last) or hi[k] != last[k]: # pylint: disable=E1136
1146 break
1148 while len(hi) > 0 and lastk < len(hi):
1149 fo = [root] + list(hi[:lastk + 1])
1150 readme = os.path.join(*(fo + ["README.txt"]))
1151 if os.path.exists(readme):
1152 fLOG("[build_notebooks_gallery] found", readme)
1153 with open(readme, "r", encoding="utf-8") as f:
1154 try:
1155 rows.extend(["", f.read(), ""])
1156 except UnicodeDecodeError as e: # pragma: no cover
1157 raise ValueError(
1158 f"Issue with file '{readme}'") from e
1159 else:
1160 fLOG("[build_notebooks_gallery] not found", readme)
1161 rows.append("")
1162 rows.append(hi[lastk])
1163 rows.append(
1164 level[min(lastk, len(level) - 1)] * len(hi[lastk]))
1165 rows.append("")
1166 lastk += 1
1168 # It starts the next gallery.
1169 last = hi
1170 rows.append(".. toctree::")
1171 rows.append(" :maxdepth: 1")
1172 if layout == "table":
1173 rows.append(" :hidden:")
1174 rows.append("")
1176 # append a link to a notebook
1177 fLOG("[build_notebooks_gallery] adding",
1178 rs0, " title ", titles.get(r0, None))
1179 rows.append(f" notebooks/{rs0}")
1180 stack_file.append(r0)
1182 if len(stack_file) > 0:
1183 # It adds the thumbnails.
1184 if layout == "table" and len(stack_file) > 0:
1185 rows.extend(["", "", ".. list-table::",
1186 " :header-rows: 0", " :widths: 3 5 15", ""])
1188 for nbf in stack_file:
1189 rs = os.path.splitext(os.path.split(nbf)[-1])[0]
1190 link = rs.replace("_", "") + "rst"
1191 r = build_thumbail_in_gallery(
1192 nbf, folder, folder_index, link, layout)
1193 rows.append(r)
1195 # done
1196 rows.append("")
1198 # links to coverage
1199 rows.extend(["", "", ".. toctree::", " :hidden: ", "",
1200 " all_notebooks_coverage", ""])
1202 with open(fileout, "w", encoding="utf8") as f:
1203 f.write("\n".join(rows))
1204 return fileout
1207def build_all_notebooks_coverage(nbs, fileout, module_name, dump=None, badge=True, too_old=30, fLOG=noLOG):
1208 """
1209 Creates a :epkg:`rst` page (gallery) with links to all notebooks and
1210 information about coverage.
1211 It relies on function @see fn notebook_coverage.
1213 @param nbs list of notebooks to consider or tuple(full path, rst),
1214 @param fileout file to create
1215 @param module_name module name
1216 @param dump dump containing information about notebook execution (or None for the default one)
1217 @param badge builds an image with the notebook coverage
1218 @param too_old drop executions older than *too_old* days from now
1219 @param fLOG logging function
1220 @return dataframe which contains the data
1221 """
1222 from ..ipythonhelper import read_nb, notebook_coverage
1223 if dump is None:
1224 dump = os.path.normpath(os.path.join(os.path.dirname(fileout), "..", "..", "..", "..",
1225 "_notebook_dumps", f"notebook.{module_name}.txt"))
1226 if not os.path.exists(dump):
1227 fLOG("[notebooks-coverage] No execution report about "
1228 "notebook at '{0}' (fileout='{1}')".format(dump,
1229 os.path.dirname(fileout)))
1230 return None
1231 report0 = notebook_coverage(nbs, dump, too_old=too_old)
1232 fLOG("[notebooks-coverage] report shape", report0.shape)
1234 from numpy import isnan
1236 # Fill nan values.
1237 for i in report0.index:
1238 nbcell = report0.loc[i, "nbcell"]
1239 if isnan(nbcell):
1240 # It loads the notebook.
1241 nbfile = report0.loc[i, "notebooks"]
1242 nb = read_nb(nbfile)
1243 report0.loc[i, "nbcell"] = len(nb)
1244 report0.loc[i, "nbrun"] = 0
1246 # Add links.
1247 cols = ['notebooks', 'date', 'etime',
1248 'nbcell', 'nbrun', 'nbvalid', 'success', 'time']
1249 report = report0[cols].copy()
1250 report["notebooks"] = report["notebooks"].apply(
1251 lambda x: "/".join(os.path.normpath(x).replace("\\", "/").split("/")[-2:]) if isinstance(x, str) else x)
1252 report["last_name"] = report["notebooks"].apply(
1253 lambda x: os.path.split(x)[-1] if isinstance(x, str) else x)
1255 report1 = report.copy()
1257 def clean_link(link):
1258 return link.replace("_", "").replace(".ipynb", ".rst").replace(".", "") if isinstance(link, str) else link
1260 report["notebooks"] = report.apply(lambda row: ':ref:`{0} <{1}>`'.format(
1261 row["notebooks"], clean_link(row["last_name"])), axis=1)
1262 report["title"] = report["last_name"].apply(
1263 lambda x: f':ref:`{clean_link(x)}`')
1264 rows = ["", ".. _l-notebooks-coverage:", "", "", "Notebooks Coverage",
1265 "==================", "", "Report on last executions.", ""]
1267 # Badge
1268 if badge:
1269 from ..ipythonhelper import badge_notebook_coverage
1270 img = os.path.join(os.path.dirname(fileout), "nbcov.png")
1271 cov = badge_notebook_coverage(report0, img)
1272 now = datetime.datetime.now()
1273 sdate = "%04d-%02d-%02d" % (now.year, now.month, now.day)
1274 cpy = os.path.join(os.path.dirname(fileout), f"nbcov-{sdate}.png")
1275 shutil.copy(img, cpy)
1276 badge = ["{0:0.00f}% {1}".format(
1277 cov, sdate), "", f".. image:: {os.path.split(cpy)[-1]}", ""]
1278 badge2 = ["", f".. image:: {os.path.split(img)[-1]}", ""]
1279 else:
1280 badge = []
1281 badge2 = []
1282 rows.extend(badge)
1284 # Formatting
1285 report["date"] = report["date"].apply(
1286 lambda x: x.split()[0] if isinstance(x, str) else x)
1287 report["etime"] = report["etime"].apply(
1288 lambda x: f"{x:1.3f}" if isinstance(x, float) else x)
1289 report["time"] = report["time"].apply(
1290 lambda x: f"{x:1.3f}" if isinstance(x, float) else x)
1292 def int2str(x):
1293 if isnan(x):
1294 return ""
1295 else:
1296 return int(x)
1298 report["coverage"] = report["nbrun"] / report["nbcell"]
1299 report["nbcell"] = report["nbcell"].apply(int2str)
1300 report["nbrun"] = report["nbrun"].apply(int2str)
1301 report["nbvalid"] = report["nbvalid"].apply(int2str)
1302 report["coverage"] = report["coverage"].apply(
1303 lambda x: f"{int(x * 100)}%" if isinstance(x, float) else "")
1304 report = report[['notebooks', 'title', 'date', 'success', 'etime',
1305 'nbcell', 'nbrun', 'nbvalid', 'time', 'coverage']].copy()
1306 report.columns = ['name', 'title', 'last execution', 'success', 'time',
1307 'nb cells', 'nb runs', 'nb valid', 'exe time', 'coverage']
1308 report = report[['coverage', 'exe time', 'last execution', 'name', 'title',
1309 'success', 'time', 'nb cells', 'nb runs', 'nb valid']]
1311 # Add results.
1312 text = df2rst(report.sort_values("name").reset_index(
1313 drop=True), index=True, list_table=True)
1314 rows.append(text)
1315 rows.extend(badge2)
1317 fLOG("[notebooks-coverage] writing", fileout)
1318 with open(fileout, "w", encoding="utf-8") as f:
1319 f.write("\n".join(rows))
1320 return report1