Coverage for src/pyquickhelper/helpgen/process_notebooks.py : 96%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains the main function to generate the documentation
5for a module designed the same way as this one, @see fn generate_help_sphinx.
7"""
8import datetime
9import json
10import os
11import sys
12import shutil
13import warnings
14from io import StringIO
15from nbconvert.exporters.base import ExporterNameError
17from .utils_sphinx_doc_helpers import HelpGenException
18from .conf_path_tools import find_latex_path, find_pandoc_path
19from .post_process import (
20 post_process_latex_output, post_process_latex_output_any,
21 post_process_rst_output, post_process_html_output,
22 post_process_slides_output, post_process_python_output)
23from .helpgen_exceptions import NotebookConvertError
24from .install_js_dep import install_javascript_tools
25from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE
26from .process_notebook_api import nb2rst
27from ..loghelper.flog import run_cmd, fLOG, noLOG
28from ..pandashelper import df2rst
29from ..filehelper.synchelper import has_been_updated, explore_folder
32template_examples = """
34List of programs
35++++++++++++++++
37.. toctree::
38 :maxdepth: 2
40.. autosummary:: __init__.py
41 :toctree: %s/
42 :template: modules.rst
44Another list
45++++++++++++
47"""
50def find_pdflatex(latex_path):
51 """
52 Returns the executable for latex.
54 @param latex_path path to look (only on Windows)
55 @return executable
56 """
57 if sys.platform.startswith("win"): # pragma: no cover
58 lat = os.path.join(latex_path, "xelatex.exe")
59 if os.path.exists(lat):
60 return lat
61 lat = os.path.join(latex_path, "pdflatex.exe")
62 if os.path.exists(lat):
63 return lat
64 raise FileNotFoundError(
65 "Unable to find pdflatex or xelatex in '{0}'".format(latex_path))
66 if sys.platform.startswith("darwin"): # pragma: no cover
67 try:
68 err = run_cmd("/Library/TeX/texbin/xelatex --help", wait=True)[1]
69 if len(err) == 0:
70 return "/Library/TeX/texbin/xelatex"
71 raise FileNotFoundError( # pragma: no cover
72 "Unable to run xelatex\n{0}".format(err))
73 except Exception:
74 return "/Library/TeX/texbin/pdflatex"
75 try:
76 err = run_cmd("xelatex --help", wait=True)[1]
77 if len(err) == 0:
78 return "xelatex"
79 else:
80 raise FileNotFoundError(
81 "Unable to run xelatex\n{0}".format(err))
82 except Exception: # pragma: no cover
83 return "pdflatex"
86def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=None,
87 formats="ipynb,html,python,rst,slides,pdf,github",
88 fLOG=fLOG, exc=True, remove_unicode_latex=False, nblinks=None,
89 notebook_replacements=None):
90 """
91 Converts notebooks into :epkg:`html`, :epkg:`rst`, :epkg:`latex`,
92 :epkg:`pdf`, :epkg:`python`, :epkg:`docx` using
93 :epkg:`nbconvert`.
95 @param notebooks list of notebooks or comma separated values
96 @param outfold folder which will contains the outputs
97 @param build temporary folder which contains all produced files
98 @param pandoc_path path to pandoc
99 @param formats list of formats to convert into (pdf format means latex
100 then compilation), or comma separated values
101 @param latex_path path to the latex compiler
102 @param fLOG logging function
103 @param exc raises an exception (True) or a warning (False) if an error happens
104 @param nblinks dictionary ``{ref: url}`` or a string in :epkg:`json`
105 format
106 @param remove_unicode_latex remove unicode characters for latex (to avoid failing)
107 @param notebook_replacements string replacement in a notebook before conversion
108 or a string in :epkg:`json` format
109 @return list of tuple *[(file, created or skipped)]*
111 This function relies on :epkg:`pandoc`.
112 It also needs modules :epkg:`pywin32`,
113 :epkg:`pygments`.
115 :epkg:`pywin32` might have some issues
116 to find its DLL, look @see fn import_pywin32.
118 The latex compilation uses :epkg:`MiKTeX`.
119 The conversion into Word document directly uses pandoc.
120 It still has an issue with table.
122 Some latex templates (for nbconvert) uses ``[commandchars=\\\\\\{\\}]{\\|}`` which allows commands ``\\\\`` and it does not compile.
123 The one used here is ``report``.
124 Some others bugs can be found at: `schlichtanders/latex_test.html <https://gist.github.com/schlichtanders/e108ed0be80108178af2>`_.
125 For example, you must not let spaces between symbol ``$`` and the
126 formulas it indicates.
128 If *pandoc_path* is None, uses @see fn find_pandoc_path to guess it.
129 If *latex_path* is None, uses @see fn find_latex_path to guess it.
131 .. exref::
132 :title: Convert a notebook into multiple formats
134 ::
136 from pyquickhelper.ipythonhelper import process_notebooks
137 process_notebooks("td1a_correction_session7.ipynb",
138 "dest_folder", "dest_folder",
139 formats=("ipynb", "html", "python", "rst", "slides", "pdf",
140 "docx", "github")])
142 For latex and pdf, a custom processor was added to handle raw data
143 and add ``\\begin{verbatim}`` and ``\\end{verbatim}``.
144 Format *github* adds a link to file on :epkg:`github`.
146 .. todoext::
147 :title: check differences between _process_notebooks_in_private and _process_notebooks_in_private_cmd
148 :tag: bug
150 For :epkg:`latex` and :epkg:`pdf`,
151 the custom preprocessor is not taken into account.
152 by function @see fn _process_notebooks_in_private.
153 """
154 if isinstance(notebooks, str):
155 notebooks = notebooks.split(',')
156 if isinstance(formats, str):
157 formats = formats.split(',')
158 if isinstance(notebook_replacements, str):
159 notebook_replacements = json.loads(notebook_replacements)
160 if isinstance(nblinks, str):
161 nblinks = json.loads(nblinks)
162 if build is None:
163 raise ValueError("build cannot be None")
165 res = _process_notebooks_in(notebooks=notebooks, outfold=outfold, build=build,
166 latex_path=latex_path, pandoc_path=pandoc_path,
167 formats=formats, fLOG=fLOG, exc=exc, nblinks=nblinks,
168 remove_unicode_latex=remove_unicode_latex,
169 notebook_replacements=notebook_replacements)
170 if "slides" in formats:
171 # we copy javascript dependencies, reveal.js
172 reveal = os.path.join(outfold, "reveal.js")
173 if not os.path.exists(reveal):
174 install_javascript_tools(None, dest=outfold)
175 reveal = os.path.join(build, "reveal.js")
176 if not os.path.exists(reveal):
177 install_javascript_tools(None, dest=build)
178 return res
181def _process_notebooks_in_private(fnbcexe, list_args, options_args):
182 """
183 This function fails in nbconvert 6.0 when the conversion
184 is called more than once. The conversion probably changes the
185 initial state.
186 """
187 out = StringIO()
188 err = StringIO()
189 memo_out = sys.stdout
190 memo_err = sys.stderr
191 sys.stdout = out
192 sys.stderr = err
193 try:
194 if list_args:
195 fnbcexe(argv=list_args, **options_args)
196 else:
197 fnbcexe(**options_args)
198 exc = None
199 except SystemExit as e: # pragma: no cover
200 exc = e
201 except IndentationError as e: # pragma: no cover
202 # This is change in IPython 6.0.0.
203 # The conversion fails on IndentationError.
204 # We switch to another one.
205 from ..ipythonhelper import read_nb
206 i = list_args.index("--template")
207 format = list_args[i + 1]
208 if format == "python":
209 i = list_args.index("--output")
210 dest = list_args[i + 1]
211 if not dest.endswith(".py"):
212 dest += ".py"
213 src = list_args[-1]
214 nb = read_nb(src)
215 code = nb.to_python()
216 with open(dest, "w", encoding="utf-8") as f:
217 f.write(code)
218 exc = None
219 else:
220 # We do nothing in this case.
221 exc = e
222 except (AttributeError, FileNotFoundError, ValueError) as e:
223 exc = e
224 except ExporterNameError as e: # pragma: no cover
225 exc = e
226 sys.stdout = memo_out
227 sys.stderr = memo_err
228 out = out.getvalue()
229 err = err.getvalue()
230 if exc:
231 if "Unsupported mimetype 'text/html'" in str(exc):
232 from nbconvert.nbconvertapp import main
233 main(argv=list_args, **options_args)
234 return "", ""
235 env = "\n".join("{0}={1}".format(k, v)
236 for k, v in sorted(os.environ.items()))
237 raise RuntimeError( # pragma: no cover
238 "Notebook conversion failed.\nfnbcexe\n{}\noptions_args\n{}"
239 "\n--ARGS--\n{}\n--OUT--\n{}\n--ERR--\n{}\n--ENVIRON--\n{}"
240 "".format(fnbcexe, options_args, list_args, out, err,
241 env)) from exc
242 return out, err
245def _process_notebooks_in_private_cmd(fnbcexe, list_args, options_args, fLOG):
246 this = os.path.join(os.path.dirname(
247 os.path.abspath(__file__)), "process_notebooks_cmd.py")
248 res = []
249 for c in list_args:
250 if c[0] == '"' or c[-1] == '"' or ' ' not in c:
251 res.append(c)
252 else:
253 res.append('"{0}"'.format(c))
254 sargs = " ".join(res)
255 cmd = '"{0}" "{1}" {2}'.format(
256 sys.executable.replace("w.exe", ".exe"), this, sargs)
257 fLOG("[_process_notebooks_in_private_cmd]", cmd)
258 return run_cmd(cmd, wait=True, fLOG=fLOG)
261def _preprocess_notebook(notebook_content):
262 """
263 Preprocesses the content of a notebook.
265 @param notebook_content notebook content
266 @return modified content
267 """
268 def walk_through(field):
269 if isinstance(field, list):
270 for f in field:
271 walk_through(f)
272 elif isinstance(field, dict):
273 if (field.get('version_major', -1) == 2 and
274 field.get('version_minor', -1) == 0):
275 field['version_minor'] = 2
276 elif (field.get('nbformat', -1) == 4 and
277 field.get('nbformat_minor', -1) in (0, 1)):
278 field['nbformat_minor'] = 2
279 for _, v in field.items():
280 walk_through(v)
282 content = json.loads(notebook_content)
283 walk_through(content)
284 new_content = json.dumps(content)
285 return new_content
288def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_path=None,
289 formats=("ipynb", "html", "python", "rst",
290 "slides", "pdf", "github"),
291 fLOG=fLOG, exc=True, nblinks=None, remove_unicode_latex=False,
292 notebook_replacements=None):
293 """
294 The notebook conversion does not handle images from url
295 for :epkg:`pdf` and :epkg:`docx`. They could be downloaded first
296 and replaced by local files.
298 .. note::
300 :epkg:`nbconvert` introduced a commit which breaks
301 the conversion of notebooks in latex if they have
302 a cell outputting *svg*
303 (see `PR 910 <https://github.com/jupyter/nbconvert/pull/910>`_).
305 Use `xelatex <https://doc.ubuntu-fr.org/xelatex>`_ if possible.
306 """
307 from nbconvert.nbconvertapp import main as nbconvert_main
308 if pandoc_path is None:
309 pandoc_path = find_pandoc_path()
311 if latex_path is None:
312 latex_path = find_latex_path()
314 if isinstance(notebooks, str):
315 notebooks = [notebooks]
317 if "PANDOCPY" in os.environ and sys.platform.startswith("win"): # pragma: no cover
318 exe = os.environ["PANDOCPY"]
319 exe = exe.rstrip("\\/")
320 if exe.endswith("\\Scripts"):
321 exe = exe[:len(exe) - len("Scripts") - 1]
322 if not os.path.exists(exe):
323 raise FileNotFoundError(exe)
324 fLOG("[_process_notebooks_in] ** using PANDOCPY", exe)
325 else:
326 if sys.platform.startswith("win"): # pragma: no cover
327 from .utils_pywin32 import import_pywin32
328 try:
329 import_pywin32()
330 except ModuleNotFoundError as e:
331 warnings.warn(e)
332 exe = os.path.split(sys.executable)[0]
334 extensions = {"ipynb": ".ipynb", "latex": ".tex", "elatex": ".tex", "pdf": ".pdf",
335 "html": ".html", "rst": ".rst", "python": ".py", "docx": ".docx",
336 "word": ".docx", "slides": ".slides.html"}
338 files = []
339 skipped = []
341 # main(argv=None, **kwargs)
342 fnbc = nbconvert_main
344 if "slides" in formats:
345 build_slide = os.path.join(build, "bslides")
346 if not os.path.exists(build_slide):
347 os.mkdir(build_slide)
349 copied_images = dict()
351 for notebook_in in notebooks:
352 thisfiles = []
354 # we copy available images (only notebook folder)
355 # in case they are used in latex
356 currentdir = os.path.abspath(os.path.dirname(notebook_in))
357 for curfile in os.listdir(currentdir):
358 ext = os.path.splitext(curfile)[1]
359 if ext in {'.png', '.jpg', '.bmp', '.gif', '.jpeg', '.svg', '.mp4'}:
360 src = os.path.join(currentdir, curfile)
361 if src not in copied_images:
362 dest = os.path.join(build, curfile)
363 shutil.copy(src, build)
364 fLOG("[_process_notebooks_in] copy '{}' to '{}'.".format(
365 src, build))
366 copied_images[src] = dest
368 # copy of the notebook into the build folder
369 # and changes the source
370 _name = os.path.splitext(os.path.split(notebook_in)[-1])[0]
371 _name += '.ipynb'
372 notebook = os.path.join(build, _name)
373 fLOG("[_process_notebooks_in] -- copy notebook '{}' to '{}'.".format(
374 notebook_in, notebook))
375 with open(notebook_in, "r", encoding="utf-8") as _f:
376 content = _f.read()
377 content = _preprocess_notebook(content)
378 with open(notebook, "w", encoding="utf-8") as _f:
379 _f.write(content)
381 # next
382 nbout = os.path.split(notebook)[-1]
383 if " " in nbout:
384 raise HelpGenException(
385 "spaces are not allowed in notebooks file names: "
386 "{0}".format(notebook))
387 nbout = os.path.splitext(nbout)[0]
388 for format in formats:
390 if format == "github":
391 # we add a link on the rst page in that case
392 continue
394 if format not in extensions:
395 raise NotebookConvertError( # pragma: no cover
396 "Unable to find format: '{}' in {}".format(
397 format, ", ".join(extensions.keys())))
399 # output
400 format_ = format
401 outputfile_noext = os.path.join(build, nbout)
402 if format == 'html':
403 outputfile = outputfile_noext + '2html' + extensions[format]
404 outputfile_noext_fixed = outputfile_noext + '2html'
405 else:
406 outputfile = outputfile_noext + extensions[format]
407 outputfile_noext_fixed = outputfile_noext
408 trueoutputfile = outputfile
409 pandoco = "docx" if format in ("word", "docx") else None
411 # The function checks it was not done before.
412 if os.path.exists(trueoutputfile):
413 dto = os.stat(trueoutputfile).st_mtime
414 dtnb = os.stat(notebook).st_mtime
415 if dtnb < dto: # pragma: no cover
416 fLOG("[_process_notebooks_in] -- skipping notebook", format,
417 notebook, "(", trueoutputfile, ")")
418 if trueoutputfile not in thisfiles:
419 thisfiles.append(trueoutputfile)
420 if pandoco is None:
421 skipped.append(trueoutputfile)
422 continue
423 out2 = os.path.splitext(
424 trueoutputfile)[0] + "." + pandoco
425 if os.path.exists(out2):
426 skipped.append(trueoutputfile)
427 continue
429 # if the format is slides, we update the metadata
430 options_args = {}
431 if format == "slides":
432 nb_slide = add_tag_for_slideshow(notebook, build_slide)
433 fnbcexe = fnbc
434 else:
435 nb_slide = None
436 fnbcexe = fnbc
438 # compilation
439 list_args = []
440 custom_config = os.path.join(os.path.abspath(
441 os.path.dirname(__file__)), "_nbconvert_config.py")
442 if format == "pdf":
443 if not os.path.exists(custom_config):
444 raise FileNotFoundError(custom_config)
445 # title = os.path.splitext(
446 # os.path.split(notebook)[-1])[0].replace("_", " ")
447 list_args.extend(['--config', '"%s"' % custom_config])
448 format = "latex"
449 compilation = True
450 thisfiles.append(os.path.splitext(outputfile)[0] + ".tex")
451 elif format in ("latex", "elatex"):
452 if not os.path.exists(custom_config):
453 raise FileNotFoundError(custom_config)
454 list_args.extend(['--config', '"%s"' % custom_config])
455 compilation = False
456 format = "latex"
457 elif format in ("word", "docx"):
458 format = "html"
459 compilation = False
460 elif format in ("slides", ):
461 list_args.extend(["--reveal-prefix", "reveal.js"])
462 compilation = False
463 else:
464 compilation = False
466 # output
467 # set templates to None to avoid error
468 # No template sub-directory with name 'article' found in the following paths:
469 templ = {'html': None, 'latex': None,
470 'elatex': None}.get(format, format)
471 fLOG("[_process_notebooks_in] ### convert into '{}' (done: {}): '{}' -> '{}'".format(
472 format_, os.path.exists(outputfile), notebook, outputfile))
474 list_args.extend(["--output", outputfile_noext_fixed])
475 if templ is not None and format != "slides":
476 list_args.extend(["--template", templ])
478 # execution
479 if format not in ("ipynb", ):
480 # nbconvert is messing up with static variables in sphinx or
481 # docutils if format is slides, not sure about the others
482 if format in ('rst', ):
483 fLOG("[_process_notebooks_in] NBcn:", format, options_args)
484 nb2rst(notebook, outputfile, post_process=False)
485 err = ""
486 c = ""
487 elif nbconvert_main != fnbcexe or format not in (
488 "slides", "elatex", "latex", "pdf", "html"):
489 if options_args:
490 fLOG("[_process_notebooks_in] NBp*:",
491 format, options_args)
492 else:
493 list_args.extend(["--to", format,
494 notebook if nb_slide is None else nb_slide])
495 fLOG(
496 "[_process_notebooks_in] NBc* format='{}' args={}".format(format, list_args))
497 fLOG("[_process_notebooks_in] cwd='{}'".format(os.getcwd()))
499 c = " ".join(list_args)
500 out, err = _process_notebooks_in_private(
501 fnbcexe, list_args, options_args)
502 else:
503 # conversion into slides alter Jinja2 environment
504 # jinja2.exceptions.TemplateNotFound: rst
505 if options_args:
506 fLOG("[_process_notebooks_in] NBp+:",
507 format, options_args)
508 else:
509 list_args.extend(["--to", format,
510 notebook if nb_slide is None else nb_slide])
511 fLOG("[_process_notebooks_in] NBc+:", format, list_args)
512 fLOG("[_process_notebooks_in]", os.getcwd())
514 c = " ".join(list_args)
515 out, err = _process_notebooks_in_private_cmd(
516 fnbcexe, list_args, options_args, fLOG)
518 if "raise ImportError" in err or "Unknown exporter" in err:
519 raise ImportError(
520 "cmd: {0} {1}\n--ERR--\n{2}".format(fnbcexe, list_args, err))
521 if len(err) > 0:
522 if format in ("elatex", "latex"):
523 # There might be some errors because the latex script needs to be post-processed
524 # sometimes (wrong characters such as " or formulas not
525 # captured as formulas).
526 if err and "usage: process_notebooks_cmd.py" in err:
527 raise RuntimeError(
528 "Unable to convert a notebook\n----\n{}----\n{}\n"
529 "---ERR---\n{}\n---OUT---\n{}".format(
530 fnbcexe, list_args, err, out))
531 fLOG("[_process_notebooks_in] LATEX --ERR--\n" + err)
532 fLOG("[_process_notebooks_in] LATEX --OUT--\n" + out)
533 else:
534 err = err.lower()
535 if "critical" in err or "bad config" in err:
536 raise HelpGenException(
537 "CMD:\n{0}\n[nberror]\n{1}".format(list_args, err))
538 else:
539 # format ipynb
540 # we do nothing
541 pass
543 format = extensions[format].strip(".")
545 # we add the file to the list of generated files
546 if outputfile not in thisfiles:
547 thisfiles.append(outputfile)
549 fLOG("[_process_notebooks_in] -",
550 format, compilation, outputfile)
552 if compilation:
553 # compilation latex
554 if not sys.platform.startswith("win") or os.path.exists(latex_path):
555 lat = find_pdflatex(latex_path)
557 tex = set(_ for _ in thisfiles if os.path.splitext(
558 _)[-1] == ".tex")
559 if len(tex) != 1:
560 raise FileNotFoundError(
561 "No latex file was generated or more than one (={0}), nb={1}\nthisfile=\n{2}".format(
562 len(tex), notebook, "\n".join(thisfiles)))
563 tex = list(tex)[0]
564 try:
565 post_process_latex_output_any(
566 tex, custom_latex_processing=None, nblinks=nblinks,
567 remove_unicode=remove_unicode_latex, fLOG=fLOG)
568 except FileNotFoundError as e:
569 mes = ("[_process_notebooks_in-ERROR] Unable to to convert into latex"
570 "notebook %r due to %r.") % (tex, e)
571 warnings.warn(mes, RuntimeWarning)
572 fLOG(mes)
573 continue
575 # -interaction=batchmode
576 c = '"{0}" "{1}" -max-print-line=900 -output-directory="{2}"'.format(
577 lat, tex, os.path.split(tex)[0])
578 fLOG("[_process_notebooks_in] ** LATEX compilation (b)", c)
579 if not sys.platform.startswith("win"):
580 c = c.replace('"', '')
581 if sys.platform.startswith("win"):
582 change_path = None
583 else:
584 # On Linux the parameter --output-directory is sometimes ignored.
585 # And it only works from the current directory.
586 change_path = os.path.split(tex)[0]
587 out, err = run_cmd(
588 c, wait=True, log_error=False, shell=sys.platform.startswith("win"),
589 catch_exit=True, prefix_log="[latex] ", change_path=change_path)
590 if out is not None and ("Output written" in out or 'bytes written' in out):
591 # The output was produced. We ignore the return code.
592 fLOG("[_process_notebooks_in] WARNINGS: "
593 "Latex compilation had warnings:", c)
594 out += "\n--ERR--\n" + err
595 err = ""
596 if len(err) > 0:
597 raise HelpGenException(
598 "CMD:\n{0}\n[nberror]\n{1}\nOUT:\n{2}------".format(c, err, out))
599 f = os.path.join(build, nbout + ".pdf")
600 if not os.path.exists(f): # pragma: no cover
601 # On Linux the parameter --output-directory is sometimes ignored.
602 # And it only works from the current directory.
603 # We check again.
604 loc = os.path.split(f)[-1]
605 if os.path.exists(loc):
606 # We move the file.
607 moved = True
608 shutil.move(loc, f)
609 else:
610 moved = False
611 if not os.path.exists(f):
612 files = "\n".join(os.listdir(build))
613 msg = "Content of '{0}':\n{1}\n----\n'{2}' moved? {3}\nCMD:\n{4}".format(
614 build, files, loc, moved, c)
615 raise HelpGenException(
616 "Missing file: '{0}'\nCMD\n{4}nOUT:\n{2}\n[nberror]\n{1}\n-----\n{3}".format(f, err, out, msg, c))
617 thisfiles.append(f)
618 else:
619 fLOG("[_process_notebooks_in] unable to find latex in", latex_path)
621 elif pandoco is not None: # pragma: no cover
622 # compilation pandoc
623 fLOG("[_process_notebooks_in] ** pandoc compilation (b)", pandoco)
624 inputfile = os.path.splitext(outputfile)[0] + ".html"
625 outfilep = os.path.splitext(outputfile)[0] + "." + pandoco
627 # for some files, the following error might appear:
628 # Stack space overflow: current size 33692 bytes.
629 # Use `+RTS -Ksize -RTS' to increase it.
630 # it usually means there is something wrong (circular
631 # reference, ...)
632 if sys.platform.startswith("win"):
633 c = '"{0}\\pandoc.exe" +RTS -K32m -RTS -f html -t {1} "{2}" -o "{3}"'.format(
634 pandoc_path, pandoco, inputfile, outfilep)
635 else:
636 c = 'pandoc +RTS -K32m -RTS -f html -t {0} "{1}" -o "{2}"'.format(
637 pandoco, outputfile, outfilep)
639 if not sys.platform.startswith("win"):
640 c = c.replace('"', '')
641 out, err = run_cmd(
642 c, wait=True, log_error=False, shell=sys.platform.startswith("win"))
643 if len(err) > 0:
644 lines = err.strip("\r\n").split("\n")
645 # we filter out the message
646 # pandoc.exe: Could not find image `https://
647 left = [
648 _ for _ in lines if _ and "Could not find image `http" not in _]
649 if len(left) > 0:
650 raise HelpGenException(
651 "issue with cmd: %s\n[nberror]\n%s" % (c, err))
652 for _ in lines:
653 fLOG("[_process_notebooks_in] w, pandoc issue: {0}".format(
654 _.strip("\n\r")))
655 outputfile = outfilep
656 format = "docx"
658 nb_replacements = notebook_replacements.get(
659 format, None) if notebook_replacements else None
661 if format == "html":
662 # we add a link to the notebook
663 if not os.path.exists(outputfile):
664 raise FileNotFoundError( # pragma: no cover
665 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join(
666 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + c)
667 thisfiles += add_link_to_notebook(outputfile, notebook, "pdf" in formats, False,
668 "python" in formats, "slides" in formats,
669 exc=exc, nblinks=nblinks, fLOG=fLOG,
670 notebook_replacements=nb_replacements)
672 elif format == "slides.html":
673 # we add a link to the notebook
674 if not os.path.exists(outputfile):
675 raise FileNotFoundError( # pragma: no cover
676 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join(
677 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + str(list_args))
678 thisfiles += add_link_to_notebook(outputfile, notebook,
679 "pdf" in formats, False, "python" in formats,
680 "slides" in formats, exc=exc,
681 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
683 elif format == "ipynb":
684 # we just copy the notebook
685 thisfiles += add_link_to_notebook(outputfile, notebook,
686 "ipynb" in formats, False, "python" in formats,
687 "slides" in formats, exc=exc,
688 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
690 elif format == "rst":
691 # It adds a link to the notebook.
692 thisfiles += add_link_to_notebook(
693 outputfile, notebook, "pdf" in formats, "html" in formats, "python" in formats,
694 "slides" in formats, exc=exc, github="github" in formats,
695 notebook=notebook, nblinks=nblinks, fLOG=fLOG)
697 elif format in ("tex", "elatex", "latex", "pdf"):
698 thisfiles += add_link_to_notebook(outputfile, notebook, False, False,
699 False, False, exc=exc, nblinks=nblinks,
700 fLOG=fLOG, notebook_replacements=nb_replacements)
702 elif format in ("py", "python"):
703 post_process_python_output(
704 outputfile, True, nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements)
706 elif format in ["docx", "word"]:
707 pass
709 else:
710 raise HelpGenException("unexpected format " + format)
712 files.extend(thisfiles)
713 fLOG("[_process_notebooks_in] ### conversion into '{}' done into '{}'.".format(
714 format_, outputfile))
716 copy = []
717 for f in files:
718 dest = os.path.join(outfold, os.path.split(f)[-1])
719 if not f.endswith(".tex"):
721 if sys.version_info >= (3, 4):
722 try:
723 shutil.copy(f, outfold)
724 fLOG("[_process_notebooks_in] copy ",
725 f, " to ", outfold, "[", dest, "]")
726 except shutil.SameFileError:
727 fLOG("[_process_notebooks_in] w,file ",
728 dest, "already exists")
729 else: # pragma: no cover
730 try:
731 shutil.copy(f, outfold)
732 fLOG("[_process_notebooks_in] copy ",
733 f, " to ", outfold, "[", dest, "]")
734 except shutil.Error as e:
735 if "are the same file" in str(e):
736 fLOG("[_process_notebooks_in] w,file ",
737 dest, "already exists")
738 else:
739 raise e
741 if not os.path.exists(dest):
742 raise FileNotFoundError(dest)
743 copy.append((dest, True))
745 # image
746 for image in os.listdir(build):
747 if image.endswith(".png") or image.endswith(".html") or \
748 image.endswith(".pdf") or image.endswith(".svg") or \
749 image.endswith(".jpg") or image.endswith(".gif") or \
750 image.endswith(".xml") or image.endswith(".jpeg"):
751 image = os.path.join(build, image)
752 dest = os.path.join(outfold, os.path.split(image)[-1])
754 try:
755 shutil.copy(image, outfold)
756 fLOG("[_process_notebooks_in] copy ",
757 image, " to ", outfold, "[", dest, "]")
758 except shutil.SameFileError:
759 fLOG("[_process_notebooks_in] w,file ",
760 dest, "already exists")
762 if not os.path.exists(dest):
763 raise FileNotFoundError(dest) # pragma: no cover
764 copy.append((dest, True))
766 return copy + [(_, False) for _ in skipped]
769def add_link_to_notebook(file, nb, pdf, html, python, slides, exc=True,
770 github=False, notebook=None, nblinks=None, fLOG=None,
771 notebook_replacements=None):
772 """
773 Adds a link to the notebook in :epkg:`HTML` format and does a little bit of cleaning
774 for various format.
776 @param file notebook.html
777 @param nb notebook (.ipynb)
778 @param pdf if True, add a link to the PDF, assuming it will exists at the same location
779 @param html if True, add a link to the HTML conversion
780 @param python if True, add a link to the Python conversion
781 @param slides if True, add a link to the HTML slides
782 @param exc raises an exception (True) or a warning (False)
783 @param github add a link to the notebook on github
784 @param notebook location of the notebook (file might be a copy)
785 @param nblinks dictionary ``{ref: url}``
786 @param notebook_replacements stirng replacement in notebooks
787 @param fLOG logging function
788 @return list of generated files
790 The function does some cleaning too in the files.
791 """
792 core, ext = os.path.splitext(file)
793 if core.endswith(".slides"):
794 ext = ".slides" + ext
795 fLOG("[add_link_to_notebook] add_link_to_notebook", ext, " file ", file)
797 fold = os.path.split(file)[0]
798 res = [os.path.join(fold, os.path.split(nb)[-1])]
799 newr = has_been_updated(nb, res[-1])[0]
800 if newr:
801 shutil.copy(nb, fold)
803 if ext == ".ipynb":
804 return res
805 if ext == ".pdf":
806 return res
807 if ext == ".html":
808 post_process_html_output(
809 file, pdf, python, slides, exc=exc, nblinks=nblinks,
810 fLOG=fLOG, notebook_replacements=notebook_replacements)
811 return res
812 if ext == ".slides.html":
813 post_process_slides_output(
814 file, pdf, python, slides, exc=exc, nblinks=nblinks,
815 fLOG=fLOG, notebook_replacements=notebook_replacements)
816 return res
817 if ext == ".slides2p.html":
818 post_process_slides_output(
819 file, pdf, python, slides, exc=exc, nblinks=nblinks,
820 fLOG=fLOG, notebook_replacements=notebook_replacements)
821 return res
822 if ext == ".tex":
823 post_process_latex_output(
824 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG,
825 notebook_replacements=notebook_replacements)
826 return res
827 if ext == ".py":
828 post_process_python_output(
829 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG,
830 notebook_replacements=notebook_replacements)
831 return res
832 if ext == ".rst":
833 post_process_rst_output(
834 file, html, pdf, python, slides, is_notebook=True, exc=exc,
835 github=github, notebook=notebook, nblinks=nblinks, fLOG=fLOG,
836 notebook_replacements=notebook_replacements)
837 return res
838 raise HelpGenException(
839 "Unable to add a link to this extension: %r" % ext)
842def build_thumbail_in_gallery(nbfile, folder_snippet, relative, rst_link, layout, snippet_folder=None, fLOG=None):
843 """
844 Returns :epkg:`rst` code for a notebook.
846 @param nbfile notebook file
847 @param folder_snippet where to store the snippet
848 @param relative the path to the snippet will be relative to this folder
849 @param rst_link rst link
850 @param layout ``'classic'`` or ``'table'``
851 @param snippet_folder folder where to find custom snippet for notebooks,
852 the snippet should have the same name as the notebook
853 itself, snippet must have extension ``.png``
854 @return RST
856 Modifies the function to bypass the generation of a snippet
857 if a custom one was found. Parameter *snippet_folder* was added.
858 """
859 from ..ipythonhelper import read_nb
860 nb = read_nb(nbfile)
861 _, desc = nb.get_description()
863 if snippet_folder is not None and os.path.exists(snippet_folder):
864 custom_snippet = os.path.join(snippet_folder, os.path.splitext(
865 os.path.split(nbfile)[-1])[0] + '.png')
866 else:
867 custom_snippet = None
869 if custom_snippet is not None and os.path.exists(custom_snippet):
870 # reading a custom snippet
871 if fLOG:
872 fLOG("[build_thumbail_in_gallery] custom snippet '{0}'".format(
873 custom_snippet))
874 try:
875 from PIL import Image
876 except ImportError:
877 import Image
878 image = Image.open(custom_snippet)
879 else:
880 # generating an image
881 if fLOG:
882 fLOG(
883 "[build_thumbail_in_gallery] build snippet from '{0}'".format(nbfile))
884 image = nb.get_thumbnail()
886 if image is None:
887 image = nb.get_thumbnail(use_default=True)
889 if image is None:
890 raise ValueError(
891 "The snippet cannot be null, notebook='{0}'.".format(nbfile))
892 name = os.path.splitext(os.path.split(nbfile)[-1])[0]
893 name += ".thumb"
894 full = os.path.join(folder_snippet, name)
896 dirname = os.path.dirname(full)
897 if not os.path.exists(dirname):
898 raise FileNotFoundError( # pragma: no cover
899 "Unable to find folder '{0}'\nfolder_snippet='{1}'\nrelative='{2}'\nnbfile='{3}'".format(
900 dirname, folder_snippet, relative, nbfile))
902 if isinstance(image, str):
903 # SVG
904 full += ".svg"
905 name += ".svg"
906 with open(full, "w", encoding="utf-8") as f:
907 f.write(image)
908 else:
909 # Image
910 full += ".png"
911 name += ".png"
912 image.save(full)
914 rel = os.path.relpath(full, start=relative).replace("\\", "/")
915 nb_name = rel.replace(".thumb.png", ".html")
916 if layout == "classic":
917 rst = THUMBNAIL_TEMPLATE.format(
918 snippet=desc, thumbnail=rel, ref_name=rst_link)
919 elif layout == "table":
920 rst = THUMBNAIL_TEMPLATE_TABLE.format(
921 snippet=desc, thumbnail=rel, ref_name=rst_link, nb_name=nb_name)
922 else:
923 raise ValueError(
924 "layout must be 'classic' or 'table'") # pragma: no cover
925 return rst
928def add_tag_for_slideshow(ipy, folder, encoding="utf8"):
929 """
930 Modifies a notebook to add tag for a slideshow.
932 @param ipy notebook file
933 @param folder where to write the new notebook
934 @param encoding encoding
935 @return written file
936 """
937 from ..ipythonhelper import read_nb
938 filename = os.path.split(ipy)[-1]
939 output = os.path.join(folder, filename)
940 nb = read_nb(ipy, encoding=encoding, kernel=False)
941 nb.add_tag_slide()
942 nb.to_json(output)
943 return output
946def build_notebooks_gallery(nbs, fileout, layout="classic", neg_pattern=None,
947 snippet_folder=None, fLOG=noLOG):
948 """
949 Creates a :epkg:`rst` page (gallery) with links to all notebooks.
950 For each notebook, it creates a snippet.
952 @param nbs list of notebooks to consider or tuple(full path, rst),
953 @param fileout file to create
954 @param layout ``'classic'`` or ``'table'``
955 @param neg_pattern do not consider notebooks matching this regular expression
956 @param snippet_folder folder where to find custom snippet for notebooks,
957 the snippet should have the same name as the notebook
958 itself, snippet must have extension ``.png``
959 @param fLOG logging function
960 @return created file name
962 Example for parameter *nbs*:
964 ::
966 ('challenges\\city_tour\\city_tour_1.ipynb',
967 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1.ipynb')
968 ('challenges\\city_tour\\city_tour_1_solution.ipynb',
969 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1_solution.ipynb')
970 ('challenges\\city_tour\\city_tour_data_preparation.ipynb',
971 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_data_preparation.ipynb')
972 ('challenges\\city_tour\\city_tour_long.ipynb',
973 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_long.ipynb')
974 ('cheat_sheets\\chsh_files.ipynb',
975 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_files.ipynb')
976 ('cheat_sheets\\chsh_geo.ipynb',
977 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_geo.ipynb')
979 *nbs* can be a folder, in that case, the function will build
980 the list of all notebooks in that folder.
981 *nbs* can be a list of tuple.
982 the function adds a thumbnail, organizes the list of notebook
983 as a galley, it adds a link on notebook coverage.
984 The function bypasses the generation of a snippet
985 if a custom one was found.
986 """
987 from ..ipythonhelper import read_nb
988 if not isinstance(nbs, list):
989 fold = nbs
990 nbs = explore_folder(
991 fold, ".*[.]ipynb", neg_pattern=neg_pattern, fullname=True)[1]
992 if len(nbs) == 0:
993 raise FileNotFoundError( # pragma: no cover
994 "Unable to find notebooks in folder '{0}'.".format(nbs))
995 nbs = [(os.path.relpath(n, fold), n) for n in nbs]
997 # Go through the list of notebooks.
998 fLOG("[build_notebooks_gallery]", len(nbs), "notebooks")
999 hier = set()
1000 rst = []
1001 containers = {}
1002 for tu in nbs:
1003 if isinstance(tu, (tuple, list)):
1004 if tu[0] is None or ("/" not in tu[0] and "\\" not in tu[0]):
1005 rst.append((tuple(), tu[1]))
1006 else:
1007 way = tuple(tu[0].replace("\\", "/").split("/")[:-1])
1008 hier.add(way)
1009 rst.append((way, tu[1]))
1010 else:
1011 rst.append((tuple(), tu))
1012 name = rst[-1][1]
1013 ext = os.path.splitext(name)[-1]
1014 if ext != ".ipynb":
1015 raise ValueError( # pragma: no cover
1016 "One file is not a notebook: {0}".format(rst[-1][1]))
1017 dirname, na = os.path.split(name)
1018 if dirname not in containers:
1019 containers[dirname] = []
1020 containers[dirname].append(na)
1021 rst.sort()
1023 folder_index = os.path.dirname(os.path.normpath(fileout))
1024 folder = os.path.join(folder_index, "notebooks")
1025 if not os.path.exists(folder):
1026 os.mkdir(folder)
1028 # reordering based on titles
1029 titles = {}
1030 reord = []
1031 for hi, nbf in rst:
1032 nb = read_nb(nbf)
1033 title = nb.get_description()[0]
1034 titles[nbf] = title
1035 reord.append((hi, title, nbf))
1036 reord.sort()
1037 rst = [_[:1] + _[-1:] for _ in reord]
1039 # containers
1040 containers = list(sorted((k, v) for k, v in containers.items()))
1042 # find root
1043 hi, rs = rst[0]
1044 if len(hi) == 0:
1045 root = os.path.dirname(rs)
1046 else:
1047 spl = rs.replace("\\", "/").split("/")
1048 ro = spl[:-len(hi) - 1]
1049 root = "/".join(ro)
1051 # look for README.txt
1052 fLOG("[build_notebooks_gallery] root", root)
1053 rows = ["", ":orphan:", ""]
1054 exp = os.path.join(root, "README.txt")
1055 if os.path.exists(exp):
1056 fLOG("[build_notebooks_gallery] found", exp)
1057 with open(exp, "r", encoding="utf-8") as f:
1058 try:
1059 rows.extend(["", ".. _l-notebooks:", "", f.read(), ""])
1060 except UnicodeDecodeError as e: # pragma: no cover
1061 raise ValueError("Issue with file '{0}'".format(exp)) from e
1062 else:
1063 fLOG("[build_notebooks_gallery] not found", exp)
1064 rows.extend(["", ".. _l-notebooks:", "", "", "Notebooks Gallery",
1065 "=================", ""])
1067 rows.extend(["", ":ref:`l-notebooks-coverage`", "",
1068 "", ".. contents::", " :depth: 1",
1069 " :local:", ""])
1071 # produces the final files
1072 if len(hier) == 0:
1073 # case where there is no hierarchy
1074 fLOG("[build_notebooks_gallery] no hierarchy")
1075 rows.append(".. toctree::")
1076 rows.append(" :maxdepth: 1")
1077 if layout == "table":
1078 rows.append(" :hidden:")
1079 rows.append("")
1080 for hi, file in rst:
1081 rs = os.path.splitext(os.path.split(file)[-1])[0]
1082 fLOG("[build_notebooks_gallery] adding",
1083 rs, " title ", titles.get(file, None))
1084 rows.append(" notebooks/{0}".format(rs))
1085 if layout == "table" and len(rst) > 0:
1086 rows.extend(["", "", ".. list-table::",
1087 " :header-rows: 0", " :widths: 3 5 15", ""])
1089 for _, file in rst:
1090 link = os.path.splitext(os.path.split(file)[-1])[0]
1091 link = link.replace("_", "") + "rst"
1092 if not os.path.exists(file):
1093 raise FileNotFoundError( # pragma: no cover
1094 "Unable to find: '{0}'\nRST=\n{1}".format(
1095 file, "\n".join(str(_) for _ in rst)))
1096 r = build_thumbail_in_gallery(
1097 file, folder, folder_index, link, layout,
1098 snippet_folder=snippet_folder, fLOG=fLOG)
1099 rows.append(r)
1100 else:
1101 # case where there are subfolders
1102 fLOG("[build_notebooks_gallery] subfolders")
1103 already = "\n".join(rows)
1104 level = "-+^"
1105 rows.append("")
1106 if ".. contents::" not in already:
1107 rows.append(".. contents::")
1108 rows.append(" :local:")
1109 rows.append(" :depth: 2")
1110 rows.append("")
1111 stack_file = []
1112 last = None
1113 for hi, r in rst:
1114 rs0 = os.path.splitext(os.path.split(r)[-1])[0]
1115 r0 = r
1116 if hi != last:
1117 fLOG("[build_notebooks_gallery] new level", hi)
1118 # It adds the thumbnail.
1119 if layout == "table" and len(stack_file) > 0:
1120 rows.extend(
1121 ["", "", ".. list-table::", " :header-rows: 0", " :widths: 3 5 15", ""])
1123 for nbf in stack_file:
1124 fLOG("[build_notebooks_gallery] ", nbf)
1125 rs = os.path.splitext(os.path.split(nbf)[-1])[0]
1126 link = rs.replace("_", "") + "rst"
1127 r = build_thumbail_in_gallery(
1128 nbf, folder, folder_index, link, layout)
1129 rows.append(r)
1130 fLOG("[build_notebooks_gallery] saw {0} files".format(
1131 len(stack_file)))
1132 stack_file = []
1134 # It switches to the next gallery.
1135 if layout == "classic":
1136 rows.append(".. raw:: html")
1137 rows.append("")
1138 rows.append(" <div style='clear:both'></div>")
1139 rows.append("")
1141 # It adds menus and subfolders.
1142 lastk = 0
1143 for k in range(0, len(hi)):
1144 lastk = k
1145 if last is None or k >= len(last) or hi[k] != last[k]: # pylint: disable=E1136
1146 break
1148 while len(hi) > 0 and lastk < len(hi):
1149 fo = [root] + list(hi[:lastk + 1])
1150 readme = os.path.join(*(fo + ["README.txt"]))
1151 if os.path.exists(readme):
1152 fLOG("[build_notebooks_gallery] found", readme)
1153 with open(readme, "r", encoding="utf-8") as f:
1154 try:
1155 rows.extend(["", f.read(), ""])
1156 except UnicodeDecodeError as e: # pragma: no cover
1157 raise ValueError(
1158 "Issue with file '{0}'".format(readme)) from e
1159 else:
1160 fLOG("[build_notebooks_gallery] not found", readme)
1161 rows.append("")
1162 rows.append(hi[lastk])
1163 rows.append(
1164 level[min(lastk, len(level) - 1)] * len(hi[lastk]))
1165 rows.append("")
1166 lastk += 1
1168 # It starts the next gallery.
1169 last = hi
1170 rows.append(".. toctree::")
1171 rows.append(" :maxdepth: 1")
1172 if layout == "table":
1173 rows.append(" :hidden:")
1174 rows.append("")
1176 # append a link to a notebook
1177 fLOG("[build_notebooks_gallery] adding",
1178 rs0, " title ", titles.get(r0, None))
1179 rows.append(" notebooks/{0}".format(rs0))
1180 stack_file.append(r0)
1182 if len(stack_file) > 0:
1183 # It adds the thumbnails.
1184 if layout == "table" and len(stack_file) > 0:
1185 rows.extend(["", "", ".. list-table::",
1186 " :header-rows: 0", " :widths: 3 5 15", ""])
1188 for nbf in stack_file:
1189 rs = os.path.splitext(os.path.split(nbf)[-1])[0]
1190 link = rs.replace("_", "") + "rst"
1191 r = build_thumbail_in_gallery(
1192 nbf, folder, folder_index, link, layout)
1193 rows.append(r)
1195 # done
1196 rows.append("")
1198 # links to coverage
1199 rows.extend(["", "", ".. toctree::", " :hidden: ", "",
1200 " all_notebooks_coverage", ""])
1202 with open(fileout, "w", encoding="utf8") as f:
1203 f.write("\n".join(rows))
1204 return fileout
1207def build_all_notebooks_coverage(nbs, fileout, module_name, dump=None, badge=True, too_old=30, fLOG=noLOG):
1208 """
1209 Creates a :epkg:`rst` page (gallery) with links to all notebooks and
1210 information about coverage.
1211 It relies on function @see fn notebook_coverage.
1213 @param nbs list of notebooks to consider or tuple(full path, rst),
1214 @param fileout file to create
1215 @param module_name module name
1216 @param dump dump containing information about notebook execution (or None for the default one)
1217 @param badge builds an image with the notebook coverage
1218 @param too_old drop executions older than *too_old* days from now
1219 @param fLOG logging function
1220 @return dataframe which contains the data
1221 """
1222 from ..ipythonhelper import read_nb, notebook_coverage
1223 if dump is None:
1224 dump = os.path.normpath(os.path.join(os.path.dirname(fileout), "..", "..", "..", "..",
1225 "_notebook_dumps", "notebook.{0}.txt".format(module_name)))
1226 if not os.path.exists(dump):
1227 fLOG("[notebooks-coverage] No execution report about "
1228 "notebook at '{0}' (fileout='{1}')".format(dump,
1229 os.path.dirname(fileout)))
1230 return None
1231 report0 = notebook_coverage(nbs, dump, too_old=too_old)
1232 fLOG("[notebooks-coverage] report shape", report0.shape)
1234 from numpy import isnan
1236 # Fill nan values.
1237 for i in report0.index:
1238 nbcell = report0.loc[i, "nbcell"]
1239 if isnan(nbcell):
1240 # It loads the notebook.
1241 nbfile = report0.loc[i, "notebooks"]
1242 nb = read_nb(nbfile)
1243 report0.loc[i, "nbcell"] = len(nb)
1244 report0.loc[i, "nbrun"] = 0
1246 # Add links.
1247 cols = ['notebooks', 'date', 'etime',
1248 'nbcell', 'nbrun', 'nbvalid', 'success', 'time']
1249 report = report0[cols].copy()
1250 report["notebooks"] = report["notebooks"].apply(
1251 lambda x: "/".join(os.path.normpath(x).replace("\\", "/").split("/")[-2:]) if isinstance(x, str) else x)
1252 report["last_name"] = report["notebooks"].apply(
1253 lambda x: os.path.split(x)[-1] if isinstance(x, str) else x)
1255 report1 = report.copy()
1257 def clean_link(link):
1258 return link.replace("_", "").replace(".ipynb", ".rst").replace(".", "") if isinstance(link, str) else link
1260 report["notebooks"] = report.apply(lambda row: ':ref:`{0} <{1}>`'.format(
1261 row["notebooks"], clean_link(row["last_name"])), axis=1)
1262 report["title"] = report["last_name"].apply(
1263 lambda x: ':ref:`{0}`'.format(clean_link(x)))
1264 rows = ["", ".. _l-notebooks-coverage:", "", "", "Notebooks Coverage",
1265 "==================", "", "Report on last executions.", ""]
1267 # Badge
1268 if badge:
1269 from ..ipythonhelper import badge_notebook_coverage
1270 img = os.path.join(os.path.dirname(fileout), "nbcov.png")
1271 cov = badge_notebook_coverage(report0, img)
1272 now = datetime.datetime.now()
1273 sdate = "%04d-%02d-%02d" % (now.year, now.month, now.day)
1274 cpy = os.path.join(os.path.dirname(fileout), "nbcov-%s.png" % sdate)
1275 shutil.copy(img, cpy)
1276 badge = ["{0:0.00f}% {1}".format(
1277 cov, sdate), "", ".. image:: {0}".format(os.path.split(cpy)[-1]), ""]
1278 badge2 = ["", ".. image:: {0}".format(os.path.split(img)[-1]), ""]
1279 else:
1280 badge = []
1281 badge2 = []
1282 rows.extend(badge)
1284 # Formatting
1285 report["date"] = report["date"].apply(
1286 lambda x: x.split()[0] if isinstance(x, str) else x)
1287 report["etime"] = report["etime"].apply(
1288 lambda x: "%1.3f" % x if isinstance(x, float) else x)
1289 report["time"] = report["time"].apply(
1290 lambda x: "%1.3f" % x if isinstance(x, float) else x)
1292 def int2str(x):
1293 if isnan(x):
1294 return ""
1295 else:
1296 return int(x)
1298 report["coverage"] = report["nbrun"] / report["nbcell"]
1299 report["nbcell"] = report["nbcell"].apply(int2str)
1300 report["nbrun"] = report["nbrun"].apply(int2str)
1301 report["nbvalid"] = report["nbvalid"].apply(int2str)
1302 report["coverage"] = report["coverage"].apply(
1303 lambda x: "{0}%".format(int(x * 100)) if isinstance(x, float) else "")
1304 report = report[['notebooks', 'title', 'date', 'success', 'etime',
1305 'nbcell', 'nbrun', 'nbvalid', 'time', 'coverage']].copy()
1306 report.columns = ['name', 'title', 'last execution', 'success', 'time',
1307 'nb cells', 'nb runs', 'nb valid', 'exe time', 'coverage']
1308 report = report[['coverage', 'exe time', 'last execution', 'name', 'title',
1309 'success', 'time', 'nb cells', 'nb runs', 'nb valid']]
1311 # Add results.
1312 text = df2rst(report.sort_values("name").reset_index(
1313 drop=True), index=True, list_table=True)
1314 rows.append(text)
1315 rows.extend(badge2)
1317 fLOG("[notebooks-coverage] writing", fileout)
1318 with open(fileout, "w", encoding="utf-8") as f:
1319 f.write("\n".join(rows))
1320 return report1