Coverage for pyquickhelper/helpgen/process_notebooks.py: 91%

577 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-03 02:21 +0200

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Contains the main function to generate the documentation 

5for a module designed the same way as this one, @see fn generate_help_sphinx. 

6 

7""" 

8import datetime 

9import json 

10import os 

11import sys 

12import shutil 

13import warnings 

14from io import StringIO 

15from nbconvert.exporters.base import ExporterNameError 

16 

17from .utils_sphinx_doc_helpers import HelpGenException 

18from .conf_path_tools import find_latex_path, find_pandoc_path 

19from .post_process import ( 

20 post_process_latex_output, post_process_latex_output_any, 

21 post_process_rst_output, post_process_html_output, 

22 post_process_slides_output, post_process_python_output) 

23from .helpgen_exceptions import NotebookConvertError 

24from .install_js_dep import install_javascript_tools 

25from .style_css_template import THUMBNAIL_TEMPLATE, THUMBNAIL_TEMPLATE_TABLE 

26from .process_notebook_api import nb2rst 

27from ..loghelper.flog import run_cmd, fLOG, noLOG 

28from ..pandashelper import df2rst 

29from ..filehelper.synchelper import has_been_updated, explore_folder 

30 

31 

32template_examples = """ 

33 

34List of programs 

35++++++++++++++++ 

36 

37.. toctree:: 

38 :maxdepth: 2 

39 

40.. autosummary:: __init__.py 

41 :toctree: %s/ 

42 :template: modules.rst 

43 

44Another list 

45++++++++++++ 

46 

47""" 

48 

49 

50def find_pdflatex(latex_path): 

51 """ 

52 Returns the executable for latex. 

53 

54 @param latex_path path to look (only on Windows) 

55 @return executable 

56 """ 

57 if sys.platform.startswith("win"): # pragma: no cover 

58 lat = os.path.join(latex_path, "xelatex.exe") 

59 if os.path.exists(lat): 

60 return lat 

61 lat = os.path.join(latex_path, "pdflatex.exe") 

62 if os.path.exists(lat): 

63 return lat 

64 raise FileNotFoundError( 

65 f"Unable to find pdflatex or xelatex in '{latex_path}'") 

66 if sys.platform.startswith("darwin"): # pragma: no cover 

67 try: 

68 err = run_cmd("/Library/TeX/texbin/xelatex --help", wait=True)[1] 

69 if len(err) == 0: 

70 return "/Library/TeX/texbin/xelatex" 

71 raise FileNotFoundError( # pragma: no cover 

72 f"Unable to run xelatex\n{err}") 

73 except Exception: 

74 return "/Library/TeX/texbin/pdflatex" 

75 try: 

76 err = run_cmd("xelatex --help", wait=True)[1] 

77 if len(err) == 0: 

78 return "xelatex" 

79 else: 

80 raise FileNotFoundError( 

81 f"Unable to run xelatex\n{err}") 

82 except Exception: # pragma: no cover 

83 return "pdflatex" 

84 

85 

86def process_notebooks(notebooks, outfold, build, latex_path=None, pandoc_path=None, 

87 formats="ipynb,html,python,rst,slides,pdf,github", 

88 fLOG=fLOG, exc=True, remove_unicode_latex=False, nblinks=None, 

89 notebook_replacements=None): 

90 """ 

91 Converts notebooks into :epkg:`html`, :epkg:`rst`, :epkg:`latex`, 

92 :epkg:`pdf`, :epkg:`python`, :epkg:`docx` using 

93 :epkg:`nbconvert`. 

94 

95 @param notebooks list of notebooks or comma separated values 

96 @param outfold folder which will contains the outputs 

97 @param build temporary folder which contains all produced files 

98 @param pandoc_path path to pandoc 

99 @param formats list of formats to convert into (pdf format means latex 

100 then compilation), or comma separated values 

101 @param latex_path path to the latex compiler 

102 @param fLOG logging function 

103 @param exc raises an exception (True) or a warning (False) if an error happens 

104 @param nblinks dictionary ``{ref: url}`` or a string in :epkg:`json` 

105 format 

106 @param remove_unicode_latex remove unicode characters for latex (to avoid failing) 

107 @param notebook_replacements string replacement in a notebook before conversion 

108 or a string in :epkg:`json` format 

109 @return list of tuple *[(file, created or skipped)]* 

110 

111 This function relies on :epkg:`pandoc`. 

112 It also needs modules :epkg:`pywin32`, 

113 :epkg:`pygments`. 

114 

115 :epkg:`pywin32` might have some issues 

116 to find its DLL, look @see fn import_pywin32. 

117 

118 The latex compilation uses :epkg:`MiKTeX`. 

119 The conversion into Word document directly uses pandoc. 

120 It still has an issue with table. 

121 

122 Some latex templates (for nbconvert) uses ``[commandchars=\\\\\\{\\}]{\\|}`` which allows commands ``\\\\`` and it does not compile. 

123 The one used here is ``report``. 

124 Some others bugs can be found at: `schlichtanders/latex_test.html <https://gist.github.com/schlichtanders/e108ed0be80108178af2>`_. 

125 For example, you must not let spaces between symbol ``$`` and the 

126 formulas it indicates. 

127 

128 If *pandoc_path* is None, uses @see fn find_pandoc_path to guess it. 

129 If *latex_path* is None, uses @see fn find_latex_path to guess it. 

130 

131 .. exref:: 

132 :title: Convert a notebook into multiple formats 

133 

134 :: 

135 

136 from pyquickhelper.ipythonhelper import process_notebooks 

137 process_notebooks("td1a_correction_session7.ipynb", 

138 "dest_folder", "dest_folder", 

139 formats=("ipynb", "html", "python", "rst", "slides", "pdf", 

140 "docx", "github")]) 

141 

142 For latex and pdf, a custom processor was added to handle raw data 

143 and add ``\\begin{verbatim}`` and ``\\end{verbatim}``. 

144 Format *github* adds a link to file on :epkg:`github`. 

145 

146 .. todoext:: 

147 :title: check differences between _process_notebooks_in_private and _process_notebooks_in_private_cmd 

148 :tag: bug 

149 

150 For :epkg:`latex` and :epkg:`pdf`, 

151 the custom preprocessor is not taken into account. 

152 by function @see fn _process_notebooks_in_private. 

153 """ 

154 if isinstance(notebooks, str): 

155 notebooks = notebooks.split(',') 

156 if isinstance(formats, str): 

157 formats = formats.split(',') 

158 if isinstance(notebook_replacements, str): 

159 notebook_replacements = json.loads(notebook_replacements) 

160 if isinstance(nblinks, str): 

161 nblinks = json.loads(nblinks) 

162 if build is None: 

163 raise ValueError( # pragma: no cover 

164 "build cannot be None") 

165 

166 res = _process_notebooks_in(notebooks=notebooks, outfold=outfold, build=build, 

167 latex_path=latex_path, pandoc_path=pandoc_path, 

168 formats=formats, fLOG=fLOG, exc=exc, nblinks=nblinks, 

169 remove_unicode_latex=remove_unicode_latex, 

170 notebook_replacements=notebook_replacements) 

171 if "slides" in formats: 

172 # we copy javascript dependencies, reveal.js 

173 reveal = os.path.join(outfold, "reveal.js") 

174 if not os.path.exists(reveal): 

175 install_javascript_tools(None, dest=outfold) 

176 reveal = os.path.join(build, "reveal.js") 

177 if not os.path.exists(reveal): 

178 install_javascript_tools(None, dest=build) 

179 return res 

180 

181 

182def _process_notebooks_in_private(fnbcexe, list_args, options_args): 

183 """ 

184 This function fails in nbconvert 6.0 when the conversion 

185 is called more than once. The conversion probably changes the 

186 initial state. 

187 """ 

188 out = StringIO() 

189 err = StringIO() 

190 memo_out = sys.stdout 

191 memo_err = sys.stderr 

192 sys.stdout = out 

193 sys.stderr = err 

194 try: 

195 if list_args: 

196 fnbcexe(argv=list_args, **options_args) 

197 else: 

198 fnbcexe(**options_args) 

199 exc = None 

200 except SystemExit as e: # pragma: no cover 

201 exc = e 

202 except IndentationError as e: # pragma: no cover 

203 # This is change in IPython 6.0.0. 

204 # The conversion fails on IndentationError. 

205 # We switch to another one. 

206 from ..ipythonhelper import read_nb 

207 i = list_args.index("--template") 

208 format = list_args[i + 1] 

209 if format == "python": 

210 i = list_args.index("--output") 

211 dest = list_args[i + 1] 

212 if not dest.endswith(".py"): 

213 dest += ".py" 

214 src = list_args[-1] 

215 nb = read_nb(src) 

216 code = nb.to_python() 

217 with open(dest, "w", encoding="utf-8") as f: 

218 f.write(code) 

219 exc = None 

220 else: 

221 # We do nothing in this case. 

222 exc = e 

223 except (AttributeError, FileNotFoundError, ValueError) as e: 

224 exc = e 

225 except ExporterNameError as e: # pragma: no cover 

226 exc = e 

227 sys.stdout = memo_out 

228 sys.stderr = memo_err 

229 out = out.getvalue() 

230 err = err.getvalue() 

231 if exc: 

232 if "Unsupported mimetype 'text/html'" in str(exc): 

233 from nbconvert.nbconvertapp import main 

234 main(argv=list_args, **options_args) 

235 return "", "" 

236 env = "\n".join(f"{k}={v}" 

237 for k, v in sorted(os.environ.items())) 

238 raise RuntimeError( # pragma: no cover 

239 "Notebook conversion failed.\nfnbcexe\n{}\noptions_args\n{}" 

240 "\n--ARGS--\n{}\n--OUT--\n{}\n--ERR--\n{}\n--ENVIRON--\n{}" 

241 "".format(fnbcexe, options_args, list_args, out, err, 

242 env)) from exc 

243 return out, err 

244 

245 

246def _process_notebooks_in_private_cmd(fnbcexe, list_args, options_args, fLOG): 

247 this = os.path.join(os.path.dirname( 

248 os.path.abspath(__file__)), "process_notebooks_cmd.py") 

249 res = [] 

250 for c in list_args: 

251 if c[0] == '"' or c[-1] == '"' or ' ' not in c: 

252 res.append(c) 

253 else: 

254 res.append(f'"{c}"') 

255 sargs = " ".join(res) 

256 cmd = f"\"{sys.executable.replace('w.exe', '.exe')}\" \"{this}\" {sargs}" 

257 fLOG("[_process_notebooks_in_private_cmd]", cmd) 

258 return run_cmd(cmd, wait=True, fLOG=fLOG) 

259 

260 

261def _preprocess_notebook(notebook_content): 

262 """ 

263 Preprocesses the content of a notebook. 

264 

265 @param notebook_content notebook content 

266 @return modified content 

267 """ 

268 def walk_through(field): 

269 if isinstance(field, list): 

270 for f in field: 

271 walk_through(f) 

272 elif isinstance(field, dict): 

273 if (field.get('version_major', -1) == 2 and 

274 field.get('version_minor', -1) == 0): 

275 field['version_minor'] = 2 

276 elif (field.get('nbformat', -1) == 4 and 

277 field.get('nbformat_minor', -1) in (0, 1)): 

278 field['nbformat_minor'] = 2 

279 for _, v in field.items(): 

280 walk_through(v) 

281 

282 content = json.loads(notebook_content) 

283 walk_through(content) 

284 new_content = json.dumps(content) 

285 return new_content 

286 

287 

288def _process_notebooks_in(notebooks, outfold, build, latex_path=None, pandoc_path=None, 

289 formats=("ipynb", "html", "python", "rst", 

290 "slides", "pdf", "github"), 

291 fLOG=fLOG, exc=True, nblinks=None, remove_unicode_latex=False, 

292 notebook_replacements=None): 

293 """ 

294 The notebook conversion does not handle images from url 

295 for :epkg:`pdf` and :epkg:`docx`. They could be downloaded first 

296 and replaced by local files. 

297 

298 .. note:: 

299 

300 :epkg:`nbconvert` introduced a commit which breaks 

301 the conversion of notebooks in latex if they have 

302 a cell outputting *svg* 

303 (see `PR 910 <https://github.com/jupyter/nbconvert/pull/910>`_). 

304 

305 Use `xelatex <https://doc.ubuntu-fr.org/xelatex>`_ if possible. 

306 """ 

307 from nbconvert.nbconvertapp import main as nbconvert_main 

308 if pandoc_path is None: 

309 pandoc_path = find_pandoc_path() 

310 

311 if latex_path is None: 

312 latex_path = find_latex_path() 

313 

314 if isinstance(notebooks, str): 

315 notebooks = [notebooks] 

316 

317 if "PANDOCPY" in os.environ and sys.platform.startswith("win"): # pragma: no cover 

318 exe = os.environ["PANDOCPY"] 

319 exe = exe.rstrip("\\/") 

320 if exe.endswith("\\Scripts"): 

321 exe = exe[:len(exe) - len("Scripts") - 1] 

322 if not os.path.exists(exe): 

323 raise FileNotFoundError(exe) 

324 fLOG("[_process_notebooks_in] ** using PANDOCPY", exe) 

325 else: 

326 if sys.platform.startswith("win"): # pragma: no cover 

327 from .utils_pywin32 import import_pywin32 

328 try: 

329 import_pywin32() 

330 except ModuleNotFoundError as e: 

331 warnings.warn(e) 

332 exe = os.path.split(sys.executable)[0] 

333 

334 extensions = {"ipynb": ".ipynb", "latex": ".tex", "elatex": ".tex", "pdf": ".pdf", 

335 "html": ".html", "rst": ".rst", "python": ".py", "docx": ".docx", 

336 "word": ".docx", "slides": ".slides.html"} 

337 

338 files = [] 

339 skipped = [] 

340 

341 # main(argv=None, **kwargs) 

342 fnbc = nbconvert_main 

343 

344 if "slides" in formats: 

345 build_slide = os.path.join(build, "bslides") 

346 if not os.path.exists(build_slide): 

347 os.mkdir(build_slide) 

348 

349 copied_images = dict() 

350 

351 for notebook_in in notebooks: 

352 thisfiles = [] 

353 

354 # we copy available images (only notebook folder) 

355 # in case they are used in latex 

356 currentdir = os.path.abspath(os.path.dirname(notebook_in)) 

357 for curfile in os.listdir(currentdir): 

358 ext = os.path.splitext(curfile)[1] 

359 if ext in {'.png', '.jpg', '.bmp', '.gif', '.jpeg', '.svg', '.mp4'}: 

360 src = os.path.join(currentdir, curfile) 

361 if src not in copied_images: 

362 dest = os.path.join(build, curfile) 

363 shutil.copy(src, build) 

364 fLOG(f"[_process_notebooks_in] copy '{src}' to '{build}'.") 

365 copied_images[src] = dest 

366 

367 # copy of the notebook into the build folder 

368 # and changes the source 

369 _name = os.path.splitext(os.path.split(notebook_in)[-1])[0] 

370 _name += '.ipynb' 

371 notebook = os.path.join(build, _name) 

372 fLOG("[_process_notebooks_in] -- copy notebook '{}' to '{}'.".format( 

373 notebook_in, notebook)) 

374 with open(notebook_in, "r", encoding="utf-8") as _f: 

375 content = _f.read() 

376 content = _preprocess_notebook(content) 

377 with open(notebook, "w", encoding="utf-8") as _f: 

378 _f.write(content) 

379 

380 # next 

381 nbout = os.path.split(notebook)[-1] 

382 if " " in nbout: 

383 raise HelpGenException( # pragma: no cover 

384 f"spaces are not allowed in notebooks file names: {notebook}") 

385 nbout = os.path.splitext(nbout)[0] 

386 for format in formats: 

387 

388 if format == "github": 

389 # we add a link on the rst page in that case 

390 continue 

391 

392 if format not in extensions: 

393 raise NotebookConvertError( # pragma: no cover 

394 "Unable to find format: '{}' in {}".format( 

395 format, ", ".join(extensions.keys()))) 

396 

397 # output 

398 format_ = format 

399 outputfile_noext = os.path.join(build, nbout) 

400 if format == 'html': 

401 outputfile = outputfile_noext + '2html' + extensions[format] 

402 outputfile_noext_fixed = outputfile_noext + '2html' 

403 else: 

404 outputfile = outputfile_noext + extensions[format] 

405 outputfile_noext_fixed = outputfile_noext 

406 trueoutputfile = outputfile 

407 pandoco = "docx" if format in ("word", "docx") else None 

408 

409 # The function checks it was not done before. 

410 if os.path.exists(trueoutputfile): 

411 dto = os.stat(trueoutputfile).st_mtime 

412 dtnb = os.stat(notebook).st_mtime 

413 if dtnb < dto: # pragma: no cover 

414 fLOG("[_process_notebooks_in] -- skipping notebook", format, 

415 notebook, "(", trueoutputfile, ")") 

416 if trueoutputfile not in thisfiles: 

417 thisfiles.append(trueoutputfile) 

418 if pandoco is None: 

419 skipped.append(trueoutputfile) 

420 continue 

421 out2 = os.path.splitext( 

422 trueoutputfile)[0] + "." + pandoco 

423 if os.path.exists(out2): 

424 skipped.append(trueoutputfile) 

425 continue 

426 

427 # if the format is slides, we update the metadata 

428 options_args = {} 

429 if format == "slides": 

430 nb_slide = add_tag_for_slideshow(notebook, build_slide) 

431 fnbcexe = fnbc 

432 else: 

433 nb_slide = None 

434 fnbcexe = fnbc 

435 

436 # compilation 

437 list_args = [] 

438 custom_config = os.path.join(os.path.abspath( 

439 os.path.dirname(__file__)), "_nbconvert_config.py") 

440 if format == "pdf": 

441 if not os.path.exists(custom_config): 

442 raise FileNotFoundError( # pragma: no cover 

443 custom_config) 

444 # title = os.path.splitext( 

445 # os.path.split(notebook)[-1])[0].replace("_", " ") 

446 list_args.extend(['--config', f'"{custom_config}"']) 

447 format = "latex" 

448 compilation = True 

449 thisfiles.append(os.path.splitext(outputfile)[0] + ".tex") 

450 elif format in ("latex", "elatex"): 

451 if not os.path.exists(custom_config): 

452 raise FileNotFoundError( # pragma: no cover 

453 custom_config) 

454 list_args.extend(['--config', f'"{custom_config}"']) 

455 compilation = False 

456 format = "latex" 

457 elif format in ("word", "docx"): 

458 format = "html" 

459 compilation = False 

460 elif format in ("slides", ): 

461 list_args.extend(["--reveal-prefix", "reveal.js"]) 

462 compilation = False 

463 else: 

464 compilation = False 

465 

466 # output 

467 # set templates to None to avoid error 

468 # No template sub-directory with name 'article' found in the following paths: 

469 templ = {'html': None, 'latex': None, 

470 'elatex': None}.get(format, format) 

471 fLOG("[_process_notebooks_in] ### convert into '{}' (done: {}): '{}' -> '{}'".format( 

472 format_, os.path.exists(outputfile), notebook, outputfile)) 

473 

474 list_args.extend(["--output", outputfile_noext_fixed]) 

475 if templ is not None and format != "slides": 

476 list_args.extend(["--template", templ]) 

477 

478 # execution 

479 if format not in ("ipynb", ): 

480 # nbconvert is messing up with static variables in sphinx or 

481 # docutils if format is slides, not sure about the others 

482 if format in ('rst', ): 

483 fLOG("[_process_notebooks_in] NBcn:", format, options_args) 

484 nb2rst(notebook, outputfile, post_process=False) 

485 err = "" 

486 c = "" 

487 elif nbconvert_main != fnbcexe or format not in ( 

488 "slides", "elatex", "latex", "pdf", "html"): 

489 if options_args: 

490 fLOG("[_process_notebooks_in] NBp*:", 

491 format, options_args) 

492 else: 

493 list_args.extend(["--to", format, 

494 notebook if nb_slide is None else nb_slide]) 

495 fLOG( 

496 f"[_process_notebooks_in] NBc* format='{format}' args={list_args}") 

497 fLOG(f"[_process_notebooks_in] cwd='{os.getcwd()}'") 

498 

499 c = " ".join(list_args) 

500 out, err = _process_notebooks_in_private( 

501 fnbcexe, list_args, options_args) 

502 else: 

503 # conversion into slides alter Jinja2 environment 

504 # jinja2.exceptions.TemplateNotFound: rst 

505 if options_args: 

506 fLOG("[_process_notebooks_in] NBp+:", 

507 format, options_args) 

508 else: 

509 list_args.extend(["--to", format, 

510 notebook if nb_slide is None else nb_slide]) 

511 fLOG("[_process_notebooks_in] NBc+:", format, list_args) 

512 fLOG("[_process_notebooks_in]", os.getcwd()) 

513 

514 c = " ".join(list_args) 

515 out, err = _process_notebooks_in_private_cmd( 

516 fnbcexe, list_args, options_args, fLOG) 

517 

518 if "raise ImportError" in err or "Unknown exporter" in err: 

519 raise ImportError( 

520 f"cmd: {fnbcexe} {list_args}\n--ERR--\n{err}") 

521 if len(err) > 0: 

522 if format in ("elatex", "latex"): 

523 # There might be some errors because the latex script needs to be post-processed 

524 # sometimes (wrong characters such as " or formulas not 

525 # captured as formulas). 

526 if err and "usage: process_notebooks_cmd.py" in err: 

527 raise RuntimeError( # pragma: no cover 

528 "Unable to convert a notebook\n----\n{}----\n{}\n" 

529 "---ERR---\n{}\n---OUT---\n{}".format( 

530 fnbcexe, list_args, err, out)) 

531 fLOG("[_process_notebooks_in] LATEX --ERR--\n" + err) 

532 fLOG("[_process_notebooks_in] LATEX --OUT--\n" + out) 

533 else: 

534 err = err.lower() 

535 if "critical" in err or "bad config" in err: 

536 raise HelpGenException( # pragma: no cover 

537 f"CMD:\n{list_args}\n[nberror]\n{err}") 

538 else: 

539 # format ipynb 

540 # we do nothing 

541 pass 

542 

543 format = extensions[format].strip(".") 

544 

545 # we add the file to the list of generated files 

546 if outputfile not in thisfiles: 

547 thisfiles.append(outputfile) 

548 

549 fLOG("[_process_notebooks_in] -", 

550 format, compilation, outputfile) 

551 

552 if compilation: 

553 # compilation latex 

554 if not sys.platform.startswith("win") or os.path.exists(latex_path): 

555 lat = find_pdflatex(latex_path) 

556 

557 tex = set(_ for _ in thisfiles if os.path.splitext( 

558 _)[-1] == ".tex") 

559 if len(tex) != 1: 

560 raise FileNotFoundError( # pragma: no cover 

561 "No latex file was generated or more than one (={0}), nb={1}\nthisfile=\n{2}".format( 

562 len(tex), notebook, "\n".join(thisfiles))) 

563 tex = list(tex)[0] 

564 try: 

565 post_process_latex_output_any( 

566 tex, custom_latex_processing=None, nblinks=nblinks, 

567 remove_unicode=remove_unicode_latex, fLOG=fLOG) 

568 except FileNotFoundError as e: # pragma: no cover 

569 mes = ("[_process_notebooks_in-ERROR] Unable to convert into latex" 

570 "notebook %r due to %r.") % (tex, e) 

571 warnings.warn(mes, RuntimeWarning) 

572 fLOG(mes) 

573 continue 

574 

575 # -interaction=batchmode 

576 c = '"{0}" "{1}" -max-print-line=900 -output-directory="{2}"'.format( 

577 lat, tex, os.path.split(tex)[0]) 

578 fLOG("[_process_notebooks_in] ** LATEX compilation (b)", c) 

579 if not sys.platform.startswith("win"): 

580 c = c.replace('"', '') 

581 if sys.platform.startswith("win"): 

582 change_path = None 

583 else: 

584 # On Linux the parameter --output-directory is sometimes ignored. 

585 # And it only works from the current directory. 

586 change_path = os.path.split(tex)[0] 

587 out, err = run_cmd( 

588 c, wait=True, log_error=False, shell=sys.platform.startswith("win"), 

589 catch_exit=True, prefix_log="[latex] ", change_path=change_path) 

590 if out is not None and ("Output written" in out or 'bytes written' in out): 

591 # The output was produced. We ignore the return code. 

592 fLOG("[_process_notebooks_in] WARNINGS: " 

593 "Latex compilation had warnings:", c) 

594 out += "\n--ERR--\n" + err 

595 err = "" 

596 if len(err) > 0: 

597 raise HelpGenException( # pragma: no cover 

598 f"CMD:\n{c}\n[nberror]\n{err}\nOUT:\n{out}------") 

599 f = os.path.join(build, nbout + ".pdf") 

600 if not os.path.exists(f): # pragma: no cover 

601 # On Linux the parameter --output-directory is sometimes ignored. 

602 # And it only works from the current directory. 

603 # We check again. 

604 loc = os.path.split(f)[-1] 

605 if os.path.exists(loc): 

606 # We move the file. 

607 moved = True 

608 shutil.move(loc, f) 

609 else: 

610 moved = False 

611 if not os.path.exists(f): 

612 files = "\n".join(os.listdir(build)) 

613 msg = "Content of '{0}':\n{1}\n----\n'{2}' moved? {3}\nCMD:\n{4}".format( 

614 build, files, loc, moved, c) 

615 raise HelpGenException( 

616 f"Missing file: '{f}'\nCMD\n{c}nOUT:\n{out}\n[nberror]\n{err}\n-----\n{msg}") 

617 thisfiles.append(f) 

618 else: 

619 fLOG("[_process_notebooks_in] unable to find latex in", latex_path) 

620 

621 elif pandoco is not None: # pragma: no cover 

622 # compilation pandoc 

623 fLOG("[_process_notebooks_in] ** pandoc compilation (b)", pandoco) 

624 inputfile = os.path.splitext(outputfile)[0] + ".html" 

625 outfilep = os.path.splitext(outputfile)[0] + "." + pandoco 

626 

627 # for some files, the following error might appear: 

628 # Stack space overflow: current size 33692 bytes. 

629 # Use `+RTS -Ksize -RTS' to increase it. 

630 # it usually means there is something wrong (circular 

631 # reference, ...) 

632 if sys.platform.startswith("win"): 

633 c = '"{0}\\pandoc.exe" +RTS -K32m -RTS -f html -t {1} "{2}" -o "{3}"'.format( 

634 pandoc_path, pandoco, inputfile, outfilep) 

635 else: 

636 c = 'pandoc +RTS -K32m -RTS -f html -t {0} "{1}" -o "{2}"'.format( 

637 pandoco, outputfile, outfilep) 

638 

639 if not sys.platform.startswith("win"): 

640 c = c.replace('"', '') 

641 out, err = run_cmd( 

642 c, wait=True, log_error=False, shell=sys.platform.startswith("win")) 

643 if len(err) > 0: 

644 lines = err.strip("\r\n").split("\n") 

645 # we filter out the message 

646 # pandoc.exe: Could not find image `https:// 

647 left = [ 

648 _ for _ in lines if _ and "Could not find image `http" not in _] 

649 if len(left) > 0: 

650 raise HelpGenException( 

651 f"issue with cmd: {c}\n[nberror]\n{err}") 

652 for _ in lines: 

653 fLOG("[_process_notebooks_in] w, pandoc issue: {0}".format( 

654 _.strip("\n\r"))) 

655 outputfile = outfilep 

656 format = "docx" 

657 

658 nb_replacements = notebook_replacements.get( 

659 format, None) if notebook_replacements else None 

660 

661 if format == "html": 

662 # we add a link to the notebook 

663 if not os.path.exists(outputfile): 

664 raise FileNotFoundError( # pragma: no cover 

665 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join( 

666 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + c) 

667 thisfiles += add_link_to_notebook(outputfile, notebook, "pdf" in formats, False, 

668 "python" in formats, "slides" in formats, 

669 exc=exc, nblinks=nblinks, fLOG=fLOG, 

670 notebook_replacements=nb_replacements) 

671 

672 elif format == "slides.html": 

673 # we add a link to the notebook 

674 if not os.path.exists(outputfile): 

675 raise FileNotFoundError( # pragma: no cover 

676 outputfile + "\nCONTENT in " + os.path.dirname(outputfile) + ":\n" + "\n".join( 

677 os.listdir(os.path.dirname(outputfile))) + "\n[nberror]\n" + err + "\nOUT:\n" + out + "\nCMD:\n" + str(list_args)) 

678 thisfiles += add_link_to_notebook(outputfile, notebook, 

679 "pdf" in formats, False, "python" in formats, 

680 "slides" in formats, exc=exc, 

681 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements) 

682 

683 elif format == "ipynb": 

684 # we just copy the notebook 

685 thisfiles += add_link_to_notebook(outputfile, notebook, 

686 "ipynb" in formats, False, "python" in formats, 

687 "slides" in formats, exc=exc, 

688 nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements) 

689 

690 elif format == "rst": 

691 # It adds a link to the notebook. 

692 thisfiles += add_link_to_notebook( 

693 outputfile, notebook, "pdf" in formats, "html" in formats, "python" in formats, 

694 "slides" in formats, exc=exc, github="github" in formats, 

695 notebook=notebook, nblinks=nblinks, fLOG=fLOG) 

696 

697 elif format in ("tex", "elatex", "latex", "pdf"): 

698 thisfiles += add_link_to_notebook(outputfile, notebook, False, False, 

699 False, False, exc=exc, nblinks=nblinks, 

700 fLOG=fLOG, notebook_replacements=nb_replacements) 

701 

702 elif format in ("py", "python"): 

703 post_process_python_output( 

704 outputfile, True, nblinks=nblinks, fLOG=fLOG, notebook_replacements=nb_replacements) 

705 

706 elif format in ["docx", "word"]: 

707 pass 

708 

709 else: 

710 raise HelpGenException( # pragma: no cover 

711 "unexpected format " + format) 

712 

713 files.extend(thisfiles) 

714 fLOG("[_process_notebooks_in] ### conversion into '{}' done into '{}'.".format( 

715 format_, outputfile)) 

716 

717 copy = [] 

718 for f in files: 

719 dest = os.path.join(outfold, os.path.split(f)[-1]) 

720 if not f.endswith(".tex"): 

721 

722 if sys.version_info >= (3, 4): 

723 try: 

724 shutil.copy(f, outfold) 

725 fLOG("[_process_notebooks_in] copy ", 

726 f, " to ", outfold, "[", dest, "]") 

727 except shutil.SameFileError: 

728 fLOG("[_process_notebooks_in] w,file ", 

729 dest, "already exists") 

730 else: # pragma: no cover 

731 try: 

732 shutil.copy(f, outfold) 

733 fLOG("[_process_notebooks_in] copy ", 

734 f, " to ", outfold, "[", dest, "]") 

735 except shutil.Error as e: 

736 if "are the same file" in str(e): 

737 fLOG("[_process_notebooks_in] w,file ", 

738 dest, "already exists") 

739 else: 

740 raise e 

741 

742 if not os.path.exists(dest): 

743 raise FileNotFoundError(dest) # pragma: no cover 

744 copy.append((dest, True)) 

745 

746 # image 

747 for image in os.listdir(build): 

748 if (image.endswith(".png") or image.endswith(".html") or 

749 image.endswith(".pdf") or image.endswith(".svg") or 

750 image.endswith(".jpg") or image.endswith(".gif") or 

751 image.endswith(".xml") or image.endswith(".jpeg")): 

752 image = os.path.join(build, image) 

753 dest = os.path.join(outfold, os.path.split(image)[-1]) 

754 

755 try: 

756 shutil.copy(image, outfold) 

757 fLOG("[_process_notebooks_in] copy ", 

758 image, " to ", outfold, "[", dest, "]") 

759 except shutil.SameFileError: 

760 fLOG("[_process_notebooks_in] w,file ", 

761 dest, "already exists") 

762 

763 if not os.path.exists(dest): 

764 raise FileNotFoundError(dest) # pragma: no cover 

765 copy.append((dest, True)) 

766 

767 return copy + [(_, False) for _ in skipped] 

768 

769 

770def add_link_to_notebook(file, nb, pdf, html, python, slides, exc=True, 

771 github=False, notebook=None, nblinks=None, fLOG=None, 

772 notebook_replacements=None): 

773 """ 

774 Adds a link to the notebook in :epkg:`HTML` format and does a little bit of cleaning 

775 for various format. 

776 

777 @param file notebook.html 

778 @param nb notebook (.ipynb) 

779 @param pdf if True, add a link to the PDF, assuming it will exists at the same location 

780 @param html if True, add a link to the HTML conversion 

781 @param python if True, add a link to the Python conversion 

782 @param slides if True, add a link to the HTML slides 

783 @param exc raises an exception (True) or a warning (False) 

784 @param github add a link to the notebook on github 

785 @param notebook location of the notebook (file might be a copy) 

786 @param nblinks dictionary ``{ref: url}`` 

787 @param notebook_replacements stirng replacement in notebooks 

788 @param fLOG logging function 

789 @return list of generated files 

790 

791 The function does some cleaning too in the files. 

792 """ 

793 core, ext = os.path.splitext(file) 

794 if core.endswith(".slides"): 

795 ext = ".slides" + ext 

796 fLOG("[add_link_to_notebook] add_link_to_notebook", ext, " file ", file) 

797 

798 fold = os.path.split(file)[0] 

799 res = [os.path.join(fold, os.path.split(nb)[-1])] 

800 newr = has_been_updated(nb, res[-1])[0] 

801 if newr: 

802 shutil.copy(nb, fold) 

803 

804 if ext == ".ipynb": 

805 return res 

806 if ext == ".pdf": 

807 return res 

808 if ext == ".html": 

809 post_process_html_output( 

810 file, pdf, python, slides, exc=exc, nblinks=nblinks, 

811 fLOG=fLOG, notebook_replacements=notebook_replacements) 

812 return res 

813 if ext == ".slides.html": 

814 post_process_slides_output( 

815 file, pdf, python, slides, exc=exc, nblinks=nblinks, 

816 fLOG=fLOG, notebook_replacements=notebook_replacements) 

817 return res 

818 if ext == ".slides2p.html": 

819 post_process_slides_output( 

820 file, pdf, python, slides, exc=exc, nblinks=nblinks, 

821 fLOG=fLOG, notebook_replacements=notebook_replacements) 

822 return res 

823 if ext == ".tex": 

824 post_process_latex_output( 

825 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG, 

826 notebook_replacements=notebook_replacements) 

827 return res 

828 if ext == ".py": 

829 post_process_python_output( 

830 file, True, exc=exc, nblinks=nblinks, fLOG=fLOG, 

831 notebook_replacements=notebook_replacements) 

832 return res 

833 if ext == ".rst": 

834 post_process_rst_output( 

835 file, html, pdf, python, slides, is_notebook=True, exc=exc, 

836 github=github, notebook=notebook, nblinks=nblinks, fLOG=fLOG, 

837 notebook_replacements=notebook_replacements) 

838 return res 

839 raise HelpGenException( # pragma: no cover 

840 f"Unable to add a link to this extension: {ext!r}") 

841 

842 

843def build_thumbail_in_gallery(nbfile, folder_snippet, relative, rst_link, layout, snippet_folder=None, fLOG=None): 

844 """ 

845 Returns :epkg:`rst` code for a notebook. 

846 

847 @param nbfile notebook file 

848 @param folder_snippet where to store the snippet 

849 @param relative the path to the snippet will be relative to this folder 

850 @param rst_link rst link 

851 @param layout ``'classic'`` or ``'table'`` 

852 @param snippet_folder folder where to find custom snippet for notebooks, 

853 the snippet should have the same name as the notebook 

854 itself, snippet must have extension ``.png`` 

855 @return RST 

856 

857 Modifies the function to bypass the generation of a snippet 

858 if a custom one was found. Parameter *snippet_folder* was added. 

859 """ 

860 from ..ipythonhelper import read_nb 

861 nb = read_nb(nbfile) 

862 _, desc = nb.get_description() 

863 

864 if snippet_folder is not None and os.path.exists(snippet_folder): 

865 custom_snippet = os.path.join(snippet_folder, os.path.splitext( 

866 os.path.split(nbfile)[-1])[0] + '.png') 

867 else: 

868 custom_snippet = None 

869 

870 if custom_snippet is not None and os.path.exists(custom_snippet): 

871 # reading a custom snippet 

872 if fLOG: 

873 fLOG( 

874 f"[build_thumbail_in_gallery] custom snippet '{custom_snippet}'") 

875 try: 

876 from PIL import Image 

877 except ImportError: # pragma: no cover 

878 import Image 

879 image = Image.open(custom_snippet) 

880 else: 

881 # generating an image 

882 if fLOG: 

883 fLOG( 

884 f"[build_thumbail_in_gallery] build snippet from '{nbfile}'") 

885 image = nb.get_thumbnail() 

886 

887 if image is None: 

888 image = nb.get_thumbnail(use_default=True) 

889 

890 if image is None: 

891 raise ValueError( # pragma: no cover 

892 f"The snippet cannot be null, notebook='{nbfile}'.") 

893 name = os.path.splitext(os.path.split(nbfile)[-1])[0] 

894 name += ".thumb" 

895 full = os.path.join(folder_snippet, name) 

896 

897 dirname = os.path.dirname(full) 

898 if not os.path.exists(dirname): 

899 raise FileNotFoundError( # pragma: no cover 

900 "Unable to find folder '{0}'\nfolder_snippet='{1}'\nrelative='{2}'\nnbfile='{3}'".format( 

901 dirname, folder_snippet, relative, nbfile)) 

902 

903 if isinstance(image, str): 

904 # SVG 

905 full += ".svg" 

906 name += ".svg" 

907 with open(full, "w", encoding="utf-8") as f: 

908 f.write(image) 

909 else: 

910 # Image 

911 full += ".png" 

912 name += ".png" 

913 image.save(full) 

914 

915 rel = os.path.relpath(full, start=relative).replace("\\", "/") 

916 nb_name = rel.replace(".thumb.png", ".html") 

917 if layout == "classic": 

918 rst = THUMBNAIL_TEMPLATE.format( 

919 snippet=desc, thumbnail=rel, ref_name=rst_link) 

920 elif layout == "table": 

921 rst = THUMBNAIL_TEMPLATE_TABLE.format( 

922 snippet=desc, thumbnail=rel, ref_name=rst_link, nb_name=nb_name) 

923 else: 

924 raise ValueError( 

925 "layout must be 'classic' or 'table'") # pragma: no cover 

926 return rst 

927 

928 

929def add_tag_for_slideshow(ipy, folder, encoding="utf8"): 

930 """ 

931 Modifies a notebook to add tag for a slideshow. 

932 

933 @param ipy notebook file 

934 @param folder where to write the new notebook 

935 @param encoding encoding 

936 @return written file 

937 """ 

938 from ..ipythonhelper import read_nb 

939 filename = os.path.split(ipy)[-1] 

940 output = os.path.join(folder, filename) 

941 nb = read_nb(ipy, encoding=encoding, kernel=False) 

942 nb.add_tag_slide() 

943 nb.to_json(output) 

944 return output 

945 

946 

947def build_notebooks_gallery(nbs, fileout, layout="classic", neg_pattern=None, 

948 snippet_folder=None, fLOG=noLOG): 

949 """ 

950 Creates a :epkg:`rst` page (gallery) with links to all notebooks. 

951 For each notebook, it creates a snippet. 

952 

953 @param nbs list of notebooks to consider or tuple(full path, rst), 

954 @param fileout file to create 

955 @param layout ``'classic'`` or ``'table'`` 

956 @param neg_pattern do not consider notebooks matching this regular expression 

957 @param snippet_folder folder where to find custom snippet for notebooks, 

958 the snippet should have the same name as the notebook 

959 itself, snippet must have extension ``.png`` 

960 @param fLOG logging function 

961 @return created file name 

962 

963 Example for parameter *nbs*: 

964 

965 :: 

966 

967 ('challenges\\city_tour\\city_tour_1.ipynb', 

968 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1.ipynb') 

969 ('challenges\\city_tour\\city_tour_1_solution.ipynb', 

970 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_1_solution.ipynb') 

971 ('challenges\\city_tour\\city_tour_data_preparation.ipynb', 

972 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_data_preparation.ipynb') 

973 ('challenges\\city_tour\\city_tour_long.ipynb', 

974 'ensae_projects\\_doc\\notebooks\\challenges\\city_tour\\city_tour_long.ipynb') 

975 ('cheat_sheets\\chsh_files.ipynb', 

976 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_files.ipynb') 

977 ('cheat_sheets\\chsh_geo.ipynb', 

978 'ensae_projects\\_doc\\notebooks\\cheat_sheets\\chsh_geo.ipynb') 

979 

980 *nbs* can be a folder, in that case, the function will build 

981 the list of all notebooks in that folder. 

982 *nbs* can be a list of tuple. 

983 the function adds a thumbnail, organizes the list of notebook 

984 as a galley, it adds a link on notebook coverage. 

985 The function bypasses the generation of a snippet 

986 if a custom one was found. 

987 """ 

988 from ..ipythonhelper import read_nb 

989 if not isinstance(nbs, list): 

990 fold = nbs 

991 nbs = explore_folder( 

992 fold, ".*[.]ipynb", neg_pattern=neg_pattern, fullname=True)[1] 

993 if len(nbs) == 0: 

994 raise FileNotFoundError( # pragma: no cover 

995 f"Unable to find notebooks in folder '{nbs}'.") 

996 nbs = [(os.path.relpath(n, fold), n) for n in nbs] 

997 

998 # Go through the list of notebooks. 

999 fLOG("[build_notebooks_gallery]", len(nbs), "notebooks") 

1000 hier = set() 

1001 rst = [] 

1002 containers = {} 

1003 for tu in nbs: 

1004 if isinstance(tu, (tuple, list)): 

1005 if tu[0] is None or ("/" not in tu[0] and "\\" not in tu[0]): 

1006 rst.append((tuple(), tu[1])) 

1007 else: 

1008 way = tuple(tu[0].replace("\\", "/").split("/")[:-1]) 

1009 hier.add(way) 

1010 rst.append((way, tu[1])) 

1011 else: 

1012 rst.append((tuple(), tu)) 

1013 name = rst[-1][1] 

1014 ext = os.path.splitext(name)[-1] 

1015 if ext != ".ipynb": 

1016 raise ValueError( # pragma: no cover 

1017 f"One file is not a notebook: {rst[-1][1]}") 

1018 dirname, na = os.path.split(name) 

1019 if dirname not in containers: 

1020 containers[dirname] = [] 

1021 containers[dirname].append(na) 

1022 rst.sort() 

1023 

1024 folder_index = os.path.dirname(os.path.normpath(fileout)) 

1025 folder = os.path.join(folder_index, "notebooks") 

1026 if not os.path.exists(folder): 

1027 os.mkdir(folder) 

1028 

1029 # reordering based on titles 

1030 titles = {} 

1031 reord = [] 

1032 for hi, nbf in rst: 

1033 nb = read_nb(nbf) 

1034 title = nb.get_description()[0] 

1035 titles[nbf] = title 

1036 reord.append((hi, title, nbf)) 

1037 reord.sort() 

1038 rst = [_[:1] + _[-1:] for _ in reord] 

1039 

1040 # containers 

1041 containers = list(sorted((k, v) for k, v in containers.items())) 

1042 

1043 # find root 

1044 hi, rs = rst[0] 

1045 if len(hi) == 0: 

1046 root = os.path.dirname(rs) 

1047 else: 

1048 spl = rs.replace("\\", "/").split("/") 

1049 ro = spl[:-len(hi) - 1] 

1050 root = "/".join(ro) 

1051 

1052 # look for README.txt 

1053 fLOG("[build_notebooks_gallery] root", root) 

1054 rows = ["", ":orphan:", ""] 

1055 exp = os.path.join(root, "README.txt") 

1056 if os.path.exists(exp): 

1057 fLOG("[build_notebooks_gallery] found", exp) 

1058 with open(exp, "r", encoding="utf-8") as f: 

1059 try: 

1060 rows.extend(["", ".. _l-notebooks:", "", f.read(), ""]) 

1061 except UnicodeDecodeError as e: # pragma: no cover 

1062 raise ValueError(f"Issue with file '{exp}'") from e 

1063 else: 

1064 fLOG("[build_notebooks_gallery] not found", exp) 

1065 rows.extend(["", ".. _l-notebooks:", "", "", "Notebooks Gallery", 

1066 "=================", ""]) 

1067 

1068 rows.extend(["", ":ref:`l-notebooks-coverage`", "", 

1069 "", ".. contents::", " :depth: 1", 

1070 " :local:", ""]) 

1071 

1072 # produces the final files 

1073 if len(hier) == 0: 

1074 # case where there is no hierarchy 

1075 fLOG("[build_notebooks_gallery] no hierarchy") 

1076 rows.append(".. toctree::") 

1077 rows.append(" :maxdepth: 1") 

1078 if layout == "table": 

1079 rows.append(" :hidden:") 

1080 rows.append("") 

1081 for hi, file in rst: 

1082 rs = os.path.splitext(os.path.split(file)[-1])[0] 

1083 fLOG("[build_notebooks_gallery] adding", 

1084 rs, " title ", titles.get(file, None)) 

1085 rows.append(f" notebooks/{rs}") 

1086 if layout == "table" and len(rst) > 0: 

1087 rows.extend(["", "", ".. list-table::", 

1088 " :header-rows: 0", " :widths: 3 5 15", ""]) 

1089 

1090 for _, file in rst: 

1091 link = os.path.splitext(os.path.split(file)[-1])[0] 

1092 link = link.replace("_", "") + "rst" 

1093 if not os.path.exists(file): 

1094 raise FileNotFoundError( # pragma: no cover 

1095 "Unable to find: '{0}'\nRST=\n{1}".format( 

1096 file, "\n".join(str(_) for _ in rst))) 

1097 r = build_thumbail_in_gallery( 

1098 file, folder, folder_index, link, layout, 

1099 snippet_folder=snippet_folder, fLOG=fLOG) 

1100 rows.append(r) 

1101 else: 

1102 # case where there are subfolders 

1103 fLOG("[build_notebooks_gallery] subfolders") 

1104 already = "\n".join(rows) 

1105 level = "-+^" 

1106 rows.append("") 

1107 if ".. contents::" not in already: 

1108 rows.append(".. contents::") 

1109 rows.append(" :local:") 

1110 rows.append(" :depth: 2") 

1111 rows.append("") 

1112 stack_file = [] 

1113 last = None 

1114 for hi, r in rst: 

1115 rs0 = os.path.splitext(os.path.split(r)[-1])[0] 

1116 r0 = r 

1117 if hi != last: 

1118 fLOG("[build_notebooks_gallery] new level", hi) 

1119 # It adds the thumbnail. 

1120 if layout == "table" and len(stack_file) > 0: 

1121 rows.extend( 

1122 ["", "", ".. list-table::", " :header-rows: 0", " :widths: 3 5 15", ""]) 

1123 

1124 for nbf in stack_file: 

1125 fLOG("[build_notebooks_gallery] ", nbf) 

1126 rs = os.path.splitext(os.path.split(nbf)[-1])[0] 

1127 link = rs.replace("_", "") + "rst" 

1128 r = build_thumbail_in_gallery( 

1129 nbf, folder, folder_index, link, layout) 

1130 rows.append(r) 

1131 fLOG(f"[build_notebooks_gallery] saw {len(stack_file)} files") 

1132 stack_file = [] 

1133 

1134 # It switches to the next gallery. 

1135 if layout == "classic": 

1136 rows.append(".. raw:: html") 

1137 rows.append("") 

1138 rows.append(" <div style='clear:both'></div>") 

1139 rows.append("") 

1140 

1141 # It adds menus and subfolders. 

1142 lastk = 0 

1143 for k in range(0, len(hi)): 

1144 lastk = k 

1145 if last is None or k >= len(last) or hi[k] != last[k]: # pylint: disable=E1136 

1146 break 

1147 

1148 while len(hi) > 0 and lastk < len(hi): 

1149 fo = [root] + list(hi[:lastk + 1]) 

1150 readme = os.path.join(*(fo + ["README.txt"])) 

1151 if os.path.exists(readme): 

1152 fLOG("[build_notebooks_gallery] found", readme) 

1153 with open(readme, "r", encoding="utf-8") as f: 

1154 try: 

1155 rows.extend(["", f.read(), ""]) 

1156 except UnicodeDecodeError as e: # pragma: no cover 

1157 raise ValueError( 

1158 f"Issue with file '{readme}'") from e 

1159 else: 

1160 fLOG("[build_notebooks_gallery] not found", readme) 

1161 rows.append("") 

1162 rows.append(hi[lastk]) 

1163 rows.append( 

1164 level[min(lastk, len(level) - 1)] * len(hi[lastk])) 

1165 rows.append("") 

1166 lastk += 1 

1167 

1168 # It starts the next gallery. 

1169 last = hi 

1170 rows.append(".. toctree::") 

1171 rows.append(" :maxdepth: 1") 

1172 if layout == "table": 

1173 rows.append(" :hidden:") 

1174 rows.append("") 

1175 

1176 # append a link to a notebook 

1177 fLOG("[build_notebooks_gallery] adding", 

1178 rs0, " title ", titles.get(r0, None)) 

1179 rows.append(f" notebooks/{rs0}") 

1180 stack_file.append(r0) 

1181 

1182 if len(stack_file) > 0: 

1183 # It adds the thumbnails. 

1184 if layout == "table" and len(stack_file) > 0: 

1185 rows.extend(["", "", ".. list-table::", 

1186 " :header-rows: 0", " :widths: 3 5 15", ""]) 

1187 

1188 for nbf in stack_file: 

1189 rs = os.path.splitext(os.path.split(nbf)[-1])[0] 

1190 link = rs.replace("_", "") + "rst" 

1191 r = build_thumbail_in_gallery( 

1192 nbf, folder, folder_index, link, layout) 

1193 rows.append(r) 

1194 

1195 # done 

1196 rows.append("") 

1197 

1198 # links to coverage 

1199 rows.extend(["", "", ".. toctree::", " :hidden: ", "", 

1200 " all_notebooks_coverage", ""]) 

1201 

1202 with open(fileout, "w", encoding="utf8") as f: 

1203 f.write("\n".join(rows)) 

1204 return fileout 

1205 

1206 

1207def build_all_notebooks_coverage(nbs, fileout, module_name, dump=None, badge=True, too_old=30, fLOG=noLOG): 

1208 """ 

1209 Creates a :epkg:`rst` page (gallery) with links to all notebooks and 

1210 information about coverage. 

1211 It relies on function @see fn notebook_coverage. 

1212 

1213 @param nbs list of notebooks to consider or tuple(full path, rst), 

1214 @param fileout file to create 

1215 @param module_name module name 

1216 @param dump dump containing information about notebook execution (or None for the default one) 

1217 @param badge builds an image with the notebook coverage 

1218 @param too_old drop executions older than *too_old* days from now 

1219 @param fLOG logging function 

1220 @return dataframe which contains the data 

1221 """ 

1222 from ..ipythonhelper import read_nb, notebook_coverage 

1223 if dump is None: 

1224 dump = os.path.normpath(os.path.join(os.path.dirname(fileout), "..", "..", "..", "..", 

1225 "_notebook_dumps", f"notebook.{module_name}.txt")) 

1226 if not os.path.exists(dump): 

1227 fLOG("[notebooks-coverage] No execution report about " 

1228 "notebook at '{0}' (fileout='{1}')".format(dump, 

1229 os.path.dirname(fileout))) 

1230 return None 

1231 report0 = notebook_coverage(nbs, dump, too_old=too_old) 

1232 fLOG("[notebooks-coverage] report shape", report0.shape) 

1233 

1234 from numpy import isnan 

1235 

1236 # Fill nan values. 

1237 for i in report0.index: 

1238 nbcell = report0.loc[i, "nbcell"] 

1239 if isnan(nbcell): 

1240 # It loads the notebook. 

1241 nbfile = report0.loc[i, "notebooks"] 

1242 nb = read_nb(nbfile) 

1243 report0.loc[i, "nbcell"] = len(nb) 

1244 report0.loc[i, "nbrun"] = 0 

1245 

1246 # Add links. 

1247 cols = ['notebooks', 'date', 'etime', 

1248 'nbcell', 'nbrun', 'nbvalid', 'success', 'time'] 

1249 report = report0[cols].copy() 

1250 report["notebooks"] = report["notebooks"].apply( 

1251 lambda x: "/".join(os.path.normpath(x).replace("\\", "/").split("/")[-2:]) if isinstance(x, str) else x) 

1252 report["last_name"] = report["notebooks"].apply( 

1253 lambda x: os.path.split(x)[-1] if isinstance(x, str) else x) 

1254 

1255 report1 = report.copy() 

1256 

1257 def clean_link(link): 

1258 return link.replace("_", "").replace(".ipynb", ".rst").replace(".", "") if isinstance(link, str) else link 

1259 

1260 report["notebooks"] = report.apply(lambda row: ':ref:`{0} <{1}>`'.format( 

1261 row["notebooks"], clean_link(row["last_name"])), axis=1) 

1262 report["title"] = report["last_name"].apply( 

1263 lambda x: f':ref:`{clean_link(x)}`') 

1264 rows = ["", ".. _l-notebooks-coverage:", "", "", "Notebooks Coverage", 

1265 "==================", "", "Report on last executions.", ""] 

1266 

1267 # Badge 

1268 if badge: 

1269 from ..ipythonhelper import badge_notebook_coverage 

1270 img = os.path.join(os.path.dirname(fileout), "nbcov.png") 

1271 cov = badge_notebook_coverage(report0, img) 

1272 now = datetime.datetime.now() 

1273 sdate = "%04d-%02d-%02d" % (now.year, now.month, now.day) 

1274 cpy = os.path.join(os.path.dirname(fileout), f"nbcov-{sdate}.png") 

1275 shutil.copy(img, cpy) 

1276 badge = ["{0:0.00f}% {1}".format( 

1277 cov, sdate), "", f".. image:: {os.path.split(cpy)[-1]}", ""] 

1278 badge2 = ["", f".. image:: {os.path.split(img)[-1]}", ""] 

1279 else: 

1280 badge = [] 

1281 badge2 = [] 

1282 rows.extend(badge) 

1283 

1284 # Formatting 

1285 report["date"] = report["date"].apply( 

1286 lambda x: x.split()[0] if isinstance(x, str) else x) 

1287 report["etime"] = report["etime"].apply( 

1288 lambda x: f"{x:1.3f}" if isinstance(x, float) else x) 

1289 report["time"] = report["time"].apply( 

1290 lambda x: f"{x:1.3f}" if isinstance(x, float) else x) 

1291 

1292 def int2str(x): 

1293 if isnan(x): 

1294 return "" 

1295 else: 

1296 return int(x) 

1297 

1298 report["coverage"] = report["nbrun"] / report["nbcell"] 

1299 report["nbcell"] = report["nbcell"].apply(int2str) 

1300 report["nbrun"] = report["nbrun"].apply(int2str) 

1301 report["nbvalid"] = report["nbvalid"].apply(int2str) 

1302 report["coverage"] = report["coverage"].apply( 

1303 lambda x: f"{int(x * 100)}%" if isinstance(x, float) else "") 

1304 report = report[['notebooks', 'title', 'date', 'success', 'etime', 

1305 'nbcell', 'nbrun', 'nbvalid', 'time', 'coverage']].copy() 

1306 report.columns = ['name', 'title', 'last execution', 'success', 'time', 

1307 'nb cells', 'nb runs', 'nb valid', 'exe time', 'coverage'] 

1308 report = report[['coverage', 'exe time', 'last execution', 'name', 'title', 

1309 'success', 'time', 'nb cells', 'nb runs', 'nb valid']] 

1310 

1311 # Add results. 

1312 text = df2rst(report.sort_values("name").reset_index( 

1313 drop=True), index=True, list_table=True) 

1314 rows.append(text) 

1315 rows.extend(badge2) 

1316 

1317 fLOG("[notebooks-coverage] writing", fileout) 

1318 with open(fileout, "w", encoding="utf-8") as f: 

1319 f.write("\n".join(rows)) 

1320 return report1