Coverage for src/pyquickhelper/helpgen/post_process.py : 85%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains the main function to generate the documentation
5for a module designed the same way as this one, @see fn generate_help_sphinx.
6"""
7import os
8import re
9import warnings
10import glob
11from .utils_sphinx_doc_helpers import HelpGenException
14template_examples = """
16List of programs
17++++++++++++++++
19.. toctree::
20 :maxdepth: 2
22.. autosummary:: __init__.py
23 :toctree: %s/
24 :template: modules.rst
26Another list
27++++++++++++
29"""
32def update_notebook_link(text, format, nblinks, fLOG):
33 """
34 A notebook can contain a link ``[anchor](find://...)``
35 and it will be converted into: ``:ref:...`` in rst format.
37 @param text text to look into
38 @param format format
39 @param nblinks list of mappings *(reference: url)*
40 @param fLOG logging function
41 @return modified text
42 """
43 def get_url_from_nblinks(nblinks, url, format):
44 if isinstance(nblinks, dict):
45 if (url, format) in nblinks:
46 url = nblinks[url, format]
47 elif url in nblinks:
48 url = nblinks[url]
49 if url.startswith("find://"):
50 short = url[7:]
51 if (short, format) in nblinks:
52 url = nblinks[short, format]
53 elif short in nblinks:
54 url = nblinks[short]
55 else:
56 url = nblinks(url, format)
57 if url.startswith("find://"):
58 if format == 'python':
59 url = url[7:]
60 else: # pragma: no cover
61 snb = "\n".join("'{0}': '{1}'".format(k, v)
62 for k, v in sorted(nblinks.items()))
63 extension = (
64 "You shoud add links into variable 'nblinks' "
65 "into documentation configuration file.")
66 extension += "\nnblinks={0}".format(nblinks)
67 raise HelpGenException(
68 "Unable to find a replacement for '{0}' format='{1}' in \n{2}\n{3}".format(
69 url, format, snb, extension))
70 return url
72 if nblinks is None:
73 nblinks = {}
74 if format == "rst":
75 def reprst(le):
76 anc, url = le.groups()
77 url = get_url_from_nblinks(nblinks, url, format)
78 if "://" in url:
79 new_url = "`{0} <{1}>`_".format(anc, url)
80 else:
81 new_url = ":ref:`{0} <{1}>`".format(anc, url)
82 if fLOG:
83 fLOG(" [update_notebook_link]1 add in ",
84 format, ":", new_url)
85 return new_url
86 reg = re.compile("`([^`]+?) <find://([^`<>]+?)>`_")
87 new_text = reg.sub(reprst, text)
88 elif format in ("html", "slides", "slides2"):
89 def rephtml(le):
90 anc, url = le.groups()
91 url = get_url_from_nblinks(nblinks, url, format)
92 new_url = "<a href=\"{0}.html\">{1}</a>".format(anc, url)
93 if fLOG:
94 fLOG(" [update_notebook_link]2 add in ",
95 format, ":", new_url)
96 return new_url
97 reg = re.compile("<a href=\\\"find://([^\\\"]+?)\\\">([^`<>]+?)</a>")
98 new_text = reg.sub(rephtml, text)
99 elif format in ("ipynb", "python"):
100 def repipy(le):
101 anc, url = le.groups()
102 url = get_url_from_nblinks(nblinks, "find://" + url, format)
103 if not url.startswith("http"):
104 mes = "\n".join("{0}: '{1}'".format(k, v)
105 for k, v in sorted(nblinks.items()))
106 extension = "You should add this link into the documentation " \
107 "configuration file in variable 'nblinks'."
108 raise HelpGenException( # pragma: no cover
109 "A reference was not found: '{0}' - '{1}' "
110 "format={2}, nblinks=\n{3}\n{4}".format(
111 anc, url, format, mes, extension))
112 new_url = "[{0}]({1})".format(anc, url)
113 if fLOG:
114 fLOG(" [update_notebook_link]3 add in ",
115 format, ":", new_url)
116 return new_url
117 reg = re.compile("[\\[]([^[]+?)[\\]][(]find://([^ ]+)[)]")
118 new_text = reg.sub(repipy, text)
119 elif format in ("latex", "elatex"):
120 def replat(le):
121 url, anc = le.groups()
122 url = get_url_from_nblinks(nblinks, url, format)
123 if not url.endswith(".html") and not url.endswith(".js") and not url.endswith(".css"):
124 url += ".html"
125 new_url = "\\href{{{0}}}{{{1}}}".format(url, anc)
126 if fLOG:
127 fLOG(" [update_notebook_link]4 add in ",
128 format, ":", new_url)
129 return new_url
130 reg = re.compile("\\\\href{find://([^{} ]+?)}{([^{}]+)}")
131 new_text = reg.sub(replat, text)
132 # {\hyperref[\detokenize{c_classes/classes:chap-classe}]
133 # {\sphinxcrossref{\DUrole{std,std-ref}{Classes}}}}
134 else:
135 raise NotImplementedError( # pragma: no cover
136 "Unsupported format '{0}'\n{1}".format(format, text))
137 return new_text
140def _notebook_replacements(nbtext, notebook_replacements, fLOG=None):
141 """
142 Makes some replacements in a notebook.
144 @param nbtext text to process
145 @param notebook_replacements dictionary of replacements
146 @param fLOG logging function
147 @return text
148 """
149 if notebook_replacements is None:
150 return nbtext
151 for k, v in notebook_replacements:
152 if k in nbtext:
153 fLOG(
154 "[_notebook_replacements] replace '{0}' -> '{1}'".format(k, v))
155 nbtext = nbtext.replace(k, v)
156 if '"nbformat": 4,' in nbtext: # pragma: no cover
157 rep = ['"nbformat_minor": 0', '"nbformat_minor": 1',
158 '"nbformat_minor": 2']
159 for r in rep:
160 if r in nbtext:
161 nbtext = nbtext.replace(r, '"nbformat_minor": 4')
162 return nbtext
165def post_process_latex_output(root, doall, latex_book=False, exc=True,
166 custom_latex_processing=None, nblinks=None,
167 remove_unicode=True, fLOG=None, notebook_replacements=None):
168 """
169 Postprocesses the latex file produced by :epkg:`sphinx`.
171 @param root root path or latex file to process
172 @param doall do all transformations
173 @param latex_book customized for a book
174 @param exc raises an exception or a warning
175 @param custom_latex_processing function which does some post processing of the full latex file
176 @param nblinks dictionary ``{ ref : url }`` where to look for references
177 @param remove_unicode remove unicode characters (fails with latex)
178 @param notebook_replacements string replacement in notebooks
179 @param fLOG logging function
180 """
181 if os.path.isfile(root):
182 file = root
183 if fLOG:
184 fLOG("[post_process_latex_output] clean %r" % file)
185 with open(file, "r", encoding="utf8") as f:
186 content = f.read()
187 with open(file + ".tex1~", "w", encoding="utf8") as f:
188 f.write(content)
189 content = post_process_latex(
190 content, doall, latex_book=latex_book, exc=exc,
191 custom_latex_processing=custom_latex_processing, nblinks=nblinks,
192 file=file, remove_unicode=remove_unicode, fLOG=fLOG,
193 notebook_replacements=notebook_replacements)
194 with open(file, "w", encoding="utf8") as f:
195 f.write(content)
196 else:
197 build = os.path.join(root, "_doc", "sphinxdoc", "build", "latex")
198 if not os.path.exists(build):
199 raise FileNotFoundError(build)
200 for tex in os.listdir(build):
201 if tex.endswith(".tex"):
202 file = os.path.join(build, tex)
203 fLOG("[post_process_latex_output] modify file", file)
204 with open(file, "r", encoding="utf8") as f:
205 content = f.read()
206 with open(file + ".tex2~", "w", encoding="utf8") as f:
207 f.write(content)
208 content = post_process_latex(
209 content, doall, info=file, latex_book=latex_book, exc=exc,
210 custom_latex_processing=custom_latex_processing, nblinks=nblinks,
211 file=file, remove_unicode=remove_unicode, fLOG=fLOG,
212 notebook_replacements=notebook_replacements)
213 with open(file, "w", encoding="utf8") as f:
214 f.write(content)
217def post_process_python_output(root, doall, exc=True, nblinks=None, fLOG=None, notebook_replacements=None):
218 """
219 Postprocesses the python file produced by :epkg:`sphinx`.
221 @param root root path or python file to process
222 @param doall unused
223 @param exc raise an exception if needed
224 @param nblinks dictionary ``{ref: url}``
225 @param notebook_replacements string replacement in notebooks
226 @param fLOG logging function
227 """
228 if os.path.isfile(root):
229 file = root
230 if fLOG:
231 fLOG("[post_process_python_output] clean %r" % file)
232 with open(file, "r", encoding="utf8") as f:
233 content = f.read()
234 content = post_process_python(
235 content, doall, nblinks=nblinks, file=file, fLOG=fLOG,
236 notebook_replacements=notebook_replacements)
237 with open(file, "w", encoding="utf8") as f:
238 f.write(content)
239 else:
240 build = os.path.join(root, "_doc", "sphinxdoc", "build", "latex")
241 if not os.path.exists(build) and exc:
242 raise FileNotFoundError(build)
243 for tex in os.listdir(build):
244 if tex.endswith(".tex"):
245 file = os.path.join(build, tex)
246 fLOG("[post_process_python_output] modify file", file)
247 with open(file, "r", encoding="utf8") as f:
248 content = f.read()
249 content = post_process_python(
250 content, doall, info=file, nblinks=nblinks, file=file, fLOG=fLOG)
251 with open(file, "w", encoding="utf8") as f:
252 f.write(content)
255def post_process_latex_output_any(file, custom_latex_processing, nblinks=None,
256 remove_unicode=False, fLOG=None, notebook_replacements=None):
257 """
258 Postprocesses the latex file produced by :epkg:`sphinx`.
260 @param file latex filename
261 @param custom_latex_processing function which does some post processing of the full latex file
262 @param nblinks dictionary ``{url: link}``
263 @param remove_unicode remove unicode characters
264 @param notebook_replacements string replacement in notebooks
265 @param fLOG logging function
266 """
267 if fLOG:
268 fLOG("[post_process_latex_output_any] ** post_process_latex_output_any ", file)
269 if not os.path.exists(file):
270 raise FileNotFoundError( # pragma: no cover
271 "Unable to find '{}', other files in the same folder\n{}".format(
272 file, "\n".join(os.listdir(os.path.dirname(file)))))
273 with open(file, "r", encoding="utf8") as f:
274 content = f.read()
275 with open(file + ".tex3.u{0}~".format(1 if remove_unicode else 0), "w", encoding="utf8") as f:
276 f.write(content)
277 content = post_process_latex(content, True, info=file, nblinks=nblinks, file=file,
278 remove_unicode=remove_unicode, fLOG=fLOG,
279 notebook_replacements=notebook_replacements)
280 with open(file, "w", encoding="utf8") as f:
281 f.write(content)
284def post_process_rst_output(file, html, pdf, python, slides, is_notebook=False,
285 exc=True, github=False, notebook=None, nblinks=None, fLOG=None,
286 notebook_replacements=None):
287 """
288 Processes a :epkg:`rst` file generated from the conversion of a notebook.
290 @param file filename
291 @param pdf if True, add a link to the :epkg:`pdf`,
292 assuming it will exists at the same location
293 @param html if True, add a link to the :epkg:`html` conversion
294 @param python if True, add a link to the :epkg:`Python` conversion
295 @param slides if True, add a link to the slides conversion
296 @param is_notebook does something more if the file is a notebook
297 @param exc raises an exception (True) or a warning (False)
298 @param github add a link to the notebook on :epkg:`github`
299 @param notebook location of the notebook, file might be a copy
300 @param nblinks links added to a notebook, dictionary ``{ref: url}``
301 @param notebook_replacements string replacement in notebooks
302 @param fLOG logging function
304 The function adds the following replacement
305 ``st = st.replace("\\\\mathbb{1}", "\\\\mathbf{1\\\\!\\\\!1}")``.
306 and checks that audio is only included in :epkg:`HTML`.
307 """
308 if fLOG:
309 fLOG("[post_process_rst_output] clean %r" % file)
311 name = os.path.split(file)[1]
312 noext = os.path.splitext(name)[0]
313 with open(file, "r", encoding="utf8") as f:
314 lines = f.readlines()
315 with open(file + "~", "w", encoding="utf8") as f:
316 f.write("".join(lines))
318 # Probably not the best way to fix that.
319 # For some reason, nbconvert adds None as the first row.
320 if lines[0] == 'None\n':
321 lines[0] = '\n' # pragma: no cover
323 if any(line == 'None\n' for line in lines):
324 raise HelpGenException( # pragma: no cover
325 "One row unexpectedly contains only None in '{}'\n{}".format(
326 file, "".join(lines[:20])))
328 # Removes empty lines in inserted code, also adds line number.
329 def startss(line):
330 for b in ["::", ".. parsed-literal::", ".. code:: python",
331 ".. code-block:: python"]:
332 if line.startswith(b):
333 return b
334 return None
336 codeb = [".. code:: python", ".. code-block:: python"]
337 inbloc = False
338 for pos, line in enumerate(lines):
339 if not inbloc:
340 b = startss(line)
341 if b is None:
342 pass
343 else:
344 if b in codeb:
345 # we remove line number for the notebooks
346 if "notebook" not in file:
347 lines[pos] = "{0}\n :linenos:\n\n".format(codeb[-1])
348 else:
349 lines[pos] = "{0}\n\n".format(codeb[-1])
350 inbloc = True
351 memopos = pos
352 else:
353 if len(line.strip(" \r\n")) == 0 and pos < len(lines) - 1 and \
354 lines[pos + 1].startswith(" ") and len(lines[pos + 1].strip(" \r\n")) > 0:
355 lines[pos] = ""
357 elif not line.startswith(" ") and line != "\n":
358 inbloc = False
360 if lines[memopos].startswith("::"):
361 code = "".join(
362 (_[4:] if _.startswith(" ") else _) for _ in lines[memopos + 1:pos])
363 if len(code) == 0:
364 fLOG( # pragma: no cover
365 "[post_process_rst_output] EMPTY-SECTION in ", file)
366 else:
367 try:
368 cmp = compile(code, "", "exec")
369 if cmp is not None:
370 lines[memopos] = "{0}\n :linenos:\n".format(
371 ".. code-block:: python")
372 except Exception: # pragma: no cover
373 pass
375 memopos = None
377 # code and images
378 imgreg = re.compile("[.][.] image:: (.*)")
379 for pos in range(0, len(lines)):
380 # lines[pos] = lines[pos].replace(".. code:: python","::")
381 if lines[pos].strip().startswith(".. image::"):
382 # we assume every image should be placed in the same folder as the
383 # notebook itself
384 img = imgreg.findall(lines[pos])
385 if len(img) == 0:
386 raise HelpGenException( # pragma: no cover
387 "Unable to extract image name in '{0}'".format(lines[pos]))
388 nameimg = img[0]
389 short = nameimg.replace("%5C", "/")
390 short = os.path.split(short)[-1]
391 lines[pos] = lines[pos].replace(nameimg, short)
393 # title
394 for pos, line in enumerate(lines):
395 line = line.strip("\n\r")
396 if len(line) > 0 and line == "=" * len(line):
397 # lines[pos] = lines[pos].replace("=", "*")
398 pos2 = pos - 1
399 li = len(lines[pos])
400 while len(lines[pos2]) != li:
401 pos2 -= 1
402 sep = "" if lines[pos2].endswith("\n") else "\n"
403 lines[pos2] = "{0}{2}{1}".format(lines[pos], lines[pos2], sep)
404 for p in range(pos2 + 1, pos):
405 if lines[p] == "\n": # pragma: no cover
406 lines[p] = ""
407 break
409 pos += 1
410 if pos >= len(lines):
411 mes = "Unable to find a title in notebook '{0}'".format(file)
412 if exc:
413 raise HelpGenException(mes) # pragma: no cover
414 warnings.warn(mes, UserWarning)
416 # label
417 labelname = name.replace(" ", "").replace("_", "").replace(
418 ":", "").replace(".", "").replace(",", "")
419 label = "\n.. _{0}:\n\n".format(labelname)
420 lines.insert(0, label)
422 # links
423 links = ['**Links:** :download:`notebook <{0}.ipynb>`'.format(noext)]
424 if html:
425 links.append(':downloadlink:`html <{0}2html.html>`'.format(noext))
426 if pdf:
427 links.append(':download:`PDF <{0}.pdf>`'.format(noext))
428 if python:
429 links.append(':download:`python <{0}.py>`'.format(noext))
430 if slides:
431 links.append(':downloadlink:`slides <{0}.slides.html>`'.format(noext))
433 if github:
434 if notebook is None:
435 raise ValueError( # pragma: no cover
436 "Cannot add a link on github, notebook is None for "
437 "file='{0}'".format(file))
438 docname = notebook
439 folder = docname
440 git = os.path.join(folder, ".git")
441 while len(folder) > 0 and not os.path.exists(git):
442 folder = os.path.split(folder)[0]
443 git = os.path.join(folder, ".git")
444 if len(folder) > 0:
445 path = docname[len(folder):]
446 tried = []
447 if path.strip('/\\').startswith('build'):
448 # The notebook may be in a build folder but is not
449 # the original notebook. The function does something
450 # if the path starts with `build`.
451 subfolds = os.listdir(folder)
452 for sub in subfolds:
453 fulls = os.path.join(folder, sub)
454 if not os.path.isdir(fulls):
455 continue
456 if not ('_doc' in sub or 'notebook' in sub or 'example' in sub):
457 continue
458 # Search for another version of the file.
459 last_name = os.path.split(docname)[-1]
460 tried.append((last_name, fulls))
461 selected = glob.glob(
462 fulls + "/**/" + last_name, recursive=True)
463 if len(selected) == 1:
464 docname = selected[0]
465 path = docname[len(folder):]
466 break
467 if "blob/master/build" in path or "build/notebooks" in path:
468 # raise RuntimeError( # pragma: no cover
469 warnings.warn( # pragma: no cover
470 "Unexpected substring found in %r in folder %r\n"
471 "--TRIED--\n%r" % (path, folder, "\n".join(map(str, tried))))
472 links.append(
473 ":githublink:`GitHub|{0}|*`".format(path.replace("\\", "/").lstrip("/")))
474 lines[pos] = "{0}\n\n.. only:: html\n\n {1}\n\n".format(
475 lines[pos], ", ".join(links))
477 # we remove the
478 # <div
479 # style="position:absolute;
480 # ....
481 # </div>
482 reg = re.compile(
483 "([.]{2} raw[:]{2} html[\\n ]+<div[\\n ]+style=.?position:absolute;(.|\\n)*?[.]{2} raw[:]{2} html[\\n ]+</div>)")
484 merged = "".join(lines)
485 r = reg.findall(merged)
486 if len(r) > 0:
487 fLOG("[post_process_rst_output] *** remove div absolute in ", file)
488 for spa in r:
489 rep = spa[0]
490 nbl = len(rep.split("\n"))
491 merged = merged.replace(rep, "\n" * nbl)
492 lines = [(_ + "\n") for _ in merged.split("\n")]
494 # bullets
495 for pos, line in enumerate(lines):
496 if pos == 0:
497 continue
498 if len(line) > 0 and (line.startswith("- ") or line.startswith("* ")) \
499 and pos < len(lines) - 1:
500 next = lines[pos + 1]
501 prev = lines[pos - 1]
502 if (next.startswith("- ") or next.startswith("* ")) \
503 and not (prev.startswith("- ") or prev.startswith("* ")) \
504 and not prev.startswith(" "):
505 lines[pos - 1] += "\n"
506 elif line.startswith("- ") and next.startswith(" ") \
507 and not prev.startswith(" ") and not prev.startswith("- "):
508 lines[pos - 1] += "\n"
509 elif line.startswith("- "):
510 pass
512 # remove last ::
513 i = len(lines)
514 for i in range(len(lines), 1, -1):
515 s = lines[i - 1].strip(" \n\r")
516 if len(s) != 0 and s != "::":
517 break
519 if i < len(lines):
520 del lines[i:]
522 # specific treatment for notebooks
523 if is_notebook:
524 # change links <#Alink --> <#alink
525 reg = re.compile("(<#[A-Z][a-zA-Z0-9_+-]+>)")
526 for i, line in enumerate(lines):
527 r = reg.search(line)
528 if r:
529 memo = r.groups()[0]
530 new_memo = "<#" + memo[2].lower() + memo[3:]
531 new_memo = new_memo.replace("+", "")
532 line = line.replace(memo, new_memo)
533 lines[i] = line
535 # checking for find://
536 content = "".join(lines)
537 content = update_notebook_link(content, "rst", nblinks=nblinks, fLOG=fLOG)
538 if "find://" in content:
539 raise HelpGenException( # pragma: no cover
540 "find:// was found in '{0}'.\nYou should "
541 "add or extend 'nblinks' in conf.py.".format(file))
543 # notebooks replacements
544 content = _notebook_replacements(content, notebook_replacements, fLOG)
546 # replaces the function
547 content = content.replace("\\mathbb{1}", "\\mathbf{1\\!\\!1}")
549 with open(file, "w", encoding="utf8") as f:
550 f.write(content)
553def post_process_html_output(file, pdf, python, slides, exc=True,
554 nblinks=None, fLOG=None,
555 notebook_replacements=None):
556 """
557 Processes a HTML file generated from the conversion of a notebook.
559 @param file filename
560 @param pdf if True, add a link to the PDF, assuming it will exists
561 at the same location
562 @param python if True, add a link to the Python conversion
563 @param slides if True, add a link to the slides conversion
564 @param exc raises an exception (True) or a warning (False)
565 @param nblinks dictionary ``{ref: url}``
566 @param notebook_replacements string replacement in notebooks
567 @param fLOG logging function
568 """
569 if not os.path.exists(file):
570 raise FileNotFoundError(file) # pragma: no cover
571 if fLOG:
572 fLOG("[post_process_html_output] clean %r" % file)
573 with open(file, "r", encoding="utf8") as f:
574 text = f.read()
576 # mathjax
577 text = text.replace(
578 "https://c328740.ssl.cf1.rackcdn.com/mathjax/latest/MathJax.js?config=TeX-AMS_HTML",
579 "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML")
581 # notebook replacements
582 if fLOG:
583 fLOG("[post_process_html_output] nb:", notebook_replacements)
584 text = _notebook_replacements(text, notebook_replacements, fLOG)
586 text = update_notebook_link(text, "html", nblinks=nblinks, fLOG=fLOG)
587 if "find://" in text:
588 raise HelpGenException( # pragma: no cover
589 "find:// was found in '{0}'.\nYou should add "
590 "or extend 'nblinks' in conf.py.".format(file))
592 # js
593 if fLOG:
594 fLOG("[post_process_html_output] js: replacements")
595 repl = {'https://unpkg.com/@jupyter-widgets/html-manager@^0.20.0/dist/embed-amd.js':
596 '../_static/embed-amd.js'}
597 lines = text.split('\n')
598 new_lines = []
599 for line in lines:
600 if "https://cdnjs.cloudflare.com/ajax/libs/require.js" in line:
601 if fLOG:
602 fLOG("[post_process_html_output] js: skip %r" % line)
603 continue
604 new_lines.append(line)
605 text = "\n".join(new_lines)
606 for k, v in repl.items():
607 if k in text:
608 if fLOG: # pragma: no cover
609 fLOG("[post_process_html_output] js: replace %r -> %r" % (k, v))
610 text = text.replace(k, v)
612 with open(file, "w", encoding="utf8") as f:
613 f.write(text)
616def post_process_slides_output(file, pdf, python, slides, exc=True,
617 nblinks=None, fLOG=None,
618 notebook_replacements=None):
619 """
620 Processes a :epkg:`HTML` file generated from the conversion of a notebook.
622 @param file filename
623 @param pdf if True, add a link to the PDF, assuming it will
624 exists at the same location
625 @param python if True, add a link to the Python conversion
626 @param slides if True, add a link to the slides conversion
627 @param exc raises an exception (True) or a warning (False)
628 @param nblinks dictionary ``{ref: url}``
629 @param notebook_replacements string replacement in notebooks
630 @param fLOG logging function
631 """
632 if (len(file) > 5000 or not os.path.exists(file)) and "<html" in file:
633 text = file # pragma: no cover
634 save = False # pragma: no cover
635 else:
636 if not os.path.exists(file):
637 raise FileNotFoundError(file) # pragma: no cover
638 if fLOG:
639 fLOG("[post_process_slides_output] clean %r" % file)
640 # fold, name = os.path.split(file)
641 with open(file, "r", encoding="utf8") as f:
642 text = f.read()
643 save = True
645 # reveal.js
646 require = "require(" in text
647 text = text.replace("reveal.js/dist/reveal.css",
648 "reveal.js/css/reveal.css")
649 text = text.replace("reveal.js/dist/theme/simple.css",
650 "reveal.js/css/theme/simple.css")
651 text = text.replace("https://unpkg.com/@jupyter-widgets/html-manager@0.20.0/dist/embed-amd.js",
652 "embed-amd.js")
653 lines = text.split("\n")
654 for i, line in enumerate(lines):
655 if '<script src="reveal.js/lib/js/head.min.js"></script>' in line:
656 lines[i] = (
657 '<script src="reveal.js/js/jquery.min.js"></script>\n' + lines[i])
658 if '<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js"></script>' in line:
659 lines[i] = ""
660 if '<script src="https://cdnjs.cloudflare.com/ajax/libs/require.js/' in line:
661 lines[i] = ""
662 if lines[i] == "</script>" and require:
663 lines[i] += '\n<script src="require.js"></script>'
664 require = False
665 text = "\n".join(lines)
667 # mathjax
668 text = text.replace("https://c328740.ssl.cf1.rackcdn.com/mathjax/latest/MathJax.js?config=TeX-AMS_HTML",
669 "https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS_HTML")
670 text = update_notebook_link(text, "slides", nblinks=nblinks, fLOG=fLOG)
671 if "find://" in text:
672 raise HelpGenException( # pragma: no cover
673 "find:// was found in '{0}'.\nYou should add "
674 "or extend 'nblinks' in conf.py.".format(file))
676 # notebook replacements
677 text = _notebook_replacements(text, notebook_replacements, fLOG)
679 if save:
680 with open(file, "w", encoding="utf8") as f:
681 f.write(text)
682 return text
685def post_process_latex(st, doall, info=None, latex_book=False, exc=True,
686 custom_latex_processing=None, nblinks=None, file=None,
687 remove_unicode=False, fLOG=None, notebook_replacements=None):
688 """
689 Modifies a :epkg:`latex` file after its generation by :epkg:`sphinx`.
691 @param st string
692 @param doall do all transformations
693 @param info for more understandable error messages
694 @param latex_book customized for a book
695 @param exc raises an exception or a warning
696 @param custom_latex_processing function which takes and returns a string,
697 final post processing
698 @param nblinks dictionary ``{ref: url}``
699 @param file only used when an exception is raised
700 @param remove_unicode remove unicode character (fails when converting into PDF)
701 @param notebook_replacements string replacement in notebooks
702 @param fLOG logging function
703 @return string
705 *SVG* included in a notebook (or in *RST* file) usually do not word.
706 :epkg:`Inkscape` should be used to convert them into Latex.
707 The function is less strict on the checking of `$`.
708 The function replaces ``\\mathbb{1}`` by ``\\mathbf{1\\!\\!1}``.
710 .. index:: chinese characters, latex, unicode
712 .. faqref::
713 :title: Why a ¿ is showing the final PDF?
715 Unicode, chinese characters are an issue because the latex compiler
716 prompts on those if the necessary packages are not installed.
717 `pdflatex <https://en.wikipedia.org/w/index.php?title=PdfTeX&redirect=no>`_
718 does not accepts inline chinese
719 characters, `xetex <https://en.wikipedia.org/wiki/XeTeX>`_
720 should be used instead:
721 see `How to input Traditional Chinese in pdfLaTeX
722 <http://tex.stackexchange.com/questions/200449/how-to-input-traditional-chinese-in-pdflatex>`_.
723 Until this is being implemented, the unicode will unfortunately be removed
724 in this function.
726 .. versionchanged:: 1.9
727 Removes the uses of package `parskip
728 <https://ctan.org/pkg/parskip?lang=en>`_, it uses command
729 ``\\DeclareRelease`` which is not always recognized.
730 """
731 if fLOG:
732 fLOG("[post_process_latex] ** enter post_process_latex",
733 doall, "%post_process_latex" in st)
734 weird_character = set(chr(i) for i in range(1, 9))
736 def clean_unicode(c):
737 if c == "’":
738 return "'"
739 if c == "…":
740 return "..."
741 if ord(c) >= 255 or c in weird_character:
742 return "\\textquestiondown "
743 return c
745 def clean_line(line):
746 if line.startswith("\\documentclass"):
747 line = line.replace("{None}", "{report}")
748 return line
750 lines = st.split("\n")
751 lines = list(map(clean_line, lines))
752 st = "\n".join("".join(map(clean_unicode, line)) for line in lines)
754 # we count the number of times we have \$ (which is unexpected unless the
755 # currency is used.
756 dollar = st.split("\\$")
757 if len(dollar) > 0 and (
758 info is None or os.path.splitext(info)[-1] != ".html"):
759 # it could be an issue, for the time being, we raise
760 # an exception if a formula is too long
761 exp = re.compile(r"(.{200}[\\]\$\$)")
762 found = 0
763 records = []
764 for m in exp.finditer(st):
765 found += 1
766 p1, p2 = m.start(), m.end()
767 sub = st[p1:p2].strip(" \r\n").replace(
768 "\n", " ").replace("\r", "").replace("\t", " ")
769 sub2 = sub[-10:]
770 records.append((info, p1, p2, sub, sub2, ""))
771 if len(records) > 0: # pragma: no cover
772 messages = [str(i) + ":" + ("unexpected \\$ in a latex file:\n {0}\n" +
773 "at position: {1},{2}\n substring: {3}\n " +
774 "around: {4}\n temp=[{5}]").format(*rec)
775 for i, rec in enumerate(records)]
776 for mes in messages:
777 warnings.warn(mes, UserWarning)
779 st = st.replace("<br />", "\\\\")
780 st = st.replace("»", '"')
781 st = st.replace("\\mathbb{1}", "\\mathbf{1\\!\\!1}")
782 st = st.replace(
783 "\\documentclass[11pt]{article}", "\\documentclass[10pt]{article}")
785 if not doall and not latex_book:
786 st = st.replace(
787 "\\maketitle", "\\maketitle\n\n\\newchapter{Introduction}")
789 st = st.replace("%5C", "/") \
790 .replace("%3A", ":") \
791 .replace("\\includegraphics{notebooks\\", "\\includegraphics {")
792 st = st.replace(
793 "\\begin{document}", "\\setlength{\\parindent}{0cm}%s\\begin {document}" % "\n")
794 st = st.replace("DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\\\\{\\}}",
795 "DefineVerbatimEnvironment{Highlighting}{Verbatim} {commandchars=\\\\\\{\\},fontsize=\\small}")
796 st = st.replace("\\textquotesingle{}", "'")
797 st = st.replace("\u0001", "\\u1")
798 st = st.replace("\\begin{notice}{note}\\end{notice}", "")
800 # hyperref
801 if doall and "%post_process_latex" not in st:
802 st = "%post_process_latex\n" + st
803 reg = re.compile("hyperref[\\[]([a-zA-Z0-9]+)[\\]][\\{](.*?)[\\}]")
804 allhyp = reg.findall(st)
805 sections = []
806 for id, section in allhyp:
807 sec = r"\subsection{%s} \label{%s}" % (section, id)
808 sections.append((id, section, sec))
809 elif not doall and not latex_book:
810 sections = []
811 # first section
812 lines = st.split("\n")
813 for i, line in enumerate(lines):
814 if "\\section" in line:
815 lines[i] = "\\newchapter{Documentation}\n" + lines[i]
816 break
817 st = "\n".join(lines)
818 else:
819 sections = []
821 if len(sections) > 0:
822 lines = st.split("\n")
823 for i, line in enumerate(lines):
824 for _, section, sec in sections:
825 if line.strip("\r\n ") == section:
826 fLOG(" **", section, " --> ", sec)
827 lines[i] = sec
828 st = "\n".join(lines)
830 if not latex_book:
831 st = st.replace("\\chapter", "\\section")
832 st = st.replace("\\newchapter", "\\chapter")
834 comment_out = [
835 '\\usepackage{parskip}',
836 '\\usepackage{fontspec}',
837 '\\begin{center}\\rule{0.5\\linewidth}{\\linethickness}\\end{center}\n',
838 ]
839 for co in comment_out:
840 if co in st:
841 st = st.replace(co, "%" + co)
842 if "\\usepackage{multirow}" in st:
843 st = st.replace(
844 "\\usepackage{svg}\\usepackage{multirow}",
845 "\\usepackage{multirow}\\usepackage{amssymb}\\usepackage{latexsym}\\usepackage{amsfonts}\\usepackage{ulem}\\usepackage{textcomp}")
846 elif "\\usepackage{hyperref}" in st:
847 st = st.replace(
848 "\\usepackage{svg}\\usepackage{hyperref}",
849 "\\usepackage{hyperref}\\usepackage{amssymb}\\usepackage{latexsym}\\usepackage{amsfonts}\\usepackage{ulem}\\usepackage{textcomp}")
850 else:
851 raise HelpGenException( # pragma: no cover
852 "unable to add new instructions usepackage in file {0}".format(info))
854 # SVG does not work unless it is converted (nbconvert should handle that
855 # case)
856 reg = re.compile("([\\\\]includegraphics[{].*?[.]svg[}])")
857 fall = reg.findall(st)
858 for found in fall:
859 st = st.replace(found, "%" + found)
861 # fix references
862 st = update_notebook_link(st, "latex", nblinks=nblinks, fLOG=fLOG)
863 if "find://" in st:
864 raise HelpGenException( # pragma: no cover
865 "find:// was found in '{0}'\nYou should add or extend "
866 "'nblinks' in conf.py.\n{1}".format(file, st))
868 # notebook replacements
869 st = _notebook_replacements(st, notebook_replacements, fLOG)
871 # end
872 if custom_latex_processing is not None:
873 st = custom_latex_processing(st) # pragma: no cover
875 if remove_unicode:
876 encoding = 'ascii'
877 else:
878 encoding = 'utf-8'
879 st0 = st
880 bst = st.encode(encoding, errors='replace')
881 st = bst.decode(encoding, errors='replace')
882 if st0 != st and fLOG:
883 fLOG("[post_process_latex] characters were removed for encoding", encoding)
884 return st
887def post_process_python(st, doall, info=None, nblinks=None, file=None, fLOG=None, notebook_replacements=None):
888 """
889 Modifies a python file after its generation by :epkg:`sphinx`.
891 @param st string
892 @param doall do all transformations
893 @param info for more understandable error messages
894 @param nblinks dictionary ``{ref: url}``
895 @param file used only when an exception is raised
896 @param fLOG logging function
897 @param notebook_replacements string replacement in notebooks
898 @return string
899 """
900 st = st.strip("\n \r\t")
901 st = st.replace("# coding: utf-8", "# -*- coding: utf-8 -*-")
902 st = update_notebook_link(st, "python", nblinks=nblinks, fLOG=fLOG)
903 if "find://" in st:
904 raise HelpGenException( # pragma: no cover
905 "find:// was found in '{0}'.\nYou should add or extend "
906 "'nblinks' in conf.py.".format(file))
908 # notebook replacements
909 st = _notebook_replacements(st, notebook_replacements, fLOG)
911 return st
914def remove_character_under32(s):
915 """
916 Removes :epkg:`ASCII` characters in *[0..31]*.
918 @param s string to process
919 @return filtered string
920 """
921 ls = ""
922 for c in s:
923 d = ord(c)
924 if 0 <= d < 32:
925 ls += " "
926 else:
927 ls += c
928 return ls