Coverage for pyquickhelper/helpgen/sphinx_main.py: 80%
568 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains the main function to generate the documentation
5for a module designed the same way as this one, @see fn generate_help_sphinx.
6"""
7import os
8import sys
9import shutil
10import warnings
11from datetime import datetime
12from io import StringIO
13from docutils.parsers.rst import directives, roles
14from sphinx.cmd.build import main as build_main
15from ..filehelper import remove_folder
16from ..loghelper import python_path_append
17from ..loghelper.process_script import execute_script_get_local_variables, dictionary_as_class
18from ..loghelper.flog import run_cmd, fLOG
19from .utils_sphinx_doc import prepare_file_for_sphinx_help_generation
20from .utils_sphinx_doc_helpers import HelpGenException, ImportErrorHelpGen
21from .conf_path_tools import find_latex_path, find_pandoc_path
22from ..filehelper.synchelper import explore_folder
23from ..filehelper import synchronize_folder
24from .post_process import post_process_latex_output
25from .process_notebooks import process_notebooks, build_notebooks_gallery, build_all_notebooks_coverage
26from .sphinx_helper import post_process_html_nb_output_static_file
27from .install_js_dep import install_javascript_tools
28from .sphinx_main_helper import setup_environment_for_help, get_executables_path, generate_changes_repo
29from .sphinx_main_helper import compile_latex_output_final, replace_placeholder_by_recent_blogpost
30from .sphinx_main_helper import format_history, enumerate_copy_images_for_slides
31from .sphinx_main_verification import verification_html_format
32from .sphinx_main_missing_html_files import add_missing_files
33from .style_css_template import style_figure_notebook
34from .post_process_custom import find_custom_latex_processing
35from ..sphinxext.blog_post_list import BlogPostList
36from ..sphinxext.sphinx_blog_extension import BlogPostDirective, BlogPostDirectiveAgg
37from ..sphinxext.sphinx_runpython_extension import RunPythonDirective
38from ..sphinxext.sphinx_postcontents_extension import PostContentsDirective
39from ..sphinxext.sphinx_tocdelay_extension import TocDelayDirective
40from ..sphinxext.sphinx_youtube_extension import YoutubeDirective
41from ..sphinxext.sphinx_sharenet_extension import ShareNetDirective, sharenet_role
42from ..sphinxext.sphinx_downloadlink_extension import process_downloadlink_role
43from ..sphinxext.sphinx_video_extension import VideoDirective
44from ..sphinxext.sphinx_image_extension import SimpleImageDirective
45from ..sphinxext.sphinximages.sphinxtrib.images import ImageDirective
46from ..sphinxext.sphinx_template_extension import tpl_role
47from ..sphinxext.sphinx_epkg_extension import epkg_role
48from ..sphinxext.sphinx_bigger_extension import bigger_role
49from ..sphinxext.sphinx_githublink_extension import githublink_role
50from ..sphinxext.sphinx_gitlog_extension import gitlog_role
51from ..sphinxext.sphinx_mathdef_extension import MathDef
52from ..sphinxext.sphinx_quote_extension import QuoteNode
53from ..sphinxext.sphinx_blocref_extension import BlocRef
54from ..sphinxext.sphinx_exref_extension import ExRef
55from ..sphinxext.sphinx_faqref_extension import FaqRef
56from ..sphinxext.sphinx_nbref_extension import NbRef
57from ..sphinxext.sphinx_cmdref_extension import CmdRef
58from ..sphinxext.sphinx_todoext_extension import TodoExt
59from ..sphinxext.sphinx_collapse_extension import CollapseDirective
60from ..sphinxext.sphinx_gdot_extension import GDotDirective
62template_examples = """
64List of programs
65++++++++++++++++
67.. toctree::
68 :maxdepth: 2
70.. autosummary:: __init__.py
71 :toctree: %s/
72 :template: modules.rst
74Another list
75++++++++++++
77"""
80def generate_help_sphinx(project_var_name, clean=False, root=".",
81 filter_commit=lambda c: c.strip() != "documentation",
82 extra_ext=None,
83 nbformats=("ipynb", "slides", "html", "python",
84 "rst", "pdf", "github"),
85 layout=None,
86 module_name=None, from_repo=True, add_htmlhelp=False,
87 copy_add_ext=None, direct_call=False, fLOG=fLOG,
88 parallel=1, extra_paths=None, fexclude=None):
89 """
90 Runs the help generation:
92 - copies every file in another folder,
93 - replaces comments in doxygen format into rst format,
94 - replaces local import by global import (tweaking sys.path too),
95 - calls sphinx to generate the documentation.
97 @param project_var_name project name
98 @param clean if True, cleans the previous documentation first
99 (:epkg:`html` files)
100 @param root see below
101 @param filter_commit function which accepts a commit to show on the documentation
102 (based on the comment)
103 @param extra_ext list of file extensions to document (not .py)
104 @param nbformats requested formats for the notebooks conversion
105 @param layout list of formats sphinx should generate such as html, latex, pdf, docx,
106 it is a list of tuple (layout, build directory, parameters to override),
107 if None --> ``[("html", "build", {})]``
108 @param module_name name of the module (must be the folder name ``src/module_name``
109 if None, ``module_name``
110 will be replaced by *project_var_name*
111 @param from_repo if True, assumes the sources come from a source repository,
112 False otherwise
113 @param add_htmlhelp run :epkg:`HTML` Help too (only on :epkg:`Windows`)
114 @param copy_add_ext additional file extension to copy
115 @param direct_call direct call to sphinx with *sphinx_build* if *True*
116 or run a command line in an another process to get
117 a clear environment
118 @param parallel degree of parallelization
119 @param extra_paths extra paths when importing configuration
120 @param fexclude function which tells which file not to copy in the folder
121 used to build the documentation
122 @param fLOG logging function
124 The result is stored in path: ``root/_doc/sphinxdoc/source``.
125 We assume the file ``root/_doc/sphinxdoc/source/conf.py`` exists
126 as well as ``root/_doc/sphinxdoc/source/index.rst``.
128 If you generate latex/pdf files, you should add variables ``latex_path`` and ``pandoc_path``
129 in your file ``conf.py`` which defines the help.
131 You can exclud some part while generating the documentation by adding:
133 * ``# -- HELP BEGIN EXCLUDE --``
134 * ``# -- HELP END EXCLUDE --``
136 ::
138 latex_path = r"C:/Program Files/MiKTeX 2.9/miktex/bin/x64"
139 pandoc_path = r"%USERPROFILE%/AppData/Local/Pandoc"
141 .. exref::
142 :title: Run help generation
143 :index: extension, extra extension, ext
145 ::
147 # from the main folder which contains folder src or the sources
148 generate_help_sphinx("pyquickhelper")
150 By default, the function only consider files end by ``.py`` and ``.rst`` but you could
151 add other files sharing the same extensions by adding this one
152 in the ``extra_ext`` list.
154 The function requires:
156 - :epkg:`pandoc`
157 - latex
159 @warning Some themes such as `Bootstrap Sphinx Theme <http://ryan-roemer.github.io/sphinx-bootstrap-theme/>`_
160 do not work on Internet Explorer. In that case, the
161 file ``<python_path>/Lib/site-packages/sphinx/themes/basic/layout.html``
162 must be modified to add the following line (just below ``Content-Type``).
164 ::
166 <meta http-equiv="X-UA-Compatible" content="IE=edge" />
168 .. index:: PEP8, autopep8
170 The code should follow as much as possible the sytle convention
171 `PEP8 <https://www.python.org/dev/peps/pep-0008/>`_.
172 The module `autopep8 <https://pypi.python.org/pypi/autopep8>`_ can modify a file or all files contained in one folder
173 by running the following command line:
175 ::
177 autopep8 <folder> --recursive --in-place --pep8-passes 2000 --verbose
179 **About encoding:** utf-8 without BOM is the recommanded option.
181 **About languages:** only one language can be specificied even if you have
182 multiple configuration file. Only the language specified in the main
183 ``conf.py`` will be considered.
185 **About blog posts:** the function uses sphinx directives ``blogpost`` and ``blogpostagg`` to create
186 a simple blog aggregator. Blog posts will be aggregated by months and categories.
187 Link to others parts to the documentation are possible.
188 The function also create a file ``rss.xml`` which contains the ten last added blog post.
189 This file contains an absolute link to the blog posts. However, because the documentation
190 can be published anywhere, the string ``__BLOG_ROOT__`` was inserted
191 instead of the absolute url to the website. It must be replaced before uploaded
192 or the parameter *blog_root* can be specified in the configuration file ``conf.py``.
194 @warning Parameter *add_htmlhelp* calls `Html Help WorkShop
195 <https://msdn.microsoft.com/en-us/library/windows/desktop/ms669985%28v=vs.85%29.aspx>`_.
196 It also changes the encoding of the HTMLoutput into cp1552 (encoding for Windows)
197 instead of utf-8.
199 @warning An issue was raised on Linux due to the use of ``.. only:: html``
200 (``AttributeError: Can't pickle local object 'setup.<locals>.<lambda>'``).
201 It disappeared when using only one thread and not 2 as
202 it was previously. Parameter *parallel* was introduced to
203 make that change and the default value is not 1.
205 .. index:: SVG, Inkscape
207 **Others necessary tools:**
209 SVG included in a notebook (or any RST file) requires `Inkscape <https://inkscape.org/>`_
210 to be converted into Latex.
212 .. faqref::
213 :title: How to dd an extra layer to the documentation?
215 The following `commit <https://github.com/sdpython/python3_module_template/commit/
216 75d765a293f65a37b3208601d17d3b0daa891af6>`_
217 on project `python3_module_template <https://github.com/sdpython/python3_module_template/>`_
218 shows which changes needs to be done to add an extra layer of for the documentation.
220 The function assumes :epkg:`IPython` 3 is installed.
221 It might no work for earlier versions (notebooks).
222 Parameters *from_repo*, *use_run_cmd* were added.
223 Notebook conversion to slides is implemented,
224 install :epkg:`reveal.js` if not installed.
225 Parameter *add_htmlhelp* was added. It runs HtmlHelp on Windows ::
227 "C:\\Program Files (x86)\\HTML Help Workshop\\hhc.exe" build\\htmlhelp\\<module>.hhp
229 The documentation includes blog (with sphinx command ``.. blogpost::``
230 and python scripts ``.. runpython::``. The second command runs a python
231 script which outputs RST documntation adds it to the current documentation.
232 The function automatically adds custom role and custom directive ``sharenet``.
233 The function directly calls
234 `sphinx <https://www.sphinx-doc.org/en/master/>`_,
235 `nbconvert <https://nbconvert.readthedocs.io/en/latest/>`_.
236 When there are too many notebooks, the notebook index is difficult to read.
237 It does not require to get script location.
238 Not enough stable from virtual environment.
240 Set ``BOKEH_DOCS_MISSING_API_KEY_OK`` to 1.
241 bokeh sphinx extension requires that or a key for the google API (???).
242 The function was updated to use Sphinx 1.6.2.
243 However, you should read blog post
244 :ref:`Bug in Sphinx 1.6.2 for custom css <sphinx-162-bug-custom-css>`
245 if you have any trouble with custom css.
246 Add a report in ``all_notebooks.rst`` about notebook coverage.
247 Parameter *parallel* was added.
248 The parameter *nblayout* in the configuration file specifies
249 the layout for the notebook gallery. ``'classic'`` or ``'table'``.
250 The parameter *nbneg_pattern* can be used to remove notebooks from
251 the gallery if they match this regular expression.
252 It automatically adds video and image directives.
253 *remove_unicode* can set to False or True in the documentation
254 configuration file to allow or remove unicode characters
255 before compiling the latex output.
256 Import ``conf.py`` in a separate process before running
257 the generation of the documentation. Do not import it
258 directly.
259 """
260 datetime_rows = [("begin", datetime.now())]
262 fLOG("---- JENKINS BEGIN DOCUMENTATION ----")
263 if layout is None:
264 layout = [("html", "build", {})]
265 fLOG("[generate_help_sphinx] ---- layout", layout)
266 setup_environment_for_help(fLOG=fLOG)
267 # we keep a clean list of modules
268 # sphinx configuration is a module and the function loads and unloads it
269 list_modules_start = set(sys.modules.keys())
271 if add_htmlhelp: # pragma: no cover
272 if not sys.platform.startswith("win"):
273 raise ValueError("add_htmlhelp is True and the OS is not Windows")
274 fLOG("[generate_help_sphinx] add add_htmlhelp")
276 if extra_ext is None:
277 extra_ext = []
279 def lay_build_override_newconf(t3):
280 if isinstance(t3, str):
281 lay, build, override, newconf = t3, "build", {}, None
282 elif len(t3) == 1:
283 lay, build, override, newconf = t3[0], "build", {}, None
284 elif len(t3) == 2:
285 lay, build, override, newconf = t3[0], t3[1], {}, None
286 elif len(t3) == 3:
287 lay, build, override, newconf = t3[0], t3[1], t3[2], None
288 else:
289 lay, build, override, newconf = t3
290 return lay, build, override, newconf
292 directives.register_directive("blogpost", BlogPostDirective)
293 directives.register_directive("blogpostagg", BlogPostDirectiveAgg)
294 directives.register_directive("runpython", RunPythonDirective)
295 directives.register_directive("sharenet", ShareNetDirective)
296 directives.register_directive("video", VideoDirective)
297 directives.register_directive("simpleimage", SimpleImageDirective)
298 directives.register_directive("image", ImageDirective)
299 directives.register_directive("todoext", TodoExt)
300 directives.register_directive("mathdef", MathDef)
301 directives.register_directive("quote", QuoteNode)
302 directives.register_directive("blocref", BlocRef)
303 directives.register_directive("exref", ExRef)
304 directives.register_directive("faqref", FaqRef)
305 directives.register_directive("nbref", NbRef)
306 directives.register_directive("cmdref", CmdRef)
307 directives.register_directive("postcontents", PostContentsDirective)
308 directives.register_directive("tocdelay", TocDelayDirective)
309 directives.register_directive("youtube", YoutubeDirective)
310 directives.register_directive("thumbnail", ImageDirective)
311 directives.register_directive("collapse", CollapseDirective)
312 directives.register_directive("gdot", GDotDirective)
313 roles.register_canonical_role("sharenet", sharenet_role)
314 roles.register_canonical_role("bigger", bigger_role)
315 roles.register_canonical_role("githublink", githublink_role)
316 roles.register_canonical_role("gitlog", gitlog_role)
317 roles.register_canonical_role("tpl", tpl_role)
318 roles.register_canonical_role("epkg", epkg_role)
319 roles.register_canonical_role("downloadlink", process_downloadlink_role)
321 if "conf" in sys.modules:
322 raise ImportError( # pragma: no cover
323 "module conf was imported, this function expects not to:\n{0}".format(
324 sys.modules["conf"].__file__))
326 ############
327 # root_source
328 ############
329 root = os.path.abspath(root)
330 froot = root
331 root_sphinxdoc = os.path.join(root, "_doc", "sphinxdoc")
332 root_source = os.path.join(root_sphinxdoc, "source")
333 root_package = os.path.join(root, "src")
334 if not os.path.exists(root_package):
335 root_package = root
336 if not os.path.exists(root_package):
337 raise FileNotFoundError( # pragma: no cover
338 f"Unable to find source root from '{root}'.")
339 fLOG(f"[generate_help_sphinx] root='{root}'")
340 fLOG(f"[generate_help_sphinx] root_package='{root_package}'")
341 fLOG(f"[generate_help_sphinx] root_source='{root_source}'")
342 fLOG(f"[generate_help_sphinx] root_sphinxdoc='{root_sphinxdoc}'")
343 conf_paths = [root_source, root_package]
344 if extra_paths:
345 conf_paths.extend(extra_paths)
347 ########################################
348 # we import conf_base, specific to multi layers
349 ########################################
350 confb = os.path.join(root_source, "conf_base.py")
351 if os.path.exists(confb): # pragma: no cover
352 code = "from conf_base import *"
353 with python_path_append(conf_paths):
354 try:
355 module_conf = execute_script_get_local_variables(
356 code, folder=root_source, check=True)
357 except RuntimeError as e:
358 raise ImportError("Unable to import conf_base '{}' from '{}'\nsys.path=\n{}".format(
359 confb, root_source, "\n".join(sys.path))) from e
361 if module_conf is None:
362 raise ImportError(
363 f"Unable to import '{confb}' which defines the help generation")
364 if 'ERROR' in module_conf:
365 msg = "\n".join(["paths:"] + conf_paths + [
366 "-----------------------",
367 module_conf['ERROR']])
368 raise ImportError(msg)
369 conf_base = dictionary_as_class(module_conf)
370 fLOG("[generate_help_sphinx] conf_base.__file__='{0}'".format(
371 os.path.abspath(conf_base.__file__))) # pylint: disable=E1101
373 copypath = list(sys.path)
375 # stores static path for every layout, we store them to copy
376 html_static_paths = []
377 build_paths = []
378 all_tocs = []
379 parameters = []
381 ###################################
382 # import others conf, we must do it now
383 # it takes too long to do it after if there is an error
384 # we assume the configuration are not too different
385 # about language for example, latex_path, pandoc_path
386 #################################################
387 for t3 in layout:
388 lay, build, override, newconf = lay_build_override_newconf(t3)
389 if newconf is None:
390 continue
391 fLOG(f"[generate_help_sphinx] newconf: '{newconf}' - {t3}")
392 # we need to import this file to guess the template directory and
393 # add missing templates
394 folds = os.path.join(root_sphinxdoc, newconf)
395 _import_conf_extract_parameter(root, root_source, folds, build, newconf,
396 all_tocs, build_paths, parameters,
397 html_static_paths, fLOG)
399 ################################################################
400 # we add the source path to the list of path to considered before importing
401 # import conf.py
402 ################################################################
403 with python_path_append(conf_paths):
404 try:
405 module_conf = execute_script_get_local_variables(
406 "from conf import *", folder=root_source, check=True)
407 except ImportError as e: # pragma: no cover
408 raise ImportError("Unable to import 'conf.py' from '{0}', sys.path=\n{1}\nBEFORE:\n{2}".format(
409 root_source, "\n".join(sys.path), "\n".join(copypath))) from e
410 if module_conf is None:
411 raise ImportError( # pragma: no cover
412 "unable to import 'conf.py' which defines the help generation")
413 if 'ERROR' in module_conf: # pragma: no cover
414 msg = "\n".join(["paths:"] + conf_paths + [
415 "----------------------- ERROR:",
416 module_conf['ERROR'],
417 "------------------------ root_source:",
418 root_source])
419 raise ImportError(msg)
420 if len(module_conf) == 0:
421 raise ImportError(
422 "No extracted local variable.") # pragma: no cover
423 theconf = dictionary_as_class(module_conf)
424 fLOG("[generate_help_sphinx] conf.__file__='{0}'".format(
425 os.path.abspath(theconf.__file__))) # pylint: disable=E1101
426 tocs = add_missing_files(root, theconf, "__INSERT__", fLOG)
427 all_tocs.extend(tocs)
429 ##############################
430 # some checkings on the configuration
431 ##############################
432 galleries = _check_sphinx_configuration(theconf, fLOG)
433 galleries = [os.path.relpath(g, start=root_source) for g in galleries]
435 ##############################################################
436 # Extracts variables from the configuration.
437 # We store the html_static_path in html_static_paths for the base conf
438 # We extract other information from the configuration
439 ##############################################################
440 html_static_path = theconf.__dict__.get("html_static_path", "_static")
441 if isinstance(html_static_path, list):
442 html_static_path = html_static_path[0]
443 html_static_path = os.path.join(root_source, html_static_path)
444 if not os.path.exists(html_static_path):
445 raise FileNotFoundError( # pragma: no cover
446 f"no static path: {html_static_path!r}.")
447 html_static_paths.append(html_static_path)
448 build_paths.append(
449 os.path.normpath(os.path.join(html_static_path, "..", "..", "build", "html")))
450 custom_latex_processing = theconf.__dict__.get(
451 "custom_latex_processing", None)
452 if custom_latex_processing is not None: # pragma: no cover
453 # The configuration file is pickled by sphinx
454 # and parameter should not be functions.
455 if isinstance(custom_latex_processing, str):
456 custom_latex_processing = find_custom_latex_processing( # pylint: disable=E1111
457 custom_latex_processing)
458 res = custom_latex_processing("dummy phrase")
459 if res is None:
460 raise ValueError(
461 "Result of function custom_latex_processing should not be None.")
462 remove_unicode = theconf.__dict__.get("remove_unicode", False)
463 snippet_folder = theconf.__dict__.get(
464 "notebook_custom_snippet_folder", None)
465 if snippet_folder:
466 snippet_folder = os.path.join(
467 os.path.dirname(theconf.__file__), snippet_folder) # pylint: disable=E1101
469 notebook_replacements = theconf.__dict__.get("notebook_replacements", None)
470 if notebook_replacements is not None and not isinstance(notebook_replacements, dict):
471 raise TypeError( # pragma: no cover
472 "latex_notebook_replacements should be a dictionary not {0}".format(
473 type(notebook_replacements)))
475 ####################################
476 # modifies the version number in conf.py
477 ####################################
478 readme = os.path.join(root, "README.rst")
479 if not os.path.exists(readme):
480 raise FileNotFoundError(readme) # pragma: no cover
481 shutil.copy(readme, root_source)
482 license = os.path.join(root, "LICENSE.txt")
483 if not os.path.exists(license):
484 raise FileNotFoundError(license) # pragma: no cover
485 shutil.copy(license, root_source)
486 history = os.path.join(root, "HISTORY.rst")
487 if os.path.exists(history):
488 dest = os.path.join(root_source, "HISTORY.rst")
489 format_history(history, dest)
491 ##########
492 # language
493 ##########
494 language = theconf.__dict__.get("language", "en")
495 use_sys = theconf.__dict__.get("enable_disabled_parts", None)
496 latex_book = theconf.__dict__.get('latex_book', False)
497 nbexamples_conf = theconf.__dict__.get('example_gallery_config', None)
498 # examples_conf = theconf.__dict__.get('sphinx_gallery_conf', None)
500 ##########
501 # auto_rst_generation
502 ##########
503 auto_rst_generation = theconf.__dict__.get("auto_rst_generation", True)
505 ospath = os.environ["PATH"]
506 latex_path = theconf.__dict__.get("latex_path", find_latex_path())
507 # graphviz_dot = theconf.__dict__.get("graphviz_dot", find_graphviz_dot())
508 pandoc_path = theconf.__dict__.get("pandoc_path", find_pandoc_path())
509 if os.path.isfile(latex_path):
510 latex_path = os.path.dirname(latex_path)
512 ##########
513 # nblinks: references for the notebooks, dictionary {(ref, format): link}
514 ##########
515 nblayout = theconf.__dict__.get("nblayout", "classic")
516 nblinks = theconf.__dict__.get("nblinks", None)
517 nbneg_pattern = theconf.__dict__.get("nbneg_pattern", None)
518 if nblinks is not None and len(nblinks) > 0:
519 fLOG("[generate_help_sphinx] NBLINKS - BEGIN")
520 for i, (k, v) in enumerate(sorted(nblinks.items())):
521 fLOG(f" {i + 1}/{len(nblinks)} - '{k}': '{v}'")
522 fLOG("[generate_help_sphinx] NBLINKS - END")
523 fLOG(f"[generate_help_sphinx] nbneg_pattern='{nbneg_pattern}'")
525 # add to PATH
526 sep = ";" if sys.platform.startswith("win") else ":"
527 if latex_path not in ospath:
528 os.environ["PATH"] += sep + latex_path
529 if pandoc_path not in ospath:
530 os.environ["PATH"] += sep + pandoc_path
532 #########
533 # changes
534 #########
535 datetime_rows = [("changes", datetime.now())]
536 chan = os.path.join(root, "_doc", "sphinxdoc", "source", "filechanges.rst")
537 if "modify_commit" in theconf.__dict__:
538 modify_commit = theconf.modify_commit # pylint: disable=E1101
539 else:
540 modify_commit = None
541 generate_changes_repo(
542 chan, root, filter_commit=filter_commit, exception_if_empty=from_repo,
543 fLOG=fLOG, modify_commit=modify_commit)
545 ######################################
546 # we copy javascript dependencies, reveal.js
547 ######################################
548 datetime_rows = [("javascript", datetime.now())]
549 fLOG("[generate_help_sphinx] JAVASCRIPT:", html_static_paths)
550 fLOG("[generate_help_sphinx] ROOT:", root_sphinxdoc)
551 fLOG("[generate_help_sphinx] BUILD:", build_paths)
552 for html_static_path in html_static_paths:
553 found = install_javascript_tools(
554 root_sphinxdoc, dest=html_static_path, fLOG=fLOG)
555 fLOG(f"[generate_help_sphinx] [javascript]: '{found}'")
557 ############################
558 # we copy the extended styles (notebook, snippets)
559 ############################
560 datetime_rows = [("copy", datetime.now())]
561 for html_static_path in html_static_paths:
562 dest = os.path.join(html_static_path, style_figure_notebook[0])
563 fLOG(" CREATE-CSS", dest)
564 with open(dest, "w", encoding="utf-8") as f:
565 f.write(style_figure_notebook[1])
567 # We should not need that.
568 # for build in build_paths:
569 # dest = os.path.join(build, "_downloads")
570 # if not os.path.exists(dest):
571 # os.makedirs(dest)
572 # install_javascript_tools(
573 # root_sphinxdoc, dest=dest, fLOG=fLOG)
575 ##############
576 # copy the files
577 ##############
578 fLOG("---- JENKINS BEGIN DOCUMENTATION COPY FILES ----")
579 optional_dirs = []
580 mapped_function = [(f".*[.]{ext.strip('.')}$", None)
581 for ext in extra_ext]
583 ###################################
584 # we save the module already imported
585 ###################################
586 if module_name is None:
587 module_name = project_var_name
589 sys_modules = set(sys.modules.keys())
591 ####################
592 # generates extra files
593 ####################
594 datetime_rows = [("prepare", datetime.now())]
595 try:
596 dest_doc = os.path.join(root, "_doc", "sphinxdoc", "source")
597 fLOG(f"[generate_help_sphinx] module_name='{module_name}'")
598 fLOG(f"[generate_help_sphinx] project_var_name='{project_var_name}'")
599 fLOG(
600 f"[generate_help_sphinx] root='{root}' root_package='{root_package}'")
601 fLOG(f"[generate_help_sphinx] dest_doc='{dest_doc}'")
602 subfolders = []
603 if root_package.endswith("src"):
604 subfolders.append(("src/" + module_name, module_name))
605 else:
606 subfolders.append((module_name, module_name))
607 fLOG(f"[generate_help_sphinx] subfolders={subfolders}")
608 prepare_file_for_sphinx_help_generation(
609 {}, root, dest_doc, subfolders=subfolders, silent=True,
610 rootrep=(f"_doc.sphinxdoc.source.{module_name}.", ""),
611 optional_dirs=optional_dirs, mapped_function=mapped_function,
612 replace_relative_import=False, module_name=module_name,
613 copy_add_ext=copy_add_ext, use_sys=use_sys, fexclude=fexclude,
614 auto_rst_generation=auto_rst_generation, fLOG=fLOG)
616 except ImportErrorHelpGen as e: # pragma: no cover
617 fLOG(
618 "[generate_help_sphinx] major failure, no solution found yet, please run again the script")
619 fLOG("[generate_help_sphinx] list of added modules:")
620 remove = [k for k in sys.modules if k not in sys_modules]
621 for k in sorted(remove):
622 fLOG("[generate_help_sphinx] ", k)
624 raise e
626 fLOG("[generate_help_sphinx] end of prepare_file_for_sphinx_help_generation")
627 fLOG("---- JENKINS END DOCUMENTATION COPY FILES ----")
629 ######
630 # blog
631 ######
632 datetime_rows = [("blog", datetime.now())]
633 fLOG("---- JENKINS BEGIN DOCUMENTATION BLOGS ----")
634 fLOG("[generate_help_sphinx] begin blogs")
635 blog_fold = os.path.join(
636 os.path.join(root, "_doc/sphinxdoc/source", "blog"))
638 if os.path.exists(blog_fold):
639 fLOG("[generate_help_sphinx] BlogPostList")
640 plist = BlogPostList(blog_fold, language=language, fLOG=fLOG,
641 conf=theconf)
642 fLOG("[generate_help_sphinx] BlogPostList.write_aggregated")
643 plist.write_aggregated(
644 blog_fold,
645 blog_title=theconf.__dict__.get("blog_title", project_var_name),
646 blog_description=theconf.__dict__.get(
647 "blog_description", "blog associated to " + project_var_name),
648 blog_root=theconf.__dict__.get("blog_root", "__BLOG_ROOT__"))
649 else:
650 plist = None
652 fLOG("[generate_help_sphinx] end blogs")
653 fLOG("---- JENKINS END DOCUMENTATION BLOGS ----")
655 ###########
656 # notebooks
657 ###########
658 datetime_rows = [("notebooks", datetime.now())]
659 fLOG("---- JENKINS BEGIN DOCUMENTATION NOTEBOOKS ----")
660 fLOG("[generate_help_sphinx] begin notebooks")
661 indextxtnote = None
662 indexlistnote = []
663 notebook_dir = os.path.abspath(os.path.join(root, "_doc", "notebooks"))
664 notebook_doc = os.path.abspath(
665 os.path.join(root, "_doc", "sphinxdoc", "source", "notebooks"))
666 if os.path.exists(notebook_dir):
667 fLOG(f" look into '{notebook_dir}'")
668 fLOG(f" -pattern '{nbneg_pattern}'")
669 notebooks = explore_folder(notebook_dir, pattern=".*[.]ipynb", neg_pattern=nbneg_pattern,
670 fullname=True, fLOG=fLOG)[1]
671 notebooks = [_ for _ in notebooks if (
672 "checkpoint" not in _ and "/build/" not in _.replace("\\", "/"))]
673 fLOG(f" found {len(notebooks)} notebooks")
674 if len(notebooks) > 0:
675 fLOG("[generate_help_sphinx] **** notebooks", nbformats)
676 build = os.path.join(root, "build", "notebooks")
677 if not os.path.exists(build):
678 os.makedirs(build)
679 indextxtnote = os.path.join(build, "index_notebooks.txt")
680 with open(indextxtnote, "w", encoding="utf-8") as f:
681 for note in notebooks:
682 no = os.path.relpath(note, notebook_dir)
683 indexlistnote.append((no, note))
684 f.write(no + "\n")
685 if not os.path.exists(notebook_doc):
686 os.mkdir(notebook_doc)
687 nbs_all = process_notebooks(notebooks, build=build, outfold=notebook_doc,
688 formats=nbformats, latex_path=latex_path,
689 pandoc_path=pandoc_path, fLOG=fLOG, nblinks=nblinks,
690 notebook_replacements=notebook_replacements)
691 nbs_all = set(_[0] for _ in nbs_all
692 if os.path.splitext(_[0])[-1] == ".rst")
693 if len(nbs_all) != len(indexlistnote): # pragma: no cover
694 ext1 = "nbs_all:\n{0}".format("\n".join(nbs_all))
695 ext2 = "indexlistnote:\n{0}".format(
696 "\n".join(str(_) for _ in indexlistnote))
697 raise ValueError("Different lengths {0} != {1}\n{2}\n{3}".format(
698 len(nbs_all), len(indexlistnote), ext1, ext2))
699 nbs = indexlistnote
700 fLOG("[generate_help_sphinx] *#* NB, add:", len(nbs))
701 nbs.sort()
702 build_notebooks_gallery(nbs, os.path.join(
703 notebook_doc, "..", "all_notebooks.rst"), layout=nblayout,
704 snippet_folder=snippet_folder, fLOG=fLOG)
705 build_all_notebooks_coverage(nbs, os.path.join(
706 notebook_doc, "..", "all_notebooks_coverage.rst"), module_name, fLOG=fLOG)
708 imgs = [os.path.join(notebook_dir, _)
709 for _ in os.listdir(notebook_dir) if ".png" in _]
710 if len(imgs) > 0:
711 gallery_dirs = nbexamples_conf.get(
712 'gallery_dirs', None) if nbexamples_conf else None
713 for img in imgs:
714 shutil.copy(img, notebook_doc)
715 if gallery_dirs:
716 for d in gallery_dirs:
717 if not os.path.exists(d):
718 os.makedirs(d)
719 shutil.copy(img, d)
720 else:
721 fLOG(f"---- no folder '{notebook_dir}'")
723 fLOG("[generate_help_sphinx] end notebooks")
724 fLOG("---- JENKINS END DOCUMENTATION NOTEBOOKS ----")
726 #############################################
727 # replace placeholder as blog posts list into tocs files
728 #############################################
729 datetime_rows = [("replace", datetime.now())]
730 fLOG("[generate_help_sphinx] blog placeholder")
731 if plist is not None:
732 replace_placeholder_by_recent_blogpost(
733 all_tocs, plist, "__INSERT__", fLOG=fLOG)
735 #################################
736 # run the documentation generation
737 #################################
738 datetime_rows = [("sphinx", datetime.now())]
739 fLOG("[generate_help_sphinx] prepare for SPHINX")
740 temp = os.environ["PATH"]
741 pyts = get_executables_path()
742 sepj = ";" if sys.platform.startswith("win") else ":"
743 script = sepj.join(pyts)
744 fLOG("[generate_help_sphinx] adding " + script)
745 temp = script + sepj + temp
746 os.environ["PATH"] = temp
747 fLOG("[generate_help_sphinx] changing PATH", temp)
748 pa = os.getcwd()
750 # bokeh trick
751 updates_env = dict(BOKEH_DOCS_MISSING_API_KEY_OK=1)
752 for k, v in updates_env.items():
753 if k not in os.environ:
754 os.environ[k] = str(v)
756 thispath = os.path.normpath(root)
757 docpath = os.path.normpath(os.path.join(thispath, "_doc", "sphinxdoc"))
759 ################
760 # checks encoding
761 ################
762 datetime_rows = [("encoding", datetime.now())]
763 fLOG("---- JENKINS BEGIN DOCUMENTATION ENCODING ----")
764 fLOG("[generate_help_sphinx] checking encoding utf8...")
765 for rt, _, files in os.walk(docpath):
766 for name in files:
767 thn = os.path.join(rt, name)
768 if name.endswith(".rst"):
769 try:
770 with open(thn, "r", encoding="utf8") as f:
771 f.read()
772 except UnicodeDecodeError as e: # pragma: no cover
773 raise HelpGenException(
774 "issue with encoding in a file", thn) from e
775 except Exception as e: # pragma: no cover
776 raise HelpGenException("issue with file ", thn) from e
778 fLOG("[generate_help_sphinx] running sphinx... from", docpath)
779 if not os.path.exists(docpath):
780 raise FileNotFoundError(docpath) # pragma: no cover
781 fLOG("---- JENKINS END DOCUMENTATION ENCODING ----")
783 os.chdir(docpath)
785 #####################
786 # builds command lines
787 #####################
788 datetime_rows = [("build", datetime.now())]
789 fLOG("[generate_help_sphinx] sphinx command lines")
790 cmds = []
791 lays = []
792 cmds_post = []
793 for t3 in layout:
794 lay, build, override, newconf = lay_build_override_newconf(t3)
796 if lay == "pdf":
797 lay = "elatex"
799 if clean and sys.platform.startswith("win"):
800 if os.path.exists(build):
801 for fold in os.listdir(build):
802 remove_folder(os.path.join(build, fold))
803 remove_folder(build)
805 over_ = [f"{k}={v}" for k, v in override.items()]
806 over = []
807 for o in over_:
808 over.append("-D")
809 over.append(o)
811 sconf = [] if newconf is None else ["-c", newconf]
813 cmd = ["sphinx-build", "-j%d" % parallel, "-v", "-T", "-b", f"{lay}",
814 "-d", f"{build}/doctrees"] + over + sconf + ["source", f"{build}/{lay}"]
815 if lay in ('latex', 'pdf', 'elatex'):
816 cmd.extend(["-D", "imgmath_image_format=png"])
817 cmds.append((cmd, build, lay))
818 fLOG("[generate_help_sphinx] run:", cmd)
819 lays.append(lay)
821 if add_htmlhelp and lay == "html": # pragma: no cover
822 # we cannot execute htmlhelp in the same folder
823 # as it changes the encoding
824 cmd = ["sphinx-build", "-j%d" % parallel, "-v", "-T", "-b", f"{lay}help",
825 "-d", f"{build}/doctrees"] + over + sconf + ["source", f"{build}/{lay}html"]
826 cmd.extend(["-D", "imgmath_image_format=png"])
827 cmds.append((cmd, build, "add_htmlhelp"))
828 fLOG("[generate_help_sphinx] run:", cmd)
829 lays.append(lay)
830 hhp = os.path.join(build, lay + "help", module_name + "_doc.hhp")
831 cmdp = '"C:\\Program Files (x86)\\HTML Help Workshop\\hhc.exe" ' + \
832 f'"{hhp}"'
833 cmds_post.append(cmdp)
835 # cmd = "make {0}".format(lay)
837 ###############################################################
838 # run cmds (prefer to use os.system instead of run_cmd if it gets stuck)
839 ###############################################################
840 datetime_rows = [("cmd", datetime.now())]
841 fLOG("[generate_help_sphinx] RUN SPHINX")
842 for cmd, build, kind in cmds:
843 fLOG("---- JENKINS BEGIN DOCUMENTATION SPHINX ----")
844 fLOG(
845 "##################################################################################################")
846 fLOG(
847 "##################### run sphinx #################################################################")
848 fLOG(
849 "##################################################################################################")
850 fLOG("[generate_help_sphinx]", cmd)
851 fLOG("[generate_help_sphinx] from ", os.getcwd())
852 fLOG("[generate_help_sphinx] PATH ", os.environ["PATH"])
854 existing = list(sorted(sys.modules.keys()))
855 for ex in existing:
856 if ex[0] == '_':
857 doesrem = False
858 elif ex not in list_modules_start:
859 doesrem = True
860 for pr in ('pywintypes', 'pandas', 'IPython', 'jupyter', 'numpy', 'scipy', 'matplotlib',
861 'pyquickhelper', 'yaml', 'xlsxwriter', '_csv',
862 '_lsprof', '_multiprocessing', '_overlapped', '_sqlite3',
863 'alabaster', 'asyncio', 'babel', 'bokeh', 'cProfile',
864 'certifi', 'colorsys', 'concurrent', 'csv', 'cycler',
865 'dateutil', 'decorator', 'docutils', 'fractions', 'gc',
866 'getopt', 'getpass', 'hmac', 'http', 'imp', 'ipython_genutils',
867 'jinja2', 'mimetypes', 'multiprocessing', 'pathlib',
868 'pickleshare', 'profile', 'prompt_toolkit', 'pstats',
869 'pydoc', 'py', 'pygments', 'requests', 'runpy', 'simplegeneric',
870 'sphinx', 'sqlite3', 'tornado', 'traitlets', 'typing', 'wcwidth',
871 'pythoncom', 'distutils', 'six', 'webbrowser', 'win32api', 'win32com',
872 'sphinxcontrib', 'zmq', 'nbformat', 'nbconvert',
873 'encodings', 'entrypoints', 'html', 'ipykernel', 'isodate',
874 'jsonschema', 'jupyter_client', 'mistune', 'nbbrowserpdf',
875 'notebook', 'pyparsing', 'zmq', 'jupyter_core',
876 'timeit', 'sphinxcontrib_images_lightbox2', 'win32con'):
877 if ex == pr or ex.startswith(pr + "."):
878 doesrem = False
879 else:
880 doesrem = False
881 if doesrem:
882 fLOG(
883 f"[generate_help_sphinx] remove '{ex}' from sys.modules")
884 del sys.modules[ex]
886 fLOG(
887 "##################################################################################################")
888 fLOG(f"[generate_help_sphinx] direct_call={direct_call}")
889 fLOG(f"[generate_help_sphinx] cmd='''{cmd}'''")
890 if isinstance(cmd, list):
891 fLOG(f"[generate_help_sphinx] cmd='''{' '.join(cmd)}'''")
892 fLOG(f"[generate_help_sphinx] kind='{kind}'")
893 fLOG(f"[generate_help_sphinx] build='{build}'")
894 fLOG(f"[generate_help_sphinx] direct_call={direct_call}")
896 # direct call
897 with python_path_append(root_source):
898 if direct_call: # pragma: no cover
899 # mostly to debug
900 out = StringIO()
901 err = StringIO()
902 memo_out = sys.stdout
903 memo_err = sys.stderr
904 sys.stdout = out
905 sys.stderr = out
906 try:
907 build_main(cmd[1:])
908 except SystemExit as e: # pragma: no cover
909 raise SystemExit("Unable to run Sphinx\n--CMD\n{0}\n--ERR--\n{1}\n--CWD--\n{2}\n--OUT--\n{3}\n--".format(
910 cmd, err.getvalue(), os.getcwd(), out.getvalue())) from e
911 sys.stdout = memo_out
912 sys.stderr = memo_err
913 out = out.getvalue()
914 err = err.getvalue()
915 lines = ['***OUT/***'] + out.split('\n') + ['***OUT\\***']
916 lines = [
917 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
918 out = "\n".join(lines)
919 else:
920 def customfLOG(*args, **kwargs):
921 "filter out some lines"
922 args = [
923 _ for _ in args if "toctree contains reference to document 'blog/" not in _]
924 if args:
925 fLOG(*args, **kwargs)
927 out, err = _process_sphinx_in_private_cmd(cmd, fLOG=customfLOG)
928 lines = ['***OUT//***'] + out.split('\n') + ['***OUT\\\\***']
929 lines = [
930 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
931 out = "\n".join(lines)
933 fLOG(
934 f"[generate_help_sphinx] end cmd len(out)={len(out)} len(err)={len(err)}")
936 if len(err) > 0 or len(out) > 0:
937 if ((len(err) > 0 and "Exception occurred:" in err) or
938 (len(out) > 0 and "Exception occurred:" in out)):
939 def keep_line(_): # pragma: no cover
940 if "RemovedInSphinx" in _:
941 return False
942 if "while setting up extension" in _:
943 return False
944 if "toctree contains reference to document 'blog/" in _:
945 return False
946 return True
947 out = "\n".join(
948 filter(lambda _: keep_line(_), out.split("\n")))
949 raise HelpGenException( # pragma: no cover
950 "Sphinx raised an exception (direct_call={3})\n--CMD--\n{0}\n--OUT--\n{1}\n[sphinxerror]-3\n{2}".format(
951 cmd, out, err, direct_call))
953 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
954 fLOG("[generate_help_sphinx]", kind, "~~~~", cmd)
955 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
956 warnings.warn(
957 "Sphinx went through errors. Check if any of them is important.\n---OUT---\n{0}\n[sphinxerror]-2\n{1}\n----".format(
958 out, err), UserWarning)
959 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
961 if kind == "html":
962 fLOG("#########################################################")
963 fLOG("[generate_help_sphinx] check that index.html exists")
964 findex = os.path.join(build, kind, "index.html")
965 if not os.path.exists(findex):
966 raise FileNotFoundError( # pragma: no cover
967 "something went wrong, unable to find {0}\n"
968 "--CMD--\n{1}\n--OUT--\n{2}\n--ERR--\n{3}\n"
969 "--LAY--\n{4}\n--INDEX--\n{5}\n-----\nPYTHONPATH={6}"
970 "\n-----"
971 "".format(findex, cmd, out, err, kind,
972 os.path.abspath(findex),
973 os.environ.get('PYTHONPATH', 'empty')))
975 fLOG("#########################################################")
976 verification_html_format(os.path.join(build, kind), fLOG=fLOG)
978 fLOG(
979 "##################################################################################################")
980 fLOG(
981 "##################### end run sphinx #############################################################")
982 fLOG(
983 "##################################################################################################")
984 fLOG("---- JENKINS END DOCUMENTATION SPHINX ----")
986 # we copy the extended styles (notebook, snippets) (again in build folders)
987 # we should not need that
988 for build_path in build_paths:
989 if not os.path.exists(build_path):
990 fLOG(
991 f"[generate_help_sphinx] build_path not found '{build_path}'")
992 continue
993 dest = os.path.join(build_path, "_static", style_figure_notebook[0])
994 if not os.path.exists(dest): # pragma: no cover
995 fLOG("[generate_help_sphinx] CREATE-CSS2", dest)
996 with open(dest, "w", encoding="utf-8") as f:
997 f.write(style_figure_notebook[1])
999 if add_htmlhelp: # pragma: no cover
1000 # we call HtmlHelp
1001 fLOG("[generate_help_sphinx] run HTMLHELP")
1002 for cmd in cmds_post:
1003 fLOG("running", cmd)
1004 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
1005 communicate=True, timeout=600)
1006 fLOG(out)
1007 if len(err) > 0:
1008 mes = "Sphinx went through errors. Check if any of them is important.\nOUT:\n{0}\n[sphinxerror]-1\n{1}"
1009 warnings.warn(mes.format(out, err), UserWarning)
1010 fLOG("[generate_help_sphinx] end run HTMLHELP")
1012 #####################################
1013 # we copy some file such as rss.xml
1014 #####################################
1015 fLOG("---- JENKINS BEGIN COPY RSS.XML ----")
1016 tocopy = [os.path.join(docpath, "source", "blog", "rss.xml")]
1017 for toco in tocopy:
1018 if os.path.exists(toco):
1019 fLOG(f"[generate_help_sphinx] copy '{os.path.split(toco)[-1]}'")
1020 for build_path in build_paths:
1021 dest = os.path.join(build_path, "_downloads")
1022 if os.path.exists(dest):
1023 shutil.copy(toco, dest)
1024 else:
1025 fLOG(
1026 f"[generate_help_sphinx] not found '{os.path.split(toco)[-1]}'")
1028 fLOG("---- JENKINS END COPY RSS.XML ----")
1030 #####################################
1031 # we copy the coverage files if it is missing
1032 #####################################
1033 datetime_rows = [("converage", datetime.now())]
1034 fLOG("---- JENKINS BEGIN DOCUMENTATION COVERAGE ----")
1035 fLOG("[generate_help_sphinx] copy coverage")
1036 covfold = os.path.join(docpath, "source", "coverage")
1037 if os.path.exists(covfold): # pragma: no cover
1038 fLOG("[generate_help_sphinx] coverage folder:", covfold)
1039 allfiles = os.listdir(covfold)
1040 allf = [_ for _ in allfiles if _.endswith(".rst")]
1041 if len(allf) == 0:
1042 # no rst file --> we copy
1043 allfiles = [os.path.join(covfold, _) for _ in allfiles]
1044 allfiles = [_ for _ in allfiles if os.path.isfile(_)]
1045 for lay in lays:
1046 layfolder = os.path.join(docpath, build, lay)
1047 fLOG("[generate_help_sphinx] coverage docpath:", docpath, " -- ",
1048 build, " -- ", lay, " ---- ", layfolder)
1049 if os.path.exists(layfolder):
1050 covbuild = os.path.join(layfolder, "coverage")
1051 fLOG("[coverage] covbuild", covbuild)
1052 if not os.path.exists(covbuild):
1053 os.mkdir(covbuild)
1054 for f in allfiles:
1055 fLOG("[generate_help_sphinx] coverage copy ",
1056 f, " to ", covbuild)
1057 shutil.copy(f, covbuild)
1058 else:
1059 fLOG("[sphinxerror]-B coverage files with rst in", covfold)
1060 else:
1061 fLOG("[generate_help_sphinx] no coverage files", covfold)
1062 fLOG("---- JENKINS END DOCUMENTATION COVERAGE ----")
1064 #########################################################
1065 # we copy javascript dependencies to build _download/javascript
1066 #########################################################
1067 datetime_rows = [("javascript", datetime.now())]
1068 # for every layout
1069 fLOG("[generate_help_sphinx] [reveal.js] JAVASCRIPT: COPY", html_static_paths)
1070 fLOG("[generate_help_sphinx] [reveal.js] BUILD:", build_paths)
1071 for subf in ["html"]:
1072 for html_static_path, build_path in zip(html_static_paths, build_paths):
1073 for sname in ["_downloads", "notebooks"]:
1074 builddoc = os.path.join(build_path, subf, sname)
1075 if not os.path.exists(builddoc):
1076 builddoc = os.path.join(build_path, "..", subf, sname)
1077 if not os.path.exists(builddoc):
1078 builddoc = os.path.join(build_path, sname)
1079 if os.path.exists(builddoc):
1080 # no download, there is probably no notebooks
1081 # so it is not needed
1082 fLOG("[generate_help_sphinx] copy javascript static files from",
1083 html_static_path, "to", builddoc)
1084 copy = synchronize_folder(
1085 html_static_path, builddoc, copy_1to2=True, fLOG=fLOG)
1086 fLOG("[generate_help_sphinx] javascript",
1087 len(copy), "files copied")
1088 for gal in galleries:
1089 gal_path = os.path.join(html_static_path, "..", gal)
1090 if not os.path.exists(gal_path):
1091 continue
1092 folders = ['.'] + [f for f in os.listdir(gal_path)
1093 if os.path.isdir(f)]
1094 for fold in folders:
1095 gal_src = os.path.join(gal_path, fold, "images")
1096 if not os.path.exists(gal_src):
1097 continue
1098 gal_dst = os.path.join(
1099 build_path, gal, fold, "_images")
1100 if not os.path.exists(gal_dst):
1101 os.makedirs(gal_dst)
1102 fLOG("[generate_help_sphinx] copy images to ",
1103 gal_src, "to", gal_dst)
1104 copy = synchronize_folder(
1105 gal_src, gal_dst, copy_1to2=True, fLOG=fLOG)
1106 fLOG("[generate_help_sphinx] notebooks",
1107 len(copy), "files copied")
1109 gal_src = os.path.join(
1110 build_path, "_images", "math")
1111 if os.path.exists(gal_src):
1112 gal_dst = os.path.join(
1113 build_path, gal, "_images", "math")
1114 if not os.path.exists(gal_dst):
1115 os.makedirs(gal_dst)
1116 fLOG(f"[generate_help_sphinx] copy images to "
1117 f"{gal_src!r} to {gal_dst!r}")
1118 copy = synchronize_folder(
1119 gal_src, gal_dst, copy_1to2=True, fLOG=fLOG)
1120 fLOG("[generate_help_sphinx] notebooks",
1121 len(copy), "files copied")
1122 else:
1123 fLOG(
1124 "[generate_help_sphinx] [reveal.js] no need, no folder", builddoc)
1126 ######
1127 # next
1128 ######
1129 datetime_rows = [("latex", datetime.now())]
1130 fLOG("[generate_help_sphinx] LATEX")
1131 if "latex" in layout or "elatex" in layout:
1132 fLOG("[generate_help_sphinx] post_process_latex_output", froot)
1133 post_process_latex_output(
1134 froot, False, custom_latex_processing=custom_latex_processing)
1136 if "pdf" in layout:
1137 fLOG("[generate_help_sphinx] compile_latex_output_final",
1138 froot, "**", latex_path)
1139 compile_latex_output_final(
1140 froot, latex_path, False, latex_book=latex_book, fLOG=fLOG,
1141 custom_latex_processing=custom_latex_processing,
1142 remove_unicode=remove_unicode)
1144 if "html" in layout:
1145 nbf = os.path.join(build, "html", "notebooks")
1146 if os.path.exists(nbf):
1147 post_process_html_nb_output_static_file(nbf, fLOG=fLOG)
1148 post_process_html_nb_output_static_file(
1149 os.path.join(build, "html", "_downloads"), fLOG=fLOG)
1151 for build_path in build_paths:
1152 src = os.path.join(build_path, "_images")
1153 dest = os.path.join(build_path, "notebooks")
1154 if os.path.exists(src) and os.path.exists(dest):
1155 fLOG("[generate_help_sphinx] [imgs] look for images in ", src)
1156 for img in enumerate_copy_images_for_slides(src, dest):
1157 fLOG("[generate_help_sphinx] [imgs] copy image for slides:", img)
1159 ######
1160 # copy pdf to html
1161 ######
1162 latex = os.path.join(build_path, "latex")
1163 html = os.path.join(build_path, "latex")
1164 if os.path.exists(html) and os.path.exists(latex): # pragma: no cover
1165 pdfs = os.listdir(latex)
1166 for pdf in pdfs:
1167 ext = os.path.splitext(pdf)[-1]
1168 if ext != '.pdf':
1169 continue
1170 full = os.path.join(latex, pdf)
1171 fLOG("[generate_help_sphinx] [pdf] copy:", pdf)
1172 shutil.copy(full, html)
1174 #####
1175 # end
1176 #####
1177 os.chdir(pa)
1178 fLOG("################################")
1179 fLOG("#### END - check log for success")
1180 fLOG("################################")
1181 for i, row in enumerate(datetime_rows):
1182 if i == 0:
1183 a = row[1]
1184 else:
1185 a = datetime_rows[i - 1][1]
1186 b = row[1]
1187 d = b - a
1188 mes = "[generate_help_sphinx] {0}{1}: {2} [{3} --> {4}]".format(
1189 row[0], " " * (15 - len(row[0])), d, a, b)
1190 fLOG(mes)
1191 fLOG("---- JENKINS END DOCUMENTATION ----")
1194def _process_sphinx_in_private_cmd(list_args, fLOG):
1195 this = os.path.join(os.path.dirname(
1196 os.path.abspath(__file__)), "process_sphinx_cmd.py")
1197 res = []
1198 for i, c in enumerate(list_args):
1199 if i == 0 and c in ("sphinx-main", "sphinx-build"):
1200 continue
1201 if c[0] == '"' or c[-1] == '"' or ' ' not in c:
1202 res.append(c)
1203 else:
1204 res.append(f'"{c}"')
1205 sargs = " ".join(res)
1206 cmd = f"\"{sys.executable.replace('w.exe', '.exe')}\" \"{this}\" {sargs}"
1207 fLOG(" ", cmd)
1208 fLOG("[generate_help_sphinx] _process_sphinx_in_private_cmd BEGIN")
1209 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
1210 communicate=False, tell_if_no_output=120)
1211 fLOG("[generate_help_sphinx] _process_sphinx_in_private_cmd END")
1212 lines = out.split('\n')
1213 lines = [
1214 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
1215 out = "\n".join(lines)
1216 return out, err
1219def _check_sphinx_configuration(conf, fLOG):
1220 """
1221 Operates some verification on the configuration.
1223 :param conf: :epkg:`sphinx` configuration
1224 :param fLOG: logging function
1225 :return: list of gallery folders
1226 """
1227 galleries = []
1228 clean_folders = []
1229 if hasattr(conf, "sphinx_gallery_conf"):
1230 sphinx_gallery_conf = conf.sphinx_gallery_conf
1231 if len(sphinx_gallery_conf["examples_dirs"]) != len(sphinx_gallery_conf["gallery_dirs"]):
1232 add = "\nexamples_dirs={0}\ngallery_dirs={1}".format(
1233 sphinx_gallery_conf["examples_dirs"], sphinx_gallery_conf["gallery_dirs"])
1234 raise ValueError( # pragma: no cover
1235 'sphinx_gallery_conf["examples_dirs"] and sphinx_gallery_conf["gallery_dirs"] '
1236 'do not have the same size.' + add)
1237 if len(sphinx_gallery_conf["examples_dirs"]) > 0:
1238 fLOG(
1239 f"[sphinx-gallery] {len(sphinx_gallery_conf['examples_dirs'])} discovered")
1240 for a, b in zip(sphinx_gallery_conf["examples_dirs"],
1241 sphinx_gallery_conf["gallery_dirs"]):
1242 fLOG(f"[sphinx-gallery] src '{a}'")
1243 fLOG(f"[sphinx-gallery] dest '{b}'")
1244 clean_folders.append(a)
1245 if len(sphinx_gallery_conf["gallery_dirs"]) > 0:
1246 galleries.extend(sphinx_gallery_conf["gallery_dirs"])
1247 for cl in clean_folders:
1248 fLOG(f"[sphinx-gallery] clean '{cl}'")
1249 for temp in os.listdir(cl): # pragma: no cover
1250 if temp.startswith("temp_"):
1251 aaa = os.path.join(cl, temp)
1252 fLOG(f"[sphinx-gallery] remove '{cl}'")
1253 remove_folder(aaa)
1254 return galleries
1257def _import_conf_extract_parameter(root, root_source, folds, build, newconf,
1258 all_tocs, build_paths, parameters,
1259 html_static_paths, fLOG):
1260 """
1261 Imports the configuration file and extracts some
1262 of the parameters it defines.
1263 Fills the following lists.
1265 @param root folder of the package
1266 @param root_source folder of the sources
1267 @param folds folder of the documentation
1268 @param build build path
1269 @param newconf unused except in an error message
1270 @param all_tocs list to fill
1271 @param build_paths list to fill
1272 @param parameters list to fill
1273 @param html_static_paths list to fill
1274 @param fLOG logging function
1276 * all_tocs
1277 * build_paths
1278 * parameters
1279 * html_static_paths
1280 """
1281 # trick, we place the good folder in the first position
1282 with python_path_append(folds):
1283 if fLOG:
1284 fLOG(
1285 f"[_import_conf_extract_parameter] import from '{folds}'")
1286 try:
1287 module_conf = execute_script_get_local_variables(
1288 "from conf import *", folder=folds)
1289 except Exception as ee: # pragma: no cover
1290 raise HelpGenException(
1291 "Unable to import a config file (root_source='{0}').".format(
1292 folds), os.path.join(folds, "conf.py")) from ee
1293 if 'ERROR' in module_conf:
1294 raise ImportError( # pragma: no cover
1295 "\n" + module_conf['ERROR'] + "\n")
1296 if len(module_conf) == 0:
1297 raise ImportError( # pragma: no cover
1298 "Unable to extract local variable from conf.py.")
1300 if module_conf is None:
1301 raise ImportError( # pragma: no cover
1302 f"Unable to import '{newconf}' which defines the help generation")
1303 thenewconf = dictionary_as_class(module_conf)
1304 if fLOG:
1305 fLOG("[_import_conf_extract_parameter] import:", thenewconf.drop(
1306 "epkg_dictionary", "latex_elements", "imgmath_latex_preamble", "preamble"))
1308 tocs = add_missing_files(root, thenewconf, "__INSERT__", fLOG)
1309 all_tocs.extend(tocs)
1311 # we store the html_static_path in html_static_paths
1312 html_static_path = thenewconf.__dict__.get(
1313 "html_static_path", "_static")
1314 if isinstance(html_static_path, list):
1315 html_static_path = html_static_path[0]
1316 html_static_path = os.path.normpath(
1317 os.path.join(root_source, html_static_path))
1318 if not os.path.exists(html_static_path):
1319 raise FileNotFoundError( # pragma: no cover
1320 "no static path:" + html_static_path)
1321 html_static_paths.append(html_static_path)
1322 build_paths.append(
1323 os.path.normpath(os.path.join(html_static_path, "..", "..", build, "html")))
1324 pp = dict(latex_book=thenewconf.latex_book) # pylint: disable=E1101
1325 parameters.append(pp)