Coverage for src/pyquickhelper/helpgen/sphinx_main.py : 92%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains the main function to generate the documentation
5for a module designed the same way as this one, @see fn generate_help_sphinx.
6"""
7import os
8import sys
9import shutil
10import warnings
11from datetime import datetime
12from io import StringIO
13from docutils.parsers.rst import directives, roles
14from sphinx.cmd.build import main as build_main
15from ..filehelper import remove_folder
16from ..loghelper import python_path_append
17from ..loghelper.process_script import execute_script_get_local_variables, dictionary_as_class
18from ..loghelper.flog import run_cmd, fLOG
19from .utils_sphinx_doc import prepare_file_for_sphinx_help_generation
20from .utils_sphinx_doc_helpers import HelpGenException, ImportErrorHelpGen
21from .conf_path_tools import find_latex_path, find_pandoc_path
22from ..filehelper.synchelper import explore_folder
23from ..filehelper import synchronize_folder
24from .post_process import post_process_latex_output
25from .process_notebooks import process_notebooks, build_notebooks_gallery, build_all_notebooks_coverage
26from .sphinx_helper import post_process_html_nb_output_static_file
27from .install_js_dep import install_javascript_tools
28from .sphinx_main_helper import setup_environment_for_help, get_executables_path, generate_changes_repo
29from .sphinx_main_helper import compile_latex_output_final, replace_placeholder_by_recent_blogpost
30from .sphinx_main_helper import format_history, enumerate_copy_images_for_slides
31from .sphinx_main_verification import verification_html_format
32from .sphinx_main_missing_html_files import add_missing_files
33from .style_css_template import style_figure_notebook
34from .post_process_custom import find_custom_latex_processing
35from ..sphinxext.blog_post_list import BlogPostList
36from ..sphinxext.sphinx_blog_extension import BlogPostDirective, BlogPostDirectiveAgg
37from ..sphinxext.sphinx_runpython_extension import RunPythonDirective
38from ..sphinxext.sphinx_postcontents_extension import PostContentsDirective
39from ..sphinxext.sphinx_tocdelay_extension import TocDelayDirective
40from ..sphinxext.sphinx_youtube_extension import YoutubeDirective
41from ..sphinxext.sphinx_sharenet_extension import ShareNetDirective, sharenet_role
42from ..sphinxext.sphinx_downloadlink_extension import process_downloadlink_role
43from ..sphinxext.sphinx_video_extension import VideoDirective
44from ..sphinxext.sphinx_image_extension import SimpleImageDirective
45from ..sphinxext.sphinximages.sphinxtrib.images import ImageDirective
46from ..sphinxext.sphinx_template_extension import tpl_role
47from ..sphinxext.sphinx_epkg_extension import epkg_role
48from ..sphinxext.sphinx_bigger_extension import bigger_role
49from ..sphinxext.sphinx_githublink_extension import githublink_role
50from ..sphinxext.sphinx_gitlog_extension import gitlog_role
51from ..sphinxext.sphinx_mathdef_extension import MathDef
52from ..sphinxext.sphinx_quote_extension import QuoteNode
53from ..sphinxext.sphinx_blocref_extension import BlocRef
54from ..sphinxext.sphinx_exref_extension import ExRef
55from ..sphinxext.sphinx_faqref_extension import FaqRef
56from ..sphinxext.sphinx_nbref_extension import NbRef
57from ..sphinxext.sphinx_cmdref_extension import CmdRef
58from ..sphinxext.sphinx_todoext_extension import TodoExt
59from ..sphinxext.sphinx_collapse_extension import CollapseDirective
60from ..sphinxext.sphinx_gdot_extension import GDotDirective
62template_examples = """
64List of programs
65++++++++++++++++
67.. toctree::
68 :maxdepth: 2
70.. autosummary:: __init__.py
71 :toctree: %s/
72 :template: modules.rst
74Another list
75++++++++++++
77"""
80def generate_help_sphinx(project_var_name, clean=False, root=".",
81 filter_commit=lambda c: c.strip() != "documentation",
82 extra_ext=None,
83 nbformats=("ipynb", "slides", "html", "python",
84 "rst", "pdf", "github"),
85 layout=None,
86 module_name=None, from_repo=True, add_htmlhelp=False,
87 copy_add_ext=None, direct_call=False, fLOG=fLOG,
88 parallel=1, extra_paths=None, fexclude=None):
89 """
90 Runs the help generation:
92 - copies every file in another folder,
93 - replaces comments in doxygen format into rst format,
94 - replaces local import by global import (tweaking sys.path too),
95 - calls sphinx to generate the documentation.
97 @param project_var_name project name
98 @param clean if True, cleans the previous documentation first
99 (:epkg:`html` files)
100 @param root see below
101 @param filter_commit function which accepts a commit to show on the documentation
102 (based on the comment)
103 @param extra_ext list of file extensions to document (not .py)
104 @param nbformats requested formats for the notebooks conversion
105 @param layout list of formats sphinx should generate such as html, latex, pdf, docx,
106 it is a list of tuple (layout, build directory, parameters to override),
107 if None --> ``[("html", "build", {})]``
108 @param module_name name of the module (must be the folder name ``src/module_name``
109 if None, ``module_name``
110 will be replaced by *project_var_name*
111 @param from_repo if True, assumes the sources come from a source repository,
112 False otherwise
113 @param add_htmlhelp run :epkg:`HTML` Help too (only on :epkg:`Windows`)
114 @param copy_add_ext additional file extension to copy
115 @param direct_call direct call to sphinx with *sphinx_build* if *True*
116 or run a command line in an another process to get
117 a clear environment
118 @param parallel degree of parallelization
119 @param extra_paths extra paths when importing configuration
120 @param fexclude function which tells which file not to copy in the folder
121 used to build the documentation
122 @param fLOG logging function
124 The result is stored in path: ``root/_doc/sphinxdoc/source``.
125 We assume the file ``root/_doc/sphinxdoc/source/conf.py`` exists
126 as well as ``root/_doc/sphinxdoc/source/index.rst``.
128 If you generate latex/pdf files, you should add variables ``latex_path`` and ``pandoc_path``
129 in your file ``conf.py`` which defines the help.
131 You can exclud some part while generating the documentation by adding:
133 * ``# -- HELP BEGIN EXCLUDE --``
134 * ``# -- HELP END EXCLUDE --``
136 ::
138 latex_path = r"C:/Program Files/MiKTeX 2.9/miktex/bin/x64"
139 pandoc_path = r"%USERPROFILE%/AppData/Local/Pandoc"
141 .. exref::
142 :title: Run help generation
143 :index: extension, extra extension, ext
145 ::
147 # from the main folder which contains folder src or the sources
148 generate_help_sphinx("pyquickhelper")
150 By default, the function only consider files end by ``.py`` and ``.rst`` but you could
151 add other files sharing the same extensions by adding this one
152 in the ``extra_ext`` list.
154 The function requires:
156 - :epkg:`pandoc`
157 - latex
159 @warning Some themes such as `Bootstrap Sphinx Theme <http://ryan-roemer.github.io/sphinx-bootstrap-theme/>`_
160 do not work on Internet Explorer. In that case, the
161 file ``<python_path>/Lib/site-packages/sphinx/themes/basic/layout.html``
162 must be modified to add the following line (just below ``Content-Type``).
164 ::
166 <meta http-equiv="X-UA-Compatible" content="IE=edge" />
168 .. index:: PEP8, autopep8
170 The code should follow as much as possible the sytle convention
171 `PEP8 <https://www.python.org/dev/peps/pep-0008/>`_.
172 The module `autopep8 <https://pypi.python.org/pypi/autopep8>`_ can modify a file or all files contained in one folder
173 by running the following command line:
175 ::
177 autopep8 <folder> --recursive --in-place --pep8-passes 2000 --verbose
179 **About encoding:** utf-8 without BOM is the recommanded option.
181 **About languages:** only one language can be specificied even if you have
182 multiple configuration file. Only the language specified in the main
183 ``conf.py`` will be considered.
185 **About blog posts:** the function uses sphinx directives ``blogpost`` and ``blogpostagg`` to create
186 a simple blog aggregator. Blog posts will be aggregated by months and categories.
187 Link to others parts to the documentation are possible.
188 The function also create a file ``rss.xml`` which contains the ten last added blog post.
189 This file contains an absolute link to the blog posts. However, because the documentation
190 can be published anywhere, the string ``__BLOG_ROOT__`` was inserted
191 instead of the absolute url to the website. It must be replaced before uploaded
192 or the parameter *blog_root* can be specified in the configuration file ``conf.py``.
194 @warning Parameter *add_htmlhelp* calls `Html Help WorkShop
195 <https://msdn.microsoft.com/en-us/library/windows/desktop/ms669985%28v=vs.85%29.aspx>`_.
196 It also changes the encoding of the HTMLoutput into cp1552 (encoding for Windows)
197 instead of utf-8.
199 @warning An issue was raised on Linux due to the use of ``.. only:: html``
200 (``AttributeError: Can't pickle local object 'setup.<locals>.<lambda>'``).
201 It disappeared when using only one thread and not 2 as
202 it was previously. Parameter *parallel* was introduced to
203 make that change and the default value is not 1.
205 .. index:: SVG, Inkscape
207 **Others necessary tools:**
209 SVG included in a notebook (or any RST file) requires `Inkscape <https://inkscape.org/>`_
210 to be converted into Latex.
212 .. faqref::
213 :title: How to dd an extra layer to the documentation?
215 The following `commit <https://github.com/sdpython/python3_module_template/commit/
216 75d765a293f65a37b3208601d17d3b0daa891af6>`_
217 on project `python3_module_template <https://github.com/sdpython/python3_module_template/>`_
218 shows which changes needs to be done to add an extra layer of for the documentation.
220 The function assumes :epkg:`IPython` 3 is installed.
221 It might no work for earlier versions (notebooks).
222 Parameters *from_repo*, *use_run_cmd* were added.
223 Notebook conversion to slides is implemented,
224 install :epkg:`reveal.js` if not installed.
225 Calls the function @see fn _setup_hook to initialize
226 the module before generating the documentation.
227 Parameter *add_htmlhelp* was added. It runs HtmlHelp on Windows ::
229 "C:\\Program Files (x86)\\HTML Help Workshop\\hhc.exe" build\\htmlhelp\\<module>.hhp
231 The documentation includes blog (with sphinx command ``.. blogpost::``
232 and python scripts ``.. runpython::``. The second command runs a python
233 script which outputs RST documntation adds it to the current documentation.
234 The function automatically adds custom role and custom directive ``sharenet``.
235 The function directly calls
236 `sphinx <https://www.sphinx-doc.org/en/master/>`_,
237 `nbconvert <https://nbconvert.readthedocs.io/en/latest/>`_.
238 When there are too many notebooks, the notebook index is difficult to read.
239 It does not require to get script location.
240 Not enough stable from virtual environment.
242 Set ``BOKEH_DOCS_MISSING_API_KEY_OK`` to 1.
243 bokeh sphinx extension requires that or a key for the google API (???).
244 The function was updated to use Sphinx 1.6.2.
245 However, you should read blog post
246 :ref:`Bug in Sphinx 1.6.2 for custom css <sphinx-162-bug-custom-css>`
247 if you have any trouble with custom css.
248 Add a report in ``all_notebooks.rst`` about notebook coverage.
249 Parameter *parallel* was added.
250 The parameter *nblayout* in the configuration file specifies
251 the layout for the notebook gallery. ``'classic'`` or ``'table'``.
252 The parameter *nbneg_pattern* can be used to remove notebooks from
253 the gallery if they match this regular expression.
254 It automatically adds video and image directives.
255 *remove_unicode* can set to False or True in the documentation
256 configuration file to allow or remove unicode characters
257 before compiling the latex output.
258 Import ``conf.py`` in a separate process before running
259 the generation of the documentation. Do not import it
260 directly.
261 """
262 datetime_rows = [("begin", datetime.now())]
264 fLOG("---- JENKINS BEGIN DOCUMENTATION ----")
265 if layout is None:
266 layout = [("html", "build", {})]
267 fLOG("[generate_help_sphinx] ---- layout", layout)
268 setup_environment_for_help(fLOG=fLOG)
269 # we keep a clean list of modules
270 # sphinx configuration is a module and the function loads and unloads it
271 list_modules_start = set(sys.modules.keys())
273 if add_htmlhelp: # pragma: no cover
274 if not sys.platform.startswith("win"):
275 raise ValueError("add_htmlhelp is True and the OS is not Windows")
276 fLOG("[generate_help_sphinx] add add_htmlhelp")
278 if extra_ext is None:
279 extra_ext = []
281 def lay_build_override_newconf(t3):
282 if isinstance(t3, str):
283 lay, build, override, newconf = t3, "build", {}, None
284 elif len(t3) == 1:
285 lay, build, override, newconf = t3[0], "build", {}, None
286 elif len(t3) == 2:
287 lay, build, override, newconf = t3[0], t3[1], {}, None
288 elif len(t3) == 3:
289 lay, build, override, newconf = t3[0], t3[1], t3[2], None
290 else:
291 lay, build, override, newconf = t3
292 return lay, build, override, newconf
294 directives.register_directive("blogpost", BlogPostDirective)
295 directives.register_directive("blogpostagg", BlogPostDirectiveAgg)
296 directives.register_directive("runpython", RunPythonDirective)
297 directives.register_directive("sharenet", ShareNetDirective)
298 directives.register_directive("video", VideoDirective)
299 directives.register_directive("simpleimage", SimpleImageDirective)
300 directives.register_directive("image", ImageDirective)
301 directives.register_directive("todoext", TodoExt)
302 directives.register_directive("mathdef", MathDef)
303 directives.register_directive("quote", QuoteNode)
304 directives.register_directive("blocref", BlocRef)
305 directives.register_directive("exref", ExRef)
306 directives.register_directive("faqref", FaqRef)
307 directives.register_directive("nbref", NbRef)
308 directives.register_directive("cmdref", CmdRef)
309 directives.register_directive("postcontents", PostContentsDirective)
310 directives.register_directive("tocdelay", TocDelayDirective)
311 directives.register_directive("youtube", YoutubeDirective)
312 directives.register_directive("thumbnail", ImageDirective)
313 directives.register_directive("collapse", CollapseDirective)
314 directives.register_directive("gdot", GDotDirective)
315 roles.register_canonical_role("sharenet", sharenet_role)
316 roles.register_canonical_role("bigger", bigger_role)
317 roles.register_canonical_role("githublink", githublink_role)
318 roles.register_canonical_role("gitlog", gitlog_role)
319 roles.register_canonical_role("tpl", tpl_role)
320 roles.register_canonical_role("epkg", epkg_role)
321 roles.register_canonical_role("downloadlink", process_downloadlink_role)
323 if "conf" in sys.modules:
324 raise ImportError( # pragma: no cover
325 "module conf was imported, this function expects not to:\n{0}".format(
326 sys.modules["conf"].__file__))
328 ############
329 # root_source
330 ############
331 root = os.path.abspath(root)
332 froot = root
333 root_sphinxdoc = os.path.join(root, "_doc", "sphinxdoc")
334 root_source = os.path.join(root_sphinxdoc, "source")
335 root_package = os.path.join(root, "src")
336 if not os.path.exists(root_package):
337 root_package = root
338 if not os.path.exists(root_package):
339 raise FileNotFoundError( # pragma: no cover
340 "Unable to find source root from '{}'.".format(root))
341 fLOG("[generate_help_sphinx] root='{0}'".format(root))
342 fLOG("[generate_help_sphinx] root_package='{0}'".format(root_package))
343 fLOG("[generate_help_sphinx] root_source='{0}'".format(root_source))
344 fLOG("[generate_help_sphinx] root_sphinxdoc='{0}'".format(root_sphinxdoc))
345 conf_paths = [root_source, root_package]
346 if extra_paths:
347 conf_paths.extend(extra_paths)
349 ########################################
350 # we import conf_base, specific to multi layers
351 ########################################
352 confb = os.path.join(root_source, "conf_base.py")
353 if os.path.exists(confb): # pragma: no cover
354 code = "from conf_base import *"
355 with python_path_append(conf_paths):
356 try:
357 module_conf = execute_script_get_local_variables(
358 code, folder=root_source, check=True)
359 except RuntimeError as e:
360 raise ImportError("Unable to import conf_base '{}' from '{}'\nsys.path=\n{}".format(
361 confb, root_source, "\n".join(sys.path))) from e
363 if module_conf is None:
364 raise ImportError(
365 "Unable to import '{0}' which defines the help generation".format(confb))
366 if 'ERROR' in module_conf:
367 msg = "\n".join(["paths:"] + conf_paths + [
368 "-----------------------",
369 module_conf['ERROR']])
370 raise ImportError(msg)
371 conf_base = dictionary_as_class(module_conf)
372 fLOG("[generate_help_sphinx] conf_base.__file__='{0}'".format(
373 os.path.abspath(conf_base.__file__))) # pylint: disable=E1101
375 copypath = list(sys.path)
377 # stores static path for every layout, we store them to copy
378 html_static_paths = []
379 build_paths = []
380 all_tocs = []
381 parameters = []
383 ###################################
384 # import others conf, we must do it now
385 # it takes too long to do it after if there is an error
386 # we assume the configuration are not too different
387 # about language for example, latex_path, pandoc_path
388 #################################################
389 for t3 in layout:
390 lay, build, override, newconf = lay_build_override_newconf(t3)
391 if newconf is None:
392 continue
393 fLOG("[generate_help_sphinx] newconf: '{}' - {}".format(newconf, t3))
394 # we need to import this file to guess the template directory and
395 # add missing templates
396 folds = os.path.join(root_sphinxdoc, newconf)
397 _import_conf_extract_parameter(root, root_source, folds, build, newconf,
398 all_tocs, build_paths, parameters,
399 html_static_paths, fLOG)
401 ################################################################
402 # we add the source path to the list of path to considered before importing
403 # import conf.py
404 ################################################################
405 with python_path_append(conf_paths):
406 try:
407 module_conf = execute_script_get_local_variables(
408 "from conf import *", folder=root_source, check=True)
409 except ImportError as e: # pragma: no cover
410 raise ImportError("Unable to import 'conf.py' from '{0}', sys.path=\n{1}\nBEFORE:\n{2}".format(
411 root_source, "\n".join(sys.path), "\n".join(copypath))) from e
412 if module_conf is None:
413 raise ImportError( # pragma: no cover
414 "unable to import 'conf.py' which defines the help generation")
415 if 'ERROR' in module_conf:
416 msg = "\n".join(["paths:"] + conf_paths + [
417 "----------------------- ERROR:",
418 module_conf['ERROR'],
419 "------------------------ root_source:",
420 root_source])
421 raise ImportError(msg)
422 if len(module_conf) == 0:
423 raise ImportError(
424 "No extracted local variable.") # pragma: no cover
425 theconf = dictionary_as_class(module_conf)
426 fLOG("[generate_help_sphinx] conf.__file__='{0}'".format(
427 os.path.abspath(theconf.__file__))) # pylint: disable=E1101
428 tocs = add_missing_files(root, theconf, "__INSERT__", fLOG)
429 all_tocs.extend(tocs)
431 ##############################
432 # some checkings on the configuration
433 ##############################
434 _check_sphinx_configuration(theconf, fLOG)
436 ##############################################################
437 # Extracts variables from the configuration.
438 # We store the html_static_path in html_static_paths for the base conf
439 # We extract other information from the configuration
440 ##############################################################
441 html_static_path = theconf.__dict__.get("html_static_path", "phdoc_static")
442 if isinstance(html_static_path, list):
443 html_static_path = html_static_path[0]
444 html_static_path = os.path.join(root_source, html_static_path)
445 if not os.path.exists(html_static_path):
446 raise FileNotFoundError( # pragma: no cover
447 "no static path:" + html_static_path)
448 html_static_paths.append(html_static_path)
449 build_paths.append(
450 os.path.normpath(os.path.join(html_static_path, "..", "..", "build", "html")))
451 custom_latex_processing = theconf.__dict__.get(
452 "custom_latex_processing", None)
453 if custom_latex_processing is not None: # pragma: no cover
454 # The configuration file is pickled by sphinx
455 # and parameter should not be functions.
456 if isinstance(custom_latex_processing, str):
457 custom_latex_processing = find_custom_latex_processing( # pylint: disable=E1111
458 custom_latex_processing)
459 res = custom_latex_processing("dummy phrase")
460 if res is None:
461 raise ValueError(
462 "Result of function custom_latex_processing should not be None.")
463 remove_unicode = theconf.__dict__.get("remove_unicode", False)
464 snippet_folder = theconf.__dict__.get(
465 "notebook_custom_snippet_folder", None)
466 if snippet_folder:
467 snippet_folder = os.path.join(
468 os.path.dirname(theconf.__file__), snippet_folder) # pylint: disable=E1101
470 notebook_replacements = theconf.__dict__.get("notebook_replacements", None)
471 if notebook_replacements is not None and not isinstance(notebook_replacements, dict):
472 raise TypeError("latex_notebook_replacements should be a dictionary not {0}".format(
473 type(notebook_replacements)))
475 ####################################
476 # modifies the version number in conf.py
477 ####################################
478 readme = os.path.join(root, "README.rst")
479 if not os.path.exists(readme):
480 raise FileNotFoundError(readme) # pragma: no cover
481 shutil.copy(readme, root_source)
482 license = os.path.join(root, "LICENSE.txt")
483 if not os.path.exists(license):
484 raise FileNotFoundError(license) # pragma: no cover
485 shutil.copy(license, root_source)
486 history = os.path.join(root, "HISTORY.rst")
487 if os.path.exists(history):
488 dest = os.path.join(root_source, "HISTORY.rst")
489 format_history(history, dest)
491 ##########
492 # language
493 ##########
494 language = theconf.__dict__.get("language", "en")
495 use_sys = theconf.__dict__.get("enable_disabled_parts", None)
496 latex_book = theconf.__dict__.get('latex_book', False)
497 nbexamples_conf = theconf.__dict__.get('example_gallery_config', None)
498 # examples_conf = theconf.__dict__.get('sphinx_gallery_conf', None)
500 ##########
501 # auto_rst_generation
502 ##########
503 auto_rst_generation = theconf.__dict__.get("auto_rst_generation", True)
505 ospath = os.environ["PATH"]
506 latex_path = theconf.__dict__.get("latex_path", find_latex_path())
507 # graphviz_dot = theconf.__dict__.get("graphviz_dot", find_graphviz_dot())
508 pandoc_path = theconf.__dict__.get("pandoc_path", find_pandoc_path())
509 if os.path.isfile(latex_path):
510 latex_path = os.path.dirname(latex_path)
512 ##########
513 # nblinks: references for the notebooks, dictionary {(ref, format): link}
514 ##########
515 nblayout = theconf.__dict__.get("nblayout", "classic")
516 nblinks = theconf.__dict__.get("nblinks", None)
517 nbneg_pattern = theconf.__dict__.get("nbneg_pattern", None)
518 if nblinks is not None and len(nblinks) > 0:
519 fLOG("[generate_help_sphinx] NBLINKS - BEGIN")
520 for i, (k, v) in enumerate(sorted(nblinks.items())):
521 fLOG(" {0}/{1} - '{2}': '{3}'".format(i + 1, len(nblinks), k, v))
522 fLOG("[generate_help_sphinx] NBLINKS - END")
523 fLOG("[generate_help_sphinx] nbneg_pattern='{0}'".format(nbneg_pattern))
525 # add to PATH
526 sep = ";" if sys.platform.startswith("win") else ":"
527 if latex_path not in ospath:
528 os.environ["PATH"] += sep + latex_path
529 if pandoc_path not in ospath:
530 os.environ["PATH"] += sep + pandoc_path
532 #########
533 # changes
534 #########
535 datetime_rows = [("changes", datetime.now())]
536 chan = os.path.join(root, "_doc", "sphinxdoc", "source", "filechanges.rst")
537 if "modify_commit" in theconf.__dict__:
538 modify_commit = theconf.modify_commit # pylint: disable=E1101
539 else:
540 modify_commit = None
541 generate_changes_repo(
542 chan, root, filter_commit=filter_commit, exception_if_empty=from_repo,
543 fLOG=fLOG, modify_commit=modify_commit)
545 ######################################
546 # we copy javascript dependencies, reveal.js
547 ######################################
548 datetime_rows = [("javascript", datetime.now())]
549 fLOG("[generate_help_sphinx] JAVASCRIPT:", html_static_paths)
550 fLOG("[generate_help_sphinx] ROOT:", root_sphinxdoc)
551 fLOG("[generate_help_sphinx] BUILD:", build_paths)
552 for html_static_path in html_static_paths:
553 found = install_javascript_tools(
554 root_sphinxdoc, dest=html_static_path, fLOG=fLOG)
555 fLOG("[generate_help_sphinx] [javascript]: '{0}'".format(found))
557 ############################
558 # we copy the extended styles (notebook, snippets)
559 ############################
560 datetime_rows = [("copy", datetime.now())]
561 for html_static_path in html_static_paths:
562 dest = os.path.join(html_static_path, style_figure_notebook[0])
563 fLOG(" CREATE-CSS", dest)
564 with open(dest, "w", encoding="utf-8") as f:
565 f.write(style_figure_notebook[1])
567 # We should not need that.
568 # for build in build_paths:
569 # dest = os.path.join(build, "_downloads")
570 # if not os.path.exists(dest):
571 # os.makedirs(dest)
572 # install_javascript_tools(
573 # root_sphinxdoc, dest=dest, fLOG=fLOG)
575 ##############
576 # copy the files
577 ##############
578 fLOG("---- JENKINS BEGIN DOCUMENTATION COPY FILES ----")
579 optional_dirs = []
580 mapped_function = [(".*[.]%s$" % ext.strip("."), None)
581 for ext in extra_ext]
583 ###################################
584 # we save the module already imported
585 ###################################
586 if module_name is None:
587 module_name = project_var_name
589 sys_modules = set(sys.modules.keys())
591 ####################
592 # generates extra files
593 ####################
594 datetime_rows = [("prepare", datetime.now())]
595 try:
596 dest_doc = os.path.join(root, "_doc", "sphinxdoc", "source")
597 fLOG("[generate_help_sphinx] module_name='{}'".format(module_name))
598 fLOG("[generate_help_sphinx] project_var_name='{}'".format(project_var_name))
599 fLOG("[generate_help_sphinx] root='{}' root_package='{}'"
600 "".format(root, root_package))
601 fLOG("[generate_help_sphinx] dest_doc='{}'".format(dest_doc))
602 subfolders = []
603 if root_package.endswith("src"):
604 subfolders.append(("src/" + module_name, module_name))
605 else:
606 subfolders.append((module_name, module_name))
607 fLOG("[generate_help_sphinx] subfolders={0}".format(subfolders))
608 prepare_file_for_sphinx_help_generation(
609 {}, root, dest_doc, subfolders=subfolders, silent=True,
610 rootrep=("_doc.sphinxdoc.source.%s." % (module_name,), ""),
611 optional_dirs=optional_dirs, mapped_function=mapped_function,
612 replace_relative_import=False, module_name=module_name,
613 copy_add_ext=copy_add_ext, use_sys=use_sys, fexclude=fexclude,
614 auto_rst_generation=auto_rst_generation, fLOG=fLOG)
616 except ImportErrorHelpGen as e: # pragma: no cover
617 fLOG(
618 "[generate_help_sphinx] major failure, no solution found yet, please run again the script")
619 fLOG("[generate_help_sphinx] list of added modules:")
620 remove = [k for k in sys.modules if k not in sys_modules]
621 for k in sorted(remove):
622 fLOG("[generate_help_sphinx] ", k)
624 raise e
626 fLOG("[generate_help_sphinx] end of prepare_file_for_sphinx_help_generation")
627 fLOG("---- JENKINS END DOCUMENTATION COPY FILES ----")
629 ######
630 # blog
631 ######
632 datetime_rows = [("blog", datetime.now())]
633 fLOG("---- JENKINS BEGIN DOCUMENTATION BLOGS ----")
634 fLOG("[generate_help_sphinx] begin blogs")
635 blog_fold = os.path.join(
636 os.path.join(root, "_doc/sphinxdoc/source", "blog"))
638 if os.path.exists(blog_fold):
639 fLOG("[generate_help_sphinx] BlogPostList")
640 plist = BlogPostList(blog_fold, language=language, fLOG=fLOG)
641 fLOG("[generate_help_sphinx] BlogPostList.write_aggregated")
642 plist.write_aggregated(blog_fold,
643 blog_title=theconf.__dict__.get(
644 "blog_title", project_var_name),
645 blog_description=theconf.__dict__.get(
646 "blog_description", "blog associated to " + project_var_name),
647 blog_root=theconf.__dict__.get("blog_root", "__BLOG_ROOT__"))
648 else:
649 plist = None
651 fLOG("[generate_help_sphinx] end blogs")
652 fLOG("---- JENKINS END DOCUMENTATION BLOGS ----")
654 ###########
655 # notebooks
656 ###########
657 datetime_rows = [("notebooks", datetime.now())]
658 fLOG("---- JENKINS BEGIN DOCUMENTATION NOTEBOOKS ----")
659 fLOG("[generate_help_sphinx] begin notebooks")
660 indextxtnote = None
661 indexlistnote = []
662 notebook_dir = os.path.abspath(os.path.join(root, "_doc", "notebooks"))
663 notebook_doc = os.path.abspath(
664 os.path.join(root, "_doc", "sphinxdoc", "source", "notebooks"))
665 if os.path.exists(notebook_dir):
666 fLOG(" look into '{0}'".format(notebook_dir))
667 fLOG(" -pattern '{0}'".format(nbneg_pattern))
668 notebooks = explore_folder(notebook_dir, pattern=".*[.]ipynb", neg_pattern=nbneg_pattern,
669 fullname=True, fLOG=fLOG)[1]
670 notebooks = [_ for _ in notebooks if (
671 "checkpoint" not in _ and "/build/" not in _.replace("\\", "/"))]
672 fLOG(" found {0} notebooks".format(len(notebooks)))
673 if len(notebooks) > 0:
674 fLOG("[generate_help_sphinx] **** notebooks", nbformats)
675 build = os.path.join(root, "build", "notebooks")
676 if not os.path.exists(build):
677 os.makedirs(build)
678 indextxtnote = os.path.join(build, "index_notebooks.txt")
679 with open(indextxtnote, "w", encoding="utf-8") as f:
680 for note in notebooks:
681 no = os.path.relpath(note, notebook_dir)
682 indexlistnote.append((no, note))
683 f.write(no + "\n")
684 if not os.path.exists(notebook_doc):
685 os.mkdir(notebook_doc)
686 nbs_all = process_notebooks(notebooks, build=build, outfold=notebook_doc,
687 formats=nbformats, latex_path=latex_path,
688 pandoc_path=pandoc_path, fLOG=fLOG, nblinks=nblinks,
689 notebook_replacements=notebook_replacements)
690 nbs_all = set(_[0] for _ in nbs_all
691 if os.path.splitext(_[0])[-1] == ".rst")
692 if len(nbs_all) != len(indexlistnote): # pragma: no cover
693 ext1 = "nbs_all:\n{0}".format("\n".join(nbs_all))
694 ext2 = "indexlistnote:\n{0}".format(
695 "\n".join(str(_) for _ in indexlistnote))
696 raise ValueError("Different lengths {0} != {1}\n{2}\n{3}".format(
697 len(nbs_all), len(indexlistnote), ext1, ext2))
698 nbs = indexlistnote
699 fLOG("[generate_help_sphinx] *#* NB, add:", len(nbs))
700 nbs.sort()
701 build_notebooks_gallery(nbs, os.path.join(
702 notebook_doc, "..", "all_notebooks.rst"), layout=nblayout,
703 snippet_folder=snippet_folder, fLOG=fLOG)
704 build_all_notebooks_coverage(nbs, os.path.join(
705 notebook_doc, "..", "all_notebooks_coverage.rst"), module_name, fLOG=fLOG)
707 imgs = [os.path.join(notebook_dir, _)
708 for _ in os.listdir(notebook_dir) if ".png" in _]
709 if len(imgs) > 0:
710 gallery_dirs = nbexamples_conf.get(
711 'gallery_dirs', None) if nbexamples_conf else None
712 for img in imgs:
713 shutil.copy(img, notebook_doc)
714 if gallery_dirs:
715 for d in gallery_dirs:
716 if not os.path.exists(d):
717 os.makedirs(d)
718 shutil.copy(img, d)
719 else:
720 fLOG("---- no folder '{0}'".format(notebook_dir))
722 fLOG("[generate_help_sphinx] end notebooks")
723 fLOG("---- JENKINS END DOCUMENTATION NOTEBOOKS ----")
725 #############################################
726 # replace placeholder as blog posts list into tocs files
727 #############################################
728 datetime_rows = [("replace", datetime.now())]
729 fLOG("[generate_help_sphinx] blog placeholder")
730 if plist is not None:
731 replace_placeholder_by_recent_blogpost(
732 all_tocs, plist, "__INSERT__", fLOG=fLOG)
734 #################################
735 # run the documentation generation
736 #################################
737 datetime_rows = [("sphinx", datetime.now())]
738 fLOG("[generate_help_sphinx] prepare for SPHINX")
739 temp = os.environ["PATH"]
740 pyts = get_executables_path()
741 sepj = ";" if sys.platform.startswith("win") else ":"
742 script = sepj.join(pyts)
743 fLOG("[generate_help_sphinx] adding " + script)
744 temp = script + sepj + temp
745 os.environ["PATH"] = temp
746 fLOG("[generate_help_sphinx] changing PATH", temp)
747 pa = os.getcwd()
749 # bokeh trick
750 updates_env = dict(BOKEH_DOCS_MISSING_API_KEY_OK=1)
751 for k, v in updates_env.items():
752 if k not in os.environ:
753 os.environ[k] = str(v)
755 thispath = os.path.normpath(root)
756 docpath = os.path.normpath(os.path.join(thispath, "_doc", "sphinxdoc"))
758 ################
759 # checks encoding
760 ################
761 datetime_rows = [("encoding", datetime.now())]
762 fLOG("---- JENKINS BEGIN DOCUMENTATION ENCODING ----")
763 fLOG("[generate_help_sphinx] checking encoding utf8...")
764 for rt, _, files in os.walk(docpath):
765 for name in files:
766 thn = os.path.join(rt, name)
767 if name.endswith(".rst"):
768 try:
769 with open(thn, "r", encoding="utf8") as f:
770 f.read()
771 except UnicodeDecodeError as e: # pragma: no cover
772 raise HelpGenException(
773 "issue with encoding in a file", thn) from e
774 except Exception as e: # pragma: no cover
775 raise HelpGenException("issue with file ", thn) from e
777 fLOG("[generate_help_sphinx] running sphinx... from", docpath)
778 if not os.path.exists(docpath):
779 raise FileNotFoundError(docpath)
780 fLOG("---- JENKINS END DOCUMENTATION ENCODING ----")
782 os.chdir(docpath)
784 #####################
785 # builds command lines
786 #####################
787 datetime_rows = [("build", datetime.now())]
788 fLOG("[generate_help_sphinx] sphinx command lines")
789 cmds = []
790 lays = []
791 cmds_post = []
792 for t3 in layout:
793 lay, build, override, newconf = lay_build_override_newconf(t3)
795 if lay == "pdf":
796 lay = "elatex"
798 if clean and sys.platform.startswith("win"):
799 if os.path.exists(build):
800 for fold in os.listdir(build):
801 remove_folder(os.path.join(build, fold))
802 remove_folder(build)
804 over_ = ["{0}={1}".format(k, v) for k, v in override.items()]
805 over = []
806 for o in over_:
807 over.append("-D")
808 over.append(o)
810 sconf = [] if newconf is None else ["-c", newconf]
812 cmd = ["sphinx-build", "-j%d" % parallel, "-v", "-T", "-b", "{0}".format(lay),
813 "-d", "{0}/doctrees".format(build)] + over + sconf + ["source", "{0}/{1}".format(build, lay)]
814 if lay in ('latex', 'pdf', 'elatex'):
815 cmd.extend(["-D", "imgmath_image_format=png"])
816 cmds.append((cmd, build, lay))
817 fLOG("[generate_help_sphinx] run:", cmd)
818 lays.append(lay)
820 if add_htmlhelp and lay == "html": # pragma: no cover
821 # we cannot execute htmlhelp in the same folder
822 # as it changes the encoding
823 cmd = ["sphinx-build", "-j%d" % parallel, "-v", "-T", "-b", "{0}help".format(lay),
824 "-d", "{0}/doctrees".format(build)] + over + sconf + ["source", "{0}/{1}html".format(build, lay)]
825 cmd.extend(["-D", "imgmath_image_format=png"])
826 cmds.append((cmd, build, "add_htmlhelp"))
827 fLOG("[generate_help_sphinx] run:", cmd)
828 lays.append(lay)
829 hhp = os.path.join(build, lay + "help", module_name + "_doc.hhp")
830 cmdp = '"C:\\Program Files (x86)\\HTML Help Workshop\\hhc.exe" ' + \
831 '"%s"' % hhp
832 cmds_post.append(cmdp)
834 # cmd = "make {0}".format(lay)
836 ###############################################################
837 # run cmds (prefer to use os.system instead of run_cmd if it gets stuck)
838 ###############################################################
839 datetime_rows = [("cmd", datetime.now())]
840 fLOG("[generate_help_sphinx] RUN SPHINX")
841 for cmd, build, kind in cmds:
842 fLOG("---- JENKINS BEGIN DOCUMENTATION SPHINX ----")
843 fLOG(
844 "##################################################################################################")
845 fLOG(
846 "##################### run sphinx #################################################################")
847 fLOG(
848 "##################################################################################################")
849 fLOG("[generate_help_sphinx]", cmd)
850 fLOG("[generate_help_sphinx] from ", os.getcwd())
851 fLOG("[generate_help_sphinx] PATH ", os.environ["PATH"])
853 existing = list(sorted(sys.modules.keys()))
854 for ex in existing:
855 if ex[0] == '_':
856 doesrem = False
857 elif ex not in list_modules_start:
858 doesrem = True
859 for pr in ('pywintypes', 'pandas', 'IPython', 'jupyter', 'numpy', 'scipy', 'matplotlib',
860 'pyquickhelper', 'yaml', 'xlsxwriter', '_csv',
861 '_lsprof', '_multiprocessing', '_overlapped', '_sqlite3',
862 'alabaster', 'asyncio', 'babel', 'bokeh', 'cProfile',
863 'certifi', 'colorsys', 'concurrent', 'csv', 'cycler',
864 'dateutil', 'decorator', 'docutils', 'fractions', 'gc',
865 'getopt', 'getpass', 'hmac', 'http', 'imp', 'ipython_genutils',
866 'jinja2', 'mimetypes', 'multiprocessing', 'pathlib',
867 'pickleshare', 'profile', 'prompt_toolkit', 'pstats',
868 'pydoc', 'py', 'pygments', 'requests', 'runpy', 'simplegeneric',
869 'sphinx', 'sqlite3', 'tornado', 'traitlets', 'typing', 'wcwidth',
870 'pythoncom', 'distutils', 'six', 'webbrowser', 'win32api', 'win32com',
871 'sphinxcontrib', 'zmq', 'nbformat', 'nbconvert',
872 'encodings', 'entrypoints', 'html', 'ipykernel', 'isodate',
873 'jsonschema', 'jupyter_client', 'mistune', 'nbbrowserpdf',
874 'notebook', 'pyparsing', 'zmq', 'jupyter_core',
875 'timeit', 'sphinxcontrib_images_lightbox2', 'win32con'):
876 if ex == pr or ex.startswith(pr + "."):
877 doesrem = False
878 else:
879 doesrem = False
880 if doesrem:
881 fLOG(
882 "[generate_help_sphinx] remove '{0}' from sys.modules".format(ex))
883 del sys.modules[ex]
885 fLOG(
886 "##################################################################################################")
887 fLOG("[generate_help_sphinx] direct_call={0}".format(direct_call))
888 fLOG("[generate_help_sphinx] cmd='''{0}'''".format(cmd))
889 if isinstance(cmd, list):
890 fLOG("[generate_help_sphinx] cmd='''{0}'''".format(" ".join(cmd)))
891 fLOG("[generate_help_sphinx] kind='{0}'".format(kind))
892 fLOG("[generate_help_sphinx] build='{0}'".format(build))
893 fLOG("[generate_help_sphinx] direct_call={0}".format(direct_call))
895 # direct call
896 with python_path_append(root_source):
897 if direct_call:
898 # mostly to debug
899 out = StringIO()
900 err = StringIO()
901 memo_out = sys.stdout
902 memo_err = sys.stderr
903 sys.stdout = out
904 sys.stderr = out
905 try:
906 build_main(cmd[1:])
907 except SystemExit as e: # pragma: no cover
908 raise SystemExit("Unable to run Sphinx\n--CMD\n{0}\n--ERR--\n{1}\n--CWD--\n{2}\n--OUT--\n{3}\n--".format(
909 cmd, err.getvalue(), os.getcwd(), out.getvalue())) from e
910 sys.stdout = memo_out
911 sys.stderr = memo_err
912 out = out.getvalue()
913 err = err.getvalue()
914 lines = ['***OUT/***'] + out.split('\n') + ['***OUT\\***']
915 lines = [
916 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
917 out = "\n".join(lines)
918 else:
919 def customfLOG(*args, **kwargs):
920 "filter out some lines"
921 args = [
922 _ for _ in args if "toctree contains reference to document 'blog/" not in _]
923 if args:
924 fLOG(*args, **kwargs)
926 out, err = _process_sphinx_in_private_cmd(cmd, fLOG=customfLOG)
927 lines = ['***OUT//***'] + out.split('\n') + ['***OUT\\\\***']
928 lines = [
929 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
930 out = "\n".join(lines)
932 fLOG("[generate_help_sphinx] end cmd len(out)={0} len(err)={1}".format(
933 len(out), len(err)))
935 if len(err) > 0 or len(out) > 0:
936 if ((len(err) > 0 and "Exception occurred:" in err) or
937 (len(out) > 0 and "Exception occurred:" in out)):
938 def keep_line(_): # pragma: no cover
939 if "RemovedInSphinx" in _:
940 return False
941 if "while setting up extension" in _:
942 return False
943 if "toctree contains reference to document 'blog/" in _:
944 return False
945 return True
946 out = "\n".join(
947 filter(lambda _: keep_line(_), out.split("\n")))
948 raise HelpGenException(
949 "Sphinx raised an exception (direct_call={3})\n--CMD--\n{0}\n--OUT--\n{1}\n[sphinxerror]-3\n{2}".format(
950 cmd, out, err, direct_call))
952 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
953 fLOG("[generate_help_sphinx]", kind, "~~~~", cmd)
954 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
955 warnings.warn(
956 "Sphinx went through errors. Check if any of them is important.\n---OUT---\n{0}\n[sphinxerror]-2\n{1}\n----".format(
957 out, err), UserWarning)
958 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
960 if kind == "html":
961 fLOG("#########################################################")
962 fLOG("[generate_help_sphinx] check that index.html exists")
963 findex = os.path.join(build, kind, "index.html")
964 if not os.path.exists(findex):
965 raise FileNotFoundError(
966 "something went wrong, unable to find {0}\n"
967 "--CMD--\n{1}\n--OUT--\n{2}\n--ERR--\n{3}\n"
968 "--LAY--\n{4}\n--INDEX--\n{5}\n-----\nPYTHONPATH={6}"
969 "\n-----"
970 "".format(findex, cmd, out, err, kind,
971 os.path.abspath(findex),
972 os.environ.get('PYTHONPATH', 'empty')))
974 fLOG("#########################################################")
975 verification_html_format(os.path.join(build, kind), fLOG=fLOG)
977 fLOG(
978 "##################################################################################################")
979 fLOG(
980 "##################### end run sphinx #############################################################")
981 fLOG(
982 "##################################################################################################")
983 fLOG("---- JENKINS END DOCUMENTATION SPHINX ----")
985 # we copy the extended styles (notebook, snippets) (again in build folders)
986 # we should not need that
987 for build_path in build_paths:
988 if not os.path.exists(build_path):
989 fLOG("[generate_help_sphinx] build_path not found '{0}'".format(
990 build_path))
991 continue
992 dest = os.path.join(build_path, "_static", style_figure_notebook[0])
993 if not os.path.exists(dest): # pragma: no cover
994 fLOG("[generate_help_sphinx] CREATE-CSS2", dest)
995 with open(dest, "w", encoding="utf-8") as f:
996 f.write(style_figure_notebook[1])
998 if add_htmlhelp: # pragma: no cover
999 # we call HtmlHelp
1000 fLOG("[generate_help_sphinx] run HTMLHELP")
1001 for cmd in cmds_post:
1002 fLOG("running", cmd)
1003 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
1004 communicate=True, timeout=600)
1005 fLOG(out)
1006 if len(err) > 0:
1007 mes = "Sphinx went through errors. Check if any of them is important.\nOUT:\n{0}\n[sphinxerror]-1\n{1}"
1008 warnings.warn(mes.format(out, err), UserWarning)
1009 fLOG("[generate_help_sphinx] end run HTMLHELP")
1011 #####################################
1012 # we copy some file such as rss.xml
1013 #####################################
1014 fLOG("---- JENKINS BEGIN COPY RSS.XML ----")
1015 tocopy = [os.path.join(docpath, "source", "blog", "rss.xml")]
1016 for toco in tocopy:
1017 if os.path.exists(toco):
1018 fLOG("[generate_help_sphinx] copy '{}'".format(
1019 os.path.split(toco)[-1]))
1020 for build_path in build_paths:
1021 dest = os.path.join(build_path, "_downloads")
1022 if os.path.exists(dest):
1023 shutil.copy(toco, dest)
1024 else:
1025 fLOG("[generate_help_sphinx] not found '{}'".format(
1026 os.path.split(toco)[-1]))
1028 fLOG("---- JENKINS END COPY RSS.XML ----")
1030 #####################################
1031 # we copy the coverage files if it is missing
1032 #####################################
1033 datetime_rows = [("converage", datetime.now())]
1034 fLOG("---- JENKINS BEGIN DOCUMENTATION COVERAGE ----")
1035 fLOG("[generate_help_sphinx] copy coverage")
1036 covfold = os.path.join(docpath, "source", "coverage")
1037 if os.path.exists(covfold): # pragma: no cover
1038 fLOG("[generate_help_sphinx] coverage folder:", covfold)
1039 allfiles = os.listdir(covfold)
1040 allf = [_ for _ in allfiles if _.endswith(".rst")]
1041 if len(allf) == 0:
1042 # no rst file --> we copy
1043 allfiles = [os.path.join(covfold, _) for _ in allfiles]
1044 allfiles = [_ for _ in allfiles if os.path.isfile(_)]
1045 for lay in lays:
1046 layfolder = os.path.join(docpath, build, lay)
1047 fLOG("[generate_help_sphinx] coverage docpath:", docpath, " -- ",
1048 build, " -- ", lay, " ---- ", layfolder)
1049 if os.path.exists(layfolder):
1050 covbuild = os.path.join(layfolder, "coverage")
1051 fLOG("[coverage] covbuild", covbuild)
1052 if not os.path.exists(covbuild):
1053 os.mkdir(covbuild)
1054 for f in allfiles:
1055 fLOG("[generate_help_sphinx] coverage copy ",
1056 f, " to ", covbuild)
1057 shutil.copy(f, covbuild)
1058 else:
1059 fLOG("[sphinxerror]-B coverage files with rst in", covfold)
1060 else:
1061 fLOG("[generate_help_sphinx] no coverage files", covfold)
1062 fLOG("---- JENKINS END DOCUMENTATION COVERAGE ----")
1064 #########################################################
1065 # we copy javascript dependencies to build _download/javascript
1066 #########################################################
1067 datetime_rows = [("javascript", datetime.now())]
1068 # for every layout
1069 fLOG("[generate_help_sphinx] [reveal.js] JAVASCRIPT: COPY", html_static_paths)
1070 fLOG("[generate_help_sphinx] [reveal.js] BUILD:", build_paths)
1071 for subf in ["html"]:
1072 for html_static_path, build_path in zip(html_static_paths, build_paths):
1073 for sname in ["_downloads", "notebooks"]:
1074 builddoc = os.path.join(build_path, subf, sname)
1075 if not os.path.exists(builddoc):
1076 builddoc = os.path.join(build_path, "..", subf, sname)
1077 if not os.path.exists(builddoc):
1078 builddoc = os.path.join(build_path, sname)
1079 if os.path.exists(builddoc):
1080 # no download, there is probably no notebooks
1081 # so it is not needed
1082 fLOG("[generate_help_sphinx] copy javascript static files from",
1083 html_static_path, "to", builddoc)
1084 copy = synchronize_folder(
1085 html_static_path, builddoc, copy_1to2=True, fLOG=fLOG)
1086 fLOG("[generate_help_sphinx] javascript",
1087 len(copy), "files copied")
1088 else:
1089 fLOG(
1090 "[generate_help_sphinx] [reveal.js] no need, no folder", builddoc)
1092 ######
1093 # next
1094 ######
1095 datetime_rows = [("latex", datetime.now())]
1096 fLOG("[generate_help_sphinx] LATEX")
1097 if "latex" in layout or "elatex" in layout:
1098 fLOG("[generate_help_sphinx] post_process_latex_output", froot)
1099 post_process_latex_output(
1100 froot, False, custom_latex_processing=custom_latex_processing)
1102 if "pdf" in layout:
1103 fLOG("[generate_help_sphinx] compile_latex_output_final",
1104 froot, "**", latex_path)
1105 compile_latex_output_final(
1106 froot, latex_path, False, latex_book=latex_book, fLOG=fLOG,
1107 custom_latex_processing=custom_latex_processing,
1108 remove_unicode=remove_unicode)
1110 if "html" in layout:
1111 nbf = os.path.join(build, "html", "notebooks")
1112 if os.path.exists(nbf):
1113 post_process_html_nb_output_static_file(nbf, fLOG=fLOG)
1114 post_process_html_nb_output_static_file(
1115 os.path.join(build, "html", "_downloads"), fLOG=fLOG)
1117 for build_path in build_paths:
1118 src = os.path.join(build_path, "_images")
1119 dest = os.path.join(build_path, "notebooks")
1120 if os.path.exists(src) and os.path.exists(dest):
1121 fLOG("[generate_help_sphinx] [imgs] look for images in ", src)
1122 for img in enumerate_copy_images_for_slides(src, dest):
1123 fLOG("[generate_help_sphinx] [imgs] copy image for slides:", img)
1125 ######
1126 # copy pdf to html
1127 ######
1128 latex = os.path.join(build_path, "latex")
1129 html = os.path.join(build_path, "latex")
1130 if os.path.exists(html) and os.path.exists(latex): # pragma: no cover
1131 pdfs = os.listdir(latex)
1132 for pdf in pdfs:
1133 ext = os.path.splitext(pdf)[-1]
1134 if ext != '.pdf':
1135 continue
1136 full = os.path.join(latex, pdf)
1137 fLOG("[generate_help_sphinx] [pdf] copy:", pdf)
1138 shutil.copy(full, html)
1140 #####
1141 # end
1142 #####
1143 os.chdir(pa)
1144 fLOG("################################")
1145 fLOG("#### END - check log for success")
1146 fLOG("################################")
1147 for i, row in enumerate(datetime_rows):
1148 if i == 0:
1149 a = row[1]
1150 else:
1151 a = datetime_rows[i - 1][1]
1152 b = row[1]
1153 d = b - a
1154 mes = "[generate_help_sphinx] {0}{1}: {2} [{3} --> {4}]".format(
1155 row[0], " " * (15 - len(row[0])), d, a, b)
1156 fLOG(mes)
1157 fLOG("---- JENKINS END DOCUMENTATION ----")
1160def _process_sphinx_in_private_cmd(list_args, fLOG):
1161 this = os.path.join(os.path.dirname(
1162 os.path.abspath(__file__)), "process_sphinx_cmd.py")
1163 res = []
1164 for i, c in enumerate(list_args):
1165 if i == 0 and c in ("sphinx-main", "sphinx-build"):
1166 continue
1167 if c[0] == '"' or c[-1] == '"' or ' ' not in c:
1168 res.append(c)
1169 else:
1170 res.append('"{0}"'.format(c))
1171 sargs = " ".join(res)
1172 cmd = '"{0}" "{1}" {2}'.format(
1173 sys.executable.replace("w.exe", ".exe"), this, sargs)
1174 fLOG(" ", cmd)
1175 fLOG("[generate_help_sphinx] _process_sphinx_in_private_cmd BEGIN")
1176 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
1177 communicate=False, tell_if_no_output=120)
1178 fLOG("[generate_help_sphinx] _process_sphinx_in_private_cmd END")
1179 lines = out.split('\n')
1180 lines = [
1181 _ for _ in lines if "toctree contains reference to document 'blog/" not in _]
1182 out = "\n".join(lines)
1183 return out, err
1186def _check_sphinx_configuration(conf, fLOG):
1187 """
1188 Operates some verification on the configuration.
1190 @param conf :epkg:`sphinx` configuration
1191 @param fLOG logging function
1192 """
1193 clean_folders = []
1194 if hasattr(conf, "sphinx_gallery_conf"):
1195 sphinx_gallery_conf = conf.sphinx_gallery_conf
1196 if len(sphinx_gallery_conf["examples_dirs"]) != len(sphinx_gallery_conf["gallery_dirs"]):
1197 add = "\nexamples_dirs={0}\ngallery_dirs={1}".format(
1198 sphinx_gallery_conf["examples_dirs"], sphinx_gallery_conf["gallery_dirs"])
1199 raise ValueError(
1200 'sphinx_gallery_conf["examples_dirs"] and sphinx_gallery_conf["gallery_dirs"] do not have the same size.' + add)
1201 if len(sphinx_gallery_conf["examples_dirs"]) > 0:
1202 fLOG(
1203 "[sphinx-gallery] {0} discovered".format(len(sphinx_gallery_conf["examples_dirs"])))
1204 for a, b in zip(sphinx_gallery_conf["examples_dirs"],
1205 sphinx_gallery_conf["gallery_dirs"]):
1206 fLOG("[sphinx-gallery] src '{0}'".format(a))
1207 fLOG("[sphinx-gallery] dest '{0}'".format(b))
1208 clean_folders.append(a)
1209 for cl in clean_folders:
1210 fLOG("[sphinx-gallery] clean '{0}'".format(cl))
1211 for temp in os.listdir(cl): # pragma: no cover
1212 if temp.startswith("temp_"):
1213 aaa = os.path.join(cl, temp)
1214 fLOG("[sphinx-gallery] remove '{0}'".format(cl))
1215 remove_folder(aaa)
1218def _import_conf_extract_parameter(root, root_source, folds, build, newconf,
1219 all_tocs, build_paths, parameters,
1220 html_static_paths, fLOG):
1221 """
1222 Imports the configuration file and extracts some
1223 of the parameters it defines.
1224 Fills the following lists.
1226 @param root folder of the package
1227 @param root_source folder of the sources
1228 @param folds folder of the documentation
1229 @param build build path
1230 @param newconf unused except in an error message
1231 @param all_tocs list to fill
1232 @param build_paths list to fill
1233 @param parameters list to fill
1234 @param html_static_paths list to fill
1235 @param fLOG logging function
1237 * all_tocs
1238 * build_paths
1239 * parameters
1240 * html_static_paths
1241 """
1242 # trick, we place the good folder in the first position
1243 with python_path_append(folds):
1244 if fLOG:
1245 fLOG(
1246 "[_import_conf_extract_parameter] import from '{0}'".format(folds))
1247 try:
1248 module_conf = execute_script_get_local_variables(
1249 "from conf import *", folder=folds)
1250 except Exception as ee: # pragma: no cover
1251 raise HelpGenException(
1252 "Unable to import a config file (root_source='{0}').".format(
1253 folds), os.path.join(folds, "conf.py")) from ee
1254 if 'ERROR' in module_conf:
1255 raise ImportError("\n" + module_conf['ERROR'] + "\n")
1256 if len(module_conf) == 0:
1257 raise ImportError("Unable to extract local variable from conf.py.")
1259 if module_conf is None:
1260 raise ImportError( # pragma: no cover
1261 "Unable to import '{0}' which defines the help generation".format(newconf))
1262 thenewconf = dictionary_as_class(module_conf)
1263 if fLOG:
1264 fLOG("[_import_conf_extract_parameter] import:", thenewconf.drop(
1265 "epkg_dictionary", "latex_elements", "imgmath_latex_preamble", "preamble"))
1267 tocs = add_missing_files(root, thenewconf, "__INSERT__", fLOG)
1268 all_tocs.extend(tocs)
1270 # we store the html_static_path in html_static_paths
1271 html_static_path = thenewconf.__dict__.get(
1272 "html_static_path", "phdoc_static")
1273 if isinstance(html_static_path, list):
1274 html_static_path = html_static_path[0]
1275 html_static_path = os.path.normpath(
1276 os.path.join(root_source, html_static_path))
1277 if not os.path.exists(html_static_path):
1278 raise FileNotFoundError( # pragma: no cover
1279 "no static path:" + html_static_path)
1280 html_static_paths.append(html_static_path)
1281 build_paths.append(
1282 os.path.normpath(os.path.join(html_static_path, "..", "..", build, "html")))
1283 pp = dict(latex_book=thenewconf.latex_book) # pylint: disable=E1101
1284 parameters.append(pp)