Coverage for src/pyquickhelper/helpgen/sphinx_main_helper.py : 86%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Contains helpers for the main function @see fn generate_help_sphinx.
6"""
7import os
8import sys
9import datetime
10import shutil
11import subprocess
12import re
14from ..loghelper import run_cmd, RunCmdException, fLOG
15from ..loghelper.run_cmd import parse_exception_message
16from ..loghelper.pyrepo_helper import SourceRepository
17from ..pandashelper.tblformat import df2rst
18from ..filehelper import explore_folder_iterfile
19from .utils_sphinx_doc_helpers import HelpGenException
20from .post_process import post_process_latex_output
21from .process_notebooks import find_pdflatex
24template_examples = """
26List of programs
27++++++++++++++++
29.. toctree::
30 :maxdepth: 2
32.. autosummary:: __init__.py
33 :toctree: %s/
34 :template: modules.rst
36Another list
37++++++++++++
39"""
42def setup_environment_for_help(fLOG=fLOG):
43 """
44 Modifies environment variables to be able to use external tools
45 such as :epkg:`Inkscape`.
46 """
47 if sys.platform.startswith("win"):
48 prog = os.environ["ProgramFiles"]
49 inkscape = os.path.join(prog, "Inkscape")
50 if not os.path.exists(inkscape):
51 raise FileNotFoundError(
52 "Inkscape is not installed, expected at: {0}".format(inkscape))
53 path = os.environ["PATH"]
54 if inkscape not in path:
55 fLOG("SETUP: add path to %path%", inkscape)
56 os.environ["PATH"] = path + ";" + inkscape
57 else:
58 pass
61def get_executables_path():
62 """
63 Returns the paths to :epkg:`Python`,
64 :epkg:`Python` Scripts.
66 @return a list of paths
67 """
68 res = [os.path.split(sys.executable)[0]]
69 res.extend([os.path.join(res[-1], "Scripts"),
70 os.path.join(res[-1], "bin")])
71 return res
74def my_date_conversion(sdate):
75 """
76 Converts a date into a datetime.
78 @param sdate string
79 @return date
80 """
81 first = sdate.split(" ")[0]
82 trois = first.replace(".", "-").replace("/", "-").split("-")
83 return datetime.datetime(int(trois[0]), int(trois[1]), int(trois[2]))
86def produce_code_graph_changes(df):
87 """
88 Returns the code for a graph which counts the number
89 of changes per week over the last year.
91 @param df dataframe (has a column date with format ``YYYY-MM-DD``)
92 @return graph
94 The call to :epkg:`datetime.datetime.strptime`
95 introduced exceptions::
97 File "<frozen importlib._bootstrap>", line 2212, in _find_and_load_unlocked
98 File "<frozen importlib._bootstrap>", line 321, in _call_with_frames_removed
99 File "<frozen importlib._bootstrap>", line 2254, in _gcd_import
100 File "<frozen importlib._bootstrap>", line 2237, in _find_and_load
101 File "<frozen importlib._bootstrap>", line 2224, in _find_and_load_unlocked
103 when generating the documentation for another project. The reason
104 is still unclear. It was replaced by a custom function.
105 """
106 def year_week(x):
107 dt = datetime.datetime(x.year, x.month, x.day)
108 return dt.isocalendar()[:2]
110 def to_str(x):
111 year, week = year_week(x)
112 return "%d-w%02d" % (year, week)
114 df = df.copy()
115 df["dt"] = df.apply(lambda r: my_date_conversion(r["date"]), axis=1)
116 df = df[["dt"]]
117 now = datetime.datetime.now()
118 last = now - datetime.timedelta(365)
119 df = df[df.dt >= last]
120 df["week"] = df['dt'].apply(to_str)
121 df["commits"] = 1
123 val = []
124 for alldays in range(0, 365):
125 a = now - datetime.timedelta(alldays)
126 val.append({"dt": a, "week": to_str(a), "commits": 0})
128 # we move pandas here because it imports matplotlib
129 # which is not always wise when you need to modify the backend
130 import pandas
131 df = pandas.concat([df, pandas.DataFrame(val)], sort=True)
133 gr = df[["week", "commits"]].groupby("week", as_index=False).sum()
134 xl = list(gr["week"])
135 x = list(range(len(xl)))
136 y = list(gr["commits"])
138 typstr = str
140 code = """
141 import matplotlib.pyplot as plt
142 x = __X__
143 y = __Y__
144 xl = __XL__
145 plt.close('all')
146 plt.style.use('ggplot')
147 fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(10, 4))
148 ax.bar(x, y)
149 tig = ax.get_xticks()
150 labs = []
151 for t in tig:
152 if t in x:
153 labs.append(xl[x.index(t)])
154 else:
155 labs.append("")
156 ax.set_xticklabels(labs)
157 ax.grid(True)
158 ax.set_title("commits")
159 plt.show()
160 """.replace(" ", "") \
161 .replace("__X__", typstr(x)) \
162 .replace("__XL__", typstr(xl)) \
163 .replace("__Y__", typstr(y))
165 return code
168def generate_changes_repo(chan, source, exception_if_empty=True,
169 filter_commit=lambda c: c.strip() != "documentation",
170 fLOG=fLOG, modify_commit=None):
171 """
172 Generates a :epkg:`RST` tables containing the changes stored
173 by a :epkg:`SVN` or :epkg:`GIT` repository,
174 the outcome is stored in a file.
175 The log comment must start with ``*`` to be taken into account.
177 @param chan filename to write (or None if you don't need to)
178 @param source source folder to get changes for
179 @param exception_if_empty raises an exception if empty
180 @param filter_commit function which accepts a commit to show on the documentation (based on the comment)
181 @param fLOG logging function
182 @param modify_commit function which rewrite the commit text (see below)
183 @return string (rst tables with the changes)
185 :epkg:`pandas` is not imported in the function itself but at the beginning of the module. It
186 seemed to cause soe weird exceptions when generating the documentation for another module::
188 File "<frozen importlib._bootstrap>", line 2212, in _find_and_load_unlocked
189 File "<frozen importlib._bootstrap>", line 321, in _call_with_frames_removed
190 File "<frozen importlib._bootstrap>", line 2254, in _gcd_import
191 File "<frozen importlib._bootstrap>", line 2237, in _find_and_load
192 File "<frozen importlib._bootstrap>", line 2224, in _find_and_load_unlocked
194 Doing that helps. The cause still remains obscure.
195 If not None, function *modify_commit* is called the following way (see below).
196 *nbch* is the commit number. *date* can be returned as a datetime or a string.
198 ::
200 nbch, date, author, comment = modify_commit(nbch, date, author, comment)
201 """
202 # builds the changes files
203 try:
204 src = SourceRepository(commandline=True)
205 logs = src.log(path=source)
206 except Exception as eee:
207 if exception_if_empty:
208 fLOG("[sphinxerror]-9 unable to retrieve log from " + source)
209 raise HelpGenException(
210 "unable to retrieve log in " + source + "\n" + str(eee)) from eee
211 logs = [("none", 0, datetime.datetime.now(), "-")]
212 fLOG("[sphinxerror]-8", eee)
214 if len(logs) == 0:
215 fLOG("[sphinxerror]-7 unable to retrieve log from " + source)
216 if exception_if_empty:
217 raise HelpGenException("retrieved logs are empty in " + source)
218 else:
219 fLOG("info, retrieved ", len(logs), " commits")
221 rows = []
222 rows.append(
223 """\n.. _l-changes:\n\n\nChanges\n=======\n\n__CODEGRAPH__\n\nList of recent changes:\n""")
225 typstr = str
227 values = []
228 for i, row in enumerate(logs):
229 n = len(logs) - i
230 author, nbch, date, comment = row[:4]
231 last = row[-1]
232 if last.startswith("http"):
233 nbch = "`%s <%s>`_" % (typstr(nbch), last)
235 if filter_commit(comment):
236 if modify_commit is not None:
237 nbch, date, author, comment = modify_commit(
238 nbch, date, author, comment)
239 if isinstance(date, datetime.datetime):
240 ds = "%04d-%02d-%02d" % (date.year, date.month, date.day)
241 else:
242 ds = date
243 if isinstance(nbch, int):
244 values.append(
245 ["%d" % n, "%04d" % nbch, "%s" % ds, author, comment.strip("*")])
246 else:
247 values.append(
248 ["%d" % n, "%s" % nbch, "%s" % ds, author, comment.strip("*")])
250 if len(values) == 0 and exception_if_empty:
251 raise HelpGenException(
252 "Logs were not empty but there was no comment starting with '*' from '{0}'\n".format(source) +
253 "\n".join([typstr(_) for _ in logs]))
255 if len(values) > 0:
256 import pandas
257 tbl = pandas.DataFrame(
258 columns=["#", "change number", "date", "author", "comment"], data=values)
259 rows.append(
260 "\n\n" + df2rst(tbl, list_table=True) + "\n\n")
262 final = "\n".join(rows)
264 if len(values) > 0:
265 code = produce_code_graph_changes(tbl)
266 code = code.split("\n")
267 code = [" " + _ for _ in code]
268 code = "\n".join(code)
269 code = ".. plot::\n" + code + "\n"
270 final = final.replace("__CODEGRAPH__", code)
272 if chan is not None:
273 with open(chan, "w", encoding="utf8") as f:
274 f.write(final)
275 return final
278def compile_latex_output_final(root, latex_path, doall, afile=None, latex_book=False, fLOG=fLOG,
279 custom_latex_processing=None, remove_unicode=False):
280 """
281 Compiles the :epkg:`latex` documents.
283 @param root root
284 @param latex_path path to the compiler
285 @param doall do more transformation of the latex file before compiling it
286 @param afile process a specific file
287 @param latex_book do some customized transformation for a book
288 @param fLOG logging function
289 @param custom_latex_processing function which does some post processing of the full latex file
290 @param remove_unicode remove unicode characters before compiling it
292 .. faqreq:
293 :title: The PDF is corrupted, SVG are not there
295 :epkg:`SVG` graphs are not well processed by the latex compilation.
296 It usually goes through the following instruction:
298 ::
300 \\sphinxincludegraphics{{seance4_projection_population_correction_51_0}.svg}
302 And produces the following error:
304 ::
306 ! LaTeX Error: Unknown graphics extension: .svg.
308 This function does not stop if the latex compilation but if the PDF
309 is corrupted, the log should be checked to see the errors.
310 """
311 latex_exe = find_pdflatex(latex_path)
312 processed = 0
313 tried = []
314 for subfolder in ['latex', 'elatex']:
315 build = os.path.join(root, "_doc", "sphinxdoc", "build", subfolder)
316 if not os.path.exists(build):
317 build = root
318 tried.append(build)
319 for tex in os.listdir(build):
320 if tex.endswith(".tex") and (afile is None or afile in tex):
321 processed += 1
322 file = os.path.join(build, tex)
323 if doall:
324 # -interaction=batchmode
325 c = '"{0}" "{1}" -max-print-line=900 -buf-size=10000000 -output-directory="{2}"'.format(
326 latex_exe, file, build)
327 else:
328 c = '"{0}" "{1}" -max-print-line=900 -buf-size=10000000 -interaction=nonstopmode -output-directory="{2}"'.format(
329 latex_exe, file, build)
330 fLOG("[compile_latex_output_final] LATEX compilation (c)", c)
331 post_process_latex_output(file, doall, latex_book=latex_book, fLOG=fLOG,
332 custom_latex_processing=custom_latex_processing,
333 remove_unicode=remove_unicode)
334 if sys.platform.startswith("win"):
335 change_path = None
336 else:
337 # On Linux the parameter --output-directory is sometimes ignored.
338 # And it only works from the current directory.
339 change_path = os.path.split(file)[0]
340 try:
341 out, err = run_cmd(c, wait=True, log_error=False, catch_exit=True, communicate=False,
342 tell_if_no_output=120, fLOG=fLOG, prefix_log="[latex] ", change_path=change_path)
343 except Exception as e:
344 # An exception is raised when the return code is an error. We
345 # check that PDF file was written.
346 out, err = parse_exception_message(e)
347 if err is not None and len(err) == 0 and out is not None and "Output written" in out:
348 # The output was produced. We ignore the return code.
349 fLOG("WARNINGS: Latex compilation had warnings:", c)
350 else:
351 raise OSError(
352 "Unable to execute\n{0}".format(c)) from e
354 if len(err) > 0 and "Output written on " not in out:
355 raise HelpGenException(
356 "CMD:\n{0}\n[sphinxerror]-6\n{1}\n---OUT:---\n{2}".format(c, err, out))
358 # second compilation
359 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
360 fLOG("~~~~ LATEX compilation (d)", c)
361 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
362 try:
363 out, err = run_cmd(
364 c, wait=True, log_error=False, communicate=False, fLOG=fLOG,
365 tell_if_no_output=600, prefix_log="[latex] ", change_path=change_path)
366 except (subprocess.CalledProcessError, RunCmdException):
367 fLOG("[sphinxerror]-5 LATEX ERROR: check the logs")
368 err = ""
369 out = ""
370 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
371 fLOG("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
372 if len(err) > 0 and "Output written on " not in out:
373 raise HelpGenException(err)
374 if processed == 0:
375 raise FileNotFoundError("Unable to find any latex file in folders\n{0}".format(
376 "\n".join(tried)))
379def replace_placeholder_by_recent_blogpost(all_tocs, plist, placeholder, nb_post=5, fLOG=fLOG):
380 """
381 Replaces a place holder by a list of blog post.
383 @param all_tocs list of files to look into
384 @param plist list of blog post
385 @param placeholder place holder to replace
386 @param nb_post number of blog post to display
387 @param fLOG logging function
388 """
389 def make_link(post):
390 name = os.path.splitext(os.path.split(post.FileName)[-1])[0]
391 s = """<a href="{{ pathto('',1) }}/blog/%s/%s.html">%s - %s</a>""" % (
392 post.Year, name, post.Date, post.Title)
393 return s
395 end = min(nb_post, len(plist))
396 for toc in all_tocs:
397 with open(toc, "r", encoding="utf8") as f:
398 content = f.read()
399 if placeholder in content:
400 fLOG(" *** update", toc)
401 links = [make_link(post) for post in plist[:end]]
402 content = content.replace(placeholder, "\n<br />".join(links))
403 with open(toc, "w", encoding="utf8") as f:
404 f.write(content)
407_pattern_images = ".*(([.]png)|([.]gif])|([.]jpeg])|([.]jpg])|([.]svg]))$"
410def enumerate_copy_images_for_slides(src, dest, pattern=_pattern_images):
411 """
412 Copies images, initial intent was for slides,
413 once converted into html, link to images are relative to
414 the folder which contains them, we copy the images from
415 ``_images`` to ``_downloads``.
417 @param src sources
418 @param dest destination
419 @param pattern see @see fn explore_folder_iterfile
420 @return enumerator of copied files
421 """
422 iter = explore_folder_iterfile(src, pattern=pattern)
423 for img in iter:
424 d = os.path.join(dest, os.path.split(img)[-1])
425 if os.path.exists(d):
426 os.remove(d)
427 shutil.copy(img, dest)
428 yield d
431def format_history(src, dest, format="basic"):
432 """
433 Formats history based on module
434 `releases <https://github.com/bitprophet/releases>`_.
436 @param src source history (file)
437 @param dest destination (file)
439 Parameter *format* was added. :epkg:`Sphinx`
440 extension *release* no longer used but the
441 formatting is still available.
442 """
443 with open(src, "r", encoding="utf-8") as f:
444 lines = f.readlines()
446 new_lines = []
447 if format == "release":
448 tag = None
449 for i in range(0, len(lines)):
450 line = lines[i].rstrip("\r\t\n ")
451 if line.startswith("===") and i > 0:
452 rel = lines[i - 1].rstrip("\r\t\n ")
453 if "." in rel:
454 del new_lines[-1]
455 res = "* :release:`{0}`".format(rel)
456 res = res.replace("(", "<").replace(")", ">")
457 if new_lines[-1].startswith("==="):
458 new_lines.append("")
459 new_lines.append(res)
460 tag = None
461 else:
462 new_lines.append(line)
463 elif len(line) > 0:
464 if line.startswith("**"):
465 ll = line.lower().strip("*")
466 if ll in ('bug', 'bugfix', 'bugfixes'):
467 tag = "bug"
468 elif ll in ('features', 'feature'):
469 tag = "feature"
470 elif ll in ('support', 'support'):
471 tag = "support"
472 else:
473 raise ValueError(
474 "Line {0}, unable to infer tag from '{1}'".format(i, line))
475 else:
476 nline = line.lstrip("* ")
477 if nline.startswith("`"):
478 if tag is None:
479 tag = 'issue'
480 res = "* :{0}:{1}".format(tag, nline)
481 if new_lines[-1].startswith("==="):
482 new_lines.append("")
483 new_lines.append(res)
484 elif nline.startswith("#"):
485 if tag is None:
486 tag = 'issue'
487 spl = nline.split(':')
488 nb, doc = spl[0], ':'.join(spl[1:])
489 res = "* :{0}:`{1}`: {2}".format(
490 tag, nb.strip("#"), doc.strip(' '))
491 if new_lines[-1].startswith("==="):
492 new_lines.append("")
493 new_lines.append(res)
494 else:
495 new_lines.append(line)
496 if line.startswith(".. _"):
497 new_lines.append("")
498 elif format == "basic":
499 reg = re.compile(
500 "(.*?)((`([0-9]+)`:)|([#]([0-9]+):))(.*?)[(]([-0-9]{10})[)]")
501 for line in lines:
502 match = reg.search(line)
503 if match:
504 gr = match.groups()
505 issue = gr[3]
506 if issue is None or len(issue) == 0:
507 issue = gr[5]
508 desc = gr[6].strip()
509 date = gr[7].strip()
510 new_line = "{0}:issue:`{1}`: {2} ({3})".format(
511 gr[0], issue, desc, date)
512 new_lines.append(new_line)
513 else:
514 new_lines.append(line.strip("\n\r"))
515 else:
516 raise ValueError("Unexpected value for format '{0}'".format(format))
518 with open(dest, "w", encoding="utf-8") as f:
519 f.write("\n".join(new_lines))