Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Some automation helpers to test notebooks and check they are still working fine.
5"""
6import os
7from pyquickhelper.loghelper import noLOG
8from pyquickhelper.ipythonhelper import execute_notebook_list
9import pyensae
12def ls_notebooks(subfolder):
13 """
14 list the notebooks in a particular subfolder
16 @param subfolder subfolder (related to this module)
17 @return list of files
18 """
19 this = os.path.abspath(os.path.dirname(__file__))
20 docnote = os.path.join(
21 this,
22 "..",
23 "..",
24 "..",
25 "_doc",
26 "notebooks",
27 subfolder)
28 notes = [
29 os.path.normpath(
30 os.path.join(
31 docnote,
32 _)) for _ in os.listdir(docnote)]
34 keepnote = []
35 for note in notes:
36 ext = os.path.splitext(note)[-1]
37 if ext != ".ipynb":
38 continue
39 keepnote.append(note)
40 return keepnote
43def get_additional_paths():
44 """
45 returns a list of paths to add before running the notebooks,
46 paths to pyquickhelper, pyensae, pymmails
48 @return list of paths
49 """
50 import pyquickhelper
51 import pymyinstall
52 import pyrsslocal
53 import mlstatpy
54 import jyquickhelper
55 addpath = [os.path.dirname(pyquickhelper.__file__),
56 os.path.dirname(pyensae.__file__),
57 os.path.dirname(pyrsslocal.__file__),
58 os.path.dirname(pymyinstall.__file__),
59 os.path.dirname(mlstatpy.__file__),
60 os.path.dirname(jyquickhelper.__file__),
61 os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."),
62 ]
63 try:
64 import ensae_teaching_cs
65 addpath.append(os.path.dirname(ensae_teaching_cs.__file__))
66 except ImportError: # pragma: no cover
67 pass
68 addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
69 return addpath
72def clean_function_notebook(code):
73 """
74 Default cleaning for notebooks cells when unittesting.
76 @param code cell content
77 @return modified code
78 """
79 code = code.replace(
80 'run_cmd("exemple.xlsx"',
81 'skip_run_cmd("exemple.xlsx"')
83 skip = ["faire une chose avec la probabilité 0.7",
84 "# déclenche une exception",
85 "# pour lancer Excel",
86 "for k in list_exercice_1 :",
87 "return ....",
88 "return [ .... ]",
89 "def __init__(self, ...) :",
90 "if random.random() <= 0.7 :",
91 "dictionnaire_depart.items() [0]",
92 "iterateur(0,10) [0]",
93 "# ...... à remplir",
94 'String.Join(",", a.Select(c=>c.ToString()).ToArray())',
95 "# elle n'existe pas encore",
96 "# boucle sur les 24 heures de la journée",
97 "from ggplot import *",
98 "geocode = True",
99 # ggplot calls method show and it opens window blocking the offline
100 # execution
101 ]
102 rep = [("# ...", "pass # "),
103 ("%timeit -n1 -r1 ", ""),
104 ("%timeit", "#%timeit"),
105 ("show(p)", "#show(p)"),
106 ("show(tabs)", "#show(tabs)"),
107 ('http://www.gutenberg.org/cache/epub/4647/pg4647.txt',
108 'http://www.xavierdupre.fr/enseignement/complements/pg4647.txt'),
109 ]
110 spl = ["# ......",
111 "# elle n'existe pas encore",
112 ]
114 for s in skip:
115 if s in code:
116 return ""
118 for s in spl:
119 if s in code:
120 code = code.split(s)[0]
122 for s in rep:
123 code = code.replace(s[0], s[1])
125 return code
128def execute_notebooks(folder, notebooks, filter, clean_function=None,
129 fLOG=noLOG, deepfLOG=noLOG, detailed_log=None):
130 """
131 Executes a list of notebooks.
133 @param folder folder
134 @param notebooks list of notebooks
135 @param filter function which validate the notebooks
136 @param clean_function cleaning function to apply to the code before running it
137 @param fLOG logging function
138 @param deepfLOG logging function used to run the notebook
139 @param detailed_log log the output while running the notebook (when the notebook execution fails due to timeout
140 @return dictionary tuple (statistics, { notebook_file: (isSuccess, outout) })
142 The signature of function ``filter`` is::
144 def filter(i, filename):
145 return True or False
147 """
149 def valid_cell(cell):
150 if "%system" in cell:
151 return False # pragma: no cover
152 if "df.plot(...)" in cell:
153 return False # pragma: no cover
154 if 'df["difference"] = ...' in cell:
155 return False # pragma: no cover
156 if 'print(next(it))' in cell:
157 return False # pragma: no cover
158 if "est d'indice 8 et non plus 9" in cell:
159 return False
160 return True
162 addpaths = get_additional_paths()
163 if filter:
164 notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
165 if len(notebooks) == 0:
166 raise ValueError( # pragma: no cover
167 "Empty list of notebooks.")
168 if clean_function is None:
169 clean_function = clean_function_notebook # pragma: no cover
170 return execute_notebook_list(
171 folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths,
172 clean_function=clean_function)