Coverage for src/code_beatrix/automation/notebook_test_helper.py: 88%
52 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-29 13:45 +0200
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-29 13:45 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Some automation helpers to test notebooks and check they are still working fine.
5"""
6import os
7from pyquickhelper.loghelper import noLOG
8from pyquickhelper.ipythonhelper import execute_notebook_list
11def ls_notebooks(subfolder):
12 """
13 Returns the list of notebooks in a particular subfolder.
15 @param subfolder subfolder (related to this module)
16 @return list of files
17 """
18 this = os.path.abspath(os.path.dirname(__file__))
19 docnote = os.path.join(
20 this,
21 "..",
22 "..",
23 "..",
24 "_doc",
25 "notebooks",
26 subfolder)
27 notes = [
28 os.path.normpath(
29 os.path.join(
30 docnote,
31 _)) for _ in os.listdir(docnote)]
33 keepnote = []
34 for note in notes:
35 ext = os.path.splitext(note)[-1]
36 if ext != ".ipynb":
37 continue
38 keepnote.append(note)
39 return keepnote
42def get_additional_paths():
43 """
44 Returns a list of paths to add before running the notebooks,
45 paths to :epkg:`pyquickhelper`, ...
47 @return list of paths
48 """
49 import pyquickhelper
50 import jyquickhelper
51 import ensae_projects
52 import pyensae
53 addpath = [os.path.dirname(pyquickhelper.__file__),
54 os.path.dirname(jyquickhelper.__file__),
55 os.path.dirname(ensae_projects.__file__),
56 os.path.dirname(pyensae.__file__),
57 os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."),
58 ]
59 addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath]
60 return addpath
63def clean_function_notebook(code):
64 """
65 Cleans cells when unittesting notebooks.
67 @param code cell content
68 @return modified code
69 """
70 code = code.replace(
71 'run_cmd("exemple.xlsx"',
72 'skip_run_cmd("exemple.xlsx"')
74 skip = ["faire une chose avec la probabilité 0.7",
75 "# déclenche une exception",
76 "# pour lancer Excel",
77 "for k in list_exercice_1 :",
78 "return ....",
79 "return [ .... ]",
80 "def __init__(self, ...) :",
81 "if random.random() <= 0.7 :",
82 "dictionnaire_depart.items() [0]",
83 "iterateur(0,10) [0]",
84 "# ...... à remplir",
85 'String.Join(",", a.Select(c=>c.ToString()).ToArray())',
86 "# elle n'existe pas encore",
87 "from ggplot import *",
88 # ggplot calls method show and it opens window blocking the offline
89 # execution
90 ]
91 rep = [("# ...", "pass # "),
92 ("%timeit", "#%timeit"),
93 ]
94 spl = ["# ......",
95 "# elle n'existe pas encore",
96 ]
98 for s in skip:
99 if s in code:
100 return ""
102 for s in spl:
103 if s in code:
104 code = code.split(s)[0]
106 for s in rep:
107 code = code.replace(s[0], s[1])
109 return code
112def execute_notebooks(folder, notebooks, filter, clean_function=None,
113 fLOG=noLOG, deepfLOG=noLOG):
114 """
115 Executes a list of notebooks.
117 @param folder folder
118 @param notebooks list of notebooks
119 @param filter function which validate the notebooks
120 @param clean_function cleaning function to apply to the code before running it
121 @param fLOG logging function
122 @param deepfLOG logging function used to run the notebook
123 @return dictionary { notebook_file: (isSuccess, outout) }
125 The signature of function ``filter`` is::
127 def filter( i, filename) : return True or False
129 """
131 def valid_cell(cell):
132 if "%system" in cell:
133 return False
134 if "df.plot(...)" in cell:
135 return False
136 if 'df["difference"] = ...' in cell:
137 return False
138 return True
140 addpaths = get_additional_paths()
141 if filter:
142 notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)]
143 if len(notebooks) == 0:
144 raise ValueError("Empty list of notebooks.")
145 return execute_notebook_list(
146 folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths,
147 clean_function=clean_function)