Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Some automation helpers to test notebooks and check they are still working fine. 

5""" 

6import os 

7from pyquickhelper.loghelper import noLOG 

8from pyquickhelper.ipythonhelper import execute_notebook_list 

9import pyensae 

10 

11 

12def ls_notebooks(subfolder): 

13 """ 

14 list the notebooks in a particular subfolder 

15 

16 @param subfolder subfolder (related to this module) 

17 @return list of files 

18 """ 

19 this = os.path.abspath(os.path.dirname(__file__)) 

20 docnote = os.path.join( 

21 this, 

22 "..", 

23 "..", 

24 "..", 

25 "_doc", 

26 "notebooks", 

27 subfolder) 

28 notes = [ 

29 os.path.normpath( 

30 os.path.join( 

31 docnote, 

32 _)) for _ in os.listdir(docnote)] 

33 

34 keepnote = [] 

35 for note in notes: 

36 ext = os.path.splitext(note)[-1] 

37 if ext != ".ipynb": 

38 continue 

39 keepnote.append(note) 

40 return keepnote 

41 

42 

43def get_additional_paths(): 

44 """ 

45 returns a list of paths to add before running the notebooks, 

46 paths to pyquickhelper, pyensae, pymmails 

47 

48 @return list of paths 

49 """ 

50 import pyquickhelper 

51 import pymyinstall 

52 import pyrsslocal 

53 import mlstatpy 

54 import jyquickhelper 

55 addpath = [os.path.dirname(pyquickhelper.__file__), 

56 os.path.dirname(pyensae.__file__), 

57 os.path.dirname(pyrsslocal.__file__), 

58 os.path.dirname(pymyinstall.__file__), 

59 os.path.dirname(mlstatpy.__file__), 

60 os.path.dirname(jyquickhelper.__file__), 

61 os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."), 

62 ] 

63 try: 

64 import ensae_teaching_cs 

65 addpath.append(os.path.dirname(ensae_teaching_cs.__file__)) 

66 except ImportError: # pragma: no cover 

67 pass 

68 addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath] 

69 return addpath 

70 

71 

72def clean_function_notebook(code): 

73 """ 

74 Default cleaning for notebooks cells when unittesting. 

75 

76 @param code cell content 

77 @return modified code 

78 """ 

79 code = code.replace( 

80 'run_cmd("exemple.xlsx"', 

81 'skip_run_cmd("exemple.xlsx"') 

82 

83 skip = ["faire une chose avec la probabilité 0.7", 

84 "# déclenche une exception", 

85 "# pour lancer Excel", 

86 "for k in list_exercice_1 :", 

87 "return ....", 

88 "return [ .... ]", 

89 "def __init__(self, ...) :", 

90 "if random.random() <= 0.7 :", 

91 "dictionnaire_depart.items() [0]", 

92 "iterateur(0,10) [0]", 

93 "# ...... à remplir", 

94 'String.Join(",", a.Select(c=>c.ToString()).ToArray())', 

95 "# elle n'existe pas encore", 

96 "# boucle sur les 24 heures de la journée", 

97 "from ggplot import *", 

98 "geocode = True", 

99 # ggplot calls method show and it opens window blocking the offline 

100 # execution 

101 ] 

102 rep = [("# ...", "pass # "), 

103 ("%timeit -n1 -r1 ", ""), 

104 ("%timeit", "#%timeit"), 

105 ("show(p)", "#show(p)"), 

106 ("show(tabs)", "#show(tabs)"), 

107 ('http://www.gutenberg.org/cache/epub/4647/pg4647.txt', 

108 'http://www.xavierdupre.fr/enseignement/complements/pg4647.txt'), 

109 ] 

110 spl = ["# ......", 

111 "# elle n'existe pas encore", 

112 ] 

113 

114 for s in skip: 

115 if s in code: 

116 return "" 

117 

118 for s in spl: 

119 if s in code: 

120 code = code.split(s)[0] 

121 

122 for s in rep: 

123 code = code.replace(s[0], s[1]) 

124 

125 return code 

126 

127 

128def execute_notebooks(folder, notebooks, filter, clean_function=None, 

129 fLOG=noLOG, deepfLOG=noLOG, detailed_log=None): 

130 """ 

131 Executes a list of notebooks. 

132 

133 @param folder folder 

134 @param notebooks list of notebooks 

135 @param filter function which validate the notebooks 

136 @param clean_function cleaning function to apply to the code before running it 

137 @param fLOG logging function 

138 @param deepfLOG logging function used to run the notebook 

139 @param detailed_log log the output while running the notebook (when the notebook execution fails due to timeout 

140 @return dictionary tuple (statistics, { notebook_file: (isSuccess, outout) }) 

141 

142 The signature of function ``filter`` is:: 

143 

144 def filter(i, filename): 

145 return True or False 

146 

147 """ 

148 

149 def valid_cell(cell): 

150 if "%system" in cell: 

151 return False # pragma: no cover 

152 if "df.plot(...)" in cell: 

153 return False # pragma: no cover 

154 if 'df["difference"] = ...' in cell: 

155 return False # pragma: no cover 

156 if 'print(next(it))' in cell: 

157 return False # pragma: no cover 

158 if "est d'indice 8 et non plus 9" in cell: 

159 return False 

160 return True 

161 

162 addpaths = get_additional_paths() 

163 if filter: 

164 notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] 

165 if len(notebooks) == 0: 

166 raise ValueError( # pragma: no cover 

167 "Empty list of notebooks.") 

168 if clean_function is None: 

169 clean_function = clean_function_notebook # pragma: no cover 

170 return execute_notebook_list( 

171 folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths, 

172 clean_function=clean_function)