Coverage for src/code_beatrix/automation/notebook_test_helper.py: 88%

52 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-04-29 13:45 +0200

1# -*- coding: utf-8 -*- 

2""" 

3@file 

4@brief Some automation helpers to test notebooks and check they are still working fine. 

5""" 

6import os 

7from pyquickhelper.loghelper import noLOG 

8from pyquickhelper.ipythonhelper import execute_notebook_list 

9 

10 

11def ls_notebooks(subfolder): 

12 """ 

13 Returns the list of notebooks in a particular subfolder. 

14 

15 @param subfolder subfolder (related to this module) 

16 @return list of files 

17 """ 

18 this = os.path.abspath(os.path.dirname(__file__)) 

19 docnote = os.path.join( 

20 this, 

21 "..", 

22 "..", 

23 "..", 

24 "_doc", 

25 "notebooks", 

26 subfolder) 

27 notes = [ 

28 os.path.normpath( 

29 os.path.join( 

30 docnote, 

31 _)) for _ in os.listdir(docnote)] 

32 

33 keepnote = [] 

34 for note in notes: 

35 ext = os.path.splitext(note)[-1] 

36 if ext != ".ipynb": 

37 continue 

38 keepnote.append(note) 

39 return keepnote 

40 

41 

42def get_additional_paths(): 

43 """ 

44 Returns a list of paths to add before running the notebooks, 

45 paths to :epkg:`pyquickhelper`, ... 

46 

47 @return list of paths 

48 """ 

49 import pyquickhelper 

50 import jyquickhelper 

51 import ensae_projects 

52 import pyensae 

53 addpath = [os.path.dirname(pyquickhelper.__file__), 

54 os.path.dirname(jyquickhelper.__file__), 

55 os.path.dirname(ensae_projects.__file__), 

56 os.path.dirname(pyensae.__file__), 

57 os.path.join(os.path.abspath(os.path.dirname(__file__)), ".."), 

58 ] 

59 addpath = [os.path.normpath(os.path.join(_, "..")) for _ in addpath] 

60 return addpath 

61 

62 

63def clean_function_notebook(code): 

64 """ 

65 Cleans cells when unittesting notebooks. 

66 

67 @param code cell content 

68 @return modified code 

69 """ 

70 code = code.replace( 

71 'run_cmd("exemple.xlsx"', 

72 'skip_run_cmd("exemple.xlsx"') 

73 

74 skip = ["faire une chose avec la probabilité 0.7", 

75 "# déclenche une exception", 

76 "# pour lancer Excel", 

77 "for k in list_exercice_1 :", 

78 "return ....", 

79 "return [ .... ]", 

80 "def __init__(self, ...) :", 

81 "if random.random() <= 0.7 :", 

82 "dictionnaire_depart.items() [0]", 

83 "iterateur(0,10) [0]", 

84 "# ...... à remplir", 

85 'String.Join(",", a.Select(c=>c.ToString()).ToArray())', 

86 "# elle n'existe pas encore", 

87 "from ggplot import *", 

88 # ggplot calls method show and it opens window blocking the offline 

89 # execution 

90 ] 

91 rep = [("# ...", "pass # "), 

92 ("%timeit", "#%timeit"), 

93 ] 

94 spl = ["# ......", 

95 "# elle n'existe pas encore", 

96 ] 

97 

98 for s in skip: 

99 if s in code: 

100 return "" 

101 

102 for s in spl: 

103 if s in code: 

104 code = code.split(s)[0] 

105 

106 for s in rep: 

107 code = code.replace(s[0], s[1]) 

108 

109 return code 

110 

111 

112def execute_notebooks(folder, notebooks, filter, clean_function=None, 

113 fLOG=noLOG, deepfLOG=noLOG): 

114 """ 

115 Executes a list of notebooks. 

116 

117 @param folder folder 

118 @param notebooks list of notebooks 

119 @param filter function which validate the notebooks 

120 @param clean_function cleaning function to apply to the code before running it 

121 @param fLOG logging function 

122 @param deepfLOG logging function used to run the notebook 

123 @return dictionary { notebook_file: (isSuccess, outout) } 

124 

125 The signature of function ``filter`` is:: 

126 

127 def filter( i, filename) : return True or False 

128 

129 """ 

130 

131 def valid_cell(cell): 

132 if "%system" in cell: 

133 return False 

134 if "df.plot(...)" in cell: 

135 return False 

136 if 'df["difference"] = ...' in cell: 

137 return False 

138 return True 

139 

140 addpaths = get_additional_paths() 

141 if filter: 

142 notebooks = [_ for i, _ in enumerate(notebooks) if filter(i, _)] 

143 if len(notebooks) == 0: 

144 raise ValueError("Empty list of notebooks.") 

145 return execute_notebook_list( 

146 folder, notebooks, fLOG=fLOG, valid=valid_cell, additional_path=addpaths, 

147 clean_function=clean_function)