Coverage for src/ensae_teaching_cs/automation/jenkins_helper.py: 50%

88 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-01-27 05:44 +0100

1""" 

2@file 

3@brief Set up a jenkins server with all the necessary job 

4""" 

5import os 

6import re 

7import sys 

8from pyquickhelper.loghelper import noLOG 

9from pyquickhelper.jenkinshelper import setup_jenkins_server_yml 

10from pyquickhelper.jenkinshelper.yaml_helper import load_yaml 

11from .teaching_modules import get_teaching_modules 

12 

13 

14def get_interpreter(platform=None): 

15 """ 

16 Returns the default interpreter. 

17 

18 @param platform platform 

19 """ 

20 if platform == sys.platform: 

21 return os.path.dirname(sys.executable) 

22 elif platform.startswith("win"): 

23 return "c:\\Python%d%d%d_x64" % sys.version_info[:3] 

24 elif platform.startswith("darwin"): 

25 return "/usr/local/bin" 

26 return "/usr/local/bin" 

27 

28 

29def engines_default(prefix="c:\\", prefix_python="c:\\", prefix_conda="c:\\", platform=None): 

30 """ 

31 Returns a dictionary with default values for a :epkg:`Jenkins` server. 

32 

33 @param prefix prefix for Jenkins location 

34 @param prefix_python prefix for Python distribution 

35 @param prefix_conda prefix for Anaconda or Miniconda distribution 

36 @param platform platform to use 

37 @return dictionary 

38 

39 .. warning:: 

40 

41 Virtual environment with conda must be created on the same disk 

42 as the original interpreter. The other scenario is not supported. 

43 """ 

44 if platform is None: 

45 platform = sys.platform 

46 if platform.startswith("win"): 

47 res = dict(anaconda2=os.path.join(prefix_conda, "Anaconda2"), 

48 anaconda3=os.path.join(prefix_conda, "Anaconda3"), 

49 default=get_interpreter(platform=platform), 

50 py38=get_interpreter(platform=platform), 

51 py37=get_interpreter(platform=platform), 

52 py36=os.path.join(prefix_python, "Python36_x64"), 

53 py27=os.path.join(prefix_python, "Python27_x64"), 

54 Python38pyq=os.path.join( 

55 prefix, "jenkins", "venv", "py38", "pyq", "Scripts"), 

56 Python37pyq=os.path.join( 

57 prefix, "jenkins", "venv", "py37", "pyq", "Scripts"), 

58 Python36pyq=os.path.join(prefix, "jenkins", "venv", "py36", "pyq", "Scripts")) 

59 res["Python27"] = res["py27"] 

60 res["Python36"] = res["py36"] 

61 res["Python37"] = res["py37"] 

62 res["Python38"] = res["py38"] 

63 res["Anaconda2"] = res["anaconda2"] 

64 res["Anaconda3"] = res["anaconda3"] 

65 return res 

66 

67 res = {} 

68 for k in [-1, 0, 1]: 

69 vers = (sys.version_info[0], sys.version_info[1] + k) 

70 key = "Python%d%d" % vers 

71 res[key] = get_interpreter(platform=platform) 

72 res["py%d%d" % vers] = res[key] 

73 return res 

74 

75 

76def default_jenkins_jobs(filter=None, neg_filter=None, root=None, platform=None): 

77 """ 

78 Default list of :epkg:`Jenkins` jobs. 

79 

80 @param filter keep a subset of jobs (regular expression) 

81 @param neg_filter remove a subset of jobs (regular expression) 

82 @param root where to find yml project 

83 @param platform platform or None for the current one 

84 @return list 

85 

86 It produces a subset of the following list of jobs: 

87 

88 .. runpython:: 

89 

90 from ensae_teaching_cs.automation.jenkins_helper import default_jenkins_jobs 

91 modules = default_jenkins_jobs() 

92 text = [str(m) for m in modules] 

93 print("\\n".join(text)) 

94 """ 

95 if platform is None: 

96 platform = sys.platform 

97 plat = "win" if platform.startswith("win") else "lin" 

98 yml = [] 

99 pattern = "https://raw.githubusercontent.com/sdpython/%s/%s/.local.jenkins.{0}.yml".format( 

100 plat) 

101 modules = ["_automation"] + get_teaching_modules(branch=True) 

102 for c in modules: 

103 if ':' in c: 

104 c, branch = c.split(':') 

105 else: 

106 branch = 'master' 

107 yml.append(pattern % (c, branch)) 

108 

109 if filter is not None or neg_filter is not None: 

110 reg = re.compile(filter if filter else ".*") 

111 neg_reg = re.compile(neg_filter if neg_filter else "^$") 

112 res = default_jenkins_jobs(platform=platform) 

113 new_res = [] 

114 for row in res: 

115 if isinstance(row, str): 

116 if reg.search(row) and not neg_reg.search(row): 

117 new_res.append(row) 

118 elif isinstance(row, tuple): 

119 if reg.search(row[0]) and not neg_reg.search(row[0]): 

120 new_res.append(row) 

121 elif isinstance(row, list): 

122 # list 

123 sub = [] 

124 for item in row: 

125 if isinstance(item, str): 

126 if reg.search(item) and not neg_reg.search(item): 

127 sub.append(item) 

128 elif isinstance(item, tuple): 

129 if reg.search(item[0]) and not neg_reg.search(item[0]): 

130 sub.append(item) 

131 else: 

132 raise TypeError(f"{item} - {type(item)}") 

133 if len(sub) > 0: 

134 new_res.append(sub) 

135 else: 

136 raise TypeError(f"{row} - {type(row)}") 

137 return new_res 

138 else: 

139 context = {'Python37': 'python3.7', 

140 'Python38': 'python3.8', 'Python39': 'python3.9'} 

141 yml_data = load_yaml(pattern % 

142 ('_automation', 'master'), context=context) 

143 pyth = yml_data[0]['python'] 

144 res = [] 

145 for pyt in pyth: 

146 v = pyt['VERSION'] + 0.01 

147 vers = (int(v), int((v - int(v)) * 10)) 

148 res.extend(["standalone [local_pypi] [py%d%d]" % vers, 

149 ("pymyinstall [update_modules] [py%d%d]" % vers, 

150 "H H(0-1) * * 5")]) 

151 res.extend(('yml', c, 'H H(0-1) * * %d' % (i % 7)) 

152 for i, c in enumerate(yml)) 

153 return res 

154 

155 

156def setup_jenkins_server(js, github="sdpython", modules=None, 

157 overwrite=False, location=None, prefix="", 

158 delete_first=False, disable_schedule=False, 

159 fLOG=noLOG): 

160 """ 

161 Sets up many jobs on :epkg:`Jenkins`. 

162 

163 @param js (JenkinsExt) jenkins server (specially if you need credentials) 

164 @param github github account if it does not start with *http://*, 

165 the link to git repository of the project otherwise 

166 @param modules modules for which to generate the :epkg:`Jenkins` job 

167 (see @see fn default_jenkins_jobs which provides the default value 

168 if *modules* is None) 

169 @param overwrite do not create the job if it already exists 

170 @param location None for default or a local folder 

171 @param prefix add a prefix to the name 

172 @param delete_first remove all jobs first 

173 @param disable_schedule disable schedule for all jobs 

174 @param fLOG logging function 

175 @return list of created jobs 

176 

177 *modules* is a list defined as follows: 

178 

179 * each element can be a string or a tuple (string, schedule time) or a list 

180 * if it is a list, it contains a list of elements defined as previously 

181 * the job at position i is not scheduled, it will start after the last 

182 job at position i-1 whether or not it fails 

183 """ 

184 platform = js.platform 

185 if modules is None: 

186 modules = default_jenkins_jobs(platform=platform) 

187 r = setup_jenkins_server_yml(js, github=github, modules=modules, get_jenkins_script=None, 

188 overwrite=overwrite, location=location, prefix=prefix, 

189 delete_first=delete_first, disable_schedule=disable_schedule) 

190 return r