Coverage for pyquickhelper/jenkinshelper/jenkins_server.py: 93%
501 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Extends Jenkins Server from :epkg:`python-jenkins`.
4"""
6import os
7import sys
8import socket
9import hashlib
10import re
11from xml.sax.saxutils import escape
12import requests
13import jenkins
14from ..loghelper.flog import noLOG
15from ..pycode.windows_scripts import windows_jenkins, windows_jenkins_any
16from ..pycode.windows_scripts import windows_jenkins_27_conda, windows_jenkins_27_def
17from ..pycode.linux_scripts import linux_jenkins, linux_jenkins_any
18from ..pycode.build_helper import private_script_replacements
19from .jenkins_exceptions import JenkinsExtException, JenkinsJobException
20from .jenkins_server_template import _config_job, _trigger_up, _trigger_time, _git_repo, _task_batch_win, _task_batch_lin
21from .jenkins_server_template import _trigger_startup, _publishers, _file_creation, _wipe_repo, _artifacts, _cleanup_repo
22from .yaml_helper import enumerate_processed_yml
23from .jenkins_helper import jenkins_final_postprocessing, get_platform
25_timeout_default = 1200
27_default_engine_paths = {
28 "windows": {
29 "__PY36__": "__PY36__",
30 "__PY37__": "__PY37__",
31 "__PY38__": "__PY38__",
32 "__PY39__": "__PY39__",
33 "__PY310__": "__PY310__",
34 "__PY36_X64__": "__PY36_X64__",
35 "__PY37_X64__": "__PY37_X64__",
36 "__PY38_X64__": "__PY38_X64__",
37 "__PY39_X64__": "__PY39_X64__",
38 "__PY310_X64__": "__PY310_X64__",
39 },
40}
43def _modified_windows_jenkins(requirements_local, requirements_pypi, module="__MODULE__",
44 port="__PORT__", platform=None):
45 return private_script_replacements(
46 linux_jenkins, module,
47 (requirements_local, requirements_pypi),
48 port, raise_exception=False,
49 default_engine_paths=_default_engine_paths,
50 platform=get_platform(platform))
53def _modified_linux_jenkins(requirements_local, requirements_pypi, module="__MODULE__",
54 port="__PORT__", platform=None):
55 return private_script_replacements(
56 windows_jenkins, module,
57 (requirements_local, requirements_pypi),
58 port, raise_exception=False,
59 default_engine_paths=_default_engine_paths,
60 platform=get_platform(platform))
63def _modified_windows_jenkins_27(requirements_local, requirements_pypi, module="__MODULE__",
64 port="__PORT__", anaconda=True, platform=None):
65 return private_script_replacements(
66 windows_jenkins_27_conda if anaconda else windows_jenkins_27_def,
67 module, (requirements_local, requirements_pypi),
68 port, raise_exception=False,
69 default_engine_paths=_default_engine_paths,
70 platform=get_platform(platform))
73def _modified_windows_jenkins_any(requirements_local, requirements_pypi, module="__MODULE__",
74 port="__PORT__", platform=None):
75 res = private_script_replacements(
76 windows_jenkins_any, module,
77 (requirements_local, requirements_pypi),
78 port, raise_exception=False,
79 default_engine_paths=_default_engine_paths,
80 platform=get_platform(platform))
81 return res.replace("virtual_env_suffix=%2", "virtual_env_suffix=___SUFFIX__")
84def _modified_linux_jenkins_any(requirements_local, requirements_pypi, module="__MODULE__",
85 port="__PORT__", platform=None):
86 res = private_script_replacements(
87 linux_jenkins_any, module,
88 (requirements_local, requirements_pypi),
89 port, raise_exception=False,
90 default_engine_paths=_default_engine_paths,
91 platform=get_platform(platform))
92 return res.replace("virtual_env_suffix=%2", "virtual_env_suffix=___SUFFIX__")
95class JenkinsExt(jenkins.Jenkins):
97 """
98 Extensions for the :epkg:`Jenkins` server
99 based on module :epkg:`python-jenkins`.
101 .. index:: Jenkins, Jenkins extensions
103 Some useful :epkg:`Jenkins` extensions:
105 * `Credentials Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Credentials+Plugin>`_
106 * `Extra Column Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
107 * `Git Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Git+Client+Plugin>`_
108 * `GitHub Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Github+Plugin>`_
109 * `GitLab Client Plugin <https://wiki.jenkins-ci.org/display/JENKINS/GitLab+Plugin>`_
110 * `Matrix Project Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Matrix+Project+Plugin>`_
111 * `Build Pipeline Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Pipeline+Plugin>`_
113 The whole class can define many different engines.
114 A job can send a mail at the end of the job execution.
115 """
117 _config_job = _config_job # pylint: disable=W0127
118 _trigger_up = _trigger_up # pylint: disable=W0127
119 _trigger_time = _trigger_time # pylint: disable=W0127
120 _trigger_startup = _trigger_startup # pylint: disable=W0127
121 _git_repo = _git_repo # pylint: disable=W0127
122 _task_batch_win = _task_batch_win # pylint: disable=W0127
123 _task_batch_lin = _task_batch_lin # pylint: disable=W0127
124 _publishers = _publishers # pylint: disable=W0127
125 _wipe_repo = _wipe_repo # pylint: disable=W0127
126 _artifacts = _artifacts # pylint: disable=W0127
127 _cleanup_repo = _cleanup_repo # pylint: disable=W0127
129 def __init__(self, url, username=None, password=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
130 mock=False, engines=None, platform=None, pypi_port=8067, fLOG=noLOG,
131 mails=None):
132 """
133 @param url url of the server
134 @param username username
135 @param password password
136 @param timeout timeout
137 @param mock True by default, if False, avoid talking to the server
138 @param engines list of Python engines *{name: path to python.exe}*
139 @param platform platform of the Jenkins server
140 @param pypi_port pypi port used for the documentation server
141 @param mails (str) list of mails to contact in case of a mistaje
142 @param fLOG logging function
144 If *platform* is None, it is replace by the value returned
145 by @see fn get_platform.
146 """
147 if platform is None:
148 platform = get_platform(platform)
149 jenkins.Jenkins.__init__(
150 self, url, username, password, timeout=timeout)
151 self._mock = mock
152 self.platform = platform
153 self.pypi_port = pypi_port
154 self.mails = mails
155 self.fLOG = fLOG
156 if engines is None:
157 engines = {"default": os.path.dirname(sys.executable)}
158 self.engines = engines
159 for k, v in self.engines.items():
160 if v.endswith(".exe"):
161 raise FileNotFoundError( # pragma: no cover
162 f"{k}:{v} is not a folder")
163 if " " in v:
164 raise JenkinsJobException( # pragma: no cover
165 "No space allowed in engine path: " + v)
167 @property
168 def Engines(self):
169 """
170 @return the available engines
171 """
172 return self.engines
174 def jenkins_open(self, req, add_crumb=True, resolve_auth=True): # pragma: no cover
175 '''
176 Overloads the same method from module :epkg:`python-jenkins`
177 to replace string by bytes.
179 @param req see :epkg:`Jenkins API`
180 @param add_crumb see :epkg:`Jenkins API`
181 @param resolve_auth see :epkg:`Jenkins API`
182 '''
183 if self._mock:
184 raise JenkinsExtException("mocking server, cannot be open")
186 response = self.jenkins_request(
187 req=req, add_crumb=add_crumb, resolve_auth=resolve_auth)
188 if response is None:
189 raise jenkins.EmptyResponseException(
190 f"Error communicating with server[{self.server}]: empty response")
191 return response.content
193 def delete_job(self, name): # pragma: no cover
194 '''
195 Deletes :epkg:`Jenkins` job permanently.
197 :param name: name of :epkg:`Jenkins` job, ``str``
198 '''
199 if self._mock:
200 return
201 r = self._get_job_folder(name)
202 if r is None:
203 raise JenkinsExtException(f'delete[{name}] failed (no job)')
205 folder_url, short_name = self._get_job_folder(name)
206 if folder_url is None:
207 raise ValueError(f"folder_url is None for job '{name}'")
208 self.jenkins_open(requests.Request(
209 'POST', self._build_url(jenkins.DELETE_JOB, locals())
210 ))
211 if self.job_exists(name) or self.job_exists(short_name):
212 raise jenkins.JenkinsException(f'delete[{name}] failed')
214 def get_jobs(self, folder_depth=0, folder_depth_per_request=10, view_name=None):
215 """
216 Gets the list of all jobs recursively to the given folder depth,
217 see `get_all_jobs
218 <https://python-jenkins.readthedocs.org/en/latest/api.html
219 #jenkins.Jenkins.get_all_jobs>`_.
221 @return list of jobs, ``[ { str: str} ]``
222 """
223 return jenkins.Jenkins.get_jobs(self, folder_depth=folder_depth,
224 folder_depth_per_request=folder_depth_per_request,
225 view_name=view_name)
227 def delete_all_jobs(self): # pragma: no cover
228 """
229 Deletes all jobs permanently.
231 @return list of deleted jobs
232 """
233 jobs = self.get_jobs()
234 res = []
235 for k in jobs:
236 self.fLOG("[jenkins] remove job", k["name"])
237 self.delete_job(k["name"])
238 res.append(k["name"])
239 return res
241 def get_jenkins_job_name(self, job):
242 """
243 Infers a name for the jenkins job.
245 @param job str
246 @return name
247 """
248 if "<--" in job:
249 job = job.split("<--")[0]
250 if job.startswith("custom "):
251 return job.replace(" ", "_").replace("[", "").replace("]", "").strip("_")
252 def_prefix = ["doc", "setup", "setup_big"]
253 def_prefix.extend(self.engines.keys())
254 for prefix in def_prefix:
255 p = f"[{prefix}]"
256 if p in job:
257 job = p + " " + job.replace(" " + p, "")
258 return job.replace(" ", "_").replace("[", "").replace("]", "").strip("_")
260 def get_engine_from_job(self, job, return_key=False):
261 """
262 Extracts the engine from the job definition,
263 it should be like ``[engine]``.
265 @param job job string
266 @param return_key return the engine name too
267 @return engine or tuple(engine, name)
269 If their is no engine definition, the system
270 uses the default one (key=*default*) if it was defined.
271 Otherwise, it raises an exception.
272 """
273 res = None
274 spl = job.split()
275 for s in spl:
276 t = s.strip(" []")
277 if t in self.engines:
278 res = self.engines[t]
279 key = t
280 break
281 if res is None and "default" in self.engines:
282 res = self.engines["default"]
283 key = "default"
284 if res is None:
285 raise JenkinsJobException( # pragma: no cover
286 "Unable to find engine in job '{}', available: {}".format(
287 job, ", ".join(self.engines.keys())))
288 if "[27]" in job and "python34" in res.lower(): # pragma: no cover
289 mes = "\n".join(f" {k}={v}"
290 for k, v in sorted(self.engines.items()))
291 raise ValueError(
292 "Python mismatch in version:\nJOB = {0}\nRES = {1}\nENGINES\n{2}"
293 "".format(job, res, mes))
294 return (res, key) if return_key else res
296 def get_cmd_standalone(self, job):
297 """
298 Custom command for :epkg:`Jenkins` (such as updating conda)
300 @param job module and options
301 @return script
302 """
303 spl = job.split()
304 if spl[0] != "standalone":
305 raise JenkinsExtException( # pragma: no cover
306 "the job should start by standalone: " + job)
308 if self.platform.startswith("win"):
309 # windows
310 if "[conda_update]" in spl:
311 cmd = "__ENGINE__\\Scripts\\conda update -y --all"
312 elif "[local_pypi]" in spl:
313 cmd = "if not exist ..\\..\\local_pypi mkdir ..\\local_pypi"
314 cmd += "\nif not exist ..\\..\\local_pypi\\local_pypi_server mkdir ..\\..\\local_pypi\\local_pypi_server"
315 cmd += "\necho __ENGINE__\\..\\Scripts\\pypi-server.exe -v -u -p __PORT__ --disable-fallback "
316 cmd += "..\\..\\local_pypi\\local_pypi_server > ..\\..\\local_pypi\\local_pypi_server\\start_local_pypi.bat"
317 cmd = cmd.replace("__PORT__", str(self.pypi_port))
318 elif "[update]" in spl:
319 cmd = "__ENGINE__\\python -u -c \"from pymyinstall.packaged import update_all;"
320 cmd += "update_all(temp_folder='build/update_modules', "
321 cmd += "verbose=True, source='2')\""
322 elif "[install]" in spl:
323 cmd = "__ENGINE__\\python -u -c \"from pymyinstall.packaged import install_all;install_all"
324 cmd += "(temp_folder='build/update_modules', "
325 cmd += "verbose=True, source='2')\""
326 else:
327 raise JenkinsExtException(
328 "cannot interpret job: " + job) # pragma: no cover
330 engine = self.get_engine_from_job(job)
331 cmd = cmd.replace("__ENGINE__", engine)
332 return cmd
333 else: # pragma: no cover
334 if "[conda_update]" in spl:
335 cmd = "__ENGINE__/bin/conda update -y --all"
336 elif "[local_pypi]" in spl:
337 cmd = 'if [-f ../local_pypi ]; then mkdir "../local_pypi"; fi'
338 cmd += '\nif [-f ../local_pypi/local_pypi_server]; then mkdir "../local_pypi/local_pypi_server"; fi'
339 cmd += "\necho pypi-server -v -u -p __PORT__ --disable-fallback "
340 cmd += "../local_pypi/local_pypi_server > ../local_pypi/local_pypi_server/start_local_pypi.sh"
341 cmd = cmd.replace("__PORT__", str(self.pypi_port))
342 elif "[update]" in spl:
343 cmd = "__ENGINE__/python -u -c \"from pymyinstall.packaged import update_all;"
344 cmd += "update_all(temp_folder='build/update_modules', "
345 cmd += "verbose=True, source='2')\""
346 elif "[install]" in spl:
347 cmd = "__ENGINE__/python -u -c \"from pymyinstall.packaged import install_all;install_all"
348 cmd += "(temp_folder='build/update_modules', "
349 cmd += "verbose=True, source='2')\""
350 else:
351 raise JenkinsExtException("cannot interpret job: " + job)
353 engine = self.get_engine_from_job(job)
354 cmd = cmd.replace("__ENGINE__", engine)
355 return cmd
357 @staticmethod
358 def get_cmd_custom(job):
359 """
360 Custom script for :epkg:`Jenkins`.
362 @param job module and options
363 @return script
364 """
365 spl = job.split()
366 if spl[0] != "custom":
367 raise JenkinsExtException(
368 "the job should start by custom: " + job)
369 # we expect __SCRIPTOPTIONS__ to be replaced by a script later on
370 return "__SCRIPTOPTIONS__"
372 @staticmethod
373 def hash_string(s, le=4):
374 """
375 Hashes a string.
377 @param s string
378 @param le cut the string to the first *l* character
379 @return hashed string
380 """
381 m = hashlib.md5()
382 m.update(s.encode("ascii"))
383 r = m.hexdigest().upper()
384 if len(r) < le:
385 return r # pragma: no cover
386 m = le // 2
387 return r[:m] + r[len(r) - le + m:]
389 def extract_requirements(self, job):
390 """
391 Extracts the requirements for a job.
393 @param job job name
394 @return 3-tuple job, local requirements, pipy requirements
396 Example::
398 "pyensae <-- pyquickhelper <---- qgrid"
400 The function returns::
402 (pyensae, ["pyquickhelper"], ["qgrid"])
403 """
404 if "<--" in job:
405 spl = job.split("<--")
406 job = spl[0]
407 rl, rp = None, None
408 for o in spl[1:]:
409 if o.startswith("--"):
410 rp = [_.strip("- ") for _ in o.split(",")]
411 else:
412 rl = [_.strip() for _ in o.split(",")]
413 return job, rl, rp
414 return job, None, None
416 def get_jenkins_script(self, job):
417 """
418 Builds the :epkg:`Jenkins` script for a module and its options.
420 @param job module and options
421 @return script
423 Method @see me setup_jenkins_server describes which tags this method can interpret.
424 The method allow command such as ``[custom...]``, they will be
425 run in a virtual environment as ``setup.py custom...``.
426 Parameter *job* can be ``empty``, in that case, this function returns an empty string.
427 Requirements local and from pipy can be specified by added in the job name:
429 * ``<-- module1, module2`` for local requirements
430 """
431 job_verbose = job
433 def replacements(cmd, engine, python, suffix, module_name):
434 res = cmd.replace("__ENGINE__", engine) \
435 .replace("__PYTHON__", python) \
436 .replace("__SUFFIX__", suffix + "_" + job_hash) \
437 .replace("__PORT__", str(self.pypi_port)) \
438 .replace("__MODULE__", module_name) # suffix for the virtual environment and module name
439 if "[27]" in job:
440 res = res.replace("__PYTHON27__", python)
441 if "__DEFAULTPYTHON__" in res:
442 if "default" not in self.engines:
443 raise JenkinsExtException( # pragma: no cover
444 f"a default engine (Python 3.4) must be defined for script using Python 27, job={job}")
445 res = res.replace("__DEFAULTPYTHON__",
446 os.path.join(self.engines["default"], "python"))
448 # patch for pyquickhelper
449 if "PACTHPQ" in res:
450 if hasattr(self, "PACTHPQ"):
451 if not hasattr(self, "pyquickhelper"):
452 raise RuntimeError( # pragma: no cover
453 f"this should not happen:\n{job_verbose}\n---\n{res}")
454 if "pyquickhelper" in module_name:
455 repb = "@echo ~~SET set PYTHONPATH=src\nset PYTHONPATH=src"
456 else:
457 repb = "@echo ~~SET set PYTHONPATH={0}\nset PYTHONPATH={0}".format(self.pyquickhelper.replace(
458 "\\\\", "\\"))
459 repe = "@echo ~~SET set PYTHONPATH=\nset PYTHONPATH="
460 else:
461 repb = ""
462 repe = ""
463 res = res.replace("__PACTHPQb__", repb).replace(
464 "__PACTHPQe__", repe)
466 if "__" in res:
467 raise JenkinsJobException( # pragma: no cover
468 f"unable to interpret command line: {job_verbose}\nCMD: {cmd}\nRES:\n{res}")
470 # patch to avoid installing pyquickhelper when testing
471 # pyquickhelper
472 if module_name == "pyquickhelper":
473 lines = res.split("\n")
474 for i, line in enumerate(lines):
475 if "/simple/ pyquickhelper" in line and "--find-links http://localhost" in line:
476 lines[i] = "" # pragma: no cover
477 res = "\n".join(lines)
479 return res
481 # job hash
482 job_hash = JenkinsExt.hash_string(job)
484 # extact requirements
485 job, requirements_local, requirements_pypi = self.extract_requirements(
486 job)
487 spl = job.split()
488 module_name = spl[0]
490 if self.platform.startswith("win"):
491 # windows
492 engine, namee = self.get_engine_from_job(job, True)
493 python = os.path.join(engine, "python.exe")
495 if len(spl) == 1:
496 script = _modified_windows_jenkins(
497 requirements_local, requirements_pypi, platform=self.platform)
498 if not isinstance(script, list):
499 script = [script]
500 return [replacements(s, engine, python, namee + "_" + job_hash, module_name) for s in script]
502 if len(spl) == 0:
503 raise ValueError("job is empty") # pragma: no cover
505 if spl[0] == "standalone":
506 # conda update
507 return self.get_cmd_standalone(job)
509 if spl[0] == "custom":
510 # custom script
511 return JenkinsExt.get_cmd_custom(job)
513 if spl[0] == "empty":
514 return "" # pragma: no cover
516 if len(spl) in [2, 3, 4, 5]:
517 # step 1: define the script
519 if "[test_local_pypi]" in spl: # pragma: no cover
520 cmd = """__PYTHON__ -u setup.py test_local_pypi"""
521 cmd = "auto_setup_test_local_pypi.bat __PYTHON__"
522 elif "[update_modules]" in spl:
523 cmd = """__PYTHON__ -u -c "import sys;sys.path.append('src');from pymyinstall.packaged import update_all;""" + \
524 """update_all(temp_folder='build/update_modules', verbose=True, source='2')" """
525 elif "[UT]" in spl:
526 parameters = [_ for _ in spl if _.startswith(
527 "{") and _.endswith("}")]
528 if len(parameters) != 1:
529 raise ValueError( # pragma: no cover
530 "Unable to extract parameters for the unittests:"
531 "\n{0}".format(" ".join(spl)))
532 p = parameters[0].replace("_", " ").strip("{}")
533 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
534 "__COMMAND__", "unittests " + p)
535 elif "[LONG]" in spl:
536 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
537 "__COMMAND__", "unittests_LONG")
538 elif "[SKIP]" in spl:
539 cmd = _modified_windows_jenkins_any( # pragma: no cover
540 requirements_local, requirements_pypi, platform=self.platform).replace(
541 "__COMMAND__", "unittests_SKIP")
542 elif "[GUI]" in spl:
543 cmd = _modified_windows_jenkins_any( # pragma: no cover
544 requirements_local, requirements_pypi, platform=self.platform).replace(
545 "__COMMAND__", "unittests_GUI")
546 elif "[27]" in spl:
547 cmd = _modified_windows_jenkins_27(
548 requirements_local, requirements_pypi, anaconda=" [anaconda" in job, platform=self.platform)
549 if not isinstance(cmd, list):
550 cmd = [cmd] # pragma: no cover
551 else:
552 cmd = list(cmd)
553 if spl[0] == "pyquickhelper":
554 # exception for this job, we don't want to import pyquickhelper
555 # c:/jenkins/pymy/anaconda2_pyquickhelper_27/../virtual/pyquickhelper_conda27vir/Scripts/pip
556 # install --no-cache-dir --index
557 # http://localhost:8067/simple/ pyquickhelper
558 for i in range(0, len(cmd)):
559 lines = cmd[i].split("\n")
560 lines = [
561 (_ if "simple/ pyquickhelper" not in _ else "rem do not import pyquickhelper") for _ in lines]
562 cmd[i] = "\n".join(lines)
563 elif "[doc]" in spl:
564 # documentation
565 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi, platform=self.platform).replace(
566 "__COMMAND__", "build_sphinx")
567 else:
568 cmd = _modified_windows_jenkins(
569 requirements_local, requirements_pypi, platform=self.platform)
570 for pl in spl[1:]:
571 if pl.startswith("[custom_") and pl.endswith("]"):
572 cus = pl.strip("[]")
573 cmd = _modified_windows_jenkins_any(requirements_local, requirements_pypi,
574 platform=self.platform).replace("__COMMAND__", cus)
576 # step 2: replacement (python __PYTHON__, virtual environnement
577 # __SUFFIX__)
579 cmds = cmd if isinstance(cmd, list) else [cmd]
580 res = []
581 for cmd in cmds:
582 cmdn = replacements(cmd, engine, python,
583 namee + "_" + job_hash, module_name)
584 if "run27" in cmdn and (
585 "Python34" in cmdn or "Python35" in cmdn or
586 "Python36" in cmdn or "Python37" in cmdn or
587 "Python38" in cmdn or "Python39" in cmdn):
588 raise ValueError( # pragma: no cover
589 "Python version mismatch\nENGINE\n{2}\n----BEFORE"
590 "\n{0}\n-----\nAFTER\n-----\n{1}".format(cmd, cmdn, engine))
591 res.append(cmdn)
593 return res
594 else:
595 raise ValueError("unable to interpret: " +
596 job) # pragma: no cover
597 else: # pragma: no cover
598 # linux
599 engine, namee = self.get_engine_from_job(job, True)
600 if engine is None:
601 python = "python%d.%d" % sys.version_info[:2]
602 elif namee.startswith('py'):
603 vers = (int(namee[2:3]), int(namee[3:]))
604 python = "python%d.%d" % vers
605 else:
606 raise ValueError(
607 f"Unable to handle engine ='{engine}', namee='{namee}'.")
609 if len(spl) == 1:
610 script = _modified_linux_jenkins(
611 requirements_local, requirements_pypi, platform=self.platform)
612 if not isinstance(script, list):
613 script = [script]
614 return [replacements(s, engine, python, namee + "_" + job_hash, module_name) for s in script]
616 elif len(spl) == 0:
617 raise ValueError("job is empty")
619 elif spl[0] == "standalone":
620 # conda update
621 return self.get_cmd_standalone(job)
623 elif spl[0] == "empty":
624 return ""
626 elif len(spl) in [2, 3, 4, 5]:
627 # step 1: define the script
629 if "[test_local_pypi]" in spl:
630 cmd = """__PYTHON__ -u setup.py test_local_pypi"""
631 cmd = "auto_setup_test_local_pypi.bat __PYTHON__"
632 elif "[update_modules]" in spl:
633 cmd = """__PYTHON__ -u -c "import sys;sys.path.append('src');from pymyinstall.packaged import update_all;""" + \
634 """update_all(temp_folder='build/update_modules', verbose=True, source='2')" """
636 else:
637 cmd = _modified_linux_jenkins(
638 requirements_local, requirements_pypi, platform=self.platform)
639 for pl in spl[1:]:
640 if pl.startswith("[custom_") and pl.endswith("]"):
641 cus = pl.strip("[]")
642 cmd = _modified_linux_jenkins_any(requirements_local, requirements_pypi,
643 platform=self.platform).replace("__COMMAND__", cus)
645 # step 2: replacement (python __PYTHON__, virtual environnement
646 # __SUFFIX__)
648 cmds = cmd if isinstance(cmd, list) else [cmd]
649 res = []
650 for cmd in cmds:
651 cmdn = replacements(cmd, engine, python,
652 namee + "_" + job_hash, module_name)
653 if "run27" in cmdn and (
654 "Python34" in cmdn or "Python35" in cmdn or
655 "Python36" in cmdn or "Python37" in cmdn or
656 "Python38" in cmdn or "Python39" in cmdn):
657 raise ValueError(
658 f"Python version mismatch\nENGINE\n{engine}\n----BEFORE\n{cmd}\n-----\nAFTER\n-----\n{cmdn}")
659 res.append(cmdn)
661 return res
663 # other possibilities
664 raise NotImplementedError("On Linux, unable to interpret: " + job)
666 def adjust_scheduler(self, scheduler, adjust_scheduler=True):
667 """
668 Adjusts the scheduler to avoid having two jobs starting at the same time,
669 jobs are delayed by an hour, two hours, three hours...
671 @param scheduler existing scheduler
672 @param adjust_scheduler True to change it
673 @return new scheduler (only hours are changed)
675 The function uses member ``_scheduled_jobs``.
676 It creates it if it does not exist.
677 """
678 if not adjust_scheduler:
679 return scheduler # pragma: no cover
680 if scheduler is None:
681 raise ValueError("scheduler is None") # pragma: no cover
682 if not hasattr(self, "_scheduled_jobs"):
683 self._scheduled_jobs = {}
684 if scheduler not in self._scheduled_jobs:
685 self._scheduled_jobs[scheduler] = 1
686 return scheduler
687 else:
688 if "H(" in scheduler:
689 cp = re.compile("H[(]([0-9]+-[0-9]+)[)]")
690 f = cp.findall(scheduler)
691 if len(f) != 1:
692 raise ValueError( # pragma: no cover
693 f"Unable to find hours in the scheduler '{scheduler}', expects 'H(a-b)'")
694 a, b = f[0].split('-')
695 a0 = a
696 a = int(a)
697 b = int(b)
698 new_value = scheduler
699 rep = f'H({f[0]})'
700 iter = 0
701 while iter < 100 and (new_value in self._scheduled_jobs or (a0 == a)):
702 a += 1
703 b += 1
704 if a >= 24 or b > 24:
705 a = 0 # pragma: no cover
706 b = 1 + iter // 24 # pragma: no cover
707 r = 'H(%d-%d)' % (a, b)
708 new_value = scheduler.replace(rep, r)
709 iter += 1
710 scheduler = new_value
711 self._scheduled_jobs[
712 scheduler] = self._scheduled_jobs.get(scheduler, 0) + 1
713 return scheduler
715 def create_job_template(self, name, git_repo, credentials="", upstreams=None, script=None,
716 location=None, keep=10, scheduler=None, py27=False, description=None,
717 default_engine_paths=None, success_only=False, update=False,
718 timeout=_timeout_default, additional_requirements=None,
719 return_job=False, adjust_scheduler=True, clean_repo=True,
720 branch='master', **kwargs):
721 """
722 Adds a job to the :epkg:`Jenkins` server.
724 @param name name
725 @param credentials credentials
726 @param git_repo git repository
727 @param upstreams the build must run after... (even if failures),
728 must be None in that case
729 @param script script to execute or list of scripts
730 @param keep number of builds to keep
731 @param location location of the build
732 @param scheduler add a schedule time (upstreams must be None in that case)
733 @param py27 python 2.7 (True) or Python 3 (False)
734 @param description add a description to the job
735 @param default_engine_paths define the default location for python engine,
736 should be dictionary ``{ engine: path }``, see below.
737 @param success_only only triggers the job if the previous one was successful
738 @param update update the job instead of creating it
739 @param additional_requirements requirements for this module built by this Jenkins server,
740 otherthise, we assume they are available
741 on the installed distribution
742 @param timeout specify a timeout
743 @param kwargs additional parameters
744 @param adjust_scheduler adjust the scheduler of a job so that it is delayed if this spot
745 is already taken
746 @param return_job return job instead of submitting the job
747 @param clean_repo clean the repository before building (default is yes)
748 @param branch default branch
750 The job can be modified on Jenkins. To add a time trigger::
752 H H(13-14) * * *
754 Same trigger but once every week and not every day (Sunday for example)::
756 H H(13-14) * * 0
758 Parameter *success_only* prevents a job from running if the previous one failed.
759 Options *success_only* must be specified.
760 Parameter *update* updates a job instead of creating it.
761 """
762 if ':' in name:
763 raise ValueError( # pragma: no cover
764 f"Unexpected value name={name!r} and branch={branch!r}.")
765 if 'platform' in kwargs:
766 raise NameError( # pragma: no cover
767 "Parameter 'platform' should be set up in the constructor.")
768 if script is None:
769 if self.platform.startswith("win"):
770 if default_engine_paths is None and "default" in self.engines:
771 ver = "__PY%d%d__" % sys.version_info[:2]
772 pat = os.path.join(self.engines["default"], "python")
773 default_engine_paths = dict(
774 windows={ver: pat, "__PYTHON__": pat})
776 script = private_script_replacements(
777 windows_jenkins, "____", additional_requirements, "____",
778 raise_exception=False, platform=self.platform,
779 default_engine_paths=default_engine_paths)
781 hash = JenkinsExt.hash_string(script)
782 script = script.replace("__SUFFIX__", hash)
783 else:
784 raise JenkinsExtException(
785 "no default script for linux") # pragma: no cover
787 if upstreams is not None and len(upstreams) > 0 and scheduler is not None:
788 raise JenkinsExtException(
789 f"upstreams and scheduler cannot be not null at the same time: {name}")
791 # overwrite parameters with job_options
792 job_options = kwargs.get('job_options', None)
793 if job_options is not None:
794 job_options = job_options.copy()
795 if "scheduler" in job_options:
796 scheduler = job_options["scheduler"]
797 del job_options["scheduler"]
798 if "git_repo" in job_options:
799 git_repo = job_options["git_repo"]
800 del job_options["git_repo"]
801 if "credentials" in job_options:
802 credentials = job_options["credentials"]
803 del job_options["credentials"]
805 if upstreams is not None and len(upstreams) > 0:
806 trigger = JenkinsExt._trigger_up \
807 .replace("__UP__", ",".join(upstreams)) \
808 .replace("__FAILURE__", "SUCCESS" if success_only else "FAILURE") \
809 .replace("__ORDINAL__", "0" if success_only else "2") \
810 .replace("__COLOR__", "BLUE" if success_only else "RED")
811 elif scheduler is not None:
812 if scheduler.lower() == "startup":
813 trigger = JenkinsExt._trigger_startup
814 elif scheduler.lower() == "NONE":
815 trigger = "" # pragma: no cover
816 else:
817 new_scheduler = self.adjust_scheduler(
818 scheduler, adjust_scheduler)
819 trigger = JenkinsExt._trigger_time.replace(
820 "__SCHEDULER__", new_scheduler)
821 if description is not None:
822 description = description.replace(scheduler, new_scheduler)
823 scheduler = new_scheduler
824 else:
825 trigger = ""
827 if not isinstance(script, list):
828 script = [script]
830 underscore = re.compile("(__[A-Z_]+__)")
832 # we modify the scripts
833 script_mod = []
834 for scr in script:
835 search = underscore.search(scr)
836 if search:
837 raise ValueError( # pragma: no cover
838 "script still contains __\ndefault_engine_paths: {}\n"
839 "found: {}\nscr:\n{}\nSCRIPT:\n{}\n".format(
840 default_engine_paths, search.groups()[0],
841 scr, str(script)))
842 script_mod.append(scr)
844 # wrappers
845 bwrappers = []
847 # repo
848 if clean_repo:
849 wipe = JenkinsExt._wipe_repo
850 bwrappers.append(JenkinsExt._cleanup_repo)
851 else:
852 wipe = ""
853 if git_repo is None:
854 git_repo_xml = ""
855 else:
856 if not isinstance(git_repo, str):
857 raise TypeError( # pragma: no cover
858 f"git_repo must be str not '{git_repo}'")
859 git_repo_xml = JenkinsExt._git_repo \
860 .replace("__GITREPO__", git_repo) \
861 .replace("__WIPE__", wipe) \
862 .replace("__BRANCH__", branch) \
863 .replace("__CRED__", f"<credentialsId>{credentials}</credentialsId>")
865 # additional scripts
866 before = []
867 if job_options is not None:
868 if 'scripts' in job_options:
869 lscripts = job_options['scripts']
870 for scr in lscripts:
871 au = _file_creation.replace("__FILENAME__", scr["name"]) \
872 .replace("__CONTENT__", scr["content"])
873 if "__" in au:
874 raise RuntimeError(
875 f"Unable to fully replace expected string in:\n{au}")
876 before.append(au)
877 del job_options['scripts']
878 if len(job_options) > 0:
879 keys = ", ".join(
880 ["credentials", "git_repo", "scheduler", "scripts"])
881 raise ValueError( # pragma: no cover
882 "Unable to process options\n{0}\nYou can specify the "
883 "following options:\n{1}".format(job_options, keys))
885 # scripts
886 # tasks is XML, we need to encode s into XML format
887 if self.platform.startswith("win"):
888 scr = JenkinsExt._task_batch_win
889 else:
890 scr = JenkinsExt._task_batch_lin
891 tasks = before + [scr.replace("__SCRIPT__", escape(s))
892 for s in script_mod]
894 # location
895 if location is not None and "<--" in location:
896 raise RuntimeError( # pragma: no cover
897 "this should not happen")
898 location = "" if location is None else f"<customWorkspace>{location}</customWorkspace>"
900 # emailing
901 publishers = []
902 mails = kwargs.get("mails", None)
903 if mails is not None:
904 publishers.append( # pragma: no cover
905 JenkinsExt._publishers.replace("__MAIL__", mails))
906 publishers.append(JenkinsExt._artifacts.replace(
907 "__PATTERN__", "dist/*.whl,dist/*.zip"))
909 # replacements
910 conf = JenkinsExt._config_job
911 rep = dict(__KEEP__=str(keep),
912 __TASKS__="\n".join(tasks),
913 __TRIGGER__=trigger,
914 __LOCATION__=location,
915 __DESCRIPTION__="" if description is None else description,
916 __GITREPOXML__=git_repo_xml,
917 __TIMEOUT__=str(timeout),
918 __PUBLISHERS__="\n".join(publishers),
919 __BUILDWRAPPERS__="\n".join(bwrappers))
921 for k, v in rep.items():
922 conf = conf.replace(k, v)
924 # final processing
925 conf = jenkins_final_postprocessing(conf, py27)
927 if self._mock or return_job:
928 return conf
929 if update:
930 return self.reconfig_job(name, conf) # pragma: no cover
931 return self.create_job(name, conf) # pragma: no cover
933 def process_options(self, script, options):
934 """
935 Postprocesses a script inserted in a job definition.
937 @param script script to execute (in a list)
938 @param options dictionary with options
939 @return new script
940 """
941 if not isinstance(script, list):
942 script = [script]
943 for k, v in options.items():
944 if k == "pre":
945 script.insert(0, v)
946 elif k == "post":
947 script.append(v)
948 elif k == "pre_set":
949 script = [v + "\n" + _ for _ in script] # pragma: no cover
950 elif k == "post_set":
951 script = [_ + "\n" + v for _ in script] # pragma: no cover
952 elif k == "script":
953 script = [_.replace("__SCRIPTOPTIONS__", v) for _ in script]
954 else:
955 raise JenkinsJobException( # pragma: no cover
956 "unable to interpret options: " + str(options))
957 return script
959 def setup_jenkins_server(self, github, modules, get_jenkins_script=None, overwrite=False,
960 location=None, prefix="", credentials="", update=True, yml_engine="jinja2",
961 add_environ=True, disable_schedule=False, adjust_scheduler=True):
962 """
963 Sets up many jobs in :epkg:`Jenkins`.
965 @param github github account if it does not start with *http://*,
966 the link to git repository of the project otherwise,
967 we assume all jobs in *modules* are located on the same
968 account otherwise the function will have to called twice with
969 different parameters
970 @param modules modules for which to generate the
971 @param get_jenkins_script see @see me get_jenkins_script (default value if this parameter is None)
972 @param overwrite do not create the job if it already exists
973 @param location None for default or a local folder
974 @param prefix add a prefix to the name
975 @param credentials credentials to use for the job (string or dictionary)
976 @param update update job instead of deleting it if the job already exists
977 @param yml_engine templating engine used to process yaml config files
978 @param add_environ use of local environment variables to interpret the job
979 @param adjust_scheduler adjust the scheduler of a job so that it is delayed if this spot is already taken
980 @param disable_schedule disable scheduling for all jobs
981 @return list of created jobs
983 If *credentials* are a dictionary, the function looks up
984 into it by using the git repository as a key. If it does not find
985 it, it looks for default key. If there is not found,
986 the function assumes, there is not credentials for this git repository.
988 The function *get_jenkins_script* is called with the following parameters:
990 * job
992 The extension
993 `Extra Columns Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
994 is very useful to add extra columns to a view (the description, the output of the
995 last execution). Here is a list of useful extensions:
997 * `Build Graph View Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Graph+View+Plugin>`_
998 * `Build Pipeline Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build+Pipeline+Plugin>`_
999 * `Credentials Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Credentials+Plugin>`_
1000 * `Extra Columns Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Extra+Columns+Plugin>`_
1001 * `Git Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Git+Plugin>`_
1002 * `GitHub Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Github+Plugin>`_
1003 * `GitLab Plugin <https://wiki.jenkins-ci.org/display/JENKINS/GitLab+Plugin>`_
1004 * :epkg:`Python`
1005 * `Python Wrapper Plugin <https://wiki.jenkins-ci.org/display/JENKINS/Python+Wrapper+Plugin>`_
1006 * `Build timeout plugin <https://wiki.jenkins-ci.org/display/JENKINS/Build-timeout+Plugin>`_
1008 Tag description:
1010 * ``[engine]``: to use this specific engine (Python path)
1011 * ``[27]``: run with python 2.7
1012 * ``[LONG]``: run longer unit tests (files start by ``test_LONG_``)
1013 * ``[SKIP]``: run skipped unit tests (files start by ``test_SKIP_``)
1014 * ``[GUI]``: run skipped unit tests (files start by ``test_GUI_``)
1015 * ``[custom.+]``: run ``setup.py <custom.+>`` in a virtual environment
1016 * ``[UT] {-d_10}``: run ``setup.py unittests -d 10`` in a virtual environment, ``-d 10`` is one of the possible parameters
1018 Others tags:
1020 * ``[conda_update]``: update conda distribution
1021 * ``[update]``: update distribution
1022 * ``[install]``: update distribution
1023 * ``[local_pypi]``: write a script to run a local pypi server on port 8067 (default option)
1024 * ``pymyinstall [update_modules]``: run a script to update all modules
1025 (might have to be ran a couple of times before being successful)
1027 *modules* is a list defined as follows:
1029 * each element can be a string or a tuple (string, schedule time) or a list
1030 * if it is a list, it contains a list of elements defined as previously
1031 * if the job at position i is not scheduled, it will start after the last
1032 job at position i-1 whether or not it fails
1033 * the job can be defined as a tuple of 3 elements, the last one contains options
1035 The available options are:
1037 * pre: defines a string to insert at the beginning of a job
1038 * post: defines a string to insert at the end of a job
1039 * script: defines a full script if the job to execute is ``custom``
1041 Example ::
1043 modules=[ # update anaconda
1044 ("standalone [conda_update] [anaconda3]",
1045 "H H(0-1) * * 0"),
1046 "standalone [conda_update] [anaconda2] [27]",
1047 "standalone [local_pypi]",
1048 #"standalone [install]",
1049 #"standalone [update]",
1050 #"standalone [install] [py34]",
1051 #"standalone [update] [py34]",
1052 #"standalone [install] [winpython]",
1053 #"standalone [update] [winpython]",
1054 # pyquickhelper and others,
1055 ("pyquickhelper", "H H(2-3) * * 0"),
1056 ("pysqllike <-- pyquickhelper", None, dict(success_only=True)),
1057 ["python3_module_template <-- pyquickhelper",
1058 "pyquickhelper [27] [anaconda2]"],
1059 ["pyquickhelper [winpython]",
1060 "python3_module_template [27] [anaconda2] <-- pyquickhelper", ],
1061 ["pymyinstall <-- pyquickhelper", "pyensae <-- pyquickhelper"],
1062 ["pymmails <-- pyquickhelper", "pyrsslocal <-- pyquickhelper, pyensae"],
1063 ["pymyinstall [27] [anaconda2] <-- pyquickhelper", "pymyinstall [LONG] <-- pyquickhelper"],
1064 # update, do not move, it depends on pyquickhelper
1065 ("pyquickhelper [anaconda3]", "H H(2-3) * * 1"),
1066 ["pyquickhelper [winpython]", "pysqllike [anaconda3]",
1067 "pysqllike [winpython] <-- pyquickhelper",
1068 "python3_module_template [anaconda3] <-- pyquickhelper",
1069 "python3_module_template [winpython] <-- pyquickhelper",
1070 "pymmails [anaconda3] <-- pyquickhelper",
1071 "pymmails [winpython] <-- pyquickhelper",
1072 "pymyinstall [anaconda3] <-- pyquickhelper",
1073 "pymyinstall [winpython] <-- pyquickhelper"],
1074 ["pyensae [anaconda3] <-- pyquickhelper",
1075 "pyensae [winpython] <-- pyquickhelper",
1076 "pyrsslocal [anaconda3] <-- pyquickhelper, pyensae",
1077 "pyrsslocal [winpython] <-- pyquickhelper"],
1078 ("pymyinstall [update_modules]",
1079 "H H(0-1) * * 5"),
1080 "pymyinstall [update_modules] [winpython]",
1081 "pymyinstall [update_modules] [py34]",
1082 "pymyinstall [update_modules] [anaconda2]",
1083 "pymyinstall [update_modules] [anaconda3]",
1084 # py35
1085 ("pyquickhelper [py34]", "H H(2-3) * * 2"),
1086 ["pysqllike [py34]",
1087 "pymmails [py34] <-- pyquickhelper",
1088 "python3_module_template [py34] <-- pyquickhelper",
1089 "pymyinstall [py34] <-- pyquickhelper"],
1090 "pyensae [py34] <-- pyquickhelper",
1091 "pyrsslocal [py34] <-- pyquickhelper, pyensae",
1092 ],
1094 Example::
1096 from ensae_teaching_cs.automation.jenkins_helper import setup_jenkins_server
1097 from pyquickhelper.jenkinshelper import JenkinsExt
1099 engines = dict(Anaconda2=r"C:\\Anaconda2",
1100 Anaconda3=r"C:\\Anaconda3",
1101 py35=r"c:\\Python35_x64",
1102 py36=r"c:\\Python36_x64",
1103 default=r"c:\\Python36_x64",
1104 custom=r"c:\\CustomPython")
1106 js = JenkinsExt('http://machine:8080/', "user", "password", engines=engines)
1108 if True:
1109 js.setup_jenkins_server(github="sdpython", overwrite = True,
1110 location = r"c:\\jenkins\\pymy")
1113 Another example::
1115 import sys
1116 sys.path.append(r"C:\\<path>\\ensae_teaching_cs\\src")
1117 sys.path.append(r"C:\\<path>\\pyquickhelper\\src")
1118 sys.path.append(r"C:\\<path>\\pyensae\\src")
1119 sys.path.append(r"C:\\<path>\\pyrsslocal\\src")
1120 from ensae_teaching_cs.automation.jenkins_helper import setup_jenkins_server, JenkinsExt
1121 js = JenkinsExt("http://<machine>:8080/", <user>, <password>)
1122 js.setup_jenkins_server(location=r"c:\\jenkins\\pymy", overwrite=True, engines=engines)
1124 Parameter *credentials* can be a dictionary where the key is
1125 the git repository. Parameter *dependencies* and *no_dep*
1126 were removed. Dependencies are now specified
1127 in the job name using ``<--`` and they exclusively rely
1128 on pipy (local or remote). Add options for module
1129 *Build Timeout Plugin*.
1130 """
1131 # we do a patch for pyquickhelper
1132 all_jobs = []
1133 for jobs in modules:
1134 jobs = jobs if isinstance(jobs, list) else [jobs]
1135 for job in jobs:
1136 if isinstance(job, tuple):
1137 job = job[0]
1138 job = job.split("<--")[0]
1139 name = self.get_jenkins_job_name(job)
1140 all_jobs.append(name)
1141 all_jobs = set(all_jobs)
1142 if "pyquickhelper" in all_jobs:
1143 self.PACTHPQ = True
1144 self.pyquickhelper = os.path.join(
1145 location, "_pyquickhelper", "src")
1147 # rest of the function
1148 if get_jenkins_script is None:
1149 get_jenkins_script = JenkinsExt.get_jenkins_script
1151 if github is not None and "https://" not in github:
1152 github = "https://github.com/" + github + "/"
1154 deps = []
1155 created = []
1156 locations = []
1157 indexes = dict(order=0, dozen="A")
1158 counts = {}
1159 for jobs in modules:
1160 if isinstance(jobs, tuple):
1161 if len(jobs) == 0:
1162 raise ValueError(
1163 "Empty jobs in the list.") # pragma: no cover
1164 if jobs[0] == "yml" and len(jobs) != 3:
1165 raise ValueError( # pragma: no cover
1166 "If it is a yml jobs, the tuple should contain 3 elements: "
1167 "('yml', filename, schedule or None or dictionary).\n" +
1168 f"Not: {jobs}")
1170 branch = 'master'
1171 if isinstance(jobs, tuple) and jobs[0] == 'yml':
1172 url = jobs[1]
1173 url_spl = url.split('/')
1174 if len(url_spl) > 2:
1175 branch = url_spl[-2]
1176 else:
1177 branch = 'master'
1178 cre, ds, locs = self._setup_jenkins_server_modules_loop(
1179 jobs=jobs, counts=counts,
1180 get_jenkins_script=get_jenkins_script,
1181 location=location, adjust_scheduler=adjust_scheduler,
1182 add_environ=add_environ, yml_engine=yml_engine,
1183 overwrite=overwrite, prefix=prefix,
1184 credentials=credentials, github=github,
1185 disable_schedule=disable_schedule, jenkins_server=self,
1186 update=update, indexes=indexes, deps=deps, branch=branch)
1187 created.extend(cre)
1188 locations.extend(locs)
1189 deps.extend(ds)
1190 return created
1192 def _setup_jenkins_server_modules_loop(self, jobs, counts, get_jenkins_script, location, adjust_scheduler,
1193 add_environ, yml_engine, overwrite, prefix, credentials, github,
1194 disable_schedule, jenkins_server, update, indexes, deps, branch):
1195 if not isinstance(jobs, list):
1196 jobs = [jobs]
1197 indexes["unit"] = 0
1198 new_dep = []
1199 created = []
1200 locations = []
1201 for i, job in enumerate(jobs):
1202 indexes["unit"] += 1
1203 cre, dep, loc = self._setup_jenkins_server_job_iteration(
1204 job, counts=counts,
1205 get_jenkins_script=get_jenkins_script,
1206 location=location, adjust_scheduler=adjust_scheduler,
1207 add_environ=add_environ, yml_engine=yml_engine,
1208 overwrite=overwrite, prefix=prefix,
1209 credentials=credentials, github=github,
1210 disable_schedule=disable_schedule,
1211 jenkins_server=jenkins_server,
1212 update=update, indexes=indexes,
1213 deps=deps, i=i, branch=branch)
1214 created.extend(cre)
1215 new_dep.extend(dep)
1216 locations.extend(loc)
1217 if len(new_dep) > 20000:
1218 raise JenkinsExtException( # pragma: no cover
1219 f"unreasonable number of dependencies: {len(new_dep)}")
1220 return created, new_dep, locations
1222 def _setup_jenkins_server_job_iteration(self, job, get_jenkins_script, location, adjust_scheduler,
1223 add_environ, yml_engine, overwrite, prefix, credentials, github,
1224 disable_schedule, jenkins_server, update, indexes, deps, i,
1225 counts, branch):
1226 order = indexes["order"]
1227 dozen = indexes["dozen"]
1228 unit = indexes["unit"]
1229 new_dep = []
1230 created = []
1231 locations = []
1233 if isinstance(job, tuple):
1234 if len(job) < 2:
1235 raise JenkinsJobException( # pragma: no cover
1236 "the tuple must contain at least two elements:\nJOB:"
1237 "\n" + str(job))
1239 if job[0] == "yml":
1240 is_yml = True
1241 job = job[1:]
1242 else:
1243 is_yml = False
1245 # we extract options if any
1246 if len(job) == 3:
1247 options = job[2]
1248 if not isinstance(options, dict):
1249 raise JenkinsJobException( # pragma: no cover
1250 "The last element of the tuple must be a dictionary:\nJOB:\n" + str(options))
1251 else:
1252 options = {}
1254 # job and scheduler
1255 job, scheduler_options = job[:2]
1256 if isinstance(scheduler_options, dict):
1257 scheduler = scheduler_options.get('scheduler', None)
1258 else:
1259 scheduler = scheduler_options
1260 scheduler_options = None
1261 if scheduler is not None:
1262 order = 1
1263 if counts.get(dozen, 0) > 0:
1264 dozen = chr(ord(dozen) + 1)
1265 else:
1266 if i == 0:
1267 order += 1
1268 else:
1269 scheduler = None
1270 if i == 0:
1271 order += 1
1272 options = {}
1273 is_yml = False
1275 # all schedule are disabled if disable_schedule is True
1276 if disable_schedule:
1277 scheduler = None
1278 counts[dozen] = counts.get(dozen, 0) + 1
1280 # success_only
1281 if "success_only" in options:
1282 success_only = options["success_only"]
1283 del options["success_only"]
1284 else:
1285 success_only = False
1287 # timeout
1288 if "timeout" in options:
1289 timeout = options["timeout"]
1290 del options["timeout"]
1291 else:
1292 timeout = _timeout_default
1294 # script
1295 if not is_yml:
1296 script = get_jenkins_script(self, job)
1298 # we process the repository
1299 if "repo" in options:
1300 gitrepo = options["repo"]
1301 options = options.copy()
1302 del options["repo"]
1303 else:
1304 gitrepo = github
1306 # add a description to the job
1307 description = ["%s%02d%02d" % (dozen, order, unit)]
1308 if scheduler is not None:
1309 description.append(scheduler)
1310 try:
1311 description = " - ".join(description)
1312 except TypeError as e: # pragma: no cover
1313 raise TypeError(f"Issue with {description}.") from e
1315 # credentials
1316 if isinstance(credentials, dict): # pragma: no cover
1317 cred = credentials.get(gitrepo, None)
1318 if cred is None:
1319 cred = credentials.get("default", "")
1320 else:
1321 cred = credentials
1323 if not is_yml:
1324 mod = job.split()[0]
1325 name = self.get_jenkins_job_name(job)
1326 jname = prefix + name
1328 try:
1329 j = jenkins_server.get_job_config(
1330 jname) if not jenkins_server._mock else None
1331 except jenkins.NotFoundException: # pragma: no cover
1332 j = None
1333 except jenkins.JenkinsException as e: # pragma: no cover
1334 raise JenkinsExtException(
1335 f"unable to retrieve job config for job={job}, name={jname}") from e
1337 if overwrite or j is None:
1339 update_job = False
1340 if j is not None: # pragma: no cover
1341 if update:
1342 update_job = True
1343 else:
1344 self.fLOG("[jenkins] delete job", jname)
1345 jenkins_server.delete_job(jname)
1347 # we post process the script
1348 script = self.process_options(script, options)
1350 # if there is a script
1351 if script is not None and len(script) > 0:
1352 new_dep.append(name)
1353 upstreams = [] if (
1354 scheduler is not None) else deps[-1:]
1355 self.fLOG("[jenkins] create job", jname, " - ", job,
1356 " : ", scheduler, " / ", upstreams)
1358 # set up location
1359 if location is None:
1360 loc = None # pragma: no cover
1361 else:
1362 if "_" in jname:
1363 loc = os.path.join(location, name, jname)
1364 else:
1365 loc = os.path.join(location, name, "_" + jname)
1367 if mod in ("standalone", "custom"):
1368 gpar = None
1369 elif gitrepo is None:
1370 raise JenkinsJobException( # pragma: no cover
1371 "gitrepo cannot must not be None if standalone or "
1372 "custom is not defined,\njob=" + str(job))
1373 elif gitrepo.endswith(".git"):
1374 gpar = gitrepo
1375 else:
1376 gpar = gitrepo + f"{mod}/"
1378 # create the template
1379 r = jenkins_server.create_job_template(jname, git_repo=gpar, upstreams=upstreams, script=script,
1380 location=loc, scheduler=scheduler, py27="[27]" in job,
1381 description=description, credentials=cred, success_only=success_only,
1382 update=update_job, timeout=timeout, adjust_scheduler=adjust_scheduler,
1383 mails=self.mails)
1385 # check some inconsistencies
1386 if "[27]" in job and "Anaconda3" in script:
1387 raise JenkinsExtException( # pragma: no cover
1388 f"incoherence for job {job}, script:\n{script}")
1390 locations.append((job, loc))
1391 created.append((job, name, loc, job, r))
1392 else: # pragma: no cover
1393 # skip the job
1394 loc = None if location is None else os.path.join(
1395 location, jname)
1396 locations.append((job, loc))
1397 self.fLOG("[jenkins] skipping",
1398 job, "location", loc)
1399 elif j is not None:
1400 new_dep.append(name)
1402 else:
1403 # yml file
1404 if location is not None:
1405 options["root_path"] = location
1406 for k, v in self.engines.items():
1407 if k not in options:
1408 options[k] = v
1409 jobdef = job[0] if isinstance(job, tuple) else job
1411 done = {}
1412 for aj, name, var in enumerate_processed_yml(
1413 jobdef, context=options, engine=yml_engine,
1414 add_environ=add_environ, server=self, git_repo=gitrepo,
1415 scheduler=scheduler, description=description, credentials=cred,
1416 success_only=success_only, timeout=timeout, platform=self.platform,
1417 adjust_scheduler=adjust_scheduler, overwrite=overwrite,
1418 build_location=location, mails=self.mails,
1419 job_options=scheduler_options, branch=branch):
1420 if name in done:
1421 s = "A name '{0}' was already used for a job, from:\n{1}\nPROCESS:\n{2}" # pragma: no cover
1422 raise ValueError( # pragma: no cover
1423 s.format(name, jobdef, "\n".join(sorted(set(done.keys())))))
1424 done[name] = (aj, name, var)
1425 loc = None if location is None else os.path.join(
1426 location, name)
1427 self.fLOG(
1428 f"[jenkins] adding i={len(created)}: '{name}' var='{var}'")
1429 created.append((job, name, loc, job, aj))
1431 indexes["order"] = order
1432 indexes["dozen"] = dozen
1433 indexes["unit"] = unit
1434 return created, new_dep, locations