Coverage for pyquickhelper/jenkinshelper/yaml_helper.py: 95%
388 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Parse a file *.yml* and convert it into a set of actions.
4"""
5import os
6import re
7from ..texthelper.templating import apply_template
8from ..filehelper import read_content_ufs
9from .yaml_helper_yaml import yaml_load
10from .jenkins_helper import get_platform
13_jenkins_split = "JENKINS_SPLIT"
16def pickname(*args):
17 """
18 Picks the first string non null in the list.
20 @param l list of string
21 @return string
22 """
23 for s in args:
24 s = s.strip()
25 if s:
26 return s
27 raise ValueError( # pragma: no cover
28 f"Unable to find a non empty string in {args}")
31def load_yaml(file_or_buffer, context=None, engine="jinja2", platform=None):
32 """
33 Loads a :epkg:`yml` file (*.yml*).
35 @param file_or_buffer string or physical file or url
36 @param context variables to replace in the configuration
37 @param engine see @see fn apply_template
38 @param platform to join path differently based on the OS
39 @return see `PyYAML <http://pyyaml.org/wiki/PyYAMLDocumentation>`_
40 """
41 def replace(val, rep, into):
42 if val is None:
43 return val
44 return val.replace(rep, into)
45 content, source = read_content_ufs(file_or_buffer, add_source=True)
47 def ospathjoinp(*args, **kwargs):
48 p = kwargs.get('platform', platform)
49 return ospathjoin(*args, platform=p)
51 if context is None:
52 context = dict(replace=replace, ospathjoin=ospathjoinp,
53 pickname=pickname)
54 else:
55 fs = [("replace", replace), ("ospathjoin", ospathjoinp),
56 ("pickname", pickname)]
57 if any(_[0] not in context for _ in fs):
58 context = context.copy()
59 for k, f in fs:
60 if k not in context:
61 context[k] = f
62 if not isinstance(context, dict):
63 raise TypeError( # pragma: no cover
64 f"context must be a dictionary not {type(context)}.")
65 if "project_name" not in context:
66 project_name = infer_project_name(file_or_buffer, source)
67 else:
68 project_name = context["project_name"]
69 if project_name.endswith("__"):
70 raise ValueError( # pragma: no cover
71 f"project_name is wrong, it cannot end by '__': '{project_name}'")
72 if "project_name" not in context and project_name is not None:
73 context["project_name"] = project_name
75 if ("root_path" not in context or
76 not context["root_path"].endswith(project_name)):
77 context = context.copy()
78 context["root_path"] = ospathjoin(
79 context.get("root_path", ""), project_name, platform=platform)
81 if "root_path" in context:
82 if platform is None:
83 platform = get_platform(platform)
84 if platform.startswith("win"):
85 addition = f"set current={context['root_path']}\\%NAME_JENKINS%"
86 else:
87 addition = f"export current={context['root_path']}/$NAME_JENKINS"
88 content = f"automatedsetup:\n - {addition}\n{content}"
90 content = apply_template(content, context, engine)
91 try:
92 return yaml_load(content), project_name
93 except Exception as e: # pragma: no cover
94 raise SyntaxError(
95 f"Unable to parse content\n{content}") from e
98def evaluate_condition(cond, variables=None):
99 """
100 Evaluates a condition inserted in a :epkg:`yml` file.
102 @param cond (str) condition
103 @param variables (dict|None) dictionary
104 @return boolean
106 Example of a condition::
108 [ ${PYTHON} == "C:\\Python370_x64" ]
109 """
110 if variables is not None:
111 for k, v in variables.items():
112 rep = "${%s}" % k
113 vv = f'"{v}"'
114 # This fix should be done in another way and extracting strings
115 # in the expression and check escape characters.
116 cond = cond.replace('"C:\\Py', '"C:\\\\Py')
117 cond = cond.replace(rep, vv)
118 cond = cond.replace(rep.upper(), vv)
119 cond = cond.strip()
120 if cond.startswith("[") and cond.endswith("]"):
121 e = eval(cond)
122 return all(e)
123 try:
124 ev = eval(cond)
125 return ev
126 except SyntaxError as e:
127 raise SyntaxError(
128 f"Unable to interpret '{cond}'\nvariables: {variables}") from e
131def interpret_instruction(inst, variables=None):
132 """
133 Interprets an instruction with if statement.
135 @param inst (str) instruction
136 @param variables (dict|None)
137 @return (str|None)
139 Example of a statement::
141 - if [ ${PYTHON} == "C:\\\\Python391_x64" ] then python setup.py build_sphinx fi
143 Another example::
145 - if [ ${VERSION} == "3.9" and ${DIST} == "std" ]
146 then
147 --CMD=$PYINT -u scikit-learn/bench_plot_polynomial_features_partial_fit.py;;
148 --NAME=SKL_POLYF_PF;;
149 fi
151 In this second syntax, lines must end with ``;;``.
152 If an instruction cannot be interpreted, it is left
153 left unchanged as the function assumes it can only be solved
154 in a bash script.
156 Switch to ``;;`` instead of ``;`` as a instruction separator
157 for conditional instructions.
158 """
159 if isinstance(inst, list):
160 res = [interpret_instruction(_, variables) for _ in inst]
161 if any(res):
162 return [_ for _ in res if _ is not None]
163 return None
164 if isinstance(inst, tuple):
165 if len(inst) != 2 or inst[1] is None:
166 raise ValueError( # pragma: no cover
167 f"Unable to interpret '{inst}'.")
168 return (inst[0], interpret_instruction(inst[1], variables))
169 if isinstance(inst, dict):
170 return inst
171 if isinstance(inst, (int, float)):
172 return inst
174 inst = inst.replace("\n", " ")
175 exp = re.compile("^ *if +(.*) +then +(.*)( +else +(.*))? +fi *$")
176 find = exp.search(inst)
177 if find:
178 gr = find.groups()
179 try:
180 e = evaluate_condition(gr[0], variables)
181 except SyntaxError:
182 # We assume the condition is a linux condition.
183 return inst
184 g = gr[1] if e else gr[3]
185 return None if g is None else interpret_instruction(g, variables)
187 if inst.startswith('--'):
188 # one format like --CMD=...; --NAME==...;
189 exp = re.compile("--([a-zA-Z]+?)=(.+?);;")
190 find = exp.findall(inst)
191 if find:
192 inst = {k.strip(): v.strip() for k, v in find}
193 inst = {k: (None if not v or len(v) == 0 else v)
194 for k, v in inst.items()}
195 return inst
196 return inst
197 return inst
200def enumerate_convert_yaml_into_instructions(obj, variables=None, add_environ=True):
201 """
202 Converts a :epkg:`yml` file into sequences of instructions,
203 conditions are interpreted.
205 @param obj yaml objects (@see fn load_yaml)
206 @param variables additional variables to be used
207 @param add_environ add environment variables available, does not
208 overwrite existing variables
209 when the job is generated
210 @return list of tuple(instructions, variables)
212 The function expects the following list
213 of steps in this order:
215 * *automatedsetup*: added by this module
216 * *language*: should be python
217 * *python*: list of interpreters (multiplies jobs)
218 * *virtualenv*: name of the virtual environment
219 * *install*: list of installation steps in the virtual environment
220 * *before_script*: list of steps to run
221 * *script*: list of script to run (multiplies jobs)
222 * *after_script*: list of steps to run
223 * *documentation*: documentation to run after the
225 Each step *multiplies jobs* creates a sequence of jobs and a :epkg:`Jenkins` job.
226 """
227 if variables is None:
228 def_variables = {}
229 else:
230 def_variables = variables.copy()
231 if 'Python37' in def_variables and 'Python38' not in def_variables:
232 raise RuntimeError( # pragma: no cover
233 f"Key 'Python38' is missing in {def_variables}.")
234 if add_environ:
235 for k, v in os.environ.items():
236 if k not in def_variables:
237 def_variables[k] = v
238 sequences = []
239 count = {}
240 steps = ["automatedsetup", "language", "python", "virtualenv", "install",
241 "before_script", "script", "after_script",
242 "documentation"]
243 for key in steps:
244 value = obj.get(key, None)
245 if key == "language":
246 if value != "python":
247 raise NotImplementedError( # pragma: no cover
248 "language must be python")
249 continue # pragma: no cover
250 if value is not None:
251 if key in {'python', 'script'} and not isinstance(value, list):
252 value = [value]
253 count[key] = len(value)
254 sequences.append((key, value))
256 for k in obj:
257 if k not in steps:
258 raise ValueError(
259 "Unexpected key '{0}' found in yaml file. Expect:\n{1}".format(k, "\n".join(steps)))
261 # multiplications
262 i_python = 0
263 i_script = 0
264 notstop = True
265 while notstop:
266 seq = []
267 add = True
268 variables = def_variables.copy()
269 for key, value in sequences:
270 if key == "python":
271 value = value[i_python]
272 if isinstance(value, dict):
273 if 'PATH' not in value:
274 raise KeyError( # pragma: no cover
275 f"The dictionary should include key 'path': {value}")
276 for k, v in sorted(value.items()):
277 if k != 'PATH':
278 variables[k] = v
279 seq.append(('INFO', (k, v)))
280 value = value["PATH"]
281 elif key == "script":
282 value = interpret_instruction(value[i_script], variables)
283 if isinstance(value, dict):
284 for k, v in sorted(value.items()):
285 if k not in ('CMD', 'CMDPY'):
286 seq.append(('INFO', (k, v)))
287 variables[k] = v
289 i_script += 1
290 if i_script >= count['script']:
291 i_script = 0
292 i_python += 1
293 if i_python >= count['python']:
294 notstop = False
295 if value is not None and value != 'None':
296 seq.append((key, value))
297 variables[key] = value
298 else:
299 add = False
300 if add:
301 r = interpret_instruction(seq, variables)
302 if r is not None:
303 yield r, variables
306def ospathjoin(*args, **kwargs):
307 """
308 Simple ``o.path.join`` for a specific platform.
310 @param args list of paths
311 @param kwargs additional parameters, among them,
312 *platform* (win32 or ...)
313 @return path
314 """
315 def build_value(*args, **kwargs):
316 platform = kwargs.get('platform', None)
317 if platform is None:
318 return os.path.join(*args)
319 elif platform.startswith("win"):
320 return "\\".join(args)
321 return "/".join(args)
323 value = build_value(*args, **kwargs)
324 if value == "/$PYINT":
325 raise RuntimeError( # pragma: no cover
326 f"Impossible values {args} - {kwargs}.")
327 return value
330def ospathdirname(lp, platform=None):
331 """
332 Simple ``o.path.dirname`` for a specific platform.
334 @param lp path
335 @param platform platform
336 @return path
337 """
338 if platform is None:
339 return os.path.dirname(lp)
340 elif platform.startswith("win"):
341 return "\\".join(lp.replace("/", "\\").split("\\")[:-1])
342 return "/".join(lp.replace("\\", "/").split("/")[:-1])
345def convert_sequence_into_batch_file(seq, variables=None, platform=None):
346 """
347 Converts a sequence of instructions into a batch file.
349 @param seq sequence of instructions
350 @param variables list of variables
351 @param platform ``get_platform(platform)`` if None
352 @return (str) batch file or a list of batch file if the constant ``JENKINS_SPLIT``
353 was found in section install (this tweak is needed when the job has to be split
354 for :epkg:`Jenkins`.
355 """
356 global _jenkins_split
357 if platform is None:
358 platform = get_platform(platform)
360 iswin = platform.startswith("win")
362 if iswin:
363 error_level = "if %errorlevel% neq 0 exit /b %errorlevel%"
364 else:
365 error_level = "if [ $? -ne 0 ]; then exit $?; fi"
367 interpreter = None
368 venv_interpreter = None
369 root_project = None
370 anaconda = False
371 conda = None
372 echo = "@echo" if iswin else "echo"
374 rowsset = []
375 if iswin:
376 rowsset.append("@echo off")
377 rowsset.append("set PATH0=%PATH%")
379 def add_path_win(rows, interpreter, platform, root_project):
380 path_inter = ospathdirname(interpreter, platform)
381 if len(path_inter) == 0:
382 raise ValueError( # pragma: no cover
383 "Unable to guess interpreter path from '{0}', platform={1}"
384 "".format(interpreter, platform))
385 if iswin:
386 rows.append(f"set PATH={path_inter};%PATH%")
387 else:
388 rows.append(f"export PATH={path_inter}:$PATH")
389 if root_project is not None:
390 if iswin:
391 rows.append(f"set ROOTPROJECT={root_project}")
392 else:
393 rows.append(f"export ROOTPROJECT={root_project}")
395 rows = []
396 splits = [rows]
397 typstr = str
399 for key, value in seq:
400 if key == "automatedsetup":
401 rows.append("")
402 rows.append(echo + " AUTOMATEDSETUP")
403 rows.append("\n".join(value))
404 rows.append("")
405 elif key == "python":
406 variables["YMLPYTHON"] = value
407 if variables.get('DIST', None) == "conda":
408 rows.append(echo + " conda")
409 anaconda = True
410 interpreter = ospathjoin(
411 value, "python", platform=platform)
412 venv_interpreter = value
413 if platform.startswith("win"):
414 conda = ospathjoin(
415 value, "Scripts", "conda", platform=platform)
416 else:
417 conda = ospathjoin(
418 value, "bin", "conda", platform=platform)
419 else:
420 if iswin:
421 interpreter = ospathjoin(
422 value, "python", platform=platform)
423 else:
424 interpreter = ospathjoin(
425 value, "$PYINT", platform=platform)
426 venv_interpreter = value
427 rows.append(echo + " interpreter=" + interpreter)
429 elif key == "virtualenv":
430 if isinstance(value, list):
431 if len(value) > 2:
432 raise ValueError( # pragma: no cover
433 "Expecting one or two values for the path of the virtual environment"
434 ":\n{0}".format(value))
435 d = value[0].copy()
436 for i in range(1, len(value)):
437 d.update(value[i])
438 value = d
439 p = value["path"] if isinstance(value, dict) else value
440 rows.append("")
441 rows.append(echo + f" CREATE VIRTUAL ENVIRONMENT in {p}")
442 if not anaconda:
443 if iswin:
444 rows.append('if not exist "{0}" mkdir "{0}"'.format(p))
445 else:
446 rows.append('if [-f {0}]; then mkdir "{0}"; fi'.format(p))
447 if anaconda:
448 pinter = ospathdirname(interpreter, platform=platform)
449 rows.append(
450 f'"{conda}" create -y -v -p "{p}" --clone "{pinter}" --offline --no-update-deps')
451 interpreter = ospathjoin(
452 p, "python", platform=platform)
453 else:
454 if iswin:
455 rows.append("set KEEPPATH=%PATH%")
456 rows.append(f"set PATH={venv_interpreter};%PATH%")
457 else:
458 rows.append("export KEEPPATH=$PATH")
459 rows.append(
460 f"export PATH={venv_interpreter}:$PATH")
461 pat = '"{0}" -m virtualenv {1}'
462 if isinstance(value, dict):
463 system_site_packages = value.get(
464 'system_site_packages', True)
465 else:
466 system_site_packages = True
467 if system_site_packages:
468 pat += " --system-site-packages"
469 rows.append(pat.format(interpreter, p))
470 if iswin:
471 rows.append("set PATH=%KEEPPATH%")
472 interpreter = ospathjoin(
473 p, "Scripts", "python", platform=platform)
474 else:
475 rows.append("export PATH=$KEEPPATH")
476 interpreter = ospathjoin(
477 p, "bin", "python", platform=platform)
478 rows.append(error_level)
480 elif key in {"install", "before_script", "script", "after_script", "documentation"}:
481 if value is not None:
482 if isinstance(value, dict):
483 if "CMD" not in value and "CMDPY" not in value:
484 raise KeyError( # pragma: no cover
485 "A script defined by a dictionary must contain key "
486 "'{0}' or '{1}' in \n{2}".format("CMD", 'CMDPY', value))
487 if "NAME" in value:
488 if iswin:
489 rows.append(f"set JOB_NAME={value['NAME']}")
490 else:
491 rows.append(f"export JOB_NAME={value['NAME']}")
492 if "CMD" in value:
493 value = value["CMD"]
494 else:
495 value = evaluate_condition(
496 value["CMDPY"], variables=variables)
497 elif isinstance(value, list):
498 starter = list(rows)
499 elif isinstance(value, typstr):
500 pass
501 else:
502 raise TypeError( # pragma: no cover
503 f"value must of type list, dict, not '{type(value)}'\n{value}")
505 rows.append("")
506 rows.append(echo + " " + key.upper())
507 add_path_win(rows, interpreter, platform, root_project)
508 if not isinstance(value, list):
509 value = [value, error_level]
510 else:
511 keep = value
512 value = []
513 for v in keep:
514 if v.startswith(_jenkins_split):
515 if "-" in v:
516 nbrem = v.split("-")[-1]
517 try:
518 nbrem = int(nbrem)
519 except ValueError: # pragma: no cover
520 raise ValueError(
521 f"Unable to interpret '{v}'")
522 else:
523 nbrem = 0
524 rows.extend(value)
525 value = []
526 st = list(starter)
527 if nbrem > 0:
528 st = st[:-nbrem]
529 splits.append(st)
530 rows = splits[-1]
531 add_path_win(rows, interpreter,
532 platform, root_project)
533 else:
534 value.append(v)
535 value.append(error_level)
536 rows.extend(value)
537 elif key == 'INFO':
538 vs = f'"{value[1]}"' if isinstance(
539 value[1], str) and " " in value[1] else value[1]
540 if iswin:
541 rowsset.append(f"SET {value[0]}={vs}")
542 else:
543 rowsset.append(f"export {value[0]}={vs}")
544 else:
545 raise ValueError( # pragma: no cover
546 f"unexpected key '{key}'")
548 splits = [rowsset + _ for _ in splits]
549 allres = []
550 for rows in splits:
551 try:
552 res = "\n".join(rows)
553 except TypeError as e: # pragma: no cover
554 raise TypeError("Unexpected type\n{0}".format(
555 "\n".join([str((type(_), _)) for _ in rows]))) from e
556 if _jenkins_split in res:
557 raise ValueError( # pragma: no cover
558 "Constant '{0}' is present in the generated script. "
559 "It can only be added to the install section."
560 "".format(_jenkins_split))
561 allres.append(res)
562 return allres if len(allres) > 1 else allres[0]
565def infer_project_name(file_or_buffer, source):
566 """
567 Infers a project name based on :epkg:`yml` file.
569 @param file_or_buffer file name
570 @param source second output of @see fn read_content_ufs
571 @return name
573 The function can infer a name for *source* in ``{'r', 'u'}``.
574 For *source* equal to ``'s'``, it returns ``'unknown_string'``.
575 """
576 if source == "r":
577 fold = os.path.dirname(file_or_buffer)
578 last = os.path.split(fold)[-1]
579 elif source == "u":
580 spl = file_or_buffer.split('/')
581 pos = -2
582 name = None
583 while len(spl) > -pos:
584 name = spl[pos]
585 if name in {'master', 'main'}:
586 pos -= 1
587 elif 'github' in name:
588 break
589 else:
590 break
591 if name is None:
592 raise ValueError( # pragma: no cover
593 f"Unable to infer project name for '{file_or_buffer}'")
594 return name
595 elif source == "s":
596 return "unknown_string"
597 else:
598 raise ValueError( # pragma: no cover
599 f"Unexpected value for add_source: '{source}' for '{file_or_buffer}'")
600 return last
603def enumerate_processed_yml(file_or_buffer, context=None, engine="jinja2", platform=None,
604 server=None, git_repo=None, add_environ=True, overwrite=False,
605 build_location=None, branch='master', **kwargs):
606 """
607 Submits or enumerates jobs based on the content of a :epkg:`yml` file.
609 @param file_or_buffer filename or string
610 @param context variables to replace in the configuration
611 @param engine see @see fn apply_template
612 @param server see @see cl JenkinsExt
613 @param platform plaform where the job will be executed
614 @param git_repo git repository (if *server* is not None)
615 @param add_environ add environment variable before interpreting the job
616 @param overwrite overwrite the job if it already exists in Jenkins
617 @param build_location location for the build
618 @param branch default branch
619 @param kwargs see @see me create_job_template
620 @return enumerator for *(job, name, variables)*
622 Example of a :epkg:`yml` file
623 `.local.jenkins.win.yml
624 <https://github.com/sdpython/pyquickhelper/blob/master/.local.jenkins.win.yml>`_.
625 A subfolder was added to the project location.
626 A scheduler can be defined as well by adding ``SCHEDULER:'* * * * *'``.
627 """
628 typstr = str
629 fLOG = kwargs.get('fLOG', None)
630 project_name = None if context is None else context.get(
631 "project_name", None)
632 obj, project_name = load_yaml(
633 file_or_buffer, context=context, platform=platform)
634 platform_set = platform or get_platform(platform)
635 for seq, var in enumerate_convert_yaml_into_instructions(obj, variables=context, add_environ=add_environ):
636 conv = convert_sequence_into_batch_file(
637 seq, variables=var, platform=platform)
639 # we extract a suffix from the command line
640 if server is not None:
641 name = "_".join([project_name, var.get('NAME', ''),
642 typstr(var.get("VERSION", '')).replace(".", ""),
643 var.get('DIST', '')])
645 if platform_set.startswith("win"):
646 if isinstance(conv, list):
647 conv = ["SET NAME_JENKINS=" +
648 name + "\n" + _ for _ in conv]
649 else:
650 conv = "SET NAME_JENKINS=" + name + "\n" + conv
651 else:
652 if isinstance(conv, list):
653 conv = ["export NAME_JENKINS=" +
654 name + "\n" + _ for _ in conv]
655 conv.append("export $(cat ~/.profile)")
656 else:
657 conv = ("export NAME_JENKINS=" + name +
658 "\nexport $(cat ~/.profile)\n" + conv)
660 import jenkins
661 try:
662 j = server.get_job_config(name) if not server._mock else None
663 except jenkins.NotFoundException: # pragma: no cover
664 j = None
665 except jenkins.JenkinsException as e: # pragma: no cover
666 from .jenkins_exceptions import JenkinsExtException
667 raise JenkinsExtException(
668 f"Unable to retrieve job config for name='{name}'.") from e
670 update_job = False
671 if j is not None:
672 if kwargs.get('update', True):
673 update_job = True
674 else:
675 if fLOG is not None: # pragma: no cover
676 fLOG("[jenkins] delete job", name)
677 server.delete_job(name)
679 if git_repo is not None and project_name not in git_repo:
680 git_repo += project_name
682 # set up location
683 if build_location is None:
684 loc = None
685 else:
686 loc = ospathjoin(build_location, project_name,
687 name, platform=platform)
689 if overwrite or j is None:
690 timeout = var.get("TIMEOUT", None)
691 scheduler = var.get("SCHEDULER", None)
692 clean_repo = var.get("CLEAN", True) in { # pylint: disable=W0130
693 True, 1, "True", "true", "1"}
694 if timeout is not None:
695 kwargs["timeout"] = timeout
696 if scheduler is not None:
697 if "FIXED" in scheduler:
698 scheduler = scheduler.replace("FIXED", "").strip()
699 adjuster_scheduler = False
700 elif "STARTUP" in scheduler:
701 adjuster_scheduler = False
702 elif 'fixed' in scheduler.lower():
703 raise ValueError( # pragma: no cover
704 "Scheduler should contain 'FIXED' in upper case.")
705 elif 'startup' in scheduler.lower():
706 raise ValueError( # pragma: no cover
707 "Scheduler should contain 'STARTUP' in upper case.")
708 else:
709 adjuster_scheduler = True
710 kwargs["scheduler"] = scheduler
711 kwargs["adjuster_scheduler"] = adjuster_scheduler
712 yield server.create_job_template(name, script=conv, git_repo=git_repo,
713 update=update_job, location=loc,
714 clean_repo=clean_repo, branch=branch,
715 **kwargs), name, var
716 else:
717 yield conv, None, var