Coverage for pyquickhelper/jenkinshelper/yaml

1"""

2@file

3@brief Parse a file *.yml* and convert it into a set of actions.

4"""

5import os

6import re

7from ..texthelper.templating import apply_template

8from ..filehelper import read_content_ufs

9from .yaml_helper_yaml import yaml_load

10from .jenkins_helper import get_platform

13_jenkins_split = "JENKINS_SPLIT"

16def pickname(*args):

17 """

18 Picks the first string non null in the list.

20 @param l list of string

21 @return string

22 """

23 for s in args:

24 s = s.strip()

25 if s:

26 return s

27 raise ValueError( # pragma: no cover

28 f"Unable to find a non empty string in {args}")

31def load_yaml(file_or_buffer, context=None, engine="jinja2", platform=None):

32 """

33 Loads a :epkg:`yml` file (*.yml*).

35 @param file_or_buffer string or physical file or url

36 @param context variables to replace in the configuration

37 @param engine see @see fn apply_template

38 @param platform to join path differently based on the OS

39 @return see `PyYAML <http://pyyaml.org/wiki/PyYAMLDocumentation>`_

40 """

41 def replace(val, rep, into):

42 if val is None:

43 return val

44 return val.replace(rep, into)

45 content, source = read_content_ufs(file_or_buffer, add_source=True)

47 def ospathjoinp(*args, **kwargs):

48 p = kwargs.get('platform', platform)

49 return ospathjoin(*args, platform=p)

51 if context is None:

52 context = dict(replace=replace, ospathjoin=ospathjoinp,

53 pickname=pickname)

54 else:

55 fs = [("replace", replace), ("ospathjoin", ospathjoinp),

56 ("pickname", pickname)]

57 if any(_[0] not in context for _ in fs):

58 context = context.copy()

59 for k, f in fs:

60 if k not in context:

61 context[k] = f

62 if not isinstance(context, dict):

63 raise TypeError( # pragma: no cover

64 f"context must be a dictionary not {type(context)}.")

65 if "project_name" not in context:

66 project_name = infer_project_name(file_or_buffer, source)

67 else:

68 project_name = context["project_name"]

69 if project_name.endswith("__"):

70 raise ValueError( # pragma: no cover

71 f"project_name is wrong, it cannot end by '__': '{project_name}'")

72 if "project_name" not in context and project_name is not None:

73 context["project_name"] = project_name

75 if ("root_path" not in context or

76 not context["root_path"].endswith(project_name)):

77 context = context.copy()

78 context["root_path"] = ospathjoin(

79 context.get("root_path", ""), project_name, platform=platform)

81 if "root_path" in context:

82 if platform is None:

83 platform = get_platform(platform)

84 if platform.startswith("win"):

85 addition = f"set current={context['root_path']}\\%NAME_JENKINS%"

86 else:

87 addition = f"export current={context['root_path']}/$NAME_JENKINS"

88 content = f"automatedsetup:\n - {addition}\n{content}"

90 content = apply_template(content, context, engine)

91 try:

92 return yaml_load(content), project_name

93 except Exception as e: # pragma: no cover

94 raise SyntaxError(

95 f"Unable to parse content\n{content}") from e

98def evaluate_condition(cond, variables=None):

99 """

100 Evaluates a condition inserted in a :epkg:`yml` file.

101

102 @param cond (str) condition

103 @param variables (dict|None) dictionary

104 @return boolean

105

106 Example of a condition::

107

108 [ ${PYTHON} == "C:\\Python370_x64" ]

109 """

110 if variables is not None:

111 for k, v in variables.items():

112 rep = "${%s}" % k

113 vv = f'"{v}"'

114 # This fix should be done in another way and extracting strings

115 # in the expression and check escape characters.

116 cond = cond.replace('"C:\\Py', '"C:\\\\Py')

117 cond = cond.replace(rep, vv)

118 cond = cond.replace(rep.upper(), vv)

119 cond = cond.strip()

120 if cond.startswith("[") and cond.endswith("]"):

121 e = eval(cond)

122 return all(e)

123 try:

124 ev = eval(cond)

125 return ev

126 except SyntaxError as e:

127 raise SyntaxError(

128 f"Unable to interpret '{cond}'\nvariables: {variables}") from e

129

130

131def interpret_instruction(inst, variables=None):

132 """

133 Interprets an instruction with if statement.

134

135 @param inst (str) instruction

136 @param variables (dict|None)

137 @return (str|None)

138

139 Example of a statement::

140

141 - if [ ${PYTHON} == "C:\\\\Python391_x64" ] then python setup.py build_sphinx fi

142

143 Another example::

144

145 - if [ ${VERSION} == "3.9" and ${DIST} == "std" ]

146 then

147 --CMD=$PYINT -u scikit-learn/bench_plot_polynomial_features_partial_fit.py;;

148 --NAME=SKL_POLYF_PF;;

149 fi

150

151 In this second syntax, lines must end with ``;;``.

152 If an instruction cannot be interpreted, it is left

153 left unchanged as the function assumes it can only be solved

154 in a bash script.

155

156 Switch to ``;;`` instead of ``;`` as a instruction separator

157 for conditional instructions.

158 """

159 if isinstance(inst, list):

160 res = [interpret_instruction(_, variables) for _ in inst]

161 if any(res):

162 return [_ for _ in res if _ is not None]

163 return None

164 if isinstance(inst, tuple):

165 if len(inst) != 2 or inst[1] is None:

166 raise ValueError( # pragma: no cover

167 f"Unable to interpret '{inst}'.")

168 return (inst[0], interpret_instruction(inst[1], variables))

169 if isinstance(inst, dict):

170 return inst

171 if isinstance(inst, (int, float)):

172 return inst

173

174 inst = inst.replace("\n", " ")

175 exp = re.compile("^ *if +(.*) +then +(.*)( +else +(.*))? +fi *$")

176 find = exp.search(inst)

177 if find:

178 gr = find.groups()

179 try:

180 e = evaluate_condition(gr[0], variables)

181 except SyntaxError:

182 # We assume the condition is a linux condition.

183 return inst

184 g = gr[1] if e else gr[3]

185 return None if g is None else interpret_instruction(g, variables)

186

187 if inst.startswith('--'):

188 # one format like --CMD=...; --NAME==...;

189 exp = re.compile("--([a-zA-Z]+?)=(.+?);;")

190 find = exp.findall(inst)

191 if find:

192 inst = {k.strip(): v.strip() for k, v in find}

193 inst = {k: (None if not v or len(v) == 0 else v)

194 for k, v in inst.items()}

195 return inst

196 return inst

197 return inst

198

199

200def enumerate_convert_yaml_into_instructions(obj, variables=None, add_environ=True):

201 """

202 Converts a :epkg:`yml` file into sequences of instructions,

203 conditions are interpreted.

204

205 @param obj yaml objects (@see fn load_yaml)

206 @param variables additional variables to be used

207 @param add_environ add environment variables available, does not

208 overwrite existing variables

209 when the job is generated

210 @return list of tuple(instructions, variables)

211

212 The function expects the following list

213 of steps in this order:

214

215 * *automatedsetup*: added by this module

216 * *language*: should be python

217 * *python*: list of interpreters (multiplies jobs)

218 * *virtualenv*: name of the virtual environment

219 * *install*: list of installation steps in the virtual environment

220 * *before_script*: list of steps to run

221 * *script*: list of script to run (multiplies jobs)

222 * *after_script*: list of steps to run

223 * *documentation*: documentation to run after the

224

225 Each step *multiplies jobs* creates a sequence of jobs and a :epkg:`Jenkins` job.

226 """

227 if variables is None:

228 def_variables = {}

229 else:

230 def_variables = variables.copy()

231 if 'Python37' in def_variables and 'Python38' not in def_variables:

232 raise RuntimeError( # pragma: no cover

233 f"Key 'Python38' is missing in {def_variables}.")

234 if add_environ:

235 for k, v in os.environ.items():

236 if k not in def_variables:

237 def_variables[k] = v

238 sequences = []

239 count = {}

240 steps = ["automatedsetup", "language", "python", "virtualenv", "install",

241 "before_script", "script", "after_script",

242 "documentation"]

243 for key in steps:

244 value = obj.get(key, None)

245 if key == "language":

246 if value != "python":

247 raise NotImplementedError( # pragma: no cover

248 "language must be python")

249 continue # pragma: no cover

250 if value is not None:

251 if key in {'python', 'script'} and not isinstance(value, list):

252 value = [value]

253 count[key] = len(value)

254 sequences.append((key, value))

255

256 for k in obj:

257 if k not in steps:

258 raise ValueError(

259 "Unexpected key '{0}' found in yaml file. Expect:\n{1}".format(k, "\n".join(steps)))

260

261 # multiplications

262 i_python = 0

263 i_script = 0

264 notstop = True

265 while notstop:

266 seq = []

267 add = True

268 variables = def_variables.copy()

269 for key, value in sequences:

270 if key == "python":

271 value = value[i_python]

272 if isinstance(value, dict):

273 if 'PATH' not in value:

274 raise KeyError( # pragma: no cover

275 f"The dictionary should include key 'path': {value}")

276 for k, v in sorted(value.items()):

277 if k != 'PATH':

278 variables[k] = v

279 seq.append(('INFO', (k, v)))

280 value = value["PATH"]

281 elif key == "script":

282 value = interpret_instruction(value[i_script], variables)

283 if isinstance(value, dict):

284 for k, v in sorted(value.items()):

285 if k not in ('CMD', 'CMDPY'):

286 seq.append(('INFO', (k, v)))

287 variables[k] = v

288

289 i_script += 1

290 if i_script >= count['script']:

291 i_script = 0

292 i_python += 1

293 if i_python >= count['python']:

294 notstop = False

295 if value is not None and value != 'None':

296 seq.append((key, value))

297 variables[key] = value

298 else:

299 add = False

300 if add:

301 r = interpret_instruction(seq, variables)

302 if r is not None:

303 yield r, variables

304

305

306def ospathjoin(*args, **kwargs):

307 """

308 Simple ``o.path.join`` for a specific platform.

309

310 @param args list of paths

311 @param kwargs additional parameters, among them,

312 *platform* (win32 or ...)

313 @return path

314 """

315 def build_value(*args, **kwargs):

316 platform = kwargs.get('platform', None)

317 if platform is None:

318 return os.path.join(*args)

319 elif platform.startswith("win"):

320 return "\\".join(args)

321 return "/".join(args)

322

323 value = build_value(*args, **kwargs)

324 if value == "/$PYINT":

325 raise RuntimeError( # pragma: no cover

326 f"Impossible values {args} - {kwargs}.")

327 return value

328

329

330def ospathdirname(lp, platform=None):

331 """

332 Simple ``o.path.dirname`` for a specific platform.

333

334 @param lp path

335 @param platform platform

336 @return path

337 """

338 if platform is None:

339 return os.path.dirname(lp)

340 elif platform.startswith("win"):

341 return "\\".join(lp.replace("/", "\\").split("\\")[:-1])

342 return "/".join(lp.replace("\\", "/").split("/")[:-1])

343

344

345def convert_sequence_into_batch_file(seq, variables=None, platform=None):

346 """

347 Converts a sequence of instructions into a batch file.

348

349 @param seq sequence of instructions

350 @param variables list of variables

351 @param platform ``get_platform(platform)`` if None

352 @return (str) batch file or a list of batch file if the constant ``JENKINS_SPLIT``

353 was found in section install (this tweak is needed when the job has to be split

354 for :epkg:`Jenkins`.

355 """

356 global _jenkins_split

357 if platform is None:

358 platform = get_platform(platform)

359

360 iswin = platform.startswith("win")

361

362 if iswin:

363 error_level = "if %errorlevel% neq 0 exit /b %errorlevel%"

364 else:

365 error_level = "if [ $? -ne 0 ]; then exit $?; fi"

366

367 interpreter = None

368 venv_interpreter = None

369 root_project = None

370 anaconda = False

371 conda = None

372 echo = "@echo" if iswin else "echo"

373

374 rowsset = []

375 if iswin:

376 rowsset.append("@echo off")

377 rowsset.append("set PATH0=%PATH%")

378

379 def add_path_win(rows, interpreter, platform, root_project):

380 path_inter = ospathdirname(interpreter, platform)

381 if len(path_inter) == 0:

382 raise ValueError( # pragma: no cover

383 "Unable to guess interpreter path from '{0}', platform={1}"

384 "".format(interpreter, platform))

385 if iswin:

386 rows.append(f"set PATH={path_inter};%PATH%")

387 else:

388 rows.append(f"export PATH={path_inter}:$PATH")

389 if root_project is not None:

390 if iswin:

391 rows.append(f"set ROOTPROJECT={root_project}")

392 else:

393 rows.append(f"export ROOTPROJECT={root_project}")

394

395 rows = []

396 splits = [rows]

397 typstr = str

398

399 for key, value in seq:

400 if key == "automatedsetup":

401 rows.append("")

402 rows.append(echo + " AUTOMATEDSETUP")

403 rows.append("\n".join(value))

404 rows.append("")

405 elif key == "python":

406 variables["YMLPYTHON"] = value

407 if variables.get('DIST', None) == "conda":

408 rows.append(echo + " conda")

409 anaconda = True

410 interpreter = ospathjoin(

411 value, "python", platform=platform)

412 venv_interpreter = value

413 if platform.startswith("win"):

414 conda = ospathjoin(

415 value, "Scripts", "conda", platform=platform)

416 else:

417 conda = ospathjoin(

418 value, "bin", "conda", platform=platform)

419 else:

420 if iswin:

421 interpreter = ospathjoin(

422 value, "python", platform=platform)

423 else:

424 interpreter = ospathjoin(

425 value, "$PYINT", platform=platform)

426 venv_interpreter = value

427 rows.append(echo + " interpreter=" + interpreter)

428

429 elif key == "virtualenv":

430 if isinstance(value, list):

431 if len(value) > 2:

432 raise ValueError( # pragma: no cover

433 "Expecting one or two values for the path of the virtual environment"

434 ":\n{0}".format(value))

435 d = value[0].copy()

436 for i in range(1, len(value)):

437 d.update(value[i])

438 value = d

439 p = value["path"] if isinstance(value, dict) else value

440 rows.append("")

441 rows.append(echo + f" CREATE VIRTUAL ENVIRONMENT in {p}")

442 if not anaconda:

443 if iswin:

444 rows.append('if not exist "{0}" mkdir "{0}"'.format(p))

445 else:

446 rows.append('if [-f {0}]; then mkdir "{0}"; fi'.format(p))

447 if anaconda:

448 pinter = ospathdirname(interpreter, platform=platform)

449 rows.append(

450 f'"{conda}" create -y -v -p "{p}" --clone "{pinter}" --offline --no-update-deps')

451 interpreter = ospathjoin(

452 p, "python", platform=platform)

453 else:

454 if iswin:

455 rows.append("set KEEPPATH=%PATH%")

456 rows.append(f"set PATH={venv_interpreter};%PATH%")

457 else:

458 rows.append("export KEEPPATH=$PATH")

459 rows.append(

460 f"export PATH={venv_interpreter}:$PATH")

461 pat = '"{0}" -m virtualenv {1}'

462 if isinstance(value, dict):

463 system_site_packages = value.get(

464 'system_site_packages', True)

465 else:

466 system_site_packages = True

467 if system_site_packages:

468 pat += " --system-site-packages"

469 rows.append(pat.format(interpreter, p))

470 if iswin:

471 rows.append("set PATH=%KEEPPATH%")

472 interpreter = ospathjoin(

473 p, "Scripts", "python", platform=platform)

474 else:

475 rows.append("export PATH=$KEEPPATH")

476 interpreter = ospathjoin(

477 p, "bin", "python", platform=platform)

478 rows.append(error_level)

479

480 elif key in {"install", "before_script", "script", "after_script", "documentation"}:

481 if value is not None:

482 if isinstance(value, dict):

483 if "CMD" not in value and "CMDPY" not in value:

484 raise KeyError( # pragma: no cover

485 "A script defined by a dictionary must contain key "

486 "'{0}' or '{1}' in \n{2}".format("CMD", 'CMDPY', value))

487 if "NAME" in value:

488 if iswin:

489 rows.append(f"set JOB_NAME={value['NAME']}")

490 else:

491 rows.append(f"export JOB_NAME={value['NAME']}")

492 if "CMD" in value:

493 value = value["CMD"]

494 else:

495 value = evaluate_condition(

496 value["CMDPY"], variables=variables)

497 elif isinstance(value, list):

498 starter = list(rows)

499 elif isinstance(value, typstr):

500 pass

501 else:

502 raise TypeError( # pragma: no cover

503 f"value must of type list, dict, not '{type(value)}'\n{value}")

504

505 rows.append("")

506 rows.append(echo + " " + key.upper())

507 add_path_win(rows, interpreter, platform, root_project)

508 if not isinstance(value, list):

509 value = [value, error_level]

510 else:

511 keep = value

512 value = []

513 for v in keep:

514 if v.startswith(_jenkins_split):

515 if "-" in v:

516 nbrem = v.split("-")[-1]

517 try:

518 nbrem = int(nbrem)

519 except ValueError: # pragma: no cover

520 raise ValueError(

521 f"Unable to interpret '{v}'")

522 else:

523 nbrem = 0

524 rows.extend(value)

525 value = []

526 st = list(starter)

527 if nbrem > 0:

528 st = st[:-nbrem]

529 splits.append(st)

530 rows = splits[-1]

531 add_path_win(rows, interpreter,

532 platform, root_project)

533 else:

534 value.append(v)

535 value.append(error_level)

536 rows.extend(value)

537 elif key == 'INFO':

538 vs = f'"{value[1]}"' if isinstance(

539 value[1], str) and " " in value[1] else value[1]

540 if iswin:

541 rowsset.append(f"SET {value[0]}={vs}")

542 else:

543 rowsset.append(f"export {value[0]}={vs}")

544 else:

545 raise ValueError( # pragma: no cover

546 f"unexpected key '{key}'")

547

548 splits = [rowsset + _ for _ in splits]

549 allres = []

550 for rows in splits:

551 try:

552 res = "\n".join(rows)

553 except TypeError as e: # pragma: no cover

554 raise TypeError("Unexpected type\n{0}".format(

555 "\n".join([str((type(_), _)) for _ in rows]))) from e

556 if _jenkins_split in res:

557 raise ValueError( # pragma: no cover

558 "Constant '{0}' is present in the generated script. "

559 "It can only be added to the install section."

560 "".format(_jenkins_split))

561 allres.append(res)

562 return allres if len(allres) > 1 else allres[0]

563

564

565def infer_project_name(file_or_buffer, source):

566 """

567 Infers a project name based on :epkg:`yml` file.

568

569 @param file_or_buffer file name

570 @param source second output of @see fn read_content_ufs

571 @return name

572

573 The function can infer a name for *source* in ``{'r', 'u'}``.

574 For *source* equal to ``'s'``, it returns ``'unknown_string'``.

575 """

576 if source == "r":

577 fold = os.path.dirname(file_or_buffer)

578 last = os.path.split(fold)[-1]

579 elif source == "u":

580 spl = file_or_buffer.split('/')

581 pos = -2

582 name = None

583 while len(spl) > -pos:

584 name = spl[pos]

585 if name in {'master', 'main'}:

586 pos -= 1

587 elif 'github' in name:

588 break

589 else:

590 break

591 if name is None:

592 raise ValueError( # pragma: no cover

593 f"Unable to infer project name for '{file_or_buffer}'")

594 return name

595 elif source == "s":

596 return "unknown_string"

597 else:

598 raise ValueError( # pragma: no cover

599 f"Unexpected value for add_source: '{source}' for '{file_or_buffer}'")

600 return last

601

602

603def enumerate_processed_yml(file_or_buffer, context=None, engine="jinja2", platform=None,

604 server=None, git_repo=None, add_environ=True, overwrite=False,

605 build_location=None, branch='master', **kwargs):

606 """

607 Submits or enumerates jobs based on the content of a :epkg:`yml` file.

608

609 @param file_or_buffer filename or string

610 @param context variables to replace in the configuration

611 @param engine see @see fn apply_template

612 @param server see @see cl JenkinsExt

613 @param platform plaform where the job will be executed

614 @param git_repo git repository (if *server* is not None)

615 @param add_environ add environment variable before interpreting the job

616 @param overwrite overwrite the job if it already exists in Jenkins

617 @param build_location location for the build

618 @param branch default branch

619 @param kwargs see @see me create_job_template

620 @return enumerator for *(job, name, variables)*

621

622 Example of a :epkg:`yml` file

623 `.local.jenkins.win.yml

624 <https://github.com/sdpython/pyquickhelper/blob/master/.local.jenkins.win.yml>`_.

625 A subfolder was added to the project location.

626 A scheduler can be defined as well by adding ``SCHEDULER:'* * * * *'``.

627 """

628 typstr = str

629 fLOG = kwargs.get('fLOG', None)

630 project_name = None if context is None else context.get(

631 "project_name", None)

632 obj, project_name = load_yaml(

633 file_or_buffer, context=context, platform=platform)

634 platform_set = platform or get_platform(platform)

635 for seq, var in enumerate_convert_yaml_into_instructions(obj, variables=context, add_environ=add_environ):

636 conv = convert_sequence_into_batch_file(

637 seq, variables=var, platform=platform)

638

639 # we extract a suffix from the command line

640 if server is not None:

641 name = "_".join([project_name, var.get('NAME', ''),

642 typstr(var.get("VERSION", '')).replace(".", ""),

643 var.get('DIST', '')])

644

645 if platform_set.startswith("win"):

646 if isinstance(conv, list):

647 conv = ["SET NAME_JENKINS=" +

648 name + "\n" + _ for _ in conv]

649 else:

650 conv = "SET NAME_JENKINS=" + name + "\n" + conv

651 else:

652 if isinstance(conv, list):

653 conv = ["export NAME_JENKINS=" +

654 name + "\n" + _ for _ in conv]

655 conv.append("export $(cat ~/.profile)")

656 else:

657 conv = ("export NAME_JENKINS=" + name +

658 "\nexport $(cat ~/.profile)\n" + conv)

659

660 import jenkins

661 try:

662 j = server.get_job_config(name) if not server._mock else None

663 except jenkins.NotFoundException: # pragma: no cover

664 j = None

665 except jenkins.JenkinsException as e: # pragma: no cover

666 from .jenkins_exceptions import JenkinsExtException

667 raise JenkinsExtException(

668 f"Unable to retrieve job config for name='{name}'.") from e

669

670 update_job = False

671 if j is not None:

672 if kwargs.get('update', True):

673 update_job = True

674 else:

675 if fLOG is not None: # pragma: no cover

676 fLOG("[jenkins] delete job", name)

677 server.delete_job(name)

678

679 if git_repo is not None and project_name not in git_repo:

680 git_repo += project_name

681

682 # set up location

683 if build_location is None:

684 loc = None

685 else:

686 loc = ospathjoin(build_location, project_name,

687 name, platform=platform)

688

689 if overwrite or j is None:

690 timeout = var.get("TIMEOUT", None)

691 scheduler = var.get("SCHEDULER", None)

692 clean_repo = var.get("CLEAN", True) in { # pylint: disable=W0130

693 True, 1, "True", "true", "1"}

694 if timeout is not None:

695 kwargs["timeout"] = timeout

696 if scheduler is not None:

697 if "FIXED" in scheduler:

698 scheduler = scheduler.replace("FIXED", "").strip()

699 adjuster_scheduler = False

700 elif "STARTUP" in scheduler:

701 adjuster_scheduler = False

702 elif 'fixed' in scheduler.lower():

703 raise ValueError( # pragma: no cover

704 "Scheduler should contain 'FIXED' in upper case.")

705 elif 'startup' in scheduler.lower():

706 raise ValueError( # pragma: no cover

707 "Scheduler should contain 'STARTUP' in upper case.")

708 else:

709 adjuster_scheduler = True

710 kwargs["scheduler"] = scheduler

711 kwargs["adjuster_scheduler"] = adjuster_scheduler

712 yield server.create_job_template(name, script=conv, git_repo=git_repo,

713 update=update_job, location=loc,

714 clean_repo=clean_repo, branch=branch,

715 **kwargs), name, var

716 else:

717 yield conv, None, var

Coverage for pyquickhelper/jenkinshelper/yaml_helper.py: 95%

388 statements