Coverage for pyquickhelper/pycode/setup_helper.py: 64%
318 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Helper for the setup
4"""
6import os
7import sys
8import re
9import warnings
10import hashlib
11import datetime
14def get_available_setup_commands():
15 """
16 Returns the list of commands :epkg:`pyquickhelper` implements
17 or allows.
18 """
19 commands = ['bdist_egg', 'bdist_msi', 'bdist_wheel', 'bdist_wininst', 'build27',
20 'build_ext', 'build_script', 'build_sphinx', 'clean_pyd', 'clean_space',
21 'copy27', 'copy_dist', 'copy_sphinx', 'history', 'lab', 'local_pypi',
22 'notebook', 'publish', 'publish_doc', 'register', 'run27', 'run_pylint',
23 'sdist', 'setupdep', 'test_local_pypi',
24 'unittests', 'unittests_GUI', 'unittests_LONG', 'unittests_SKIP',
25 'upload_docs', 'write_version', 'local_jenkins']
26 return commands
29def get_available_build_commands():
30 """
31 Returns commands which build the package.
32 """
33 return {"sdist", "bdist_wheel", "publish", "publish_doc", "register",
34 "upload_docs", "bdist_wininst", "build_ext"}
37def available_commands_list(argv):
38 """
39 Checks that on command handled by pyquickhelper is part of the arguments.
41 @param argv ``sys.args``
42 @return bool
43 """
44 commands = get_available_setup_commands()
45 for c in commands:
46 if c in argv:
47 return True
48 return False
51def process_argv_for_unittest(argv, skip_function):
52 """
53 Interprets command line arguments.
55 :param argv: list
56 :param skip_function: skipping function
57 :return: commands
58 """
59 if "-d" in argv:
60 ld = argv.index("-d")
61 if ld >= len(argv) - 1:
62 raise ValueError( # pragma: no cover
63 "Option -d should be follow by a duration in seconds.")
64 d = float(argv[ld + 1])
65 else:
66 d = None
68 if "-f" in argv:
69 lf = argv.index("-f")
70 if lf >= len(argv) - 1:
71 raise ValueError( # pragma: no cover
72 "Option -f should be follow by a file.")
73 f = argv[lf + 1]
74 else:
75 f = None
77 if "-e" in argv:
78 le = argv.index("-e")
79 if le >= len(argv) - 1:
80 raise ValueError( # pragma: no cover
81 "Option -e should be follow by a regular expression.")
82 pattern = argv[le + 1]
83 if len(pattern) >= 2 and pattern[0] == pattern[-1] == '"':
84 pattern = pattern[1:-1]
85 e = re.compile(pattern)
86 else:
87 e = None
89 if "-g" in argv:
90 lg = argv.index("-g")
91 if lg >= len(argv) - 1:
92 raise ValueError( # pragma: no cover
93 "Option -g should be follow by a regular expression.")
94 pattern = argv[lg + 1]
95 if len(pattern) >= 2 and pattern[0] == pattern[-1] == '"':
96 pattern = pattern[1:-1]
97 g = re.compile(pattern)
98 else:
99 g = None
101 if f is None and d is None and e is None and g is None:
102 return skip_function # pragma: no cover
104 def ereg(name):
105 return (e is None) or (e.search(name) is not None)
107 def greg(name):
108 return (g is None) or (g.search(name) is None)
110 if f is not None:
111 if d is not None: # pragma: no cover
112 raise NotImplementedError(
113 "Options -f and -d cannot be specified at the same time.")
115 def allow(name, code, duration): # pragma: no cover
116 name = os.path.split(name)[-1]
117 return f not in name and ereg(name) and greg(name)
118 return allow # pragma: no cover
119 else:
120 # d is not None
121 def skip_allowd(name, code, duration):
122 name = os.path.split(name)[-1]
123 cond = (
124 (duration is None or d is None or duration <= d) and
125 ereg(name) and greg(name))
126 return not cond
127 return skip_allowd
130def process_standard_options_for_setup(
131 argv, file_or_folder, project_var_name, module_name=None, unittest_modules=None,
132 pattern_copy=None,
133 requirements=None, port=8067, blog_list=None, default_engine_paths=None,
134 extra_ext=None, add_htmlhelp=False, coverage_options=None,
135 coverage_exclude_lines=None, func_sphinx_begin=None, func_sphinx_end=None,
136 additional_notebook_path=None, additional_local_path=None, copy_add_ext=None,
137 nbformats=("ipynb", "html", "python", "rst", "slides",
138 "pdf", "github"),
139 layout=None, direct_call=False,
140 additional_ut_path=None,
141 skip_function=None, covtoken=None,
142 stdout=None, stderr=None, use_run_cmd=False, filter_warning=None,
143 file_filter_pep8=None, github_owner=None,
144 existing_history=None, coverage_root='src',
145 fexclude=None, skip_issues=None, fLOG=None):
146 """
147 Processes the standard options the module pyquickhelper is
148 able to process assuming the module which calls this function
149 follows the same design as *pyquickhelper*, it will process the following
150 options:
152 .. runpython::
154 from pyquickhelper.pycode import process_standard_options_for_setup_help
155 process_standard_options_for_setup_help("--help-commands")
157 @param argv = *sys.argv*
158 @param file_or_folder file ``setup.py`` or folder which contains it
159 @param project_var_name display name of the module
160 @param module_name module name, None if equal to *project_var_name*
161 (``import <module_name>``)
162 @param unittest_modules modules added for the unit tests,
163 see @see fn py3to2_convert_tree
164 @param pattern_copy see @see fn py3to2_convert_tree
165 @param requirements dependencies, fetched with a local pipy server from
166 ``http://localhost:port/``
167 @param port port for the local pipy server
168 @param blog_list list of blog to listen for this module
169 (usually stored in ``module.__blog__``)
170 @param default_engine_paths define the default location for python engine,
171 should be dictionary *{ engine: path }*, see below.
172 @param extra_ext extra file extension to process (add a page for each of them,
173 ex ``["doc"]``)
174 @param add_htmlhelp run HTML Help too (only on Windows)
175 @param coverage_options see @see fn main_wrapper_tests
176 @param coverage_exclude_lines see @see fn main_wrapper_tests
177 @param func_sphinx_begin function called before the documentation generation,
178 it gets the same parameters as this function (all named),
179 use ``**args**``
180 @param func_sphinx_end function called after the documentation generation,
181 it gets the same parameters as this function (all named),
182 use ``**args**``
183 @param additional_notebook_path additional paths to add when launching the notebook
184 @param additional_local_path additional paths to add when running a local command
185 @param copy_add_ext additional file extensions to copy
186 @param nbformats requested formats for the notebooks conversion
187 @param layout list of formats sphinx should generate such as html, latex, pdf, docx,
188 it is a list of tuple (layout, build directory, parameters to override),
189 if None --> ``["html", "pdf"]``
190 @param additional_ut_path additional paths to add when running unit tests
191 @param skip_function function to skip unit tests, see @ee fn main_wrapper_tests
192 @param covtoken token used when publishing coverage report to
193 `codecov <https://codecov.io/>`_,
194 more in @see fn main_wrapper_tests
195 @param fLOG logging function
196 @param stdout redirect stdout for unit test if not None
197 @param stderr redirect stderr for unit test if not None
198 @param use_run_cmd to run the sphinx documentation with @see fn run_cmd and
199 not ``os.system``
200 @param filter_warning see @see fn main_wrapper_tests
201 @param file_filter_pep8 function to filter out files for which checking pep8
202 (see @see fn remove_extra_spaces_folder)
203 @param github_owner :epkg:`github` owner of the package
204 @param existing_history existing history, retrieves existing issues stored
205 in that file
206 @param coverage_root see @see fn main_wrapper_tests
207 @param direct_call @see fn generate_help_sphinx
208 @param fexclude function which tells which file not to copy in the folder
209 used to build the documentation
210 @param skip_issues skip a given list of issues when building the history
211 @return True (an option was processed) or False,
212 the file ``setup.py`` should call function ``setup``
214 The command ``build_script`` is used, the flag ``--private`` can be used to
215 avoid producing scripts to publish the module on `Pypi <https://pypi.python.org/pypi>`_.
217 An example for *default_engine_paths*::
219 default_engine_paths = {
220 "windows": {
221 "__PY35__": None,
222 "__PY36_X64__": "c:\\Python365_x64",
223 "__PY37_X64__": "c:\\Python372_x64",
224 "__PY38_X64__": "c:\\Python387_x64",
225 "__PY39_X64__": "c:\\Python391_x64",
226 "__PY310_X64__": "c:\\Python3101_x64",
227 },
228 }
230 Parameters *coverage_options*, *coverage_exclude_lines*, *copy_add_ext* were added
231 for function @see fn main_wrapper_tests.
232 Parameter *unittest_modules* accepts a list of string and 2-uple.
233 If it is a 2-uple, the first string is used to convert Python 3 code into Python 2
234 (in case the local folder is different from the module name),
235 the second string is used to add local path to the variable ``PYTHON_PATH``.
236 If it is a single string, it means both name strings are equal.
237 Parameters *func_sphinx_begin* and *func_sphinx_end* were added
238 to pre-process or post-process the documentation.
239 Parameter *additional_notebook_path* was added to specify some additional
240 paths when preparing the script *auto_cmd_notebook.bat*.
241 Parameters *layout*, *nbformats* were added for
242 function @see fn generate_help_sphinx.
243 The coverage computation can be disable by specifying
244 ``coverage_options["disable_coverage"] = True``.
245 Parameter *covtoken* was added to post the coverage report to :epkg:`codecov`.
246 Option ``-e`` and ``-g`` were added to
247 filter file by regular expressions (in with *e*, out with *g*).
248 """
249 if module_name is not None and (
250 len(module_name) == 0 or module_name[0] == '_'):
251 raise RuntimeError( # pragma: no cover
252 f"module cannot be empty or start with '_': {module_name!r}.")
253 if fLOG is None: # pragma: no cover
254 from ..loghelper.flog import noLOG
255 fLOG = noLOG
256 if skip_function is None: # pragma: no cover
257 from .utils_tests_private import default_skip_function
258 skip_function = default_skip_function
259 if pattern_copy is None:
260 # delayed import
261 from .default_regular_expression import _setup_pattern_copy
262 pattern_copy = _setup_pattern_copy
264 if layout is None:
265 layout = ["html", "pdf"]
267 if "--help" in argv or "--help-commands" in argv:
268 process_standard_options_for_setup_help(argv)
269 return not len(get_available_build_commands() & set(argv))
270 fLOG("[process_standard_options_for_setup]", argv)
271 fLOG("[process_standard_options_for_setup] python version:", sys.version_info)
273 folder = file_or_folder if os.path.isdir(
274 file_or_folder) else os.path.dirname(file_or_folder)
275 unit_test_folder = os.path.join(folder, "_unittests")
276 fLOG(f"unittest_modules={unittest_modules}")
278 if unittest_modules is None:
279 unittest_modules_py3to2 = None
280 unittest_modules_script = None
281 else: # pragma: no cover
282 unittest_modules_py3to2 = []
283 unittest_modules_script = []
284 for mod in unittest_modules:
285 if isinstance(mod, tuple):
286 unittest_modules_py3to2.append(mod[0])
287 unittest_modules_script.append(mod[1])
288 else:
289 unittest_modules_py3to2.append(mod)
290 unittest_modules_script.append(mod)
292 # dump unit test coverage?
294 def dump_coverage_fct(full=True):
295 mn = project_var_name if module_name is None else module_name
296 full_path = _get_dump_default_path(folder, mn, argv)
297 if full_path is None or full:
298 return full_path
299 else:
300 sub = os.path.split(full_path)[0]
301 sub = os.path.split(sub)[0]
302 return sub
304 # starts interpreting the commands
306 if "clean_space" in argv: # pragma: no cover
307 rem = clean_space_for_setup(
308 file_or_folder, file_filter=file_filter_pep8)
309 print("[clean_space] number of impacted files (pep8 + rst):", len(rem))
310 rem = clean_notebooks_for_numbers(file_or_folder)
311 print("[clean_space] number of impacted notebooks:", len(rem))
312 return True
314 if "run_pylint" in argv: # pragma: no cover
315 verbose = '-v' in argv
316 pos = argv.index('run_pylint')
317 ignores = [_[2:] for _ in argv if _[:2] == '-i']
318 ignores = None if len(ignores) == 0 else tuple(ignores)
319 argv = [_ for _ in argv if _ not in (
320 '-v', '-') and not _.startswith('-i')]
321 pattern = argv[pos + 1] if len(argv) > pos + 1 else ".*[.]py$"
322 neg_pattern = argv[pos + 2] if len(argv) > pos + 2 else None
323 print("[run_pylint] run_pylint for sources pattern='{0}' neg_pattern='{1}'".format(
324 pattern, neg_pattern))
325 src_folder = os.path.join(folder, "src")
326 if not os.path.exists(src_folder):
327 src_folder = folder
328 run_pylint_for_setup(src_folder, fLOG=print, pattern=pattern,
329 verbose=verbose, pylint_ignore=ignores)
330 print("[run_pylint] run_pylint for unittest")
331 run_pylint_for_setup(os.path.join(folder, "_unittests"),
332 fLOG=print, pattern=pattern, verbose=verbose,
333 pylint_ignore=ignores)
334 return True
336 elif 'history' in argv: # pragma: no cover
337 dest = ' '.join(argv).split( # pylint: disable=C0207
338 'history')[-1].strip()
339 if not dest:
340 dest = os.path.join(folder, 'HISTORY.rst')
341 if existing_history is None:
342 hfold = get_folder(file_or_folder)
343 histo = os.path.join(hfold, 'HISTORY.rst')
344 if os.path.exists(histo):
345 existing_history = histo
346 if existing_history is not None:
347 print('[history] existing ', existing_history)
348 print('[history] ', dest)
349 build_history_from_setup(
350 dest, owner=github_owner, module=project_var_name,
351 existing_history=existing_history,
352 skip_issues=skip_issues, fLOG=fLOG)
353 return True
355 elif "write_version" in argv: # pragma: no cover
356 fLOG("---- JENKINS BEGIN WRITE VERSION ----")
357 write_version_for_setup(file_or_folder, module_name=module_name)
358 fLOG("---- JENKINS BEGIN END VERSION ----")
359 return True
361 elif "clean_pyd" in argv:
362 clean_space_for_setup(file_or_folder)
363 return True
365 elif "build_sphinx" in argv:
366 # delayed import
367 try:
368 from nbconvert.nbconvertapp import main as nbconvert_main
369 if nbconvert_main is None: # pragma: no cover
370 raise AttributeError("nbconvert_main is None")
371 except AttributeError as e: # pragma: no cover
372 raise ImportError(
373 "Unable to import nbconvert, cannot generate the documentation") from e
375 if func_sphinx_begin is not None:
376 func_sphinx_begin(
377 argv=argv, file_or_folder=file_or_folder,
378 project_var_name=project_var_name,
379 module_name=module_name, unittest_modules=unittest_modules,
380 pattern_copy=pattern_copy,
381 requirements=requirements, port=port, blog_list=blog_list,
382 default_engine_paths=default_engine_paths,
383 extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
384 coverage_options=coverage_options,
385 coverage_exclude_lines=coverage_exclude_lines,
386 func_sphinx_begin=func_sphinx_begin,
387 func_sphinx_end=func_sphinx_end,
388 additional_notebook_path=additional_notebook_path,
389 nbformats=nbformats, layout=layout,
390 skip_function=skip_function,
391 addition_ut_path=additional_ut_path, fLOG=fLOG)
392 standard_help_for_setup(
393 argv, file_or_folder, project_var_name,
394 module_name=module_name, extra_ext=extra_ext,
395 add_htmlhelp=add_htmlhelp, copy_add_ext=copy_add_ext,
396 nbformats=nbformats, layout=layout,
397 use_run_cmd=use_run_cmd, fLOG=fLOG, direct_call=direct_call,
398 fexclude=fexclude)
400 if func_sphinx_end is not None:
401 func_sphinx_end(
402 argv=argv, file_or_folder=file_or_folder,
403 project_var_name=project_var_name,
404 module_name=module_name, unittest_modules=unittest_modules,
405 pattern_copy=pattern_copy,
406 requirements=requirements, port=port, blog_list=blog_list,
407 default_engine_paths=default_engine_paths,
408 extra_ext=extra_ext, add_htmlhelp=add_htmlhelp,
409 coverage_options=coverage_options,
410 coverage_exclude_lines=coverage_exclude_lines,
411 func_sphinx_begin=func_sphinx_begin,
412 func_sphinx_end=func_sphinx_end,
413 additional_notebook_path=additional_notebook_path,
414 nbformats=nbformats, layout=layout,
415 skip_function=skip_function,
416 addition_ut_path=additional_ut_path, fLOG=fLOG)
418 return True
420 elif "unittests" in argv:
421 skip_f = process_argv_for_unittest(argv, skip_function)
422 return run_unittests_for_setup(
423 file_or_folder,
424 coverage_options=coverage_options,
425 coverage_exclude_lines=coverage_exclude_lines,
426 additional_ut_path=additional_ut_path,
427 skip_function=skip_f, covtoken=covtoken,
428 stdout=stdout, stderr=stderr,
429 filter_warning=filter_warning, dump_coverage=dump_coverage_fct(),
430 add_coverage_folder=dump_coverage_fct(False),
431 coverage_root=coverage_root, fLOG=fLOG)
433 elif "unittests_LONG" in argv:
434 def skip_long(name, code, duration):
435 return "test_LONG_" not in name
436 return run_unittests_for_setup(
437 file_or_folder, skip_function=skip_long,
438 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
439 additional_ut_path=additional_ut_path,
440 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
441 fLOG=fLOG)
443 elif "unittests_SKIP" in argv:
444 def skip_skip(name, code, duration):
445 return "test_SKIP_" not in name
446 return run_unittests_for_setup(
447 file_or_folder, skip_function=skip_skip,
448 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
449 additional_ut_path=additional_ut_path,
450 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
451 fLOG=fLOG)
453 elif "unittests_GUI" in argv: # pragma: no cover
454 def skip_skip(name, code, duration):
455 return "test_GUI_" not in name
456 return run_unittests_for_setup(
457 file_or_folder, skip_function=skip_skip,
458 coverage_options=coverage_options, coverage_exclude_lines=coverage_exclude_lines,
459 additional_ut_path=additional_ut_path,
460 stdout=stdout, stderr=stderr, dump_coverage=dump_coverage_fct(),
461 fLOG=fLOG)
463 elif "build_script" in argv: # pragma: no cover
464 # delayed import
465 from .build_helper import get_extra_script_command, get_script_command, get_build_script
467 # script running setup.py
468 script = get_build_script(
469 project_var_name, requirements=requirements, port=port,
470 default_engine_paths=default_engine_paths)
471 binto = os.path.join(folder, "bin")
472 if not os.path.exists(binto):
473 os.mkdir(binto)
474 with open(os.path.join(folder, "bin", f"auto_unittest_setup_help.{get_script_extension()}"), "w") as f:
475 f.write(script)
477 for c in ("build_script", "clean_space",
478 "write_version", "clean_pyd",
479 "build_sphinx", "unittests",
480 "unittests_LONG", "unittests_SKIP", "unittests_GUI",
481 "unittests -d 5", "copy27",
482 "local_pypi", 'run_pylint'):
483 sc = get_script_command(
484 c, project_var_name, requirements=requirements, port=port, platform=sys.platform,
485 default_engine_paths=default_engine_paths, additional_local_path=additional_local_path)
486 cn = c.replace(" ", "_")
487 with open(os.path.join(folder, "bin", f"auto_setup_{cn}.{get_script_extension()}"), "w") as f:
488 f.write(sc)
490 # script running for a developper
492 for c in {"notebook", "publish", "publish_doc", "local_pypi", "run27",
493 "build27", "setupdep", "copy_dist",
494 "any_setup_command", "build_dist",
495 "copy_sphinx", "lab", "history"}:
496 if "--private" in argv and "publish" in c:
497 # we skip this to avoid producing scripts for publish
498 # functionalities
499 continue
500 sc = get_extra_script_command(c, project_var_name, requirements=requirements,
501 port=port, platform=sys.platform,
502 default_engine_paths=default_engine_paths,
503 unit_test_folder=unit_test_folder,
504 unittest_modules=unittest_modules_script,
505 additional_notebook_path=additional_notebook_path,
506 additional_local_path=additional_local_path)
507 if sc is None:
508 continue
509 if c == "setupdep":
510 folder_setup = os.path.join(folder, "build", "auto_setup")
511 if not os.path.exists(folder_setup):
512 os.makedirs(folder_setup)
513 with open(os.path.join(folder_setup, "auto_setup_dep.py"), "w") as f:
514 f.write(sc)
515 else:
516 with open(os.path.join(folder, "bin", f"auto_cmd_{c}.{get_script_extension()}"), "w") as f:
517 f.write(sc)
519 # script for anybody
520 write_module_scripts(
521 folder, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
523 # pyproj for PTVS
524 if sys.platform.startswith("win"): # pragma: no cover
525 write_pyproj(folder)
527 return True
529 elif "copy27" in argv: # pragma: no cover
530 # delayed import
531 from .py3to2 import py3to2_convert_tree
532 root = os.path.abspath(os.path.dirname(file_or_folder))
533 root = os.path.normpath(root)
534 dest = os.path.join(root, "dist_module27")
535 py3to2_convert_tree(
536 root, dest, unittest_modules=unittest_modules_py3to2, pattern_copy=pattern_copy)
537 return True
539 elif "local_pypi" in argv: # pragma: no cover
540 # delayed import
541 from ..filehelper import get_url_content_timeout
542 url = f"http://localhost:{port}/"
543 content = get_url_content_timeout(url, timeout=5)
544 if content is None or len(content) == 0:
545 raise RuntimeError("test failed for url: " + url)
546 print(content)
547 return True
549 elif 'local_jenkins' in argv: # pragma: no cover
550 pos = argv.index("local_jenkins")
551 user = argv[pos + 1]
552 password = argv[pos + 2]
553 if len(argv) > pos + 3:
554 location = argv[pos + 3]
555 else:
556 if sys.platform.startswith("win"): # pragma: no cover
557 location = "\\Jenkins"
558 else:
559 location = "somewhere/workspace"
560 if len(argv) > pos + 4:
561 server_url = argv[pos + 4]
562 else:
563 server_url = "http://localhost:8080/"
564 from ..jenkinshelper import JenkinsExt, setup_jenkins_server_yml, default_jenkins_jobs
565 modules = default_jenkins_jobs(
566 github_owner=github_owner, module_name=project_var_name if module_name is None else module_name)
567 key = "Python%d%d" % sys.version_info[:2]
568 engines = {key: os.path.abspath(os.path.dirname(sys.executable))}
569 js = JenkinsExt(server_url, user, password, engines=engines)
570 setup_jenkins_server_yml(js, github=github_owner, modules=modules, fLOG=fLOG, overwrite=True,
571 delete_first=False, location=location)
572 return True
574 else:
575 return False
578def get_script_extension():
579 """
580 Returns the scripts extension based on the system it is running on.
582 @return bat or sh
583 """
584 if sys.platform.startswith("win"): # pragma: no cover
585 return "bat"
586 return "sh"
589def get_folder(file_or_folder):
590 """
591 Returns the folder which contains ``setup.py``.
593 @param file_or_folder file ``setup.py`` or folder which contains it
594 @return folder
595 """
596 file_or_folder = os.path.abspath(file_or_folder)
597 if os.path.isdir(file_or_folder):
598 folder = file_or_folder
599 else:
600 folder = os.path.dirname(file_or_folder)
601 return folder
604def write_version_for_setup(file_or_folder, exc=False, module_name=None):
605 """
606 Extracts the version number,
607 the function writes the files ``version.txt`` in this folder.
609 @param file_or_folder file ``setup.py`` or folder which contains it
610 @param exc raises an exception if cannot look into git folder
611 @param module_name module name
612 @return version number
613 """
614 # delayed import to speed up import of pycode
615 from ..loghelper.pyrepo_helper import SourceRepository
616 src = SourceRepository(commandline=True)
617 ffolder = get_folder(file_or_folder)
618 try:
619 version = src.version(ffolder)
620 except Exception as e: # pragma: no cover
621 if exc:
622 raise e
623 return None
624 if version in ["0", 0, None]:
625 raise RuntimeError( # pragma: no cover
626 f"issue with version {version}")
628 # write version number
629 if version is not None:
630 with open(os.path.join(ffolder, "version.txt"), "w") as f:
631 f.write(str(version) + "\n")
633 modifies_init_file(ffolder, version, module_name=module_name)
634 return version
637def clean_space_for_setup(file_or_folder, file_filter=None):
638 """
639 .. index:: pep8
641 Does some cleaning within the module, apply :epkg:`pep8` rules.
643 @param file_or_folder file ``setup.py`` or folder which contains it
644 @param file_filter file filter (see @see fn remove_extra_spaces_folder)
645 @return impacted files
646 """
647 # delayed import
648 from .code_helper import remove_extra_spaces_folder
649 ffolder = get_folder(file_or_folder)
650 rem = remove_extra_spaces_folder(
651 ffolder, extensions=[".py", ".rst", ".md", ".bat", ".sh"],
652 file_filter=file_filter)
653 return rem
656def clean_notebooks_for_numbers(file_or_folder):
657 """
658 Upgrades notebooks to the latest format and
659 cleans notebooks execution numbers and rearranges the JSON file.
661 @param file_or_folder file ``setup.py`` or folder which contains it
662 @return impacted files
664 .. index:: notebooks
665 """
666 from ..ipythonhelper.notebook_helper import upgrade_notebook, remove_execution_number
667 from ..filehelper import explore_folder_iterfile
668 ffolder = get_folder(file_or_folder)
669 fold2 = os.path.normpath(
670 os.path.join(ffolder, "_doc", "notebooks"))
671 mod = []
672 for nbf in explore_folder_iterfile(fold2, pattern=".*[.]ipynb"):
673 t = upgrade_notebook(nbf)
674 if t:
675 mod.append(nbf) # pragma: no cover
676 # remove numbers
677 s = remove_execution_number(nbf, nbf)
678 if s:
679 mod.append(nbf)
680 return mod
683def standard_help_for_setup(argv, file_or_folder, project_var_name, module_name=None,
684 extra_ext=None, add_htmlhelp=False, copy_add_ext=None,
685 nbformats=("ipynb", "html", "python",
686 "rst", "slides", "pdf"),
687 layout=None, use_run_cmd=False, direct_call=False,
688 fexclude=None, fLOG=None):
689 """
690 Standard function which generates help assuming they follow
691 the same design as *pyquickhelper*.
693 @param argv it should be ``sys.argv``
694 @param file_or_folder file ``setup.py`` or folder which contains it
695 @param project_var_name display name of the module
696 @param module_name module name, None if equal to *project_var_name*
697 (``import <module_name>``)
698 @param extra_ext extra file extension to process (ex ``["doc"]``)
699 @param add_htmlhelp run HTML Help too (only on Windows)
700 @param copy_add_ext additional extension of files to copy
701 @param nbformats notebooks format to generate
702 @param layout layout for the documentation, if None --> ``["html", "pdf"]``
703 @param use_run_cmd use function @see fn run_cmd instead of ``os.system``
704 to build the documentation
705 @param direct_call see @see fn generate_help_sphinx
706 @param fexclude function which tells which file not to copy in the folder
707 used to build the documentation
708 @param fLOG logging function
710 The function outputs some information through function @see fn fLOG.
712 A page will be added for each extra file extension mentioned in *extra_ext* if
713 some of these were found.
714 """
715 if fLOG is None: # pragma: no cover
716 from ..loghelper.flog import noLOG
717 fLOG = noLOG
718 if "--help" in argv: # pragma: no cover
719 from ..helpgen.help_usage import get_help_usage
720 print(get_help_usage())
721 else:
722 from ..helpgen.sphinx_main import generate_help_sphinx
724 if layout is None:
725 layout = ["html", "pdf"] # pragma: no cover
726 if module_name is None:
727 module_name = project_var_name
729 ffolder = get_folder(file_or_folder)
730 source = os.path.join(ffolder, "_doc", "sphinxdoc", "source")
732 if not os.path.exists(source):
733 raise FileNotFoundError( # pragma: no cover
734 "you must get the source from GitHub to build the documentation,\nfolder {0} "
735 "should exist\n(file_or_folder={1})\n(ffolder={2})\n(cwd={3})".format(
736 source, file_or_folder, ffolder, os.getcwd()))
738 if "conf" in sys.modules: # pragma: no cover
739 warnings.warn("module conf was imported, this function expects not to:\n{0}".format(
740 sys.modules["conf"].__file__))
741 del sys.modules["conf"]
743 project_name = os.path.split(
744 os.path.split(os.path.abspath(ffolder))[0])[-1]
746 generate_help_sphinx(project_name, module_name=module_name, layout=layout,
747 extra_ext=extra_ext, nbformats=nbformats, add_htmlhelp=add_htmlhelp,
748 copy_add_ext=copy_add_ext, fLOG=fLOG, root=ffolder,
749 direct_call=direct_call, fexclude=fexclude)
752def run_unittests_for_setup(file_or_folder, skip_function=None,
753 coverage_options=None, coverage_exclude_lines=None,
754 additional_ut_path=None, covtoken=None, stdout=None,
755 stderr=None, filter_warning=None, dump_coverage=None,
756 add_coverage_folder=None, coverage_root='src', fLOG=None):
757 """
758 Runs the unit tests and computes the coverage, stores
759 the results in ``_doc/sphinxdoc/source/coverage``
760 assuming the module follows the same design as *pyquickhelper*.
762 @param file_or_folder file ``setup.py`` or folder which contains it
763 @param skip_function see @see fn main_wrapper_tests
764 @param coverage_options see @see fn main_wrapper_tests
765 @param coverage_exclude_lines see @see fn main_wrapper_tests
766 @param additional_ut_path see @see fn main_wrapper_tests
767 @param covtoken see @see fn main_wrapper_tests
768 @param stdout see @see fn main_wrapper_tests
769 @param stderr see @see fn main_wrapper_tests
770 @param filter_warning see @see fn main_wrapper_tests
771 @param coverage_root see @see fn main_wrapper_tests
772 @param dump_coverage location where to dump the coverage
773 @param add_coverage_folder additional folder where to look for other coverage reports
774 @param fLOG logging function
776 See function @see fn main_wrapper_tests.
777 The coverage computation can be disabled by specifying
778 ``coverage_options["disable_coverage"] = True``.
779 *covtoken* was added to post the coverage report to
780 `codecov <https://codecov.io/>`_.
781 Parameter *dump_coverage*
782 dumps the unit test coverage in another location.
783 """
784 # delayed import
785 from .tkinter_helper import fix_tkinter_issues_virtualenv
786 from .utils_tests import main_wrapper_tests
787 ffolder = get_folder(file_or_folder)
788 funit = os.path.join(ffolder, "_unittests")
789 if not os.path.exists(funit):
790 raise FileNotFoundError( # pragma: no cover
791 "You must get the whole source to run the unittests,"
792 "\nfolder {0} should exist".format(funit))
794 if skip_function is None: # pragma: no cover
795 from .utils_tests_private import default_skip_function
796 skip_function = default_skip_function
797 if fLOG is None: # pragma: no cover
798 from ..loghelper.flog import noLOG
799 fLOG = noLOG
801 fix_tkinter_issues_virtualenv(fLOG=fLOG)
803 cov = True
804 if coverage_options:
805 if "disable_coverage" in coverage_options and coverage_options["disable_coverage"]:
806 cov = False
808 if dump_coverage is not None and not cov:
809 dump_coverage = None
811 logfile = os.path.join(funit, "unittests.out")
812 res = main_wrapper_tests(
813 logfile, add_coverage=cov, skip_function=skip_function,
814 coverage_options=coverage_options,
815 coverage_exclude_lines=coverage_exclude_lines,
816 additional_ut_path=additional_ut_path,
817 covtoken=covtoken, stdout=stdout, stderr=stderr,
818 filter_warning=filter_warning, dump_coverage=dump_coverage,
819 add_coverage_folder=add_coverage_folder,
820 coverage_root=coverage_root, fLOG=fLOG)
821 if not isinstance(res, dict):
822 raise TypeError( # pragma: no cover
823 f"Dictionary expected not {type(res)!r}.")
824 return res
827def copy27_for_setup(file_or_folder): # pragma: no cover
828 """
829 Prepares a copy of the source for :epkg:`Python` 2.7,
830 assuming the module follows the same design as *pyquickhelper*.
832 @param file_or_folder file ``setup.py`` or folder which contains it
833 """
834 # delayed import
835 from .py3to2 import py3to2_convert_tree
836 root = get_folder(file_or_folder)
837 root = os.path.normpath(root)
838 dest = os.path.join(root, "dist_module27")
839 py3to2_convert_tree(root, dest)
842def write_pyproj(file_or_folder, location=None):
843 """
844 Creates a project
845 `pyproj <https://docs.microsoft.com/fr-fr/visualstudio/python/managing-python-projects-in-visual-studio>`_
846 to work with `PTVS <https://pytools.codeplex.com/>`_
847 (Python Tools for Visual Studio)
849 @param file_or_folder file ``setup.py`` or folder which contains it
850 @param location if not None, stores the project into this folder
852 This functionality fails with :epkg:`Python` 2.7 (encoding).
853 """
854 # delayed import
855 from ..filehelper import explore_folder_iterfile
856 from .build_helper import get_pyproj_project
858 avoid = ["dist", "build", "dist_module27",
859 "_doc", "_virtualenv", "_virtualenv27", "_venv"]
861 def filter(name):
862 if os.path.splitext(name)[-1] != ".py":
863 return False
864 if "temp_" in name:
865 return False
866 if "temp2_" in name:
867 return False # pragma: no cover
868 for a in avoid:
869 if name.startswith(a + "\\"):
870 return False # pragma: no cover
871 if name.startswith(a + "/"):
872 return False
873 return True
875 root = get_folder(file_or_folder)
876 root = os.path.normpath(root)
877 name = os.path.split(root)[-1]
878 if location is None:
879 dest = os.path.join(root, "ptvs_project.pyproj") # pragma: no cover
880 else:
881 dest = os.path.join(location, "ptvs_project.pyproj")
882 all_files = [os.path.relpath(_, root)
883 for _ in explore_folder_iterfile(root)]
884 all_files = [_ for _ in all_files if filter(_)]
885 pyproj = get_pyproj_project(name, all_files)
886 with open(dest, "w", encoding="utf8") as f:
887 f.write(pyproj.strip())
890def process_standard_options_for_setup_help(argv):
891 """
892 Prints the added options available through this module.
893 """
894 commands = {
895 "build27": "build the wheel for Python 2.7 (if available), it requires to run copy27 first",
896 "build_script": "produce various scripts to build the module",
897 "build_sphinx": "build the documentation",
898 "build_wheel": "build the wheel",
899 "clean_space": "clean unnecessary spaces in the code, applies :epkg:`pycodestyle` on all files",
900 "clean_pyd": "clean file ``*.pyd``",
901 "copy27": "create a modified copy of the module to run on Python 2.7 (if available), it requires to run copy27 first",
902 "copy_dist": "copy documentation to folder dist",
903 "copy_sphinx": "modify and copy sources to _doc/sphinxdoc/source/<module>",
904 "history": "builds the history of the package in RST",
905 "local_jenkins": "sets up jobs on a local jenkins server",
906 "run27": "run the unit tests for the Python 2.7",
907 "run_pylint": "run pylint on the sources, allowed parameters <pattern> <neg_pattern>",
908 "unittests": "run the unit tests which do not contain test_LONG, test_SKIP or test_GUI in their file name",
909 "unittests_LONG": "run the unit tests which contain test_LONG their file name",
910 "unittests_SKIP": "run the unit tests which contain test_SKIP their file name",
911 "unittests_GUI": "run the unit tests which contain test_GUI their file name",
912 "write_version": "write a file ``version.txt`` with the version number (assuming sources are host with git)",
913 }
915 if "--help-commands" in argv:
916 print("Commands processed by pyquickhelper:")
917 for k, v in sorted(commands.items()):
918 print(f" {k}{' ' * (len('copy27 ') - len(k))}{v}")
919 print()
920 elif "--help" in argv:
921 docu = 0
922 for k, v in sorted(commands.items()):
923 if k in argv:
924 docu += 1
926 if docu == 0: # pragma: no cover
927 print("pyquickhelper commands:")
928 print()
929 for k in sorted(commands):
930 process_standard_options_for_setup_help(['--help', k])
931 print()
932 else:
933 for k, v in sorted(commands.items()):
934 if k in argv:
935 docu += 1
936 print(f" setup.py {k}{' ' * (20 - len(k))}{v}")
937 if k == "unittests":
938 print(
939 f"\n {k} [-d seconds] [-f file] [-e regex] [-g regex]\n\n {v}")
940 print(
941 " -d seconds run all unit tests for which predicted duration is below a given threshold.")
942 print(
943 " -f file run all unit tests in file (do not use the full path)")
944 print(
945 " -e regex run all unit tests files matching the regular expression")
946 print(
947 " -g regex run all unit tests files not matching the regular expression")
948 print()
949 elif k == "local_jenkins": # pragma: no cover
950 print()
951 print(
952 f" {k} user password [location] [server]")
953 print(" default location is somewhere/workspace")
954 print(" default server is http://localhost:8080/")
955 print()
958def write_module_scripts(folder, platform=sys.platform, blog_list=None,
959 default_engine_paths=None, command=None):
960 """
961 Writes a couple of scripts which allow a user to be faster on some tasks
962 or to easily get information about the module.
964 @param folder where to write the script
965 @param platform platform
966 @param blog_list blog list to follow, should be attribute ``__blog__`` of the module
967 @param command None to generate scripts for all commands or a value in *[blog, doc]*.
968 @param default_engine_paths default engines (or python distributions)
969 @return list of written scripts
971 The function produces the following files:
973 * *auto_rss_list.xml*: list of rss stream to follow
974 * *auto_rss_database.db3*: stores blog posts
975 * *auto_rss_server.py*: runs a server which updates the scripts and runs a server. It also open the default browser.
976 * *auto_rss_server.(bat|sh)*: run *auto_run_server.py*, the file on Linux might be missing if there is an equivalent python script
978 .. faqref::
979 :title: How to generate auto_rss_server.py?
981 The following code generates the script *auto_rss_local.py*
982 which runs a local server to read blog posts included
983 in the documentation (it uses module
984 `pyrsslocal <http://www.xavierdupre.fr/app/pyrsslocal/helpsphinx/index.html>`_)::
986 from pyquickhelper.pycode import write_module_scripts, __blog__
987 write_module_scripts(".", blog_list=__blog__, command="blog")
988 """
989 # delayed import
990 from .build_helper import get_script_module
991 default_set = {"blog", "doc"}
992 if command is not None:
993 if command not in default_set:
994 raise ValueError( # pragma: no cover
995 f"command {command} is not available in {default_set}")
996 commands = {command}
997 else:
998 commands = default_set
1000 res = []
1001 for c in commands:
1002 sc = get_script_module(
1003 c, platform=sys.platform, blog_list=blog_list, default_engine_paths=default_engine_paths)
1004 if sc is None:
1005 continue # pragma: no cover
1006 tobin = os.path.join(folder, "bin")
1007 if not os.path.exists(tobin):
1008 os.mkdir(tobin)
1009 for item in sc:
1010 if isinstance(item, tuple):
1011 name = os.path.join(folder, "bin", item[0])
1012 with open(name, "w", encoding="utf8") as f:
1013 f.write(item[1])
1014 res.append(name)
1015 else: # pragma: no cover
1016 name = os.path.join(
1017 folder, "bin", f"auto_run_{c}.{get_script_extension()}")
1018 with open(name, "w") as f:
1019 f.write(item)
1020 res.append(name)
1021 return res
1024def _get_dump_default_path(location, module_name, argv):
1025 """
1026 Proposes a default location to dump results about unit tests execution.
1028 @param location location of the module
1029 @param module_name module name
1030 @param argv argument on the command line
1031 @return location of the dump
1033 The result is None for remote continuous integration.
1034 """
1035 from . import is_travis_or_appveyor
1036 if is_travis_or_appveyor():
1037 return None # pragma: no cover
1038 hash = hash_list(argv)
1039 setup = os.path.join(location, "setup.py")
1040 if not os.path.exists(setup):
1041 raise FileNotFoundError(setup) # pragma: no cover
1042 fold = os.path.join(location, "..", "_coverage_dumps")
1043 if not os.path.exists(fold):
1044 os.mkdir(fold)
1045 dt = datetime.datetime.now().strftime("%Y%m%dT%H%M")
1046 if module_name is None:
1047 raise ValueError("module_name cannot be None") # pragma: no cover
1048 dump = os.path.join(fold, module_name, hash, dt)
1049 if not os.path.exists(dump):
1050 os.makedirs(dump)
1051 return dump
1054def hash_list(argv, size=8):
1055 """
1056 Proposes a hash for the list of arguments.
1058 @param argv list of arguments on the command line.
1059 @param size size of the hash
1060 @return string
1061 """
1062 st = "--".join(map(str, argv))
1063 hash = hashlib.md5()
1064 hash.update(st.encode("utf-8"))
1065 res = hash.hexdigest()
1066 if len(res) > 8:
1067 return res[:8]
1068 return res # pragma: no cover
1071def build_history_from_setup(dest, owner, module, existing_history=None,
1072 skip_issues=None, fLOG=None): # pragma: no cover
1073 """
1074 Builds the history from :epkg:`github` and :epkg:`pypi`.
1076 @param dest history will be written in this file
1077 @param owner owner of the package on :epkg:`github`
1078 @param module module name
1079 @param existing_history existing history, retrieves existing issues stored
1080 in that file
1081 @param skip_issues skip a given list of issues when building the history
1082 @param fLOG logging function
1083 @return history
1084 """
1085 # delayed import
1086 from ..loghelper.history_helper import build_history, compile_history
1087 if owner is None:
1088 raise ValueError( # pragma: no cover
1089 "owner must be specified.")
1090 if "/" in owner:
1091 raise ValueError( # pragma: no cover
1092 f"owner {owner!r} cannot contain '/'.")
1093 if fLOG is None: # pragma: no cover
1094 from ..loghelper.flog import noLOG
1095 fLOG = noLOG
1096 repo = module
1097 hist = build_history(owner, repo, unpublished=True,
1098 existing_history=existing_history,
1099 skip_issues=skip_issues, fLOG=fLOG)
1100 output = compile_history(hist)
1101 if dest is not None:
1102 with open(dest, "w", encoding="utf-8") as f:
1103 f.write(output)
1104 return output
1107def run_pylint_for_setup(folder, pattern=".*[.]py$", neg_pattern=None,
1108 verbose=False, pylint_ignore=None, fLOG=None):
1109 """
1110 Applies :epkg:`pylint` on subfolder *folder*.
1112 @param folder folder where to look
1113 @param pattern file to checks
1114 @param neg_pattern negative pattern
1115 @param pylint_ignore ignore these :epkg:`pylint` warnings or errors
1116 @param verbose verbose
1117 @param fLOG logging function
1118 """
1119 # delayed import
1120 from .utils_tests_helper import check_pep8
1121 if fLOG is None: # pragma: no cover
1122 from ..loghelper.flog import noLOG
1123 fLOG = noLOG
1124 check_pep8(folder, pattern=pattern, neg_pattern=neg_pattern,
1125 pylint_ignore=pylint_ignore, verbose=verbose, fLOG=fLOG)
1128def modifies_init_file(folder, version, module_name=None):
1129 """
1130 Automatically modifies the init file.
1132 @param folder where to find the init file
1133 @param version commit number
1134 @return modified init file
1135 """
1136 def _update_version(v, nv):
1137 vs = v.split('.')
1138 if len(vs) <= 2:
1139 return '.'.join(list(vs) + [nv])
1140 if len(vs) >= 3:
1141 vs = list(vs)
1142 vs[-1] = nv
1143 return '.'.join(vs)
1144 raise ValueError( # pragma: no cover
1145 f"Unable to process '{v}' with new version '{nv}'.")
1147 filename = None
1148 if os.path.exists(folder):
1149 if os.path.isdir(folder):
1150 src = os.path.join(folder, 'src')
1151 if module_name is None:
1152 setu = os.path.join(folder, 'setup.py')
1153 if not os.path.exists(setu):
1154 raise FileNotFoundError( # pragma: no cover
1155 "Unable to find 'setup.py' in '{}' and module_name is "
1156 "None.".format(folder))
1157 reg = re.compile(
1158 "(project_var_name = ['\\\"]([a-zA-Z][a-zA-Z_0-9]+)['\\\"])")
1159 with open(setu, 'r', encoding='utf-8') as f:
1160 cst = f.read()
1161 find = reg.findall(cst)
1162 if len(find) == 0:
1163 raise FileNotFoundError( # pragma: no cover
1164 "Unable to find 'project_var_name' in 'setup.py' in '{}' "
1165 "and module_name is None.".format(folder))
1166 module_name = find[0][1]
1167 if os.path.exists(src) and module_name is not None:
1168 filename = os.path.join(src, module_name, '__init__.py')
1169 elif os.path.exists(src) and module_name is not None:
1170 filename = os.path.join(src, module_name, '__init__.py')
1171 elif module_name is not None:
1172 filename = os.path.join(folder, module_name, '__init__.py')
1173 else:
1174 raise FileNotFoundError( # pragma: no cover
1175 f"Unable to find '__init__.py' in '{folder}' (module_name is None).")
1176 if not os.path.exists(filename):
1177 raise FileNotFoundError( # pragma: no cover
1178 f"Unable to find '__init__.py' in '{folder}' (got '{filename}').")
1179 with open(filename, 'r', encoding='utf-8') as f:
1180 content = f.read()
1181 elif '__version__' in folder:
1182 content = folder
1183 else:
1184 raise ValueError(f"Unable to process '{folder}'.")
1186 reg = re.compile("(__version__ = ['\\\"]([0-9.]+)['\\\"])")
1187 lines = content.split('\n')
1188 modif = []
1189 rep = []
1190 for line in lines:
1191 if line.startswith("__version__"):
1192 find = reg.findall(line)
1193 if len(find) != 1:
1194 raise ValueError( # pragma: no cover
1195 f"Unable to find __version__ in '{line}'")
1196 v = find[0][1]
1197 nv = _update_version(v, str(version))
1198 newline = line.replace(v, nv)
1199 modif.append(newline)
1200 rep.append((line, newline))
1201 else:
1202 modif.append(line)
1203 if len(rep) == 0:
1204 raise ValueError( # pragma: no cover
1205 f"Unable to find '__version__' in \n{content}")
1207 content = '\n'.join(modif)
1208 if filename is not None:
1209 with open(filename, 'w', encoding='utf-8') as f:
1210 f.write(content)
1211 return content