Coverage for src/pymyinstall/win_installer/win_setup_main_helper.py : 12%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Helpers for function @see fn win_python_setup
4"""
5from __future__ import print_function
6import datetime
7import os
8import shutil
9import sys
10import fnmatch
11import subprocess
13from ..installcustom.install_custom_pandoc import install_pandoc
14from ..installcustom.install_custom_R import install_R
15from ..installcustom.install_custom_julia import install_julia
16from ..installcustom.install_custom_scite import install_scite
17from ..installcustom.install_custom_putty import install_putty
18from ..installcustom.install_custom_sqlitespy import install_sqlitespy
19from ..installcustom.install_custom_python import install_python
20from ..installcustom.install_custom_mingw import install_mingw
21from ..installcustom.install_custom_tdm_gcc import install_tdm_gcc
22from ..installcustom.install_custom_7z import install_7z
23from ..installcustom.install_custom_graphviz import install_graphviz
24from ..installcustom.install_custom_vs import install_vs
25from ..installcustom.install_custom import download_page
26from ..installcustom.install_custom_javajdk import install_javajdk
27from ..installcustom.install_custom_jenkins import install_jenkins
28from ..installcustom.install_custom_git import install_git
29from ..installcustom.install_custom_miktex import install_miktex
30from ..installcustom.install_custom_inkscape import install_inkscape
31from ..installhelper.link_shortcuts import add_shortcut
32from ..installhelper import run_cmd
33from ..packaged import minimal_set
35from .win_packages import _is_package_in_list
36from .pywin32_helper import import_pywin32
37from .win_extract import extract_msi, extract_exe, extract_archive, extract_copy
38from .win_exception import WinInstallException
39from .win_setup_mark_step import mark_step, is_step_done
40from .win_setup_r import r_run_script, _script as _script_r
41from .win_setup_julia import julia_run_script, _script_install as _script_julia_install
42from .win_setup_julia import _script_build as _script_julia_build, _script_init as _script_julia_init
45if sys.version_info[0] == 2:
46 from codecs import open
47 FileNotFoundError = Exception
50def dtnow():
51 """
52 shortcut, return ``datetime.datetime.now()``
53 """
54 return datetime.datetime.now()
57def copy_icons(src, dest):
58 """
59 copy all files from src to dest
61 @param src source
62 @param dest destination
63 @return operations
64 """
65 operations = []
66 if not os.path.exists(dest):
67 os.makedirs(dest)
68 operations.append(("mkdir", dest))
69 files = os.listdir(src)
70 for file in files:
71 if os.path.isdir(os.path.join(src, file)):
72 continue
73 d = os.path.join(dest, file)
74 if not os.path.exists(d):
75 shutil.copy(os.path.join(src, file), dest)
76 operations.append(("copy", file))
77 return operations
80def win_download(folder=None, module_list=None, verbose=False, fLOG=print,
81 download_only=True, selection=None, source=None, embed=True):
82 """
83 The function downloads everything needed to prepare a setup.
85 @param folder where to prepare the python version (the user must replace None)
86 @param module_list list of module to install (see @see fn minimal_set = default options)
87 @param fLOG logging function
88 @param download_only only downloads (unused, True by default)
89 @param verbose print more information
90 @param selection selection of tools to install (dictionary { toolname: version or None})
91 @param source source of python packages (see @see cl ModuleInstall)
92 @param embed use embedded version (or custom one)
93 @return list of completed operations
95 List of available tools:
97 * scite
98 * putty
99 * mingw
100 * SQLiteSpy
101 * python
102 * R
103 * vs
104 * julia
105 * 7z
106 * graphviz
107 * tdm
108 * jdk (java)
109 * jenkins
110 * git
111 * miktex
112 * inkscape
113 """
114 if selection is None:
115 raise ValueError("selection must be specified")
117 available = os.listdir(folder)
119 def is_here(program, no_wheel=False):
120 b = _is_package_in_list(program, available, no_wheel=no_wheel)
121 return b
123 operations = []
125 if not is_here("scite") and "scite" in selection:
126 if verbose:
127 fLOG("[pymy] --- download", "scite")
128 r = install_scite(dest_folder=folder, fLOG=fLOG,
129 install=False, version=selection.get("scite", None))
130 operations.append(("download", r))
131 fLOG("[pymy] done")
133 if not is_here("putty") and "putty" in selection:
134 if verbose:
135 fLOG("[pymy] --- download", "putty")
136 r = install_putty(dest_folder=folder, fLOG=fLOG,
137 install=False, version=selection.get("putty", None))
138 operations.append(("download", r))
139 fLOG("[pymy] done")
141 if not is_here("mingw") and "mingw" in selection:
142 if verbose:
143 fLOG("[pymy] --- download", "mingw")
144 r = install_mingw(dest_folder=folder, fLOG=fLOG,
145 install=False, version=selection.get("mingw", None))
146 operations.append(("download", r))
147 fLOG("[pymy] done")
149 if not is_here("miktex") and "miktex" in selection:
150 if verbose:
151 fLOG("[pymy] --- download", "miktex")
152 r = install_miktex(dest_folder=folder, fLOG=fLOG,
153 install=False, version=selection.get("miktex", None))
154 operations.append(("download", r))
155 fLOG("[pymy] done")
157 if not is_here("inkscape") and "inkscape" in selection:
158 if verbose:
159 fLOG("[pymy] --- download", "inkscape")
160 r = install_inkscape(dest_folder=folder, fLOG=fLOG,
161 install=False, version=selection.get("inkscape", None))
162 operations.append(("download", r))
163 fLOG("[pymy] done")
165 if not is_here("tdm") and "tdm" in selection:
166 if verbose:
167 fLOG("[pymy] --- download", "tdm")
168 r = install_tdm_gcc(dest_folder=folder, fLOG=fLOG,
169 install=False, version=selection.get("tdm", None))
170 operations.append(("download", r))
171 fLOG("[pymy] done")
173 if not is_here("git") and "git" in selection:
174 if verbose:
175 fLOG("[pymy] --- download", "git")
176 r = install_git(folder, fLOG=fLOG,
177 install=False, version=selection.get("git", None))
178 operations.append(("download", r))
179 fLOG("[pymy] done")
181 if not is_here("SQLiteSpy") and "sqlitespy" in selection:
182 if verbose:
183 fLOG("[pymy] --- download", "sqllitespy")
184 r = install_sqlitespy(temp_folder=folder, fLOG=fLOG,
185 install=False, version=selection.get("sqlitespy", None))
186 operations.append(("download", r))
187 fLOG("[pymy] done")
189 if not is_here("python") and "python" in selection:
190 if verbose:
191 fLOG("[pymy] --- download", "python")
192 r = install_python(
193 temp_folder=folder, fLOG=fLOG, install=False, force_download=True,
194 version=selection.get("python", None), custom=not embed)
195 operations.append(("download", r))
196 fLOG("[pymy] done")
198 if not is_here("R-") and "r" in selection:
199 if verbose:
200 fLOG("[pymy] --- download", "R")
201 r = install_R(
202 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("r", None))
203 operations.append(("download", r))
204 fLOG("[pymy] done")
206 if not is_here("vs") and "vs" in selection:
207 if verbose:
208 fLOG("[pymy] --- download", "Visual Studio Express")
209 r = install_vs(folder, fLOG=fLOG, install=False,
210 version=selection.get("vs", None))
211 operations.append(("download", r))
212 fLOG("[pymy] done")
214 if not is_here("julia") and "julia" in selection:
215 if verbose:
216 fLOG("[pymy] --- download", "julia")
217 r = install_julia(
218 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("julia", None))
219 operations.append(("download", r))
220 fLOG("[pymy] done")
222 if not is_here("pandoc") and "pandoc" in selection:
223 if verbose:
224 fLOG("[pymy] --- download", "pandoc")
225 r = install_pandoc(
226 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("pandoc", None))
227 operations.append(("download", r))
228 fLOG("[pymy] done")
230 if not is_here("7z") and "7z" in selection:
231 if verbose:
232 fLOG("[pymy] --- download", "7z")
233 r = install_7z(
234 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("7z", None))
235 operations.append(("download", r))
236 fLOG("done")
238 if not is_here("graphviz", no_wheel=True) and "graphviz" in selection:
239 if verbose:
240 fLOG("[pymy] --- download", "graphviz")
241 r = install_graphviz(
242 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("graphviz", None))
243 operations.append(("download", r))
244 fLOG("[pymy] done")
246 if not is_here("jdk", no_wheel=True) and "jdk" in selection:
247 if verbose:
248 fLOG("[pymy] --- download", "java jdk")
249 r = install_javajdk(
250 temp_folder=folder, fLOG=fLOG, install=False, force_download=True, version=selection.get("java jdk", None))
251 operations.append(("download", r))
252 fLOG("[pymy] done")
254 if not is_here("jenkins", no_wheel=True) and "jenkins" in selection:
255 if verbose:
256 fLOG("[pymy] --- download", "jenkins")
257 r = install_jenkins(folder, fLOG=fLOG, install=False,
258 version=selection.get("jenkins", None))
259 operations.append(("download", r))
260 fLOG("[pymy] done")
262 if module_list is None:
263 module_list = minimal_set()
265 for mod in module_list:
266 if is_here(mod.name + "-"):
267 continue
268 if verbose:
269 fLOG("[pymy] download module", mod.name)
270 res = mod.download(temp_folder=folder, source=source)
271 if isinstance(res, list):
272 for r in res:
273 operations.append(("download", r))
274 else:
275 operations.append(("download", res))
277 return operations
280def win_install(folders, download_folder, verbose=False, fLOG=print,
281 names="Julia Scite 7z TDM MinGW R pandoc Python SQLiteSpy Putty Graphviz".split(),
282 selection=None):
283 """
284 Install setups
286 @param folders dictionary of folders, must contain key tools, python
287 @param download_folder where the setup are
288 @param fLOG logging function
289 @param verbose print more information
290 @param names name of subfolders to be created
291 @param selection selection of tools to install
292 @return list of completed operations, executable (to make shortcuts)
294 The function installs every setup which starts by one of the string in *names*
295 and whose extension is .exe, .msi, .zip, .7z.
297 To install Python on Windows,
298 see `Using Python on Windows <https://docs.python.org/3.5/using/windows.html>`_,
299 and `What's coming for the Python 3.5 installer? <http://stevedower.id.au/blog/the-python-3-5-installer/>`_
300 """
301 operations = []
302 dfunc = {".zip": extract_archive, ".exe": extract_exe,
303 ".msi": extract_msi, "putty.exe": extract_copy,
304 ".7z": extract_archive}
306 def location(file):
307 ext = os.path.splitext(file)[-1]
308 if ext not in [".msi", ".exe", ".zip", ".7z"]:
309 return None
310 lf = file.lower()
311 for name in names:
312 lname = name.lower()
313 if name.lower() in selection and (lf.startswith(lname) or "-%s-" % lname in lf):
314 if name == "Python":
315 return folders["python"]
316 else:
317 return os.path.join(folders["tools"], name)
318 return None
320 def find_exe(loc, name):
321 name = name.lower()
322 if name.lower() == "mingw":
323 name = "gcc"
324 elif name.lower() == "tdm":
325 name = "gcc"
326 elif name.lower() == "jdk":
327 name = "java"
328 elif name.lower() == "graphviz":
329 name = "dot"
330 exp = name + ".exe"
332 for root, dirnames, filenames in os.walk(loc):
333 for filename in fnmatch.filter(filenames, '*.exe'):
334 exe = os.path.join(root, filename)
335 file = os.path.split(exe)[-1].lower()
336 if file == exp:
337 return exe
338 return None
340 # we sort to get 7z installed first as it is needed for exe files
341 cands = os.listdir(download_folder)
342 cands.sort()
344 installed = {}
346 for cand in cands:
347 loc = location(cand)
348 if loc is None:
349 continue
351 name = os.path.split(loc)[-1]
352 if not os.path.exists(loc):
353 os.mkdir(loc)
355 # already installed, checking if there is any exe file
356 exe = find_exe(loc, name)
357 if exe is not None:
358 # already done
359 fLOG("[pymy] --- already installed", exe)
360 else:
361 fLOG("[pymy] --- install", cand, " in ", loc)
362 full = os.path.join(download_folder, cand)
363 if 'tdm' in cand and 'gcc' in cand:
364 raise WinInstallException(
365 "TM must be manually installed from the setup\n{0}\nin\n{1}".format(full, loc))
366 elif 'python' in cand:
367 continue
368 # see http://stevedower.id.au/blog/the-python-3-5-installer/
369 temploc = os.path.join(download_folder, "python")
370 options = [os.path.join(
371 download_folder, cand), "/quiet", "/layout", temploc]
372 cmd = " ".join(options)
373 fLOG("[pymy] run ", cmd)
374 out, err = run_cmd(cmd, wait=True)
375 fLOG("[pymy] OUT:\n", out)
376 if err:
377 fLOG("[pymy] OUT:\n", err)
378 options = [os.path.join(temploc, cand),
379 "TargetDir={0}".format(loc), "InstallAllUsers=1",
380 "AssociateFiles=0", "CompileAll=1", "Include_symbols=1",
381 "SimpleInstall=1"]
382 cmd = " ".join(options)
383 fLOG("[pymy] run ", cmd)
384 out, err = run_cmd(cmd, wait=True)
385 fLOG("[pymy] OUT:\n", out)
386 if err:
387 fLOG("[pymy] OUT:\n", err)
388 else:
389 ext = os.path.splitext(cand)[-1]
390 filename = os.path.split(cand)[-1]
391 func = dfunc.get(filename, dfunc[ext])
393 if ext == ".exe":
394 func(
395 full, loc, verbose=verbose, fLOG=fLOG, szip=installed["7z"])
396 else:
397 func(full, loc, verbose=verbose, fLOG=fLOG)
399 operations.append(("install", cand))
400 fLOG("[pymy] done")
402 # add main executable
403 found = find_exe(loc, name)
405 # for MinGW, we check that the executable mingw-get.exe was installed
406 if found is None:
407 if name == "MinGW" and cand == "mingw-get-setup.exe":
408 exe = os.path.join(loc, "bin", "mingw-get.exe")
409 if os.path.exists(exe):
410 cmd = exe + \
411 " install binutils gcc g++ mingw32 fortran gdb mingw32 mingw w32api g77"
412 if verbose:
413 fLOG("[pymy] install MinGW", cmd)
414 retcode = subprocess.call(
415 cmd, shell=True, stdout=sys.stderr)
416 if retcode < 0:
417 raise WinInstallException(
418 "unable to execute:\nCMD:\n{0}".format(cmd))
419 found = find_exe(loc, name)
420 elif name == "VS" and cand == "vs_community.exe":
421 # Visual Studio needs to be manually installed
422 # we copy the setup to VS
423 shutil.copy(os.path.join(download_folder, cand), loc)
424 found = os.path.join(loc, cand)
426 if found is None:
427 raise FileNotFoundError("unable to find executable for name={0} in {1}, found: {2}, exe={3}, function={4}".format(
428 name, loc, found, exe, func))
429 installed[name] = found
431 return operations, installed
434def create_links_tools(folder, installed, verbose=False, fLOG=print):
435 """
436 create links for the tools
438 @param folder where links will be stored
439 @param installed dictionary *{ tool: exe file }*
440 @param verbose display more information
441 @param fLOG logging function
442 @return operations, list of tuple *("link", link file)*
443 """
444 import_pywin32()
445 operations = []
447 for k, v in installed.items():
448 if k == "R":
449 name = "test R Gui"
450 link_name = name + ".lnk"
451 dest = os.path.join(folder, link_name)
452 add_shortcut(target="tools\\R\\bin\\x64\\Rgui.exe",
453 name=name, arguments="", icon="~dp0\\tools\\icons\\r.ico",
454 folder=folder)
455 if verbose:
456 fLOG("[pymy] create link", dest)
457 operations.append(("link", link_name))
459 name = "test R Console"
460 link_name = name + ".lnk"
461 dest = os.path.join(folder, link_name)
462 add_shortcut(target="tools\\R\\bin\\x64\\R.exe",
463 name=name, arguments="", icon="~dp0\\tools\\icons\\r.ico",
464 folder=folder)
465 if verbose:
466 fLOG("[pymy] create link", dest)
467 operations.append(("link", link_name))
469 elif k == "julia":
470 name = "test Julia Console"
471 link_name = name + ".lnk"
472 dest = os.path.join(folder, link_name)
473 add_shortcut(target="tools\\julia\\bin\\julia.exe",
474 name=name, arguments="", icon="~dp0\\tools\\icons\\julia.ico",
475 folder=folder)
476 if verbose:
477 fLOG("[pymy] create link", dest)
478 operations.append(("link", link_name))
480 elif k == "python":
481 name = "test Python Console"
482 link_name = name + ".lnk"
483 dest = os.path.join(folder, link_name)
484 add_shortcut(target="python\\python.exe",
485 name=name, arguments="", icon="~dp0\\tools\\icons\\python.ico",
486 folder=folder)
487 if verbose:
488 fLOG("[pymy] create link", dest)
489 operations.append(("link", link_name))
491 return operations
494def win_download_notebooks(notebooks, folder, verbose=False, fLOG=print):
495 """
496 download notebooks and store them as documentation
498 @param notebooks list of tuple (place, url)
499 @param folder where to put them
500 @param verbose verbose
501 @param fLOG logging function
502 @return list of operations
503 """
504 operations = []
505 for path, urls in notebooks:
506 if not isinstance(urls, list):
507 urls = [urls]
508 for url in urls:
509 name = url.split("/")[-1]
510 dest = os.path.join(folder, path)
511 if not os.path.exists(dest):
512 os.makedirs(dest)
513 operations.append(("mkdir", dest))
514 dfile = os.path.join(dest, name)
515 if not os.path.exists(dfile):
516 if verbose:
517 fLOG("download notebooks", name, "from", url)
518 content = download_page(url)
519 with open(dfile, "w", encoding="utf8") as f:
520 f.write(content)
521 operations.append(("docs", dfile))
522 return operations
525def win_install_julia_step(folders, verbose=False, fLOG=print):
526 """
527 does necessary steps to setup Julia
529 @param folders installation folders
530 @param verbose verbose
531 @param fLOG logging function
532 @return list of processed operations
533 """
534 operations = []
535 if not is_step_done(folders["logs"], "julia_init"):
536 ##########################
537 # init Julia packages
538 #########################
539 fLOG("[pymy] --- init julia packages")
540 jl = os.path.join(folders["tools"], "Julia")
541 output = os.path.join(folders["logs"], "out.init.julia.txt")
542 out = julia_run_script(
543 jl, folders["python"], _script_julia_init, verbose=verbose, fLOG=fLOG)
544 with open(output, "w", encoding="utf8") as f:
545 f.write(out)
546 operations.append(("Julia", _script_julia_init))
547 operations.append(("time", dtnow()))
548 mark_step(folders["logs"], "julia_init")
550 if not is_step_done(folders["logs"], "julia_install"):
551 ##########################
552 # install Julia packages
553 #########################
554 fLOG("[pymy] --- install julia packages")
555 jl = os.path.join(folders["tools"], "Julia")
556 output = os.path.join(folders["logs"], "out.install.julia.txt")
557 out = julia_run_script(
558 jl, folders["python"], _script_julia_install, verbose=verbose, fLOG=fLOG)
559 with open(output, "w", encoding="utf8") as f:
560 f.write(out)
561 operations.append(("Julia", _script_julia_install))
562 operations.append(("time", dtnow()))
563 mark_step(folders["logs"], "julia_install")
565 if not is_step_done(folders["logs"], "julia_build"):
566 #########################
567 # build Julia packages
568 #########################
569 fLOG("[pymy] --- build julia packages")
570 jl = os.path.join(folders["tools"], "Julia")
571 output = os.path.join(folders["logs"], "out.build.julia.txt")
572 out = julia_run_script(
573 jl, folders["python"], _script_julia_build, verbose=verbose, fLOG=fLOG)
574 with open(output, "w", encoding="utf8") as f:
575 f.write(out)
576 operations.append(("Julia", _script_julia_build))
577 operations.append(("time", dtnow()))
578 mark_step(folders["logs"], "julia_build")
580 return operations
583def win_install_r_step(folders, verbose=False, fLOG=print):
584 """
585 does necessary steps to setup R
587 @param folders installation folders
588 @param verbose verbose
589 @param fLOG logging function
590 @return list of processed operations
591 """
592 operations = []
593 if not is_step_done(folders["logs"], "r_install"):
594 ######################
595 # install R packages
596 ######################
597 fLOG("[pymy] --- install R packages")
598 r = os.path.join(folders["tools"], "R")
599 output = os.path.join(folders["logs"], "out.install.r.txt")
600 out = r_run_script(r, _script_r, output)
601 with open(output, "w", encoding="utf8") as f:
602 f.write(out)
603 operations.append(("R", _script_r))
604 operations.append(("time", dtnow()))
605 mark_step(folders["logs"], "r_install")
606 else:
607 fLOG(
608 "--- skip installation of R packages or remove file log.step.r_install.txt")
610 return operations