Coverage for src/pymyinstall/installcustom/install_custom_python.py : 55%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Various functions to install `python <http://www.python.org/>`_.
5"""
6from __future__ import print_function
7import sys
8import os
9import datetime
11from ..installhelper.install_cmd_helper import run_cmd, unzip_files
12from .install_custom import download_page, download_file
14if sys.version_info[0] == 2:
15 FileNotFoundError = Exception
18def unzip7_files(filename_7z, fLOG=print, dest="."):
19 """
20 If `7z <http://www.7-zip.org/>`_ is installed, the function uses it
21 to uncompress file into *7z* format. The file *filename_7z* must not exist.
23 .. index:: 7zip, 7z
25 :param filename_7z: final destination
26 :param fLOG: logging function
27 :param dest: destination folder
29 :return: output of 7z
31 .. versionadded:: 1.1
32 """
33 if sys.platform.startswith("win"):
34 exe = r"C:\Program Files\7-Zip\7z.exe"
35 if not os.path.exists(exe):
36 raise FileNotFoundError("unable to find: {0}".format(exe))
37 else:
38 exe = "7z"
40 if not os.path.exists(filename_7z):
41 raise FileNotFoundError(filename_7z)
43 cmd = '"{0}"-y -o"{2}" x "{1}"'.format(exe, filename_7z, dest)
44 out, err = run_cmd(cmd, wait=True)
46 if err is not None and len(err) > 0:
47 raise Exception("OUT:\n{0}\nERR-A:\n{1}".format(out, err))
49 return out
52def fix_fcntl_windows(path):
53 """
54 Adds a file `fnctl.py` on :epkg:`Windows`
55 (only available on :epkg:`Linux`).
57 @param path path to the python installation
58 """
59 if not sys.platform.startswith("win"):
60 raise Exception("fcntl should only be added on Windows.")
61 dest = os.path.join(path, "Lib", "fcntl.py")
62 if os.path.exists(dest):
63 # already done
64 return
65 module = """
66 def fcntl(fd, op, arg=0):
67 return 0
68 def ioctl(fd, op, arg=0, mutable_flag=True):
69 if mutable_flag:
70 return 0
71 else:
72 return ""
73 def flock(fd, op):
74 return
75 def lockf(fd, operation, length=0, start=0, whence=0):
76 return
77 """.replace(" ", "")
78 with open(dest, "w") as f:
79 f.write(module)
82def fix_termios_windows(path):
83 """
84 Adds a file `termios.py` on :epkg:`Windows`
85 (only available on :epkg:`Linux`).
87 @param path path to the python installation
88 """
89 if not sys.platform.startswith("win"):
90 raise Exception("fcntl should only be added on Windows.")
91 dest = os.path.join(path, "Lib", "termios.py")
92 if os.path.exists(dest):
93 # already done
94 return
95 module = """
96 TCSAFLUSH = 1
97 """.replace(" ", "")
98 with open(dest, "w") as f:
99 f.write(module)
102def fix_resource_windows(path):
103 """
104 Adds a file `resource.py` on :epkg:`Windows`
105 (only available on :epkg:`Linux`).
107 @param path path to the python installation
108 """
109 if not sys.platform.startswith("win"):
110 raise Exception("fcntl should only be added on Windows.")
111 dest = os.path.join(path, "Lib", "resource.py")
112 if os.path.exists(dest):
113 # already done
114 return
115 module = """
116 """.replace(" ", "")
117 with open(dest, "w") as f:
118 f.write(module)
121def _clean_err1(err):
122 if err:
123 lines = []
124 for line in err.split("\n"):
125 if "find: ‘build’: No such file or directory" in line:
126 continue
127 if "(ignored)" in line:
128 continue
129 if "Task was destroyed but it is pending!" in line:
130 continue
131 if "[libinstall] Error 1 (ignored)" in line:
132 continue
133 if "task: <Task finished coro=<<async_generator_athrow without __name__>()" in line:
134 continue
135 if "stty: 'standard input': Inappropriate ioctl for device" in line:
136 continue
137 if "task: <Task pending coro=<<async_generator_athrow without __name__>()>>" in line:
138 continue
139 if "unhandled exception during asyncio.run() shutdown" in line:
140 continue
141 if "RuntimeError: can't send non-None value to a just-started coroutine" in line:
142 continue
143 if " which is not installed." in line:
144 continue
145 lines.append(line)
146 err = "\n".join(lines).strip() if lines else None
147 errl = err.lower()
148 if 'error' not in errl and 'exception' not in errl:
149 lines = []
150 for line in err.split("\n"):
151 if line.startswith(' '):
152 continue
153 if 'note: declared here' in line:
154 continue
155 if "In file included" in line:
156 continue
157 if "warning:" in line:
158 continue
159 if "In function " in line:
160 continue
161 lines.append(line)
162 err = "\n".join(lines).strip() if lines else None
163 return err
166def _clean_err0(err):
167 # remove a couple of warnings.
168 lines = err.split("\n")
169 lines2 = [
170 _ for _ in lines if "UserWarning: Module pymyinstall was already imported" not in _]
171 if len(lines2) < len(lines):
172 lines2 = [
173 _ for _ in lines2 if "from pip._vendor import pkg_resources" not in _]
174 return "\n".join(lines2)
177def install_python(temp_folder=".", fLOG=print, install=True, force_download=False, # pylint: disable=R0914
178 version=None, modules=None, custom=False, latest=False,
179 download_folder="download", verbose=False, make_first=False):
180 """
181 Installs :epkg:`python`.
182 It does not do it a second time if it is already installed.
184 @param temp_folder where to download the setup
185 @param fLOG logging function
186 @param install install (otherwise only download)
187 @param force_download force the downloading of python
188 @param version version to download (by default the current version of Python)
189 @param modules modules to install
190 @param custom the standalone distribution has issue when installing new packages,
191 custom is True means switching to a zip of the standard distribution,
192 see below
193 @param latest install this version of pymyinstall and not the pypi version
194 @param download_folder download folder for packages
195 @param verbose more display
196 @param make_first run *make* before *make altinstall*
197 @return temporary file
199 The version is fixed to the current version of Python and amd64.
200 The standalone distribution has an issue and raises an error for some
201 packages such as `smart_open <https://pypi.python.org/pypi/smart_open>`_:
203 ::
205 error: [Errno 2] No such file or directory: '<python>\\python36.zip\\lib2to3\\Grammar.txt'
207 In that case, you should consider using ``custom=True``.
208 The function work for :epkg:`Linux` too.
209 List of steps done in linux:
211 ::
213 mkdir install_folder
214 cd install_folder
215 curl -O https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz
216 tar xzf Python-3.7.2.tgz
217 mkdir dist372
218 cd Python-3.7.2/
219 # current folder is /home/dupre/temp/temp_py/dist372/
220 ./configure --enable-optimizations --with-ensurepip=install --prefix=/home/dupre/temp/temp_py/dist372/inst --exec-prefix=/home/dupre/temp/temp_py/dist372/bin --datadir=/home/dupre/temp/temp_py/dist372/data
221 """
222 cmds = []
224 if version is None:
225 version = "%s.%s.%s" % sys.version_info[:3]
226 versioni = tuple(int(_)
227 for _ in version.split(".")) # pylint: disable=R1728
228 link = "https://www.python.org/downloads/release/python-%s/" % version.replace(
229 ".", "")
230 page = download_page(link)
231 if page is None:
232 raise ValueError("page is None for link '{0}'".format(link))
234 if sys.platform.startswith("win"):
235 if versioni[:2] <= (3, 4):
236 raise NotImplementedError(
237 "Python <= 3.4 is not supported anymore.")
238 # The setup for Python 3.5 does not accept multiple versions,
239 # it was installed on one machine and then compressed into a 7z
240 # file
241 if versioni >= (3, 7, 0):
242 if custom:
243 if versioni > (3, 7, 0):
244 raise ValueError(
245 "Not custom zip available for Python {0}".format(versioni))
246 url = "http://www.xavierdupre.fr/enseignement/setup/Python{0}{1}-{0}.{1}.{2}-amd64.zip".format(
247 *versioni[:3])
248 else:
249 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/python-{0}.{1}.{2}-embed-amd64.zip".format(
250 *versioni[:3])
251 elif versioni >= (3, 6, 0):
252 if custom:
253 if versioni > (3, 6, 5):
254 raise ValueError(
255 "Not custom zip available for Python {0}".format(versioni))
256 url = "http://www.xavierdupre.fr/enseignement/setup/Python{0}{1}-{0}.{1}.{2}-amd64.zip".format(
257 *versioni[:3])
258 else:
259 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/python-{0}.{1}.{2}-embed-amd64.zip".format(
260 *versioni[:3])
261 elif versioni >= (3, 5, 0):
262 if custom:
263 if versioni not in [(3, 5, 3), (3, 5, 2)]:
264 raise ValueError(
265 "Not custom zip available for Python {0}".format(versioni))
266 url = "http://www.xavierdupre.fr/enseignement/setup/Python35-3.5.3-amd64.zip"
267 else:
268 url = "https://www.python.org/ftp/python/3.5.3/python-3.5.3-embed-amd64.zip"
269 else:
270 raise Exception(
271 "Unable to find a proper version for version {0}".format(version))
272 else:
273 url = "https://www.python.org/ftp/python/{0}.{1}.{2}/Python-{0}.{1}.{2}.tgz".format(
274 *versioni)
276 full = url.split("/")[-1]
277 outfile = os.path.join(temp_folder, full)
278 fLOG("[install_python] download", url)
279 local = download_file(url, outfile, fLOG=fLOG)
281 # Install
282 if install:
283 # unzip files
284 if sys.platform.startswith("win"):
285 unzip_files(local, temp_folder, fLOG=fLOG)
286 else:
287 cmd = "tar xzf {0}".format(outfile)
288 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
289 change_path=temp_folder)
290 cmds.append(cmd)
291 if err:
292 raise RuntimeError(
293 "Issue with running '{0}'\n--OUT--\n{1}\n--ERR--\n{2}\n--IN--\n{3}\n--CMDS--\n{4}".format(
294 cmd, out, err, temp_folder, "\n".join(cmds)))
295 pyinstall = os.path.join(
296 temp_folder, "Python-{0}.{1}.{2}".format(*versioni))
298 cmd = "./configure --enable-optimizations --with-ensurepip=install --prefix={0}/inst --exec-prefix={0}/bin --datadir={0}/data"
299 cmd = cmd.format(temp_folder)
300 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
301 change_path=pyinstall)
302 cmds.append(cmd)
303 if err:
304 lines = []
305 for line in err.split("\n"):
306 if "[libinstall] Error 1 (ignored)" in line:
307 continue
308 lines.append(line)
309 err = "\n".join(lines).strip() if lines else None
310 if err:
311 raise RuntimeError(
312 "Issue with running '{0}'\n--OUT--\n{1}\n--ERR--\n{2}\n--CMDS--\n{3}".format(
313 cmd, out, err, "\n".join(cmds)))
315 # See https://stackoverflow.com/questions/44708262/make-install-from-source-python-without-running-tests.
316 os.environ["EXTRATESTOPTS"] = "--list-tests"
318 if make_first:
319 cmd = "make"
320 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
321 change_path=pyinstall)
322 cmds.append(cmd)
323 err = _clean_err1(err)
324 if err:
325 raise RuntimeError(
326 "Issue while running '{0}'\n---URL---\n{1}\n---OUT---\n{2}\n"
327 "---ERR---?1-\n{3}\n---IN---\n{4}\n---CMDS---\n{5}".format(
328 cmd, url, out, err, pyinstall, "\n".join(cmds)))
330 cmd = "make altinstall"
331 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
332 change_path=pyinstall)
333 cmds.append(cmd)
334 err = _clean_err1(err)
335 if err:
336 lines = []
337 for line in err.split("\n"):
338 if "[libinstall] Error 1 (ignored)" in line:
339 continue
340 if ' which is not on PATH.' in line:
341 continue
342 lines.append(line)
343 err = "\n".join(lines).strip() if lines else None
344 if err:
345 raise RuntimeError(
346 "Issue while running '{0}'\n---URL---\n{1}\n---OUT---\n{2}\n---ERR---?2-\n{3}\n---IN---\n{4}\n---CMDS---\n{5}".format(
347 cmd, url, out, err, pyinstall, "\n".join(cmds)))
349 # has pip?
350 if sys.platform.startswith("win"):
351 pyexe = os.path.join(temp_folder, "python.exe")
352 else:
353 pyexe = os.path.join(temp_folder, "bin", "python")
354 cmd = "{0} -m pip --help"
355 cmds.append(cmd)
356 try:
357 _, err = run_cmd(cmd, wait=True)
358 has_pip = not err
359 except Exception:
360 has_pip = False
362 # get-pip
363 if not has_pip:
364 get_pip = "https://bootstrap.pypa.io/get-pip.py"
365 outfile_pip = os.path.join(temp_folder, "get-pip.py")
366 download_file(get_pip, outfile_pip, fLOG=fLOG)
368 # following issue https://github.com/pypa/get-pip/issues/7
369 if sys.platform.startswith("win"):
370 vers = "%d%d" % versioni[:2]
371 if vers in ("36", "37"):
372 pth = os.path.join(temp_folder, "python%s._pth" % vers)
373 if os.path.exists(pth):
374 with open(pth, "r") as f:
375 content = f.read()
376 content = content.replace(
377 "#import site", "import site")
378 with open(pth, "w") as f:
379 f.write(content)
381 # run get-pip.py
382 if sys.platform.startswith("win"):
383 pyexe = os.path.join(temp_folder, "python.exe")
384 else:
385 versioni3 = versioni[:3]
386 pyexe = os.path.join(
387 temp_folder, "Python-{}.{}.{}".format(*versioni3), "python")
388 if not os.path.exists(pyexe):
389 raise FileNotFoundError(pyexe)
391 # Patches for windows.
392 if install and sys.platform.startswith("win"):
393 if not custom:
394 cmd = '"{0}" -u "{1}"'.format(pyexe, outfile_pip)
395 out, err = run_cmd(cmd, wait=True, fLOG=fLOG)
396 cmds.append(cmd)
397 if len(err) > 0:
398 skip = ['Consider adding this directory to PATH',
399 'which is not on PATH.']
400 lines = err.split('\n')
401 errs = []
402 for line in lines:
403 zoo = True
404 for sk in skip:
405 if sk in line:
406 zoo = False
407 break
408 if zoo:
409 errs.append(line)
410 err = "\n".join(errs).strip(' \n\r')
411 if len(err) > 0:
412 raise Exception(
413 "Something went wrong:\nCMD\n{0}\nOUT\n{1}\nERR-B\n{2}\n---CMDS--\n{3}".format(
414 cmd, out, err, "\n".join(cmds)))
415 else:
416 from ..win_installer.win_patch import win_patch_paths
417 fLOG("[install_python] Patch scripts .exe")
418 patched = win_patch_paths(temp_folder, pyexe, fLOG=fLOG)
419 for pat in patched:
420 fLOG(" - ", pat)
422 # fix fcntl
423 fix_fcntl_windows(temp_folder)
424 fix_termios_windows(temp_folder)
425 fix_resource_windows(temp_folder)
427 # modules
428 if install and modules is not None:
429 if isinstance(modules, list):
430 raise NotImplementedError(
431 "Not implemented for a list of modules.")
433 # cmd = '"{0}" -u -c "import pip;pip.main([\'install\',
434 # \'https://github.com/sdpython/pymyinstall/archive/master.zip\'])"'.format(pyexe)
435 cmd = '"{0}" -u -c "import pip._internal;pip._internal.main([\'install\', \'pyquicksetup\'])"'.format(
436 pyexe)
437 fLOG("[install_python] " + cmd)
438 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, change_path=None)
439 cmds.append(cmd)
440 if latest:
441 folder = os.path.normpath(os.path.join(os.path.abspath(
442 os.path.dirname(__file__)), "..", "..", ".."))
443 setup = os.path.join(folder, "setup.py")
444 if not os.path.exists(setup):
445 raise FileNotFoundError(setup)
446 sep = "\\" if sys.platform.startswith("win") else "/"
447 cmd = '"{0}" -u "{1}{2}setup.py" install'.format(
448 pyexe, folder, sep)
449 change_path = folder
450 else:
451 cmd = '"{0}" -u -c "import pip._internal;pip._internal.main([\'install\', \'pymyinstall\'])"'.format(
452 pyexe)
453 change_path = None
454 fLOG("[install_python] " + cmd)
455 out, err = run_cmd(cmd, wait=True, fLOG=fLOG, change_path=change_path)
456 cmds.append(cmd)
457 err_keep = err
458 err = [_ for _ in err.split("\n")
459 if not _.startswith("pymyinstall.") and
460 not _.startswith("zip_safe flag not set; analyzing archive contents...") and
461 not _.startswith("error removing build") and
462 "UserWarning:" not in _ and
463 "warnings.warn(" not in _ and
464 "module references __file__" not in _]
465 err = "\n".join(_ for _ in err if _)
467 exp = ".zip/lib2to3/Grammar.txt"
468 if len(err) > 0 and exp not in out.replace("\\", "/").replace("//", "/"):
469 raise Exception(
470 "Something went wrong:\nCMD\n{0}\nOUT\n{1}\nERR-C\n{2}".format(
471 cmd, out, err_keep))
472 fLOG(out)
474 dirpyexe = os.path.dirname(pyexe)
475 fLOG(
476 "[install_python] add python to PATH='{0}'".format(dirpyexe))
477 path = os.environ['PATH']
478 path = ";".join([dirpyexe, path])
479 os.environ['PATH'] = path
481 fLOG("[install_python] install modules")
482 pattern = ('"{0}" -u -c "import sys;from pymyinstall.packaged import install_all;install_all'
483 '(fLOG=print, temp_folder=\'{2}\','
484 'verbose=True, source=\'2\', list_module=\'{1}\')"')
485 cmd = pattern.format(
486 pyexe, modules, download_folder.replace("\\", "/"))
487 out, err = run_cmd(cmd, wait=True, fLOG=fLOG,
488 communicate=False, catch_exit=True)
489 cmds.append(cmd)
490 fLOG("[install_python] end installed modules.")
491 if len(err) > 0:
492 # We try a second time to make sure a second pass does not help.
493 fLOG("[install_python2] install modules")
494 out_, err_ = run_cmd(
495 cmd, wait=True, fLOG=fLOG, communicate=False, catch_exit=False)
496 err__ = _clean_err0(err_)
497 if len(err__) > 0:
498 mes = "[install_python2] end installed modules. Something went wrong:\n"
499 raise Exception(
500 mes + "ERR-D-CMD\n{0}\nOUT\n{1}\nOUT2\n{3}\nERR-D\n{2}\nERR2-D\n{4}\nERR2-Dc\n{5}\n**CMD**\n{0}\n--CMDS--\n{6}".format(
501 cmd, out, err, out_, err_, err__, "\n".join(cmds)))
502 out += ("\n-------------" * 5) + "\n" + out_
503 fLOG("[install_python2] end installed modules.")
504 fLOG(out)
506 return local
509def folder_older_than(folder, delay=datetime.timedelta(30)):
510 """
511 Tells if a folder is older than a given timespan.
513 @param folder folder name
514 @param delay delay
515 @return boolean
516 """
517 folder = os.path.abspath(folder)
518 if not os.path.exists(folder):
519 return False
520 cre = os.stat(folder).st_ctime
521 dt = datetime.datetime.fromtimestamp(cre)
522 now = datetime.datetime.now()
523 delta = now - dt
524 return delta > delay