Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Implements function @see fn run_cmd.
5"""
6import sys
7import os
8import time
9import subprocess
10import threading
11import warnings
12import re
13import queue
14from .flog_fake_classes import PQHException
17class RunCmdException(Exception):
18 """
19 Raised by function @see fn run_cmd.
20 """
21 pass
24def get_interpreter_path():
25 """
26 Returns the interpreter path.
27 """
28 if sys.platform.startswith("win"):
29 return sys.executable.replace( # pragma: no cover
30 "pythonw.exe", "python.exe")
31 else:
32 return sys.executable
35def split_cmp_command(cmd, remove_quotes=True):
36 """
37 Splits a command line.
39 @param cmd command line
40 @param remove_quotes True by default
41 @return list
42 """
43 if isinstance(cmd, str):
44 spl = cmd.split()
45 res = []
46 for s in spl:
47 if len(res) == 0:
48 res.append(s)
49 elif res[-1].startswith('"') and not res[-1].endswith('"'):
50 res[-1] += " " + s
51 else:
52 res.append(s)
53 if remove_quotes:
54 nres = []
55 for _ in res:
56 if _.startswith('"') and _.endswith('"'):
57 nres.append(_.strip('"'))
58 else:
59 nres.append(_)
60 return nres
61 else:
62 return res
63 else:
64 return cmd
67def decode_outerr(outerr, encoding, encerror, msg):
68 """
69 Decodes the output or the error after running a command line instructions.
71 @param outerr output or error
72 @param encoding encoding (if None, it is replaced by ascii)
73 @param encerror how to handle errors
74 @param msg to add to the exception message
75 @return converted string
76 """
77 if encoding is None:
78 encoding = "ascii"
79 typstr = str
80 if not isinstance(outerr, bytes):
81 raise TypeError( # pragma: no cover
82 "only able to decode bytes, not " + typstr(type(outerr)))
83 try:
84 out = outerr.decode(encoding, errors=encerror)
85 return out
86 except UnicodeDecodeError as exu:
87 try:
88 out = outerr.decode(
89 "utf8" if encoding != "utf8" else "latin-1", errors=encerror)
90 return out
91 except Exception as e: # pragma: no cover
92 out = outerr.decode(encoding, errors='ignore')
93 raise Exception("issue with cmd (" + encoding + "):" +
94 typstr(msg) + "\n" + typstr(exu) + "\n-----\n" + out) from e
95 raise Exception( # pragma: no cover
96 "complete issue with cmd:" + typstr(msg))
99def skip_run_cmd(cmd, sin="", shell=True, wait=False, log_error=True,
100 stop_running_if=None, encerror="ignore",
101 encoding="utf8", change_path=None, communicate=True,
102 preprocess=True, timeout=None, catch_exit=False, fLOG=None,
103 timeout_listen=None, tell_if_no_output=None, prefix_log=None):
104 """
105 Has the same signature as @see fn run_cmd but does nothing.
106 """
107 return "", ""
110def run_cmd(cmd, sin="", shell=sys.platform.startswith("win"), wait=False, log_error=True,
111 stop_running_if=None, encerror="ignore", encoding="utf8",
112 change_path=None, communicate=True, preprocess=True, timeout=None,
113 catch_exit=False, fLOG=None, tell_if_no_output=None, prefix_log=None):
114 """
115 Runs a command line and wait for the result.
117 @param cmd command line
118 @param sin sin: what must be written on the standard input
119 @param shell if True, cmd is a shell command (and no command window is opened)
120 @param wait call ``proc.wait``
121 @param log_error if log_error, call fLOG (error)
122 @param stop_running_if the function stops waiting if some condition is fulfilled.
123 The function received the last line from the logs.
124 Signature: ``stop_waiting_if(last_out, last_err) -> bool``.
125 The function must return True to stop waiting.
126 This function can also be used to intercept the standard output
127 and the standard error while running.
128 @param encerror encoding errors (ignore by default) while converting the output into a string
129 @param encoding encoding of the output
130 @param change_path change the current path if not None (put it back after the execution)
131 @param communicate use method `communicate
132 <https://docs.python.org/3/library/subprocess.html#subprocess.Popen.communicate>`_
133 which is supposed to be safer, parameter ``wait`` must be True
134 @param preprocess preprocess the command line if necessary (not available on Windows)
135 (False to disable that option)
136 @param timeout when data is sent to stdin (``sin``), a timeout is needed to avoid
137 waiting for ever (*timeout* is in seconds)
138 @param catch_exit catch *SystemExit* exception
139 @param fLOG logging function (if not None, bypass others parameters)
140 @param tell_if_no_output tells if there is no output every *tell_if_no_output* seconds
141 @param prefix_log add a prefix to a line before printing it
142 @return content of stdout, stdres (only if wait is True)
144 .. exref::
145 :title: Run a program using the command line
147 ::
149 from pyquickhelper.loghelper import run_cmd
150 out, err = run_cmd("python setup.py install", wait=True)
152 If you are using this function to run :epkg:`git` function, parameter ``shell`` must be True.
153 The function catches *SystemExit* exception.
154 See `Constantly print Subprocess output while process is running
155 <http://stackoverflow.com/questions/4417546/constantly-print-subprocess-output-while-process-is-running/4417735>`_.
156 If *wait* is False, the function returns the started process.
157 ``__exit__`` should be called if wait if False.
158 Parameter *prefix_log* was added.
159 """
160 if prefix_log is None:
161 prefix_log = ""
162 if fLOG is not None:
163 if isinstance(cmd, (list, tuple)):
164 fLOG( # pragma: no cover
165 prefix_log + "[run_cmd] execute", " ".join(cmd))
166 else:
167 fLOG(prefix_log + "[run_cmd] execute", cmd)
169 if change_path is not None:
170 current = os.getcwd()
171 os.chdir(change_path)
173 if sys.platform.startswith("win"):
174 cmdl = cmd
175 else:
176 cmdl = split_cmp_command(cmd) if preprocess else cmd
178 if catch_exit:
179 try:
180 pproc = subprocess.Popen(cmdl,
181 shell=shell,
182 stdin=subprocess.PIPE if sin is not None and len(
183 sin) > 0 else None,
184 stdout=subprocess.PIPE if wait else None,
185 stderr=subprocess.PIPE if wait else None)
186 except SystemExit as e:
187 if change_path is not None: # pragma: no cover
188 os.chdir(current)
189 raise RunCmdException( # pragma: no cover
190 "SystemExit raised (1)") from e
192 else:
193 pproc = subprocess.Popen(cmdl,
194 shell=shell,
195 stdin=subprocess.PIPE if sin is not None and len(
196 sin) > 0 else None,
197 stdout=subprocess.PIPE if wait else None,
198 stderr=subprocess.PIPE if wait else None)
200 pproc.__enter__()
201 if isinstance(cmd, list):
202 cmd = " ".join(cmd)
204 if wait:
205 skip_out_err = False
206 out = []
207 err = []
208 err_read = False
209 skip_waiting = False
211 if communicate:
212 # communicate is True
213 if tell_if_no_output is not None:
214 raise NotImplementedError(
215 "tell_if_no_output is not implemented when communicate is True")
216 if stop_running_if is not None:
217 raise NotImplementedError(
218 "stop_running_if is not implemented when communicate is True")
219 input = sin if sin is None else sin.encode()
220 if input is not None and len(input) > 0:
221 if fLOG is not None:
222 fLOG(prefix_log + "[run_cmd] input", [input])
224 if catch_exit:
225 try:
226 stdoutdata, stderrdata = pproc.communicate(
227 input=input, timeout=timeout)
228 except SystemExit as e: # pragma: no cover
229 if change_path is not None:
230 os.chdir(current)
231 raise RunCmdException("SystemExit raised (2)") from e
232 else:
233 stdoutdata, stderrdata = pproc.communicate(
234 input=input, timeout=timeout)
236 out = decode_outerr(stdoutdata, encoding, encerror, cmd)
237 err = decode_outerr(stderrdata, encoding, encerror, cmd)
238 else:
239 # communicate is False: use of threads
240 if sin is not None and len(sin) > 0:
241 if change_path is not None:
242 os.chdir(current) # pragma: no cover
243 raise Exception(
244 "communicate should be True to send something on stdin")
245 stdout, stderr = pproc.stdout, pproc.stderr
247 begin = time.perf_counter()
248 last_update = begin
249 # with threads
250 (stdoutReader, stdoutQueue) = _AsyncLineReader.getForFd(
251 stdout, catch_exit=catch_exit)
252 (stderrReader, stderrQueue) = _AsyncLineReader.getForFd(
253 stderr, catch_exit=catch_exit)
254 runloop = True
256 while (not stdoutReader.eof() or not stderrReader.eof()) and runloop:
257 while not stdoutQueue.empty():
258 line = stdoutQueue.get()
259 decol = decode_outerr(
260 line, encoding, encerror, cmd)
261 sdecol = decol.strip("\n\r")
262 if fLOG is not None:
263 fLOG(prefix_log + sdecol)
264 out.append(sdecol)
265 last_update = time.perf_counter()
266 if stop_running_if is not None and stop_running_if(decol, None):
267 runloop = False
268 break
270 while not stderrQueue.empty():
271 line = stderrQueue.get()
272 decol = decode_outerr(
273 line, encoding, encerror, cmd)
274 sdecol = decol.strip("\n\r")
275 if fLOG is not None:
276 fLOG(prefix_log + sdecol)
277 err.append(sdecol)
278 last_update = time.perf_counter()
279 if stop_running_if is not None and stop_running_if(None, decol):
280 runloop = False
281 break
282 time.sleep(0.05)
284 delta = time.perf_counter() - last_update
285 if tell_if_no_output is not None and delta >= tell_if_no_output:
286 fLOG( # pragma: no cover
287 prefix_log + "[run_cmd] No update in {0} seconds for cmd: {1}".format(
288 "%5.1f" % (last_update - begin), cmd))
289 last_update = time.perf_counter() # pragma: no cover
290 full_delta = time.perf_counter() - begin
291 if timeout is not None and full_delta > timeout:
292 runloop = False # pragma: no cover
293 fLOG( # pragma: no cover
294 prefix_log + "[run_cmd] Timeout after {0} seconds for cmd: {1}".format(
295 "%5.1f" % full_delta, cmd))
296 break # pragma: no cover
298 if runloop:
299 # Waiting for async readers to finish...
300 stdoutReader.join()
301 stderrReader.join()
303 # Waiting for process to exit...
304 returnCode = pproc.wait()
305 err_read = True
307 if returnCode != 0: # pragma: no cover
308 if change_path is not None:
309 os.chdir(current)
310 try:
311 # we try to close the ressources
312 stdout.close()
313 stderr.close()
314 except Exception as e:
315 warnings.warn(
316 "Unable to close stdout and sterr.", RuntimeWarning)
317 if catch_exit:
318 mes = "SystemExit raised with error code {0}\nCMD:\n{1}\nCWD:\n{2}\n#---OUT---#\n{3}\n#---ERR---#\n{4}"
319 raise RunCmdException(mes.format(
320 returnCode, cmd, os.getcwd(), "\n".join(out), "\n".join(err)))
321 raise subprocess.CalledProcessError(returnCode, cmd)
323 if not skip_waiting:
324 pproc.wait()
325 else: # pragma: no cover
326 out.append("[run_cmd] killing process.")
327 fLOG(
328 prefix_log + "[run_cmd] killing process because stop_running_if returned True.")
329 pproc.kill()
330 err_read = True
331 fLOG(prefix_log + "[run_cmd] process killed.")
332 skip_out_err = True
334 out = "\n".join(out)
335 if skip_out_err:
336 err = "Process killed." # pragma: no cover
337 else:
338 if err_read:
339 err = "\n".join(err)
340 else: # pragma: no cover
341 temp = err = stderr.read()
342 try:
343 err = decode_outerr(temp, encoding, encerror, cmd)
344 except Exception:
345 err = decode_outerr(temp, encoding, "ignore", cmd)
346 stdout.close()
347 stderr.close()
349 # same path for whether communicate is False or True
350 err = err.replace("\r\n", "\n")
351 if fLOG is not None:
352 fLOG(prefix_log + "end of execution", cmd)
354 if len(err) > 0 and log_error and fLOG is not None:
355 if "\n" in err:
356 fLOG(prefix_log + "[run_cmd] stderr (log)")
357 for eline in err.split("\n"):
358 fLOG(prefix_log + eline)
359 else:
360 fLOG( # pragma: no cover
361 prefix_log + "[run_cmd] stderr (log)\n%s" % err)
363 if change_path is not None:
364 os.chdir(current)
366 pproc.__exit__(None, None, None)
367 if sys.platform.startswith("win"): # pragma: no cover
368 if err is not None:
369 err = err.strip("\n\r\t ")
370 return out.replace("\r\n", "\n"), err.replace("\r\n", "\n")
371 else:
372 if err is not None:
373 err = err.strip("\n\r\t ")
374 return out, err
375 else:
377 if change_path is not None:
378 os.chdir(current) # pragma: no cover
380 return pproc, None
383def parse_exception_message(exc):
384 """
385 Parses the message embedded in an exception and returns the standard output and error
386 if it can be found.
388 @param exc exception coming from @see fn run_cmd
389 @return out, err
390 """
391 mes = str(exc)
392 reg = re.compile(".*#---OUT---#(.*)#---ERR---#(.*)", re.DOTALL)
393 find = reg.search(mes.replace("\r", ""))
394 if find: # pragma: no cover
395 gr = find.groups()
396 out, err = gr[0], gr[1]
397 return out.strip("\n "), err.strip("\n ")
398 else:
399 return None, None
402def run_script(script, *args, **kwargs):
403 """
404 Runs a script.
406 @param script script to execute or command line starting with ``-m``
407 @param args other parameters
408 @param kwargs sent to @see fn run_cmd
409 @return out,err: content of stdout stream and stderr stream
411 Allows command line starting with ``-m``.
412 """
413 if not script.startswith('-m') and not os.path.exists(script):
414 raise PQHException("file %s not found" % script)
415 py = get_interpreter_path()
416 cmd = "%s %s" % (py, script)
417 if len(args) > 0:
418 typstr = str
419 cmd += " " + " ".join([typstr(x) for x in args])
420 out, err = run_cmd(cmd, **kwargs)
421 return out, err
424class _AsyncLineReader(threading.Thread):
426 def __init__(self, fd, outputQueue, catch_exit):
427 threading.Thread.__init__(self)
429 assert isinstance(outputQueue, queue.Queue)
430 assert callable(fd.readline)
432 self.fd = fd
433 self.catch_exit = catch_exit
434 self.outputQueue = outputQueue
436 def run(self):
437 if self.catch_exit:
438 try:
439 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')):
440 pass
441 except SystemExit as e: # pragma: no cover
442 self.outputQueue.put(str(e))
443 raise RunCmdException("SystemExit raised (3)") from e
444 else:
445 for _ in map(self.outputQueue.put, iter(self.fd.readline, b'')):
446 pass
448 def eof(self):
449 return not self.is_alive() and self.outputQueue.empty()
451 @classmethod
452 def getForFd(cls, fd, start=True, catch_exit=False):
453 q = queue.Queue()
454 reader = cls(fd, q, catch_exit)
456 if start:
457 reader.start()
459 return reader, q