1""" 

2@file 

3@brief This extension contains various functionalities to help unittesting. 

4""" 

5import os 

6import stat 

7import sys 

8import re 

9import warnings 

10import time 

11import importlib 

12from contextlib import redirect_stdout, redirect_stderr 

13from io import StringIO 

14 

15 

16def _get_PyLinterRunV(): 

17 # Separate function to speed up import. 

18 from pylint.lint import Run as PyLinterRun 

19 from pylint import __version__ as pylint_version 

20 if pylint_version >= '2.0.0': 

21 PyLinterRunV = PyLinterRun 

22 else: 

23 PyLinterRunV = lambda *args, do_exit=False: PyLinterRun( # pylint: disable=E1120, E1123 

24 *args, exit=do_exit) # pylint: disable=E1120, E1123 

25 return PyLinterRunV 

26 

27 

28def get_temp_folder(thisfile, name=None, clean=True, create=True, 

29 persistent=False, path_name="tpath"): 

30 """ 

31 Creates and returns a local temporary folder to store files 

32 when unit testing. 

33 

34 @param thisfile use ``__file__`` or the function which runs the test 

35 @param name name of the temporary folder 

36 @param clean if True, clean the folder first, it can also a function 

37 called to determine whether or not the folder should be 

38 cleaned 

39 @param create if True, creates it (empty if clean is True) 

40 @param persistent if True, create a folder at root level to reduce path length, 

41 the function checks the ``MAX_PATH`` variable and 

42 shorten the test folder is *max_path* is True on :epkg:`Windows`, 

43 on :epkg:`Linux`, it creates a folder three level ahead 

44 @param path_name test path used when *max_path* is True 

45 @return temporary folder 

46 

47 The function extracts the file which runs this test and will name 

48 the temporary folder base on the name of the method. *name* must be None. 

49 

50 Parameter *clean* can be a function. 

51 Signature is ``def clean(folder)``. 

52 """ 

53 if name is None: 

54 name = thisfile.__name__ 

55 if name.startswith("test_"): 

56 name = "temp_" + name[5:] 

57 elif not name.startswith("temp_"): 

58 name = "temp_" + name 

59 thisfile = os.path.abspath(thisfile.__func__.__code__.co_filename) 

60 final = os.path.split(name)[-1] 

61 

62 if not final.startswith("temp_") and not final.startswith("temp2_"): 

63 raise NameError("the folder '{0}' must begin with temp_".format(name)) 

64 

65 local = os.path.join( 

66 os.path.normpath(os.path.abspath(os.path.dirname(thisfile))), name) 

67 

68 if persistent: 

69 if sys.platform.startswith("win"): 

70 from ctypes.wintypes import MAX_PATH 

71 if MAX_PATH <= 300: 

72 local = os.path.join(os.path.abspath("\\" + path_name), name) 

73 else: 

74 local = os.path.join( 

75 local, "..", "..", "..", "..", path_name, name) 

76 else: 

77 local = os.path.join(local, "..", "..", "..", 

78 "..", path_name, name) 

79 local = os.path.normpath(local) 

80 

81 if name == local: 

82 raise NameError( 

83 "The folder '{0}' must be relative, not absolute".format(name)) 

84 

85 if not os.path.exists(local): 

86 if create: 

87 os.makedirs(local) 

88 mode = os.stat(local).st_mode 

89 nmode = mode | stat.S_IWRITE 

90 if nmode != mode: 

91 os.chmod(local, nmode) 

92 else: 

93 if (callable(clean) and clean(local)) or (not callable(clean) and clean): 

94 # delayed import to speed up import time of pycode 

95 from ..filehelper.synchelper import remove_folder 

96 remove_folder(local) 

97 time.sleep(0.1) 

98 if create and not os.path.exists(local): 

99 os.makedirs(local) 

100 mode = os.stat(local).st_mode 

101 nmode = mode | stat.S_IWRITE 

102 if nmode != mode: 

103 os.chmod(local, nmode) 

104 

105 return local 

106 

107 

108def _extended_refactoring(filename, line): 

109 """ 

110 Private function which does extra checkings 

111 when refactoring :epkg:`pyquickhelper`. 

112 

113 @param filename filename 

114 @param line line 

115 @return None or error message 

116 """ 

117 if "from pyquickhelper import fLOG" in line: 

118 if "test_code_style" not in filename: 

119 return "issue with fLOG" 

120 if "from pyquickhelper import noLOG" in line: 

121 if "test_code_style" not in filename: 

122 return "issue with noLOG" 

123 if "from pyquickhelper import run_cmd" in line: 

124 if "test_code_style" not in filename: 

125 return "issue with run_cmd" 

126 if "from pyquickhelper import get_temp_folder" in line: 

127 if "test_code_style" not in filename: 

128 return "issue with get_temp_folder" 

129 return None 

130 

131 

132class PEP8Exception(Exception): 

133 """ 

134 Code or style issues. 

135 """ 

136 pass 

137 

138 

139def check_pep8(folder, ignore=('E265', 'W504'), skip=None, 

140 complexity=-1, stop_after=100, fLOG=None, 

141 pylint_ignore=('C0103', 'C1801', 

142 'R0201', 'R1705', 

143 'W0108', 'W0613', 

144 'W0107', 'C0415', 

145 'C0209'), 

146 recursive=True, neg_pattern=None, extended=None, 

147 max_line_length=143, pattern=".*[.]py$", 

148 run_lint=True, verbose=False, run_cmd_filter=None): 

149 """ 

150 Checks if :epkg:`PEP8`, 

151 the function calls command :epkg:`pycodestyle` 

152 on a specific folder. 

153 

154 @param folder folder to look into 

155 @param ignore list of warnings to skip when raising an exception if 

156 :epkg:`PEP8` is not verified, see also 

157 `Error Codes <http://pep8.readthedocs.org/en/latest/intro.html#error-codes>`_ 

158 @param pylint_ignore ignore :epkg:`pylint` issues, see 

159 :epkg:`pylint error codes` 

160 @param complexity see `check_file <https://pycodestyle.pycqa.org/en/latest/api.html>`_ 

161 @param stop_after stop after *stop_after* issues 

162 @param skip skip a warning if a substring in this list is found 

163 @param neg_pattern skip files verifying this regular expressions 

164 @param extended list of tuple (name, function), see below 

165 @param max_line_length maximum allowed length of a line of code 

166 @param recursive look into subfolder 

167 @param pattern only file matching this pattern will be checked 

168 @param run_lint run :epkg:`pylint` 

169 @param verbose :epkg:`pylint` is slow, tells which file is 

170 investigated (but it is even slower) 

171 @param run_cmd_filter some files makes :epkg:`pylint` crashes (``import yaml``), 

172 the test for this is run in a separate process 

173 if the function *run_cmd_filter* returns True of the filename, 

174 *verbose* is set to True in that case 

175 @param fLOG logging function 

176 @return output 

177 

178 Functions mentioned in *extended* takes two parameters (file name and line) 

179 and they returned None or an error message or a tuple (position in the line, error message). 

180 When the return is not empty, a warning will be added to the ones 

181 printed by :epkg:`pycodestyle`. 

182 A few codes to ignore: 

183 

184 * *E501*: line too long (?? characters) 

185 * *E265*: block comments should have a space after # 

186 * *W504*: line break after binary operator, this one is raised 

187 after the code is modified by @see fn remove_extra_spaces_and_pep8. 

188 

189 The full list is available at :epkg:`PEP8 codes`. In addition, 

190 the function adds its own codes: 

191 

192 * *ECL1*: line too long for a specific reason. 

193 

194 Some errors to disable with :epkg:`pylint`: 

195 

196 * *C0103*: variable name is not conform 

197 * *C0111*: missing function docstring 

198 * *C1801*: do not use `len(SEQUENCE)` to determine if a sequence is empty 

199 * *R0201*: method could be a function 

200 * *R0205*: Class '?' inherits from object, can be safely removed from bases in python3 (pylint) 

201 * *R0901*: too many ancestors 

202 * *R0902*: too many instance attributes 

203 * *R0911*: too many return statements 

204 * *R0912*: too many branches 

205 * *R0913*: too many arguments 

206 * *R0914*: too many local variables 

207 * *R0915*: too many statements 

208 * *R1702*: too many nested blocks 

209 * *R1705*: unnecessary "else" after "return" 

210 * *W0107*: unnecessary pass statements 

211 * *W0108*: Lambda may not be necessary 

212 * *W0613*: unused argument 

213 

214 The full list is available at :epkg:`pylint error codes`. 

215 :epkg:`pylint` was added used to check the code. 

216 It produces the following list of errors 

217 :epkg:`pylint error codes`. 

218 

219 If *neg_pattern* is empty, it populates with a default value 

220 which skips unnecessary folders: 

221 ``".*[/\\\\\\\\]((_venv)|([.]git)|(__pycache__)|(temp_)).*"``. 

222 """ 

223 # delayed import to speed up import time of pycode 

224 import pycodestyle 

225 from ..filehelper.synchelper import explore_folder_iterfile 

226 if fLOG is None: 

227 from ..loghelper.flog import noLOG 

228 fLOG = noLOG 

229 

230 def extended_checkings(fname, content, buf, extended): 

231 for i, line in enumerate(content): 

232 for name, fu in extended: 

233 r = fu(fname, line) 

234 if isinstance(r, tuple): 

235 c, r = r 

236 else: 

237 c = 1 

238 if r is not None: 

239 buf.write("{0}:{1}:{4} F{2} {3}\n".format( 

240 fname, i + 1, name, r, c)) 

241 

242 def fkeep(s): 

243 if len(s) == 0: 

244 return False 

245 if skip is not None: 

246 for kip in skip: 

247 if kip in s: 

248 return False 

249 return True 

250 

251 if max_line_length is not None: 

252 if extended is None: 

253 extended = [] 

254 else: 

255 extended = extended.copy() 

256 

257 def check_lenght_line(fname, line): 

258 if len(line) > max_line_length and not line.lstrip().startswith('#'): 

259 if ">`_" in line: 

260 return "line too long (link) {0} > {1}".format(len(line), max_line_length) 

261 if ":math:`" in line: 

262 return "line too long (:math:) {0} > {1}".format(len(line), max_line_length) 

263 if "ERROR: " in line: 

264 return "line too long (ERROR:) {0} > {1}".format(len(line), max_line_length) 

265 return None 

266 

267 extended.append(("[ECL1]", check_lenght_line)) 

268 

269 if ignore is None: 

270 ignore = tuple() 

271 elif isinstance(ignore, list): 

272 ignore = tuple(ignore) 

273 

274 if neg_pattern is None: 

275 neg_pattern = ".*[/\\\\]((_venv)|([.]git)|(__pycache__)|(temp_)|([.]egg)|(bin)).*" 

276 

277 try: 

278 regneg_filter = None if neg_pattern is None else re.compile( 

279 neg_pattern) 

280 except re.error as e: 

281 raise ValueError("Unable to compile '{0}'".format(neg_pattern)) from e 

282 

283 # pycodestyle 

284 fLOG("[check_pep8] code style on '{0}'".format(folder)) 

285 files_to_check = [] 

286 skipped = [] 

287 buf = StringIO() 

288 with redirect_stdout(buf): 

289 for file in explore_folder_iterfile(folder, pattern=pattern, 

290 recursive=recursive): 

291 if regneg_filter is not None: 

292 if regneg_filter.search(file): 

293 skipped.append(file) 

294 continue 

295 if file.endswith("__init__.py"): 

296 ig = ignore + ('F401',) 

297 else: 

298 ig = ignore 

299 if file is None: 

300 raise TypeError("file cannot be None") 

301 if len(file) == 0: 

302 raise TypeError("file cannot be empty") 

303 

304 # code style 

305 files_to_check.append(file) 

306 try: 

307 style = pycodestyle.StyleGuide( 

308 ignore=ig, complexity=complexity, format='pylint', 

309 max_line_length=max_line_length) 

310 res = style.check_files([file]) 

311 except TypeError as e: 

312 ext = "This is often due to an instruction from . import... The imported module has no name." 

313 raise TypeError("Issue with pycodesyle for module '{0}' ig={1} complexity={2}\n{3}".format( 

314 file, ig, complexity, ext)) from e 

315 

316 if extended is not None: 

317 with open(file, "r", errors="ignore") as f: 

318 content = f.readlines() 

319 extended_checkings(file, content, buf, extended) 

320 

321 if res.total_errors + res.file_errors > 0: 

322 res.print_filename = True 

323 lines = [_ for _ in buf.getvalue().split("\n") if fkeep(_)] 

324 if len(lines) > stop_after: 

325 raise PEP8Exception( 

326 "{0} lines\n{1}".format(len(lines), "\n".join(lines))) 

327 

328 lines = [_ for _ in buf.getvalue().split("\n") if fkeep(_)] 

329 if len(lines) > 10: 

330 raise PEP8Exception( 

331 "{0} lines\n{1}".format(len(lines), "\n".join(lines))) 

332 

333 if len(files_to_check) == 0: 

334 mes = skipped[0] if skipped else "-no skipped file-" 

335 raise FileNotFoundError("No file found in '{0}'\n pattern='{1}'\nskipped='{2}'".format( 

336 folder, pattern, mes)) 

337 

338 # pylint 

339 if not run_lint: 

340 return "\n".join(lines) 

341 fLOG("[check_pep8] pylint with {0} files".format(len(files_to_check))) 

342 memout = sys.stdout 

343 

344 try: 

345 fLOG('', OutputStream=memout) 

346 regular_print = False 

347 except TypeError: 

348 regular_print = True 

349 

350 def myprint(s): 

351 "local print, chooses the right function" 

352 if regular_print: 

353 memout.write(s + "\n") 

354 else: 

355 fLOG(s, OutputStream=memout) 

356 

357 neg_pat = ".*temp[0-9]?_.*,doc_.*" 

358 if neg_pattern is not None: 

359 neg_pat += ',' + neg_pattern 

360 

361 if run_cmd_filter is not None: 

362 verbose = True 

363 

364 PyLinterRunV = _get_PyLinterRunV() 

365 sout = StringIO() 

366 serr = StringIO() 

367 with redirect_stdout(sout): 

368 with redirect_stderr(serr): 

369 with warnings.catch_warnings(): 

370 warnings.simplefilter("ignore", DeprecationWarning) 

371 opt = ["--ignore-patterns=" + neg_pat, "--persistent=n", 

372 '--jobs=1', '--suggestion-mode=n', "--score=n", 

373 '--max-args=30', '--max-locals=50', '--max-returns=30', 

374 '--max-branches=50', '--max-parents=25', 

375 '--max-attributes=50', '--min-public-methods=0', 

376 '--max-public-methods=100', '--max-bool-expr=10', 

377 '--max-statements=200', 

378 '--msg-template={abspath}:{line}: {msg_id}: {msg} (pylint)'] 

379 if pylint_ignore: 

380 opt.append('--disable=' + ','.join(pylint_ignore)) 

381 if max_line_length: 

382 opt.append("--max-line-length=%d" % max_line_length) 

383 if verbose: 

384 for i, name in enumerate(files_to_check): 

385 cop = list(opt) 

386 cop.append(name) 

387 if run_cmd_filter is None or not run_cmd_filter(name): 

388 myprint( 

389 "[check_pep8] lint file {0}/{1} - '{2}'\n".format(i + 1, len(files_to_check), name)) 

390 PyLinterRunV(cop, do_exit=False) 

391 else: 

392 # delayed import to speed up import time of pycode 

393 from ..loghelper import run_cmd 

394 # runs from command line 

395 myprint( 

396 "[check_pep8] cmd-lint file {0}/{1} - '{2}'\n".format(i + 1, len(files_to_check), name)) 

397 cmd = "{0} -m pylint {1}".format( 

398 sys.executable, " ".join('"{0}"'.format(_) for _ in cop)) 

399 out = run_cmd(cmd, wait=True)[0] 

400 lines.extend(_ for _ in out.split( 

401 '\n') if _.strip('\r ')) 

402 else: 

403 opt.extend(files_to_check) 

404 PyLinterRunV(opt, do_exit=False) 

405 

406 pylint_lines = sout.getvalue().split('\n') 

407 pylint_lines = [ 

408 _ for _ in pylint_lines if ( 

409 '(pylint)' in _ and fkeep(_) and _[0] != ' ' and len(_.split(':')) > 2)] 

410 pylint_lines = [_ for _ in pylint_lines if not _.startswith( 

411 "except ") and not _.startswith("else:") and not _.startswith( 

412 "try:") and "# noqa" not in _] 

413 lines.extend(pylint_lines) 

414 if len(lines) > 0: 

415 raise PEP8Exception( 

416 "{0} lines\n{1}".format(len(lines), "\n".join(lines))) 

417 

418 return "\n".join(lines) 

419 

420 

421def add_missing_development_version(names, root, hide=False): 

422 """ 

423 Looks for development version of a given module and add paths to 

424 ``sys.path`` after having checked they are working. 

425 

426 @param names name or names of the module to import 

427 @param root folder where to look (assuming all modules location 

428 at the same place in a flat hierarchy) 

429 @param hide hide warnings when importing a module (might be a lot) 

430 @return added paths 

431 """ 

432 # delayed import to speed up import time 

433 from ..loghelper import sys_path_append 

434 

435 if not isinstance(names, list): 

436 names = [names] 

437 root = os.path.abspath(root) 

438 if os.path.isfile(root): 

439 root = os.path.dirname(root) 

440 if not os.path.exists(root): 

441 raise FileNotFoundError(root) 

442 

443 spl = os.path.split(root) 

444 py27 = False 

445 if spl[-1].startswith("ut_"): 

446 if "dist_module27" in root: 

447 # python 27 

448 py27 = True 

449 newroot = os.path.join(root, "..", "..", "..", "..") 

450 else: 

451 newroot = os.path.join(root, "..", "..", "..") 

452 else: 

453 newroot = root 

454 

455 newroot = os.path.normpath(os.path.abspath(newroot)) 

456 found = os.listdir(newroot) 

457 dirs = [os.path.join(newroot, _) for _ in found] 

458 

459 paths = [] 

460 for name in names: 

461 exc = None 

462 try: 

463 if hide: 

464 with warnings.catch_warnings(record=True): 

465 importlib.import_module(name) 

466 else: 

467 importlib.import_module(name) 

468 continue 

469 except ImportError as e: 

470 # it requires a path 

471 exc = e 

472 

473 if name not in found: 

474 raise FileNotFoundError("Unable to find a subfolder '{0}' in '{1}' (py27={3})\nFOUND:\n{2}\nexc={4}".format( 

475 name, newroot, "\n".join(dirs), py27, exc)) 

476 

477 if py27: 

478 this = os.path.join(newroot, name, "dist_module27", "src") 

479 if not os.path.exists(this): 

480 this = os.path.join(newroot, name, "dist_module27") 

481 else: 

482 this = os.path.join(newroot, name, "src") 

483 if not os.path.exists(this): 

484 this = os.path.join(newroot, name) 

485 

486 if not os.path.exists(this): 

487 raise FileNotFoundError("unable to find a subfolder '{0}' in '{1}' (*py27={3})\nFOUND:\n{2}".format( 

488 this, newroot, "\n".join(dirs), py27)) 

489 with sys_path_append(this): 

490 if hide: 

491 with warnings.catch_warnings(record=True): 

492 importlib.import_module(name) 

493 else: 

494 importlib.import_module(name) 

495 paths.append(this) 

496 return paths