Coverage for pyquickhelper/pycode/code_helper.py: 76%

103 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-03 02:21 +0200

1""" 

2@file 

3@brief Various function to clean the code. 

4""" 

5import os 

6 

7 

8def remove_extra_spaces_and_pep8(filename, apply_pep8=True, aggressive=False, is_string=None): 

9 """ 

10 Removes extra spaces in a filename, replaces the file in place. 

11 

12 :param filename: file name or string (but it assumes it is python). 

13 :param apply_pep8: if True, calls :epkg:`autopep8` on the file 

14 :param aggressive: more aggressive 

15 :param is_string: force *filename* to be a string 

16 :return: number of removed extra spaces 

17 """ 

18 encoding = None 

19 initial_content = None 

20 if "\n" in filename or (is_string is not None and is_string): 

21 ext = ".py" 

22 lines = filename.replace("\r", "").split("\n") 

23 filename = None 

24 else: 

25 ext = os.path.splitext(filename)[-1] 

26 if ext in (".bat", ".py", ".sh", ".pyx", ".pxd"): 

27 try: 

28 with open(filename, "r") as f: 

29 lines = f.readlines() 

30 encoding = None 

31 except PermissionError as e: # pragma: no cover 

32 raise PermissionError(filename) from e 

33 except UnicodeDecodeError as e: # pragma: no cover 

34 try: 

35 with open(filename, "r", encoding="utf-8") as f: 

36 lines = f.readlines() 

37 encoding = "utf-8" 

38 except Exception: 

39 raise RuntimeError( 

40 f"unable to load file {filename} due to unicode errors") from e 

41 initial_content = "".join(lines) 

42 else: 

43 try: 

44 with open(filename, "r", encoding="utf-8-sig") as f: 

45 lines = f.readlines() 

46 encoding = "utf-8" 

47 except PermissionError as e: # pragma: no cover 

48 raise PermissionError(filename) from e 

49 except UnicodeDecodeError as e: # pragma: no cover 

50 try: 

51 with open(filename, "r") as f: 

52 lines = f.readlines() 

53 encoding = None 

54 except Exception: 

55 raise RuntimeError( 

56 f"unable to load file {filename} due to unicode errors") from e 

57 initial_content = "".join(lines) 

58 

59 if filename is not None and len(lines) == 0 and not filename.endswith("__init__.py"): 

60 raise ValueError( # pragma: no cover 

61 f"File '{filename}' is empty, encoding='{encoding}'.") 

62 

63 if filename is not None and ext in (".py", ".pyx", ".pxd"): 

64 if encoding is not None and len(lines) > 0 and "#-*-coding:utf-8-*-" in lines[0].replace(" ", ""): 

65 with open(filename, "r", encoding="utf8") as f: 

66 try: 

67 lines = f.readlines() 

68 except UnicodeDecodeError as e: # pragma: no cover 

69 raise RuntimeError("unable to read: " + filename) from e 

70 encoding = "utf8" 

71 else: 

72 encoding = None 

73 

74 def cdiff(lines): 

75 lines2 = [_.rstrip(" \r\n") for _ in lines] 

76 last = len(lines2) - 1 

77 while last >= 0 and len(lines2[last]) == 0: 

78 last -= 1 

79 last += 1 

80 lines2 = lines2[:last] 

81 

82 diff = len("".join(lines)) - len("\n".join(lines2)) + len(lines) 

83 return diff, lines2 

84 

85 diff, lines2 = cdiff(lines) 

86 

87 if filename is not None: 

88 ext = os.path.splitext(filename)[-1] 

89 if ext in (".py", ) and apply_pep8: 

90 # delayed import to speed up import of pycode 

91 import autopep8 

92 options = ['', '-a'] if aggressive else [''] 

93 options.extend(["--ignore=E402,E731"]) 

94 r = autopep8.fix_code( 

95 "\n".join(lines2), options=autopep8.parse_args(options)) 

96 

97 if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2): 

98 raise ValueError( # pragma: no cover 

99 "Resulting file is empty for '{3}',\ninitial number of lines " 

100 "{0} encoding='{1}' diff={2}".format( 

101 len(lines), encoding, diff, filename)) 

102 if filename is None: 

103 return r 

104 elif r != initial_content: 

105 if encoding is None: 

106 with open(filename, "w") as f: 

107 f.write(r) 

108 else: 

109 with open(filename, "w", encoding="utf8") as f: 

110 f.write(r) 

111 if r != "".join(lines): 

112 diff, lines2 = cdiff(r.split("\n")) 

113 else: 

114 diff = 0 

115 elif ext in (".rst", ".md", ".pyx", ".pxd"): 

116 lines2 = [_.replace("\r", "").rstrip("\n ") for _ in lines] 

117 rem = set() 

118 for i, line in enumerate(lines2): 

119 if i >= 1 and line == lines2[i - 1] == "": 

120 rem.add(i) 

121 lines2 = [_ for i, _ in enumerate(lines2) if i not in rem] 

122 if len(lines) > 0 and len(lines2[-1]) > 0: 

123 lines2.append("") 

124 if len(lines) > 0 and len(lines2) == 0: # pragma: no cover 

125 begin = 5 if len(lines) > 5 else len(lines) 

126 mes = "Resulting file is empty for '{4}',\ninitial number of lines {0} encoding='{1}' len(rem)={2} diff={3}\nBeginning:\n{5}" 

127 raise ValueError(mes.format(len(lines), encoding, len( 

128 rem), diff, filename, "".join(lines[:begin]))) 

129 if len(lines2) < len(lines) // 2: # pragma: no cover 

130 lines2_ = [_ for _ in lines2 if _ and _ != "\n"] 

131 lines_ = [_ for _ in lines if _ and _ != "\n"] 

132 if len(lines2_) < len(lines_) // 2: 

133 begin = 5 if len(lines_) > 5 else len(lines_) 

134 mes = "Resulting file is almost empty for '{4}',\ninitial number of lines {0} encoding='{1}' " + \ 

135 "len(rem)={2} diff={3}\nBeginning:\n{5}" 

136 raise ValueError(mes.format(len(lines_), encoding, len( 

137 rem), diff, filename, "".join(lines_[:begin]))) 

138 rl = "".join(lines) 

139 r2 = "\n".join(lines2) 

140 if r2 != rl: 

141 if encoding is None: 

142 with open(filename, "w") as f: 

143 f.write("\n".join(lines2)) 

144 else: 

145 with open(filename, "w", encoding="utf8") as f: 

146 f.write("\n".join(lines2)) 

147 diff = max(1, diff) 

148 else: 

149 diff = 0 

150 elif diff != 0: 

151 if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2): 

152 raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format( 

153 len(lines), encoding, diff, filename)) 

154 r1 = "".join(lines) 

155 r2 = "\n".join(lines2) 

156 if r2 != r1: 

157 if encoding is None: 

158 with open(filename, "w") as f: 

159 f.write("\n".join(lines2)) 

160 else: 

161 with open(filename, "w", encoding="utf8") as f: 

162 f.write("\n".join(lines2)) 

163 

164 if not os.path.exists(filename): 

165 raise FileNotFoundError( # pragma: no cover 

166 f"Issue when applying autopep8 with filename: '{filename}'.") 

167 return diff 

168 

169 

170def remove_extra_spaces_folder( 

171 folder, extensions=(".py", ".rst", ".md", ".pyx", ".pxd"), apply_pep8=True, 

172 file_filter=None): 

173 """ 

174 Removes extra files in a folder for specific file extensions. 

175 

176 @param folder folder to explore 

177 @param extensions list of file extensions to process 

178 @param apply_pep8 if True, calls :epkg:`autopep8` on the file 

179 @param file_filter None of function which filters based on the filename 

180 @return the list of modified files 

181 

182 The function does not check files having 

183 ``/build/`` or ``/dist/`` or ``temp_`` 

184 or ``/build2/`` or ``/build3/`` 

185 in their name. 

186 

187 The signature of *file_filter* is the following: 

188 

189 :: 

190 

191 def file_filter(filename): 

192 return True or False 

193 """ 

194 # delayed import to speed up import of .pycode 

195 from ..filehelper.synchelper import explore_folder 

196 neg_pattern = "|".join(f"[/\\\\]{_}[/\\\\]" for _ in ["build", "build2", "build3", 

197 "dist", "_venv", "_todo", "dist_module27", "_virtualenv"]) 

198 files = explore_folder(folder, neg_pattern=neg_pattern, fullname=True)[1] 

199 mod = [] 

200 for f in files: 

201 fl = f.lower().replace("\\", "/") 

202 if "/temp_" not in fl and "/build/" not in fl \ 

203 and "/dist/" not in fl \ 

204 and "/build2/" not in fl \ 

205 and "/build3/" not in fl \ 

206 and "/_virtualenv/" not in fl \ 

207 and ".egg/" not in fl \ 

208 and "/_venv/" not in fl \ 

209 and "/_todo/" not in fl \ 

210 and "/dist_module27" not in fl \ 

211 and "automation_done.rst" not in fl \ 

212 and "auto_import.rst" not in fl \ 

213 and os.stat(f).st_size < 200000 \ 

214 and (file_filter is None or file_filter(f)): 

215 ext = os.path.splitext(f)[-1] 

216 if ext in extensions: 

217 d = remove_extra_spaces_and_pep8(f, apply_pep8=apply_pep8) 

218 if d != 0: 

219 mod.append(f) 

220 return mod