Coverage for pyquickhelper/pycode/code_helper.py: 76%
103 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-03 02:21 +0200
1"""
2@file
3@brief Various function to clean the code.
4"""
5import os
8def remove_extra_spaces_and_pep8(filename, apply_pep8=True, aggressive=False, is_string=None):
9 """
10 Removes extra spaces in a filename, replaces the file in place.
12 :param filename: file name or string (but it assumes it is python).
13 :param apply_pep8: if True, calls :epkg:`autopep8` on the file
14 :param aggressive: more aggressive
15 :param is_string: force *filename* to be a string
16 :return: number of removed extra spaces
17 """
18 encoding = None
19 initial_content = None
20 if "\n" in filename or (is_string is not None and is_string):
21 ext = ".py"
22 lines = filename.replace("\r", "").split("\n")
23 filename = None
24 else:
25 ext = os.path.splitext(filename)[-1]
26 if ext in (".bat", ".py", ".sh", ".pyx", ".pxd"):
27 try:
28 with open(filename, "r") as f:
29 lines = f.readlines()
30 encoding = None
31 except PermissionError as e: # pragma: no cover
32 raise PermissionError(filename) from e
33 except UnicodeDecodeError as e: # pragma: no cover
34 try:
35 with open(filename, "r", encoding="utf-8") as f:
36 lines = f.readlines()
37 encoding = "utf-8"
38 except Exception:
39 raise RuntimeError(
40 f"unable to load file {filename} due to unicode errors") from e
41 initial_content = "".join(lines)
42 else:
43 try:
44 with open(filename, "r", encoding="utf-8-sig") as f:
45 lines = f.readlines()
46 encoding = "utf-8"
47 except PermissionError as e: # pragma: no cover
48 raise PermissionError(filename) from e
49 except UnicodeDecodeError as e: # pragma: no cover
50 try:
51 with open(filename, "r") as f:
52 lines = f.readlines()
53 encoding = None
54 except Exception:
55 raise RuntimeError(
56 f"unable to load file {filename} due to unicode errors") from e
57 initial_content = "".join(lines)
59 if filename is not None and len(lines) == 0 and not filename.endswith("__init__.py"):
60 raise ValueError( # pragma: no cover
61 f"File '{filename}' is empty, encoding='{encoding}'.")
63 if filename is not None and ext in (".py", ".pyx", ".pxd"):
64 if encoding is not None and len(lines) > 0 and "#-*-coding:utf-8-*-" in lines[0].replace(" ", ""):
65 with open(filename, "r", encoding="utf8") as f:
66 try:
67 lines = f.readlines()
68 except UnicodeDecodeError as e: # pragma: no cover
69 raise RuntimeError("unable to read: " + filename) from e
70 encoding = "utf8"
71 else:
72 encoding = None
74 def cdiff(lines):
75 lines2 = [_.rstrip(" \r\n") for _ in lines]
76 last = len(lines2) - 1
77 while last >= 0 and len(lines2[last]) == 0:
78 last -= 1
79 last += 1
80 lines2 = lines2[:last]
82 diff = len("".join(lines)) - len("\n".join(lines2)) + len(lines)
83 return diff, lines2
85 diff, lines2 = cdiff(lines)
87 if filename is not None:
88 ext = os.path.splitext(filename)[-1]
89 if ext in (".py", ) and apply_pep8:
90 # delayed import to speed up import of pycode
91 import autopep8
92 options = ['', '-a'] if aggressive else ['']
93 options.extend(["--ignore=E402,E731"])
94 r = autopep8.fix_code(
95 "\n".join(lines2), options=autopep8.parse_args(options))
97 if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
98 raise ValueError( # pragma: no cover
99 "Resulting file is empty for '{3}',\ninitial number of lines "
100 "{0} encoding='{1}' diff={2}".format(
101 len(lines), encoding, diff, filename))
102 if filename is None:
103 return r
104 elif r != initial_content:
105 if encoding is None:
106 with open(filename, "w") as f:
107 f.write(r)
108 else:
109 with open(filename, "w", encoding="utf8") as f:
110 f.write(r)
111 if r != "".join(lines):
112 diff, lines2 = cdiff(r.split("\n"))
113 else:
114 diff = 0
115 elif ext in (".rst", ".md", ".pyx", ".pxd"):
116 lines2 = [_.replace("\r", "").rstrip("\n ") for _ in lines]
117 rem = set()
118 for i, line in enumerate(lines2):
119 if i >= 1 and line == lines2[i - 1] == "":
120 rem.add(i)
121 lines2 = [_ for i, _ in enumerate(lines2) if i not in rem]
122 if len(lines) > 0 and len(lines2[-1]) > 0:
123 lines2.append("")
124 if len(lines) > 0 and len(lines2) == 0: # pragma: no cover
125 begin = 5 if len(lines) > 5 else len(lines)
126 mes = "Resulting file is empty for '{4}',\ninitial number of lines {0} encoding='{1}' len(rem)={2} diff={3}\nBeginning:\n{5}"
127 raise ValueError(mes.format(len(lines), encoding, len(
128 rem), diff, filename, "".join(lines[:begin])))
129 if len(lines2) < len(lines) // 2: # pragma: no cover
130 lines2_ = [_ for _ in lines2 if _ and _ != "\n"]
131 lines_ = [_ for _ in lines if _ and _ != "\n"]
132 if len(lines2_) < len(lines_) // 2:
133 begin = 5 if len(lines_) > 5 else len(lines_)
134 mes = "Resulting file is almost empty for '{4}',\ninitial number of lines {0} encoding='{1}' " + \
135 "len(rem)={2} diff={3}\nBeginning:\n{5}"
136 raise ValueError(mes.format(len(lines_), encoding, len(
137 rem), diff, filename, "".join(lines_[:begin])))
138 rl = "".join(lines)
139 r2 = "\n".join(lines2)
140 if r2 != rl:
141 if encoding is None:
142 with open(filename, "w") as f:
143 f.write("\n".join(lines2))
144 else:
145 with open(filename, "w", encoding="utf8") as f:
146 f.write("\n".join(lines2))
147 diff = max(1, diff)
148 else:
149 diff = 0
150 elif diff != 0:
151 if len(lines) > 0 and (len(lines2) == 0 or len(lines2) < len(lines) // 2):
152 raise ValueError("Resulting file is empty for '{3}',\ninitial number of lines {0} encoding='{1}' diff={2}".format(
153 len(lines), encoding, diff, filename))
154 r1 = "".join(lines)
155 r2 = "\n".join(lines2)
156 if r2 != r1:
157 if encoding is None:
158 with open(filename, "w") as f:
159 f.write("\n".join(lines2))
160 else:
161 with open(filename, "w", encoding="utf8") as f:
162 f.write("\n".join(lines2))
164 if not os.path.exists(filename):
165 raise FileNotFoundError( # pragma: no cover
166 f"Issue when applying autopep8 with filename: '{filename}'.")
167 return diff
170def remove_extra_spaces_folder(
171 folder, extensions=(".py", ".rst", ".md", ".pyx", ".pxd"), apply_pep8=True,
172 file_filter=None):
173 """
174 Removes extra files in a folder for specific file extensions.
176 @param folder folder to explore
177 @param extensions list of file extensions to process
178 @param apply_pep8 if True, calls :epkg:`autopep8` on the file
179 @param file_filter None of function which filters based on the filename
180 @return the list of modified files
182 The function does not check files having
183 ``/build/`` or ``/dist/`` or ``temp_``
184 or ``/build2/`` or ``/build3/``
185 in their name.
187 The signature of *file_filter* is the following:
189 ::
191 def file_filter(filename):
192 return True or False
193 """
194 # delayed import to speed up import of .pycode
195 from ..filehelper.synchelper import explore_folder
196 neg_pattern = "|".join(f"[/\\\\]{_}[/\\\\]" for _ in ["build", "build2", "build3",
197 "dist", "_venv", "_todo", "dist_module27", "_virtualenv"])
198 files = explore_folder(folder, neg_pattern=neg_pattern, fullname=True)[1]
199 mod = []
200 for f in files:
201 fl = f.lower().replace("\\", "/")
202 if "/temp_" not in fl and "/build/" not in fl \
203 and "/dist/" not in fl \
204 and "/build2/" not in fl \
205 and "/build3/" not in fl \
206 and "/_virtualenv/" not in fl \
207 and ".egg/" not in fl \
208 and "/_venv/" not in fl \
209 and "/_todo/" not in fl \
210 and "/dist_module27" not in fl \
211 and "automation_done.rst" not in fl \
212 and "auto_import.rst" not in fl \
213 and os.stat(f).st_size < 200000 \
214 and (file_filter is None or file_filter(f)):
215 ext = os.path.splitext(f)[-1]
216 if ext in extensions:
217 d = remove_extra_spaces_and_pep8(f, apply_pep8=apply_pep8)
218 if d != 0:
219 mod.append(f)
220 return mod