1""" 

2@file 

3@brief Helper to convert a script written in Python 3 to Python 2 

4""" 

5 

6import os 

7import re 

8import shutil 

9from ..filehelper.synchelper import explore_folder_iterfile 

10from ..loghelper.flog import noLOG 

11from .default_regular_expression import _setup_pattern_copy 

12 

13 

14class Convert3to2Exception(Exception): 

15 

16 """ 

17 exception raised for an exception happening during the conversion 

18 """ 

19 pass 

20 

21 

22def py3to2_convert_tree(folder, dest, encoding="utf8", pattern=".*[.]py$", 

23 pattern_copy=_setup_pattern_copy, 

24 unittest_modules=None, fLOG=noLOG): 

25 """ 

26 Converts files in a folder and its subfolders from python 3 to python 2, 

27 the function only considers python script (verifying *pattern*). 

28 

29 @param folder folder 

30 @param dest destination 

31 @param encoding all files will be saved with this encoding 

32 @param pattern pattern to find source code 

33 @param pattern_copy copy these files, do not modify them 

34 @param fLOG logging function 

35 @param unittest_modules modules used during unit tests but not installed 

36 @return list of copied files 

37 

38 If a folder does not exists, it will create it. 

39 The function excludes all files in subfolders 

40 starting by ``dist``, ``_doc``, ``build``, ``extensions``, ``nbextensions``. 

41 The function also exclude subfolders inside 

42 subfolders following the pattern ``ut_.*``. 

43 

44 There are some issues difficult to solve with strings. 

45 Python 2.7 is not friendly with strings. Some needed pieces of code:: 

46 

47 if sys.version_info[0]==2: 

48 from codecs import open 

49 

50 You can also read blog post :ref:`b-migration-py2py3`. 

51 

52 The variable *unittest_modules* indicates the list of 

53 modules which are not installed in :epkg:`Python` distribution 

54 but still used and placed in the same folder as the same which 

55 has to converted. 

56 

57 *unittest_modules* can be either a list or a tuple ``(module, alias)``. 

58 Then the alias appears instead of the module name. 

59 

60 The function does not convert the exception 

61 `FileNotFoundError <https://docs.python.org/3/library/exceptions.html>`_ 

62 which only exists in Python 3. The module will fail in version 2.7 

63 if this exception is raised. 

64 

65 The following page 

66 `Cheat Sheet: Writing Python 2-3 compatible code 

67 <http://python-future.org/compatible_idioms.html>`_ 

68 gives the difference between the two versions of Python 

69 and how to write compatible code. 

70 """ 

71 exclude = ("temp_", "dist", "_doc", "build", "extensions", 

72 "nbextensions", "dist_module27", "_virtualenv", "_venv") 

73 reg = re.compile(".*/ut_.*/.*/.*") 

74 

75 conv = [] 

76 for file in explore_folder_iterfile(folder, pattern=pattern): 

77 full = os.path.join(folder, file) 

78 if "site-packages" in full: 

79 continue 

80 file = os.path.relpath(file, folder) 

81 

82 # undesired sub folders 

83 ex = False 

84 for exc in exclude: 

85 if file.startswith(exc) or "\\temp_" in file or \ 

86 "/temp_" in file or "dist_module27" in file: 

87 ex = True 

88 break 

89 if ex: 

90 continue 

91 

92 # subfolders inside unit tests folder 

93 lfile = file.replace("\\", "/") 

94 if reg.search(lfile): 

95 continue 

96 

97 py2 = py3to2_convert(full, unittest_modules) 

98 destfile = os.path.join(dest, file) 

99 dirname = os.path.dirname(destfile) 

100 if not os.path.exists(dirname): 

101 os.makedirs(dirname) 

102 with open(destfile, "w", encoding="utf8") as f: 

103 f.write(py2) 

104 conv.append(destfile) 

105 

106 for file in explore_folder_iterfile(folder, pattern=pattern_copy): 

107 full = os.path.join(folder, file) 

108 file = os.path.relpath(file, folder) 

109 

110 # undesired sub folders 

111 ex = False 

112 for exc in exclude: 

113 if file.startswith(exc) or "\\temp_" in file or \ 

114 "/temp_" in file or "dist_module27" in file: 

115 ex = True 

116 break 

117 if ex: 

118 continue 

119 

120 destfile = os.path.join(dest, file) 

121 dirname = os.path.dirname(destfile) 

122 if not os.path.exists(dirname): 

123 os.makedirs(dirname) 

124 shutil.copy(full, dirname) 

125 conv.append(destfile) 

126 

127 fLOG("py3to2_convert_tree, copied", len(conv), "files") 

128 

129 return conv 

130 

131 

132def py3to2_convert(script, unittest_modules): 

133 """ 

134 converts a script into from python 3 to python 2 

135 

136 @param script script or filename 

137 @param unittest_modules modules used during unit test but not installed, 

138 @see fn py3to2_convert_tree 

139 @return string 

140 

141 See see @fn py3to2_convert_tree for more information. 

142 """ 

143 if os.path.exists(script): 

144 try: 

145 with open(script, "r", encoding="utf8") as f: 

146 content = f.read() 

147 except (UnicodeEncodeError, UnicodeDecodeError): # pragma: no cover 

148 with open(script, "r") as f: 

149 content = f.read() 

150 

151 else: 

152 content = script # pragma: no cover 

153 

154 # start processing 

155 content = py3to2_remove_raise_from(content) 

156 

157 # unicode 

158 if ("install_requires=" in content or "package_data" in content) and "setup" in content: 

159 # we skip the file setup.py as it raises an error 

160 pass 

161 else: 

162 try: 

163 content = py3to2_future(content) 

164 except Convert3to2Exception as e: # pragma: no cover 

165 raise Convert3to2Exception( 

166 'unable to convert a file due to unicode issue.\n File "{0}", line 1'.format(script)) from e 

167 

168 # some other modification 

169 content = content.replace("from queue import", "from Queue import") 

170 content = content.replace("nonlocal ", "# nonlocal ") 

171 

172 # long and unicode 

173 content = content.replace("int #long#", "long") 

174 content = content.replace("int # long#", "long") 

175 content = content.replace("str #unicode#", "unicode") 

176 content = content.replace("str # unicode#", "unicode") 

177 content = content.replace( 

178 "Programming Language :: Python :: 3", "Programming Language :: Python :: 2") 

179 content = content.replace(', sep="\\t")', ', sep="\\t".encode("ascii"))') 

180 

181 # imported modules 

182 if unittest_modules is not None: 

183 content = py3to2_imported_local_modules(content, unittest_modules) 

184 

185 # end 

186 return content 

187 

188 

189def py3to2_future(content): 

190 """ 

191 checks that import ``from __future__ import unicode_literals`` 

192 is always present, the function assumes it is a python code 

193 

194 @param content file content 

195 @return new content 

196 """ 

197 find = "from __future__ import unicode_literals" 

198 if find in content and '"{0}"'.format(find) not in content: 

199 # the second condition avoid to raise this 

200 # exception when parsing this file 

201 # this case should only happen for this file 

202 raise Convert3to2Exception( # pragma: no cover 

203 "unable to convert a file") 

204 

205 lines = content.split("\n") 

206 position = 0 

207 incomment = None 

208 while (position < len(lines) and not lines[position].startswith("import ") and 

209 not lines[position].startswith("from ") and 

210 not lines[position].startswith("def ") and 

211 not lines[position].startswith("class ")): 

212 if incomment is None: 

213 if lines[position].startswith("'''"): 

214 incomment = "'''" # pragma: no cover 

215 elif lines[position].startswith('"""'): 

216 incomment = '"""' 

217 else: 

218 if lines[position].endswith("'''"): # pragma: no cover 

219 incomment = None 

220 position += 1 

221 break 

222 if lines[position].endswith('"""'): 

223 incomment = None 

224 position += 1 

225 break 

226 position += 1 

227 

228 if position < len(lines): 

229 lines[position] = "{0}\n{1}".format(find, lines[position]) 

230 return "\n".join(lines) 

231 

232 

233def py3to2_remove_raise_from(content): 

234 """ 

235 Removes expression such as: ``raise Exception ("...") from e``. 

236 The function is very basic. It should be done with a grammar. 

237 

238 @param content file content 

239 @return script 

240 """ 

241 lines = content.split("\n") 

242 r = None 

243 for i, line in enumerate(lines): 

244 if " raise " in line: 

245 r = i 

246 if " from " in line and r is not None: 

247 spl = line.split(" from ") 

248 if len(spl[0].strip(" \n")) > 0: 

249 lines[i] = line = spl[0] + "# from " + " - ".join(spl[1:]) 

250 

251 if r is not None and i > r + 3: 

252 r = None 

253 

254 return "\n".join(lines) 

255 

256 

257def py3to2_imported_local_modules(content, unittest_modules): 

258 """ 

259 See function @see fn py3to2_convert_tree 

260 and documentation about parameter *unittest_modules*. 

261 

262 @param content script or filename 

263 @param unittest_modules modules used during unit test but not installed, 

264 @see fn py3to2_convert_tree 

265 """ 

266 lines = content.split("\n") 

267 for modname in unittest_modules: 

268 if isinstance(modname, tuple): 

269 modname, alias = modname # pragma: no cover 

270 else: 

271 alias = modname 

272 

273 s1 = '"{0}"'.format(modname) 

274 s2 = "'{0}'".format(modname) 

275 s3 = "import {0}".format(modname) 

276 s4 = '"{0}"'.format(modname.upper()) 

277 s4_rep = '"{0}27"'.format(modname.upper()) 

278 

279 if (s1 in content or s2 in content or s4 in content) and s3 in content: 

280 for i, line in enumerate(lines): 

281 if " in " in line or "ModuleInstall" in line: 

282 continue 

283 if s1 in line: 

284 line = line.replace( 

285 s1, '"..", "{0}", "dist_module27"'.format(alias)) 

286 lines[i] = line 

287 elif s2 in line: 

288 line = line.replace( # pragma: no cover 

289 s2, "'..', '{0}', 'dist_module27'".format(alias)) 

290 lines[i] = line # pragma: no cover 

291 elif s4 in line: 

292 line = line.replace(s4, s4_rep) 

293 lines[i] = line 

294 return "\n".join(lines)