Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief Class to transfer files to a website using FTP, it only transfers updated files
4"""
5from __future__ import print_function
6import re
7import os
8import warnings
9import sys
10import ftplib
11from io import BytesIO
12from time import sleep
13from random import random
14from .files_status import FilesStatus
15from ..loghelper.flog import noLOG
16from .ftp_transfer import CannotCompleteWithoutNewLoginException
19class FolderTransferFTPException(Exception):
21 """
22 custom exception for @see cl FolderTransferFTP
23 """
24 pass
27_text_extensions = {".ipynb", ".html", ".py", ".cpp", ".h", ".hpp", ".c",
28 ".cs", ".txt", ".csv", ".xml", ".css", ".js", ".r", ".doc",
29 ".ind", ".buildinfo", ".rst", ".aux", ".out", ".log", ".cc",
30 '.tmpl'}
33def content_as_binary(filename):
34 """
35 determines if filename is binary or None before transfering it
37 @param filename filename
38 @return boolean
39 """
40 global _text_extensions
41 ext = os.path.splitext(filename)[-1].lower()
42 if ext in _text_extensions:
43 return False
44 else:
45 return True
48class FolderTransferFTP:
50 """
51 This class aims at transfering a folder to a FTP website,
52 it checks that a file was updated before transfering,
53 @see cl TransferFTP .
55 .. exref::
56 :title: Transfer updated files to a website
58 The following code shows how to transfer the content of a folder to
59 website through FTP protocol.
61 ::
63 ftn = FileTreeNode("c:/somefolder")
64 ftp = TransferFTP("ftp.website.fr", "login", "password", fLOG=print)
65 fftp = FolderTransferFTP (ftn, ftp, "status_file.txt",
66 root_web = "/www/htdocs/app/pyquickhelper/helpsphinx")
68 fftp.start_transfering()
69 ftp.close()
71 The following example is more complete:
73 ::
75 import sys, os
76 from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
77 import keyring
79 user = keyring.get_password("webtransfer", "user")
80 pwd = keyring.get_password("webtransfer", "pwd")
82 ftp = TransferFTP("ftp.website.fr", user, pwd, fLOG=print)
84 location = r"local_location/GitHub/%s/dist/html"
85 this = os.path.abspath(os.path.dirname(__file__))
86 rootw = "/root/subfolder/%s/helpsphinx"
88 for module in ["pyquickhelper", "pyensae"] :
89 root = location % module
91 # documentation
92 sfile = os.path.join(this, "status_%s.txt" % module)
93 ftn = FileTreeNode(root)
94 fftp = FolderTransferFTP (ftn, ftp, sfile,
95 root_web = rootw % module,
96 fLOG=print)
98 fftp.start_transfering()
100 # setup, wheels
101 ftn = FileTreeNode(os.path.join(root,".."), filter = lambda root, path, f, dir: not dir)
102 fftp = FolderTransferFTP (ftn, ftp, sfile,
103 root_web = (rootw % module).replace("helpsphinx",""),
104 fLOG=print)
106 fftp.start_transfering()
108 ftp.close()
109 """
111 def __init__(self, file_tree_node, ftp_transfer, file_status, root_local=None,
112 root_web=None, footer_html=None, content_filter=None,
113 is_binary=content_as_binary, text_transform=None, filter_out=None,
114 exc=False, force_allow=None, fLOG=noLOG):
115 """
116 @param file_tree_node @see cl FileTreeNode
117 @param ftp_transfer @see cl TransferFTP
118 @param file_status file keeping the status for each file (date, hash of the content for the last upload)
119 @param root_local local root
120 @param root_web remote root on the website
121 @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
122 at the end of the file (before tag ``</body>``)
123 @param content_filter function which transform the content if a specific string is found
124 in the file, if the result is None, it raises an exception
125 indicating the file cannot be transfered (applies only on text files)
126 @param is_binary function which determines if content of a files is binary or not
127 @param text_transform function to transform the content of a text file before uploading it
128 @param filter_out regular expression to exclude some files, it can also be a function.
129 @param exc raise exception if not able to transfer
130 @param force_allow the class does not transfer a file containing a set of specific strings
131 except if they are in the list
132 @param fLOG logging function
134 Function *text_transform(self, filename, content)* returns the modified content.
136 If *filter_out* is a function, the signature is::
138 def filter_out(full_file_name, filename):
139 # ...
140 return True # if the file is filtered out, False otherwise
142 Function *filter_out* receives another parameter (filename)
143 to give more information when raising an exception.
144 """
145 self._ftn = file_tree_node
146 self._ftp = ftp_transfer
147 self._status = file_status
148 self._root_local = root_local if root_local is not None else file_tree_node.root
149 self._root_web = root_web if root_web is not None else ""
150 self.fLOG = fLOG
151 self._footer_html = footer_html
152 self._content_filter = content_filter
153 self._is_binary = is_binary
154 self._exc = exc
155 self._force_allow = force_allow
156 if filter_out is not None and not isinstance(filter_out, str):
157 self._filter_out = filter_out
158 else:
159 self._filter_out_reg = None if filter_out is None else re.compile(
160 filter_out)
161 self._filter_out = (lambda f: False) if filter_out is None else (
162 lambda f: self._filter_out_reg.search(f) is not None)
164 self._ft = FilesStatus(file_status)
165 self._text_transform = text_transform
167 def __str__(self):
168 """
169 usual
170 """
171 mes = ["FolderTransferFTP"]
172 mes += [" local root: {0}".format(self._root_local)]
173 mes += [" remote root: {0}".format(self._root_web)]
174 return "\n".join(mes)
176 def iter_eligible_files(self):
177 """
178 Iterates on eligible file for transfering
179 (if they have been modified).
181 @return iterator on file name
182 """
183 for f in self._ftn:
184 if f.isfile():
185 if self._filter_out(f.fullname):
186 continue
187 n = self._ft.has_been_modified_and_reason(f.fullname)[0]
188 if n:
189 yield f
191 def update_status(self, file):
192 """
193 Updates the status of a file.
195 @param file filename
196 @return @see cl FileInfo
197 """
198 r = self._ft.update_copied_file(file)
199 self._ft.save_dates()
200 return r
202 def preprocess_before_transfering(self, path, force_binary=False, force_allow=None):
203 """
204 Applies some preprocessing to the file to transfer.
205 It adds the footer for example.
206 It returns a stream which should be closed by
207 using method @see me close_stream.
209 @param path file name
210 @param force_binary impose a binary transfer
211 @param force_allow allow these strings even if they seem to be credentials
212 @return binary stream, size
214 Bypass utf-8 encoding checking when the extension is ``.rst.txt``.
215 """
216 if force_binary or self._is_binary(path):
217 size = os.stat(path).st_size
218 return open(path, "rb"), size
219 else:
220 if self._footer_html is None and self._content_filter is None:
221 size = os.stat(path).st_size
222 return open(path, "rb"), size
223 else:
224 size = os.stat(path).st_size
225 with open(path, "r", encoding="utf8") as f:
226 try:
227 content = f.read()
228 except UnicodeDecodeError as e:
229 ext = os.path.splitext(path)[-1]
230 if ext in {".js"} or path.endswith(".rst.txt"):
231 # just a warning
232 warnings.warn(
233 "FTP transfer, encoding issue with '{0}'".format(path), UserWarning)
234 return self.preprocess_before_transfering(path, True)
235 else:
236 stex = str(e).split("\n")
237 stex = "\n ".join(stex)
238 raise FolderTransferFTPException(
239 'Unable to transfer:\n File "{0}", line 1\nEXC:\n{1}'.format(path, stex)) from e
241 # footer
242 if self._footer_html is not None and os.path.splitext(
243 path)[-1].lower() in (".htm", ".html"):
244 spl = content.split("</body>")
245 if len(spl) > 1:
246 if len(spl) != 2:
247 spl = ["</body>".join(spl[:-1]), spl[-1]]
249 content = spl[0] + self._footer_html + \
250 "</body>" + spl[-1]
252 # filter
253 try:
254 content = self._content_filter(
255 content, path, force_allow=force_allow)
256 except Exception as e: # pragma: no cover
257 import traceback
258 exc_type, exc_value, exc_traceback = sys.exc_info()
259 trace = traceback.format_exception(
260 exc_type, exc_value, exc_traceback)
261 if isinstance(trace, list):
262 trace = "\n".join(trace)
263 raise FolderTransferFTPException(
264 "File '{0}' cannot be transferred (filtering exception)\nfunction:\n{1}\nEXC\n{2}\nStackTrace:\n{3}".format(
265 path, self._content_filter, e, trace)) from e
266 if content is None:
267 raise FolderTransferFTPException(
268 "File '{0}' cannot be transferred due to its content.".format(path))
270 # transform
271 if self._text_transform is not None:
272 content = self._text_transform(self, path, content)
274 # to binary
275 bcont = content.encode("utf8")
276 return BytesIO(bcont), len(bcont)
278 def close_stream(self, stream):
279 """
280 Closes a stream opened by @see me preprocess_before_transfering.
282 @param stream stream to close
283 """
284 if isinstance(stream, BytesIO):
285 pass
286 else:
287 stream.close()
289 def start_transfering(self, max_errors=20, delay=None):
290 """
291 Starts transfering files to a remote :epkg:`FTP` website.
293 :param max_errors: stops after this number of errors
294 :param delay: delay between two files
295 :return: list of transferred @see cl FileInfo
296 :raises FolderTransferFTPException: the class raises
297 an exception (@see cl FolderTransferFTPException)
298 more than *max_errors* issues happened
299 """
300 issues = []
301 done = []
302 total = list(self.iter_eligible_files())
303 sum_bytes = 0
304 for i, file in enumerate(total):
305 if i % 20 == 0:
306 self.fLOG("#### transfering %d/%d (so far %d bytes)" %
307 (i, len(total), sum_bytes))
308 relp = os.path.relpath(file.fullname, self._root_local)
309 if ".." in relp:
310 raise ValueError( # pragma: no cover
311 "The local root is not accurate:\n{0}\nFILE:\n{1}"
312 "\nRELPATH:\n{2}".format(self, file.fullname, relp))
313 path = self._root_web + "/" + os.path.split(relp)[0]
314 path = path.replace("\\", "/")
316 size = os.stat(file.fullname).st_size
317 self.fLOG("[upload % 8d bytes name=%s -- fullname=%s -- to=%s]" % (
318 size, os.path.split(file.fullname)[-1], file.fullname, path))
320 if self._exc:
321 data, size = self.preprocess_before_transfering(
322 file.fullname, force_allow=self._force_allow)
323 else:
324 try:
325 data, size = self.preprocess_before_transfering(
326 file.fullname, force_allow=self._force_allow)
327 except FolderTransferFTPException as ex: # pragma: no cover
328 stex = str(ex).split("\n")
329 stex = "\n ".join(stex)
330 warnings.warn(
331 "Unable to transfer '{0}' due to [{1}].".format(file.fullname, stex), ResourceWarning)
332 issues.append(
333 (file.fullname, "FolderTransferFTPException", ex))
334 continue
336 if size > 2**20:
337 blocksize = 2**20
338 transfered = 0
340 def callback_function_(*args, **kwargs):
341 "local function"
342 private_p = kwargs.get('private_p', None)
343 if private_p is None:
344 raise ValueError("private_p cannot be None")
345 private_p[1] += private_p[0]
346 private_p[1] = min(private_p[1], size)
347 self.fLOG(" transferred: %1.3f - %d/%d" %
348 (1.0 * private_p[1] / private_p[2], private_p[1], private_p[2]))
350 tp_ = [blocksize, transfered, size]
351 cb = lambda *args2, **kwargs2: callback_function_(
352 *args2, private_p=tp_, **kwargs2)
353 else:
354 blocksize = None
355 cb = None
357 if self._exc:
358 r = self._ftp.transfer(
359 data, path, os.path.split(file.fullname)[-1], blocksize=blocksize, callback=cb)
360 else:
361 try:
362 r = self._ftp.transfer(
363 data, path, os.path.split(file.fullname)[-1], blocksize=blocksize, callback=cb)
364 except FileNotFoundError as e: # pragma: no cover
365 r = False
366 issues.append((file.fullname, "not found", e))
367 self.fLOG("[FolderTransferFTP] - issue", e)
368 except ftplib.error_perm as ee: # pragma: no cover
369 r = False
370 issues.append((file.fullname, str(ee), ee))
371 self.fLOG("[FolderTransferFTP] - issue", ee)
372 except TimeoutError as eee: # pragma: no cover
373 r = False
374 issues.append((file.fullname, "TimeoutError", eee))
375 self.fLOG("[FolderTransferFTP] - issue", eee)
376 except EOFError as eeee: # pragma: no cover
377 r = False
378 issues.append((file.fullname, "EOFError", eeee))
379 self.fLOG("[FolderTransferFTP] - issue", eeee)
380 except ConnectionAbortedError as eeeee: # pragma: no cover
381 r = False
382 issues.append(
383 (file.fullname, "ConnectionAbortedError", eeeee))
384 self.fLOG(" issue", eeeee)
385 except ConnectionResetError as eeeeee: # pragma: no cover
386 r = False
387 issues.append(
388 (file.fullname, "ConnectionResetError", eeeeee))
389 self.fLOG("[FolderTransferFTP] - issue", eeeeee)
390 except CannotCompleteWithoutNewLoginException as e8: # pragma: no cover
391 r = False
392 issues.append(
393 (file.fullname, "CannotCompleteWithoutNewLoginException", e8))
394 self.fLOG("[FolderTransferFTP] - issue", e8)
395 except Exception as e7: # pragma: no cover
396 try:
397 import paramiko
398 except ImportError:
399 raise e7
400 if isinstance(e7, paramiko.sftp.SFTPError):
401 r = False
402 issues.append(
403 (file.fullname, "ConnectionResetError", e7))
404 self.fLOG("[FolderTransferFTP] - issue", e7)
405 else:
406 raise e7
408 self.close_stream(data)
410 sum_bytes += size
412 if r:
413 fi = self.update_status(file.fullname)
414 done.append(fi)
416 if len(issues) >= max_errors:
417 raise FolderTransferFTPException( # pragma: no cover
418 "Too many issues:\n{0}".format(
419 "\n".join("{0} -- {1} --- {2}".format(
420 a, b, str(c).replace('\n', ' ')) for a, b, c in issues)))
422 if delay is not None and delay > 0:
423 h = random()
424 delta = (h - 0.5) * delay * 0.1
425 delay_rnd = delay + delta
426 sleep(delay_rnd)
428 return done