Coverage for src/ensae_teaching_cs/automation/ftp_publish_helper.py: 30%
166 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-28 06:23 +0200
1# -*- coding: utf-8 -*-
2"""
3@file
4@brief Helpers to publish the documentation of :epkg:`python` to a website.
5"""
6import sys
7import os
8from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
9from pyquickhelper.filehelper.ftp_transfer_files import content_as_binary as pqh_content_as_binary
10from .teaching_modules import get_teaching_modules
13def trigger_on_specific_strings(content, filename=None, force_allow=None):
14 """
15 Looks for specific string such as
16 *USERNAME*, *USERDNSDOMAIN*, *HOMEPATH*, *USERNAME*, *COMPUTERNAME*,
17 *LOGONSERVER*, *USER*, *HOME*, *LOGNAME*
18 and returns None if it was found or modifies the content to remove it.
20 @param content content of a file
21 @param filename only used when an exception is raised
22 @param force_allow allow these expressions even if they seem to be credentials
23 @return modified content
24 """
25 strep = [('somewhere', 'somewhere'),
26 ('somewhere', 'somewhere')]
27 for env in ['USERNAME', 'USER']:
28 if env in os.environ and os.environ[env] != "jenkins":
29 for sub in ["_data", "GitHub"]:
30 strep.extend([(r"C:\\%s\\__home_\\%s\\" % (os.environ[env], sub), "somewhere"),
31 ("C:\\%s\\__home_\\%s\\" %
32 (os.environ[env], sub), "somewhere"),
33 ("C:\\%s\\__home_\\%s\\" %
34 (os.environ[env], sub), "somewhere"),
35 (f"C:{os.environ[env]}__home_{sub}",
36 "somewhere"),
37 (f"{os.environ[env]}__home_{sub}", "somewhere")
38 ])
39 for s, b in strep:
40 if s in content:
41 content = content.replace(s, b)
43 if force_allow is None:
44 force_allow = set()
45 else:
46 force_allow = set(force_allow)
47 lower_content = content.lower()
48 for st in ["USERNAME", "USERDNSDOMAIN", "HOMEPATH", "USERNAME",
49 "COMPUTERNAME", "LOGONSERVER", "USER", 'HOME', 'LOGNAME']:
50 if st in os.environ:
51 s = os.environ[st].lower()
52 if s == 'jenkins':
53 continue
54 if s in ('administrateur', 'administrator'):
55 continue
56 if s not in force_allow and s in lower_content:
57 raise RuntimeError(
58 f'string {st}:{s} was found in\n File "{filename}", line 1')
59 return content
62def content_as_binary(filename):
63 """
64 Overloads function `content_as_finary
65 <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/filehelper/ftp_transfer_files.html?
66 highlight=content_as_binary#pyquickhelper.filehelper.ftp_transfer_files.content_as_binary>`_ from
67 `pyquickhelper <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/pyquickhelper/>`_.
68 Determines if filename is binary or None before transfering it.
70 @param filename filename
71 @return boolean
72 """
73 if pqh_content_as_binary(filename):
74 return True
75 ff = os.path.split(filename)[-1]
76 if ff == "footer.html":
77 return True
78 return False
81def text_transform(ftpp, filename, content):
82 """
83 If filename is *rss.xml*,
84 replaces the string *__BLOG_ROOT__* by *self._root_web*.
86 @param ftpp object FolderTransferFTP
87 @param filename filename
88 @param content content of the file
89 @return new content
90 """
91 if filename.endswith("rss.xml"):
92 web = ftpp._root_web
93 if not web.startswith("http://"):
94 web = "http://" + web.strip("/")
95 ftpp.fLOG("[text_transform] replace __BLOG_ROOT__ by ", web)
96 return content.replace("__BLOG_ROOT__", web)
97 else:
98 return content
101def publish_documentation(docs, ftpsite=None, login=None, password=None,
102 footer_html=None, content_filter=trigger_on_specific_strings,
103 is_binary=content_as_binary, force_allow=None,
104 delay=0.5, exc=False, ftps='FTP',
105 page_transform=None, fLOG=print):
106 """
107 Publishes the documentation and the setups of a python module on a webiste,
108 it assumes the modules is organized the same way as :epkg:`pyquickhelper`.
110 @param docs list of dictionaries (see below)
111 @param ftpsite something like ``ftp.something.``
112 @param login login
113 @param password password
114 @param footer_html append this HTML code to any uploaded page (such a javascript code to count the audience)
115 @param content_filter filter the content of a file (it raises an exception if the result is None),
116 appies only on text files
117 @param is_binary a function to tell if a content of a file is binary or not
118 @param force_allow a file is not published if it contains something which looks like credentials
119 except if this string is part of *force_allow*
120 @param delay delay between file transferring (in average)
121 @param exc raise exception if not able to transfer
122 @param ftps use protocol FTP, TLS, or SFTP
123 @param page_transform function which transforms
124 the page before uploading it,
125 @see fn text_transform
126 @param fLOG logging function
128 *docs* is a list of dictionaries which must contain for each folder
129 to transfer:
131 - ``local``: local folder
132 - ``root_local``: local paths will be related to this root
133 - ``root_web``: prefix to add to the remote paths
134 - ``status_file``: a file where the function populates the transfered files and some information about them
136 A local file is composed by ``<local_root>/<relative_path>``, it
137 will be uploaded to ``<web_root>/<relative_path>``.
138 """
140 params = {"ftpsite": ftpsite,
141 "login": login,
142 "password": password}
144 nbnone = len([v for k, v in params.items() if v is None or len(v) == 0])
145 if nbnone > 0:
146 raise ValueError(
147 f"One of the following parameters is not specified:\n{params}")
149 nbnone = [v for k, v in params.items() if v is None or len(v) == 0]
150 if len(nbnone) > 0:
151 raise RuntimeError("one of the parameters is None:\n" + str(nbnone))
153 password = params["password"]
154 login = params["login"]
155 ftpsite = params["ftpsite"]
157 filter_out = "([/\\\\]((moduletoc.html)|(blogtoc.html)|(searchbox.html)))|([.]buildinfo)|([.]pyc)"
159 ftp = TransferFTP(ftpsite, login, password, ftps=ftps, fLOG=fLOG)
161 if page_transform is None:
162 fct_transform = text_transform
163 else:
165 def combined_transform(ftpp, filename, content):
166 text_transform(ftpp, filename, content)
167 page_transform(ftpp, filename, content)
169 fct_transform = combined_transform
171 for project in docs:
173 fLOG("######################################################################")
174 for k, v in sorted(project.items()):
175 fLOG(f"[publish_documentation] loop {k}='{v}'")
177 location = project["local"]
178 root_local = project["root_local"]
179 root_web = project["root_web"]
181 sfile = project["status_file"]
182 rootw = project["root_web"]
184 # documentation + setup
185 fLOG(f"[publish_documentation] location='{location}'")
187 ftn = FileTreeNode(root_local)
188 fftp = FolderTransferFTP(ftn, ftp, sfile, root_web=rootw, fLOG=fLOG, footer_html=footer_html,
189 content_filter=content_filter, is_binary=is_binary,
190 text_transform=fct_transform, filter_out=filter_out,
191 force_allow=force_allow, exc=exc)
193 fftp.start_transfering(delay=delay)
195 ftn = FileTreeNode(os.path.join(root_local, ".."),
196 filter=lambda root, path, f, dir: not dir)
197 fftp = FolderTransferFTP(ftn, ftp, sfile,
198 root_web=root_web.replace("helpsphinx", ""), fLOG=fLOG,
199 footer_html=footer_html, content_filter=content_filter,
200 is_binary=is_binary, text_transform=fct_transform,
201 filter_out=filter_out)
203 fftp.start_transfering()
205 ftp.close()
208def publish_teachings_to_web(login, ftpsite="ftp.xavierdupre.fr", # pylint: disable=W0102
209 tracking_id=None,
210 location="c:\\jenkins\\pymy\\%s\\%s%s\\dist\\%s",
211 rootw="/www/htdocs/app/%s/%s",
212 folder_status=".",
213 layout=[("html", "helpsphinx")],
214 modules=None, password=None, force_allow=None,
215 suffix=("_UT_%d%d_std" % sys.version_info[:2],),
216 delay=0.5, exc=False, exc_transfer=False,
217 transfer=True, additional_projects=None,
218 ftps='FTP', page_transform=None, fLOG=print):
219 """
220 Copies the documentation to the website.
222 @param login login
223 @param ftpsite ftp site
224 @param tracking_id tracking_id
225 @param location location of Jenkins build
226 @param rootw root on ftp site
227 @param folder_status folder status
228 @param modules list of modules to publish, if None, use @see fn get_teaching_modules
229 @param password if None, if will asked
230 @param layout last part of the folders
231 @param suffix suffixes to append to the project name
232 @param force_allow allow to publish files even if they contain these strings
233 whereas they seem to be credentials
234 @param delay delay between two files being transferred
235 @param exc raise exception if not found (True) or skip (False)
236 @param exc_transfer raise an exception if cannot be transfered
237 @param transfer starts transfering, otherwise returns the list of
238 transfering task to do
239 @param additional_projects additional projects
240 @param ftps use protocol FTP, TLS, or SFTP
241 @param page_transform function which transforms a page before uploading it,
242 @see fn text_transform
243 @param fLOG logging function
245 Example of use::
247 import sys
248 import os
249 from pyquickhelper.filehelper import TransferFTP, FileTreeNode, FolderTransferFTP
250 from tkinterquickhelper.funcwin import open_window_params
251 from ensae_teaching_cs.automation.ftp_publish_helper import publish_teachings_to_web
253 login = "..."
254 website = "ftp...."
255 rootw = "/www/htdocs/app/%s/%s"
257 password = None
259 publish_teachings_to_web(login, ftpsite=website,
260 tracking_id="tracking_id",
261 location="<something>\\\\%s\\\\%s%s\\\\dist\\\\%s",
262 rootw=rootw,
263 folder_status=os.path.abspath("."),
264 password=password)
266 Example of an additional projects:
268 ::
270 other_projects = [dict(status_file="status_projects.txt"),
271 root_local="...", root_web="...")]
272 """
273 if modules is None:
274 modules = get_teaching_modules(branch=False)
276 if tracking_id is None:
277 tracking_id = ""
278 footer = ""
279 else:
280 # footer = """
281 # """.format(tracking_id)
282 footer = ""
284 if password is None and transfer:
285 raise ValueError("password is empty")
287 location = os.path.abspath(location)
288 if folder_status is None:
289 folder_status = os.path.abspath(os.path.dirname(__file__))
290 else:
291 folder_status = os.path.abspath(folder_status)
293 if not isinstance(suffix, (tuple, list)):
294 suffix = [suffix]
296 projects = []
297 for module in modules:
298 fLOG(
299 f"[ensae_teaching_cs] PUBLISH '{module}' - layout '{layout}'")
300 for lay in layout:
301 for suf in suffix:
302 root = os.path.abspath(location %
303 (module, module, suf, lay[0]))
304 if transfer and os.path.exists(root):
305 break
306 if transfer and not os.path.exists(root):
307 if exc:
308 p = os.path.abspath(location %
309 (module, module, suffix[0], lay[0]))
310 raise FileNotFoundError(
311 f"First tried '{root}'\n last tried '{p}'")
312 else:
313 continue
314 fLOG(" ", root)
315 rw = rootw % (module, lay[1])
317 project = dict(status_file=os.path.join(folder_status, f"status_{module}.txt"),
318 local=root, root_local=root, root_web=rw)
319 projects.append(project)
321 if module == "ensae_teaching_cs":
322 lay = [_ for _ in layout if _[0] == "html"][0]
323 if transfer and not os.path.exists(root):
324 if exc:
325 raise FileNotFoundError(root)
326 else:
327 continue
329 def _update_path(pth):
330 for a, b in [("\\build", "\\build3"),
331 ("\\html", "\\html3"),
332 ("/build", "/build3"),
333 ("/html", "/html3")]:
334 pth = pth.replace(a, b)
335 return pth
337 local = _update_path(root)
338 fLOG(f"[ensae_teaching_cs] checking folder '{local}'")
339 fLOG(f"[ensae_teaching_cs] root is '{root}'")
340 if os.path.exists(local):
341 fLOG(f"[ensae_teaching_cs] found '{local}'")
342 project = dict(status_file=os.path.join(folder_status, f"status_3_{module}.txt"),
343 local=local, root_local=local,
344 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3"))
345 projects.append(project)
346 project = dict(status_file=os.path.join(folder_status, f"status_2_{module}.txt"),
347 local=local, root_local=local,
348 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2"))
349 projects.append(project)
350 else:
351 fLOG("[ensae_teaching_cs] looking into DOC1, DOC3")
352 root1 = root.replace("_UT_", "_DOC1_")
353 if transfer:
354 if not os.path.exists(root1):
355 if exc:
356 raise FileNotFoundError(root1)
357 else:
358 pass
359 else:
360 project = dict(status_file=os.path.join(folder_status, f"status_doc1_{module}.txt"),
361 local=root1, root_local=root1,
362 root_web=(rootw % (module, lay[1])).replace("_no_clean", ""))
363 projects.append(project)
365 root3 = root.replace("_UT_", "_DOC3_").replace("html", "html3")
366 if transfer:
367 if not os.path.exists(root3):
368 if exc:
369 raise FileNotFoundError(root3)
370 else:
371 project = dict(status_file=os.path.join(folder_status, f"status_doc3_{module}.txt"),
372 local=root3, root_local=root3,
373 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx3"))
374 projects.append(project)
375 project = dict(status_file=os.path.join(folder_status, f"status_doc2_{module}.txt"),
376 local=root3, root_local=root3,
377 root_web=(rootw % (module, lay[1])).replace("_no_clean", "").replace("/helpsphinx", "/helpsphinx2"))
378 projects.append(project)
380 # publish
381 if additional_projects:
382 for proj in additional_projects:
383 if 'root_local' not in proj:
384 if 'folder' not in proj:
385 raise KeyError(
386 f"Key 'folder' or 'root_local' must be specified in {proj}.")
387 proj = proj.copy()
388 proj['root_local'] = proj['folder']
389 if 'folder' not in proj:
390 proj = proj.copy()
391 proj['folder'] = proj['root_local']
392 if 'local' not in proj:
393 proj = proj.copy()
394 proj['local'] = os.path.dirname(proj['root_local'])
395 projects.append(proj)
397 if transfer:
398 publish_documentation(
399 projects, ftpsite=ftpsite, login=login, password=password,
400 footer_html=footer, force_allow=force_allow, delay=delay,
401 exc=exc_transfer, ftps=ftps, page_transform=page_transform,
402 fLOG=fLOG)
403 return projects