Coverage for src/ensae_teaching_cs/automation_students/projects_helper.py: 13%
69 statements
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-28 06:23 +0200
« prev ^ index » next coverage.py v7.1.0, created at 2023-04-28 06:23 +0200
1"""
2@file
3@brief A couple of functons which automates everything.
4"""
6import os
7import pandas
8from pyquickhelper.loghelper import fLOG
9from pyquickhelper.filehelper import encrypt_stream
10from pymmails import MailBoxImap, EmailMessageRenderer, EmailMessageListRenderer
11from pymmails.render.email_message_style import template_email_html_short
12from .projects_repository import ProjectsRepository
13from .mail_helper import grab_addresses
16def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt",
17 user=None, pwd=None, server="imap.gmail.com",
18 mailfolder=[
19 "ensae/ENSAE_2016_3A"],
20 date="1-Jan-2016", zipfilename="projet_3A_2016.zip",
21 zipencpwd=b"sixteenbyteskeys", dataframe=None,
22 columns={
23 "name": "nom_prenom", "group": "groupe", "subject": "sujet"},
24 skip_names=None, process_name=None,
25 title="List of emails", nolink_if=None, fLOG=fLOG):
26 """
27 The scenario is the following:
29 * You are the teacher.
30 * Students started their projects at date *t*.
31 * They can work alone or by group.
32 * They send mails, you reply.
33 * Their address mail follows the convention: ``<first name>.<last name>@anything``
34 so it is to associate a mail address to a student name.
35 * You move every mail you received in a separate folder in your inbox.
36 * Sometime, you send a mail to everybody.
37 * Finally they send their project with attachments.
38 * You want to store everything (mails and attachements) in folders, one per group.
39 * You want a summary of what was received.
40 * You want to build a zip file to share their work with others teachers.
41 * You want to update the folder if a new mail was sent.
43 This function looks into a folder of your inbox and grabs every mails and
44 attachements from a groups of students.
46 @param folder where to store the results
47 @param filemails files used to store students address,
48 the operation is done once, remove the file
49 to force the function to rebuild the information.
50 @param user user of the gmail inbox
51 @param pwd password of the gmail inbox
52 @param server gmail server, it should be ``"imap.gmail.com"``,
53 it works with others mail servers using the *IMAP* protocol
54 @param mailfolder folder in your inbox to look into,
55 there can be several
56 @param date when to start looking (do not change the format,
57 look at the default value)
58 @param zipfilename name of the zip file to create
59 @param zipencpwd the zip file is also encrypted for a safer share with this key
60 and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/
61 pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_.
62 @param dataframe dataframe which contains the definition of students groups
63 @param columns columns the function will look into, students names, group definition
64 (a unique number for all students in the same group), subject
65 @param skip_names list of names to skip
66 @param process_name to operate a transformation before matching students names with
67 their emails
68 @param title each group folder contains a html file connecting them,
69 this is its title
70 @param nolink_if The summary extracts links from url, it skips the urls which
71 contains on the substrings included in that list (None to use a default set)
72 @param fLOG logging function
73 @return @see cl ProjectsRepository
75 By default, Gmail does not let you programmatically access you own inbox,
76 you need to modify your gmail parameters to let this function do so.
77 """
78 folder = os.path.abspath(".")
79 filemails = os.path.join(folder, filemails)
80 zipfilename = os.path.join(folder, zipfilename)
81 zipfilenameenc = zipfilename + ".enc"
83 # load the groups
84 if isinstance(dataframe, pandas.DataFrame):
85 df = dataframe
86 elif dataframe.endswith("xlsx"):
87 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe)
88 df = pandas.read_excel(dataframe, engine='openpyxl')
89 else:
90 df = pandas.read_csv(dataframe, sep="\t", encoding="utf8")
92 # check mails
93 if "mail" not in columns:
94 if os.path.exists(filemails):
95 fLOG(
96 "[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails)
97 with open(filemails, "r", encoding="utf8") as f:
98 lines = f.readlines()
99 emails = [li.strip("\r\t\n ") for li in lines]
100 else:
101 fLOG(
102 "[extract_students_mails_from_gmail_and_stores_in_folders] mine address ")
103 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
104 box.login()
105 emails = grab_addresses(box, mailfolder, date, fLOG=fLOG)
106 box.logout()
108 with open(filemails, "w", encoding="utf8") as f:
109 f.write("\n".join(emails))
110 else:
111 # nothing to do mail already present
112 emails = set(df[columns["mail"]])
114 # we remove empty names
115 df = df[~df[columns["name"]].isnull()].copy()
117 if process_name:
118 df[columns["name"]] = df[columns["name"]].apply(
119 lambda f: process_name(f))
121 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder)
122 proj = ProjectsRepository(folder, fLOG=fLOG)
124 proj = ProjectsRepository.create_folders_from_dataframe(df, folder,
125 col_subject=columns[
126 "subject"], fLOG=fLOG, col_group=columns["group"],
127 col_student=columns[
128 "name"], email_function=emails, skip_if_nomail=False,
129 col_mail=columns["mail"], must_have_email=True, skip_names=skip_names)
130 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len(
131 proj.Groups))
133 # gathers mails
134 email_renderer = EmailMessageRenderer(tmpl=template_email_html_short,
135 fLOG=fLOG)
136 renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer,
137 fLOG=fLOG)
139 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG)
140 box.login()
141 proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder,
142 date=date, overwrite=False, skip_if_empty=True)
144 box.logout()
146 # cleaning files
147 for group in proj.Groups:
148 files = list(proj.enumerate_group_files(group))
149 att = [_ for _ in files if ".html" in _]
150 if len(att) <= 1:
151 fLOG(
152 f"[extract_students_mails_from_gmail_and_stores_in_folders] remove '{group}'")
153 proj.remove_group(group)
155 # unzip files and convert notebooks
156 for group in proj.Groups:
157 proj.unzip_convert(group)
159 summary = os.path.join(folder, "index.html")
160 fLOG(
161 f"[extract_students_mails_from_gmail_and_stores_in_folders] write summary '{summary}'")
162 if os.path.exists(summary):
163 os.remove(summary)
164 proj.write_run_command()
165 proj.write_summary(nolink_if=nolink_if)
167 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename)
168 if os.path.exists(zipfilename):
169 os.remove(zipfilename)
170 proj.zip_group(None, zipfilename,
171 addition=["index.html", "mail_style.css", "emails.txt"])
173 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in '{}'.".format(
174 zipfilenameenc))
175 if os.path.exists(zipfilenameenc):
176 os.remove(zipfilenameenc)
177 encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30)
179 return proj