Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief A couple of functons which automates everything. 

4""" 

5 

6import os 

7import pandas 

8from pyquickhelper.loghelper import fLOG 

9from pyquickhelper.filehelper import encrypt_stream 

10from pymmails import MailBoxImap, EmailMessageRenderer, EmailMessageListRenderer 

11from pymmails.render.email_message_style import template_email_html_short 

12from .projects_repository import ProjectsRepository 

13from .mail_helper import grab_addresses 

14 

15 

16def extract_students_mails_from_gmail_and_stores_in_folders(folder=".", filemails="emails.txt", 

17 user=None, pwd=None, server="imap.gmail.com", 

18 mailfolder=[ 

19 "ensae/ENSAE_2016_3A"], 

20 date="1-Jan-2016", zipfilename="projet_3A_2016.zip", 

21 zipencpwd=b"sixteenbyteskeys", dataframe=None, 

22 columns={ 

23 "name": "nom_prenom", "group": "groupe", "subject": "sujet"}, 

24 skip_names=None, process_name=None, 

25 title="List of emails", nolink_if=None, fLOG=fLOG): 

26 """ 

27 The scenario is the following: 

28 

29 * You are the teacher. 

30 * Students started their projects at date *t*. 

31 * They can work alone or by group. 

32 * They send mails, you reply. 

33 * Their address mail follows the convention: ``<first name>.<last name>@anything`` 

34 so it is to associate a mail address to a student name. 

35 * You move every mail you received in a separate folder in your inbox. 

36 * Sometime, you send a mail to everybody. 

37 * Finally they send their project with attachments. 

38 * You want to store everything (mails and attachements) in folders, one per group. 

39 * You want a summary of what was received. 

40 * You want to build a zip file to share their work with others teachers. 

41 * You want to update the folder if a new mail was sent. 

42 

43 This function looks into a folder of your inbox and grabs every mails and 

44 attachements from a groups of students. 

45 

46 @param folder where to store the results 

47 @param filemails files used to store students address, 

48 the operation is done once, remove the file 

49 to force the function to rebuild the information. 

50 @param user user of the gmail inbox 

51 @param pwd password of the gmail inbox 

52 @param server gmail server, it should be ``"imap.gmail.com"``, 

53 it works with others mail servers using the *IMAP* protocol 

54 @param mailfolder folder in your inbox to look into, 

55 there can be several 

56 @param date when to start looking (do not change the format, 

57 look at the default value) 

58 @param zipfilename name of the zip file to create 

59 @param zipencpwd the zip file is also encrypted for a safer share with this key 

60 and function `encrypt_stream <http://www.xavierdupre.fr/app/pyquickhelper/helpsphinx/ 

61 pyquickhelper/filehelper/encryption.html#pyquickhelper.filehelper.encryption.encrypt_stream>`_. 

62 @param dataframe dataframe which contains the definition of students groups 

63 @param columns columns the function will look into, students names, group definition 

64 (a unique number for all students in the same group), subject 

65 @param skip_names list of names to skip 

66 @param process_name to operate a transformation before matching students names with 

67 their emails 

68 @param title each group folder contains a html file connecting them, 

69 this is its title 

70 @param nolink_if The summary extracts links from url, it skips the urls which 

71 contains on the substrings included in that list (None to use a default set) 

72 @param fLOG logging function 

73 @return @see cl ProjectsRepository 

74 

75 By default, Gmail does not let you programmatically access you own inbox, 

76 you need to modify your gmail parameters to let this function do so. 

77 """ 

78 folder = os.path.abspath(".") 

79 filemails = os.path.join(folder, filemails) 

80 zipfilename = os.path.join(folder, zipfilename) 

81 zipfilenameenc = zipfilename + ".enc" 

82 

83 # load the groups 

84 if isinstance(dataframe, pandas.DataFrame): 

85 df = dataframe 

86 elif dataframe.endswith("xlsx"): 

87 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] read dataframe", dataframe) 

88 df = pandas.read_excel(dataframe, engine='openpyxl') 

89 else: 

90 df = pandas.read_csv(dataframe, sep="\t", encoding="utf8") 

91 

92 # check mails 

93 if "mail" not in columns: 

94 if os.path.exists(filemails): 

95 fLOG( 

96 "[extract_students_mails_from_gmail_and_stores_in_folders] read addresses from ", filemails) 

97 with open(filemails, "r", encoding="utf8") as f: 

98 lines = f.readlines() 

99 emails = [li.strip("\r\t\n ") for li in lines] 

100 else: 

101 fLOG( 

102 "[extract_students_mails_from_gmail_and_stores_in_folders] mine address ") 

103 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG) 

104 box.login() 

105 emails = grab_addresses(box, mailfolder, date, fLOG=fLOG) 

106 box.logout() 

107 

108 with open(filemails, "w", encoding="utf8") as f: 

109 f.write("\n".join(emails)) 

110 else: 

111 # nothing to do mail already present 

112 emails = set(df[columns["mail"]]) 

113 

114 # we remove empty names 

115 df = df[~df[columns["name"]].isnull()].copy() 

116 

117 if process_name: 

118 df[columns["name"]] = df[columns["name"]].apply( 

119 lambda f: process_name(f)) 

120 

121 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] create groups folders in", folder) 

122 proj = ProjectsRepository(folder, fLOG=fLOG) 

123 

124 proj = ProjectsRepository.create_folders_from_dataframe(df, folder, 

125 col_subject=columns[ 

126 "subject"], fLOG=fLOG, col_group=columns["group"], 

127 col_student=columns[ 

128 "name"], email_function=emails, skip_if_nomail=False, 

129 col_mail=columns["mail"], must_have_email=True, skip_names=skip_names) 

130 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] nb groups", len( 

131 proj.Groups)) 

132 

133 # gathers mails 

134 email_renderer = EmailMessageRenderer(tmpl=template_email_html_short, 

135 fLOG=fLOG) 

136 renderer = EmailMessageListRenderer(title=title, email_renderer=email_renderer, 

137 fLOG=fLOG) 

138 

139 box = MailBoxImap(user, pwd, server, ssl=True, fLOG=fLOG) 

140 box.login() 

141 proj.dump_group_mails(renderer, group=None, mailbox=box, subfolder=mailfolder, 

142 date=date, overwrite=False, skip_if_empty=True) 

143 

144 box.logout() 

145 

146 # cleaning files 

147 for group in proj.Groups: 

148 files = list(proj.enumerate_group_files(group)) 

149 att = [_ for _ in files if ".html" in _] 

150 if len(att) <= 1: 

151 fLOG( 

152 "[extract_students_mails_from_gmail_and_stores_in_folders] remove '{}'".format(group)) 

153 proj.remove_group(group) 

154 

155 # unzip files and convert notebooks 

156 for group in proj.Groups: 

157 proj.unzip_convert(group) 

158 

159 summary = os.path.join(folder, "index.html") 

160 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] write summary '{}'".format(summary)) 

161 if os.path.exists(summary): 

162 os.remove(summary) 

163 proj.write_run_command() 

164 proj.write_summary(nolink_if=nolink_if) 

165 

166 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] zip everything in", zipfilename) 

167 if os.path.exists(zipfilename): 

168 os.remove(zipfilename) 

169 proj.zip_group(None, zipfilename, 

170 addition=["index.html", "mail_style.css", "emails.txt"]) 

171 

172 fLOG("[extract_students_mails_from_gmail_and_stores_in_folders] encrypt the zip file in '{}'.".format( 

173 zipfilenameenc)) 

174 if os.path.exists(zipfilenameenc): 

175 os.remove(zipfilenameenc) 

176 encrypt_stream(zipencpwd, zipfilename, zipfilenameenc, chunksize=2 ** 30) 

177 

178 return proj