Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1""" 

2@file 

3@brief API to move files 

4""" 

5import json 

6import hashlib 

7from io import StringIO 

8from ..loghelper.flog import noLOG 

9from ..loghelper.convert_helper import str2datetime, datetime2str 

10 

11 

12class TransferAPI_FileInfo: 

13 """ 

14 Keeps tracks of transferred files. 

15 """ 

16 

17 def __init__(self, name, pieces, last_update): 

18 """ 

19 Information about a transferred file. 

20 

21 @param name name of the file 

22 @param pieces list of pieces contributing to the file 

23 @param last_update last_update 

24 """ 

25 self.name = name 

26 self.pieces = pieces 

27 self.last_update = last_update 

28 

29 def __str__(self): 

30 """ 

31 usual 

32 """ 

33 mes = "[%s,#%d,%s]" % (self.name, len(self.pieces), self.last_update) 

34 return mes 

35 

36 def add_piece(self, piece): 

37 """ 

38 Adds a piece. 

39 

40 @param piece add piece 

41 """ 

42 self.pieces.append(piece) 

43 

44 @staticmethod 

45 def read_json(s): 

46 """ 

47 Retrieves information from a :epkg:`json` string. 

48 """ 

49 st = StringIO(s) 

50 js = json.load(st) 

51 js[2] = str2datetime(js[2]) 

52 return TransferAPI_FileInfo(*js) 

53 

54 def to_json(self): 

55 """ 

56 Serializes this class info JSON. 

57 """ 

58 li = [self.name, self.pieces, datetime2str(self.last_update)] 

59 return json.dumps(li) 

60 

61 

62class TransferAPI: 

63 """ 

64 Defines an API to transfer files over a remote location. 

65 """ 

66 

67 def __init__(self, fLOG=noLOG): 

68 """ 

69 @param fLOG logging function 

70 """ 

71 self.fLOG = fLOG if fLOG else noLOG 

72 

73 def transfer(self, path, data): 

74 """ 

75 It assumes a data holds in memory, 

76 tansfer data to path. 

77 

78 @param data bytes 

79 @param path path to remove location 

80 @return boolean 

81 """ 

82 raise NotImplementedError() # pragma: no cover 

83 

84 def retrieve(self, path, exc=True): 

85 """ 

86 Retrieves data from path. 

87 

88 @param path remove location 

89 @param exc keep exception 

90 @return data 

91 """ 

92 raise NotImplementedError() # pragma: no cover 

93 

94 def retrieve_mapping(self, decrypt): 

95 """ 

96 Returns the mapping. 

97 

98 @param decrypt decrypt function 

99 @return list of key,value pair 

100 """ 

101 m = self.retrieve("__mapping__", exc=False) 

102 if m is None: 

103 return {} 

104 return TransferAPI.bytes2mapping(m) 

105 

106 def transfer_mapping(self, mapping, encrypt, filename=None): 

107 """ 

108 Transfers the mapping. 

109 

110 @param mapping mapping 

111 @param encrypt encryption function 

112 @param filename local filename 

113 @return boolean 

114 """ 

115 b = TransferAPI.mapping2bytes(mapping) 

116 if filename is not None: 

117 with open(filename, "wb") as f: 

118 f.write(b) 

119 return self.transfer("__mapping__", b) 

120 

121 @staticmethod 

122 def mapping2bytes(mapping): 

123 """ 

124 Serializes a mapping. 

125 

126 @param mapping dictionary { str, @see cl TransferAPI_FileInfo } 

127 @return bytes 

128 """ 

129 rows = [] 

130 for k, v in sorted(mapping.items()): 

131 r = "{0}\t{1}".format(k, v.to_json()) 

132 rows.append(r) 

133 return "\n".join(rows).encode() 

134 

135 @staticmethod 

136 def bytes2mapping(byt): 

137 """ 

138 Deserializes a mapping. 

139 

140 @param byt bytes 

141 @return dictionary { str, @see cl TransferAPI_FileInfo } 

142 """ 

143 lines = byt.decode().split("\n") 

144 res = {} 

145 for line in lines: 

146 spl = line.split("\t") 

147 res[spl[0]] = TransferAPI_FileInfo.read_json("\n".join(spl[1:])) 

148 return res 

149 

150 @staticmethod 

151 def checksum_md5(data): 

152 """ 

153 Computes MD5 for a file. 

154 

155 @param data some data 

156 @return string 

157 """ 

158 zero = hashlib.md5() 

159 zero.update(data) 

160 return zero.hexdigest() 

161 

162 def get_remote_path(self, data, name, piece=0): 

163 """ 

164 Produces a remote path. 

165 

166 @param data binary data to transfer (to be hashed) 

167 @param name local name 

168 @param piece pieces 

169 @return remote path 

170 

171 *~ hash of everything* 

172 """ 

173 m1 = TransferAPI.checksum_md5(name.encode() + str(piece).encode()) 

174 m2 = TransferAPI.checksum_md5(data) 

175 return m1 + "_" + m2 

176 

177 

178class MockTransferAPI(TransferAPI): 

179 """ 

180 Class used for unit test purposes, simple key, value storage. 

181 """ 

182 

183 def __init__(self, fLOG=noLOG): 

184 """ 

185 @param fLOG logging function 

186 """ 

187 TransferAPI.__init__(self, fLOG) 

188 self._storage = {} 

189 

190 def transfer(self, path, data): 

191 """ 

192 It assumes a data holds in memory, 

193 tansfer data to path. 

194 

195 @param data bytes 

196 @param path path to remove location 

197 @return boolean 

198 """ 

199 self._storage[path] = data 

200 return True 

201 

202 def retrieve(self, path, exc=True): 

203 """ 

204 Retrieves data from path. 

205 

206 @param path remove location 

207 @param exc keep exception 

208 @return data 

209 """ 

210 if exc: 

211 return self._storage[path] 

212 else: 

213 try: 

214 return self._storage[path] 

215 except KeyError: 

216 return None