Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2@file
3@brief API to move files
4"""
5import json
6import hashlib
7from io import StringIO
8from ..loghelper.flog import noLOG
9from ..loghelper.convert_helper import str2datetime, datetime2str
12class TransferAPI_FileInfo:
13 """
14 Keeps tracks of transferred files.
15 """
17 def __init__(self, name, pieces, last_update):
18 """
19 Information about a transferred file.
21 @param name name of the file
22 @param pieces list of pieces contributing to the file
23 @param last_update last_update
24 """
25 self.name = name
26 self.pieces = pieces
27 self.last_update = last_update
29 def __str__(self):
30 """
31 usual
32 """
33 mes = "[%s,#%d,%s]" % (self.name, len(self.pieces), self.last_update)
34 return mes
36 def add_piece(self, piece):
37 """
38 Adds a piece.
40 @param piece add piece
41 """
42 self.pieces.append(piece)
44 @staticmethod
45 def read_json(s):
46 """
47 Retrieves information from a :epkg:`json` string.
48 """
49 st = StringIO(s)
50 js = json.load(st)
51 js[2] = str2datetime(js[2])
52 return TransferAPI_FileInfo(*js)
54 def to_json(self):
55 """
56 Serializes this class info JSON.
57 """
58 li = [self.name, self.pieces, datetime2str(self.last_update)]
59 return json.dumps(li)
62class TransferAPI:
63 """
64 Defines an API to transfer files over a remote location.
65 """
67 def __init__(self, fLOG=noLOG):
68 """
69 @param fLOG logging function
70 """
71 self.fLOG = fLOG if fLOG else noLOG
73 def transfer(self, path, data):
74 """
75 It assumes a data holds in memory,
76 tansfer data to path.
78 @param data bytes
79 @param path path to remove location
80 @return boolean
81 """
82 raise NotImplementedError() # pragma: no cover
84 def retrieve(self, path, exc=True):
85 """
86 Retrieves data from path.
88 @param path remove location
89 @param exc keep exception
90 @return data
91 """
92 raise NotImplementedError() # pragma: no cover
94 def retrieve_mapping(self, decrypt):
95 """
96 Returns the mapping.
98 @param decrypt decrypt function
99 @return list of key,value pair
100 """
101 m = self.retrieve("__mapping__", exc=False)
102 if m is None:
103 return {}
104 return TransferAPI.bytes2mapping(m)
106 def transfer_mapping(self, mapping, encrypt, filename=None):
107 """
108 Transfers the mapping.
110 @param mapping mapping
111 @param encrypt encryption function
112 @param filename local filename
113 @return boolean
114 """
115 b = TransferAPI.mapping2bytes(mapping)
116 if filename is not None:
117 with open(filename, "wb") as f:
118 f.write(b)
119 return self.transfer("__mapping__", b)
121 @staticmethod
122 def mapping2bytes(mapping):
123 """
124 Serializes a mapping.
126 @param mapping dictionary { str, @see cl TransferAPI_FileInfo }
127 @return bytes
128 """
129 rows = []
130 for k, v in sorted(mapping.items()):
131 r = "{0}\t{1}".format(k, v.to_json())
132 rows.append(r)
133 return "\n".join(rows).encode()
135 @staticmethod
136 def bytes2mapping(byt):
137 """
138 Deserializes a mapping.
140 @param byt bytes
141 @return dictionary { str, @see cl TransferAPI_FileInfo }
142 """
143 lines = byt.decode().split("\n")
144 res = {}
145 for line in lines:
146 spl = line.split("\t")
147 res[spl[0]] = TransferAPI_FileInfo.read_json("\n".join(spl[1:]))
148 return res
150 @staticmethod
151 def checksum_md5(data):
152 """
153 Computes MD5 for a file.
155 @param data some data
156 @return string
157 """
158 zero = hashlib.md5()
159 zero.update(data)
160 return zero.hexdigest()
162 def get_remote_path(self, data, name, piece=0):
163 """
164 Produces a remote path.
166 @param data binary data to transfer (to be hashed)
167 @param name local name
168 @param piece pieces
169 @return remote path
171 *~ hash of everything*
172 """
173 m1 = TransferAPI.checksum_md5(name.encode() + str(piece).encode())
174 m2 = TransferAPI.checksum_md5(data)
175 return m1 + "_" + m2
178class MockTransferAPI(TransferAPI):
179 """
180 Class used for unit test purposes, simple key, value storage.
181 """
183 def __init__(self, fLOG=noLOG):
184 """
185 @param fLOG logging function
186 """
187 TransferAPI.__init__(self, fLOG)
188 self._storage = {}
190 def transfer(self, path, data):
191 """
192 It assumes a data holds in memory,
193 tansfer data to path.
195 @param data bytes
196 @param path path to remove location
197 @return boolean
198 """
199 self._storage[path] = data
200 return True
202 def retrieve(self, path, exc=True):
203 """
204 Retrieves data from path.
206 @param path remove location
207 @param exc keep exception
208 @return data
209 """
210 if exc:
211 return self._storage[path]
212 else:
213 try:
214 return self._storage[path]
215 except KeyError:
216 return None