Coverage for src/pyrsslocal/xmlhelper/xml_tree_node.py: 40%

1""" 

2@file 

3@brief parsing XML 

4""" 

5 

6import copy 

7 

8from pyquickhelper.loghelper.flog import guess_type_list, guess_type_value_type 

9from .xml_utils import escape 

10from .xml_exceptions import XmlException 

11 

12 

13class XMLHandlerDictNode(dict): 

14 """ 

15 Defines a node containing a dictionary. 

16 

17 @var father ancestor 

18 @var name name of the section (tag) 

19 @var buffer value or content of the section 

20 @var level level in the hierarchy 

21 @var other included sections 

22 """ 

23 

24 def __init__(self, father, name, level, root=False): 

25 """ 

26 @param father father 

27 @param name node name 

28 @param level could be infered but still 

29 @param root is it the root 

30 """ 

31 dict.__init__(self) 

32 self.father = father 

33 self.name = name 

34 self.buffer = "" 

35 self.level = level 

36 if father is None and not root: 

37 raise XmlException( 

38 "father is None and root is False, name = %s level = %d" % 

39 (name, level)) 

40 self.other = [] 

41 

42 def __cmp__(self, other): 

43 a, b = id(self), id(other) 

44 if a < b: 

45 return -1 

46 elif a == b: 

47 return 0 

48 else: 

49 return 1 

50 

51 def __lt__(self, other): 

52 return self.__cmp__(other) == -1 

53 

54 def enumerate_on_tag(self, tag, recursive=False): 

55 """ 

56 Enumerates all nodes sharing the same name: tag. 

57 

58 @param tag node name to enumerate on 

59 @param recursive if True, looks into node (name == tag) if there are 

60 sub-node with the same name 

61 @return enumeration on node 

62 """ 

63 if self.name == tag: 

64 yield self 

65 for o in self.other: 

66 if isinstance(o, tuple): 

67 if recursive: 

68 for _ in o[1].enumerate_on_tag(tag): 

69 yield _ 

70 else: 

71 yield o[1] 

72 else: 

73 for _ in o.enumerate_on_tag(tag): 

74 yield _ 

75 

76 def add_xml_content(self, content): 

77 """ 

78 Adds the content of the node itself (and all included nodes). 

79 """ 

80 self.xmlcontent = content 

81 

82 def get_xml_content(self): 

83 """ 

84 @return self.xmlcontent 

85 """ 

86 return self.xmlcontent if "xmlcontent" in self.__dict__ else None 

87 

88 def __str__(self): 

89 """ 

90 usual 

91 """ 

92 mx = 0 

93 for k in self: 

94 mx = max(len(k), mx) 

95 head = self.level * " " 

96 pile = [head + "*" + self.name] 

97 

98 try: 

99 buf = str(self.buffer) \ 

100 if self.buffer[0] in guess_type_value_type() \ 

101 else self.buffer 

102 except IndexError: 

103 buf = str(self.buffer) 

104 

105 if len(buf) > 0: 

106 t = " " * (mx - len("lst") + 1) 

107 ty = self.__dict__.get("conversion_table", {}).get(self.name, "") 

108 if ty != "": 

109 ty = " \t(%s)" % str(ty) 

110 if isinstance(buf, (list, tuple)): 

111 pile.append(head + " lst" + t + ": " + str(repr(buf) + ty)) 

112 else: 

113 pile.append(head + " val" + t + ": " + buf + ty) 

114 

115 for k in sorted(self): 

116 v = self[k] 

117 vs = str(v) if v in guess_type_value_type() or isinstance( 

118 v, 

119 tuple) else v 

120 ty = self.__dict__.get("conversion_table", {}).get(k, "") 

121 if ty != "": 

122 ty = " \t(%s)" % str(ty) 

123 if isinstance(vs, str): 

124 t = " " * (mx - len(k) + 1) 

125 pile.append(head + " " + k + t + ": " + vs + ty) 

126 elif isinstance(vs, list): 

127 t = " " * (mx - len(k) + 1) 

128 pile.append(head + " " + k + t + ": " + str(repr(vs)) + ty) 

129 elif isinstance(vs, tuple): 

130 pile.append("-" + str(vs) + ty) 

131 else: 

132 pile.append(str(vs) + ty) 

133 

134 if len(self.other) > 0: 

135 pile.append(head + " ----") 

136 soro = sorted(copy.copy(self.other)) 

137 for k, v in soro: 

138 temp = str(v) 

139 star = temp.find("*") 

140 if star != -1 and "_othercount" in self.__dict__: 

141 temp = "%s*(%d) %s" % (temp[:star], 

142 self._othercount.get(k, -1), temp[star + 1:]) 

143 pile.append(temp) 

144 

145 return "\n".join(pile) 

146 

147 def strip(self): 

148 """ 

149 Strips the buffer. 

150 """ 

151 self.buffer = self.buffer.strip() 

152 

153 def copy(self): 

154 """ 

155 Gets a copy. 

156 """ 

157 u = XMLHandlerDictNode(self, self.father, self.name, self.level) 

158 u.buffer = self.buffer 

159 u.level = self.level 

160 return u 

161 

162 def set(self, i, v): 

163 """ 

164 Changes the value of a field. 

165 @param i field 

166 @param v new value 

167 """ 

168 if i in self: 

169 if isinstance(v, XMLHandlerDictNode): 

170 self.other.append((i, v)) 

171 return v 

172 else: 

173 raise XmlException( 

174 "unable to append a new string value for an existing field %s:%s" % 

175 (i, v)) 

176 else: 

177 self[i] = v 

178 return self 

179 

180 def is_text_only(self): 

181 """ 

182 Returns True if it only contains text. 

183 """ 

184 if len(self.other) > 0: 

185 return False 

186 if len(self) > 1: 

187 return False 

188 for k, v in self.items(): 

189 if k != self.name: 

190 return False 

191 if not isinstance(v, str): 

192 return False 

193 return True 

194 

195 def rearrange(self, debug=False): 

196 """ 

197 Moves all objects to other. 

198 """ 

199 

200 # check level 

201 if self.father is not None: 

202 self.level = self.father.level + 1 

203 

204 # is is_text_only --> fill buffer, clean the rest 

205 if self.is_text_only() and len(self) == 1: 

206 k = self.keys()[0] 

207 self.buffer = self[k] 

208 self.clear() 

209 return 

210 

211 # values in self.keys also in other --> all in other 

212 # unique values in other and if text --> self 

213 count = {} 

214 for k, v in self.other: 

215 count[k] = 0 

216 v.rearrange() 

217 

218 for k, v in self.other: 

219 count[k] += 1 

220 move = [k for k, v in count.items() if v == 1] 

221 keys = {} 

222 for m in move: 

223 keys[m] = None 

224 mult = [] 

225 rem = [] 

226 i = 0 

227 for k, v in self.other: 

228 if k in keys and v.is_text_only(): 

229 if k in self: 

230 if k not in mult: 

231 tempv = self[k] 

232 if isinstance(tempv, str): 

233 tempv = XMLHandlerDictNode(self, k, self.level + 1) 

234 tempv.buffer = self[k] 

235 mult.append((k, tempv)) 

236 else: 

237 self[k] = v.buffer 

238 rem.append(i) 

239 i += 1 

240 

241 mult.reverse() 

242 for m in mult: 

243 self.other.insert(0, m) 

244 del self[m[0]] 

245 rem.reverse() 

246 for e in rem: 

247 del self.other[e] 

248 

249 # in case of self contains object --> other 

250 rem = [] 

251 for k, v in self.items(): 

252 if not isinstance(v, str) and not isinstance(v, list): 

253 v.rearrange(debug=True) 

254 if not v.is_text_only(): 

255 self.other.append((k, v)) 

256 rem.append(k) 

257 else: 

258 self[k] = v.buffer 

259 

260 for k in rem: 

261 del self[k] 

262 

263 # in case other already contains some objects of the same kind 

264 rem = [] 

265 count = {} 

266 for k, v in self.other: 

267 count[k] = 1 

268 for k, v in self.items(): 

269 if k in count: 

270 if isinstance(v, str): 

271 node = XMLHandlerDictNode(self, k, self.level + 1, False) 

272 node.buffer = v 

273 self.other.append((k, node)) 

274 else: 

275 self.other.append((k, v)) 

276 rem.append(k) 

277 

278 for k in rem: 

279 del self[k] 

280 

281 # last check 

282 if len(self) == 1: 

283 # self.popitem(), strange it works in version 2 

284 k, _ = list(self.items())[0] 

285 if k == self.name: 

286 self.buffer = self[k] 

287 del self[k] 

288 

289 def get_xml_output(self): 

290 """ 

291 @return an XML output (all lines terminated by end_of_line 

292 """ 

293 att = [""] + ["%s=\"%s\"" % 

294 (k, escape(self[k])) for k in sorted(self) if len(self[k]) <= 20] 

295 att = " ".join(att) 

296 lev = max(self.level - 1, 0) 

297 lev = " " * lev 

298 

299 if len(self.other) == 0: 

300 if len(self.buffer) == 0: 

301 return "%s<%s%s />\n" % (lev, self.name, att) 

302 else: 

303 return "%s<%s%s>%s</%s>\n" % (lev, 

304 self.name, att, self.buffer, self.name) 

305 else: 

306 res = ["%s<%s%s>\n" % (lev, self.name, att)] 

307 if len(self.buffer) > 0: 

308 res.append("%s%s\n" % (lev, escape(self.buffer))) 

309 for k in sorted(self): 

310 v = self[k] 

311 if len(v) <= 20: 

312 continue 

313 res.append("%s<%s>\n" % (lev, k)) 

314 res.append("%s%s\n" % (lev, escape(v))) 

315 res.append("%s</%s>\n" % (lev, k)) 

316 

317 other = sorted(copy.copy(self.other)) 

318 

319 for k, v in other: 

320 res.append(v.get_xml_output()) 

321 res.append("%s</%s>\n" % (lev, self.name)) 

322 return "".join(res) 

323 

324 def get_values(self, field): 

325 """ 

326 Gets all values associated to a given field name. 

327 @param field field name 

328 @return list of [ key, value ] 

329 """ 

330 res = [] 

331 if self.name == field: 

332 res.append((("", -1), self.buffer)) 

333 

334 for k, v in self.items(): 

335 if k == field: 

336 res.append(((k, -1), v)) 

337 

338 i = 0 

339 for k, v in self.other: 

340 temp = v.get_values(field) 

341 for a, b in temp: 

342 res.append(((k, i) + a, b)) 

343 i += 1 

344 

345 return res 

346 

347 def get_values_group(self, fields, nb=1): 

348 """ 

349 Gets all values associated to a list of fields 

350 (must come together in a single node, not in *self.other*). 

351 @param fields fields name (list or dictionary) 

352 @param nb at least nb fields must be filled 

353 @return list of dictionaries 

354 """ 

355 res = [] 

356 if self.name in fields: 

357 res.append((self.name, self.buffer)) 

358 

359 for k, v in self.items(): 

360 if k in fields: 

361 res.append((k, v)) 

362 

363 if len(res) >= nb: 

364 temp = {} 

365 for k, v in res: 

366 if k in temp: 

367 raise XmlException("field %s already present in '%s' (full name '%s')" % ( 

368 k, ", ".join(temp.keys()), "/".join(self.get_full_name()))) 

369 temp[k] = v 

370 for f in fields: 

371 if f not in temp: 

372 temp[f] = None 

373 res = [((self.name, -1), temp)] 

374 else: 

375 res = [] 

376 

377 i = 0 

378 for k, v in self.other: 

379 temp = v.get_values_group(fields, nb) 

380 for a, b in temp: 

381 res.append(((k, i) + a, b)) 

382 i += 1 

383 

384 return res 

385 

386 def _convert_into_list(self): 

387 """ 

388 Converts all types into lists. 

389 """ 

390 if isinstance(self.buffer, str): 

391 self.buffer = [self.buffer] 

392 

393 for k in self: 

394 v = self[k] 

395 if isinstance(v, str): 

396 self[k] = [v] 

397 

398 for k, v in self.other: 

399 v._convert_into_list() 

400 

401 def __iadd__(self, other): 

402 """ 

403 Concatenates every information. 

404 @param other other value to concatenate 

405 @return self 

406 """ 

407 self.iadd(other, False, False) 

408 return self 

409 

410 def iadd(self, other, use_list, collapse): 

411 """ 

412 Concatenates every information. 

413 @param other other value to concatenate 

414 @param use_list use a list or not 

415 @param collapse collapse all information 

416 @return self 

417 """ 

418 if self.name != other.name: 

419 raise XmlException("the two names should be equal %s != %s full names (%s != %s)" % ( 

420 self.name, other.name, "/".join(self.get_full_name()), "/".join(other.get_full_name()))) 

421 

422 # _othercount 

423 if "_othercount" not in self.__dict__: 

424 self._othercount = {} 

425 

426 # next 

427 if use_list: 

428 self._convert_into_list() 

429 

430 if use_list: 

431 if isinstance(other.buffer, list): 

432 self.buffer.extend(other.buffer) 

433 else: 

434 self.buffer.append(other.buffer) 

435 else: 

436 self.buffer += other.buffer 

437 

438 for k, v in other.items(): 

439 if k not in self: 

440 if use_list: 

441 if isinstance(v, list): 

442 self[k] = v 

443 else: 

444 self[k] = [v] 

445 else: 

446 self[k] = v 

447 else: 

448 if use_list: 

449 if isinstance(v, list): 

450 self[k].extend(v) 

451 else: 

452 self[k].append(v) 

453 else: 

454 self[k] += v 

455 

456 # count the number 

457 selfcount = {} 

458 othcount = {} 

459 for k, v in self.other: 

460 if k in selfcount: 

461 selfcount[k] += 1 

462 else: 

463 selfcount[k] = 1 

464 self._othercount[k] = max(self._othercount.get(k, 0), selfcount[k]) 

465 

466 for k, v in other.other: 

467 if k in othcount: 

468 othcount[k] += 1 

469 else: 

470 othcount[k] = 1 

471 self._othercount[k] = max(self._othercount.get(k, 0), othcount[k]) 

472 

473 if "_othercount" in other.__dict__: 

474 for k, v in other._othercount.items(): 

475 self._othercount[k] = max(self._othercount.get(k, 0), v) 

476 

477 # iadd single elements + append other from others 

478 for node in other.other: 

479 ok = False 

480 for n in self.other: 

481 if node[0] != n[0]: 

482 continue 

483 key = node[0] 

484 if selfcount.get(key, 0) == othcount.get(key, 0) == 1: 

485 n[1].iadd(node[1], use_list=use_list, collapse=collapse) 

486 ok = True 

487 break 

488 

489 if not ok: 

490 nt = copy.deepcopy(node) 

491 nt[1].parent = self 

492 nt[1]._build_othercount() 

493 if use_list: 

494 nt[1]._convert_into_list() 

495 if collapse: 

496 nt[1]._collapse(use_list) 

497 self.other.append(nt) 

498 k = node[0] 

499 

500 # count 

501 count = {} 

502 for k, v in self.other: 

503 count[k] = count.get(k, 0) + 1 

504 

505 # transfert from dict self if a key is present in self.other 

506 rem = [] 

507 for k, v in self.items(): 

508 if k in count: 

509 tn = XMLHandlerDictNode(self, k, self.level + 1, False) 

510 tn._build_othercount() 

511 if use_list: 

512 tn._convert_into_list() 

513 tn.buffer = [v] if use_list and not isinstance(v, list) else v 

514 self.other.append((k, tn)) 

515 rem.append(k) 

516 self._othercount[k] = self._othercount.get(k, 0) + 1 

517 for k in rem: 

518 del self[k] 

519 

520 # count again 

521 count = {} 

522 for k, v in self.other: 

523 if isinstance(v, str): 

524 count[k, 0] = count.get((k, 0), 0) + 1 

525 else: 

526 count[k, 1] = count.get((k, 1), 0) + 1 

527 

528 # string to object 

529 for i, tu in enumerate(self.other): 

530 k, v = tu 

531 if isinstance(v, str) and count.get((k, 1), 0) > 0: 

532 tn = XMLHandlerDictNode(self, k, self.level + 1, False) 

533 tn._build_othercount() 

534 tn.buffer = [v] if use_list else v 

535 self.other[i] = (k, tn) 

536 

537 # collapsing 

538 if collapse: 

539 self._collapse(use_list) 

540 

541 def _build_othercount(self): 

542 """ 

543 Builds *_othercount* when not present. 

544 """ 

545 if "_othercount" not in self.__dict__: 

546 self._othercount = {} 

547 for k, v in self.other: 

548 self._othercount[k] = self._othercount.get(k, 0) + 1 

549 v._build_othercount() 

550 

551 def _collapse(self, use_list): 

552 """ 

553 Collapses together all fields having the same name 

554 in the member other. 

555 @warning it should be called after iadd 

556 """ 

557 names = {} 

558 for k, v in self.other: 

559 if k in names: 

560 names[k].append(v) 

561 else: 

562 names[k] = [v] 

563 

564 del self.other[:] 

565 for k, lv in names.items(): 

566 if len(lv) > 1: 

567 self._othercount[k] = max(self._othercount.get(k, 0), len(lv)) 

568 for i in range(1, len(lv)): 

569 lv[0].iadd(lv[i], use_list=use_list, collapse=True) 

570 self.other.append((k, lv[0])) 

571 else: 

572 lv[0]._collapse(use_list) 

573 self.other.append((k, lv[0])) 

574 #self._check_ (False) 

575 

576 def _check_(self, add_root_id): 

577 """some checking 

578 """ 

579 count = {} 

580 # if add_root_id and "add_root_id" not in self.__dict__ : 

581 # fLOG (self) 

582 # raise ValueError ("unable to find add_root_id in '%s'" % self.get_full_name ()) 

583 if "_othercount" not in self.__dict__: 

584 raise XmlException("unable to find _othercount in '%s'" % 

585 "/".join(self.get_full_name())) 

586 for k, v in self.other: 

587 count[k] = count.get(k, 0) + 1 

588 if len(count) > 0: 

589 if max(count.values()) > 1: 

590 raise XmlException("max (count.values ()) > 1 in '%s' \nexp: %s" % ( 

591 "/".join(self.get_full_name()), str(count))) 

592 for k, v in self.other: 

593 if isinstance(v, list): 

594 for _ in v: 

595 _._check_(add_root_id) 

596 else: 

597 v._check_(add_root_id) 

598 

599 def _guess_type(self, tolerance=0.01, utf8=False): 

600 """ 

601 Replaces all values in the object. 

602 @param tolerance @see fn guess_type_list 

603 @param utf8 if True, all types are str 

604 @warning it should be called after _collapse 

605 """ 

606 self.buffer = (str, 10) if utf8 else guess_type_list(self.buffer) 

607 for k in self: 

608 self[k] = (str, 10) if utf8 else guess_type_list(self[k]) 

609 for k, v in self.other: 

610 v._guess_type(utf8) 

611 

612 def find_node(self, li): 

613 """ 

614 @param li list of names 

615 @return a list of nodes which correspond to the list of names 

616 """ 

617 node = [self] 

618 for el in li: 

619 temp = [] 

620 for n in node: 

621 for k, v in n.other: 

622 if k == el: 

623 temp.append(v) 

624 node = temp 

625 

626 return node 

627 

628 def find_node_value(self, li): 

629 """ 

630 @param li list of names 

631 @return a list of values 

632 """ 

633 path = li if isinstance(li, list) else li.split("/") 

634 way, last = path[:-1], path[-1] 

635 

636 if len(way) > 0 and way[0] == self.name: 

637 del way[0] 

638 

639 res = [] 

640 node = self.find_node(way) 

641 for n in node: 

642 if last == "_": 

643 res.append(n.buffer) 

644 else: 

645 res.append(n.get(last, None)) 

646 return res 

647 

648 def get_full_name(self): 

649 """ 

650 @return the list of self.name from all parents 

651 """ 

652 li = [self.name] 

653 node = self 

654 while node.father is not None: 

655 node = node.father 

656 li.append(node.name) 

657 li.reverse() 

658 return li 

659 

660 def _log_error(self): 

661 """ 

662 logs an object from the root if not already done 

663 """ 

664 root = self.get_root() 

665 if "_logged" in root.__dict__: 

666 return 

667 root._logged = True 

668 

669 def _adopt_table(self, tbl, exception): 

670 """ 

671 Adopts a table built on anoher object. 

672 @param tbl same kind of node but including members: 

673 - table 

674 - conversion_table 

675 @param exception if True, raises an exception, log otherwise 

676 

677 @warning The method could change the object itself if it does not fit. 

678 

679 @warning The method adds members 'conversion_table', 'add_root_id' 

680 """ 

681 self.conversion_table = tbl.conversion_table # field conversion 

682 self.add_root_id = tbl.add_root_id 

683 

684 memo = {} 

685 for k, v in tbl.other: 

686 memo[k] = v 

687 

688 rem = [] 

689 for k in self: 

690 if k not in tbl.conversion_table: 

691 if len(self[k]) == 0: 

692 continue 

693 if k not in memo: 

694 self._log_error() 

695 if exception: 

696 raise XmlException( 

697 "a field '%s' is not provided by the reference (path: %s)\nmemo.keys(): %s" % 

698 (k, "/".join( 

699 self.get_full_name()), str( 

700 memo.keys()))) 

701 tn = XMLHandlerDictNode(self, k, self.level + 1, False) 

702 v = self[k] 

703 tn.buffer = v 

704 self.other.append((k, tn)) 

705 rem.append(k) 

706 

707 for k in rem: 

708 del self[k] 

709 

710 count = {} 

711 for k, v in self.other: 

712 if k in count: 

713 count[k] += 1 

714 else: 

715 count[k] = 1 

716 

717 # checking if relation 11 are ok with this object 

718 if "_othercount" not in tbl.__dict__: 

719 raise XmlException("we expect _othercount to be here") 

720 for k, v in count.items(): 

721 if k not in tbl._othercount: 

722 self._log_error() 

723 if exception: 

724 raise XmlException("unable to find field '%s' (1:n) in path '%s'" % ( 

725 k, "/".join(self.get_full_name()))) 

726 elif v > 1 and tbl._othercount[k] <= 1: # pylint: disable=R1716 

727 self._log_error() 

728 if exception: 

729 raise XmlException("we expect a relation 1:1 for field '%s' in path '%s'" % ( 

730 k, "/".join(self.get_full_name()))) 

731 

732 # next 

733 for k, v in self.other: 

734 if k not in memo: 

735 # fLOG("ERROR: unable to find field '%s' (1:n) in path '%s'" % 

736 # (k, "/".join(self.get_full_name()))) 

737 self._log_error() 

738 else: 

739 v._adopt_table(memo[k], exception=exception) 

740 

741 def _transfer_to_object(self, root=True, exception=True): 

742 """ 

743 Transfers values to the object *self.table*. 

744 @param root if True, it is the root 

745 @param exception if True, raise ValueError 

746 @return the value, dictionary of dictionary of list sometimes... 

747 

748 @warning We assume fid is the key. 

749 

750 @warning If root.add_root_id is True, is assumes column root_id is root.add_root_id 

751 """ 

752 attr = {} 

753 try: 

754 v = self.conversion_table[self.name](self.buffer) 

755 except Exception as ex: 

756 if "conversion_table" not in self.__dict__: 

757 # fLOG("ERROR: unable to find conversion_table for field ", 

758 # self.name, 

759 # " in node " + "/".join(self.get_full_name())) 

760 self._log_error() 

761 #if exception : raise ValueError ("fail to convert value for field " + k) 

762 elif len(self.buffer) > 0: