Coverage for src/pyrsslocal/simple_server/simple_server_custom.py: 59%

270 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2023-02-02 02:59 +0100

1""" 

2@file 

3@brief This modules contains a class which implements a simple server. 

4""" 

5 

6import sys 

7import os 

8import subprocess 

9import copy 

10import io 

11import getpass 

12from urllib.parse import urlparse, parse_qs 

13from io import StringIO 

14from threading import Thread 

15from http.server import BaseHTTPRequestHandler, HTTPServer 

16from pyquickhelper.loghelper import fLOG 

17from pyquickhelper.filehelper import get_url_content_timeout 

18from .html_script_parser import HTMLScriptParser, HTMLScriptParserRemove 

19from .html_string import html_footer, html_header, html_debug_string 

20 

21 

22def get_path_javascript(): 

23 """ 

24 *pyrsslocal* contains some javascript script, it adds the paths 

25 to the paths where content will be looked for. 

26 

27 @return a path 

28 """ 

29 filepath = os.path.split(__file__)[0] 

30 jspath = os.path.normpath( 

31 os.path.abspath( 

32 os.path.join( 

33 filepath, 

34 "..", 

35 "javascript"))) 

36 if not os.path.exists(jspath): 

37 raise FileNotFoundError(jspath) 

38 return jspath 

39 

40 

41class SimpleHandler(BaseHTTPRequestHandler): 

42 """ 

43 Defines a simple handler used by *HTTPServer*. 

44 Firefox works better for local files. 

45 

46 This class provides the following function associated to ``/localfile``: 

47 

48 * if the url is ``http://localhost:port/localfile/<filename>``, it display this file 

49 * you add a path parameter: ``http://localhost:port/localfile/<filename>?path=<path>`` 

50 to tell the service to look into a different folder 

51 * you add a parameter ``&execute=False`` for python script if you want to display them, not to run them. 

52 * you can add a parameter ``&keep``, the class retains the folder and will look further files in this list 

53 

54 See `Python documentation <http://docs.python.org/3/library/http.server.html>`_ 

55 

56 @warning Some information about pathes are stored in a unique queue but it should be done in cookie or in session data. 

57 An instance of SimpleHandler is created for each session and it is better to assume 

58 you cannot add member to this class. 

59 """ 

60 

61 # this queue will keep some pathes which should be stored in session 

62 # information or in cookies 

63 queue_pathes = [] 

64 javascript_path = get_path_javascript() 

65 

66 def add_path(self, p): 

67 """ 

68 Adds a local path to the list of path to watch. 

69 @param p local path to data 

70 

71 *Python* documentation says list are proctected against multithreads (concurrent accesses). 

72 """ 

73 if p not in SimpleHandler.queue_pathes: 

74 SimpleHandler.queue_pathes.append(p) 

75 

76 def get_pathes(self): 

77 """ 

78 Returns a list of local path where to look for a local file. 

79 @return a list of pathes 

80 """ 

81 return copy.copy(SimpleHandler.queue_pathes) 

82 

83 def __init__(self, request, client_address, server): 

84 """ 

85 Regular constructor, an instance is created for each request, 

86 do not store any data for a longer time than a request. 

87 """ 

88 BaseHTTPRequestHandler.__init__(self, request, client_address, server) 

89 

90 def log_message(self, format, *args): # pylint: disable=W0622 

91 """ 

92 Logs an arbitrary message. Overloads the original method. 

93 

94 This is used by all other logging functions. Override 

95 it if you have specific logging wishes. 

96 

97 The first argument, FORMAT, is a format string for the 

98 message to be logged. If the format string contains 

99 any % escapes requiring parameters, they should be 

100 specified as subsequent arguments (it's just like 

101 printf!). 

102 

103 The client ip and current date/time are prefixed to 

104 every message. 

105 """ 

106 self.private_LOG("- %s - %s\n" % 

107 (self.address_string(), 

108 format % args)) 

109 

110 def LOG(self, *args): 

111 """ 

112 To log, it appends various information about the id address... 

113 @param args string to LOG or list of strings to LOG 

114 """ 

115 self.private_LOG("- %s -" % 

116 (self.address_string(),), 

117 *args) 

118 

119 def private_LOG(self, *s): 

120 """ 

121 To log 

122 @param s string to LOG or list of strings to LOG 

123 """ 

124 fLOG(*s) 

125 

126 def do_GET(self): 

127 """ 

128 What to do is case of GET request. 

129 """ 

130 parsed_path = urlparse(self.path) 

131 self.serve_content(parsed_path, "GET") 

132 # self.wfile.close() 

133 

134 def do_POST(self): 

135 """ 

136 What to do is case of POST request. 

137 """ 

138 parsed_path = urlparse(self.path) 

139 self.serve_content(parsed_path) 

140 # self.wfile.close() 

141 

142 def do_redirect(self, path="/index.html"): 

143 """ 

144 Redirection when url is just the website. 

145 @param path path to redirect to (a string) 

146 """ 

147 self.send_response(301) 

148 self.send_header('Location', path) 

149 self.end_headers() 

150 

151 def get_ftype(self, path): 

152 """ 

153 Defines the header to send (type of files) based on path. 

154 @param path location (a string) 

155 @return htype, ftype (html, css, ...) 

156 """ 

157 htype = '' 

158 ftype = '' 

159 

160 if path.endswith('.js'): 

161 htype = 'application/javascript' 

162 ftype = 'r' 

163 elif path.endswith('.css'): 

164 htype = 'text/css' 

165 ftype = 'r' 

166 elif path.endswith('.html'): 

167 htype = 'text/html' 

168 ftype = 'r' 

169 elif path.endswith('.py'): 

170 htype = 'text/html' 

171 ftype = 'execute' 

172 elif path.endswith('.png'): 

173 htype = 'image/png' 

174 ftype = 'rb' 

175 elif path.endswith('.jpg'): 

176 htype = 'image/jpeg' 

177 ftype = 'rb' 

178 elif path.endswith('.jepg'): 

179 htype = 'image/jpeg' 

180 ftype = 'rb' 

181 elif path.endswith('.ico'): 

182 htype = 'image/x-icon' 

183 ftype = 'rb' 

184 elif path.endswith('.gif'): 

185 htype = 'image/gif' 

186 ftype = 'rb' 

187 

188 return htype, ftype 

189 

190 def send_headers(self, path): 

191 """ 

192 Defines the header to send (type of files) based on path. 

193 @param path location (a string) 

194 @return type (html, css, ...) 

195 """ 

196 htype, ftype = self.get_ftype(path) 

197 

198 if htype != '': 

199 self.send_header('Content-type', htype) 

200 self.end_headers() 

201 else: 

202 self.send_header('Content-type', 'text/plain') 

203 self.end_headers() 

204 return ftype 

205 

206 def get_file_content(self, localpath, ftype, path=None): 

207 """ 

208 Returns the content of a local file. The function also looks into 

209 folders in ``self.__pathes`` to see if the file can be found in one of the 

210 folder when not found in the first one. 

211 

212 @param localpath local filename 

213 @param ftype r or rb 

214 @param path if != None, the filename will be path/localpath 

215 @return content 

216 """ 

217 if path is not None: 

218 tlocalpath = os.path.join(path, localpath) 

219 else: 

220 tlocalpath = localpath 

221 

222 if not os.path.exists(tlocalpath): 

223 for p in self.get_pathes(): 

224 self.LOG("trying ", p) 

225 tloc = os.path.join(p, localpath) 

226 if os.path.exists(tloc): 

227 tlocalpath = tloc 

228 break 

229 

230 if not os.path.exists(tlocalpath): 

231 self.send_error(404) 

232 content = "unable to find file " + localpath 

233 self.LOG(content) 

234 return content 

235 

236 if ftype in ("r", "execute"): 

237 self.LOG("reading file ", tlocalpath) 

238 with open(tlocalpath, "r", encoding="utf8") as f: 

239 return f.read() 

240 else: 

241 self.LOG("reading file ", tlocalpath) 

242 with open(tlocalpath, "rb") as f: 

243 return f.read() 

244 

245 def execute(self, localpath): 

246 """ 

247 Locally execute a python script. 

248 @param localpath local python script 

249 @return output, error 

250 """ 

251 exe = subprocess.Popen([sys.executable, localpath], 

252 stdout=subprocess.PIPE, 

253 stderr=subprocess.PIPE) 

254 out, error = exe.communicate() 

255 return out, error 

256 

257 def feed(self, any_, script_python=False, params=None): 

258 """ 

259 Displays something. 

260 

261 @param any_ string 

262 @param script_python if True, the function processes script sections 

263 @param params extra parameters, see @me process_scripts 

264 

265 A script section looks like: 

266 

267 :: 

268 

269 <script type="text/python"> 

270 from pandas import DataFrame 

271 pars = [ { "key":k, "value":v } for k,v in params ] 

272 tbl = DataFrame (pars) 

273 print ( tbl.tohtml(class_table="myclasstable") ) 

274 </script> 

275 """ 

276 if params is None: 

277 params = {} 

278 

279 if isinstance(any_, bytes): 

280 if script_python: 

281 raise SystemError("unable to execute script from bytes") 

282 self.wfile.write(any_) 

283 else: 

284 if script_python: 

285 any_ = self.process_scripts(any_, params) 

286 text = any_.encode("utf-8") 

287 self.wfile.write(text) 

288 

289 def shutdown(self): 

290 """ 

291 Shuts down the service from the service itself (not from another thread). 

292 For the time being, the function generates the following exception: 

293 

294 :: 

295 

296 Traceback (most recent call last): 

297 File "simple_server_custom.py", line 225, in <module> 

298 run_server(None) 

299 File "simple_server_custom.py", line 219, in run_server 

300 server.serve_forever() 

301 File "c:\\python33\\lib\\socketserver.py", line 237, in serve_forever 

302 poll_interval) 

303 File "c:\\python33\\lib\\socketserver.py", line 155, in _eintr_retry 

304 return func(*args) 

305 ValueError: file descriptor cannot be a negative integer (-1) 

306 

307 A better way to shut it down should is recommended. The use of the function: 

308 

309 :: 

310 

311 self.server.shutdown() 

312 

313 freezes the server because this function should not be run in the same thread. 

314 """ 

315 # self.server.close() 

316 # help(self.server.socket) 

317 # self.server.socket.shutdown(socket.SHUT_RDWR) 

318 self.server.socket.close() 

319 # self.server.shutdown() 

320 fLOG("end of shut down") 

321 

322 def main_page(self): 

323 """ 

324 Returns the main page (case the server is called 

325 with no path). 

326 @return default page 

327 """ 

328 return "index.html" 

329 

330 def serve_content(self, path, method="GET"): 

331 """ 

332 Tells what to do based on the path. The function intercepts the 

333 path ``/localfile/``, otherwise it calls ``serve_content_web``. 

334 

335 If you type ``http://localhost:8080/localfile/__file__``, 

336 it will display this file. 

337 

338 @param path ParseResult 

339 @param method GET or POST 

340 """ 

341 if path.path in ("", "/"): 

342 temp = "/" + self.main_page() 

343 self.do_redirect(temp) 

344 

345 else: 

346 params = parse_qs(path.query) 

347 params["__path__"] = path 

348 # here you might want to look into a local path... f2r = HOME + 

349 # path 

350 

351 url = path.geturl() 

352 params["__url__"] = path 

353 

354 if url.startswith("/localfile/"): 

355 localpath = path.path[len("/localfile/"):] 

356 self.LOG("localpath ", localpath, os.path.isfile(localpath)) 

357 

358 if localpath == "shutdown": 

359 self.LOG("call shutdown") 

360 self.shutdown() 

361 

362 elif localpath == "__file__": 

363 self.LOG("display file __file__", localpath) 

364 self.send_response(200) 

365 self.send_headers("__file__.txt") 

366 content = self.get_file_content(__file__, "r") 

367 self.feed(content) 

368 

369 else: 

370 self.send_response(200) 

371 _, ftype = self.get_ftype(localpath) 

372 execute = eval(params.get("execute", ["True"])[ # pylint: disable=W0123 

373 0]) # pylint: disable=W0123 

374 path = params.get("path", [None])[0] 

375 keep = eval(params.get("keep", ["False"])[ # pylint: disable=W0123 

376 0]) # pylint: disable=W0123 

377 if keep and path not in self.get_pathes(): 

378 self.LOG( 

379 "execute", 

380 execute, 

381 "- ftype", 

382 ftype, 

383 " - path", 

384 path, 

385 " keep ", 

386 keep) 

387 self.add_path(path) 

388 else: 

389 self.LOG( 

390 "execute", 

391 execute, 

392 "- ftype", 

393 ftype, 

394 " - path", 

395 path) 

396 

397 if ftype != 'execute' or not execute: 

398 content = self.get_file_content(localpath, ftype, path) 

399 ext = os.path.splitext(localpath)[-1].lower() 

400 if ext in [ 

401 ".py", ".c", ".cpp", ".hpp", ".h", ".r", ".sql", ".js", ".java", ".css"]: 

402 self.send_headers(".html") 

403 self.feed( 

404 self.html_code_renderer( 

405 localpath, 

406 content)) 

407 else: 

408 self.send_headers(localpath) 

409 self.feed(content) 

410 else: 

411 self.LOG("execute file ", localpath) 

412 out, err = self.execute(localpath) 

413 if len(err) > 0: 

414 self.send_error(404) 

415 self.feed( 

416 "Requested resource %s unavailable" % 

417 localpath) 

418 else: 

419 self.send_headers(localpath) 

420 self.feed(out) 

421 

422 elif url.startswith("/js/"): 

423 found = None 

424 for jspa in self.get_javascript_paths(): 

425 file = os.path.join(jspa, url[4:]) 

426 if os.path.exists(file): 

427 found = file 

428 

429 if found is None: 

430 self.send_response(200) 

431 self.send_headers("") 

432 self.feed( 

433 "Unable to serve content for url: '{}'.".format(path.geturl())) 

434 self.send_error(404) 

435 else: 

436 _, ft = self.get_ftype(found) 

437 if ft == "r": 

438 try: 

439 with open(found, ft, encoding="utf8") as f: # pylint: disable=W1501 

440 content = f.read() 

441 except UnicodeDecodeError: 

442 self.LOG("file is not utf8", found) 

443 with open(found, ft) as f: # pylint: disable=W1501 

444 content = f.read() 

445 else: 

446 self.LOG("reading binary") 

447 with open(found, ft) as f: # pylint: disable=W1501 

448 content = f.read() 

449 

450 self.send_response(200) 

451 self.send_headers(found) 

452 self.feed(content) 

453 

454 elif url.startswith("/debug_string/"): 

455 # debugging purposes 

456 self.send_response(200) 

457 self.send_headers("debug.html") 

458 self.feed(html_debug_string, False, params) 

459 

460 elif url.startswith("/fetchurlclean/"): 

461 self.send_response(200) 

462 self.send_headers("debug.html") 

463 url = path.path.replace("/fetchurlclean/", "") 

464 try: 

465 content = get_url_content_timeout(url) 

466 except Exception as e: 

467 content = "<html><body>ERROR (1): %s</body></html>" % e 

468 if content is None or len(content) == 0: 

469 content = "<html><body>ERROR (1): content is empty</body></html>" 

470 

471 stre = io.StringIO() 

472 pars = HTMLScriptParserRemove(outStream=stre) 

473 pars.feed(content) 

474 content = stre.getvalue() 

475 

476 self.feed(content, False, params={}) 

477 

478 elif url.startswith("/fetchurl/"): 

479 self.send_response(200) 

480 self.send_headers("debug.html") 

481 url = path.path.replace("/fetchurl/", "") 

482 try: 

483 content = get_url_content_timeout(url) 

484 except Exception as e: 

485 content = "<html><body>ERROR (2): %s</body></html>" % e 

486 self.feed(content, False, params={}) 

487 

488 else: 

489 self.serve_content_web(path, method, params) 

490 

491 def get_javascript_paths(self): 

492 """ 

493 Returns all the location where the server should 

494 look for a java script. 

495 @return list of paths 

496 """ 

497 return [SimpleHandler.javascript_path] 

498 

499 def html_code_renderer(self, localpath, content): 

500 """ 

501 Produces a :epkg:`html` code for code. 

502 

503 @param localpath local path to file (local or not) 

504 @param content content of the file 

505 @return html string 

506 """ 

507 res = [html_header % (localpath, getpass.getuser(), "code")] 

508 res.append("<pre class=\"prettyprint\">") 

509 res.append(content.replace("<", "&lt;").replace(">", "&gt;")) 

510 res.append(html_footer) 

511 return "\n".join(res) 

512 

513 def serve_content_web(self, path, method, params): 

514 """ 

515 Functions to overload (executed after serve_content). 

516 

517 @param path ParseResult 

518 @param method GET or POST 

519 @param params params parsed from the url + others 

520 """ 

521 self.send_response(200) 

522 self.send_headers("") 

523 self.feed("Unable to serve content for url: '{}'\n{}".format( 

524 path.geturl(), str(params))) 

525 self.send_error(404) 

526 

527 def process_scripts(self, content, params): 

528 """ 

529 Parses a :epkg:`HTML` string, extract script section 

530 (only python script for the time being) 

531 and returns the final page. 

532 

533 @param content html string 

534 @param params dictionary with what is known from the server 

535 @return html content 

536 """ 

537 st = StringIO() 

538 parser = HTMLScriptParser( 

539 outStream=st, 

540 catch_exception=True, 

541 context=params) 

542 parser.feed(content) 

543 res = st.getvalue() 

544 return res 

545 

546 

547class ThreadServer (Thread): 

548 """ 

549 Defines a thread which holds a web server. 

550 

551 @var server the server of run 

552 """ 

553 

554 def __init__(self, server): 

555 """ 

556 @param server to run 

557 """ 

558 Thread.__init__(self) 

559 self.server = server 

560 

561 def run(self): 

562 """ 

563 Runs the server. 

564 """ 

565 self.server.serve_forever() 

566 

567 def shutdown(self): 

568 """ 

569 Shuts down the server, if it does not work, 

570 you can still kill the thread: 

571 

572 :: 

573 

574 self.kill() 

575 """ 

576 self.server.shutdown() 

577 self.server.server_close() 

578 

579 

580def run_server(server, thread=False, port=8080): 

581 """ 

582 Runs the server. 

583 @param server if None, it becomes ``HTTPServer(('localhost', 8080), SimpleHandler)`` 

584 @param thread if True, the server is run in a thread 

585 and the function returns right away, 

586 otherwite, it runs the server. 

587 @param port port to use 

588 @return server if thread is False, the thread otherwise (the thread is started) 

589 

590 @warning If you kill the python program while the thread is still running, python interpreter might be closed completely. 

591 """ 

592 if server is None: 

593 server = HTTPServer(('localhost', port), SimpleHandler) 

594 if thread: 

595 th = ThreadServer(server) 

596 th.start() 

597 return th 

598 else: 

599 server.serve_forever() 

600 return server 

601 

602 

603if __name__ == '__main__': 

604 fLOG(OutputPrint=True) 

605 fLOG("running server") 

606 run_server(None) 

607 fLOG("end running server") 

608 

609 # http://localhost:8080/localfile/D:\Dupre\_data\informatique\support\python_td_2013\programme\td9_by_hours.json 

610 # http://localhost:8080/localfile/tag-cloud.html?path=D:\Dupre\_data\program\pyhome\pyhome3\_nrt\nrt_internet\data&keep=True 

611 # http://localhost:8080/debug_string/ 

612 

613 """ 

614 from pyquickhelper.loghelper import fLOG 

615 from pyrsslocal.internet.simple_server.simple_server_custom import run_server 

616 

617 fLOG(OutputPrint=True) 

618 fLOG("running server") 

619 run_server(None) 

620 fLOG("end running server") 

621 """