Coverage for src/pyensae/languages/antlr_grammar_build.py: 7%

1""" 

2@file 

3@brief Helpers to build grammars 

4This module requires `antlr4 <http://www.antlr.org/>`_. 

5and `antlr4-python3-runtime <https://pypi.python.org/pypi/antlr4-python3-runtime/>`_. 

6""" 

7import os 

8import sys 

9from pyquickhelper.loghelper import noLOG 

10 

11 

12def _is_syntax_is_missing(language): 

13 """ 

14 Downloads the grammar for a specific language if 

15 the files is missing. 

16 

17 @param language language: python, sqlite, ... 

18 @return grammar file 

19 """ 

20 locations = { 

21 "R": "https://github.com/antlr/grammars-v4/tree/master/r/", 

22 "SQLite": "https://github.com/antlr/grammars-v4/blob/master/sqlite/", 

23 "CSharp": "https://github.com/antlr/grammars-v4/tree/master/csharp", 

24 } 

25 

26 folder = os.path.dirname(__file__) 

27 filename = os.path.join(folder, language + ".g4") 

28 if os.path.exists(filename): 

29 return filename 

30 if language in locations: 

31 raise FileNotFoundError( 

32 "The grammar '{0}' is not available, you should get " 

33 "it from {1}".format(language, locations[language])) 

34 raise KeyError( 

35 "Unexpected language: '{0}', not in '{1}'".format( 

36 language, ",".join(locations.keys()))) 

37 

38 

39def build_grammar(g4, version="4.10.1", fLOG=noLOG): 

40 """ 

41 Compiles the grammar for a specific file. 

42 

43 @param g4 grammar format antlr4 

44 @param version version of *antlr4* to use, 4.9 

45 @param fLOG logging function 

46 @return list of files 

47 

48 The compilation must be done with `antlr4 <http://www.antlr.org/>`_. 

49 It generates a lexer and a parser which can be imported in Python. 

50 The options for the command line are described at: 

51 `antlr4 options 

52 <https://theantlrguy.atlassian.net/wiki/display/ANTLR4/Options>`_. 

53 

54 .. exref:: 

55 :title: Builds a Antlr4 grammar 

56 

57 See `grammars-v4 <https://github.com/antlr/grammars-v4>`_ 

58 

59 :: 

60 

61 build_grammar("R.g4") 

62 """ 

63 if not g4.endswith(".g4"): 

64 fold = os.path.abspath(os.path.dirname(__file__)) 

65 g4 = os.path.join(fold, g4 + ".g4") 

66 

67 url = "http://www.antlr.org/download/antlr-{0}-complete.jar".format( 

68 version) 

69 spl = url.split("/") 

70 domain, name = "/".join(spl[:-1]) + "/", spl[-1] 

71 folder = os.path.abspath(os.path.dirname(__file__)) 

72 final = os.path.join(folder, name) 

73 

74 if not os.path.exists(final): 

75 from ..datasource.http_retrieve import download_data 

76 name = download_data(name, website=domain, whereTo=folder) 

77 if not os.path.exists(name): 

78 raise FileNotFoundError("unable to download: " + url) 

79 

80 path = os.environ.get("CLASSPATH", "") 

81 if name not in path: 

82 path = ".;{0}\\antlr-{1}-complete.jar".format(folder, version) 

83 else: 

84 path = ".;{0}\\antlr-{1}-complete.jar;{2}".format( 

85 folder, 

86 version, 

87 os.environ["CLASSPATH"]) 

88 

89 os.environ["CLASSPATH"] = path 

90 fLOG("CLASSPATH", os.environ["CLASSPATH"]) 

91 

92 # we remove -rc... 

93 version = version.split("-")[0] 

94 

95 cmd = "org.antlr.v4.Tool " 

96 cmd += "-Dlanguage=Python3 " 

97 cmd += g4 

98 from pyquickhelper.loghelper import run_cmd 

99 out, err = run_cmd("java " + cmd, wait=True, fLOG=fLOG) 

100 

101 def compiled(): 

102 if "Lexer" in g4: 

103 lexer = g4.replace(".g4", ".tokens") 

104 else: 

105 lexer = g4.replace(".g4", ".py") 

106 return os.path.exists(lexer) 

107 

108 if not compiled() or (len(err) > 0 and "error" in err): 

109 

110 javapath = r'C:\Program Files\Java\jre7\bin\java.exe' 

111 os.environ["PATH"] = os.environ["PATH"] + ";" + javapath 

112 if sys.platform.startswith("win") and os.path.exists(javapath): 

113 out, err = run_cmd( 

114 '"' + javapath + '" ' + cmd, wait=True, fLOG=fLOG) 

115 if not compiled() or (len(err) > 0 and "error" in err): 

116 raise Exception( 

117 "unable to compile: " + 

118 final + 

119 "\nCLASSPATH:\n" + 

120 os.environ["CLASSPATH"] + 

121 "\nERR:\n" + 

122 err + 

123 "\nCMD:\njava " + 

124 cmd + 

125 "\nYou should do it manually.") 

126 elif err: 

127 err_lines = err.split(err) 

128 err_lines = [_ for _ in err_lines if not _.startswith("warning(")] 

129 err2 = "\n".join(err_lines).strip("\n ") 

130 if len(err2) > 0: 

131 raise Exception( 

132 "unable to compile: " + 

133 final + 

134 "\nCLASSPATH:\n" + 

135 os.environ["CLASSPATH"] + 

136 "\nERR:\n" + 

137 err + 

138 "\nCMD:\njava " + 

139 cmd) 

140 

141 if os.environ.get("USERNAME", os.environ.get("USER", "")) in g4: 

142 dest = os.path.dirname(g4) 

143 for name in os.listdir(dest): 

144 if "Parser" not in name and "Lexer" not in name and \ 

145 "Token" not in name and "Listener" not in name: 

146 continue 

147 full = os.path.join(dest, name) 

148 with open(full, "r", encoding="utf-8") as f: 

149 content = f.read() 

150 content1 = content.replace(dest, "") 

151 if content1 != content: 

152 fLOG("[build_grammar] modified", name) 

153 with open(full, "w", encoding="utf-8") as f: 

154 f.write(content1) 

155 

156 return out + "\n---ERR---\n" + err