oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat_flinkSql

Browse Source
This commit is contained in:
Erindcl 2020-12-14 14:03:28 +08:00
commit e6ee2ff411
16 changed files with 19779 additions and 15053 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/grammar/hive/HiveSql.g4
View File

@ -5,7 +5,9 @@
The ASF licenses this file to You under the Apache License, Version 2.0 The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with (the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0 http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -13,13 +15,16 @@
limitations under the License. limitations under the License.
*/ */
// HPL/SQL Procedural SQL Extension Grammar parser grammar HiveSql;
grammar HiveSql;
options { options {
tokenVocab=HiveSqlLexer; tokenVocab=HiveSqlLexer;
} }
@parser::members {
this._input = input;
}
program : block EOF; program : block EOF;
block : ((begin_end_block | stmt) T_GO?)+ ; // Multiple consecutive blocks/statements block : ((begin_end_block | stmt) T_GO?)+ ; // Multiple consecutive blocks/statements
@ -34,7 +39,7 @@ single_block_stmt : // Single BEGIN END blo
; ;
block_end : block_end :
{!this._input.LT(2).getText().equalsIgnoreCase("TRANSACTION")}? T_END {!this._input.LT(2).text.toUpperCase() === "TRANSACTION"}? T_END
; ;
proc_block : proc_block :
@ -108,7 +113,7 @@ stmt :
semicolon_stmt : semicolon_stmt :
T_SEMICOLON T_SEMICOLON
| '@' | '#' | '/' | T_CALLS | T_SHARP | T_DIV
; ;
exception_block : // Exception block exception_block : // Exception block
@ -124,7 +129,7 @@ null_stmt : // NULL statement (no operation)
; ;
expr_stmt : // Standalone expression expr_stmt : // Standalone expression
{!this._input.LT(1).getText().equalsIgnoreCase("GO")}? expr {this._input.LT(1).text.toUpperCase() !== "GO"}? expr
; ;
assignment_stmt : // Assignment statement assignment_stmt : // Assignment statement
@ -229,7 +234,7 @@ create_local_temp_table_stmt :
; ;
create_table_definition : create_table_definition :
(T_AS? T_OPEN_P select_stmt T_CLOSE_P | T_AS? select_stmt | T_OPEN_P create_table_columns T_CLOSE_P | T_LIKE table_name) create_table_options? (T_AS? T_OPEN_P select_stmt T_CLOSE_P | T_AS? select_stmt | T_OPEN_P create_table_columns T_CLOSE_P) create_table_options?
; ;
create_table_columns : create_table_columns :
@ -365,7 +370,6 @@ alter_table_add_constraint_item :
dtype : // Data types dtype : // Data types
T_CHAR T_CHAR
| T_CHARACTER
| T_BIGINT | T_BIGINT
| T_BINARY_DOUBLE | T_BINARY_DOUBLE
| T_BINARY_FLOAT | T_BINARY_FLOAT
@ -401,7 +405,7 @@ dtype : // Data types
| T_VARCHAR | T_VARCHAR
| T_VARCHAR2 | T_VARCHAR2
| T_XML | T_XML
| ident ('%' (T_TYPE | T_ROWTYPE))? // User-defined or derived data type | ident (T_PRECENT (T_TYPE | T_ROWTYPE))? // User-defined or derived data type
; ;
dtype_len : // Data type length or size specification dtype_len : // Data type length or size specification
@ -471,9 +475,9 @@ create_procedure_stmt :
create_routine_params : create_routine_params :
T_OPEN_P T_CLOSE_P T_OPEN_P T_CLOSE_P
| T_OPEN_P create_routine_param_item (T_COMMA create_routine_param_item)* T_CLOSE_P | T_OPEN_P create_routine_param_item (T_COMMA create_routine_param_item)* T_CLOSE_P
| {!this._input.LT(1).getText().equalsIgnoreCase("IS") && | {this._input.LT(1).text.toUpperCase() !== "IS" &&
!this._input.LT(1).getText().equalsIgnoreCase("AS") && this._input.LT(1).text.toUpperCase() !== "AS" &&
!(this._input.LT(1).getText().equalsIgnoreCase("DYNAMIC") && this._input.LT(2).getText().equalsIgnoreCase("RESULT")) !(this._input.LT(1).text.toUpperCase() ==="DYNAMIC" && this._input.LT(2).text.toUpperCase() === "RESULT")
}? }?
create_routine_param_item (T_COMMA create_routine_param_item)* create_routine_param_item (T_COMMA create_routine_param_item)*
; ;
@ -520,7 +524,7 @@ if_tsql_stmt :
; ;
if_bteq_stmt : if_bteq_stmt :
'.' T_IF bool_expr T_THEN single_block_stmt T_DOT T_IF bool_expr T_THEN single_block_stmt
; ;
elseif_block : elseif_block :
@ -675,7 +679,7 @@ print_stmt : // PRINT statement
; ;
quit_stmt : quit_stmt :
'.'? T_QUIT expr? T_DOT? T_QUIT expr?
; ;
raise_stmt : raise_stmt :
@ -811,7 +815,7 @@ select_list_item :
; ;
select_list_alias : select_list_alias :
{!this._input.LT(1).getText().equalsIgnoreCase("INTO") && !this._input.LT(1).getText().equalsIgnoreCase("FROM")}? T_AS? ident {this._input.LT(1).text.toUpperCase() !== "INTO" && this._input.LT(1).text.toUpperCase() !== "FROM"}? T_AS? ident
| T_OPEN_P T_TITLE L_S_STRING T_CLOSE_P | T_OPEN_P T_TITLE L_S_STRING T_CLOSE_P
; ;
@ -861,14 +865,15 @@ from_table_values_row:
; ;
from_alias_clause : from_alias_clause :
{!this._input.LT(1).getText().equalsIgnoreCase("EXEC") && {this._input.LT(1).text.toUpperCase() !== "EXEC" &&
!this._input.LT(1).getText().equalsIgnoreCase("EXECUTE") && this._input.LT(1).text.toUpperCase() !== "EXECUTE" &&
!this._input.LT(1).getText().equalsIgnoreCase("INNER") && this._input.LT(1).text.toUpperCase() !== "INNER" &&
!this._input.LT(1).getText().equalsIgnoreCase("LEFT") && this._input.LT(1).text.toUpperCase() !== "LEFT" &&
!this._input.LT(1).getText().equalsIgnoreCase("GROUP") && this._input.LT(1).text.toUpperCase() !== "GROUP" &&
!this._input.LT(1).getText().equalsIgnoreCase("ORDER") && this._input.LT(1).text.toUpperCase() !== "ORDER" &&
!this._input.LT(1).getText().equalsIgnoreCase("LIMIT") && this._input.LT(1).text.toUpperCase() !== "LIMIT" &&
!this._input.LT(1).getText().equalsIgnoreCase("WITH")}? this._input.LT(1).text.toUpperCase() !== "WITH" &&
this._input.LT(1).text.toUpperCase() !== "JOIN"}?
T_AS? ident (T_OPEN_P L_ID (T_COMMA L_ID)* T_CLOSE_P)? T_AS? ident (T_OPEN_P L_ID (T_COMMA L_ID)* T_CLOSE_P)?
; ;
@ -945,7 +950,7 @@ delete_stmt :
; ;
delete_alias : delete_alias :
{!this._input.LT(1).getText().equalsIgnoreCase("ALL")}? {this._input.LT(1).text.toUpperCase() !== "ALL"}?
T_AS? ident T_AS? ident
; ;
@ -1070,7 +1075,7 @@ expr_case_searched :
; ;
expr_cursor_attribute : expr_cursor_attribute :
ident '%' (T_ISOPEN | T_FOUND | T_NOTFOUND) ident T_PRECENT (T_ISOPEN | T_FOUND | T_NOTFOUND)
; ;
expr_agg_window_func : expr_agg_window_func :
@ -1136,7 +1141,7 @@ expr_func_params :
; ;
func_param : func_param :
{!this._input.LT(1).getText().equalsIgnoreCase("INTO")}? (ident T_EQUAL T_GREATER?)? expr {this._input.LT(1).text.toUpperCase() !== "INTO"}? (ident T_EQUAL T_GREATER?)? expr
; ;
expr_select : expr_select :
@ -1160,7 +1165,7 @@ hive_item :
; ;
host : host :
'!' host_cmd ';' // OS command T_NOTE host_cmd ';' // OS command
| host_stmt | host_stmt
; ;
@ -1173,7 +1178,7 @@ host_stmt :
; ;
file_name : file_name :
L_FILE | ('/' | '.' '/')? ident ('/' ident)* L_FILE | ('/' | T_DOT '/')? ident ('/' ident)*
; ;
date_literal : // DATE 'YYYY-MM-DD' literal date_literal : // DATE 'YYYY-MM-DD' literal
@ -1185,7 +1190,7 @@ timestamp_literal : // TIMESTAMP 'YYYY-MM-DD HH:MI:SS.FFF'
; ;
ident : ident :
'-'? (L_ID | non_reserved_words) ('.' (L_ID | non_reserved_words))* (L_ID | non_reserved_words) (T_DOT (L_ID | non_reserved_words))*
; ;
string : // String literal (single or double quoted) string : // String literal (single or double quoted)
@ -1538,3 +1543,4 @@ non_reserved_words : // Tokens that are not reserved words
| T_XML | T_XML
| T_YES | T_YES
; ;

6
src/grammar/hive/HiveSqlLexer.g4
View File

@ -1,3 +1,4 @@
lexer grammar HiveSqlLexer; lexer grammar HiveSqlLexer;
// Lexer rules // Lexer rules
@ -347,9 +348,12 @@ T_COLON : ':' ;
T_COMMA : ',' ; T_COMMA : ',' ;
T_PIPE : '||' ; T_PIPE : '||' ;
T_DIV : '/' ; T_DIV : '/' ;
T_DOT : '.' ;
T_DOT2 : '..' ; T_DOT2 : '..' ;
T_EQUAL : '=' ; T_EQUAL : '=' ;
T_EQUAL2 : '==' ; T_EQUAL2 : '==' ;
T_SHARP : '#' ;
T_NOTE : '!' ;
T_NOTEQUAL : '<>' ; T_NOTEQUAL : '<>' ;
T_NOTEQUAL2 : '!=' ; T_NOTEQUAL2 : '!=' ;
T_GREATER : '>' ; T_GREATER : '>' ;
@ -357,6 +361,8 @@ T_GREATEREQUAL : '>=' ;
T_LESS : '<' ; T_LESS : '<' ;
T_LESSEQUAL : '<=' ; T_LESSEQUAL : '<=' ;
T_MUL : '*' ; T_MUL : '*' ;
T_PRECENT : '%' ;
T_CALLS : '@' ;
T_OPEN_B : '{' ; T_OPEN_B : '{' ;
T_OPEN_P : '(' ; T_OPEN_P : '(' ;
T_OPEN_SB : '[' ; T_OPEN_SB : '[' ;

3
src/grammar/hive/README.md
View File

@ -1,3 +0,0 @@
# Hive SQL Grammar
Source file from [Hive Github](https://github.com/apache/hive/tree/master/hplsql/src/main/antlr4/org/apache/hive/hplsql)

1390
src/lib/hive/HiveSql.interp
View File

File diff suppressed because one or more lines are too long

25435
src/lib/hive/HiveSqlParser.js → src/lib/hive/HiveSql.js
View File

File diff suppressed because one or more lines are too long

786
src/lib/hive/HiveSql.tokens
View File

@ -1,384 +1,404 @@
T__0=1 T_ACTION=1
T__1=2 T_ADD2=2
T__2=3 T_ALL=3
T__3=4 T_ALLOCATE=4
T__4=5 T_ALTER=5
T__5=6 T_AND=6
T__6=7 T_ANSI_NULLS=7
T__7=8 T_ANSI_PADDING=8
T__8=9 T_AS=9
T__9=10 T_ASC=10
T_GO=11 T_ASSOCIATE=11
T_BEGIN=12 T_AT=12
T_SEMICOLON=13 T_AUTO_INCREMENT=13
T_END=14 T_AVG=14
T_EXCEPTION=15 T_BATCHSIZE=15
T_WHEN=16 T_BEGIN=16
L_ID=17 T_BETWEEN=17
T_THEN=18 T_BIGINT=18
T_NULL=19 T_BINARY_DOUBLE=19
T_SET=20 T_BINARY_FLOAT=20
T_COMMA=21 T_BINARY_INTEGER=21
T_COLON=22 T_BIT=22
T_EQUAL=23 T_BODY=23
T_OPEN_P=24 T_BREAK=24
T_CLOSE_P=25 T_BY=25
T_ALLOCATE=26 T_BYTE=26
T_CURSOR=27 T_CALL=27
T_FOR=28 T_CALLER=28
T_RESULT=29 T_CASCADE=29
T_PROCEDURE=30 T_CASE=30
T_ASSOCIATE=31 T_CASESPECIFIC=31
T_LOCATOR=32 T_CAST=32
T_LOCATORS=33 T_CHAR=33
T_WITH=34 T_CHARACTER=34
T_TRANSACTION=35 T_CHARSET=35
T_BREAK=36 T_CLIENT=36
T_CALL=37 T_CLOSE=37
T_DECLARE=38 T_CLUSTERED=38
T_AS=39 T_CMP=39
T_CONSTANT=40 T_COLLECT=40
T_CONDITION=41 T_COLLECTION=41
T_IS=42 T_COLUMN=42
T_RETURN=43 T_COMMENT=43
T_ONLY=44 T_CONSTANT=44
T_TO=45 T_COMMIT=45
T_CALLER=46 T_COMPRESS=46
T_CLIENT=47 T_CONCAT=47
T_WITHOUT=48 T_CONDITION=48
T_CONTINUE=49 T_CONSTRAINT=49
T_EXIT=50 T_CONTINUE=50
T_HANDLER=51 T_COPY=51
T_SQLEXCEPTION=52 T_COUNT=52
T_SQLWARNING=53 T_COUNT_BIG=53
T_NOT=54 T_CREATE=54
T_FOUND=55 T_CREATION=55
T_GLOBAL=56 T_CREATOR=56
T_TEMPORARY=57 T_CS=57
T_TABLE=58 T_CURRENT=58
T_CREATE=59 T_CURRENT_SCHEMA=59
T_IF=60 T_CURSOR=60
T_EXISTS=61 T_DATABASE=61
T_LOCAL=62 T_DATA=62
T_MULTISET=63 T_DATE=63
T_VOLATILE=64 T_DATETIME=64
T_LIKE=65 T_DAY=65
T_CONSTRAINT=66 T_DAYS=66
T_PRIMARY=67 T_DEC=67
T_KEY=68 T_DECIMAL=68
T_UNIQUE=69 T_DECLARE=69
T_REFERENCES=70 T_DEFAULT=70
T_IDENTITY=71 T_DEFERRED=71
L_INT=72 T_DEFINED=72
T_AUTO_INCREMENT=73 T_DEFINER=73
T_ENABLE=74 T_DEFINITION=74
T_CLUSTERED=75 T_DELETE=75
T_ASC=76 T_DELIMITED=76
T_DESC=77 T_DELIMITER=77
T_FOREIGN=78 T_DESC=78
T_ON=79 T_DESCRIBE=79
T_UPDATE=80 T_DIAGNOSTICS=80
T_DELETE=81 T_DIR=81
T_NO=82 T_DIRECTORY=82
T_ACTION=83 T_DISTINCT=83
T_RESTRICT=84 T_DISTRIBUTE=84
T_DEFAULT=85 T_DO=85
T_CASCADE=86 T_DOUBLE=86
T_LOG=87 T_DROP=87
T_FALLBACK=88 T_DYNAMIC=88
T_COMMIT=89 T_ELSE=89
T_PRESERVE=90 T_ELSEIF=90
T_ROWS=91 T_ELSIF=91
T_SEGMENT=92 T_ENABLE=92
T_CREATION=93 T_END=93
T_IMMEDIATE=94 T_ENGINE=94
T_DEFERRED=95 T_ESCAPED=95
T_PCTFREE=96 T_EXCEPT=96
T_PCTUSED=97 T_EXEC=97
T_INITRANS=98 T_EXECUTE=98
T_MAXTRANS=99 T_EXCEPTION=99
T_NOCOMPRESS=100 T_EXCLUSIVE=100
T_LOGGING=101 T_EXISTS=101
T_NOLOGGING=102 T_EXIT=102
T_STORAGE=103 T_FALLBACK=103
T_TABLESPACE=104 T_FALSE=104
T_INDEX=105 T_FETCH=105
T_IN=106 T_FIELDS=106
T_REPLACE=107 T_FILE=107
T_DISTRIBUTE=108 T_FILES=108
T_BY=109 T_FLOAT=109
T_HASH=110 T_FOR=110
T_LOGGED=111 T_FOREIGN=111
T_COMPRESS=112 T_FORMAT=112
T_YES=113 T_FOUND=113
T_DEFINITION=114 T_FROM=114
T_DROP=115 T_FULL=115
T_DATA=116 T_FUNCTION=116
T_STORED=117 T_GET=117
T_ROW=118 T_GLOBAL=118
T_FORMAT=119 T_GO=119
T_DELIMITED=120 T_GRANT=120
T_FIELDS=121 T_GROUP=121
T_TERMINATED=122 T_HANDLER=122
T_ESCAPED=123 T_HASH=123
T_COLLECTION=124 T_HAVING=124
T_ITEMS=125 T_HDFS=125
T_MAP=126 T_HIVE=126
T_KEYS=127 T_HOST=127
T_LINES=128 T_IDENTITY=128
T_DEFINED=129 T_IF=129
T_TEXTIMAGE_ON=130 T_IGNORE=130
T_COMMENT=131 T_IMMEDIATE=131
T_CHARACTER=132 T_IN=132
T_CHARSET=133 T_INCLUDE=133
T_ENGINE=134 T_INDEX=134
T_ALTER=135 T_INITRANS=135
T_ADD2=136 T_INNER=136
T_CHAR=137 T_INOUT=137
T_BIGINT=138 T_INSERT=138
T_BINARY_DOUBLE=139 T_INT=139
T_BINARY_FLOAT=140 T_INT2=140
T_BINARY_INTEGER=141 T_INT4=141
T_BIT=142 T_INT8=142
T_DATE=143 T_INTEGER=143
T_DATETIME=144 T_INTERSECT=144
T_DEC=145 T_INTERVAL=145
T_DECIMAL=146 T_INTO=146
T_DOUBLE=147 T_INVOKER=147
T_PRECISION=148 T_IS=148
T_FLOAT=149 T_ISOPEN=149
T_INT=150 T_ITEMS=150
T_INT2=151 T_JOIN=151
T_INT4=152 T_KEEP=152
T_INT8=153 T_KEY=153
T_INTEGER=154 T_KEYS=154
T_NCHAR=155 T_LANGUAGE=155
T_NVARCHAR=156 T_LEAVE=156
T_NUMBER=157 T_LEFT=157
T_NUMERIC=158 T_LIKE=158
T_PLS_INTEGER=159 T_LIMIT=159
T_REAL=160 T_LINES=160
T_RESULT_SET_LOCATOR=161 T_LOCAL=161
T_VARYING=162 T_LOCATION=162
T_SIMPLE_FLOAT=163 T_LOCATOR=163
T_SIMPLE_DOUBLE=164 T_LOCATORS=164
T_SIMPLE_INTEGER=165 T_LOCKS=165
T_SMALLINT=166 T_LOG=166
T_SMALLDATETIME=167 T_LOGGED=167
T_STRING=168 T_LOGGING=168
T_SYS_REFCURSOR=169 T_LOOP=169
T_TIMESTAMP=170 T_MAP=170
T_TINYINT=171 T_MATCHED=171
T_VARCHAR=172 T_MAX=172
T_VARCHAR2=173 T_MAXTRANS=173
T_XML=174 T_MERGE=174
T_TYPE=175 T_MESSAGE_TEXT=175
T_ROWTYPE=176 T_MICROSECOND=176
T_MAX=177 T_MICROSECONDS=177
T_BYTE=178 T_MIN=178
T_CASESPECIFIC=179 T_MULTISET=179
T_CS=180 T_NCHAR=180
T_DATABASE=181 T_NEW=181
T_SCHEMA=182 T_NVARCHAR=182
T_LOCATION=183 T_NO=183
T_OR=184 T_NOCOUNT=184
T_FUNCTION=185 T_NOCOMPRESS=185
T_RETURNS=186 T_NOLOGGING=186
T_PACKAGE=187 T_NONE=187
T_PROC=188 T_NOT=188
T_BODY=189 T_NOTFOUND=189
T_OUT=190 T_NULL=190
T_INOUT=191 T_NUMERIC=191
T_LANGUAGE=192 T_NUMBER=192
T_SQL=193 T_OBJECT=193
T_SECURITY=194 T_OFF=194
T_CREATOR=195 T_ON=195
T_DEFINER=196 T_ONLY=196
T_INVOKER=197 T_OPEN=197
T_OWNER=198 T_OR=198
T_DYNAMIC=199 T_ORDER=199
T_SETS=200 T_OUT=200
T_EXEC=201 T_OUTER=201
T_EXECUTE=202 T_OVER=202
T_INTO=203 T_OVERWRITE=203
T_ELSE=204 T_OWNER=204
T_ELSIF=205 T_PACKAGE=205
T_ELSEIF=206 T_PARTITION=206
T_INCLUDE=207 T_PCTFREE=207
T_INSERT=208 T_PCTUSED=208
T_OVERWRITE=209 T_PLS_INTEGER=209
T_VALUES=210 T_PRECISION=210
T_DIRECTORY=211 T_PRESERVE=211
T_GET=212 T_PRIMARY=212
T_DIAGNOSTICS=213 T_PRINT=213
T_MESSAGE_TEXT=214 T_PROC=214
T_ROW_COUNT=215 T_PROCEDURE=215
T_GRANT=216 T_QUALIFY=216
T_ROLE=217 T_QUERY_BAND=217
T_LEAVE=218 T_QUIT=218
T_OBJECT=219 T_QUOTED_IDENTIFIER=219
T_AT=220 T_RAISE=220
T_OPEN=221 T_REAL=221
T_FETCH=222 T_REFERENCES=222
T_FROM=223 T_REGEXP=223
T_COLLECT=224 T_REPLACE=224
T_STATISTICS=225 T_RESIGNAL=225
T_STATS=226 T_RESTRICT=226
T_COLUMN=227 T_RESULT=227
T_CLOSE=228 T_RESULT_SET_LOCATOR=228
T_CMP=229 T_RETURN=229
T_SUM=230 T_RETURNS=230
T_COPY=231 T_REVERSE=231
T_HDFS=232 T_RIGHT=232
T_BATCHSIZE=233 T_RLIKE=233
T_DELIMITER=234 T_ROLE=234
T_SQLINSERT=235 T_ROLLBACK=235
T_IGNORE=236 T_ROW=236
T_WORK=237 T_ROWS=237
T_PRINT=238 T_ROWTYPE=238
T_QUIT=239 T_ROW_COUNT=239
T_RAISE=240 T_RR=240
T_RESIGNAL=241 T_RS=241
T_SQLSTATE=242 T_PWD=242
T_VALUE=243 T_TRIM=243
T_ROLLBACK=244 T_SCHEMA=244
T_CURRENT=245 T_SECOND=245
T_CURRENT_SCHEMA=246 T_SECONDS=246
T_ANSI_NULLS=247 T_SECURITY=247
T_ANSI_PADDING=248 T_SEGMENT=248
T_NOCOUNT=249 T_SEL=249
T_QUOTED_IDENTIFIER=250 T_SELECT=250
T_XACT_ABORT=251 T_SET=251
T_OFF=252 T_SESSION=252
T_QUERY_BAND=253 T_SESSIONS=253
T_NONE=254 T_SETS=254
T_SESSION=255 T_SHARE=255
T_SIGNAL=256 T_SIGNAL=256
T_SUMMARY=257 T_SIMPLE_DOUBLE=257
T_TOP=258 T_SIMPLE_FLOAT=258
T_LIMIT=259 T_SIMPLE_INTEGER=259
T_TRUNCATE=260 T_SMALLDATETIME=260
T_USE=261 T_SMALLINT=261
T_WHILE=262 T_SQL=262
T_DO=263 T_SQLEXCEPTION=263
T_LOOP=264 T_SQLINSERT=264
T_REVERSE=265 T_SQLSTATE=265
T_DOT2=266 T_SQLWARNING=266
T_STEP=267 T_STATS=267
L_LABEL=268 T_STATISTICS=268
T_LESS=269 T_STEP=269
T_GREATER=270 T_STORAGE=270
T_USING=271 T_STORED=271
T_UNION=272 T_STRING=272
T_ALL=273 T_SUBDIR=273
T_EXCEPT=274 T_SUBSTRING=274
T_INTERSECT=275 T_SUM=275
T_SELECT=276 T_SUMMARY=276
T_SEL=277 T_SYS_REFCURSOR=277
T_DISTINCT=278 T_TABLE=278
T_TITLE=279 T_TABLESPACE=279
L_S_STRING=280 T_TEMPORARY=280
T_INNER=281 T_TERMINATED=281
T_JOIN=282 T_TEXTIMAGE_ON=282
T_LEFT=283 T_THEN=283
T_RIGHT=284 T_TIMESTAMP=284
T_FULL=285 T_TINYINT=285
T_OUTER=286 T_TITLE=286
T_WHERE=287 T_TO=287
T_GROUP=288 T_TOP=288
T_HAVING=289 T_TRANSACTION=289
T_QUALIFY=290 T_TRUE=290
T_ORDER=291 T_TRUNCATE=291
T_RR=292 T_TYPE=292
T_RS=293 T_UNION=293
T_UR=294 T_UNIQUE=294
T_AND=295 T_UPDATE=295
T_KEEP=296 T_UR=296
T_EXCLUSIVE=297 T_USE=297
T_SHARE=298 T_USING=298
T_LOCKS=299 T_VALUE=299
T_MERGE=300 T_VALUES=300
T_MATCHED=301 T_VAR=301
T_DESCRIBE=302 T_VARCHAR=302
T_BETWEEN=303 T_VARCHAR2=303
T_EQUAL2=304 T_VARYING=304
T_NOTEQUAL=305 T_VOLATILE=305
T_NOTEQUAL2=306 T_WHEN=306
T_LESSEQUAL=307 T_WHERE=307
T_GREATEREQUAL=308 T_WHILE=308
T_RLIKE=309 T_WITH=309
T_REGEXP=310 T_WITHOUT=310
T_MUL=311 T_WORK=311
T_DIV=312 T_XACT_ABORT=312
T_ADD=313 T_XML=313
T_SUB=314 T_YES=314
T_INTERVAL=315 T_ACTIVITY_COUNT=315
T_DAY=316 T_CUME_DIST=316
T_DAYS=317 T_CURRENT_DATE=317
T_MICROSECOND=318 T_CURRENT_TIMESTAMP=318
T_MICROSECONDS=319 T_CURRENT_USER=319
T_SECOND=320 T_DENSE_RANK=320
T_SECONDS=321 T_FIRST_VALUE=321
T_PIPE=322 T_LAG=322
T_CONCAT=323 T_LAST_VALUE=323
T_CASE=324 T_LEAD=324
T_ISOPEN=325 T_MAX_PART_STRING=325
T_NOTFOUND=326 T_MIN_PART_STRING=326
T_AVG=327 T_MAX_PART_INT=327
T_COUNT=328 T_MIN_PART_INT=328
T_COUNT_BIG=329 T_MAX_PART_DATE=329
T_CUME_DIST=330 T_MIN_PART_DATE=330
T_DENSE_RANK=331 T_PART_COUNT=331
T_FIRST_VALUE=332 T_PART_LOC=332
T_LAG=333 T_RANK=333
T_LAST_VALUE=334 T_ROW_NUMBER=334
T_LEAD=335 T_STDEV=335
T_MIN=336 T_SYSDATE=336
T_RANK=337 T_VARIANCE=337
T_ROW_NUMBER=338 T_USER=338
T_STDEV=339 T_ADD=339
T_VAR=340 T_COLON=340
T_VARIANCE=341 T_COMMA=341
T_OVER=342 T_PIPE=342
T_PARTITION=343 T_DIV=343
T_ACTIVITY_COUNT=344 T_DOT=344
T_CAST=345 T_DOT2=345
T_CURRENT_DATE=346 T_EQUAL=346
T_CURRENT_TIMESTAMP=347 T_EQUAL2=347
T_CURRENT_USER=348 T_SHARP=348
T_USER=349 T_NOTE=349
T_MAX_PART_STRING=350 T_NOTEQUAL=350
T_MIN_PART_STRING=351 T_NOTEQUAL2=351
T_MAX_PART_INT=352 T_GREATER=352
T_MIN_PART_INT=353 T_GREATEREQUAL=353
T_MAX_PART_DATE=354 T_LESS=354
T_MIN_PART_DATE=355 T_LESSEQUAL=355
T_PART_COUNT=356 T_MUL=356
T_PART_LOC=357 T_PRECENT=357
T_TRIM=358 T_CALLS=358
T_SUBSTRING=359 T_OPEN_B=359
T_SYSDATE=360 T_OPEN_P=360
T_HIVE=361 T_OPEN_SB=361
T_HOST=362 T_CLOSE_B=362
L_FILE=363 T_CLOSE_P=363
L_D_STRING=364 T_CLOSE_SB=364
L_DEC=365 T_SEMICOLON=365
T_TRUE=366 T_SUB=366
T_FALSE=367 L_ID=367
T_DIR=368 L_S_STRING=368
T_FILE=369 L_D_STRING=369
T_FILES=370 L_INT=370
T_NEW=371 L_DEC=371
T_PWD=372 L_WS=372
T_SESSIONS=373 L_M_COMMENT=373
T_SUBDIR=374 L_S_COMMENT=374
'@'=1 L_FILE=375
'#'=2 L_LABEL=376
'/'=3 '+'=339
'%'=4 ':'=340
'.'=5 ','=341
'*'=6 '||'=342
'!'=7 '/'=343
';'=8 '.'=344
'-'=9 '..'=345
'+'=10 '='=346
'=='=347
'#'=348
'!'=349
'<>'=350
'!='=351
'>'=352
'>='=353
'<'=354
'<='=355
'*'=356
'%'=357
'@'=358
'{'=359
'('=360
'['=361
'}'=362
')'=363
']'=364
';'=365
'-'=366

1180
src/lib/hive/HiveSqlLexer.interp
View File

File diff suppressed because one or more lines are too long

3148
src/lib/hive/HiveSqlLexer.js
View File

File diff suppressed because it is too large Load Diff

424
src/lib/hive/HiveSqlLexer.tokens
View File

@ -1,20 +1,404 @@
T__0=1 T_ACTION=1
T__1=2 T_ADD2=2
T__2=3 T_ALL=3
T__3=4 T_ALLOCATE=4
T__4=5 T_ALTER=5
T__5=6 T_AND=6
T__6=7 T_ANSI_NULLS=7
T__7=8 T_ANSI_PADDING=8
T__8=9 T_AS=9
T__9=10 T_ASC=10
'@'=1 T_ASSOCIATE=11
'#'=2 T_AT=12
'/'=3 T_AUTO_INCREMENT=13
'%'=4 T_AVG=14
'.'=5 T_BATCHSIZE=15
'*'=6 T_BEGIN=16
'!'=7 T_BETWEEN=17
';'=8 T_BIGINT=18
'-'=9 T_BINARY_DOUBLE=19
'+'=10 T_BINARY_FLOAT=20
T_BINARY_INTEGER=21
T_BIT=22
T_BODY=23
T_BREAK=24
T_BY=25
T_BYTE=26
T_CALL=27
T_CALLER=28
T_CASCADE=29
T_CASE=30
T_CASESPECIFIC=31
T_CAST=32
T_CHAR=33
T_CHARACTER=34
T_CHARSET=35
T_CLIENT=36
T_CLOSE=37
T_CLUSTERED=38
T_CMP=39
T_COLLECT=40
T_COLLECTION=41
T_COLUMN=42
T_COMMENT=43
T_CONSTANT=44
T_COMMIT=45
T_COMPRESS=46
T_CONCAT=47
T_CONDITION=48
T_CONSTRAINT=49
T_CONTINUE=50
T_COPY=51
T_COUNT=52
T_COUNT_BIG=53
T_CREATE=54
T_CREATION=55
T_CREATOR=56
T_CS=57
T_CURRENT=58
T_CURRENT_SCHEMA=59
T_CURSOR=60
T_DATABASE=61
T_DATA=62
T_DATE=63
T_DATETIME=64
T_DAY=65
T_DAYS=66
T_DEC=67
T_DECIMAL=68
T_DECLARE=69
T_DEFAULT=70
T_DEFERRED=71
T_DEFINED=72
T_DEFINER=73
T_DEFINITION=74
T_DELETE=75
T_DELIMITED=76
T_DELIMITER=77
T_DESC=78
T_DESCRIBE=79
T_DIAGNOSTICS=80
T_DIR=81
T_DIRECTORY=82
T_DISTINCT=83
T_DISTRIBUTE=84
T_DO=85
T_DOUBLE=86
T_DROP=87
T_DYNAMIC=88
T_ELSE=89
T_ELSEIF=90
T_ELSIF=91
T_ENABLE=92
T_END=93
T_ENGINE=94
T_ESCAPED=95
T_EXCEPT=96
T_EXEC=97
T_EXECUTE=98
T_EXCEPTION=99
T_EXCLUSIVE=100
T_EXISTS=101
T_EXIT=102
T_FALLBACK=103
T_FALSE=104
T_FETCH=105
T_FIELDS=106
T_FILE=107
T_FILES=108
T_FLOAT=109
T_FOR=110
T_FOREIGN=111
T_FORMAT=112
T_FOUND=113
T_FROM=114
T_FULL=115
T_FUNCTION=116
T_GET=117
T_GLOBAL=118
T_GO=119
T_GRANT=120
T_GROUP=121
T_HANDLER=122
T_HASH=123
T_HAVING=124
T_HDFS=125
T_HIVE=126
T_HOST=127
T_IDENTITY=128
T_IF=129
T_IGNORE=130
T_IMMEDIATE=131
T_IN=132
T_INCLUDE=133
T_INDEX=134
T_INITRANS=135
T_INNER=136
T_INOUT=137
T_INSERT=138
T_INT=139
T_INT2=140
T_INT4=141
T_INT8=142
T_INTEGER=143
T_INTERSECT=144
T_INTERVAL=145
T_INTO=146
T_INVOKER=147
T_IS=148
T_ISOPEN=149
T_ITEMS=150
T_JOIN=151
T_KEEP=152
T_KEY=153
T_KEYS=154
T_LANGUAGE=155
T_LEAVE=156
T_LEFT=157
T_LIKE=158
T_LIMIT=159
T_LINES=160
T_LOCAL=161
T_LOCATION=162
T_LOCATOR=163
T_LOCATORS=164
T_LOCKS=165
T_LOG=166
T_LOGGED=167
T_LOGGING=168
T_LOOP=169
T_MAP=170
T_MATCHED=171
T_MAX=172
T_MAXTRANS=173
T_MERGE=174
T_MESSAGE_TEXT=175
T_MICROSECOND=176
T_MICROSECONDS=177
T_MIN=178
T_MULTISET=179
T_NCHAR=180
T_NEW=181
T_NVARCHAR=182
T_NO=183
T_NOCOUNT=184
T_NOCOMPRESS=185
T_NOLOGGING=186
T_NONE=187
T_NOT=188
T_NOTFOUND=189
T_NULL=190
T_NUMERIC=191
T_NUMBER=192
T_OBJECT=193
T_OFF=194
T_ON=195
T_ONLY=196
T_OPEN=197
T_OR=198
T_ORDER=199
T_OUT=200
T_OUTER=201
T_OVER=202
T_OVERWRITE=203
T_OWNER=204
T_PACKAGE=205
T_PARTITION=206
T_PCTFREE=207
T_PCTUSED=208
T_PLS_INTEGER=209
T_PRECISION=210
T_PRESERVE=211
T_PRIMARY=212
T_PRINT=213
T_PROC=214
T_PROCEDURE=215
T_QUALIFY=216
T_QUERY_BAND=217
T_QUIT=218
T_QUOTED_IDENTIFIER=219
T_RAISE=220
T_REAL=221
T_REFERENCES=222
T_REGEXP=223
T_REPLACE=224
T_RESIGNAL=225
T_RESTRICT=226
T_RESULT=227
T_RESULT_SET_LOCATOR=228
T_RETURN=229
T_RETURNS=230
T_REVERSE=231
T_RIGHT=232
T_RLIKE=233
T_ROLE=234
T_ROLLBACK=235
T_ROW=236
T_ROWS=237
T_ROWTYPE=238
T_ROW_COUNT=239
T_RR=240
T_RS=241
T_PWD=242
T_TRIM=243
T_SCHEMA=244
T_SECOND=245
T_SECONDS=246
T_SECURITY=247
T_SEGMENT=248
T_SEL=249
T_SELECT=250
T_SET=251
T_SESSION=252
T_SESSIONS=253
T_SETS=254
T_SHARE=255
T_SIGNAL=256
T_SIMPLE_DOUBLE=257
T_SIMPLE_FLOAT=258
T_SIMPLE_INTEGER=259
T_SMALLDATETIME=260
T_SMALLINT=261
T_SQL=262
T_SQLEXCEPTION=263
T_SQLINSERT=264
T_SQLSTATE=265
T_SQLWARNING=266
T_STATS=267
T_STATISTICS=268
T_STEP=269
T_STORAGE=270
T_STORED=271
T_STRING=272
T_SUBDIR=273
T_SUBSTRING=274
T_SUM=275
T_SUMMARY=276
T_SYS_REFCURSOR=277
T_TABLE=278
T_TABLESPACE=279
T_TEMPORARY=280
T_TERMINATED=281
T_TEXTIMAGE_ON=282
T_THEN=283
T_TIMESTAMP=284
T_TINYINT=285
T_TITLE=286
T_TO=287
T_TOP=288
T_TRANSACTION=289
T_TRUE=290
T_TRUNCATE=291
T_TYPE=292
T_UNION=293
T_UNIQUE=294
T_UPDATE=295
T_UR=296
T_USE=297
T_USING=298
T_VALUE=299
T_VALUES=300
T_VAR=301
T_VARCHAR=302
T_VARCHAR2=303
T_VARYING=304
T_VOLATILE=305
T_WHEN=306
T_WHERE=307
T_WHILE=308
T_WITH=309
T_WITHOUT=310
T_WORK=311
T_XACT_ABORT=312
T_XML=313
T_YES=314
T_ACTIVITY_COUNT=315
T_CUME_DIST=316
T_CURRENT_DATE=317
T_CURRENT_TIMESTAMP=318
T_CURRENT_USER=319
T_DENSE_RANK=320
T_FIRST_VALUE=321
T_LAG=322
T_LAST_VALUE=323
T_LEAD=324
T_MAX_PART_STRING=325
T_MIN_PART_STRING=326
T_MAX_PART_INT=327
T_MIN_PART_INT=328
T_MAX_PART_DATE=329
T_MIN_PART_DATE=330
T_PART_COUNT=331
T_PART_LOC=332
T_RANK=333
T_ROW_NUMBER=334
T_STDEV=335
T_SYSDATE=336
T_VARIANCE=337
T_USER=338
T_ADD=339
T_COLON=340
T_COMMA=341
T_PIPE=342
T_DIV=343
T_DOT=344
T_DOT2=345
T_EQUAL=346
T_EQUAL2=347
T_SHARP=348
T_NOTE=349
T_NOTEQUAL=350
T_NOTEQUAL2=351
T_GREATER=352
T_GREATEREQUAL=353
T_LESS=354
T_LESSEQUAL=355
T_MUL=356
T_PRECENT=357
T_CALLS=358
T_OPEN_B=359
T_OPEN_P=360
T_OPEN_SB=361
T_CLOSE_B=362
T_CLOSE_P=363
T_CLOSE_SB=364
T_SEMICOLON=365
T_SUB=366
L_ID=367
L_S_STRING=368
L_D_STRING=369
L_INT=370
L_DEC=371
L_WS=372
L_M_COMMENT=373
L_S_COMMENT=374
L_FILE=375
L_LABEL=376
'+'=339
':'=340
','=341
'||'=342
'/'=343
'.'=344
'..'=345
'='=346
'=='=347
'#'=348
'!'=349
'<>'=350
'!='=351
'>'=352
'>='=353
'<'=354
'<='=355
'*'=356
'%'=357
'@'=358
'{'=359
'('=360
'['=361
'}'=362
')'=363
']'=364
';'=365
'-'=366

912
src/lib/hive/HiveSqlListener.js
View File

File diff suppressed because it is too large Load Diff

458
src/lib/hive/HiveSqlVisitor.js
View File

File diff suppressed because it is too large Load Diff

4
src/parser/hive.ts
View File

@ -1,6 +1,6 @@
import { InputStream, CommonTokenStream, Lexer } from 'antlr4'; import { InputStream, CommonTokenStream, Lexer } from 'antlr4';
import { HiveSqlLexer } from '../lib/hive/HiveSqlLexer'; import { HiveSqlLexer } from '../lib/hive/HiveSqlLexer';
import { HiveSqlParser } from '../lib/hive/HiveSqlParser'; import { HiveSql } from '../lib/hive/HiveSql';
export * from '../lib/hive/HiveSqlListener'; export * from '../lib/hive/HiveSqlListener';
export * from '../lib/hive/HiveSqlVisitor'; export * from '../lib/hive/HiveSqlVisitor';
@ -14,7 +14,7 @@ export default class HiveSQL extends BasicParser {
} }
public createParserFromLexer(lexer: Lexer) { public createParserFromLexer(lexer: Lexer) {
const tokenStream = new CommonTokenStream(lexer); const tokenStream = new CommonTokenStream(lexer);
return new HiveSqlParser(tokenStream); return new HiveSql(tokenStream);
} }
} }

10
test/parser/hive/lexer.test.ts
View File

@ -2,11 +2,15 @@ import SQLParser from '../../../src/parser/hive';
describe('HiveSQL Lexer tests', () => { describe('HiveSQL Lexer tests', () => {
const parser = new SQLParser(); const parser = new SQLParser();
// select id,name,sex from user1; test('select token counts', () => {
const sql = 'SELECT * FROM t1'; const sql = 'SELECT * FROM t1';
const tokens = parser.getAllTokens(sql); const tokens = parser.getAllTokens(sql);
expect(tokens.length).toBe(4);
});
test('token counts', () => { test('select token counts', () => {
expect(tokens.length).toBe(12); const sql = 'show create table_name;';
const tokens = parser.getAllTokens(sql);
expect(tokens.length).toBe(4);
}); });
}); });

25
test/parser/hive/listener.test.ts
View File

@ -1,17 +1,16 @@
import SQLParser, { HiveSqlListener } from '../../../src/parser/hive'; import SQLParser, { HiveSqlListener } from '../../../src/parser/hive';
describe('Hive SQL Listener Tests', () => { describe('Hive SQL Listener Tests', () => {
const expectTableName = 'user1';
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SQLParser(); const parser = new SQLParser();
test('Listener enterSelectList', async () => {
const expectTableName = 'userName';
const sql = `select ${expectTableName} from user1 where inc_day='20190601' limit 1000;`;
const parserTree = parser.parse(sql); const parserTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = ''; let result = '';
class MyListener extends HiveSqlListener { class MyListener extends HiveSqlListener {
enterTableName(ctx): void { enterSelect_list(ctx): void {
result = ctx.getText().toLowerCase(); result = ctx.getText();
} }
} }
const listenTableName: any = new MyListener(); const listenTableName: any = new MyListener();
@ -19,4 +18,18 @@ describe('Hive SQL Listener Tests', () => {
await parser.listen(listenTableName, parserTree); await parser.listen(listenTableName, parserTree);
expect(result).toBe(expectTableName); expect(result).toBe(expectTableName);
}); });
test('Listener enterCreateTable', async () => {
const sql = `drop table table_name;`;
const parserTree = parser.parse(sql);
let result = '';
class MyListener extends HiveSqlListener {
enterDrop_stmt(ctx): void {
result = ctx.getText();
}
}
const listenTableName: any = new MyListener();
await parser.listen(listenTableName, parserTree);
expect(result).toBe('droptabletable_name');
});
}); });

24
test/parser/hive/syntax.test.ts
View File

@ -2,17 +2,21 @@ import SQLParser from '../../../src/parser/hive';
describe('Hive SQL Syntax Tests', () => { describe('Hive SQL Syntax Tests', () => {
const parser = new SQLParser(); const parser = new SQLParser();
test('Create Table Statement', () => {
test('Select Statement', () => { const sql = 'CREATE TABLE person(name STRING,age INT);';
const sql = 'SELECT * FROM employee WHERE salary>30000;';
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Select 1+1', () => {
const sql = 'SELECT 1+1;';
const result = parser.validate(sql); const result = parser.validate(sql);
expect(result.length).toBe(0); expect(result.length).toBe(0);
}); });
test('Create Table Statement', () => {
const sql = `alter table dm_gis.table_name add if not exists partition (inc_day = '20190601');`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Wrong Select Statement', () => {
const sql = 'SELECT add ABC from Where ;'
const result = parser.validate(sql);
expect(result.length).toBe(2);
expect(result[0].message).toBe(`no viable alternative at input 'SELECTaddABCfromWhere'`)
expect(result[1].message).toBe(`mismatched input 'Where' expecting <EOF>`)
});
}); });

7
test/parser/hive/visitor.test.ts
View File

@ -1,15 +1,15 @@
import SQLParser, { HiveSqlVisitor } from '../../../src/parser/hive'; import SQLParser, { HiveSqlVisitor } from '../../../src/parser/hive';
describe('Generic SQL Visitor Tests', () => { describe('Generic SQL Visitor Tests', () => {
const expectTableName = 'user1'; const expectTableName = 'dm_gis.dlv_addr_tc_count';
const sql = `select id,name,sex from ${expectTableName};`; const sql = `select citycode,tc,inc_day from ${expectTableName} where inc_day='20190501' limit 100;`;
const parser = new SQLParser(); const parser = new SQLParser();
const parserTree = parser.parse(sql, (error) => { const parserTree = parser.parse(sql, (error) => {
console.log('Parse error:', error); console.log('Parse error:', error);
}); });
console.log('Parser tree string:', parser.toString(parserTree)); // console.log('Parser tree string:', parser.toString(parserTree));
test('Visitor visitTableName', () => { test('Visitor visitTableName', () => {
let result = ''; let result = '';
@ -19,6 +19,7 @@ describe('Generic SQL Visitor Tests', () => {
super.visitTable_name(ctx); super.visitTable_name(ctx);
} }
} }
const visitor: any = new MyVisitor(); const visitor: any = new MyVisitor();
visitor.visit(parserTree); visitor.visit(parserTree);