oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

modify grammar, add syntax test

Browse Source
This commit is contained in:
xigua
2020-11-05 20:37:07 +08:00
parent bfe055be71
commit 0d47b7debc
12 changed files with 40947 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
src/grammar/spark/SqlBase.g4
View File

@ -17,66 +17,62 @@
grammar SqlBase;
@parser::members {
/**
* When false, INTERSECT is given the greater precedence over the other set
* operations (UNION, EXCEPT and MINUS) as per the SQL standard.
*/
// public boolean legacy_setops_precedence_enbled = false;
legacy_setops_precedence_enbled = false;
/**
* When false, a literal with an exponent would be converted into
* double type rather than decimal type.
*/
// public boolean legacy_exponent_literal_as_decimal_enabled = false;
legacy_exponent_literal_as_decimal_enabled = false;
/**
* When true, the behavior of keywords follows ANSI SQL standard.
*/
// public boolean SQL_standard_keyword_behavior = false;
SQL_standard_keyword_behavior = false;
/**
* When false, INTERSECT is given the greater precedence over the other set
* operations (UNION, EXCEPT and MINUS) as per the SQL standard.
*/
// public boolean legacy_setops_precedence_enbled = false;
/**
* When false, a literal with an exponent would be converted into
* double type rather than decimal type.
*/
// public boolean legacy_exponent_literal_as_decimal_enabled = false;
global.legacy_exponent_literal_as_decimal_enabled = false;
/**
* When true, the behavior of keywords follows ANSI SQL standard.
*/
// public boolean SQL_standard_keyword_behavior = false;
global.legacy_setops_precedence_enbled = false;
global.legacy_exponent_literal_as_decimal_enabled = false;
global.SQL_standard_keyword_behavior = false;
}
//@lexer::members {
// /**
// * Verify whether current token is a valid decimal token (which contains dot).
// * Returns true if the character that follows the token is not a digit or letter or underscore.
// *
// * For example:
// * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
// * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
// * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
// * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
// * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
// * which is not a digit or letter or underscore.
// */
// public boolean isValidDecimal() {
// int nextChar = _input.LA(1);
// if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' ||
// nextChar == '_') {
// return false;
// } else {
// return true;
// }
// }
//
// /**
// * This method will be called when we see '/*' and try to match it as a bracketed comment.
// * If the next character is '+', it should be parsed as hint later, and we cannot match
// * it as a bracketed comment.
// *
// * Returns true if the next character is '+'.
// */
// public boolean isHint() {
// int nextChar = _input.LA(1);
// if (nextChar == '+') {
// return true;
// } else {
// return false;
// }
// }
//}
@lexer::members {
var ctx = this
/**
* Verify whether current token is a valid decimal token (which contains dot).
* Returns true if the character that follows the token is not a digit or letter or underscore.
*
* For example:
* For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'.
* For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'.
* For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'.
* For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed
* by a space. 34.E2 is a valid decimal token because it is followed by symbol '+'
* which is not a digit or letter or underscore.
*/
global.isValidDecimal = function() {
let nextChar = ctx._input.LA(1);
return !(nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' || nextChar == '_')
}
program: statement EOF;
/**
* This method will be called when we see '/*' and try to match it as a bracketed comment.
* If the next character is '+', it should be parsed as hint later, and we cannot match
* it as a bracketed comment.
*
* Returns true if the next character is '+'.
*/
global.isHint = function() {
let nextChar = ctx._input.LA(1);
return nextChar == '+'
}
}
program
: singleStatement EOF
;
singleStatement
: statement ';'* EOF

735
src/lib/spark/SqlBase.interp Normal file
View File

File diff suppressed because one or more lines are too long

565
src/lib/spark/SqlBase.tokens Normal file
View File

@ -0,0 +1,565 @@
T__0=1
T__1=2
T__2=3
T__3=4
T__4=5
T__5=6
T__6=7
T__7=8
T__8=9
T__9=10
T__10=11
ADD=12
AFTER=13
ALL=14
ALTER=15
ANALYZE=16
AND=17
ANTI=18
ANY=19
ARCHIVE=20
ARRAY=21
AS=22
ASC=23
AT=24
AUTHORIZATION=25
BETWEEN=26
BOTH=27
BUCKET=28
BUCKETS=29
BY=30
CACHE=31
CASCADE=32
CASE=33
CAST=34
CHANGE=35
CHECK=36
CLEAR=37
CLUSTER=38
CLUSTERED=39
CODEGEN=40
COLLATE=41
COLLECTION=42
COLUMN=43
COLUMNS=44
COMMENT=45
COMMIT=46
COMPACT=47
COMPACTIONS=48
COMPUTE=49
CONCATENATE=50
CONSTRAINT=51
COST=52
CREATE=53
CROSS=54
CUBE=55
CURRENT=56
CURRENT_DATE=57
CURRENT_TIME=58
CURRENT_TIMESTAMP=59
CURRENT_USER=60
DATA=61
DATABASE=62
DATABASES=63
DBPROPERTIES=64
DEFINED=65
DELETE=66
DELIMITED=67
DESC=68
DESCRIBE=69
DFS=70
DIRECTORIES=71
DIRECTORY=72
DISTINCT=73
DISTRIBUTE=74
DIV=75
DROP=76
ELSE=77
END=78
ESCAPE=79
ESCAPED=80
EXCEPT=81
EXCHANGE=82
EXISTS=83
EXPLAIN=84
EXPORT=85
EXTENDED=86
EXTERNAL=87
EXTRACT=88
FALSE=89
FETCH=90
FIELDS=91
FILTER=92
FILEFORMAT=93
FIRST=94
FOLLOWING=95
FOR=96
FOREIGN=97
FORMAT=98
FORMATTED=99
FROM=100
FULL=101
FUNCTION=102
FUNCTIONS=103
GLOBAL=104
GRANT=105
GROUP=106
GROUPING=107
HAVING=108
IF=109
IGNORE=110
IMPORT=111
IN=112
INDEX=113
INDEXES=114
INNER=115
INPATH=116
INPUTFORMAT=117
INSERT=118
INTERSECT=119
INTERVAL=120
INTO=121
IS=122
ITEMS=123
JOIN=124
KEYS=125
LAST=126
LATERAL=127
LAZY=128
LEADING=129
LEFT=130
LIKE=131
LIMIT=132
LINES=133
LIST=134
LOAD=135
LOCAL=136
LOCATION=137
LOCK=138
LOCKS=139
LOGICAL=140
MACRO=141
MAP=142
MATCHED=143
MERGE=144
MSCK=145
NAMESPACE=146
NAMESPACES=147
NATURAL=148
NO=149
NOT=150
NULL=151
NULLS=152
OF=153
ON=154
ONLY=155
OPTION=156
OPTIONS=157
OR=158
ORDER=159
OUT=160
OUTER=161
OUTPUTFORMAT=162
OVER=163
OVERLAPS=164
OVERLAY=165
OVERWRITE=166
PARTITION=167
PARTITIONED=168
PARTITIONS=169
PERCENTLIT=170
PIVOT=171
PLACING=172
POSITION=173
PRECEDING=174
PRIMARY=175
PRINCIPALS=176
PROPERTIES=177
PURGE=178
QUERY=179
RANGE=180
RECORDREADER=181
RECORDWRITER=182
RECOVER=183
REDUCE=184
REFERENCES=185
REFRESH=186
RENAME=187
REPAIR=188
REPLACE=189
RESET=190
RESTRICT=191
REVOKE=192
RIGHT=193
RLIKE=194
ROLE=195
ROLES=196
ROLLBACK=197
ROLLUP=198
ROW=199
ROWS=200
SCHEMA=201
SELECT=202
SEMI=203
SEPARATED=204
SERDE=205
SERDEPROPERTIES=206
SESSION_USER=207
SET=208
SETMINUS=209
SETS=210
SHOW=211
SKEWED=212
SOME=213
SORT=214
SORTED=215
START=216
STATISTICS=217
STORED=218
STRATIFY=219
STRUCT=220
SUBSTR=221
SUBSTRING=222
TABLE=223
TABLES=224
TABLESAMPLE=225
TBLPROPERTIES=226
TEMPORARY=227
TERMINATED=228
THEN=229
TIME=230
TO=231
TOUCH=232
TRAILING=233
TRANSACTION=234
TRANSACTIONS=235
TRANSFORM=236
TRIM=237
TRUE=238
TRUNCATE=239
TYPE=240
UNARCHIVE=241
UNBOUNDED=242
UNCACHE=243
UNION=244
UNIQUE=245
UNKNOWN=246
UNLOCK=247
UNSET=248
UPDATE=249
USE=250
USER=251
USING=252
VALUES=253
VIEW=254
VIEWS=255
WHEN=256
WHERE=257
WINDOW=258
WITH=259
ZONE=260
EQ=261
NSEQ=262
NEQ=263
NEQJ=264
LT=265
LTE=266
GT=267
GTE=268
PLUS=269
MINUS=270
ASTERISK=271
SLASH=272
PERCENT=273
TILDE=274
AMPERSAND=275
PIPE=276
CONCAT_PIPE=277
HAT=278
STRING=279
BIGINT_LITERAL=280
SMALLINT_LITERAL=281
TINYINT_LITERAL=282
INTEGER_VALUE=283
EXPONENT_VALUE=284
DECIMAL_VALUE=285
FLOAT_LITERAL=286
DOUBLE_LITERAL=287
BIGDECIMAL_LITERAL=288
IDENTIFIER=289
BACKQUOTED_IDENTIFIER=290
SIMPLE_COMMENT=291
BRACKETED_COMMENT=292
WS=293
UNRECOGNIZED=294
';'=1
'('=2
')'=3
','=4
'.'=5
'/*+'=6
'*/'=7
'->'=8
'['=9
']'=10
':'=11
'ADD'=12
'AFTER'=13
'ALL'=14
'ALTER'=15
'ANALYZE'=16
'AND'=17
'ANTI'=18
'ANY'=19
'ARCHIVE'=20
'ARRAY'=21
'AS'=22
'ASC'=23
'AT'=24
'AUTHORIZATION'=25
'BETWEEN'=26
'BOTH'=27
'BUCKET'=28
'BUCKETS'=29
'BY'=30
'CACHE'=31
'CASCADE'=32
'CASE'=33
'CAST'=34
'CHANGE'=35
'CHECK'=36
'CLEAR'=37
'CLUSTER'=38
'CLUSTERED'=39
'CODEGEN'=40
'COLLATE'=41
'COLLECTION'=42
'COLUMN'=43
'COLUMNS'=44
'COMMENT'=45
'COMMIT'=46
'COMPACT'=47
'COMPACTIONS'=48
'COMPUTE'=49
'CONCATENATE'=50
'CONSTRAINT'=51
'COST'=52
'CREATE'=53
'CROSS'=54
'CUBE'=55
'CURRENT'=56
'CURRENT_DATE'=57
'CURRENT_TIME'=58
'CURRENT_TIMESTAMP'=59
'CURRENT_USER'=60
'DATA'=61
'DATABASE'=62
'DBPROPERTIES'=64
'DEFINED'=65
'DELETE'=66
'DELIMITED'=67
'DESC'=68
'DESCRIBE'=69
'DFS'=70
'DIRECTORIES'=71
'DIRECTORY'=72
'DISTINCT'=73
'DISTRIBUTE'=74
'DIV'=75
'DROP'=76
'ELSE'=77
'END'=78
'ESCAPE'=79
'ESCAPED'=80
'EXCEPT'=81
'EXCHANGE'=82
'EXISTS'=83
'EXPLAIN'=84
'EXPORT'=85
'EXTENDED'=86
'EXTERNAL'=87
'EXTRACT'=88
'FALSE'=89
'FETCH'=90
'FIELDS'=91
'FILTER'=92
'FILEFORMAT'=93
'FIRST'=94
'FOLLOWING'=95
'FOR'=96
'FOREIGN'=97
'FORMAT'=98
'FORMATTED'=99
'FROM'=100
'FULL'=101
'FUNCTION'=102
'FUNCTIONS'=103
'GLOBAL'=104
'GRANT'=105
'GROUP'=106
'GROUPING'=107
'HAVING'=108
'IF'=109
'IGNORE'=110
'IMPORT'=111
'IN'=112
'INDEX'=113
'INDEXES'=114
'INNER'=115
'INPATH'=116
'INPUTFORMAT'=117
'INSERT'=118
'INTERSECT'=119
'INTERVAL'=120
'INTO'=121
'IS'=122
'ITEMS'=123
'JOIN'=124
'KEYS'=125
'LAST'=126
'LATERAL'=127
'LAZY'=128
'LEADING'=129
'LEFT'=130
'LIKE'=131
'LIMIT'=132
'LINES'=133
'LIST'=134
'LOAD'=135
'LOCAL'=136
'LOCATION'=137
'LOCK'=138
'LOCKS'=139
'LOGICAL'=140
'MACRO'=141
'MAP'=142
'MATCHED'=143
'MERGE'=144
'MSCK'=145
'NAMESPACE'=146
'NAMESPACES'=147
'NATURAL'=148
'NO'=149
'NULL'=151
'NULLS'=152
'OF'=153
'ON'=154
'ONLY'=155
'OPTION'=156
'OPTIONS'=157
'OR'=158
'ORDER'=159
'OUT'=160
'OUTER'=161
'OUTPUTFORMAT'=162
'OVER'=163
'OVERLAPS'=164
'OVERLAY'=165
'OVERWRITE'=166
'PARTITION'=167
'PARTITIONED'=168
'PARTITIONS'=169
'PERCENT'=170
'PIVOT'=171
'PLACING'=172
'POSITION'=173
'PRECEDING'=174
'PRIMARY'=175
'PRINCIPALS'=176
'PROPERTIES'=177
'PURGE'=178
'QUERY'=179
'RANGE'=180
'RECORDREADER'=181
'RECORDWRITER'=182
'RECOVER'=183
'REDUCE'=184
'REFERENCES'=185
'REFRESH'=186
'RENAME'=187
'REPAIR'=188
'REPLACE'=189
'RESET'=190
'RESTRICT'=191
'REVOKE'=192
'RIGHT'=193
'ROLE'=195
'ROLES'=196
'ROLLBACK'=197
'ROLLUP'=198
'ROW'=199
'ROWS'=200
'SCHEMA'=201
'SELECT'=202
'SEMI'=203
'SEPARATED'=204
'SERDE'=205
'SERDEPROPERTIES'=206
'SESSION_USER'=207
'SET'=208
'MINUS'=209
'SETS'=210
'SHOW'=211
'SKEWED'=212
'SOME'=213
'SORT'=214
'SORTED'=215
'START'=216
'STATISTICS'=217
'STORED'=218
'STRATIFY'=219
'STRUCT'=220
'SUBSTR'=221
'SUBSTRING'=222
'TABLE'=223
'TABLES'=224
'TABLESAMPLE'=225
'TBLPROPERTIES'=226
'TERMINATED'=228
'THEN'=229
'TIME'=230
'TO'=231
'TOUCH'=232
'TRAILING'=233
'TRANSACTION'=234
'TRANSACTIONS'=235
'TRANSFORM'=236
'TRIM'=237
'TRUE'=238
'TRUNCATE'=239
'TYPE'=240
'UNARCHIVE'=241
'UNBOUNDED'=242
'UNCACHE'=243
'UNION'=244
'UNIQUE'=245
'UNKNOWN'=246
'UNLOCK'=247
'UNSET'=248
'UPDATE'=249
'USE'=250
'USER'=251
'USING'=252
'VALUES'=253
'VIEW'=254
'VIEWS'=255
'WHEN'=256
'WHERE'=257
'WINDOW'=258
'WITH'=259
'ZONE'=260
'<=>'=262
'<>'=263
'!='=264
'<'=265
'>'=267
'+'=269
'-'=270
'*'=271
'/'=272
'%'=273
'~'=274
'&'=275
'|'=276
'||'=277
'^'=278

903
src/lib/spark/SqlBaseLexer.interp Normal file
View File

File diff suppressed because one or more lines are too long

2436
src/lib/spark/SqlBaseLexer.js Normal file
View File

File diff suppressed because it is too large Load Diff

565
src/lib/spark/SqlBaseLexer.tokens Normal file
View File

@ -0,0 +1,565 @@
T__0=1
T__1=2
T__2=3
T__3=4
T__4=5
T__5=6
T__6=7
T__7=8
T__8=9
T__9=10
T__10=11
ADD=12
AFTER=13
ALL=14
ALTER=15
ANALYZE=16
AND=17
ANTI=18
ANY=19
ARCHIVE=20
ARRAY=21
AS=22
ASC=23
AT=24
AUTHORIZATION=25
BETWEEN=26
BOTH=27
BUCKET=28
BUCKETS=29
BY=30
CACHE=31
CASCADE=32
CASE=33
CAST=34
CHANGE=35
CHECK=36
CLEAR=37
CLUSTER=38
CLUSTERED=39
CODEGEN=40
COLLATE=41
COLLECTION=42
COLUMN=43
COLUMNS=44
COMMENT=45
COMMIT=46
COMPACT=47
COMPACTIONS=48
COMPUTE=49
CONCATENATE=50
CONSTRAINT=51
COST=52
CREATE=53
CROSS=54
CUBE=55
CURRENT=56
CURRENT_DATE=57
CURRENT_TIME=58
CURRENT_TIMESTAMP=59
CURRENT_USER=60
DATA=61
DATABASE=62
DATABASES=63
DBPROPERTIES=64
DEFINED=65
DELETE=66
DELIMITED=67
DESC=68
DESCRIBE=69
DFS=70
DIRECTORIES=71
DIRECTORY=72
DISTINCT=73
DISTRIBUTE=74
DIV=75
DROP=76
ELSE=77
END=78
ESCAPE=79
ESCAPED=80
EXCEPT=81
EXCHANGE=82
EXISTS=83
EXPLAIN=84
EXPORT=85
EXTENDED=86
EXTERNAL=87
EXTRACT=88
FALSE=89
FETCH=90
FIELDS=91
FILTER=92
FILEFORMAT=93
FIRST=94
FOLLOWING=95
FOR=96
FOREIGN=97
FORMAT=98
FORMATTED=99
FROM=100
FULL=101
FUNCTION=102
FUNCTIONS=103
GLOBAL=104
GRANT=105
GROUP=106
GROUPING=107
HAVING=108
IF=109
IGNORE=110
IMPORT=111
IN=112
INDEX=113
INDEXES=114
INNER=115
INPATH=116
INPUTFORMAT=117
INSERT=118
INTERSECT=119
INTERVAL=120
INTO=121
IS=122
ITEMS=123
JOIN=124
KEYS=125
LAST=126
LATERAL=127
LAZY=128
LEADING=129
LEFT=130
LIKE=131
LIMIT=132
LINES=133
LIST=134
LOAD=135
LOCAL=136
LOCATION=137
LOCK=138
LOCKS=139
LOGICAL=140
MACRO=141
MAP=142
MATCHED=143
MERGE=144
MSCK=145
NAMESPACE=146
NAMESPACES=147
NATURAL=148
NO=149
NOT=150
NULL=151
NULLS=152
OF=153
ON=154
ONLY=155
OPTION=156
OPTIONS=157
OR=158
ORDER=159
OUT=160
OUTER=161
OUTPUTFORMAT=162
OVER=163
OVERLAPS=164
OVERLAY=165
OVERWRITE=166
PARTITION=167
PARTITIONED=168
PARTITIONS=169
PERCENTLIT=170
PIVOT=171
PLACING=172
POSITION=173
PRECEDING=174
PRIMARY=175
PRINCIPALS=176
PROPERTIES=177
PURGE=178
QUERY=179
RANGE=180
RECORDREADER=181
RECORDWRITER=182
RECOVER=183
REDUCE=184
REFERENCES=185
REFRESH=186
RENAME=187
REPAIR=188
REPLACE=189
RESET=190
RESTRICT=191
REVOKE=192
RIGHT=193
RLIKE=194
ROLE=195
ROLES=196
ROLLBACK=197
ROLLUP=198
ROW=199
ROWS=200
SCHEMA=201
SELECT=202
SEMI=203
SEPARATED=204
SERDE=205
SERDEPROPERTIES=206
SESSION_USER=207
SET=208
SETMINUS=209
SETS=210
SHOW=211
SKEWED=212
SOME=213
SORT=214
SORTED=215
START=216
STATISTICS=217
STORED=218
STRATIFY=219
STRUCT=220
SUBSTR=221
SUBSTRING=222
TABLE=223
TABLES=224
TABLESAMPLE=225
TBLPROPERTIES=226
TEMPORARY=227
TERMINATED=228
THEN=229
TIME=230
TO=231
TOUCH=232
TRAILING=233
TRANSACTION=234
TRANSACTIONS=235
TRANSFORM=236
TRIM=237
TRUE=238
TRUNCATE=239
TYPE=240
UNARCHIVE=241
UNBOUNDED=242
UNCACHE=243
UNION=244
UNIQUE=245
UNKNOWN=246
UNLOCK=247
UNSET=248
UPDATE=249
USE=250
USER=251
USING=252
VALUES=253
VIEW=254
VIEWS=255
WHEN=256
WHERE=257
WINDOW=258
WITH=259
ZONE=260
EQ=261
NSEQ=262
NEQ=263
NEQJ=264
LT=265
LTE=266
GT=267
GTE=268
PLUS=269
MINUS=270
ASTERISK=271
SLASH=272
PERCENT=273
TILDE=274
AMPERSAND=275
PIPE=276
CONCAT_PIPE=277
HAT=278
STRING=279
BIGINT_LITERAL=280
SMALLINT_LITERAL=281
TINYINT_LITERAL=282
INTEGER_VALUE=283
EXPONENT_VALUE=284
DECIMAL_VALUE=285
FLOAT_LITERAL=286
DOUBLE_LITERAL=287
BIGDECIMAL_LITERAL=288
IDENTIFIER=289
BACKQUOTED_IDENTIFIER=290
SIMPLE_COMMENT=291
BRACKETED_COMMENT=292
WS=293
UNRECOGNIZED=294
';'=1
'('=2
')'=3
','=4
'.'=5
'/*+'=6
'*/'=7
'->'=8
'['=9
']'=10
':'=11
'ADD'=12
'AFTER'=13
'ALL'=14
'ALTER'=15
'ANALYZE'=16
'AND'=17
'ANTI'=18
'ANY'=19
'ARCHIVE'=20
'ARRAY'=21
'AS'=22
'ASC'=23
'AT'=24
'AUTHORIZATION'=25
'BETWEEN'=26
'BOTH'=27
'BUCKET'=28
'BUCKETS'=29
'BY'=30
'CACHE'=31
'CASCADE'=32
'CASE'=33
'CAST'=34
'CHANGE'=35
'CHECK'=36
'CLEAR'=37
'CLUSTER'=38
'CLUSTERED'=39
'CODEGEN'=40
'COLLATE'=41
'COLLECTION'=42
'COLUMN'=43
'COLUMNS'=44
'COMMENT'=45
'COMMIT'=46
'COMPACT'=47
'COMPACTIONS'=48
'COMPUTE'=49
'CONCATENATE'=50
'CONSTRAINT'=51
'COST'=52
'CREATE'=53
'CROSS'=54
'CUBE'=55
'CURRENT'=56
'CURRENT_DATE'=57
'CURRENT_TIME'=58
'CURRENT_TIMESTAMP'=59
'CURRENT_USER'=60
'DATA'=61
'DATABASE'=62
'DBPROPERTIES'=64
'DEFINED'=65
'DELETE'=66
'DELIMITED'=67
'DESC'=68
'DESCRIBE'=69
'DFS'=70
'DIRECTORIES'=71
'DIRECTORY'=72
'DISTINCT'=73
'DISTRIBUTE'=74
'DIV'=75
'DROP'=76
'ELSE'=77
'END'=78
'ESCAPE'=79
'ESCAPED'=80
'EXCEPT'=81
'EXCHANGE'=82
'EXISTS'=83
'EXPLAIN'=84
'EXPORT'=85
'EXTENDED'=86
'EXTERNAL'=87
'EXTRACT'=88
'FALSE'=89
'FETCH'=90
'FIELDS'=91
'FILTER'=92
'FILEFORMAT'=93
'FIRST'=94
'FOLLOWING'=95
'FOR'=96
'FOREIGN'=97
'FORMAT'=98
'FORMATTED'=99
'FROM'=100
'FULL'=101
'FUNCTION'=102
'FUNCTIONS'=103
'GLOBAL'=104
'GRANT'=105
'GROUP'=106
'GROUPING'=107
'HAVING'=108
'IF'=109
'IGNORE'=110
'IMPORT'=111
'IN'=112
'INDEX'=113
'INDEXES'=114
'INNER'=115
'INPATH'=116
'INPUTFORMAT'=117
'INSERT'=118
'INTERSECT'=119
'INTERVAL'=120
'INTO'=121
'IS'=122
'ITEMS'=123
'JOIN'=124
'KEYS'=125
'LAST'=126
'LATERAL'=127
'LAZY'=128
'LEADING'=129
'LEFT'=130
'LIKE'=131
'LIMIT'=132
'LINES'=133
'LIST'=134
'LOAD'=135
'LOCAL'=136
'LOCATION'=137
'LOCK'=138
'LOCKS'=139
'LOGICAL'=140
'MACRO'=141
'MAP'=142
'MATCHED'=143
'MERGE'=144
'MSCK'=145
'NAMESPACE'=146
'NAMESPACES'=147
'NATURAL'=148
'NO'=149
'NULL'=151
'NULLS'=152
'OF'=153
'ON'=154
'ONLY'=155
'OPTION'=156
'OPTIONS'=157
'OR'=158
'ORDER'=159
'OUT'=160
'OUTER'=161
'OUTPUTFORMAT'=162
'OVER'=163
'OVERLAPS'=164
'OVERLAY'=165
'OVERWRITE'=166
'PARTITION'=167
'PARTITIONED'=168
'PARTITIONS'=169
'PERCENT'=170
'PIVOT'=171
'PLACING'=172
'POSITION'=173
'PRECEDING'=174
'PRIMARY'=175
'PRINCIPALS'=176
'PROPERTIES'=177
'PURGE'=178
'QUERY'=179
'RANGE'=180
'RECORDREADER'=181
'RECORDWRITER'=182
'RECOVER'=183
'REDUCE'=184
'REFERENCES'=185
'REFRESH'=186
'RENAME'=187
'REPAIR'=188
'REPLACE'=189
'RESET'=190
'RESTRICT'=191
'REVOKE'=192
'RIGHT'=193
'ROLE'=195
'ROLES'=196
'ROLLBACK'=197
'ROLLUP'=198
'ROW'=199
'ROWS'=200
'SCHEMA'=201
'SELECT'=202
'SEMI'=203
'SEPARATED'=204
'SERDE'=205
'SERDEPROPERTIES'=206
'SESSION_USER'=207
'SET'=208
'MINUS'=209
'SETS'=210
'SHOW'=211
'SKEWED'=212
'SOME'=213
'SORT'=214
'SORTED'=215
'START'=216
'STATISTICS'=217
'STORED'=218
'STRATIFY'=219
'STRUCT'=220
'SUBSTR'=221
'SUBSTRING'=222
'TABLE'=223
'TABLES'=224
'TABLESAMPLE'=225
'TBLPROPERTIES'=226
'TERMINATED'=228
'THEN'=229
'TIME'=230
'TO'=231
'TOUCH'=232
'TRAILING'=233
'TRANSACTION'=234
'TRANSACTIONS'=235
'TRANSFORM'=236
'TRIM'=237
'TRUE'=238
'TRUNCATE'=239
'TYPE'=240
'UNARCHIVE'=241
'UNBOUNDED'=242
'UNCACHE'=243
'UNION'=244
'UNIQUE'=245
'UNKNOWN'=246
'UNLOCK'=247
'UNSET'=248
'UPDATE'=249
'USE'=250
'USER'=251
'USING'=252
'VALUES'=253
'VIEW'=254
'VIEWS'=255
'WHEN'=256
'WHERE'=257
'WINDOW'=258
'WITH'=259
'ZONE'=260
'<=>'=262
'<>'=263
'!='=264
'<'=265
'>'=267
'+'=269
'-'=270
'*'=271
'/'=272
'%'=273
'~'=274
'&'=275
'|'=276
'||'=277
'^'=278

2445
src/lib/spark/SqlBaseListener.js Normal file
View File

File diff suppressed because it is too large Load Diff

31285
src/lib/spark/SqlBaseParser.js Normal file
View File

File diff suppressed because one or more lines are too long

1636
src/lib/spark/SqlBaseVisitor.js Normal file
View File

File diff suppressed because it is too large Load Diff

21
test/parser/spark/lexer.test.ts
View File

@ -1,14 +1,21 @@
import SQLParser from '../../../src/parser/spark';
const log = console.log.bind(console);
describe('SparkSQL Lexer tests', () => {
const mysqlParser = new SQLParser();
const parser = new SQLParser();
// const sql = 'select id,name,sex from user1;';
const sql = 'select * from person where age >= 20 order by age desc limit 2;';
const tokens = mysqlParser.getAllTokens(sql);
console.log('tokens', tokens);
test('select id,name from user1;', () => {
const sql = `select id,name from user1;`;
const tokens = parser.getAllTokens(sql);
const length = tokens.length;
log('tokens length', length);
expect(tokens.length).toBe(10);
});
test('token counts', () => {
expect(tokens.length).toBe(28);
test('SELECT * FROM t WHERE x = 1 AND y = 2;', () => {
const sql = `SELECT * FROM t WHERE x = 1 AND y = 2;`;
const tokens = parser.getAllTokens(sql);
expect(tokens.length).toBe(24);
});
});

4
test/parser/spark/listener.test.ts
View File

@ -7,7 +7,7 @@ describe('Spark SQL Listener Tests', () => {
const parserTree = parser.parse(sql);
test('Listener enterTableName', async () => {
test('Listener enterTableName', () => {
let result = '';
class MyListener extends SqlBaseListener {
enterTableName(ctx): void {
@ -16,7 +16,7 @@ describe('Spark SQL Listener Tests', () => {
}
const listenTableName: any = new MyListener();
await parser.listen(listenTableName, parserTree);
parser.listen(listenTableName, parserTree);
expect(result).toBe(expectTableName);
});
});

325
test/parser/spark/syntax.test.ts
View File

@ -1,27 +1,318 @@
/* eslint-disable max-len */
import SQLParser from '../../../src/parser/spark';
const log = console.log.bind(console);
// const log = console.log.bind(console);
const error = console.log.bind(console, '***** error\n');
const validateTest = (sqls) => {
const parser = new SQLParser();
sqls.forEach((sql, i) => {
const result = parser.validate(sql);
if (result.length !== 0) {
error(i, sql);
error(result);
}
expect(result.length).toBe(0);
});
};
describe('Spark SQL Syntax Tests', () => {
const parser = new SQLParser();
test('Select Statement', () => {
const sql = 'select id,name from user1;';
const result = parser.validate(sql);
log('result', result);
expect(result.length).toBe(0);
test('ALTER Statement', () => {
const sqls = [
`ALTER DATABASE inventory SET DBPROPERTIES ('Edited-by' = 'John', 'Edit-date' = '01/01/2001');`,
`ALTER TABLE Student RENAME TO StudentInfo;`,
`ALTER VIEW tempdb1.v1 RENAME TO tempdb1.v2;`,
];
validateTest(sqls);
});
test('Select 1+1', () => {
const sql = 'SELECT 1+1;';
const result = parser.validate(sql);
expect(result.length).toBe(0);
test('CREATE Statement', () => {
const sqls = [
`CREATE DATABASE IF NOT EXISTS customer_db;`,
`CREATE FUNCTION simple_udf AS 'SimpleUdf'
USING JAR '/tmp/SimpleUdf.jar';`,
`CREATE OR REPLACE FUNCTION simple_udf AS 'SimpleUdfR'
USING JAR '/tmp/SimpleUdfR.jar';`,
`CREATE TABLE student (id INT, name STRING, age INT) USING CSV;`,
`CREATE TABLE student (id INT, name STRING, age INT)
USING CSV
PARTITIONED BY (age)
CLUSTERED BY (Id) INTO 4 buckets;`,
`CREATE OR REPLACE VIEW experienced_employee
(ID COMMENT 'Unique identification number', Name)
COMMENT 'View for experienced employees'
AS SELECT id, name FROM all_employee
WHERE working_years > 5;`,
`CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies
AS SELECT mo.member_id, mb.full_name, mo.movie_title
FROM movies AS mo INNER JOIN members AS mb
ON mo.member_id = mb.id;`,
];
validateTest(sqls);
});
test('select', () => {
const sql = 'select * from person where age >= 20 order by age desc limit 2';
const result = parser.validate(sql);
log('result', result);
expect(result.length).toBe(0);
test('DROP Statement', () => {
const sqls = [
`DROP DATABASE inventory_db CASCADE;`,
`DROP DATABASE IF EXISTS inventory_db CASCADE;`,
`DROP FUNCTION test_avg;`,
`DROP TEMPORARY FUNCTION IF EXISTS test_avg;`,
`DROP TABLE userdb.employeetable;`,
`DROP TABLE IF EXISTS employeetable;`,
`DROP VIEW userdb.employeeView;`,
`DROP VIEW IF EXISTS employeeView;`,
];
validateTest(sqls);
});
test('TRUNCATE Statement', () => {
const sqls = [
`TRUNCATE TABLE Student partition(age=10);`,
];
validateTest(sqls);
});
test('REPAIR TABLE Statement', () => {
const sqls = [
`MSCK REPAIR TABLE t1;`,
];
validateTest(sqls);
});
test('USE Database Statement', () => {
const sqls = [
`USE userdb;`,
];
validateTest(sqls);
});
test('INSERT Statement', () => {
const sqls = [
`INSERT INTO students VALUES
('Amy Smith', '123 Park Ave, San Jose', 111111);`,
`INSERT INTO students TABLE visiting_students;`,
`INSERT OVERWRITE students VALUES
('Ashua Hill', '456 Erica Ct, Cupertino', 111111),
('Brian Reed', '723 Kern Ave, Palo Alto', 222222);`,
`INSERT OVERWRITE students TABLE visiting_students;`,
`INSERT OVERWRITE students
FROM applicants SELECT name, address, id applicants WHERE qualified = true;`,
`INSERT OVERWRITE DIRECTORY '/tmp/destination'
USING parquet
OPTIONS (col1 1, col2 2, col3 'test')
SELECT * FROM test_table;`,
`INSERT OVERWRITE DIRECTORY
USING parquet
OPTIONS ('path' '/tmp/destination', col1 1, col2 2, col3 'test')
SELECT * FROM test_table;`,
`INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
STORED AS orc
SELECT * FROM test_table;`,
`INSERT OVERWRITE LOCAL DIRECTORY '/tmp/destination'
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
SELECT * FROM test_table;`,
];
validateTest(sqls);
});
test('LOAD Statement', () => {
const sqls = [
`LOAD DATA LOCAL INPATH '/user/hive/warehouse/students' OVERWRITE INTO TABLE test_load;`,
`LOAD DATA LOCAL INPATH '/user/hive/warehouse/test_partition/c2=2/c3=3'
OVERWRITE INTO TABLE test_load_partition PARTITION (c2=2, c3=3);`,
];
validateTest(sqls);
});
test('SELECT WHERE, GROUP BY, HAVING, ORDER BY Statement', () => {
const sqls = [
`SELECT * FROM person WHERE id > 200 ORDER BY id;`,
`SELECT * FROM person AS parent
WHERE EXISTS (
SELECT 1 FROM person AS child
WHERE parent.id = child.id AND child.age IS NULL
);`,
`SELECT id, sum(quantity) FROM dealer GROUP BY id ORDER BY id;`,
`SELECT city, car_model, sum(quantity) AS sum FROM dealer
GROUP BY city, car_model WITH CUBE
ORDER BY city, car_model;`,
`SELECT city, sum(quantity) AS sum FROM dealer GROUP BY city HAVING city = 'Fremont';`,
`SELECT sum(quantity) AS sum FROM dealer HAVING sum(quantity) > 10;`,
`SELECT name, age FROM person ORDER BY age;`,
`SELECT * FROM person ORDER BY name ASC, age DESC;`,
];
validateTest(sqls);
});
test('SELECT SORT BY, CLUSTER BY, DISTRIBUTE BY, LIMIT Statement', () => {
const sqls = [
`SELECT /*+ REPARTITION(zip_code) */ age, name, zip_code FROM person SORT BY age DESC NULLS FIRST;`,
`SELECT /*+ REPARTITION(zip_code) */ name, age, zip_code FROM person
SORT BY name ASC, age DESC;`,
`SELECT age, name FROM person CLUSTER BY age;`,
`SELECT age, name FROM person DISTRIBUTE BY age;`,
`SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');`,
`SELECT name, age FROM person ORDER BY name LIMIT length('SPARK');`,
];
validateTest(sqls);
});
test('SELECT Common Table Expression Statement', () => {
const sqls = [
`SELECT * FROM t WHERE x = 1 AND y = 2;`,
`SELECT max(c) FROM (
WITH t(c) AS (SELECT 1)
SELECT * FROM t
);`,
`CREATE VIEW v AS
WITH t(a, b, c, d) AS (SELECT 1, 2, 3, 4)
SELECT * FROM t;`,
`SET spark.sql.legacy.ctePrecedencePolicy = CORRECTED;
WITH
t AS (SELECT 1),
t2 AS (
WITH t AS (SELECT 2)
SELECT * FROM t
)
SELECT * FROM t2;`,
];
validateTest(sqls);
});
test('SELECT HINTS, INLINE TABLE, JOIN, LIKE Predicate, Set Operators, Sampling Queries Statement', () => {
const sqls = [
`SELECT /*+ REPARTITION_BY_RANGE(3, c) */ * FROM t;`,
`SELECT /*+ BROADCAST(t1), MERGE(t1, t2) */ * FROM t1 INNER JOIN t2 ON t1.key = t2.key;`,
`SELECT * FROM VALUES ("one", array(0, 1)), ("two", array(2, 3)) AS data(a, b);`,
`SELECT * FROM employee ANTI JOIN department ON employee.deptno = department.deptno;`,
`SELECT * FROM person WHERE name LIKE '%\\_%';`,
`SELECT * FROM person WHERE name LIKE '%$_%' ESCAPE '$';`,
`SELECT c FROM number1 EXCEPT ALL (SELECT c FROM number2);`,
`(SELECT c FROM number1) INTERSECT ALL (SELECT c FROM number2);`,
`SELECT c FROM number1 UNION ALL (SELECT c FROM number2);`,
`SELECT * FROM test TABLESAMPLE (5 ROWS);`,
`SELECT * FROM test TABLESAMPLE (BUCKET 4 OUT OF 10);`,
];
validateTest(sqls);
});
test('SELECT Table-valued Functions, Window Functions, CASE, LATERAL VIEW, PIVOT Statement', () => {
const sqls = [
`SELECT * FROM range(6 + cos(3));`,
`SELECT inline(array(struct(1, 'a'), struct(2, 'b')));`,
`SELECT * FROM test LATERAL VIEW explode (ARRAY(3,4)) AS c2;`,
`SELECT name, dept, RANK() OVER (PARTITION BY dept ORDER BY salary) AS rank FROM employees;`,
`SELECT name, dept, salary, MIN(salary) OVER (PARTITION BY dept ORDER BY salary) AS min
FROM employees;`,
`SELECT id, CASE id WHEN 100 then 'bigger' WHEN id > 300 THEN '300' ELSE 'small' END FROM person;`,
`SELECT * FROM person
WHERE
CASE 1 = 1
WHEN 100 THEN 'big'
WHEN 200 THEN 'bigger'
WHEN 300 THEN 'biggest'
ELSE 'small'
END = 'small';`,
`SELECT * FROM person
LATERAL VIEW EXPLODE(ARRAY(30, 60)) tabelName AS c_age
LATERAL VIEW EXPLODE(ARRAY(40, 80)) AS d_age;`,
`SELECT * FROM person
LATERAL VIEW OUTER EXPLODE(ARRAY()) tabelName AS c_age;`,
`SELECT * FROM person
PIVOT (
SUM(age) AS a, AVG(class) AS c
FOR name IN ('John' AS john, 'Mike' AS mike)
);`,
];
validateTest(sqls);
});
test('EXPLAIN Statement', () => {
const sqls = [
`EXPLAIN select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;`,
`EXPLAIN EXTENDED select k, sum(v) from values (1, 2), (1, 3) t(k, v) group by k;`,
];
validateTest(sqls);
});
test('ANALYZE Statement', () => {
const sqls = [
`ANALYZE TABLE students COMPUTE STATISTICS NOSCAN;`,
`ANALYZE TABLE students COMPUTE STATISTICS FOR COLUMNS name;`,
];
validateTest(sqls);
});
test('CACHE TABLE, UNCACHE TABLE, CLEAR CACHE, REFRESH TABLE, REFRESH Statement', () => {
const sqls = [
`CACHE TABLE testCache OPTIONS ('storageLevel' 'DISK_ONLY') SELECT * FROM testData;`,
`UNCACHE TABLE t1;`,
`CLEAR CACHE;`,
`REFRESH TABLE tbl1;`,
`REFRESH "hdfs://path/to/table";`,
];
validateTest(sqls);
});
test('DESCRIBE DATABASE, TABLE, FUNCTION, QUERY Statement', () => {
const sqls = [
`DESCRIBE DATABASE employees;`,
`DESCRIBE DATABASE EXTENDED employees;`,
`DESCRIBE TABLE customer;`,
`DESCRIBE TABLE EXTENDED customer PARTITION (state = 'AR');`,
`DESC FUNCTION abs;`,
`DESC FUNCTION max;`,
`DESC FUNCTION EXTENDED explode;`,
`DESCRIBE QUERY SELECT age, sum(age) FROM person GROUP BY age;`,
`DESCRIBE QUERY WITH all_names_cte
AS (SELECT name from person) SELECT * FROM all_names_cte;`,
`DESC QUERY VALUES(100, 'John', 10000.20D) AS employee(id, name, salary);`,
];
validateTest(sqls);
});
test('SHOW COLUMNS, CREATE TABLE, DATABASES, FUNCTIONS, PARTITIONS, TABLE EXTENDED, TABLES, TBLPROPERTIES, VIEWS Statement', () => {
const sqls = [
`SHOW COLUMNS IN customer;`,
`SHOW CREATE TABLE test;`,
`SHOW DATABASES LIKE 'pay*';`,
`SHOW SCHEMAS;`,
`SHOW FUNCTIONS trim;`,
`SHOW SYSTEM FUNCTIONS salesdb.max;`,
`SHOW FUNCTIONS LIKE 't[a-z][a-z][a-z]';`,
`SHOW PARTITIONS customer;`,
`SHOW PARTITIONS customer PARTITION (city = 'San Jose');`,
`SHOW TABLE EXTENDED LIKE 'employee';`,
// `SHOW TABLE EXTENDED IN default LIKE 'empl*' PARTITION ('grade=1');`,
`SHOW TABLES;`,
`SHOW TABLES FROM userdb;`,
`SHOW TABLES LIKE 'sam*|suj';`,
`SHOW TBLPROPERTIES customer ('created.date');`,
`SHOW VIEWS;`,
`SHOW VIEWS LIKE 'sam|suj|temp*';`,
];
validateTest(sqls);
});
test('SET, RESET Statement', () => {
const sqls = [
`SET spark.sql.variable.substitute=false;`,
`RESET`,
];
validateTest(sqls);
});
test('ADD, LIST Statement', () => {
const sqls = [
`ADD FILE /tmp/test;`,
`ADD FILE "/path/to/some/directory";`,
`ADD JAR /tmp/test.jar;`,
`ADD JAR '/some/other.jar';`,
`LIST FILE;`,
`LIST FILE /tmp/test /some/random/file /another/random/file;`,
`LIST JAR;`,
`LIST JAR /tmp/test.jar /some/random.jar /another/random.jar;`,
];
validateTest(sqls);
});
});