oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feat/flink dt sql (#75)

Browse Source 
* feat: dt self develop grammar

* feat: dt self develop grammar
This commit is contained in:
nankaNULL 2023-03-21 09:53:25 +08:00 committed by GitHub
parent 4f02f9aca0
commit c0842b3e07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 5416 additions and 4864 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/grammar/flinksql/FlinkSqlLexer.g4
View File

@ -252,6 +252,14 @@ JAVA: 'JAVA';
SCALA: 'SCALA'; SCALA: 'SCALA';
PYTHON: 'PYTHON'; PYTHON: 'PYTHON';
JAR: 'JAR'; JAR: 'JAR';
FILE: 'FILE';
PYTHON_FILES: 'PYTHON_FILES';
PYTHON_REQUIREMENTS: 'PYTHON_REQUIREMENTS';
PYTHON_DEPENDENCIES: 'PYTHON_DEPENDENCIES';
PYTHON_JAR: 'PYTHON_JAR';
PYTHON_ARCHIVES: 'PYTHON_ARCHIVES';
PYTHON_PARAMETER: 'PYTHON_PARAMETER';
ENGINE: 'ENGINE';
CATALOGS: 'CATALOGS'; CATALOGS: 'CATALOGS';
VIEWS: 'VIEWS'; VIEWS: 'VIEWS';
JARS: 'JARS'; JARS: 'JARS';
@ -345,7 +353,9 @@ REAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+
BIT_STRING: BIT_STRING_L; BIT_STRING: BIT_STRING_L;
ID_LITERAL: ID_LITERAL_FRAG; ID_LITERAL: ID_LITERAL_FRAG;
PLUS_ID_LITERAL: PLUS_ID_LITERAL_FRAG; PLUS_ID_LITERAL: PLUS_ID_LITERAL_FRAG;
FILE_PATH: FILE_PATH_STRING;
fragment FILE_PATH_STRING: ([/\\] (~([/\\ ]))*)+;
fragment JAR_FILE_PARTTARN: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; fragment JAR_FILE_PARTTARN: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`';
fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+; fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+;
fragment ID_LITERAL_FRAG: [A-Z_0-9a-z]*?[A-Z_a-z]+?[A-Z_0-9a-z]*; fragment ID_LITERAL_FRAG: [A-Z_0-9a-z]*?[A-Z_a-z]+?[A-Z_0-9a-z]*;

11
src/grammar/flinksql/FlinkSqlParser.g4
View File

@ -14,7 +14,7 @@ sqlStatements
sqlStatement sqlStatement
: ddlStatement | dmlStatement | describeStatement | explainStatement | useStatement | showStatememt : ddlStatement | dmlStatement | describeStatement | explainStatement | useStatement | showStatememt
| loadStatement | unloadStatememt | setStatememt | resetStatememt | jarStatememt | loadStatement | unloadStatememt | setStatememt | resetStatememt | jarStatememt | dtAddStatement
; ;
emptyStatement emptyStatement
@ -86,6 +86,15 @@ jarStatememt
: (ADD | REMOVE) JAR jarFileName : (ADD | REMOVE) JAR jarFileName
; ;
// 数栈平台自研的添加文件语法
dtAddStatement
: ADD JAR WITH FILE_PATH (AS uid)?
| ADD FILE WITH FILE_PATH (AS uid)? (RENAME uid)?
| ADD (PYTHON_FILES | PYTHON_REQUIREMENTS | PYTHON_DEPENDENCIES | PYTHON_JAR | PYTHON_ARCHIVES) WITH FILE_PATH RENAME uid
| ADD PYTHON_PARAMETER FILE_PATH
| ADD ENGINE FILE WITH FILE_PATH RENAME uid KEY uid
;
// Create statements // Create statements
createTable createTable

30
src/lib/flinksql/FlinkSqlLexer.interp
View File

File diff suppressed because one or more lines are too long

3432
src/lib/flinksql/FlinkSqlLexer.js
View File

File diff suppressed because it is too large Load Diff

297
src/lib/flinksql/FlinkSqlLexer.tokens
View File

@ -241,80 +241,89 @@ JAVA=240
SCALA=241 SCALA=241
PYTHON=242 PYTHON=242
JAR=243 JAR=243
CATALOGS=244 FILE=244
VIEWS=245 PYTHON_FILES=245
JARS=246 PYTHON_REQUIREMENTS=246
PRIMARY=247 PYTHON_DEPENDENCIES=247
KEY=248 PYTHON_JAR=248
PERIOD=249 PYTHON_ARCHIVES=249
SYSTEM_TIME=250 PYTHON_PARAMETER=250
ENFORCED=251 ENGINE=251
METADATA=252 CATALOGS=252
VIRTUAL=253 VIEWS=253
STRING=254 JARS=254
ARRAY=255 PRIMARY=255
MAP=256 KEY=256
CHAR=257 PERIOD=257
VARCHAR=258 SYSTEM_TIME=258
BINARY=259 ENFORCED=259
VARBINARY=260 METADATA=260
BYTES=261 VIRTUAL=261
DECIMAL=262 STRING=262
DEC=263 ARRAY=263
NUMERIC=264 MAP=264
TINYINT=265 CHAR=265
SMALLINT=266 VARCHAR=266
INT=267 BINARY=267
INTEGER=268 VARBINARY=268
BIGINT=269 BYTES=269
FLOAT=270 DECIMAL=270
DOUBLE=271 DEC=271
DATE=272 NUMERIC=272
TIME=273 TINYINT=273
TIMESTAMP=274 SMALLINT=274
TIMESTAMP_LTZ=275 INT=275
MULTISET=276 INTEGER=276
BOOLEAN=277 BIGINT=277
RAW=278 FLOAT=278
ROW=279 DOUBLE=279
NULL=280 DATE=280
DATETIME=281 TIME=281
EQUAL_SYMBOL=282 TIMESTAMP=282
GREATER_SYMBOL=283 TIMESTAMP_LTZ=283
LESS_SYMBOL=284 MULTISET=284
EXCLAMATION_SYMBOL=285 BOOLEAN=285
BIT_NOT_OP=286 RAW=286
BIT_OR_OP=287 ROW=287
BIT_AND_OP=288 NULL=288
BIT_XOR_OP=289 DATETIME=289
DOT=290 EQUAL_SYMBOL=290
LS_BRACKET=291 GREATER_SYMBOL=291
RS_BRACKET=292 LESS_SYMBOL=292
LR_BRACKET=293 EXCLAMATION_SYMBOL=293
RR_BRACKET=294 BIT_NOT_OP=294
COMMA=295 BIT_OR_OP=295
SEMICOLON=296 BIT_AND_OP=296
AT_SIGN=297 BIT_XOR_OP=297
SINGLE_QUOTE_SYMB=298 DOT=298
DOUBLE_QUOTE_SYMB=299 LS_BRACKET=299
REVERSE_QUOTE_SYMB=300 RS_BRACKET=300
COLON_SYMB=301 LR_BRACKET=301
ASTERISK_SIGN=302 RR_BRACKET=302
UNDERLINE_SIGN=303 COMMA=303
HYPNEN_SIGN=304 SEMICOLON=304
ADD_SIGN=305 AT_SIGN=305
PENCENT_SIGN=306 SINGLE_QUOTE_SYMB=306
DOUBLE_VERTICAL_SIGN=307 DOUBLE_QUOTE_SYMB=307
DOUBLE_HYPNEN_SIGN=308 REVERSE_QUOTE_SYMB=308
SLASH_SIGN=309 COLON_SYMB=309
DOT_ID=310 ASTERISK_SIGN=310
PLUS_DOT_ID=311 UNDERLINE_SIGN=311
STRING_LITERAL=312 HYPNEN_SIGN=312
DIG_LITERAL=313 ADD_SIGN=313
REAL_LITERAL=314 PENCENT_SIGN=314
BIT_STRING=315 DOUBLE_VERTICAL_SIGN=315
ID_LITERAL=316 DOUBLE_HYPNEN_SIGN=316
PLUS_ID_LITERAL=317 SLASH_SIGN=317
DOT_ID=318
PLUS_DOT_ID=319
STRING_LITERAL=320
DIG_LITERAL=321
REAL_LITERAL=322
BIT_STRING=323
ID_LITERAL=324
PLUS_ID_LITERAL=325
FILE_PATH=326
'SELECT'=4 'SELECT'=4
'FROM'=5 'FROM'=5
'ADD'=6 'ADD'=6
@ -555,69 +564,77 @@ PLUS_ID_LITERAL=317
'SCALA'=241 'SCALA'=241
'PYTHON'=242 'PYTHON'=242
'JAR'=243 'JAR'=243
'CATALOGS'=244 'FILE'=244
'VIEWS'=245 'PYTHON_FILES'=245
'JARS'=246 'PYTHON_REQUIREMENTS'=246
'PRIMARY'=247 'PYTHON_DEPENDENCIES'=247
'KEY'=248 'PYTHON_JAR'=248
'PERIOD'=249 'PYTHON_ARCHIVES'=249
'SYSTEM_TIME'=250 'PYTHON_PARAMETER'=250
'ENFORCED'=251 'ENGINE'=251
'METADATA'=252 'CATALOGS'=252
'VIRTUAL'=253 'VIEWS'=253
'STRING'=254 'JARS'=254
'ARRAY'=255 'PRIMARY'=255
'MAP'=256 'KEY'=256
'CHAR'=257 'PERIOD'=257
'VARCHAR'=258 'SYSTEM_TIME'=258
'BINARY'=259 'ENFORCED'=259
'VARBINARY'=260 'METADATA'=260
'BYTES'=261 'VIRTUAL'=261
'DECIMAL'=262 'STRING'=262
'DEC'=263 'ARRAY'=263
'NUMERIC'=264 'MAP'=264
'TINYINT'=265 'CHAR'=265
'SMALLINT'=266 'VARCHAR'=266
'INT'=267 'BINARY'=267
'INTEGER'=268 'VARBINARY'=268
'BIGINT'=269 'BYTES'=269
'FLOAT'=270 'DECIMAL'=270
'DOUBLE'=271 'DEC'=271
'DATE'=272 'NUMERIC'=272
'TIME'=273 'TINYINT'=273
'TIMESTAMP'=274 'SMALLINT'=274
'TIMESTAMP_LTZ'=275 'INT'=275
'MULTISET'=276 'INTEGER'=276
'BOOLEAN'=277 'BIGINT'=277
'RAW'=278 'FLOAT'=278
'ROW'=279 'DOUBLE'=279
'NULL'=280 'DATE'=280
'DATETIME'=281 'TIME'=281
'='=282 'TIMESTAMP'=282
'>'=283 'TIMESTAMP_LTZ'=283
'<'=284 'MULTISET'=284
'!'=285 'BOOLEAN'=285
'~'=286 'RAW'=286
'|'=287 'ROW'=287
'&'=288 'NULL'=288
'^'=289 'DATETIME'=289
'.'=290 '='=290
'['=291 '>'=291
']'=292 '<'=292
'('=293 '!'=293
')'=294 '~'=294
','=295 '|'=295
';'=296 '&'=296
'@'=297 '^'=297
'\''=298 '.'=298
'"'=299 '['=299
'`'=300 ']'=300
':'=301 '('=301
'*'=302 ')'=302
'_'=303 ','=303
'-'=304 ';'=304
'+'=305 '@'=305
'%'=306 '\''=306
'||'=307 '"'=307
'--'=308 '`'=308
'/'=309 ':'=309
'*'=310
'_'=311
'-'=312
'+'=313
'%'=314
'||'=315
'--'=316
'/'=317

21
src/lib/flinksql/FlinkSqlParser.interp
View File

File diff suppressed because one or more lines are too long

6146
src/lib/flinksql/FlinkSqlParser.js
View File

File diff suppressed because one or more lines are too long

297
src/lib/flinksql/FlinkSqlParser.tokens
View File

@ -241,80 +241,89 @@ JAVA=240
SCALA=241 SCALA=241
PYTHON=242 PYTHON=242
JAR=243 JAR=243
CATALOGS=244 FILE=244
VIEWS=245 PYTHON_FILES=245
JARS=246 PYTHON_REQUIREMENTS=246
PRIMARY=247 PYTHON_DEPENDENCIES=247
KEY=248 PYTHON_JAR=248
PERIOD=249 PYTHON_ARCHIVES=249
SYSTEM_TIME=250 PYTHON_PARAMETER=250
ENFORCED=251 ENGINE=251
METADATA=252 CATALOGS=252
VIRTUAL=253 VIEWS=253
STRING=254 JARS=254
ARRAY=255 PRIMARY=255
MAP=256 KEY=256
CHAR=257 PERIOD=257
VARCHAR=258 SYSTEM_TIME=258
BINARY=259 ENFORCED=259
VARBINARY=260 METADATA=260
BYTES=261 VIRTUAL=261
DECIMAL=262 STRING=262
DEC=263 ARRAY=263
NUMERIC=264 MAP=264
TINYINT=265 CHAR=265
SMALLINT=266 VARCHAR=266
INT=267 BINARY=267
INTEGER=268 VARBINARY=268
BIGINT=269 BYTES=269
FLOAT=270 DECIMAL=270
DOUBLE=271 DEC=271
DATE=272 NUMERIC=272
TIME=273 TINYINT=273
TIMESTAMP=274 SMALLINT=274
TIMESTAMP_LTZ=275 INT=275
MULTISET=276 INTEGER=276
BOOLEAN=277 BIGINT=277
RAW=278 FLOAT=278
ROW=279 DOUBLE=279
NULL=280 DATE=280
DATETIME=281 TIME=281
EQUAL_SYMBOL=282 TIMESTAMP=282
GREATER_SYMBOL=283 TIMESTAMP_LTZ=283
LESS_SYMBOL=284 MULTISET=284
EXCLAMATION_SYMBOL=285 BOOLEAN=285
BIT_NOT_OP=286 RAW=286
BIT_OR_OP=287 ROW=287
BIT_AND_OP=288 NULL=288
BIT_XOR_OP=289 DATETIME=289
DOT=290 EQUAL_SYMBOL=290
LS_BRACKET=291 GREATER_SYMBOL=291
RS_BRACKET=292 LESS_SYMBOL=292
LR_BRACKET=293 EXCLAMATION_SYMBOL=293
RR_BRACKET=294 BIT_NOT_OP=294
COMMA=295 BIT_OR_OP=295
SEMICOLON=296 BIT_AND_OP=296
AT_SIGN=297 BIT_XOR_OP=297
SINGLE_QUOTE_SYMB=298 DOT=298
DOUBLE_QUOTE_SYMB=299 LS_BRACKET=299
REVERSE_QUOTE_SYMB=300 RS_BRACKET=300
COLON_SYMB=301 LR_BRACKET=301
ASTERISK_SIGN=302 RR_BRACKET=302
UNDERLINE_SIGN=303 COMMA=303
HYPNEN_SIGN=304 SEMICOLON=304
ADD_SIGN=305 AT_SIGN=305
PENCENT_SIGN=306 SINGLE_QUOTE_SYMB=306
DOUBLE_VERTICAL_SIGN=307 DOUBLE_QUOTE_SYMB=307
DOUBLE_HYPNEN_SIGN=308 REVERSE_QUOTE_SYMB=308
SLASH_SIGN=309 COLON_SYMB=309
DOT_ID=310 ASTERISK_SIGN=310
PLUS_DOT_ID=311 UNDERLINE_SIGN=311
STRING_LITERAL=312 HYPNEN_SIGN=312
DIG_LITERAL=313 ADD_SIGN=313
REAL_LITERAL=314 PENCENT_SIGN=314
BIT_STRING=315 DOUBLE_VERTICAL_SIGN=315
ID_LITERAL=316 DOUBLE_HYPNEN_SIGN=316
PLUS_ID_LITERAL=317 SLASH_SIGN=317
DOT_ID=318
PLUS_DOT_ID=319
STRING_LITERAL=320
DIG_LITERAL=321
REAL_LITERAL=322
BIT_STRING=323
ID_LITERAL=324
PLUS_ID_LITERAL=325
FILE_PATH=326
'SELECT'=4 'SELECT'=4
'FROM'=5 'FROM'=5
'ADD'=6 'ADD'=6
@ -555,69 +564,77 @@ PLUS_ID_LITERAL=317
'SCALA'=241 'SCALA'=241
'PYTHON'=242 'PYTHON'=242
'JAR'=243 'JAR'=243
'CATALOGS'=244 'FILE'=244
'VIEWS'=245 'PYTHON_FILES'=245
'JARS'=246 'PYTHON_REQUIREMENTS'=246
'PRIMARY'=247 'PYTHON_DEPENDENCIES'=247
'KEY'=248 'PYTHON_JAR'=248
'PERIOD'=249 'PYTHON_ARCHIVES'=249
'SYSTEM_TIME'=250 'PYTHON_PARAMETER'=250
'ENFORCED'=251 'ENGINE'=251
'METADATA'=252 'CATALOGS'=252
'VIRTUAL'=253 'VIEWS'=253
'STRING'=254 'JARS'=254
'ARRAY'=255 'PRIMARY'=255
'MAP'=256 'KEY'=256
'CHAR'=257 'PERIOD'=257
'VARCHAR'=258 'SYSTEM_TIME'=258
'BINARY'=259 'ENFORCED'=259
'VARBINARY'=260 'METADATA'=260
'BYTES'=261 'VIRTUAL'=261
'DECIMAL'=262 'STRING'=262
'DEC'=263 'ARRAY'=263
'NUMERIC'=264 'MAP'=264
'TINYINT'=265 'CHAR'=265
'SMALLINT'=266 'VARCHAR'=266
'INT'=267 'BINARY'=267
'INTEGER'=268 'VARBINARY'=268
'BIGINT'=269 'BYTES'=269
'FLOAT'=270 'DECIMAL'=270
'DOUBLE'=271 'DEC'=271
'DATE'=272 'NUMERIC'=272
'TIME'=273 'TINYINT'=273
'TIMESTAMP'=274 'SMALLINT'=274
'TIMESTAMP_LTZ'=275 'INT'=275
'MULTISET'=276 'INTEGER'=276
'BOOLEAN'=277 'BIGINT'=277
'RAW'=278 'FLOAT'=278
'ROW'=279 'DOUBLE'=279
'NULL'=280 'DATE'=280
'DATETIME'=281 'TIME'=281
'='=282 'TIMESTAMP'=282
'>'=283 'TIMESTAMP_LTZ'=283
'<'=284 'MULTISET'=284
'!'=285 'BOOLEAN'=285
'~'=286 'RAW'=286
'|'=287 'ROW'=287
'&'=288 'NULL'=288
'^'=289 'DATETIME'=289
'.'=290 '='=290
'['=291 '>'=291
']'=292 '<'=292
'('=293 '!'=293
')'=294 '~'=294
','=295 '|'=295
';'=296 '&'=296
'@'=297 '^'=297
'\''=298 '.'=298
'"'=299 '['=299
'`'=300 ']'=300
':'=301 '('=301
'*'=302 ')'=302
'_'=303 ','=303
'-'=304 ';'=304
'+'=305 '@'=305
'%'=306 '\''=306
'||'=307 '"'=307
'--'=308 '`'=308
'/'=309 ':'=309
'*'=310
'_'=311
'-'=312
'+'=313
'%'=314
'||'=315
'--'=316
'/'=317

9
src/lib/flinksql/FlinkSqlParserListener.js
View File

@ -182,6 +182,15 @@ FlinkSqlParserListener.prototype.exitJarStatememt = function(ctx) {
}; };
// Enter a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserListener.prototype.enterDtAddStatement = function(ctx) {
};
// Exit a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserListener.prototype.exitDtAddStatement = function(ctx) {
};
// Enter a parse tree produced by FlinkSqlParser#createTable. // Enter a parse tree produced by FlinkSqlParser#createTable.
FlinkSqlParserListener.prototype.enterCreateTable = function(ctx) { FlinkSqlParserListener.prototype.enterCreateTable = function(ctx) {
}; };

6
src/lib/flinksql/FlinkSqlParserVisitor.js
View File

@ -126,6 +126,12 @@ FlinkSqlParserVisitor.prototype.visitJarStatememt = function(ctx) {
}; };
// Visit a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserVisitor.prototype.visitDtAddStatement = function(ctx) {
return this.visitChildren(ctx);
};
// Visit a parse tree produced by FlinkSqlParser#createTable. // Visit a parse tree produced by FlinkSqlParser#createTable.
FlinkSqlParserVisitor.prototype.visitCreateTable = function(ctx) { FlinkSqlParserVisitor.prototype.visitCreateTable = function(ctx) {
return this.visitChildren(ctx); return this.visitChildren(ctx);

21
test/parser/flinksql/syntax.test.ts
View File

@ -189,13 +189,28 @@ describe('FlinkSQL Syntax Tests', () => {
REMOVE JAR '<path_to_filename>.jar' REMOVE JAR '<path_to_filename>.jar'
`; `;
const result = parser.validate(sql); const result = parser.validate(sql);
console.log(result); expect(result.length).toBe(0);
});
test('DT Add Statement', () => {
const sql = `
ADD JAR WITH /home/test.jar;
ADD JAR WITH /home/test.jar AS test.jar;
ADD FILE WITH /home/admin/sftp/dttest_cn/DsCenter_1687/krb5.conf AS krb5.conf;
ADD FILE WITH /home/admin/sftp/dttest_cn/DsCenter_1687/krb5.conf RENAME test.conf;
ADD PYTHON_FILES WITH /test.py RENAME test.py;
ADD PYTHON_REQUIREMENTS WITH /requirements.txt RENAME requirements.txt;
ADD PYTHON_JAR WITH /python_jar.jar RENAME py_jar.jar;
ADD PYTHON_PARAMETER /parameter.py;
ADD ENGINE FILE WITH /filePath RENAME theName KEY theKey;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0); expect(result.length).toBe(0);
}); });
test('Test invalid Double Line Comment statement', () => { test('Test invalid Double Line Comment statement', () => {
let sql = `-test comment\n`; const sql = `-test comment\n`;
let result = parser.validate(sql); const result = parser.validate(sql);
expect(result.length).toBe(1); expect(result.length).toBe(1);
}); });