Feat/flink dt sql (#75)

* feat: dt self develop grammar

* feat: dt self develop grammar
This commit is contained in:
nankaNULL 2023-03-21 09:53:25 +08:00 committed by GitHub
parent 4f02f9aca0
commit c0842b3e07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 5416 additions and 4864 deletions

View File

@ -252,6 +252,14 @@ JAVA: 'JAVA';
SCALA: 'SCALA';
PYTHON: 'PYTHON';
JAR: 'JAR';
FILE: 'FILE';
PYTHON_FILES: 'PYTHON_FILES';
PYTHON_REQUIREMENTS: 'PYTHON_REQUIREMENTS';
PYTHON_DEPENDENCIES: 'PYTHON_DEPENDENCIES';
PYTHON_JAR: 'PYTHON_JAR';
PYTHON_ARCHIVES: 'PYTHON_ARCHIVES';
PYTHON_PARAMETER: 'PYTHON_PARAMETER';
ENGINE: 'ENGINE';
CATALOGS: 'CATALOGS';
VIEWS: 'VIEWS';
JARS: 'JARS';
@ -345,7 +353,9 @@ REAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+
BIT_STRING: BIT_STRING_L;
ID_LITERAL: ID_LITERAL_FRAG;
PLUS_ID_LITERAL: PLUS_ID_LITERAL_FRAG;
FILE_PATH: FILE_PATH_STRING;
fragment FILE_PATH_STRING: ([/\\] (~([/\\ ]))*)+;
fragment JAR_FILE_PARTTARN: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`';
fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+;
fragment ID_LITERAL_FRAG: [A-Z_0-9a-z]*?[A-Z_a-z]+?[A-Z_0-9a-z]*;

View File

@ -14,7 +14,7 @@ sqlStatements
sqlStatement
: ddlStatement | dmlStatement | describeStatement | explainStatement | useStatement | showStatememt
| loadStatement | unloadStatememt | setStatememt | resetStatememt | jarStatememt
| loadStatement | unloadStatememt | setStatememt | resetStatememt | jarStatememt | dtAddStatement
;
emptyStatement
@ -86,6 +86,15 @@ jarStatememt
: (ADD | REMOVE) JAR jarFileName
;
// 数栈平台自研的添加文件语法
dtAddStatement
: ADD JAR WITH FILE_PATH (AS uid)?
| ADD FILE WITH FILE_PATH (AS uid)? (RENAME uid)?
| ADD (PYTHON_FILES | PYTHON_REQUIREMENTS | PYTHON_DEPENDENCIES | PYTHON_JAR | PYTHON_ARCHIVES) WITH FILE_PATH RENAME uid
| ADD PYTHON_PARAMETER FILE_PATH
| ADD ENGINE FILE WITH FILE_PATH RENAME uid KEY uid
;
// Create statements
createTable

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

View File

@ -241,80 +241,89 @@ JAVA=240
SCALA=241
PYTHON=242
JAR=243
CATALOGS=244
VIEWS=245
JARS=246
PRIMARY=247
KEY=248
PERIOD=249
SYSTEM_TIME=250
ENFORCED=251
METADATA=252
VIRTUAL=253
STRING=254
ARRAY=255
MAP=256
CHAR=257
VARCHAR=258
BINARY=259
VARBINARY=260
BYTES=261
DECIMAL=262
DEC=263
NUMERIC=264
TINYINT=265
SMALLINT=266
INT=267
INTEGER=268
BIGINT=269
FLOAT=270
DOUBLE=271
DATE=272
TIME=273
TIMESTAMP=274
TIMESTAMP_LTZ=275
MULTISET=276
BOOLEAN=277
RAW=278
ROW=279
NULL=280
DATETIME=281
EQUAL_SYMBOL=282
GREATER_SYMBOL=283
LESS_SYMBOL=284
EXCLAMATION_SYMBOL=285
BIT_NOT_OP=286
BIT_OR_OP=287
BIT_AND_OP=288
BIT_XOR_OP=289
DOT=290
LS_BRACKET=291
RS_BRACKET=292
LR_BRACKET=293
RR_BRACKET=294
COMMA=295
SEMICOLON=296
AT_SIGN=297
SINGLE_QUOTE_SYMB=298
DOUBLE_QUOTE_SYMB=299
REVERSE_QUOTE_SYMB=300
COLON_SYMB=301
ASTERISK_SIGN=302
UNDERLINE_SIGN=303
HYPNEN_SIGN=304
ADD_SIGN=305
PENCENT_SIGN=306
DOUBLE_VERTICAL_SIGN=307
DOUBLE_HYPNEN_SIGN=308
SLASH_SIGN=309
DOT_ID=310
PLUS_DOT_ID=311
STRING_LITERAL=312
DIG_LITERAL=313
REAL_LITERAL=314
BIT_STRING=315
ID_LITERAL=316
PLUS_ID_LITERAL=317
FILE=244
PYTHON_FILES=245
PYTHON_REQUIREMENTS=246
PYTHON_DEPENDENCIES=247
PYTHON_JAR=248
PYTHON_ARCHIVES=249
PYTHON_PARAMETER=250
ENGINE=251
CATALOGS=252
VIEWS=253
JARS=254
PRIMARY=255
KEY=256
PERIOD=257
SYSTEM_TIME=258
ENFORCED=259
METADATA=260
VIRTUAL=261
STRING=262
ARRAY=263
MAP=264
CHAR=265
VARCHAR=266
BINARY=267
VARBINARY=268
BYTES=269
DECIMAL=270
DEC=271
NUMERIC=272
TINYINT=273
SMALLINT=274
INT=275
INTEGER=276
BIGINT=277
FLOAT=278
DOUBLE=279
DATE=280
TIME=281
TIMESTAMP=282
TIMESTAMP_LTZ=283
MULTISET=284
BOOLEAN=285
RAW=286
ROW=287
NULL=288
DATETIME=289
EQUAL_SYMBOL=290
GREATER_SYMBOL=291
LESS_SYMBOL=292
EXCLAMATION_SYMBOL=293
BIT_NOT_OP=294
BIT_OR_OP=295
BIT_AND_OP=296
BIT_XOR_OP=297
DOT=298
LS_BRACKET=299
RS_BRACKET=300
LR_BRACKET=301
RR_BRACKET=302
COMMA=303
SEMICOLON=304
AT_SIGN=305
SINGLE_QUOTE_SYMB=306
DOUBLE_QUOTE_SYMB=307
REVERSE_QUOTE_SYMB=308
COLON_SYMB=309
ASTERISK_SIGN=310
UNDERLINE_SIGN=311
HYPNEN_SIGN=312
ADD_SIGN=313
PENCENT_SIGN=314
DOUBLE_VERTICAL_SIGN=315
DOUBLE_HYPNEN_SIGN=316
SLASH_SIGN=317
DOT_ID=318
PLUS_DOT_ID=319
STRING_LITERAL=320
DIG_LITERAL=321
REAL_LITERAL=322
BIT_STRING=323
ID_LITERAL=324
PLUS_ID_LITERAL=325
FILE_PATH=326
'SELECT'=4
'FROM'=5
'ADD'=6
@ -555,69 +564,77 @@ PLUS_ID_LITERAL=317
'SCALA'=241
'PYTHON'=242
'JAR'=243
'CATALOGS'=244
'VIEWS'=245
'JARS'=246
'PRIMARY'=247
'KEY'=248
'PERIOD'=249
'SYSTEM_TIME'=250
'ENFORCED'=251
'METADATA'=252
'VIRTUAL'=253
'STRING'=254
'ARRAY'=255
'MAP'=256
'CHAR'=257
'VARCHAR'=258
'BINARY'=259
'VARBINARY'=260
'BYTES'=261
'DECIMAL'=262
'DEC'=263
'NUMERIC'=264
'TINYINT'=265
'SMALLINT'=266
'INT'=267
'INTEGER'=268
'BIGINT'=269
'FLOAT'=270
'DOUBLE'=271
'DATE'=272
'TIME'=273
'TIMESTAMP'=274
'TIMESTAMP_LTZ'=275
'MULTISET'=276
'BOOLEAN'=277
'RAW'=278
'ROW'=279
'NULL'=280
'DATETIME'=281
'='=282
'>'=283
'<'=284
'!'=285
'~'=286
'|'=287
'&'=288
'^'=289
'.'=290
'['=291
']'=292
'('=293
')'=294
','=295
';'=296
'@'=297
'\''=298
'"'=299
'`'=300
':'=301
'*'=302
'_'=303
'-'=304
'+'=305
'%'=306
'||'=307
'--'=308
'/'=309
'FILE'=244
'PYTHON_FILES'=245
'PYTHON_REQUIREMENTS'=246
'PYTHON_DEPENDENCIES'=247
'PYTHON_JAR'=248
'PYTHON_ARCHIVES'=249
'PYTHON_PARAMETER'=250
'ENGINE'=251
'CATALOGS'=252
'VIEWS'=253
'JARS'=254
'PRIMARY'=255
'KEY'=256
'PERIOD'=257
'SYSTEM_TIME'=258
'ENFORCED'=259
'METADATA'=260
'VIRTUAL'=261
'STRING'=262
'ARRAY'=263
'MAP'=264
'CHAR'=265
'VARCHAR'=266
'BINARY'=267
'VARBINARY'=268
'BYTES'=269
'DECIMAL'=270
'DEC'=271
'NUMERIC'=272
'TINYINT'=273
'SMALLINT'=274
'INT'=275
'INTEGER'=276
'BIGINT'=277
'FLOAT'=278
'DOUBLE'=279
'DATE'=280
'TIME'=281
'TIMESTAMP'=282
'TIMESTAMP_LTZ'=283
'MULTISET'=284
'BOOLEAN'=285
'RAW'=286
'ROW'=287
'NULL'=288
'DATETIME'=289
'='=290
'>'=291
'<'=292
'!'=293
'~'=294
'|'=295
'&'=296
'^'=297
'.'=298
'['=299
']'=300
'('=301
')'=302
','=303
';'=304
'@'=305
'\''=306
'"'=307
'`'=308
':'=309
'*'=310
'_'=311
'-'=312
'+'=313
'%'=314
'||'=315
'--'=316
'/'=317

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -241,80 +241,89 @@ JAVA=240
SCALA=241
PYTHON=242
JAR=243
CATALOGS=244
VIEWS=245
JARS=246
PRIMARY=247
KEY=248
PERIOD=249
SYSTEM_TIME=250
ENFORCED=251
METADATA=252
VIRTUAL=253
STRING=254
ARRAY=255
MAP=256
CHAR=257
VARCHAR=258
BINARY=259
VARBINARY=260
BYTES=261
DECIMAL=262
DEC=263
NUMERIC=264
TINYINT=265
SMALLINT=266
INT=267
INTEGER=268
BIGINT=269
FLOAT=270
DOUBLE=271
DATE=272
TIME=273
TIMESTAMP=274
TIMESTAMP_LTZ=275
MULTISET=276
BOOLEAN=277
RAW=278
ROW=279
NULL=280
DATETIME=281
EQUAL_SYMBOL=282
GREATER_SYMBOL=283
LESS_SYMBOL=284
EXCLAMATION_SYMBOL=285
BIT_NOT_OP=286
BIT_OR_OP=287
BIT_AND_OP=288
BIT_XOR_OP=289
DOT=290
LS_BRACKET=291
RS_BRACKET=292
LR_BRACKET=293
RR_BRACKET=294
COMMA=295
SEMICOLON=296
AT_SIGN=297
SINGLE_QUOTE_SYMB=298
DOUBLE_QUOTE_SYMB=299
REVERSE_QUOTE_SYMB=300
COLON_SYMB=301
ASTERISK_SIGN=302
UNDERLINE_SIGN=303
HYPNEN_SIGN=304
ADD_SIGN=305
PENCENT_SIGN=306
DOUBLE_VERTICAL_SIGN=307
DOUBLE_HYPNEN_SIGN=308
SLASH_SIGN=309
DOT_ID=310
PLUS_DOT_ID=311
STRING_LITERAL=312
DIG_LITERAL=313
REAL_LITERAL=314
BIT_STRING=315
ID_LITERAL=316
PLUS_ID_LITERAL=317
FILE=244
PYTHON_FILES=245
PYTHON_REQUIREMENTS=246
PYTHON_DEPENDENCIES=247
PYTHON_JAR=248
PYTHON_ARCHIVES=249
PYTHON_PARAMETER=250
ENGINE=251
CATALOGS=252
VIEWS=253
JARS=254
PRIMARY=255
KEY=256
PERIOD=257
SYSTEM_TIME=258
ENFORCED=259
METADATA=260
VIRTUAL=261
STRING=262
ARRAY=263
MAP=264
CHAR=265
VARCHAR=266
BINARY=267
VARBINARY=268
BYTES=269
DECIMAL=270
DEC=271
NUMERIC=272
TINYINT=273
SMALLINT=274
INT=275
INTEGER=276
BIGINT=277
FLOAT=278
DOUBLE=279
DATE=280
TIME=281
TIMESTAMP=282
TIMESTAMP_LTZ=283
MULTISET=284
BOOLEAN=285
RAW=286
ROW=287
NULL=288
DATETIME=289
EQUAL_SYMBOL=290
GREATER_SYMBOL=291
LESS_SYMBOL=292
EXCLAMATION_SYMBOL=293
BIT_NOT_OP=294
BIT_OR_OP=295
BIT_AND_OP=296
BIT_XOR_OP=297
DOT=298
LS_BRACKET=299
RS_BRACKET=300
LR_BRACKET=301
RR_BRACKET=302
COMMA=303
SEMICOLON=304
AT_SIGN=305
SINGLE_QUOTE_SYMB=306
DOUBLE_QUOTE_SYMB=307
REVERSE_QUOTE_SYMB=308
COLON_SYMB=309
ASTERISK_SIGN=310
UNDERLINE_SIGN=311
HYPNEN_SIGN=312
ADD_SIGN=313
PENCENT_SIGN=314
DOUBLE_VERTICAL_SIGN=315
DOUBLE_HYPNEN_SIGN=316
SLASH_SIGN=317
DOT_ID=318
PLUS_DOT_ID=319
STRING_LITERAL=320
DIG_LITERAL=321
REAL_LITERAL=322
BIT_STRING=323
ID_LITERAL=324
PLUS_ID_LITERAL=325
FILE_PATH=326
'SELECT'=4
'FROM'=5
'ADD'=6
@ -555,69 +564,77 @@ PLUS_ID_LITERAL=317
'SCALA'=241
'PYTHON'=242
'JAR'=243
'CATALOGS'=244
'VIEWS'=245
'JARS'=246
'PRIMARY'=247
'KEY'=248
'PERIOD'=249
'SYSTEM_TIME'=250
'ENFORCED'=251
'METADATA'=252
'VIRTUAL'=253
'STRING'=254
'ARRAY'=255
'MAP'=256
'CHAR'=257
'VARCHAR'=258
'BINARY'=259
'VARBINARY'=260
'BYTES'=261
'DECIMAL'=262
'DEC'=263
'NUMERIC'=264
'TINYINT'=265
'SMALLINT'=266
'INT'=267
'INTEGER'=268
'BIGINT'=269
'FLOAT'=270
'DOUBLE'=271
'DATE'=272
'TIME'=273
'TIMESTAMP'=274
'TIMESTAMP_LTZ'=275
'MULTISET'=276
'BOOLEAN'=277
'RAW'=278
'ROW'=279
'NULL'=280
'DATETIME'=281
'='=282
'>'=283
'<'=284
'!'=285
'~'=286
'|'=287
'&'=288
'^'=289
'.'=290
'['=291
']'=292
'('=293
')'=294
','=295
';'=296
'@'=297
'\''=298
'"'=299
'`'=300
':'=301
'*'=302
'_'=303
'-'=304
'+'=305
'%'=306
'||'=307
'--'=308
'/'=309
'FILE'=244
'PYTHON_FILES'=245
'PYTHON_REQUIREMENTS'=246
'PYTHON_DEPENDENCIES'=247
'PYTHON_JAR'=248
'PYTHON_ARCHIVES'=249
'PYTHON_PARAMETER'=250
'ENGINE'=251
'CATALOGS'=252
'VIEWS'=253
'JARS'=254
'PRIMARY'=255
'KEY'=256
'PERIOD'=257
'SYSTEM_TIME'=258
'ENFORCED'=259
'METADATA'=260
'VIRTUAL'=261
'STRING'=262
'ARRAY'=263
'MAP'=264
'CHAR'=265
'VARCHAR'=266
'BINARY'=267
'VARBINARY'=268
'BYTES'=269
'DECIMAL'=270
'DEC'=271
'NUMERIC'=272
'TINYINT'=273
'SMALLINT'=274
'INT'=275
'INTEGER'=276
'BIGINT'=277
'FLOAT'=278
'DOUBLE'=279
'DATE'=280
'TIME'=281
'TIMESTAMP'=282
'TIMESTAMP_LTZ'=283
'MULTISET'=284
'BOOLEAN'=285
'RAW'=286
'ROW'=287
'NULL'=288
'DATETIME'=289
'='=290
'>'=291
'<'=292
'!'=293
'~'=294
'|'=295
'&'=296
'^'=297
'.'=298
'['=299
']'=300
'('=301
')'=302
','=303
';'=304
'@'=305
'\''=306
'"'=307
'`'=308
':'=309
'*'=310
'_'=311
'-'=312
'+'=313
'%'=314
'||'=315
'--'=316
'/'=317

View File

@ -182,6 +182,15 @@ FlinkSqlParserListener.prototype.exitJarStatememt = function(ctx) {
};
// Enter a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserListener.prototype.enterDtAddStatement = function(ctx) {
};
// Exit a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserListener.prototype.exitDtAddStatement = function(ctx) {
};
// Enter a parse tree produced by FlinkSqlParser#createTable.
FlinkSqlParserListener.prototype.enterCreateTable = function(ctx) {
};

View File

@ -126,6 +126,12 @@ FlinkSqlParserVisitor.prototype.visitJarStatememt = function(ctx) {
};
// Visit a parse tree produced by FlinkSqlParser#dtAddStatement.
FlinkSqlParserVisitor.prototype.visitDtAddStatement = function(ctx) {
return this.visitChildren(ctx);
};
// Visit a parse tree produced by FlinkSqlParser#createTable.
FlinkSqlParserVisitor.prototype.visitCreateTable = function(ctx) {
return this.visitChildren(ctx);

View File

@ -189,13 +189,28 @@ describe('FlinkSQL Syntax Tests', () => {
REMOVE JAR '<path_to_filename>.jar'
`;
const result = parser.validate(sql);
console.log(result);
expect(result.length).toBe(0);
});
test('DT Add Statement', () => {
const sql = `
ADD JAR WITH /home/test.jar;
ADD JAR WITH /home/test.jar AS test.jar;
ADD FILE WITH /home/admin/sftp/dttest_cn/DsCenter_1687/krb5.conf AS krb5.conf;
ADD FILE WITH /home/admin/sftp/dttest_cn/DsCenter_1687/krb5.conf RENAME test.conf;
ADD PYTHON_FILES WITH /test.py RENAME test.py;
ADD PYTHON_REQUIREMENTS WITH /requirements.txt RENAME requirements.txt;
ADD PYTHON_JAR WITH /python_jar.jar RENAME py_jar.jar;
ADD PYTHON_PARAMETER /parameter.py;
ADD ENGINE FILE WITH /filePath RENAME theName KEY theKey;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test invalid Double Line Comment statement', () => {
let sql = `-test comment\n`;
let result = parser.validate(sql);
const sql = `-test comment\n`;
const result = parser.validate(sql);
expect(result.length).toBe(1);
});