fix: fix flinksql create table grammar, add test (#65)

* fix: fix flinksql create table grammar, add test

* feat: add cross join, left outer join and time temporal join

* test: supplement flinksql join test

* fix: fix catalog table grammar, add test

* fix: fix flinksql data type, add test

* fix: delete console

* feat: add query with clause, add test

src/grammar/flinksql/FlinkSqlLexer.g4

@@ -230,8 +230,10 @@ WS:                           'WS';
 SYSTEM:                       'SYSTEM';
 INCLUDING:                    'INCLUDING';
 EXCLUDING:                    'EXCLUDING';
+OVERWRITING:                  'OVERWRITING';
 CONSTRAINTS:                  'CONSTRAINTS';
 GENERATED:                    'GENERATED';
+WATERMARKS:                   'WATERMARKS';
 CATALOG:                      'CATALOG';
 LANGUAGE:                     'LANGUAGE';
 CATALOGS:                     'CATALOGS';
@@ -240,7 +242,9 @@ PRIMARY:                      'PRIMARY';
 KEY:                          'KEY';
 PERIOD:                       'PERIOD';
 SYSTEM_TIME:                  'SYSTEM_TIME';
-
+ENFORCED:                     'ENFORCED';
+METADATA:                     'METADATA';
+VIRTUAL:                      'VIRTUAL';
 
 // DATA TYPE Keywords
 
@@ -253,15 +257,19 @@ BINARY:                       'BINARY';
 VARBINARY:                    'VARBINARY';
 BYTES:                        'BYTES';
 DECIMAL:                      'DECIMAL';
+DEC:                          'DEC';
+NUMERIC:                      'NUMERIC';
 TINYINT:                      'TINYINT';
 SMALLINT:                     'SMALLINT';
 INT:                          'INT';
+INTEGER:                      'INTEGER';
 BIGINT:                       'BIGINT';
 FLOAT:                        'FLOAT';
 DOUBLE:                       'DOUBLE';
 DATE:                         'DATE';
 TIME:                         'TIME';
 TIMESTAMP:                    'TIMESTAMP';
+TIMESTAMP_LTZ:                'TIMESTAMP_LTZ';
 MULTISET:                     'MULTISET';
 BOOLEAN:                      'BOOLEAN';
 RAW:                          'RAW';

src/grammar/flinksql/FlinkSqlParser.g4

@@ -50,7 +50,7 @@ showStatememt
 // Create statements
 
 createTable
-    : CREATE TABLE uid
+    : CREATE TABLE ifNotExists? sourceTable
     LR_BRACKET 
         columnOptionDefinition (COMMA columnOptionDefinition)*
         (COMMA watermarkDefinition)?
@@ -64,7 +64,13 @@ createTable
     ;
 
 columnOptionDefinition
-    : columnName columnType lengthOneDimension? columnAlias?
+    : physicalColumnDefinition
+    | metadataColumnDefinition
+    | computedColumnDefinition
+    ;
+
+physicalColumnDefinition
+    : columnName columnType columnConstraint? commentSpec?
     ;
 
 columnName
@@ -76,28 +82,77 @@ columnNameList
     ;
 
 columnType
-    : typeName=(CHAR | VARCHAR | STRING | BINARY | VARBINARY | BYTES
+    : typeName=(DATE | BOOLEAN | NULL)
-    | DECIMAL | TINYINT | SMALLINT | INT | BIGINT | FLOAT | DOUBLE
+    | typeName=(CHAR | VARCHAR | STRING | BINARY | VARBINARY | BYTES
-    | DATE | TIME | TIMESTAMP
+        | TINYINT | SMALLINT | INT | INTEGER | BIGINT
-    | ARRAY | MAP | MULTISET | ROW
+        | TIME | TIMESTAMP | TIMESTAMP_LTZ | DATETIME
-    | BOOLEAN | RAW | NULL
+    ) lengthOneDimension?
-    | DATETIME)
+    | typeName=(DECIMAL | DEC | NUMERIC | FLOAT | DOUBLE) lengthTwoOptionalDimension?
+    | type=(ARRAY | MULTISET) lengthOneTypeDimension?
+    | type=MAP mapTypeDimension?
+    | type=ROW rowTypeDimension?
+    | type=RAW lengthTwoStringDimension?
     ;
 
 lengthOneDimension
     : '(' decimalLiteral ')'
     ;
 
+lengthTwoOptionalDimension
+    : '(' decimalLiteral (',' decimalLiteral)? ')'
+    ;
+
+lengthTwoStringDimension
+    : '(' stringLiteral (',' stringLiteral)? ')'
+    ;
+
+lengthOneTypeDimension
+    : LESS_SYMBOL columnType GREATER_SYMBOL
+    ;
+
+mapTypeDimension
+    : LESS_SYMBOL columnType (COMMA columnType) GREATER_SYMBOL
+    ;
+
+rowTypeDimension
+    : LESS_SYMBOL columnName columnType (COMMA columnName columnType)* GREATER_SYMBOL
+    ;
+
+columnConstraint
+    :(CONSTRAINT constraintName)? PRIMARY KEY (NOT ENFORCED)?
+    ;
+
 commentSpec
     : COMMENT STRING_LITERAL
     ;
 
+metadataColumnDefinition
+    : columnName columnType METADATA (FROM metadataKey)? VIRTUAL?
+    ;
+
+metadataKey
+    : STRING_LITERAL
+    ;
+
+computedColumnDefinition
+    : columnName AS computedColumnExpression commentSpec?
+    ;
+
+// 计算表达式
+computedColumnExpression
+    : expression
+    ;
+
 watermarkDefinition
     : WATERMARK FOR expression AS expression
     ;
 
 tableConstraint
-    : (CONSTRAINT identifier)? PRIMARY KEY '(' columnNameList ')'
+    : (CONSTRAINT constraintName)? PRIMARY KEY '(' columnNameList ')' (NOT ENFORCED)?
+    ;
+
+constraintName
+    : identifier
     ;
 
 selfDefinitionClause // 数栈自定义语句 ‘PERIOD FOR SYSTEM_TIME’
@@ -124,12 +179,16 @@ transformArgument
     ;
 
 likeDefinition
-    : LIKE identifier likeOption
+    : LIKE sourceTable (LR_BRACKET likeOption* RR_BRACKET)?
+    ;
+
+sourceTable
+    : uid
     ;
 
 likeOption
-    : (INCLUDING | EXCLUDING) (ALL | CONSTRAINTS)
+    : (INCLUDING | EXCLUDING) (ALL | CONSTRAINTS | PARTITIONS)
-    | (INCLUDING | EXCLUDING) (GENERATED | OPTIONS)
+    | (INCLUDING | EXCLUDING | OVERWRITING) (GENERATED | OPTIONS | WATERMARKS)
     ;
 
 createCatalog
@@ -219,6 +278,7 @@ valuesRowDefinition
 
 queryStatement
     : valuesCaluse
+    | WITH withItem (COMMA withItem)* queryStatement
     | '(' queryStatement ')'
     | left=queryStatement operator=(INTERSECT | UNION | EXCEPT) ALL? right=queryStatement orderByCaluse? limitClause?
     | selectClause orderByCaluse? limitClause?
@@ -229,6 +289,14 @@ valuesCaluse
     : VALUES expression (COMMA expression )*
     ;
 
+withItem
+    : withItemName (LR_BRACKET columnName (COMMA columnName)* RR_BRACKET)? AS LR_BRACKET queryStatement RR_BRACKET
+    ;
+
+withItemName
+    : identifier
+    ;
+
 selectStatement
     : selectClause fromClause whereClause? groupByClause? havingClause? windowClause?
     ;
@@ -247,7 +315,8 @@ fromClause
 
 tableExpression
     : tableReference (COMMA tableReference)*
-    | tableExpression NATURAL? (LEFT | RIGHT | FULL | INNER)? JOIN tableExpression joinCondition?
+    | tableExpression NATURAL? (LEFT | RIGHT | FULL | INNER)? OUTER? JOIN tableExpression joinCondition?
+    | tableExpression CROSS JOIN tableExpression
     ;
 
 tableReference
@@ -255,11 +324,24 @@ tableReference
     ;
 
 tablePrimary
-    : TABLE? expression
+    : TABLE? tablePath systemTimePeriod? (AS? correlationName)?
-    | LATERAL TABLE LR_BRACKET uid LR_BRACKET expression (COMMA expression)* RR_BRACKET RR_BRACKET
+    | LATERAL TABLE LR_BRACKET functionName LR_BRACKET expression (COMMA expression)* RR_BRACKET RR_BRACKET
+    | LATERAL? LR_BRACKET queryStatement RR_BRACKET
     | UNNEST LR_BRACKET expression RR_BRACKET
     ;
 
+tablePath
+    : uid
+    ;
+
+systemTimePeriod
+    : FOR SYSTEM_TIME AS OF dateTimeExpression
+    ;
+
+dateTimeExpression
+    : expression
+    ;
+
 joinCondition
     : ON booleanExpression
     | USING LR_BRACKET uid (COMMA uid)* RR_BRACKET
@@ -407,6 +489,10 @@ dereferenceDefinition
 
 // base common
 
+correlationName
+    : identifier
+    ;
+
 qualifiedName
     : identifier | dereferenceDefinition
     ;

src/grammar/generic/SqlParser.g4

@@ -2072,7 +2072,7 @@ dataType
     | typeName=DOUBLE PRECISION?
           lengthTwoDimension? (SIGNED | UNSIGNED)? ZEROFILL?            #dimensionDataType
     | typeName=(DECIMAL | DEC | FIXED | NUMERIC | FLOAT | FLOAT4 | FLOAT8)
-      lengthTwoOptionalDimension? (SIGNED | UNSIGNED)? ZEROFILL?    #dimensionDataType
+      lengthTwoOptionalDimension? (SIGNED | UNSIGNED)? ZEROFILL?    #dimensionDataType 
     | typeName=(
         DATE | TINYBLOB | BLOB | MEDIUMBLOB | LONGBLOB
         | BOOL | BOOLEAN | SERIAL

src/lib/flinksql/FlinkSqlLexer.tokens

@@ -219,76 +219,85 @@ WS=218
 SYSTEM=219
 INCLUDING=220
 EXCLUDING=221
-CONSTRAINTS=222
+OVERWRITING=222
-GENERATED=223
+CONSTRAINTS=223
-CATALOG=224
+GENERATED=224
-LANGUAGE=225
+WATERMARKS=225
-CATALOGS=226
+CATALOG=226
-VIEWS=227
+LANGUAGE=227
-PRIMARY=228
+CATALOGS=228
-KEY=229
+VIEWS=229
-PERIOD=230
+PRIMARY=230
-SYSTEM_TIME=231
+KEY=231
-STRING=232
+PERIOD=232
-ARRAY=233
+SYSTEM_TIME=233
-MAP=234
+ENFORCED=234
-CHAR=235
+METADATA=235
-VARCHAR=236
+VIRTUAL=236
-BINARY=237
+STRING=237
-VARBINARY=238
+ARRAY=238
-BYTES=239
+MAP=239
-DECIMAL=240
+CHAR=240
-TINYINT=241
+VARCHAR=241
-SMALLINT=242
+BINARY=242
-INT=243
+VARBINARY=243
-BIGINT=244
+BYTES=244
-FLOAT=245
+DECIMAL=245
-DOUBLE=246
+DEC=246
-DATE=247
+NUMERIC=247
-TIME=248
+TINYINT=248
-TIMESTAMP=249
+SMALLINT=249
-MULTISET=250
+INT=250
-BOOLEAN=251
+INTEGER=251
-RAW=252
+BIGINT=252
-ROW=253
+FLOAT=253
-NULL=254
+DOUBLE=254
-DATETIME=255
+DATE=255
-EQUAL_SYMBOL=256
+TIME=256
-GREATER_SYMBOL=257
+TIMESTAMP=257
-LESS_SYMBOL=258
+TIMESTAMP_LTZ=258
-EXCLAMATION_SYMBOL=259
+MULTISET=259
-BIT_NOT_OP=260
+BOOLEAN=260
-BIT_OR_OP=261
+RAW=261
-BIT_AND_OP=262
+ROW=262
-BIT_XOR_OP=263
+NULL=263
-DOT=264
+DATETIME=264
-LS_BRACKET=265
+EQUAL_SYMBOL=265
-RS_BRACKET=266
+GREATER_SYMBOL=266
-LR_BRACKET=267
+LESS_SYMBOL=267
-RR_BRACKET=268
+EXCLAMATION_SYMBOL=268
-COMMA=269
+BIT_NOT_OP=269
-SEMICOLON=270
+BIT_OR_OP=270
-AT_SIGN=271
+BIT_AND_OP=271
-SINGLE_QUOTE_SYMB=272
+BIT_XOR_OP=272
-DOUBLE_QUOTE_SYMB=273
+DOT=273
-REVERSE_QUOTE_SYMB=274
+LS_BRACKET=274
-COLON_SYMB=275
+RS_BRACKET=275
-ASTERISK_SIGN=276
+LR_BRACKET=276
-UNDERLINE_SIGN=277
+RR_BRACKET=277
-HYPNEN_SIGN=278
+COMMA=278
-ADD_SIGN=279
+SEMICOLON=279
-PENCENT_SIGN=280
+AT_SIGN=280
-DOUBLE_VERTICAL_SIGN=281
+SINGLE_QUOTE_SYMB=281
-DOUBLE_HYPNEN_SIGN=282
+DOUBLE_QUOTE_SYMB=282
-SLASH_SIGN=283
+REVERSE_QUOTE_SYMB=283
-DOT_ID=284
+COLON_SYMB=284
-PLUS_DOT_ID=285
+ASTERISK_SIGN=285
-STRING_LITERAL=286
+UNDERLINE_SIGN=286
-DIG_LITERAL=287
+HYPNEN_SIGN=287
-REAL_LITERAL=288
+ADD_SIGN=288
-BIT_STRING=289
+PENCENT_SIGN=289
-ID_LITERAL=290
+DOUBLE_VERTICAL_SIGN=290
-PLUS_ID_LITERAL=291
+DOUBLE_HYPNEN_SIGN=291
+SLASH_SIGN=292
+DOT_ID=293
+PLUS_DOT_ID=294
+STRING_LITERAL=295
+DIG_LITERAL=296
+REAL_LITERAL=297
+BIT_STRING=298
+ID_LITERAL=299
+PLUS_ID_LITERAL=300
 'SELECT'=4
 'FROM'=5
 'ADD'=6
@@ -507,65 +516,74 @@ PLUS_ID_LITERAL=291
 'SYSTEM'=219
 'INCLUDING'=220
 'EXCLUDING'=221
-'CONSTRAINTS'=222
+'OVERWRITING'=222
-'GENERATED'=223
+'CONSTRAINTS'=223
-'CATALOG'=224
+'GENERATED'=224
-'LANGUAGE'=225
+'WATERMARKS'=225
-'CATALOGS'=226
+'CATALOG'=226
-'VIEWS'=227
+'LANGUAGE'=227
-'PRIMARY'=228
+'CATALOGS'=228
-'KEY'=229
+'VIEWS'=229
-'PERIOD'=230
+'PRIMARY'=230
-'SYSTEM_TIME'=231
+'KEY'=231
-'STRING'=232
+'PERIOD'=232
-'ARRAY'=233
+'SYSTEM_TIME'=233
-'MAP'=234
+'ENFORCED'=234
-'CHAR'=235
+'METADATA'=235
-'VARCHAR'=236
+'VIRTUAL'=236
-'BINARY'=237
+'STRING'=237
-'VARBINARY'=238
+'ARRAY'=238
-'BYTES'=239
+'MAP'=239
-'DECIMAL'=240
+'CHAR'=240
-'TINYINT'=241
+'VARCHAR'=241
-'SMALLINT'=242
+'BINARY'=242
-'INT'=243
+'VARBINARY'=243
-'BIGINT'=244
+'BYTES'=244
-'FLOAT'=245
+'DECIMAL'=245
-'DOUBLE'=246
+'DEC'=246
-'DATE'=247
+'NUMERIC'=247
-'TIME'=248
+'TINYINT'=248
-'TIMESTAMP'=249
+'SMALLINT'=249
-'MULTISET'=250
+'INT'=250
-'BOOLEAN'=251
+'INTEGER'=251
-'RAW'=252
+'BIGINT'=252
-'ROW'=253
+'FLOAT'=253
-'NULL'=254
+'DOUBLE'=254
-'DATETIME'=255
+'DATE'=255
-'='=256
+'TIME'=256
-'>'=257
+'TIMESTAMP'=257
-'<'=258
+'TIMESTAMP_LTZ'=258
-'!'=259
+'MULTISET'=259
-'~'=260
+'BOOLEAN'=260
-'|'=261
+'RAW'=261
-'&'=262
+'ROW'=262
-'^'=263
+'NULL'=263
-'.'=264
+'DATETIME'=264
-'['=265
+'='=265
-']'=266
+'>'=266
-'('=267
+'<'=267
-')'=268
+'!'=268
-','=269
+'~'=269
-';'=270
+'|'=270
-'@'=271
+'&'=271
-'\''=272
+'^'=272
-'"'=273
+'.'=273
-'`'=274
+'['=274
-':'=275
+']'=275
-'*'=276
+'('=276
-'_'=277
+')'=277
-'-'=278
+','=278
-'+'=279
+';'=279
-'%'=280
+'@'=280
-'||'=281
+'\''=281
-'--'=282
+'"'=282
-'/'=283
+'`'=283
+':'=284
+'*'=285
+'_'=286
+'-'=287
+'+'=288
+'%'=289
+'||'=290
+'--'=291
+'/'=292

src/lib/flinksql/FlinkSqlParser.tokens

@@ -219,76 +219,85 @@ WS=218
 SYSTEM=219
 INCLUDING=220
 EXCLUDING=221
-CONSTRAINTS=222
+OVERWRITING=222
-GENERATED=223
+CONSTRAINTS=223
-CATALOG=224
+GENERATED=224
-LANGUAGE=225
+WATERMARKS=225
-CATALOGS=226
+CATALOG=226
-VIEWS=227
+LANGUAGE=227
-PRIMARY=228
+CATALOGS=228
-KEY=229
+VIEWS=229
-PERIOD=230
+PRIMARY=230
-SYSTEM_TIME=231
+KEY=231
-STRING=232
+PERIOD=232
-ARRAY=233
+SYSTEM_TIME=233
-MAP=234
+ENFORCED=234
-CHAR=235
+METADATA=235
-VARCHAR=236
+VIRTUAL=236
-BINARY=237
+STRING=237
-VARBINARY=238
+ARRAY=238
-BYTES=239
+MAP=239
-DECIMAL=240
+CHAR=240
-TINYINT=241
+VARCHAR=241
-SMALLINT=242
+BINARY=242
-INT=243
+VARBINARY=243
-BIGINT=244
+BYTES=244
-FLOAT=245
+DECIMAL=245
-DOUBLE=246
+DEC=246
-DATE=247
+NUMERIC=247
-TIME=248
+TINYINT=248
-TIMESTAMP=249
+SMALLINT=249
-MULTISET=250
+INT=250
-BOOLEAN=251
+INTEGER=251
-RAW=252
+BIGINT=252
-ROW=253
+FLOAT=253
-NULL=254
+DOUBLE=254
-DATETIME=255
+DATE=255
-EQUAL_SYMBOL=256
+TIME=256
-GREATER_SYMBOL=257
+TIMESTAMP=257
-LESS_SYMBOL=258
+TIMESTAMP_LTZ=258
-EXCLAMATION_SYMBOL=259
+MULTISET=259
-BIT_NOT_OP=260
+BOOLEAN=260
-BIT_OR_OP=261
+RAW=261
-BIT_AND_OP=262
+ROW=262
-BIT_XOR_OP=263
+NULL=263
-DOT=264
+DATETIME=264
-LS_BRACKET=265
+EQUAL_SYMBOL=265
-RS_BRACKET=266
+GREATER_SYMBOL=266
-LR_BRACKET=267
+LESS_SYMBOL=267
-RR_BRACKET=268
+EXCLAMATION_SYMBOL=268
-COMMA=269
+BIT_NOT_OP=269
-SEMICOLON=270
+BIT_OR_OP=270
-AT_SIGN=271
+BIT_AND_OP=271
-SINGLE_QUOTE_SYMB=272
+BIT_XOR_OP=272
-DOUBLE_QUOTE_SYMB=273
+DOT=273
-REVERSE_QUOTE_SYMB=274
+LS_BRACKET=274
-COLON_SYMB=275
+RS_BRACKET=275
-ASTERISK_SIGN=276
+LR_BRACKET=276
-UNDERLINE_SIGN=277
+RR_BRACKET=277
-HYPNEN_SIGN=278
+COMMA=278
-ADD_SIGN=279
+SEMICOLON=279
-PENCENT_SIGN=280
+AT_SIGN=280
-DOUBLE_VERTICAL_SIGN=281
+SINGLE_QUOTE_SYMB=281
-DOUBLE_HYPNEN_SIGN=282
+DOUBLE_QUOTE_SYMB=282
-SLASH_SIGN=283
+REVERSE_QUOTE_SYMB=283
-DOT_ID=284
+COLON_SYMB=284
-PLUS_DOT_ID=285
+ASTERISK_SIGN=285
-STRING_LITERAL=286
+UNDERLINE_SIGN=286
-DIG_LITERAL=287
+HYPNEN_SIGN=287
-REAL_LITERAL=288
+ADD_SIGN=288
-BIT_STRING=289
+PENCENT_SIGN=289
-ID_LITERAL=290
+DOUBLE_VERTICAL_SIGN=290
-PLUS_ID_LITERAL=291
+DOUBLE_HYPNEN_SIGN=291
+SLASH_SIGN=292
+DOT_ID=293
+PLUS_DOT_ID=294
+STRING_LITERAL=295
+DIG_LITERAL=296
+REAL_LITERAL=297
+BIT_STRING=298
+ID_LITERAL=299
+PLUS_ID_LITERAL=300
 'SELECT'=4
 'FROM'=5
 'ADD'=6
@@ -507,65 +516,74 @@ PLUS_ID_LITERAL=291
 'SYSTEM'=219
 'INCLUDING'=220
 'EXCLUDING'=221
-'CONSTRAINTS'=222
+'OVERWRITING'=222
-'GENERATED'=223
+'CONSTRAINTS'=223
-'CATALOG'=224
+'GENERATED'=224
-'LANGUAGE'=225
+'WATERMARKS'=225
-'CATALOGS'=226
+'CATALOG'=226
-'VIEWS'=227
+'LANGUAGE'=227
-'PRIMARY'=228
+'CATALOGS'=228
-'KEY'=229
+'VIEWS'=229
-'PERIOD'=230
+'PRIMARY'=230
-'SYSTEM_TIME'=231
+'KEY'=231
-'STRING'=232
+'PERIOD'=232
-'ARRAY'=233
+'SYSTEM_TIME'=233
-'MAP'=234
+'ENFORCED'=234
-'CHAR'=235
+'METADATA'=235
-'VARCHAR'=236
+'VIRTUAL'=236
-'BINARY'=237
+'STRING'=237
-'VARBINARY'=238
+'ARRAY'=238
-'BYTES'=239
+'MAP'=239
-'DECIMAL'=240
+'CHAR'=240
-'TINYINT'=241
+'VARCHAR'=241
-'SMALLINT'=242
+'BINARY'=242
-'INT'=243
+'VARBINARY'=243
-'BIGINT'=244
+'BYTES'=244
-'FLOAT'=245
+'DECIMAL'=245
-'DOUBLE'=246
+'DEC'=246
-'DATE'=247
+'NUMERIC'=247
-'TIME'=248
+'TINYINT'=248
-'TIMESTAMP'=249
+'SMALLINT'=249
-'MULTISET'=250
+'INT'=250
-'BOOLEAN'=251
+'INTEGER'=251
-'RAW'=252
+'BIGINT'=252
-'ROW'=253
+'FLOAT'=253
-'NULL'=254
+'DOUBLE'=254
-'DATETIME'=255
+'DATE'=255
-'='=256
+'TIME'=256
-'>'=257
+'TIMESTAMP'=257
-'<'=258
+'TIMESTAMP_LTZ'=258
-'!'=259
+'MULTISET'=259
-'~'=260
+'BOOLEAN'=260
-'|'=261
+'RAW'=261
-'&'=262
+'ROW'=262
-'^'=263
+'NULL'=263
-'.'=264
+'DATETIME'=264
-'['=265
+'='=265
-']'=266
+'>'=266
-'('=267
+'<'=267
-')'=268
+'!'=268
-','=269
+'~'=269
-';'=270
+'|'=270
-'@'=271
+'&'=271
-'\''=272
+'^'=272
-'"'=273
+'.'=273
-'`'=274
+'['=274
-':'=275
+']'=275
-'*'=276
+'('=276
-'_'=277
+')'=277
-'-'=278
+','=278
-'+'=279
+';'=279
-'%'=280
+'@'=280
-'||'=281
+'\''=281
-'--'=282
+'"'=282
-'/'=283
+'`'=283
+':'=284
+'*'=285
+'_'=286
+'-'=287
+'+'=288
+'%'=289
+'||'=290
+'--'=291
+'/'=292

src/lib/flinksql/FlinkSqlParserListener.js

@@ -1,4 +1,4 @@
-// Generated from /Users/ziv/Workspace/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.8
+// Generated from /Users/yuwan/Desktop/yuwan/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.8
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
@@ -128,6 +128,15 @@ FlinkSqlParserListener.prototype.exitColumnOptionDefinition = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#physicalColumnDefinition.
+FlinkSqlParserListener.prototype.enterPhysicalColumnDefinition = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#physicalColumnDefinition.
+FlinkSqlParserListener.prototype.exitPhysicalColumnDefinition = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#columnName.
 FlinkSqlParserListener.prototype.enterColumnName = function(ctx) {
 };
@@ -164,6 +173,60 @@ FlinkSqlParserListener.prototype.exitLengthOneDimension = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#lengthTwoOptionalDimension.
+FlinkSqlParserListener.prototype.enterLengthTwoOptionalDimension = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#lengthTwoOptionalDimension.
+FlinkSqlParserListener.prototype.exitLengthTwoOptionalDimension = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#lengthTwoStringDimension.
+FlinkSqlParserListener.prototype.enterLengthTwoStringDimension = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#lengthTwoStringDimension.
+FlinkSqlParserListener.prototype.exitLengthTwoStringDimension = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#lengthOneTypeDimension.
+FlinkSqlParserListener.prototype.enterLengthOneTypeDimension = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#lengthOneTypeDimension.
+FlinkSqlParserListener.prototype.exitLengthOneTypeDimension = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#mapTypeDimension.
+FlinkSqlParserListener.prototype.enterMapTypeDimension = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#mapTypeDimension.
+FlinkSqlParserListener.prototype.exitMapTypeDimension = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#rowTypeDimension.
+FlinkSqlParserListener.prototype.enterRowTypeDimension = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#rowTypeDimension.
+FlinkSqlParserListener.prototype.exitRowTypeDimension = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#columnConstraint.
+FlinkSqlParserListener.prototype.enterColumnConstraint = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#columnConstraint.
+FlinkSqlParserListener.prototype.exitColumnConstraint = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#commentSpec.
 FlinkSqlParserListener.prototype.enterCommentSpec = function(ctx) {
 };
@@ -173,6 +236,42 @@ FlinkSqlParserListener.prototype.exitCommentSpec = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#metadataColumnDefinition.
+FlinkSqlParserListener.prototype.enterMetadataColumnDefinition = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#metadataColumnDefinition.
+FlinkSqlParserListener.prototype.exitMetadataColumnDefinition = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#metadataKey.
+FlinkSqlParserListener.prototype.enterMetadataKey = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#metadataKey.
+FlinkSqlParserListener.prototype.exitMetadataKey = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#computedColumnDefinition.
+FlinkSqlParserListener.prototype.enterComputedColumnDefinition = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#computedColumnDefinition.
+FlinkSqlParserListener.prototype.exitComputedColumnDefinition = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#computedColumnExpression.
+FlinkSqlParserListener.prototype.enterComputedColumnExpression = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#computedColumnExpression.
+FlinkSqlParserListener.prototype.exitComputedColumnExpression = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#watermarkDefinition.
 FlinkSqlParserListener.prototype.enterWatermarkDefinition = function(ctx) {
 };
@@ -191,6 +290,15 @@ FlinkSqlParserListener.prototype.exitTableConstraint = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#constraintName.
+FlinkSqlParserListener.prototype.enterConstraintName = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#constraintName.
+FlinkSqlParserListener.prototype.exitConstraintName = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#selfDefinitionClause.
 FlinkSqlParserListener.prototype.enterSelfDefinitionClause = function(ctx) {
 };
@@ -254,6 +362,15 @@ FlinkSqlParserListener.prototype.exitLikeDefinition = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#sourceTable.
+FlinkSqlParserListener.prototype.enterSourceTable = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#sourceTable.
+FlinkSqlParserListener.prototype.exitSourceTable = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#likeOption.
 FlinkSqlParserListener.prototype.enterLikeOption = function(ctx) {
 };
@@ -434,6 +551,24 @@ FlinkSqlParserListener.prototype.exitValuesCaluse = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#withItem.
+FlinkSqlParserListener.prototype.enterWithItem = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#withItem.
+FlinkSqlParserListener.prototype.exitWithItem = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#withItemName.
+FlinkSqlParserListener.prototype.enterWithItemName = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#withItemName.
+FlinkSqlParserListener.prototype.exitWithItemName = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#selectStatement.
 FlinkSqlParserListener.prototype.enterSelectStatement = function(ctx) {
 };
@@ -497,6 +632,33 @@ FlinkSqlParserListener.prototype.exitTablePrimary = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#tablePath.
+FlinkSqlParserListener.prototype.enterTablePath = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#tablePath.
+FlinkSqlParserListener.prototype.exitTablePath = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#systemTimePeriod.
+FlinkSqlParserListener.prototype.enterSystemTimePeriod = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#systemTimePeriod.
+FlinkSqlParserListener.prototype.exitSystemTimePeriod = function(ctx) {
+};
+
+
+// Enter a parse tree produced by FlinkSqlParser#dateTimeExpression.
+FlinkSqlParserListener.prototype.enterDateTimeExpression = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#dateTimeExpression.
+FlinkSqlParserListener.prototype.exitDateTimeExpression = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#joinCondition.
 FlinkSqlParserListener.prototype.enterJoinCondition = function(ctx) {
 };
@@ -857,6 +1019,15 @@ FlinkSqlParserListener.prototype.exitDereferenceDefinition = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#correlationName.
+FlinkSqlParserListener.prototype.enterCorrelationName = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#correlationName.
+FlinkSqlParserListener.prototype.exitCorrelationName = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#qualifiedName.
 FlinkSqlParserListener.prototype.enterQualifiedName = function(ctx) {
 };

src/lib/flinksql/FlinkSqlParserVisitor.js

@@ -1,4 +1,4 @@
-// Generated from /Users/ziv/Workspace/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.8
+// Generated from /Users/yuwan/Desktop/yuwan/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.8
 // jshint ignore: start
 var antlr4 = require('antlr4/index');
 
@@ -90,6 +90,12 @@ FlinkSqlParserVisitor.prototype.visitColumnOptionDefinition = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#physicalColumnDefinition.
+FlinkSqlParserVisitor.prototype.visitPhysicalColumnDefinition = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#columnName.
 FlinkSqlParserVisitor.prototype.visitColumnName = function(ctx) {
   return this.visitChildren(ctx);
@@ -114,12 +120,72 @@ FlinkSqlParserVisitor.prototype.visitLengthOneDimension = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#lengthTwoOptionalDimension.
+FlinkSqlParserVisitor.prototype.visitLengthTwoOptionalDimension = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#lengthTwoStringDimension.
+FlinkSqlParserVisitor.prototype.visitLengthTwoStringDimension = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#lengthOneTypeDimension.
+FlinkSqlParserVisitor.prototype.visitLengthOneTypeDimension = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#mapTypeDimension.
+FlinkSqlParserVisitor.prototype.visitMapTypeDimension = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#rowTypeDimension.
+FlinkSqlParserVisitor.prototype.visitRowTypeDimension = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#columnConstraint.
+FlinkSqlParserVisitor.prototype.visitColumnConstraint = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#commentSpec.
 FlinkSqlParserVisitor.prototype.visitCommentSpec = function(ctx) {
   return this.visitChildren(ctx);
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#metadataColumnDefinition.
+FlinkSqlParserVisitor.prototype.visitMetadataColumnDefinition = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#metadataKey.
+FlinkSqlParserVisitor.prototype.visitMetadataKey = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#computedColumnDefinition.
+FlinkSqlParserVisitor.prototype.visitComputedColumnDefinition = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#computedColumnExpression.
+FlinkSqlParserVisitor.prototype.visitComputedColumnExpression = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#watermarkDefinition.
 FlinkSqlParserVisitor.prototype.visitWatermarkDefinition = function(ctx) {
   return this.visitChildren(ctx);
@@ -132,6 +198,12 @@ FlinkSqlParserVisitor.prototype.visitTableConstraint = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#constraintName.
+FlinkSqlParserVisitor.prototype.visitConstraintName = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#selfDefinitionClause.
 FlinkSqlParserVisitor.prototype.visitSelfDefinitionClause = function(ctx) {
   return this.visitChildren(ctx);
@@ -174,6 +246,12 @@ FlinkSqlParserVisitor.prototype.visitLikeDefinition = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#sourceTable.
+FlinkSqlParserVisitor.prototype.visitSourceTable = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#likeOption.
 FlinkSqlParserVisitor.prototype.visitLikeOption = function(ctx) {
   return this.visitChildren(ctx);
@@ -294,6 +372,18 @@ FlinkSqlParserVisitor.prototype.visitValuesCaluse = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#withItem.
+FlinkSqlParserVisitor.prototype.visitWithItem = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#withItemName.
+FlinkSqlParserVisitor.prototype.visitWithItemName = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#selectStatement.
 FlinkSqlParserVisitor.prototype.visitSelectStatement = function(ctx) {
   return this.visitChildren(ctx);
@@ -336,6 +426,24 @@ FlinkSqlParserVisitor.prototype.visitTablePrimary = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#tablePath.
+FlinkSqlParserVisitor.prototype.visitTablePath = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#systemTimePeriod.
+FlinkSqlParserVisitor.prototype.visitSystemTimePeriod = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
+// Visit a parse tree produced by FlinkSqlParser#dateTimeExpression.
+FlinkSqlParserVisitor.prototype.visitDateTimeExpression = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#joinCondition.
 FlinkSqlParserVisitor.prototype.visitJoinCondition = function(ctx) {
   return this.visitChildren(ctx);
@@ -576,6 +684,12 @@ FlinkSqlParserVisitor.prototype.visitDereferenceDefinition = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#correlationName.
+FlinkSqlParserVisitor.prototype.visitCorrelationName = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#qualifiedName.
 FlinkSqlParserVisitor.prototype.visitQualifiedName = function(ctx) {
   return this.visitChildren(ctx);

test/parser/flinksql/syntax.test.ts

@@ -102,6 +102,19 @@ describe('FlinkSQL Syntax Tests', () => {
     });
 
     // query statements
+    test('Test With clause', () => {
+        const sql = `
+            WITH orders_with_total AS (
+                SELECT order_id, price + tax AS total
+                FROM Orders
+            )
+            SELECT order_id, SUM(total)
+            FROM orders_with_total
+            GROUP BY order_id;
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
     test('Test simple Select Statement', () => {
         const sql = `SELECT product, amount FROM Orders;`;
         const result = parser.validate(sql);
@@ -143,6 +156,42 @@ describe('FlinkSQL Syntax Tests', () => {
         const result = parser.validate(sql);
         expect(result.length).toBe(0);
     });
+    // test left outer join
+    test('Test Select Statement with left outer join', () => {
+        const sql = `
+            SELECT order_id, res
+            FROM Orders
+            LEFT OUTER JOIN LATERAL TABLE(table_func(order_id)) t(res)
+            ON TRUE
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // test cross join
+    test('Test Select Statement with cross join', () => {
+        const sql = `
+            SELECT order_id, tag
+            FROM Orders CROSS JOIN UNNEST(tags) AS t (tag)
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // test for time temporal join
+    test('Test Select Statement with time temporal join', () => {
+        const sql = `SELECT o.order_id, o.total, c.country, c.zip
+            FROM Orders AS o
+            JOIN Customers FOR SYSTEM_TIME AS OF o.proc_time AS c
+            ON o.customer_id = c.id;
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // test for catalog table
+    test('Test Select Statement with catalog table', () => {
+        const sql = `SELECT * FROM catalog1.db1.table1;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
 
     // describe statements
     test('Test simple Describe Statement', () => {

test/parser/flinksql/syntax/createTable.test.ts

@@ -0,0 +1,158 @@
+import { FlinkSQL } from '../../../../src';
+
+describe('FlinkSQL Create Table Syntax Tests', () => {
+    const parser = new FlinkSQL();
+
+    // simple create table statement
+    test('Test simple CreateTable Statement', () => {
+        const sql = `
+            CREATE TABLE MyTable (
+                'user_id' BIGINT,
+                'name' STRING
+            ) WITH (
+                'connector'='oracle-x'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // Metadata Columns statement
+    test('Test Metadata Columns Statement', () => {
+        const sql = `
+            CREATE TABLE MyTable (
+                'user_id' BIGINT,
+                'name' STRING,
+                'timestamp' BIGINT METADATA,       -- part of the query-to-sink schema
+                'offset' BIGINT METADATA VIRTUAL,  -- not part of the query-to-sink schema
+                'record_time' TIMESTAMP_LTZ(3) METADATA FROM 'timestamp'    -- reads and writes a Kafka record's timestamp
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // Computed Columns statement
+    test('Test Computed Columns Statement', () => {
+        const sql = `
+            CREATE TABLE MyTable (
+                'user_id' BIGINT,
+                'price' DOUBLE,
+                'quantity' DOUBLE,
+                'cost' AS price * quanitity  -- evaluate expression and supply the result to queries
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // WATERMARK statement
+    test('Test WATERMARK Statement', () => {
+        const sql = `
+            CREATE TABLE MyTable (
+                'user' BIGINT,
+                product STRING,
+                order_time TIMESTAMP(3),
+                WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // primary key statement
+    test('Test Primary Key Statement', () => {
+        const sql = `
+            CREATE TABLE MyTable (
+                id int,
+                PRIMARY KEY (id) NOT ENFORCED
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // PARTITIONED BY statement
+    test('Test PARTITIONED BY Statement', () => {
+        const sql = `
+            CREATE TABLE Orders_in_file (
+                'user' BIGINT,
+                product STRING,
+                order_time_string STRING,
+                order_time AS to_timestamp(order_time)
+            )
+            PARTITIONED BY ('user') 
+            WITH ( 
+                'connector' = 'filesystem',
+                'path' = '...'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // like statement
+    test('Test Like Statement', () => {
+        const sql = `
+            CREATE TABLE Orders_with_watermark (
+                id int,
+                -- Add watermark definition
+                WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND 
+            ) WITH (
+                -- Overwrite the startup-mode
+                'scan.startup.mode' = 'latest-offset'
+            )
+            LIKE Orders_in_file (
+                -- Exclude everything besides the computed columns which we need to generate the watermark for.
+                -- We do not want to have the partitions or filesystem options as those do not apply to kafka. 
+                EXCLUDING ALL
+                INCLUDING GENERATED
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // create catalog table
+    test('Test Create Catalog Table Statement', () => {
+        const sql = `
+            CREATE TABLE catalog1.db1.table1 (
+                id int
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    // data type
+    test('Test Data Type Statement', () => {
+        const sql = `
+            CREATE TABLE catalog1.db1.table1 (
+                attr0 string,
+                attr1 boolean,
+                attr3 decimal(38,18),
+                attr4 TINYINT,
+                attr5 smallint,
+                attr6 int,
+                attr7 bigint,
+                attr8 float,
+                attr9 double,
+                attr10 date,
+                attr11 time,
+                attr12 timestamp(3),
+                attr13 array<string>,
+                attr14 row<attr15 float, attr16 timestamp(3)>,
+                attr17 map<int, bigint>,
+                name1 VARCHAR(64),
+                message ROW<data ROW<UPO_TIMESTAMP VARCHAR(20)>>,
+                raw RAW('class', 'snapshot')
+            ) WITH (
+                'connector' = 'kafka'
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+});