feat(flinksql): add some lexer

src/grammar/flinksql/FlinkSqlLexer.g4

@@ -299,10 +299,37 @@ ROW:                          'ROW';
 NULL:                         'NULL';
 
 
-// single character Keywords
+// Operators. Comparation
 
-EQUAL:                        '=';
-COMMA:                        ',';
+EQUAL_SYMBOL:                        '=';
+GREATER_SYMBOL:                      '>';
+LESS_SYMBOL:                         '<';
+EXCLAMATION_SYMBOL:                  '!';
+
+
+// Operators. Bit
+
+BIT_NOT_OP:                          '~';
+BIT_OR_OP:                           '|';
+BIT_AND_OP:                          '&';
+BIT_XOR_OP:                          '^';
+
+
+// Constructors symbols
+
+DOT:                                 '.';
+LR_BRACKET:                          '(';
+RR_BRACKET:                          ')';
+COMMA:                               ',';
+SEMICOLON:                           ';';
+AT_SIGN:                             '@';
+ZERO_DECIMAL:                        '0';
+ONE_DECIMAL:                         '1';
+TWO_DECIMAL:                         '2';
+SINGLE_QUOTE_SYMB:                   '\'';
+DOUBLE_QUOTE_SYMB:                   '"';
+REVERSE_QUOTE_SYMB:                  '`';
+COLON_SYMB:                          ':';
 
 
 fragment ID_LITERAL:                 [A-Z_0-9a-z]*?[A-Z_a-z]+?[A-Z_0-9a-z]*;

src/grammar/flinksql/FlinkSqlParser.g4

@@ -5,13 +5,21 @@ options { tokenVocab=FlinkSqlLexer; }
 program: statement EOF;
 
 statement
-    : sqlStatement EOF
+    : sqlStatements EOF
+    ;
+
+sqlStatements
+    : (sqlStatement SEMICOLON | emptyStatement)*
     ;
 
 sqlStatement
     : ddlStatement | dmlStatement
     ;
 
+emptyStatement
+    : SEMICOLON
+    ;
+
 ddlStatement
     : createTable | createDatabase | createView | createFunction
     | alterTable | alterDatabase | alterFunction
@@ -24,13 +32,13 @@ dmlStatement
 
 createTable
     : CREATE TABLE tableName
-    ( 
-        columnOptionDefinition (',' columnOptionDefinition)*
-    )
+    LR_BRACKET 
+        columnOptionDefinition (COMMA columnOptionDefinition)*
+    RR_BRACKET
     partitionDefinition?
-    WITH (
-        withOptionDefinition (',' withOptionDefinition)*
-    )
+    WITH LR_BRACKET
+        withOptionDefinition (COMMA withOptionDefinition)*
+    RR_BRACKET
     ;
 
 
@@ -67,7 +75,7 @@ partitionColumnName
     ;
 
 withOptionDefinition
-    : REVERSE_QUOTE_ID EQUAL REVERSE_QUOTE_ID
+    : REVERSE_QUOTE_ID EQUAL_SYMBOL REVERSE_QUOTE_ID
     ;
 
 createDatabase

src/lib/flinksql/FlinkSqlLexer.tokens

@@ -280,8 +280,27 @@ BOOLEAN=279
 RAW=280
 ROW=281
 NULL=282
-EQUAL=283
-COMMA=284
+EQUAL_SYMBOL=283
+GREATER_SYMBOL=284
+LESS_SYMBOL=285
+EXCLAMATION_SYMBOL=286
+BIT_NOT_OP=287
+BIT_OR_OP=288
+BIT_AND_OP=289
+BIT_XOR_OP=290
+DOT=291
+LR_BRACKET=292
+RR_BRACKET=293
+COMMA=294
+SEMICOLON=295
+AT_SIGN=296
+ZERO_DECIMAL=297
+ONE_DECIMAL=298
+TWO_DECIMAL=299
+SINGLE_QUOTE_SYMB=300
+DOUBLE_QUOTE_SYMB=301
+REVERSE_QUOTE_SYMB=302
+COLON_SYMB=303
 'SELECT'=5
 'FROM'=6
 'ADD'=7
@@ -558,4 +577,23 @@ COMMA=284
 'ROW'=281
 'NULL'=282
 '='=283
-','=284
+'>'=284
+'<'=285
+'!'=286
+'~'=287
+'|'=288
+'&'=289
+'^'=290
+'.'=291
+'('=292
+')'=293
+','=294
+';'=295
+'@'=296
+'0'=297
+'1'=298
+'2'=299
+'\''=300
+'"'=301
+'`'=302
+':'=303

src/lib/flinksql/FlinkSqlParser.tokens

@@ -280,8 +280,27 @@ BOOLEAN=279
 RAW=280
 ROW=281
 NULL=282
-EQUAL=283
-COMMA=284
+EQUAL_SYMBOL=283
+GREATER_SYMBOL=284
+LESS_SYMBOL=285
+EXCLAMATION_SYMBOL=286
+BIT_NOT_OP=287
+BIT_OR_OP=288
+BIT_AND_OP=289
+BIT_XOR_OP=290
+DOT=291
+LR_BRACKET=292
+RR_BRACKET=293
+COMMA=294
+SEMICOLON=295
+AT_SIGN=296
+ZERO_DECIMAL=297
+ONE_DECIMAL=298
+TWO_DECIMAL=299
+SINGLE_QUOTE_SYMB=300
+DOUBLE_QUOTE_SYMB=301
+REVERSE_QUOTE_SYMB=302
+COLON_SYMB=303
 'SELECT'=5
 'FROM'=6
 'ADD'=7
@@ -558,4 +577,23 @@ COMMA=284
 'ROW'=281
 'NULL'=282
 '='=283
-','=284
+'>'=284
+'<'=285
+'!'=286
+'~'=287
+'|'=288
+'&'=289
+'^'=290
+'.'=291
+'('=292
+')'=293
+','=294
+';'=295
+'@'=296
+'0'=297
+'1'=298
+'2'=299
+'\''=300
+'"'=301
+'`'=302
+':'=303

src/lib/flinksql/FlinkSqlParserListener.js

@@ -29,6 +29,15 @@ FlinkSqlParserListener.prototype.exitStatement = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#sqlStatements.
+FlinkSqlParserListener.prototype.enterSqlStatements = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#sqlStatements.
+FlinkSqlParserListener.prototype.exitSqlStatements = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#sqlStatement.
 FlinkSqlParserListener.prototype.enterSqlStatement = function(ctx) {
 };
@@ -38,6 +47,15 @@ FlinkSqlParserListener.prototype.exitSqlStatement = function(ctx) {
 };
 
 
+// Enter a parse tree produced by FlinkSqlParser#emptyStatement.
+FlinkSqlParserListener.prototype.enterEmptyStatement = function(ctx) {
+};
+
+// Exit a parse tree produced by FlinkSqlParser#emptyStatement.
+FlinkSqlParserListener.prototype.exitEmptyStatement = function(ctx) {
+};
+
+
 // Enter a parse tree produced by FlinkSqlParser#ddlStatement.
 FlinkSqlParserListener.prototype.enterDdlStatement = function(ctx) {
 };

src/lib/flinksql/FlinkSqlParserVisitor.js

@@ -24,12 +24,24 @@ FlinkSqlParserVisitor.prototype.visitStatement = function(ctx) {
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#sqlStatements.
+FlinkSqlParserVisitor.prototype.visitSqlStatements = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#sqlStatement.
 FlinkSqlParserVisitor.prototype.visitSqlStatement = function(ctx) {
   return this.visitChildren(ctx);
 };
 
 
+// Visit a parse tree produced by FlinkSqlParser#emptyStatement.
+FlinkSqlParserVisitor.prototype.visitEmptyStatement = function(ctx) {
+  return this.visitChildren(ctx);
+};
+
+
 // Visit a parse tree produced by FlinkSqlParser#ddlStatement.
 FlinkSqlParserVisitor.prototype.visitDdlStatement = function(ctx) {
   return this.visitChildren(ctx);

test/parser/flinksql/syntax.test.ts

@@ -6,15 +6,14 @@ describe('FlinkSQL Syntax Tests', () => {
     test('Test simple CreateTable Statement', () => {
         const sql = `
             CREATE TABLE Orders (
-                user BIGINT,
-                product STRING
+                user BIGINT
             ) WITH ( 
                 "connector" = "kafka",
                 "scan.startup.mode" = "earliest-offset"
             );
         `;
         const result = parser.validate(sql);
-
+        console.log(result);
         expect(result.length).toBe(0);
     });
 });