Merge branch 'dev' into feat_sparkSql

test/parser/flinksql/lexer.test.ts

@@ -0,0 +1,11 @@
+import SQLParser from '../../../src/parser/flinksql';
+describe('FlinkSQL Lexer tests', () => {
+    const parser = new SQLParser();
+
+    const sql = 'SELECT * FROM table1';
+    const tokens = parser.getAllTokens(sql);
+
+    test('token counts', () => {
+        expect(tokens.length).toBe(7);
+    });
+});

test/parser/flinksql/listener.test.ts

@@ -0,0 +1,24 @@
+import
+SQLParser, { FlinkSqlParserListener }
+    from '../../../src/parser/flinksql';
+
+describe('Flink SQL Listener Tests', () => {
+    const expectTableName = 'user1';
+    const sql = `select id,name,sex from ${expectTableName};`;
+    const parser = new SQLParser();
+
+    const parserTree = parser.parse(sql);
+
+    test('Listener enterTableName', async () => {
+        let result = '';
+        class MyListener extends FlinkSqlParserListener {
+            enterTableExpression(ctx): void {
+                result = ctx.getText().toLowerCase();
+            }
+        }
+        const listenTableName: any = new MyListener();
+
+        await parser.listen(listenTableName, parserTree);
+        expect(result).toBe(expectTableName);
+    });
+});

test/parser/flinksql/syntax.test.ts

@@ -0,0 +1,221 @@
+import SQLParser from '../../../src/parser/flinksql';
+import sqlMockData from '../../mock/flinkSql';
+
+describe('FlinkSQL Syntax Tests', () => {
+    const parser = new SQLParser();
+
+    // Create statements
+    test('Test simple CreateTable Statement', () => {
+        const sql = `
+            CREATE TABLE Orders (
+                user BIGINT
+            ) WITH ( 
+                "connector" = "kafka",
+                "scan.startup.mode" = "earliest-offset"
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple CreateDatabase Statement', () => {
+        const sql = `
+            CREATE DATABASE IF NOT EXISTS dataApi
+            WITH ( 
+                "owner" = "admin"
+            );
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple CreateView Statement', () => {
+        const sql = `
+            CREATE TEMPORARY VIEW IF NOT EXISTS tempView
+            AS SELECT product, amount FROM Orders;
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple CreateFunction Statement', () => {
+        const sql = `
+            CREATE TEMPORARY FUNCTION IF NOT EXISTS tempFunction AS 'SimpleUdf';
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // Alter statements
+    test('Test simple AlterTable Statement', () => {
+        const sql = `ALTER TABLE Orders RENAME TO NewOrders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple AlterDatabase Statement', () => {
+        const sql = `ALTER DATABASE tempDB SET ("key1"="value1");`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple AlterFunction Statement', () => {
+        const sql = `
+            ALTER TEMPORARY FUNCTION IF EXISTS tempFunction AS 'SimpleUdf';
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // Drop statements
+    test('Test simple DropTable Statement', () => {
+        const sql = `DROP TABLE IF EXISTS Orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple DropDatabase Statement', () => {
+        const sql = `DROP DATABASE IF EXISTS Orders RESTRICT;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple DropView Statement', () => {
+        const sql = `DROP TEMPORARY VIEW IF EXISTS Orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test simple DropFunction Statement', () => {
+        const sql = `DROP TEMPORARY FUNCTION IF EXISTS Orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // insert statements
+    test('Test one simple Insert Statement', () => {
+        const sql = `
+            INSERT INTO students VALUES
+            ('Amy Smith', '123 Park Ave, San Jose', 111111);
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test two simple Insert Statement', () => {
+        const sql = `
+            INSERT INTO students PARTITION (student_id = 444444)
+            SELECT name, address FROM persons WHERE name = "Dora Williams";
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // query statements
+    test('Test simple Select Statement', () => {
+        const sql = `SELECT product, amount FROM Orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with where clause', () => {
+        const sql = `SELECT * FROM person WHERE id = 200 OR id = 300;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with group by clause', () => {
+        const sql = `SELECT id, sum(quantity) FROM dealer GROUP BY id;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with having clause', () => {
+        const sql = `
+            SELECT city, sum(quantity) AS sum 
+            FROM dealer GROUP BY city HAVING city = 'Fremont';
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with order by clause', () => {
+        const sql = `SELECT name, age FROM person ORDER BY age;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with limit clause', () => {
+        const sql = `SELECT name, age FROM person ORDER BY name LIMIT 2;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test Select Statement with join', () => {
+        const sql = `
+            SELECT id, name, employee.deptno, deptname FROM employee
+            FULL JOIN department ON employee.deptno = department.deptno;
+        `;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // describe statements
+    test('Test simple Describe Statement', () => {
+        const sql = `DESCRIBE Orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // describe statements
+    test('Test simple Explain Statement', () => {
+        const sql = `EXPLAIN tempTable FOR SELECT k, SUM(v) FROM oneTable;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // use statements
+    test('Test simple Use Statement', () => {
+        const sql = `USE CATALOG orders;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    // show statements
+    test('Test simple Show Statement', () => {
+        const sql = `SHOW CATALOGS;`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+
+    test('Test complex sql Statement one', () => {
+        const sql = sqlMockData.sqlStrOne;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement two', () => {
+        const sql = sqlMockData.sqlStrTwo;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement three', () => {
+        const sql = sqlMockData.sqlStrThree;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement four', () => {
+        const sql = sqlMockData.sqlStrFour;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement five', () => {
+        const sql = sqlMockData.sqlStrFive;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement six', () => {
+        const sql = sqlMockData.sqlStrSix;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement seven', () => {
+        const sql = sqlMockData.sqlStrSeven;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement eight', () => {
+        const sql = sqlMockData.sqlStrEight;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Test complex sql Statement nine', () => {
+        const sql = sqlMockData.allSqlStr;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+});

test/parser/flinksql/visitor.test.ts

@@ -0,0 +1,27 @@
+import SQLParser, { FlinkSqlParserVisitor } from '../../../src/parser/flinksql';
+
+describe('Flink SQL Visitor Tests', () => {
+    const expectTableName = 'user1';
+    const sql = `select id,name,sex from ${expectTableName};`;
+    const parser = new SQLParser();
+
+    const parserTree = parser.parse(sql, (error) => {
+        console.log('Parse error:', error);
+    });
+
+    // console.log('Parser tree string:', parser.toString(parserTree));
+
+    test('Visitor visitTableName', () => {
+        let result = '';
+        class MyVisitor extends FlinkSqlParserVisitor {
+            visitTableExpression(ctx): void {
+                result = ctx.getText().toLowerCase();
+                super.visitTableExpression(ctx);
+            }
+        }
+        const visitor: any = new MyVisitor();
+        visitor.visit(parserTree);
+
+        expect(result).toBe(expectTableName);
+    });
+});

test/parser/hive/lexer.test.ts

@@ -2,11 +2,15 @@ import SQLParser from '../../../src/parser/hive';
 
 describe('HiveSQL Lexer tests', () => {
     const parser = new SQLParser();
-    // select id,name,sex from user1;
-    const sql = 'SELECT * FROM t1';
-    const tokens = parser.getAllTokens(sql);
+    test('select token counts', () => {
+        const sql = 'SELECT * FROM t1';
+        const tokens = parser.getAllTokens(sql);
+        expect(tokens.length).toBe(4);
+    });
 
-    test('token counts', () => {
-        expect(tokens.length).toBe(12);
+    test('select token counts', () => {
+        const sql = 'show create table_name;';
+        const tokens = parser.getAllTokens(sql);
+        expect(tokens.length).toBe(4);
     });
 });

test/parser/hive/listener.test.ts

@@ -1,17 +1,16 @@
 import SQLParser, { HiveSqlListener } from '../../../src/parser/hive';
 
 describe('Hive SQL Listener Tests', () => {
-    const expectTableName = 'user1';
-    const sql = `select id,name,sex from ${expectTableName};`;
     const parser = new SQLParser();
+    test('Listener enterSelectList', async () => {
+        const expectTableName = 'userName';
+        const sql = `select ${expectTableName} from user1 where inc_day='20190601' limit 1000;`;
+        const parserTree = parser.parse(sql);
 
-    const parserTree = parser.parse(sql);
-
-    test('Listener enterTableName', async () => {
         let result = '';
         class MyListener extends HiveSqlListener {
-            enterTableName(ctx): void {
-                result = ctx.getText().toLowerCase();
+            enterSelect_list(ctx): void {
+                result = ctx.getText();
             }
         }
         const listenTableName: any = new MyListener();
@@ -19,4 +18,18 @@ describe('Hive SQL Listener Tests', () => {
         await parser.listen(listenTableName, parserTree);
         expect(result).toBe(expectTableName);
     });
+    test('Listener enterCreateTable', async () => {
+        const sql = `drop table table_name;`;
+        const parserTree = parser.parse(sql);
+        let result = '';
+        class MyListener extends HiveSqlListener {
+            enterDrop_stmt(ctx): void {
+                result = ctx.getText();
+            }
+        }
+        const listenTableName: any = new MyListener();
+
+        await parser.listen(listenTableName, parserTree);
+        expect(result).toBe('droptabletable_name');
+    });
 });

test/parser/hive/syntax.test.ts

@@ -2,17 +2,21 @@ import SQLParser from '../../../src/parser/hive';
 
 describe('Hive SQL Syntax Tests', () => {
     const parser = new SQLParser();
-
-    test('Select Statement', () => {
-        const sql = 'SELECT * FROM employee WHERE salary>30000;';
-        const result = parser.validate(sql);
-
-        expect(result.length).toBe(0);
-    });
-
-    test('Select 1+1', () => {
-        const sql = 'SELECT 1+1;';
+    test('Create Table Statement', () => {
+        const sql = 'CREATE TABLE person(name STRING,age INT);';
         const result = parser.validate(sql);
         expect(result.length).toBe(0);
     });
-});
+    test('Create Table Statement', () => {
+        const sql = `alter table dm_gis.table_name add if not exists partition (inc_day = '20190601');`;
+        const result = parser.validate(sql);
+        expect(result.length).toBe(0);
+    });
+    test('Wrong Select Statement', () => {
+        const sql = 'SELECT add ABC from Where ;'
+        const result = parser.validate(sql);
+        expect(result.length).toBe(2);
+        expect(result[0].message).toBe(`no viable alternative at input 'SELECTaddABCfromWhere'`)
+        expect(result[1].message).toBe(`mismatched input 'Where' expecting <EOF>`)
+    });
+});

test/parser/hive/visitor.test.ts

@@ -1,15 +1,15 @@
 import SQLParser, { HiveSqlVisitor } from '../../../src/parser/hive';
 
 describe('Generic SQL Visitor Tests', () => {
-    const expectTableName = 'user1';
-    const sql = `select id,name,sex from ${expectTableName};`;
+    const expectTableName = 'dm_gis.dlv_addr_tc_count';
+    const sql = `select citycode,tc,inc_day from ${expectTableName} where inc_day='20190501' limit 100;`;
     const parser = new SQLParser();
 
     const parserTree = parser.parse(sql, (error) => {
         console.log('Parse error:', error);
     });
 
-    console.log('Parser tree string:', parser.toString(parserTree));
+    // console.log('Parser tree string:', parser.toString(parserTree));
 
     test('Visitor visitTableName', () => {
         let result = '';
@@ -19,6 +19,7 @@ describe('Generic SQL Visitor Tests', () => {
                 super.visitTable_name(ctx);
             }
         }
+
         const visitor: any = new MyVisitor();
         visitor.visit(parserTree);