oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'dev' into feat_sparkSql

Browse Source
This commit is contained in:
xigua
2020-12-17 14:59:52 +08:00
parent edbc87e8f1 e6ee2ff411
commit 8418c42c26
36 changed files with 46392 additions and 15210 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
test/parser/flinksql/lexer.test.ts Normal file
View File

@ -0,0 +1,11 @@
import SQLParser from '../../../src/parser/flinksql';
describe('FlinkSQL Lexer tests', () => {
const parser = new SQLParser();
const sql = 'SELECT * FROM table1';
const tokens = parser.getAllTokens(sql);
test('token counts', () => {
expect(tokens.length).toBe(7);
});
});

24
test/parser/flinksql/listener.test.ts Normal file
View File

@ -0,0 +1,24 @@
import
SQLParser, { FlinkSqlParserListener }
from '../../../src/parser/flinksql';
describe('Flink SQL Listener Tests', () => {
const expectTableName = 'user1';
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SQLParser();
const parserTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
class MyListener extends FlinkSqlParserListener {
enterTableExpression(ctx): void {
result = ctx.getText().toLowerCase();
}
}
const listenTableName: any = new MyListener();
await parser.listen(listenTableName, parserTree);
expect(result).toBe(expectTableName);
});
});

221
test/parser/flinksql/syntax.test.ts Normal file
View File

@ -0,0 +1,221 @@
import SQLParser from '../../../src/parser/flinksql';
import sqlMockData from '../../mock/flinkSql';
describe('FlinkSQL Syntax Tests', () => {
const parser = new SQLParser();
// Create statements
test('Test simple CreateTable Statement', () => {
const sql = `
CREATE TABLE Orders (
user BIGINT
) WITH (
"connector" = "kafka",
"scan.startup.mode" = "earliest-offset"
);
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple CreateDatabase Statement', () => {
const sql = `
CREATE DATABASE IF NOT EXISTS dataApi
WITH (
"owner" = "admin"
);
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple CreateView Statement', () => {
const sql = `
CREATE TEMPORARY VIEW IF NOT EXISTS tempView
AS SELECT product, amount FROM Orders;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple CreateFunction Statement', () => {
const sql = `
CREATE TEMPORARY FUNCTION IF NOT EXISTS tempFunction AS 'SimpleUdf';
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// Alter statements
test('Test simple AlterTable Statement', () => {
const sql = `ALTER TABLE Orders RENAME TO NewOrders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple AlterDatabase Statement', () => {
const sql = `ALTER DATABASE tempDB SET ("key1"="value1");`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple AlterFunction Statement', () => {
const sql = `
ALTER TEMPORARY FUNCTION IF EXISTS tempFunction AS 'SimpleUdf';
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// Drop statements
test('Test simple DropTable Statement', () => {
const sql = `DROP TABLE IF EXISTS Orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple DropDatabase Statement', () => {
const sql = `DROP DATABASE IF EXISTS Orders RESTRICT;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple DropView Statement', () => {
const sql = `DROP TEMPORARY VIEW IF EXISTS Orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test simple DropFunction Statement', () => {
const sql = `DROP TEMPORARY FUNCTION IF EXISTS Orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// insert statements
test('Test one simple Insert Statement', () => {
const sql = `
INSERT INTO students VALUES
('Amy Smith', '123 Park Ave, San Jose', 111111);
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test two simple Insert Statement', () => {
const sql = `
INSERT INTO students PARTITION (student_id = 444444)
SELECT name, address FROM persons WHERE name = "Dora Williams";
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// query statements
test('Test simple Select Statement', () => {
const sql = `SELECT product, amount FROM Orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with where clause', () => {
const sql = `SELECT * FROM person WHERE id = 200 OR id = 300;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with group by clause', () => {
const sql = `SELECT id, sum(quantity) FROM dealer GROUP BY id;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with having clause', () => {
const sql = `
SELECT city, sum(quantity) AS sum
FROM dealer GROUP BY city HAVING city = 'Fremont';
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with order by clause', () => {
const sql = `SELECT name, age FROM person ORDER BY age;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with limit clause', () => {
const sql = `SELECT name, age FROM person ORDER BY name LIMIT 2;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Select Statement with join', () => {
const sql = `
SELECT id, name, employee.deptno, deptname FROM employee
FULL JOIN department ON employee.deptno = department.deptno;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// describe statements
test('Test simple Describe Statement', () => {
const sql = `DESCRIBE Orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// describe statements
test('Test simple Explain Statement', () => {
const sql = `EXPLAIN tempTable FOR SELECT k, SUM(v) FROM oneTable;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// use statements
test('Test simple Use Statement', () => {
const sql = `USE CATALOG orders;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
// show statements
test('Test simple Show Statement', () => {
const sql = `SHOW CATALOGS;`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement one', () => {
const sql = sqlMockData.sqlStrOne;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement two', () => {
const sql = sqlMockData.sqlStrTwo;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement three', () => {
const sql = sqlMockData.sqlStrThree;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement four', () => {
const sql = sqlMockData.sqlStrFour;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement five', () => {
const sql = sqlMockData.sqlStrFive;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement six', () => {
const sql = sqlMockData.sqlStrSix;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement seven', () => {
const sql = sqlMockData.sqlStrSeven;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement eight', () => {
const sql = sqlMockData.sqlStrEight;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test complex sql Statement nine', () => {
const sql = sqlMockData.allSqlStr;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
});

27
test/parser/flinksql/visitor.test.ts Normal file
View File

@ -0,0 +1,27 @@
import SQLParser, { FlinkSqlParserVisitor } from '../../../src/parser/flinksql';
describe('Flink SQL Visitor Tests', () => {
const expectTableName = 'user1';
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SQLParser();
const parserTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
// console.log('Parser tree string:', parser.toString(parserTree));
test('Visitor visitTableName', () => {
let result = '';
class MyVisitor extends FlinkSqlParserVisitor {
visitTableExpression(ctx): void {
result = ctx.getText().toLowerCase();
super.visitTableExpression(ctx);
}
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);
expect(result).toBe(expectTableName);
});
});

14
test/parser/hive/lexer.test.ts
View File

@ -2,11 +2,15 @@ import SQLParser from '../../../src/parser/hive';
describe('HiveSQL Lexer tests', () => {
const parser = new SQLParser();
// select id,name,sex from user1;
const sql = 'SELECT * FROM t1';
const tokens = parser.getAllTokens(sql);
test('select token counts', () => {
const sql = 'SELECT * FROM t1';
const tokens = parser.getAllTokens(sql);
expect(tokens.length).toBe(4);
});
test('token counts', () => {
expect(tokens.length).toBe(12);
test('select token counts', () => {
const sql = 'show create table_name;';
const tokens = parser.getAllTokens(sql);
expect(tokens.length).toBe(4);
});
});

27
test/parser/hive/listener.test.ts
View File

@ -1,17 +1,16 @@
import SQLParser, { HiveSqlListener } from '../../../src/parser/hive';
describe('Hive SQL Listener Tests', () => {
const expectTableName = 'user1';
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SQLParser();
test('Listener enterSelectList', async () => {
const expectTableName = 'userName';
const sql = `select ${expectTableName} from user1 where inc_day='20190601' limit 1000;`;
const parserTree = parser.parse(sql);
const parserTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
class MyListener extends HiveSqlListener {
enterTableName(ctx): void {
result = ctx.getText().toLowerCase();
enterSelect_list(ctx): void {
result = ctx.getText();
}
}
const listenTableName: any = new MyListener();
@ -19,4 +18,18 @@ describe('Hive SQL Listener Tests', () => {
await parser.listen(listenTableName, parserTree);
expect(result).toBe(expectTableName);
});
test('Listener enterCreateTable', async () => {
const sql = `drop table table_name;`;
const parserTree = parser.parse(sql);
let result = '';
class MyListener extends HiveSqlListener {
enterDrop_stmt(ctx): void {
result = ctx.getText();
}
}
const listenTableName: any = new MyListener();
await parser.listen(listenTableName, parserTree);
expect(result).toBe('droptabletable_name');
});
});

26
test/parser/hive/syntax.test.ts
View File

@ -2,17 +2,21 @@ import SQLParser from '../../../src/parser/hive';
describe('Hive SQL Syntax Tests', () => {
const parser = new SQLParser();
test('Select Statement', () => {
const sql = 'SELECT * FROM employee WHERE salary>30000;';
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Select 1+1', () => {
const sql = 'SELECT 1+1;';
test('Create Table Statement', () => {
const sql = 'CREATE TABLE person(name STRING,age INT);';
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
});
test('Create Table Statement', () => {
const sql = `alter table dm_gis.table_name add if not exists partition (inc_day = '20190601');`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Wrong Select Statement', () => {
const sql = 'SELECT add ABC from Where ;'
const result = parser.validate(sql);
expect(result.length).toBe(2);
expect(result[0].message).toBe(`no viable alternative at input 'SELECTaddABCfromWhere'`)
expect(result[1].message).toBe(`mismatched input 'Where' expecting <EOF>`)
});
});

7
test/parser/hive/visitor.test.ts
View File

@ -1,15 +1,15 @@
import SQLParser, { HiveSqlVisitor } from '../../../src/parser/hive';
describe('Generic SQL Visitor Tests', () => {
const expectTableName = 'user1';
const sql = `select id,name,sex from ${expectTableName};`;
const expectTableName = 'dm_gis.dlv_addr_tc_count';
const sql = `select citycode,tc,inc_day from ${expectTableName} where inc_day='20190501' limit 100;`;
const parser = new SQLParser();
const parserTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
console.log('Parser tree string:', parser.toString(parserTree));
// console.log('Parser tree string:', parser.toString(parserTree));
test('Visitor visitTableName', () => {
let result = '';
@ -19,6 +19,7 @@ describe('Generic SQL Visitor Tests', () => {
super.visitTable_name(ctx);
}
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);

81
test/utils/index.test.ts
View File

@ -1,48 +1,41 @@
import * as utils from '../../src/utils';
describe('utils', () => {
describe('split sql', () => {
test('single', () => {
let sql = 'select id,name from user';
let result = utils.splitSql(sql);
expect(result).toEqual([sql.length - 1]);
sql += ';';
result = utils.splitSql(sql);
expect(result).toEqual([sql.length - 1]);
});
test('multiple', () => {
const sql = `-- a ;
select * from a;
select user from b`;
const result = utils.splitSql(sql);
expect(result).toEqual([34, 65]);
});
test('error sql', () => {
const sql = `CREATE TABLE MyResult(
a double,
b timestamp,
c timestamp
)WITH(
type ='mysql,
url ='jdbc:mysql://1.1.1.1:3306/hi?charset=utf8',
userName ='name',
password ='123',
tableName ='user'
);`;
const result = utils.splitSql(sql);
expect(result).toEqual([337]);
const sql2 = `CREATE TABLE MyResult(
a double,
b timestamp,
c timestamp
)WITH(
type ='mysql,
url ='jdbc:mysql://1.1.1.1:3306/hi?charset=utf8',
userName ='name',
password ='123',
tableName ='user'
)`;
const result2 = utils.splitSql(sql2);
expect(result2).toEqual([336]);
});
test('split single sql', () => {
const sql = 'select id,name from user';
const result = utils.splitSql(sql);
expect(result.length).toEqual(1);
});
test('split multiple sql', () => {
const sql = `-- a ;
select * from a;
/*
xxx
xxx
*/
select user from b`;
const result = utils.splitSql(sql);
expect(result.length).toEqual(2);
});
test('lexer', () => {
const sql = `-- a ;
select * from a;
/*
xxx
xxx
*/
select user from b;`;
const result = utils.lexer(sql);
expect(result.length).toEqual(4);
});
test('cleanSql', () => {
const sql = `-- a ;
select * from a;
/*
xxx
xxx
*/
select user from b`;
const result = utils.cleanSql(sql);
expect(result.indexOf('xxx')).toEqual(-1);
});
});