oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor/basic parser (#182)

Browse Source 
* feat: replace errorCollector with ParserErrorListener to collect lexer error

* refactor: remove useless method in basicParser

* feat: correct splitSQLByStatement method

* feat: rename parse to parseWithCache and add new parse method

* refactor: rename parserTree to parseTree

* test: rename parserTree to parseTree

* refactor: rename parserError to parseError

* feat: export ErrorHandler type

* feat: use errorhandler form params

* test: basic parser unit tests

* style: lint via prettier
This commit is contained in:
Hayden
2023-10-16 17:59:28 +08:00
committed by GitHub
parent 2f1325d4fc
commit f9dbd9fc23
22 changed files with 321 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
test/common/basicParser.test.ts Normal file
View File

@ -0,0 +1,132 @@
import { CommonTokenStream } from 'antlr4ts';
import { ErrorHandler, FlinkSQL } from '../../src';
import { FlinkSqlLexer } from '../../src/lib/flinksql/FlinkSqlLexer';
describe('BasicParser unit tests', () => {
const flinkParser = new FlinkSQL();
test('Create lexer', () => {
const sql = 'SELECT * FROM tb1';
const lexer = flinkParser.createLexer(sql);
expect(lexer).not.toBeUndefined();
expect(lexer).not.toBeNull();
});
test('Create lexer with errorHandler', () => {
const sql = '袋鼠云数栈UED团队';
const errors: any[] = [];
const errorHandler: ErrorHandler<any> = (err) => {
errors.push(err);
};
const lexer = flinkParser.createLexer(sql, errorHandler);
const tokenStream = new CommonTokenStream(lexer);
tokenStream.fill();
expect(errors.length).not.toBe(0);
});
test('Create parser', () => {
const sql = 'SELECT * FROM tb1';
const parser = flinkParser.createParser(sql);
expect(parser).not.toBeUndefined();
expect(parser).not.toBeNull();
});
test('Create parser with errorHandler (lexer error)', () => {
const sql = '袋鼠云数栈UED团队';
const errors: any[] = [];
const errorHandler: ErrorHandler<any> = (err) => {
errors.push(err);
};
const parser = flinkParser.createParser(sql, errorHandler);
parser.program();
expect(errors.length).not.toBe(0);
});
test('Create parser with errorHandler (parse error)', () => {
const sql = 'SHOW TA';
const errors: any[] = [];
const errorHandler: ErrorHandler<any> = (err) => {
errors.push(err);
};
const parser = flinkParser.createParser(sql, errorHandler);
parser.program();
expect(errors.length).not.toBe(0);
});
test('Parse right input', () => {
const sql = 'SELECT * FROM tb1';
const errors: any[] = [];
const errorHandler: ErrorHandler<any> = (err) => {
errors.push(err);
};
const parseTree = flinkParser.parse(sql, errorHandler);
expect(parseTree).not.toBeUndefined();
expect(parseTree).not.toBeNull();
expect(errors.length).toBe(0);
});
test('Parse wrong input', () => {
const sql = '袋鼠云数栈UED团队';
const errors: any[] = [];
const errorHandler: ErrorHandler<any> = (err) => {
errors.push(err);
};
const parseTree = flinkParser.parse(sql, errorHandler);
expect(parseTree).not.toBeUndefined();
expect(parseTree).not.toBeNull();
expect(errors.length).not.toBe(0);
});
test('Get All tokens', () => {
const sql = 'SELECT * FROM tbl1;';
const tokens = flinkParser.getAllTokens(sql);
expect(tokens.length).toBe(8);
expect(tokens[0].type).toBe(FlinkSqlLexer.KW_SELECT);
expect(tokens[1].type).toBe(FlinkSqlLexer.SPACE);
expect(tokens[2].type).toBe(FlinkSqlLexer.ASTERISK_SIGN);
expect(tokens[3].type).toBe(FlinkSqlLexer.SPACE);
expect(tokens[4].type).toBe(FlinkSqlLexer.KW_FROM);
expect(tokens[5].type).toBe(FlinkSqlLexer.SPACE);
expect(tokens[6].type).toBe(FlinkSqlLexer.ID_LITERAL);
expect(tokens[7].type).toBe(FlinkSqlLexer.SEMICOLON);
});
test('Get All tokens with error', () => {
const sql = '袋鼠云数栈UED团队';
const tokens = flinkParser.getAllTokens(sql);
expect(tokens.length).toBe(1);
expect(tokens[0].type).toBe(FlinkSqlLexer.ID_LITERAL);
});
test('Split sql', () => {
const sql = 'SHOW TABLES;\nSELECT * FROM tb;';
const sqlSlices = flinkParser.splitSQLByStatement(sql);
expect(sqlSlices.length).toBe(2);
expect(sqlSlices[0].text).toBe('SHOW TABLES;');
expect(sql.slice(sqlSlices[0].startIndex, sqlSlices[0].endIndex + 1)).toBe(
sqlSlices[0].text
);
expect(sqlSlices[0].startLine).toBe(1);
expect(sqlSlices[0].endLine).toBe(1);
expect(sqlSlices[1].text).toBe('SELECT * FROM tb;');
expect(sql.slice(sqlSlices[1].startIndex, sqlSlices[1].endIndex + 1)).toBe(
sqlSlices[1].text
);
expect(sqlSlices[1].startLine).toBe(2);
expect(sqlSlices[1].endLine).toBe(2);
});
test('Split sql with errors', () => {
const sql = 'SHOW TABLES;\nSELECT * FOM tb;';
const sqlSlices = flinkParser.splitSQLByStatement(sql);
expect(sqlSlices).toBeNull();
});
});

4
test/parser/flinksql/listener.test.ts
View File

@ -8,7 +8,7 @@ describe('Flink SQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new FlinkSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
@ -19,7 +19,7 @@ describe('Flink SQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree);
await parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/flinksql/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('Flink SQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new FlinkSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -25,7 +25,7 @@ describe('Flink SQL Visitor Tests', () => {
};
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(result).toBe(expectTableName);
});

4
test/parser/generic/listener.test.ts
View File

@ -7,7 +7,7 @@ describe('Generic SQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new GenericSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
@ -18,7 +18,7 @@ describe('Generic SQL Listener Tests', () => {
}
const listenTableName: any = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree);
await parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/generic/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('Generic SQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new GenericSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -23,7 +23,7 @@ describe('Generic SQL Visitor Tests', () => {
};
}
const visitor = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(result).toBe(expectTableName);
});

8
test/parser/hive/listener.test.ts
View File

@ -8,7 +8,7 @@ describe('HiveSQL Listener Tests', () => {
test('Listener enterSelectList', async () => {
const expectTableName = 'username';
const sql = `select ${expectTableName} from tablename where inc_day='20190601' limit 1000;`;
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
let result = '';
class MyListener implements HiveSqlParserListener {
@ -18,12 +18,12 @@ describe('HiveSQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree as ProgramContext);
await parser.listen(listenTableName as ParseTreeListener, parseTree as ProgramContext);
expect(result).toBe(expectTableName.toUpperCase());
});
test('Listener enterCreateTable', async () => {
const sql = `drop table table_name;`;
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
let result = '';
class MyListener implements HiveSqlParserListener {
enterDropTableStatement(ctx) {
@ -32,7 +32,7 @@ describe('HiveSQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree as ProgramContext);
await parser.listen(listenTableName as ParseTreeListener, parseTree as ProgramContext);
expect(result).toBe('DROPTABLETABLE_NAME');
});
});

4
test/parser/hive/visitor.test.ts
View File

@ -8,7 +8,7 @@ describe('HiveSQL Visitor Tests', () => {
const sql = `select citycode,tc,inc_day from ${expectTableName} where inc_day='20190501' limit 100;`;
const parser = new HiveSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -25,7 +25,7 @@ describe('HiveSQL Visitor Tests', () => {
}
const visitor = new MyVisitor();
visitor.visit(parserTree as ProgramContext);
visitor.visit(parseTree as ProgramContext);
expect(result).toBe(expectTableName);
});

4
test/parser/pgsql/listener.test.ts
View File

@ -8,7 +8,7 @@ describe('PostgresSQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new PostgresSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
@ -19,7 +19,7 @@ describe('PostgresSQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree);
await parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/pgsql/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('Generic SQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new PostgresSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -26,7 +26,7 @@ describe('Generic SQL Visitor Tests', () => {
}
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(result).toBe(expectTableName);
});

4
test/parser/plsql/listener.test.ts
View File

@ -7,7 +7,7 @@ describe('PLSQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new PLSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
@ -18,7 +18,7 @@ describe('PLSQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree);
await parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/plsql/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('PLSQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new PLSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Visitor visitTable_ref_list', () => {
let result = '';
@ -20,7 +20,7 @@ describe('PLSQL Visitor Tests', () => {
};
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(result).toBe(expectTableName);
});

4
test/parser/spark/listener.test.ts
View File

@ -7,7 +7,7 @@ describe('Spark SQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SparkSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener exitRelationPrimary', () => {
let result = '';
@ -18,7 +18,7 @@ describe('Spark SQL Listener Tests', () => {
}
const listenTableName = new MyListener();
parser.listen(listenTableName as ParseTreeListener, parserTree);
parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/spark/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('Spark SQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new SparkSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -25,7 +25,7 @@ describe('Spark SQL Visitor Tests', () => {
};
}
const visitor = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(visitor.result).toBe(expectTableName);
});

4
test/parser/trinosql/listener.test.ts
View File

@ -7,7 +7,7 @@ describe('trino SQL Listener Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new trinoSQL();
const parserTree = parser.parse(sql);
const parseTree = parser.parse(sql);
test('Listener enterTableName', async () => {
let result = '';
@ -18,7 +18,7 @@ describe('trino SQL Listener Tests', () => {
}
const listenTableName = new MyListener();
await parser.listen(listenTableName as ParseTreeListener, parserTree);
await parser.listen(listenTableName as ParseTreeListener, parseTree);
expect(result).toBe(expectTableName);
});
});

4
test/parser/trinosql/visitor.test.ts
View File

@ -7,7 +7,7 @@ describe('trino SQL Visitor Tests', () => {
const sql = `select id,name,sex from ${expectTableName};`;
const parser = new trinoSQL();
const parserTree = parser.parse(sql, (error) => {
const parseTree = parser.parse(sql, (error) => {
console.log('Parse error:', error);
});
@ -22,7 +22,7 @@ describe('trino SQL Visitor Tests', () => {
};
}
const visitor: any = new MyVisitor();
visitor.visit(parserTree);
visitor.visit(parseTree);
expect(result).toBe(expectTableName);
});