oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: improve flinksql insertStatement

Browse Source 
* feat: feat: improve flinksql insertStatement

* feat: complete insertFromSelectQueries syntax unit tests

* feat: complete insertValuesIntoTable syntax unit tests

* feat: improve insertValuesIntoTable sql case data

* feat: complete insertMultipeTables unit cases

* feat: depracate insertMulStatement grammar for 1.1.4

* fix: depracate unless files

* test: recover jest.config info

* feat:  improve  insertMultipleTable.sql  test case

* feat: build new flinksql parser and lexer

* feat: generator new flink parser and lexer file

* test: improve readSQL logic

* test: fix insertMultipleTable sql error

* test:  recover jest.config.js info
This commit is contained in:
mumiao 2023-05-12 09:31:54 +08:00 committed by GitHub
parent c41beeaa5a
commit dc417e1381
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 5933 additions and 2332 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
jest.config.js
View File

@ -7,7 +7,6 @@ module.exports = {
// All imported modules in your tests should be mocked automatically
// automock: false,
// Stop running tests after `n` failures
// bail: 0,
@ -85,7 +84,7 @@ module.exports = {
"ts",
"tsx",
"json",
"node"
"node",
],
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
@ -154,10 +153,7 @@ module.exports = {
// testLocationInResults: false,
// The glob patterns Jest uses to detect test files
testMatch: [
"**/__tests__/**/*.[jt]s?(x)",
"**/?(*.)+(spec|test).[tj]s?(x)"
],
testMatch: ["**/__tests__/**/*.[jt]s?(x)", "**/?(*.)+(spec|test).[tj]s?(x)"],
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [
@ -175,17 +171,18 @@ module.exports = {
// A map from regular expressions to paths to transformers
transform: {
'\\.[jt]sx?$': ['ts-jest', {
"\\.[jt]sx?$": [
"ts-jest",
{
tsconfig: {
noUnusedLocals: false,
}
}]
},
},
],
},
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
transformIgnorePatterns: [
'/node_modules/(?!antlr4)'
],
transformIgnorePatterns: ["/node_modules/(?!antlr4)"],
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
// unmockedModulePathPatterns: undefined,
@ -199,6 +196,6 @@ module.exports = {
// Whether to use watchman for file crawling
// watchman: true,
moduleNameMapper: {
'^antlr4$': '<rootDir>/node_modules/antlr4/src/antlr4/index.web.js',
}
"^antlr4$": "<rootDir>/node_modules/antlr4/src/antlr4/index.web.js",
},
};

3727
pnpm-lock.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

7
src/grammar/flinksql/FlinkSqlParser.g4
View File

@ -329,7 +329,7 @@ dropFunction
// Insert statements
insertStatement
: (EXECUTE? insertSimpleStatement) | insertMulStatementCompatibility | (EXECUTE insertMulStatement)
: (EXECUTE? insertSimpleStatement) | (EXECUTE insertMulStatement)
;
insertSimpleStatement
@ -354,15 +354,10 @@ valuesRowDefinition
RR_BRACKET
;
insertMulStatementCompatibility
: BEGIN STATEMENT SET SEMICOLON (insertSimpleStatement SEMICOLON)+ END
;
insertMulStatement
: STATEMENT SET BEGIN (insertSimpleStatement SEMICOLON)+ END
;
// Select statements
queryStatement

2
src/lib/flinksql/FlinkSqlLexer.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlLexer.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlLexer.g4 by ANTLR 4.12.0
// noinspection ES6UnusedImports,JSUnusedGlobalSymbols,JSUnusedLocalSymbols
import {
ATN,

3
src/lib/flinksql/FlinkSqlParser.interp
View File

File diff suppressed because one or more lines are too long

4298
src/lib/flinksql/FlinkSqlParser.ts
View File

File diff suppressed because it is too large Load Diff

13
src/lib/flinksql/FlinkSqlParserListener.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
import {ParseTreeListener} from "antlr4";
@ -77,7 +77,6 @@ import { InsertSimpleStatementContext } from "./FlinkSqlParser";
import { InsertPartitionDefinitionContext } from "./FlinkSqlParser";
import { ValuesDefinitionContext } from "./FlinkSqlParser";
import { ValuesRowDefinitionContext } from "./FlinkSqlParser";
import { InsertMulStatementCompatibilityContext } from "./FlinkSqlParser";
import { InsertMulStatementContext } from "./FlinkSqlParser";
import { QueryStatementContext } from "./FlinkSqlParser";
import { ValuesCaluseContext } from "./FlinkSqlParser";
@ -932,16 +931,6 @@ export default class FlinkSqlParserListener extends ParseTreeListener {
* @param ctx the parse tree
*/
exitValuesRowDefinition?: (ctx: ValuesRowDefinitionContext) => void;
/**
* Enter a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
*/
enterInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => void;
/**
* Exit a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
*/
exitInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => void;
/**
* Enter a parse tree produced by `FlinkSqlParser.insertMulStatement`.
* @param ctx the parse tree

9
src/lib/flinksql/FlinkSqlParserVisitor.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
import {ParseTreeVisitor} from 'antlr4';
@ -77,7 +77,6 @@ import { InsertSimpleStatementContext } from "./FlinkSqlParser";
import { InsertPartitionDefinitionContext } from "./FlinkSqlParser";
import { ValuesDefinitionContext } from "./FlinkSqlParser";
import { ValuesRowDefinitionContext } from "./FlinkSqlParser";
import { InsertMulStatementCompatibilityContext } from "./FlinkSqlParser";
import { InsertMulStatementContext } from "./FlinkSqlParser";
import { QueryStatementContext } from "./FlinkSqlParser";
import { ValuesCaluseContext } from "./FlinkSqlParser";
@ -637,12 +636,6 @@ export default class FlinkSqlParserVisitor<Result> extends ParseTreeVisitor<Resu
* @return the visitor result
*/
visitValuesRowDefinition?: (ctx: ValuesRowDefinitionContext) => Result;
/**
* Visit a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
* @return the visitor result
*/
visitInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => Result;
/**
* Visit a parse tree produced by `FlinkSqlParser.insertMulStatement`.
* @param ctx the parse tree

1
src/parser/common/basicParser.ts
View File

@ -44,7 +44,6 @@ export default abstract class BasicParser {
parser.addErrorListener(new ParserErrorCollector(syntaxErrors));
parser.program();
return lexerError.concat(syntaxErrors);
}

47
test/parser/flinksql/syntax/fixtures/insertFromSelectQueries.sql Normal file
View File

@ -0,0 +1,47 @@
INSERT INTO country_page_view
SELECT user,
cnt
FROM page_view_source;
INSERT INTO catalog1.db1.country_page_view
SELECT user,
cnt
FROM page_view_source;
--- Execute InsertStatement
EXECUTE
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
cnt
FROM page_view_source;
--- Partition Clause: Static Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
cnt
FROM page_view_source;
--- Partition Clause: Dynamic Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30')
SELECT user,
cnt,
country
FROM page_view_source;
--- Column List Statement
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China') (date, country)
SELECT user,
cnt
FROM page_view_source;
--- Insert Method: OverWrite
INSERT OVERWRITE country_page_view PARTITION (date = '2019-8-30')
SELECT user,
cnt,
country
FROM page_view_source;

11
test/parser/flinksql/syntax/fixtures/insertMultipleTable.sql Normal file
View File

@ -0,0 +1,11 @@
EXECUTE STATEMENT SET
BEGIN
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
END;

22
test/parser/flinksql/syntax/fixtures/insertValuesIntoTable.sql Normal file
View File

@ -0,0 +1,22 @@
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT OverWrite country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
INSERT INTO catalog1.db1.country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);

100
test/parser/flinksql/syntax/insertStatement.test.ts
View File

@ -1,83 +1,29 @@
import FlinkSQL from "../../../../src/parser/flinksql";
import FlinkSQL from '../../../../src/parser/flinksql';
import { readSQL } from '../../../helper';
describe('FlinkSQL Create Table Syntax Tests', () => {
const parser = new FlinkSQL();
// insert statements
test('Test one simple Insert Statement', () => {
const sql = `
INSERT INTO country_page_view
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Insert Overwrite Statement', () => {
const sql = `
INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test execute Statement', () => {
const sql = `
EXECUTE INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Partition Clause Statement', () => {
const sql = `
INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Column List Statement', () => {
const sql = `
INSERT INTO emps PARTITION (x='ab', y='bc') (x, y) SELECT * FROM emps;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Insert Values Statement', () => {
const sql = `
INSERT INTO students
VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test insert into multiple tables Statement for 1.14', () => {
const sql = `
BEGIN STATEMENT SET;
INSERT INTO pageviews
SELECT page_id, count(1)
FROM pageviews
GROUP BY page_id;
const features = {
InsertFromSelectQueries: readSQL(__dirname, 'insertFromSelectQueries.sql'),
InsertValuesIntoTable: readSQL(__dirname, 'insertValuesIntoTable.sql'),
InsertMultipleTable: readSQL(__dirname, 'insertMultipleTable.sql')
};
INSERT INTO uniqueview
SELECT page_id, count(distinct user_id)
FROM pageviews
GROUP BY page_id;
END;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test insert into multiple tables Statement for 1.15', () => {
const sql = `
EXECUTE STATEMENT SET
BEGIN
INSERT INTO students VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
INSERT INTO students VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
END;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
describe('FlinkSQL Insert Syntax Tests', () => {
features.InsertFromSelectQueries.forEach((insertFromSelectQueries) => {
it(insertFromSelectQueries, () => {
expect(parser.validate(insertFromSelectQueries).length).toBe(0);
});
});
features.InsertValuesIntoTable.forEach((insertValuesIntoTable) => {
it(insertValuesIntoTable, () => {
expect(parser.validate(insertValuesIntoTable).length).toBe(0);
});
});
features.InsertMultipleTable.forEach((insertMultipleTable) => {
it(insertMultipleTable, () => {
expect(parser.validate(insertMultipleTable).length).toBe(0);
});
});
});