oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:DTStack/dt-sql-parser

Browse Source
This commit is contained in:
wewoor 2023-05-12 10:47:46 +08:00
commit 526a58ad2e
14 changed files with 5960 additions and 2339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
jest.config.js
View File

@ -7,7 +7,6 @@ module.exports = {
// All imported modules in your tests should be mocked automatically
// automock: false,
// Stop running tests after `n` failures
// bail: 0,
@ -85,7 +84,7 @@ module.exports = {
"ts",
"tsx",
"json",
"node"
"node",
],
// A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
@ -154,10 +153,7 @@ module.exports = {
// testLocationInResults: false,
// The glob patterns Jest uses to detect test files
testMatch: [
"**/__tests__/**/*.[jt]s?(x)",
"**/?(*.)+(spec|test).[tj]s?(x)"
],
testMatch: ["**/__tests__/**/*.[jt]s?(x)", "**/?(*.)+(spec|test).[tj]s?(x)"],
// An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
// testPathIgnorePatterns: [
@ -175,17 +171,18 @@ module.exports = {
// A map from regular expressions to paths to transformers
transform: {
'\\.[jt]sx?$': ['ts-jest', {
"\\.[jt]sx?$": [
"ts-jest",
{
tsconfig: {
noUnusedLocals: false,
}
}]
},
},
],
},
// An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
transformIgnorePatterns: [
'/node_modules/(?!antlr4)'
],
transformIgnorePatterns: ["/node_modules/(?!antlr4)"],
// An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
// unmockedModulePathPatterns: undefined,
@ -198,7 +195,7 @@ module.exports = {
// Whether to use watchman for file crawling
// watchman: true,
moduleNameMapper : {
'^antlr4$': '<rootDir>/node_modules/antlr4/src/antlr4/index.web.js',
}
moduleNameMapper: {
"^antlr4$": "<rootDir>/node_modules/antlr4/src/antlr4/index.web.js",
},
};

3727
pnpm-lock.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

7
src/grammar/flinksql/FlinkSqlParser.g4
View File

@ -329,7 +329,7 @@ dropFunction
// Insert statements
insertStatement
: (EXECUTE? insertSimpleStatement) | insertMulStatementCompatibility | (EXECUTE insertMulStatement)
: (EXECUTE? insertSimpleStatement) | (EXECUTE insertMulStatement)
;
insertSimpleStatement
@ -354,15 +354,10 @@ valuesRowDefinition
RR_BRACKET
;
insertMulStatementCompatibility
: BEGIN STATEMENT SET SEMICOLON (insertSimpleStatement SEMICOLON)+ END
;
insertMulStatement
: STATEMENT SET BEGIN (insertSimpleStatement SEMICOLON)+ END
;
// Select statements
queryStatement

2
src/lib/flinksql/FlinkSqlLexer.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlLexer.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlLexer.g4 by ANTLR 4.12.0
// noinspection ES6UnusedImports,JSUnusedGlobalSymbols,JSUnusedLocalSymbols
import {
ATN,

3
src/lib/flinksql/FlinkSqlParser.interp
View File

File diff suppressed because one or more lines are too long

4298
src/lib/flinksql/FlinkSqlParser.ts
View File

File diff suppressed because it is too large Load Diff

13
src/lib/flinksql/FlinkSqlParserListener.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
import {ParseTreeListener} from "antlr4";
@ -77,7 +77,6 @@ import { InsertSimpleStatementContext } from "./FlinkSqlParser";
import { InsertPartitionDefinitionContext } from "./FlinkSqlParser";
import { ValuesDefinitionContext } from "./FlinkSqlParser";
import { ValuesRowDefinitionContext } from "./FlinkSqlParser";
import { InsertMulStatementCompatibilityContext } from "./FlinkSqlParser";
import { InsertMulStatementContext } from "./FlinkSqlParser";
import { QueryStatementContext } from "./FlinkSqlParser";
import { ValuesCaluseContext } from "./FlinkSqlParser";
@ -932,16 +931,6 @@ export default class FlinkSqlParserListener extends ParseTreeListener {
* @param ctx the parse tree
*/
exitValuesRowDefinition?: (ctx: ValuesRowDefinitionContext) => void;
/**
* Enter a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
*/
enterInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => void;
/**
* Exit a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
*/
exitInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => void;
/**
* Enter a parse tree produced by `FlinkSqlParser.insertMulStatement`.
* @param ctx the parse tree

9
src/lib/flinksql/FlinkSqlParserVisitor.ts
View File

@ -1,4 +1,4 @@
// Generated from /Users/mortalYoung/Projects/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
// Generated from /Users/zhenglin/Documents/parser/dt-sql-parser/src/grammar/flinksql/FlinkSqlParser.g4 by ANTLR 4.12.0
import {ParseTreeVisitor} from 'antlr4';
@ -77,7 +77,6 @@ import { InsertSimpleStatementContext } from "./FlinkSqlParser";
import { InsertPartitionDefinitionContext } from "./FlinkSqlParser";
import { ValuesDefinitionContext } from "./FlinkSqlParser";
import { ValuesRowDefinitionContext } from "./FlinkSqlParser";
import { InsertMulStatementCompatibilityContext } from "./FlinkSqlParser";
import { InsertMulStatementContext } from "./FlinkSqlParser";
import { QueryStatementContext } from "./FlinkSqlParser";
import { ValuesCaluseContext } from "./FlinkSqlParser";
@ -637,12 +636,6 @@ export default class FlinkSqlParserVisitor<Result> extends ParseTreeVisitor<Resu
* @return the visitor result
*/
visitValuesRowDefinition?: (ctx: ValuesRowDefinitionContext) => Result;
/**
* Visit a parse tree produced by `FlinkSqlParser.insertMulStatementCompatibility`.
* @param ctx the parse tree
* @return the visitor result
*/
visitInsertMulStatementCompatibility?: (ctx: InsertMulStatementCompatibilityContext) => Result;
/**
* Visit a parse tree produced by `FlinkSqlParser.insertMulStatement`.
* @param ctx the parse tree

1
src/parser/common/basicParser.ts
View File

@ -44,7 +44,6 @@ export default abstract class BasicParser {
parser.addErrorListener(new ParserErrorCollector(syntaxErrors));
parser.program();
return lexerError.concat(syntaxErrors);
}

34
test/helper.ts
View File

@ -1,10 +1,30 @@
import fs from 'fs';
import path from 'path';
export const readSQL = (dirname: string, fileName: string, isSegment = true) => {
const sqlFiles = fs.readFileSync(path.join(dirname, 'fixtures', fileName), 'utf-8')
if (!isSegment) return [sqlFiles];
return sqlFiles.split(';')
.filter(Boolean)
.map((i) => i.trim());
}
export const readSQL = (dirname: string, fileName: string) => {
const content = fs.readFileSync(path.join(dirname, 'fixtures', fileName), 'utf-8');
const result: string[] = [];
let tmp = '';
for (let index = 0; index < content.length; index++) {
const char = content[index];
tmp += char;
const isMulti = tmp.includes('EXECUTE STATEMENT SET');
if (!isMulti) {
// 非批量的先简单按照分号切割
if (tmp.endsWith(';')) {
result.push(tmp.trim());
tmp = '';
}
} else {
if (tmp.endsWith('END;')) {
result.push(tmp.trim());
tmp = '';
}
}
}
return result;
};

47
test/parser/flinksql/syntax/fixtures/insertFromSelectQueries.sql Normal file
View File

@ -0,0 +1,47 @@
INSERT INTO country_page_view
SELECT user,
cnt
FROM page_view_source;
INSERT INTO catalog1.db1.country_page_view
SELECT user,
cnt
FROM page_view_source;
--- Execute InsertStatement
EXECUTE
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
cnt
FROM page_view_source;
--- Partition Clause: Static Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
cnt
FROM page_view_source;
--- Partition Clause: Dynamic Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30')
SELECT user,
cnt,
country
FROM page_view_source;
--- Column List Statement
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China') (date, country)
SELECT user,
cnt
FROM page_view_source;
--- Insert Method: OverWrite
INSERT OVERWRITE country_page_view PARTITION (date = '2019-8-30')
SELECT user,
cnt,
country
FROM page_view_source;

11
test/parser/flinksql/syntax/fixtures/insertMultipleTable.sql Normal file
View File

@ -0,0 +1,11 @@
EXECUTE STATEMENT SET
BEGIN
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
END;

22
test/parser/flinksql/syntax/fixtures/insertValuesIntoTable.sql Normal file
View File

@ -0,0 +1,22 @@
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT OverWrite country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT INTO country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
INSERT INTO catalog1.db1.country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);

98
test/parser/flinksql/syntax/insertStatement.test.ts
View File

@ -1,83 +1,29 @@
import FlinkSQL from "../../../../src/parser/flinksql";
import FlinkSQL from '../../../../src/parser/flinksql';
import { readSQL } from '../../../helper';
describe('FlinkSQL Create Table Syntax Tests', () => {
const parser = new FlinkSQL();
// insert statements
test('Test one simple Insert Statement', () => {
const sql = `
INSERT INTO country_page_view
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Insert Overwrite Statement', () => {
const sql = `
INSERT OVERWRITE country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test execute Statement', () => {
const sql = `
EXECUTE INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Partition Clause Statement', () => {
const sql = `
INSERT INTO country_page_view PARTITION (date='2019-8-30', country='China')
SELECT user, cnt FROM page_view_source;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Column List Statement', () => {
const sql = `
INSERT INTO emps PARTITION (x='ab', y='bc') (x, y) SELECT * FROM emps;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test Insert Values Statement', () => {
const sql = `
INSERT INTO students
VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
test('Test insert into multiple tables Statement for 1.14', () => {
const sql = `
BEGIN STATEMENT SET;
INSERT INTO pageviews
SELECT page_id, count(1)
FROM pageviews
GROUP BY page_id;
const parser = new FlinkSQL();
INSERT INTO uniqueview
SELECT page_id, count(distinct user_id)
FROM pageviews
GROUP BY page_id;
const features = {
InsertFromSelectQueries: readSQL(__dirname, 'insertFromSelectQueries.sql'),
InsertValuesIntoTable: readSQL(__dirname, 'insertValuesIntoTable.sql'),
InsertMultipleTable: readSQL(__dirname, 'insertMultipleTable.sql')
};
END;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
describe('FlinkSQL Insert Syntax Tests', () => {
features.InsertFromSelectQueries.forEach((insertFromSelectQueries) => {
it(insertFromSelectQueries, () => {
expect(parser.validate(insertFromSelectQueries).length).toBe(0);
});
});
features.InsertValuesIntoTable.forEach((insertValuesIntoTable) => {
it(insertValuesIntoTable, () => {
expect(parser.validate(insertValuesIntoTable).length).toBe(0);
});
});
features.InsertMultipleTable.forEach((insertMultipleTable) => {
it(insertMultipleTable, () => {
expect(parser.validate(insertMultipleTable).length).toBe(0);
});
test('Test insert into multiple tables Statement for 1.15', () => {
const sql = `
EXECUTE STATEMENT SET
BEGIN
INSERT INTO students VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
INSERT INTO students VALUES ('fred flintstone', 35, 1.28), ('barney rubble', 32, 2.32);
END;
`;
const result = parser.validate(sql);
expect(result.length).toBe(0);
});
});