oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: support spark sql auto complete (#179)

Browse Source 
* refactor: spark sql g4

* feat: support spark sql suggestion

* test: spark sql suggestion unit test

* test: hive spell check

* feat: spark sql keyword has multiple values

* test: KW_NOT KW_RLIKE split into two value

---------

Co-authored-by: liuyi <liuyi@dtstack.com>
This commit is contained in:
琉易
2023-10-12 17:08:21 +08:00
committed by GitHub
parent 53ead45ff5
commit 4d1dfa676f
18 changed files with 10121 additions and 9017 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
test/parser/spark/suggestion/fixtures/syntaxSuggestion.sql Normal file
View File

@ -0,0 +1,19 @@
INSERT INTO db.tb ;
SELECT * FROM db.;
CREATE TABLE db. VALUES;
DROP TABLE IF EXISTS db.a;
CREATE OR REPLACE VIEW db.v;
DROP VIEW db.v ;
CREATE FUNCTION fn1;
SELECT name, calculate_age(birthday) AS age FROM students;
CREATE DATABASE db;
DROP SCHEMA IF EXISTS sch;

18
test/parser/spark/suggestion/fixtures/tokenSuggestion.sql Normal file
View File

@ -0,0 +1,18 @@
ALTER
;
CREATE
;
DELETE
;
DESCRIBE
;
DROP
;
INSERT
;
LOAD
;
SHOW
;
EXPORT
;

146
test/parser/spark/suggestion/syntaxSuggestion.test.ts Normal file
View File

@ -0,0 +1,146 @@
import fs from 'fs';
import path from 'path';
import { CaretPosition, SyntaxContextType } from '../../../../src/parser/common/basic-parser-types';
import SparkSQL from '../../../../src/parser/spark';
const syntaxSql = fs.readFileSync(path.join(__dirname, 'fixtures', 'syntaxSuggestion.sql'), 'utf-8');
describe('Spark SQL Syntax Suggestion', () => {
const parser = new SparkSQL();
test('Validate Syntax SQL', () => {
expect(parser.validate(syntaxSql).length).not.toBe(0);
expect(parser.validate(syntaxSql).length).not.toBe(0);
expect(parser.validate(syntaxSql).length).not.toBe(0);
});
test('Insert table ', () => {
const pos: CaretPosition = {
lineNumber: 1,
column: 18,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.TABLE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.', 'tb']);
});
test('Select table ', () => {
const pos: CaretPosition = {
lineNumber: 3,
column: 18,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.TABLE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.']);
});
test('Create table ', () => {
const pos: CaretPosition = {
lineNumber: 5,
column: 17,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.TABLE_CREATE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.']);
});
test('DROP table ', () => {
const pos: CaretPosition = {
lineNumber: 7,
column: 26,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.TABLE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.', 'a']);
});
test('Create view ', () => {
const pos: CaretPosition = {
lineNumber: 9,
column: 28,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.VIEW_CREATE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.', 'v']);
});
test('Drop view ', () => {
const pos: CaretPosition = {
lineNumber: 11,
column: 15,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.VIEW);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db', '.', 'v']);
});
test('Create function ', () => {
const pos: CaretPosition = {
lineNumber: 13,
column: 20,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.FUNCTION_CREATE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['fn1']);
});
test('Use function', () => {
const pos: CaretPosition = {
lineNumber: 15,
column: 27,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.FUNCTION);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['calculate_age']);
});
test('Create database', () => {
const pos: CaretPosition = {
lineNumber: 17,
column: 19,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.DATABASE_CREATE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['db']);
});
test('Drop database', () => {
const pos: CaretPosition = {
lineNumber: 19,
column: 26,
};
const syntaxes = parser.getSuggestionAtCaretPosition(syntaxSql, pos)?.syntax;
const suggestion = syntaxes?.find((syn) => syn.syntaxContextType === SyntaxContextType.DATABASE);
expect(suggestion).not.toBeUndefined();
expect(suggestion?.wordRanges.map((token) => token.text))
.toEqual(['sch']);
});
});

200
test/parser/spark/suggestion/tokenSuggestion.test.ts Normal file
View File

@ -0,0 +1,200 @@
import fs from 'fs';
import path from 'path';
import { CaretPosition } from '../../../../src/parser/common/basic-parser-types';
import SparkSQL from '../../../../src/parser/spark';
const tokenSql = fs.readFileSync(path.join(__dirname, 'fixtures', 'tokenSuggestion.sql'), 'utf-8');
describe('Spark SQL Syntax Suggestion', () => {
const parser = new SparkSQL();
test('After ALTER', () => {
const pos: CaretPosition = {
lineNumber: 1,
column: 7,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'TABLE',
'INDEX',
'VIEW',
'DATABASE',
'NAMESPACE',
'SCHEMA',
]);
});
test('After CREATE', () => {
const pos: CaretPosition = {
lineNumber: 3,
column: 8,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'TEMPORARY',
'INDEX',
'ROLE',
'FUNCTION',
'OR',
'GLOBAL',
'VIEW',
'TABLE',
'EXTERNAL',
'DATABASE',
'NAMESPACE',
'SCHEMA',
]);
});
test('After DELETE', () => {
const pos: CaretPosition = {
lineNumber: 5,
column: 8,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual(['FROM']);
});
test('After DESCRIBE', () => {
const pos: CaretPosition = {
lineNumber: 7,
column: 10,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'WITH',
'SELECT',
'MAP',
'REDUCE',
'FROM',
'TABLE',
'VALUES',
'QUERY',
'EXTENDED',
'FORMATTED',
'DATABASE',
'FUNCTION',
]);
});
test('After DROP', () => {
const pos: CaretPosition = {
lineNumber: 9,
column: 6,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'TEMPORARY',
'INDEX',
'ROLE',
'FUNCTION',
'VIEW',
'TABLE',
'DATABASE',
'NAMESPACE',
'SCHEMA',
]);
});
test('After INSERT', () => {
const pos: CaretPosition = {
lineNumber: 11,
column: 8,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'OVERWRITE',
'INTO',
]);
});
test('After LOAD', () => {
const pos: CaretPosition = {
lineNumber: 13,
column: 6,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'DATA',
]);
});
test('After SHOW', () => {
const pos: CaretPosition = {
lineNumber: 15,
column: 6,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual([
'LOCKS',
'INDEXES',
'TRANSACTIONS',
'CREATE',
'COMPACTIONS',
'CURRENT',
'ROLES',
'PRINCIPALS',
'ROLE',
'GRANT',
'CATALOGS',
'FUNCTIONS',
'ALL',
'SYSTEM',
'USER',
'PARTITIONS',
'VIEWS',
'COLUMNS',
'TBLPROPERTIES',
'TABLE',
'TABLES',
'DATABASES',
'NAMESPACES',
'SCHEMAS',
]);
});
test('After EXPORT', () => {
const pos: CaretPosition = {
lineNumber: 17,
column: 8,
};
const suggestion = parser.getSuggestionAtCaretPosition(
tokenSql,
pos,
)?.keywords;
expect(suggestion).toEqual(['TABLE']);
});
});

6
test/parser/spark/syntax/fixtures/kwMultipleValues.sql Normal file
View File

@ -0,0 +1,6 @@
SELECT * FROM table_name WHERE NOT (age > 30);
SELECT * FROM table_name WHERE ! (age > 30);
SELECT * FROM table_name WHERE name RLIKE 'M+';
SELECT * FROM table_name WHERE name REGEXP 'M+';

23
test/parser/spark/syntax/kwMultipleValues.test.ts Normal file
View File

@ -0,0 +1,23 @@
import SparkSQL from '../../../../src/parser/spark';
import { readSQL } from '../../../helper';
const parser = new SparkSQL();
/**
* 关键词有多个值
* KW_NOT: 'NOT' | '!'
* KW_RLIKE: 'RLIKE' | 'REGEXP';
*/
const features = {
kwMultipleValues: readSQL(__dirname, 'kwMultipleValues.sql'),
};
describe('SparkSQL Insert Syntax Tests', () => {
Object.keys(features).forEach((key) => {
features[key].forEach((sql) => {
it(sql, () => {
expect(parser.validate(sql).length).toBe(0);
});
});
});
});