oushu/lava-oushudb-dt-sql-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: improve keywords definations (#100)

Browse Source 
* feat: improve keywords definations and add prefix named KW_

* test: capitalize keywords in alter sql test cases

* test: capitalize keywords in create sql test cases

* feat: merge interval rule and timeIntervalExpression rule in flinksql grammar

* test: put keywords in quotation marks around insert sql test cases

* test: put keywords in quotation marks around select sql test cases

* feat: improve uid rule

* test: rename sum to _sum because it is a reserved keyword

* docs: improve comments in flinkSqlLexer
This commit is contained in:
Hayden
2023-05-19 14:28:25 +08:00
committed by GitHub
parent a026ae0592
commit 40c911597b
19 changed files with 12652 additions and 11856 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/parser/flinksql/syntax.test.ts
View File

@ -39,7 +39,7 @@ describe('FlinkSQL Syntax Tests', () => {
});
test('Test Select Statement with having clause', () => {
const sql = `
SELECT city, sum(quantity) AS sum
SELECT city, sum(quantity) AS _sum
FROM dealer GROUP BY city HAVING city = 'Fremont';
`;
const result = parser.validate(sql);

2
test/parser/flinksql/syntax/fixtures/alterDatabase.sql
View File

@ -1,3 +1,3 @@
ALTER DATABASE tempDB SET ("key1"="value1");
alter database db1 set ('key1' = 'value1','key2.a' = 'value2.a');
ALTER DATABASE db1 SET ('key1' = 'value1','key2.a' = 'value2.a');

8
test/parser/flinksql/syntax/fixtures/alterFunction.sql
View File

@ -1,12 +1,12 @@
ALTER FUNCTION tempFunction AS 'SimpleUdf';
alter temporary function function1 as 'org.apache.flink.function.function1';
ALTER temporary FUNCTION function1 AS 'org.apache.flink.function.function1';
alter temporary function function1 as 'org.apache.flink.function.function1' language scala;
ALTER temporary FUNCTION function1 AS 'org.apache.flink.function.function1' LANGUAGE scala;
alter temporary system function function1 as 'org.apache.flink.function.function1';
ALTER temporary SYSTEM FUNCTION function1 AS 'org.apache.flink.function.function1';
alter temporary system function function1 as 'org.apache.flink.function.function1' language java;
ALTER temporary SYSTEM FUNCTION function1 AS 'org.apache.flink.function.function1' LANGUAGE java;
ALTER TEMPORARY SYSTEM FUNCTION IF EXISTS tempFunction AS 'SimpleUdf';

76
test/parser/flinksql/syntax/fixtures/alterTable.sql
View File

@ -1,65 +1,65 @@
-- Refer: https://github.com/apache/flink/blob/master/flink-table/flink-sql-parser/src/test/java/org/apache/flink/sql/parser/FlinkSqlParserImplTest.java#L2016
-- Just for simple alter table statements, it not include alter table columns statements
alter table
t1 rename to t2;
ALTER TABLE
t1 RENAME TO t2;
alter table
if exists t1 rename to t2;
ALTER TABLE
IF EXISTS t1 RENAME TO t2;
alter table
c1.d1.t1 rename to t2;
ALTER TABLE
c1.d1.t1 RENAME TO t2;
alter table
if exists c1.d1.t1 rename to t2;
ALTER TABLE
IF EXISTS c1.d1.t1 RENAME TO t2;
alter table
t1 rename a to b;
ALTER TABLE
t1 RENAME a TO b;
alter table
if exists t1 rename a to b;
ALTER TABLE
IF EXISTS t1 RENAME a TO b;
alter table
if exists t1 rename a.x to a.y;
ALTER TABLE
IF EXISTS t1 RENAME a.x TO a.y;
alter table
ALTER TABLE
t1
set
('key1' = 'value1');
alter table
if exists t1
ALTER TABLE
IF EXISTS t1
set
('key1' = 'value1');
alter table
ALTER TABLE
t1
add
constraint ct1 primary key(a, b);
ADD
CONSTRAINT ct1 PRIMARY KEY(a, b);
alter table
ALTER TABLE
t1
add
constraint ct1 primary key(a, b) not enforced;
ADD
CONSTRAINT ct1 PRIMARY KEY(a, b) NOT ENFORCED;
alter table
if exists t1
add
constraint ct1 primary key(a, b) not enforced;
ALTER TABLE
IF EXISTS t1
ADD
CONSTRAINT ct1 PRIMARY KEY(a, b) NOT ENFORCED;
alter table
ALTER TABLE
t1
add
unique(a, b);
ADD
UNIQUE(a, b);
alter table
if exists t1
add
unique(a, b);
ALTER TABLE
IF EXISTS t1
ADD
UNIQUE(a, b);
alter table
t1 drop constraint ct1;
ALTER TABLE
t1 DROP CONSTRAINT ct1;
alter table
if exists t1 drop constraint ct1;
ALTER TABLE
IF EXISTS t1 DROP CONSTRAINT ct1;

2
test/parser/flinksql/syntax/fixtures/createFunction.sql
View File

@ -25,4 +25,4 @@ CREATE TEMPORARY FUNCTION function1 AS 'org.apache.flink.function.function1' LAN
CREATE TEMPORARY SYSTEM FUNCTION function1 AS 'org.apache.flink.function.function1' LANGUAGE SCALA USING JAR '/path/to/test.jar';
CREATE FUNCTION function1 AS 'org.apache.flink.function.function1' LANGUAGE JAVA USING JAR 'file:///path/to/test.jar',
jar 'hdfs:///path/to/test2.jar';
JAR 'hdfs:///path/to/test2.jar';

106
test/parser/flinksql/syntax/fixtures/createTable.sql
View File

@ -27,14 +27,14 @@ CREATE TABLE MyTable (
WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
) WITH ('connector' = 'kafka');
CREATE TABLE MyTable (id int, PRIMARY KEY (id) NOT ENFORCED) WITH ('connector' = 'kafka');
CREATE TABLE MyTable (id INT, PRIMARY KEY (id) NOT ENFORCED) WITH ('connector' = 'kafka');
CREATE TABLE tbl1 (
a bigint,
h varchar,
a BIGINT,
h VARCHAR,
g AS 2 * (a + 1),
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
b varchar,
b VARCHAR,
proc AS PROCTIME(),
meta STRING METADATA,
my_meta STRING METADATA FROM 'meta',
@ -57,7 +57,7 @@ CREATE TABLE Orders_in_file (
);
CREATE TABLE Orders_with_watermark (
id int,
id INT,
-- Add watermark definition
WATERMARK FOR order_time AS order_time - INTERVAL '5' SECOND
) WITH (
@ -80,35 +80,35 @@ FROM
WHERE
mod(id, 10) = 0;
CREATE TABLE catalog1.db1.table1 (id int) WITH ('connector' = 'kafka');
CREATE TABLE catalog1.db1.table1 (id INT) WITH ('connector' = 'kafka');
CREATE TABLE catalog1.db1.table1 (
attr0 STRING,
attr1 boolean,
attr3 decimal(38, 18),
attr1 BOOLEAN,
attr3 DECIMAL(38, 18),
attr4 TINYINT,
attr5 smallint,
attr6 int,
attr7 bigint,
attr8 float,
attr9 double,
attr10 date,
attr11 time,
attr12 timestamp(3),
attr13 array<STRING>,
attr14 ROW<attr15 float, attr16 timestamp(3)>,
attr17 MAP<int, bigint>,
attr5 SMALLINT,
attr6 INT,
attr7 BIGINT,
attr8 FLOAT,
attr9 DOUBLE,
attr10 DATE,
attr11 TIME,
attr12 TIMESTAMP(3),
attr13 ARRAY<STRING>,
attr14 ROW<attr15 FLOAT, attr16 TIMESTAMP(3)>,
attr17 MAP<INT, BIGINT>,
name1 VARCHAR(64),
message ROW<data ROW<UPO_TIMESTAMP VARCHAR(20)>>,
raw RAW('class', 'snapshot')
) WITH ('connector' = 'kafka');
CREATE TABLE IF NOT EXISTS tbl1 (
a bigint,
h varchar,
a BIGINT,
h VARCHAR,
g AS 2 * (a + 1),
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
b varchar,
b VARCHAR,
proc AS PROCTIME(),
PRIMARY KEY (a, b) NOT ENFORCED
) PARTITIONED BY (a, h) WITH (
@ -117,11 +117,11 @@ CREATE TABLE IF NOT EXISTS tbl1 (
);
CREATE TABLE tbl1 (
a bigint COMMENT 'test column comment AAA.',
h varchar,
a BIGINT COMMENT 'test column comment AAA.',
h VARCHAR,
g AS 2 * (a + 1),
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
b varchar,
b VARCHAR,
proc AS PROCTIME(),
meta STRING METADATA,
my_meta STRING METADATA FROM 'meta',
@ -133,11 +133,11 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
a bigint COMMENT 'test column comment AAA.',
h varchar,
a BIGINT COMMENT 'test column comment AAA.',
h VARCHAR,
g AS 2 * (a + 1) COMMENT 'test computed column.',
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
b varchar,
b VARCHAR,
proc AS PROCTIME(),
PRIMARY KEY (a, b) NOT ENFORCED
) COMMENT 'test table comment ABC.' PARTITIONED BY (a, h) WITH (
@ -146,11 +146,11 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
a bigint,
h varchar,
a BIGINT,
h VARCHAR,
g AS 2 * (a + 1),
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
b varchar,
b VARCHAR,
proc AS PROCTIME(),
PRIMARY KEY (a, b) NOT ENFORCED
) WITH (
@ -159,8 +159,8 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
a bigint PRIMARY KEY NOT ENFORCED COMMENT 'test column comment AAA.',
h varchar CONSTRAINT ct1 PRIMARY KEY NOT ENFORCED,
a BIGINT PRIMARY KEY NOT ENFORCED COMMENT 'test column comment AAA.',
h VARCHAR CONSTRAINT ct1 PRIMARY KEY NOT ENFORCED,
g AS 2 * (a + 1),
ts AS toTimestamp(b, 'yyyy-MM-dd HH:mm:ss'),
proc AS PROCTIME()
@ -170,16 +170,16 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
ts timestamp(3),
id varchar,
watermark FOR ts AS ts - INTERVAL '3' SECOND
ts TIMESTAMP(3),
id VARCHAR,
WATERMARK FOR ts AS ts - INTERVAL '3' SECOND
) WITH (
'connector' = 'kafka',
'kafka.topic' = 'log.test'
);
CREATE TABLE tbl1 (
log_ts varchar,
log_ts VARCHAR,
ts AS to_timestamp(log_ts),
WATERMARK FOR ts AS ts + INTERVAL '1' SECOND
) WITH (
@ -188,7 +188,7 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
f1 ROW<q1 bigint, q2 ROW<t1 timestamp, t2 varchar>, q3 boolean>,
f1 ROW<q1 BIGINT, q2 ROW<t1 TIMESTAMP, t2 VARCHAR>, q3 BOOLEAN>,
WATERMARK FOR f1.q2.t1 AS NOW()
) WITH (
'connector' = 'kafka',
@ -196,36 +196,36 @@ CREATE TABLE tbl1 (
);
CREATE TABLE tbl1 (
a ARRAY<bigint>,
b MAP<int, varchar>,
c ROW<cc0 int, cc1 float, cc2 varchar>,
d MULTISET<varchar>,
a ARRAY<BIGINT>,
b MAP<INT, VARCHAR>,
c ROW<cc0 INT, cc1 FLOAT, cc2 VARCHAR>,
d MULTISET<VARCHAR>,
PRIMARY KEY (a, b) NOT ENFORCED
) with (
) WITH (
'x' = 'y',
'asd' = 'data'
);
CREATE TABLE tbl1 (
a ARRAY<ARRAY<bigint>>,
b MAP<MAP<int, varchar>, ARRAY<varchar>>,
c ROW<cc0 ARRAY<int>, cc1 float, cc2 varchar>,
d MULTISET<ARRAY<int>>,
a ARRAY<ARRAY<BIGINT>>,
b MAP<MAP<INT, VARCHAR>, ARRAY<VARCHAR>>,
c ROW<cc0 ARRAY<INT>, cc1 FLOAT, cc2 VARCHAR>,
d MULTISET<ARRAY<INT>>,
f TIMESTAMP(9),
PRIMARY KEY (a, b) NOT ENFORCED
) with (
) WITH (
'x' = 'y',
'asd' = 'data'
);
CREATE TABLE tbl1 (
a ARRAY<ARRAY<bigint>>,
b MAP<MAP<int, varchar>, ARRAY<varchar>>,
c ROW<cc0 ARRAY<int>, cc1 float, cc2 varchar>,
d MULTISET<ARRAY<int>>,
a ARRAY<ARRAY<BIGINT>>,
b MAP<MAP<INT, VARCHAR>, ARRAY<VARCHAR>>,
c ROW<cc0 ARRAY<INT>, cc1 FLOAT, cc2 VARCHAR>,
d MULTISET<ARRAY<INT>>,
f TIMESTAMP(9),
PRIMARY KEY (a, b) NOT ENFORCED
) with (
) WITH (
'x' = 'y',
'asd' = 'data'
) LIKE Orders (

24
test/parser/flinksql/syntax/fixtures/insertFromSelectQueries.sql
View File

@ -1,43 +1,43 @@
INSERT INTO country_page_view
SELECT user,
SELECT `user`,
cnt
FROM page_view_source;
INSERT INTO catalog1.db1.country_page_view
SELECT user,
SELECT `user`,
cnt
FROM page_view_source;
--- Execute InsertStatement
EXECUTE
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
INSERT INTO country_page_view PARTITION (`date` = '2019-8-30', country = 'China')
SELECT `user`,
cnt
FROM page_view_source;
--- Partition Clause: Static Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China')
SELECT user,
INSERT INTO country_page_view PARTITION (`date` = '2019-8-30', country = 'China')
SELECT `user`,
cnt
FROM page_view_source;
--- Partition Clause: Dynamic Partition
INSERT INTO country_page_view PARTITION (date = '2019-8-30')
SELECT user,
INSERT INTO country_page_view PARTITION (`date` = '2019-8-30')
SELECT `user`,
cnt,
country
FROM page_view_source;
--- Column List Statement
INSERT INTO country_page_view PARTITION (date = '2019-8-30', country = 'China') (date, country)
SELECT user,
INSERT INTO country_page_view PARTITION (`date` = '2019-8-30', country = 'China') (`date`, country)
SELECT `user`,
cnt
FROM page_view_source;
--- Insert Method: OverWrite
INSERT OVERWRITE country_page_view PARTITION (date = '2019-8-30')
SELECT user,
INSERT OVERWRITE country_page_view PARTITION (`date` = '2019-8-30')
SELECT `user`,
cnt,
country
FROM page_view_source;

2
test/parser/flinksql/syntax/fixtures/insertValuesIntoTable.sql
View File

@ -8,7 +8,7 @@ VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);
EXECUTE
INSERT OverWrite country_page_view
INSERT OVERWRITE country_page_view
VALUES ('Chinese', 'mumiao', 18),
('Amercian', 'georage', 22);

14
test/parser/flinksql/syntax/fixtures/selectAggregation.sql
View File

@ -66,33 +66,33 @@ GROUP BY
-- Group Window Aggregation
SELECT
user,
`user`,
TUMBLE_START(order_time, INTERVAL '1' DAY) AS wStart,
SUM(amount) FROM Orders
GROUP BY
TUMBLE(order_time, INTERVAL '1' DAY),
user;
`user`;
SELECT
user,
`user`,
TUMBLE_START(order_time, INTERVAL '1' DAY) AS wStart,
SUM(amount) FROM Orders
GROUP BY
HOP(order_time, INTERVAL '1' DAY),
user;
`user`;
SELECT
user,
`user`,
TUMBLE_START(order_time, INTERVAL '1' DAY) AS wStart,
SUM(amount) FROM Orders
GROUP BY
SESSION(order_time, INTERVAL '1' DAY),
user;
`user`;
-- Having
SELECT SUM(amount)
FROM Orders
GROUP BY users
GROUP BY `users`
HAVING SUM(amount) > 50;
-- Over Aggregation