Feat check hive create syntax (#157)

* check and update abort and alter sql

* delete superfluous space

* update grammar lib after change alter parser

* update alter parser to be compatible

* check create sql

---------

Co-authored-by: zhaoge <>

src/grammar/hive/HiveSqlParser.g4

@@ -225,7 +225,7 @@ orReplace
     ;
 
 createDatabaseStatement
-    : KW_CREATE db_schema
+    : KW_CREATE KW_REMOTE? db_schema
         ifNotExists?
         name=id_
         databaseComment?
@@ -1687,7 +1687,7 @@ createTableStatement
     ;
 
 createDataConnectorStatement
-    : KW_CREATE KW_DATACONNECTOR ifNotExists? name=id_ dataConnectorType dataConnectorUrl dataConnectorComment?
+    : KW_CREATE KW_DATACONNECTOR ifNotExists? name=id_ dataConnectorType? dataConnectorUrl? dataConnectorComment?
         (KW_WITH KW_DCPROPERTIES dcprops=dcProperties)?
     ;
 

src/lib/hive/HiveSqlLexer.ts

@@ -1,4 +1,4 @@
-// Generated from /Users/hayden/Desktop/dt-works/dt-sql-parser/src/grammar/hive/HiveSqlLexer.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from /Users/xuxiaoqi/Documents/dt-sql-parser-copy/src/grammar/hive/HiveSqlLexer.g4 by ANTLR 4.9.0-SNAPSHOT
 
 
 import { ATN } from "antlr4ts/atn/ATN";

src/lib/hive/HiveSqlParserListener.ts

@@ -1,4 +1,4 @@
-// Generated from /Users/hayden/Desktop/dt-works/dt-sql-parser/src/grammar/hive/HiveSqlParser.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from /Users/xuxiaoqi/Documents/dt-sql-parser-copy/src/grammar/hive/HiveSqlParser.g4 by ANTLR 4.9.0-SNAPSHOT
 
 
 import { ParseTreeListener } from "antlr4ts/tree/ParseTreeListener";

src/lib/hive/HiveSqlParserVisitor.ts

@@ -1,4 +1,4 @@
-// Generated from /Users/hayden/Desktop/dt-works/dt-sql-parser/src/grammar/hive/HiveSqlParser.g4 by ANTLR 4.9.0-SNAPSHOT
+// Generated from /Users/xuxiaoqi/Documents/dt-sql-parser-copy/src/grammar/hive/HiveSqlParser.g4 by ANTLR 4.9.0-SNAPSHOT
 
 
 import { ParseTreeVisitor } from "antlr4ts/tree/ParseTreeVisitor";

test/parser/hive/syntax/fixtures/createConnector.sql

@@ -1,7 +1,28 @@
+CREATE CONNECTOR pg_local;
+
+CREATE CONNECTOR IF NOT EXISTS mysql_local;
+
+CREATE CONNECTOR pg_local TYPE 'mysql';
+
+CREATE CONNECTOR IF NOT EXISTS mysql_local
+TYPE 'mysql';
+
+CREATE CONNECTOR pg_local TYPE 'mysql' URL 'jdbc:mysql://localhost:5432';
+
 CREATE CONNECTOR IF NOT EXISTS mysql_local
 TYPE 'mysql' 
 URL 'jdbc:mysql://localhost:5432';
 
+CREATE CONNECTOR pg_local TYPE 'mysql' URL 'jdbc:mysql://localhost:5432' COMMENT '这是一个 postgres 连接器';
+
+CREATE CONNECTOR pg_local 
+TYPE 'postgres' 
+URL 'jdbc:postgresql://localhost:5432' 
+COMMENT '这是一个 postgres 连接器';
+
+CREATE CONNECTOR pg_local TYPE 'mysql' URL 'jdbc:mysql://localhost:5432' COMMENT '这是一个 postgres 连接器' 
+WITH DCPROPERTIES ("hive.sql.dbcp.username"="postgres", "hive.sql.dbcp.password"="postgres");
+
 CREATE CONNECTOR pg_local 
 TYPE 'postgres' 
 URL 'jdbc:postgresql://localhost:5432' 

test/parser/hive/syntax/fixtures/createDatabase.sql

@@ -1,15 +1,27 @@
 CREATE DATABASE mydb;
 
+CREATE REMOTE DATABASE mydb;
+
+CREATE REMOTE DATABASE IF NOT EXISTS mydb;
+
 CREATE SCHEMA myschema;
 
-CREATE DATABASE IF NOT EXISTS mydb
+CREATE REMOTE SCHEMA myschema;
-    COMMENT 'my test db'
-    LOCATION '/myhive/myoutdb'
-    MANAGEDLOCATION '/myhive/myindb'
-    WITH DBPROPERTIES  ('creator'='ypc','date'='2021-03-09');
 
-CREATE SCHEMA IF NOT EXISTS myschema 
+CREATE REMOTE SCHEMA IF NOT EXISTS myschema;
-    COMMENT 'my test myschema'
+
-    LOCATION '/myhive/myoutschema'
+CREATE DATABASE IF NOT EXISTS mydb COMMENT 'my test db';
-    MANAGEDLOCATION '/myhive/myinschema'
+
-    WITH DBPROPERTIES  ('creator'='ypc','date'='2021-03-09');
+CREATE DATABASE IF NOT EXISTS mydb COMMENT 'my test db' LOCATION '/myhive/myoutdb';
+
+CREATE DATABASE IF NOT EXISTS mydb COMMENT 'my test db' LOCATION '/myhive/myoutdb' MANAGEDLOCATION '/myhive/myindb';
+
+CREATE REMOTE DATABASE IF NOT EXISTS mydb COMMENT 'my test db' LOCATION '/myhive/myoutdb' MANAGEDLOCATION '/myhive/myindb' WITH DBPROPERTIES ('creator' = 'ypc', 'date' = '2021-03-09');
+
+CREATE SCHEMA IF NOT EXISTS myschema COMMENT 'my test myschema';
+
+CREATE SCHEMA IF NOT EXISTS myschema COMMENT 'my test myschema' LOCATION '/myhive/myoutschema';
+
+CREATE SCHEMA IF NOT EXISTS myschema COMMENT 'my test myschema' LOCATION '/myhive/myoutschema' MANAGEDLOCATION '/myhive/myinschema';
+
+CREATE SCHEMA IF NOT EXISTS myschema COMMENT 'my test myschema' LOCATION '/myhive/myoutschema' MANAGEDLOCATION '/myhive/myinschema' WITH DBPROPERTIES ('creator' = 'ypc', 'date' = '2021-03-09');

test/parser/hive/syntax/fixtures/createIndex.sql

@@ -1,3 +1,7 @@
+CREATE INDEX table01_index 
+ON TABLE table01 (column2) 
+AS 'org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler';
+
 CREATE INDEX table01_index 
 ON TABLE table01 (column2) 
 AS 'org.apache.hadoop.hive.ql.index.compact.CompactIndexHandler'
@@ -30,3 +34,60 @@ ON TABLE table06 (column7)
 AS 'COMPACT' 
 ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 
 STORED AS TEXTFILE;
+
+CREATE INDEX table07_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 
+STORED AS TEXTFILE;
+
+CREATE INDEX table08_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+IDXPROPERTIES ("prop3"="value3", "prop4"="value4")
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 
+STORED AS TEXTFILE;
+
+CREATE INDEX table08_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+IDXPROPERTIES ("prop3"="value3", "prop4"="value4")
+IN TABLE indextable1
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' 
+STORED AS TEXTFILE;
+
+CREATE INDEX table08_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+IDXPROPERTIES ("prop3"="value3", "prop4"="value4")
+IN TABLE indextable1
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE 
+LOCATION '/dhs_path';
+
+CREATE INDEX table08_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+IDXPROPERTIES ("prop3"="value3", "prop4"="value4")
+IN TABLE indextable1
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE 
+LOCATION '/dhs_path' 
+TBLPROPERTIES ("prop1"="value1", "prop2"="value2");
+
+CREATE INDEX table08_index 
+ON TABLE table06 (column7) 
+AS 'COMPACT' 
+WITH DEFERRED REBUILD
+IDXPROPERTIES ("prop3"="value3", "prop4"="value4")
+IN TABLE indextable1
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+STORED AS TEXTFILE 
+LOCATION '/dhs_path' 
+TBLPROPERTIES ("prop1"="value1", "prop2"="value2")
+COMMENT '这是一个索引';

test/parser/hive/syntax/fixtures/createTable.sql

@@ -1,14 +1,84 @@
 CREATE TEMPORARY TABLE list_bucket_multiple (col1 STRING, col2 INT, col3 STRING);
 
-CREATE TRANSACTIONAL TABLE transactional_table_test(
+CREATE TEMPORARY EXTERNAL TABLE list_bucket_multiple (col1 STRING, col2 INT, col3 STRING);
-    key STRING, 
+
-    value STRING
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING);
-) 
+
-PARTITIONED BY(ds STRING) STORED AS ORC;
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) COMMENT 'this is a comment';
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) COMMENT 'this is a comment1' PARTITIONED BY (`date` STRING COMMENT 'column_comment');
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) COMMENT 'this is a comment2' PARTITIONED BY (`date` STRING COMMENT 'column_comment') CLUSTERED BY (col1, col2) INTO 32 BUCKETS;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) COMMENT 'this is a comment3' PARTITIONED BY (`date` STRING COMMENT 'column_comment') CLUSTERED BY (col1, col2) SORTED BY (col1 ASC) INTO 22 BUCKETS;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) COMMENT 'this is a comment4' PARTITIONED BY (`date` STRING COMMENT 'column_comment') CLUSTERED BY (col1, col2) SORTED BY (col1 ASC) INTO 34 BUCKETS SKEWED BY (col1, col2) ON (('s1', 1), ('s3', 3), ('s13', 13), ('s78', 78)) STORED AS DIRECTORIES;
+
+CREATE TABLE page_view(
+    viewTime INT,
+    userid BIGINT,
+    page_url STRING,
+    referrer_url STRING,
+    ip STRING COMMENT 'IP Address of the User'
+) COMMENT 'This is the page view table' PARTITIONED BY(dt STRING, country STRING) CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY '\002' MAP KEYS TERMINATED BY '\003' STORED AS SEQUENCEFILE;
+
+CREATE TEMPORARY EXTERNAL TABLE page_view(
+    viewTime INT,
+    userid BIGINT,
+    page_url STRING,
+    referrer_url STRING,
+    ip STRING COMMENT 'IP Address of the User'
+) COMMENT 'This is the page view table' PARTITIONED BY(dt STRING, country STRING) CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY '\002' MAP KEYS TERMINATED BY '\003' STORED AS SEQUENCEFILE;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS page_view(
+    viewTime INT,
+    userid BIGINT,
+    page_url STRING,
+    referrer_url STRING,
+    ip STRING COMMENT 'IP Address of the User'
+) COMMENT 'This is the page view table' PARTITIONED BY(dt STRING, country STRING) CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY '\002' MAP KEYS TERMINATED BY '\003' STORED AS SEQUENCEFILE;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS page_view(
+    viewTime INT,
+    userid BIGINT,
+    page_url STRING,
+    referrer_url STRING,
+    ip STRING COMMENT 'IP Address of the User'
+) COMMENT 'This is the page view table' PARTITIONED BY(dt STRING, country STRING) CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY '\002' MAP KEYS TERMINATED BY '\003' STORED AS SEQUENCEFILE LOCATION '/hsd_path';
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS page_view(
+    viewTime INT,
+    userid BIGINT,
+    page_url STRING,
+    referrer_url STRING,
+    ip STRING COMMENT 'IP Address of the User'
+) COMMENT 'This is the page view table' PARTITIONED BY(dt STRING, country STRING) CLUSTERED BY(userid) SORTED BY(viewTime) INTO 32 BUCKETS ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' COLLECTION ITEMS TERMINATED BY '\002' MAP KEYS TERMINATED BY '\003' STORED AS SEQUENCEFILE LOCATION '/hsd_path' AS
+SELECT
+    (key % 1024) new_key,
+    concat(key, value) key_value_pair
+FROM
+    key_value_store SORT BY new_key,
+    key_value_pair;
+
+
+CREATE TABLE list_bucket_single (key STRING, value STRING)
+  SKEWED BY (key) ON (1,5,6) STORED AS DIRECTORIES;
+
+CREATE TRANSACTIONAL TABLE transactional_table_test(key STRING, value STRING) PARTITIONED BY(ds STRING) STORED AS ORC;
 
 CREATE TABLE IF NOT EXISTS copy_table LIKE origin_table;
 
-CREATE TABLE IF NOT EXISTS derived_table AS SELECT * FROM origin_table;
+CREATE TEMPORARY TABLE IF NOT EXISTS copy_table LIKE origin_table;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS copy_table LIKE origin_table;
+
+CREATE TEMPORARY EXTERNAL TABLE IF NOT EXISTS copy_table LIKE origin_table LOCATION '/hdfs_path';
+
+CREATE TABLE IF NOT EXISTS derived_table AS
+SELECT
+    *
+FROM
+    origin_table;
 
 CREATE TABLE `mydb.t1`(
     `id` INT,
@@ -16,39 +86,22 @@ CREATE TABLE `mydb.t1`(
     `addr` STRING,
     `tel` STRING,
     `hobby` ARRAY < STRING >,
-    `add` MAP<STRING,STRING>
+    `add` MAP < STRING,
-)
+    STRING >
-PARTITIONED BY(`date` STRING) 
+) PARTITIONED BY(`date` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY '-' MAP KEYS TERMINATED BY ':';
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-COLLECTION ITEMS TERMINATED BY '-'
-MAP KEYS TERMINATED BY ':';
 
 CREATE EXTERNAL TABLE mydb.ext_table(
     id INT,
     name STRING,
     hobby ARRAY < STRING >,
-    add     MAP<STRING,STRING>
+    add
-)
+        MAP < STRING,
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+        STRING >
-COLLECTION ITEMS TERMINATED BY '-'
+) ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' COLLECTION ITEMS TERMINATED BY '-' MAP KEYS TERMINATED BY ':' LOCATION '/user/mydb/ext_table' TBLPROPERTIES('author' = 'hayden', 'desc' = '一个外部测试表');
-MAP KEYS TERMINATED BY ':'
-LOCATION '/user/mydb/ext_table'
-TBLPROPERTIES('author'='hayden','desc'='一个外部测试表')
-;
 
 CREATE MANAGED TABLE managed_table (
     id INT COMMENT 'ID',
     name STRING COMMENT '名称'
-)
+) COMMENT '测试分桶' CLUSTERED BY(id) SORTED BY (id) INTO 4 BUCKETS STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler';
-COMMENT '测试分桶'
-CLUSTERED BY(id) SORTED BY (id) INTO 4 BUCKETS
-STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler';
-
-CREATE TABLE list_bucket_multiple (
-	col1 STRING,
- 	col2 INT,
- 	col3 STRING
-)
-SKEWED BY (col1, col2) ON (('s1',1), ('s3',3), ('s13',13), ('s78',78)) 
-STORED AS DIRECTORIES;
 
+CREATE TABLE list_bucket_multiple (col1 STRING, col2 INT, col3 STRING) SKEWED BY (col1, col2) ON (('s1', 1), ('s3', 3), ('s13', 13), ('s78', 78)) STORED AS DIRECTORIES;

test/parser/hive/syntax/fixtures/createView.sql

@@ -1,4 +1,7 @@
 -- Common View
+CREATE VIEW mydb.bro_view
+AS SELECT * FROM mydb.sale_tbl;
+
 CREATE VIEW IF NOT EXISTS mydb.bro_view
 COMMENT '一个测试视图'
 AS SELECT * FROM mydb.sale_tbl;
@@ -18,6 +21,11 @@ AS SELECT DISTINCT id, `name`, runtime
 
 
 -- Materialized View
+CREATE MATERIALIZED VIEW IF NOT EXISTS mydb.bro_view
+DISABLE REWRITE
+COMMENT '一个测试视图'
+AS SELECT * FROM mydb.sale_tbl;
+
 CREATE MATERIALIZED VIEW druid_wiki_mv
 COMMENT '这是一个物化视图'
 STORED AS PARQUET