--********************************************************************-- -- Flink SQL 快速入门示例 创建表 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- -- 执行创建临时表 DDL，不需要指定catalog.database CREATE TABLE orders ( order_uid BIGINT, product_id BIGINT, price DECIMAL(32, 2), order_time TIMESTAMP(3) ) WITH ( 'connector' = 'datagen' ); --********************************************************************-- -- Flink SQL 快速入门示例 INSERT INTO --********************************************************************-- -- 定义数据源表 CREATE TABLE server_logs ( client_ip STRING, client_identity STRING, userid STRING, user_agent STRING, log_time TIMESTAMP(3), request_line STRING, status_code STRING, size INT ) WITH ( 'connector' = 'faker', 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '-', 'fields.user_agent.expression' = '#{Internet.userAgentAny}', 'fields.log_time.expression' = '#{date.past ''15'',''5'',''SECONDS''}', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}', 'fields.size.expression' = '#{number.numberBetween ''100'',''10000000''}' ); -- 定义结果表，实际应用中会选择 Kafka、JDBC 等作为结果表 CREATE TABLE client_errors ( log_time TIMESTAMP(3), request_line STRING, status_code STRING, size INT ) WITH ( 'connector' = 'stream-x' ); -- 写入数据到结果表 INSERT INTO client_errors SELECT log_time, request_line, status_code, size FROM server_logs WHERE status_code SIMILAR TO '4[0-9][0-9]'; --********************************************************************-- -- Flink SQL 快速入门示例 Statement Set --********************************************************************-- -- 定义数据源表 CREATE TABLE server_logs ( client_ip STRING, client_identity STRING, userid STRING, user_agent STRING, log_time TIMESTAMP(3), request_line STRING, status_code STRING, size INT, WATERMARK FOR log_time AS log_time - INTERVAL '30' SECONDS ) WITH ( 'connector' = 'faker', -- Faker 连接器仅在 VVR-4.0.12 及以上支持 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '-', 'fields.user_agent.expression' = '#{Internet.userAgentAny}', 'fields.log_time.expression' = '#{date.past ''15'',''5'',''SECONDS''}', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}', 'fields.size.expression' = '#{number.numberBetween ''100'',''10000000''}' ); -- 定义结果表1 CREATE TABLE aggregations1 ( `browser` STRING, `status_code` STRING, `end_time` TIMESTAMP(3), `requests` BIGINT NOT NULL ) WITH ( 'connector' = 'blackhole' ); -- 定义结果表2 CREATE TABLE aggregations2 ( `browser` STRING, `status_code` STRING, `requests` BIGINT NOT NULL ) WITH ( 'connector' = 'stream-x' ); -- This is a shared view that will be used by both insert into statements CREATE VIEW browsers AS SELECT REGEXP_EXTRACT(user_agent,'[^\/]+') AS browser, status_code, log_time FROM server_logs; -- 封装多个INSERT INTO语句到一个STATEMENT SET语句中 BEGIN STATEMENT SET; -- 5min窗口粒度聚合 INSERT INTO aggregations1 SELECT browser, status_code, TUMBLE_ROWTIME(log_time, INTERVAL '5' MINUTE) AS end_time, COUNT(*) requests FROM browsers GROUP BY browser, status_code, TUMBLE(log_time, INTERVAL '5' MINUTE); -- 1h窗口粒度聚合 INSERT INTO aggregations2 SELECT browser, status_code, COUNT(*) requests FROM browsers GROUP BY browser, status_code, TUMBLE(log_time, INTERVAL '1' HOUR); END; --********************************************************************-- -- Flink SQL 快速入门示例 Watermark -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.doctor_sightings ( doctor STRING, sighting_time TIMESTAMP(3), -- 通过watermark把sighting_time标识为事件时间，定义最大的乱序时间：期望所有的记录在目击发生后的15秒内都到达。 WATERMARK FOR sighting_time AS sighting_time - INTERVAL '15' SECONDS ) WITH ( 'connector' = 'faker', 'fields.doctor.expression' = '#{dr_who.the_doctors}', 'fields.sighting_time.expression' = '#{date.past ''15'',''SECONDS''}' ); SELECT doctor, -- 在滚动窗口中使用sighting_time字段 TUMBLE_ROWTIME(sighting_time, INTERVAL '1' MINUTE) AS sighting_time, COUNT(*) AS sightings FROM dt_catalog.dt_db.doctor_sightings GROUP BY TUMBLE(sighting_time, INTERVAL '1' MINUTE), doctor; --********************************************************************-- -- Flink SQL 快速入门示例 GROUP BY -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- -- 定义数据源表 CREATE TABLE dt_catalog.dt_db.server_logs ( client_ip STRING, client_identity STRING, userid STRING, user_agent STRING, log_time TIMESTAMP(3), request_line STRING, status_code STRING, size INT ) WITH ( 'connector' = 'faker', 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '-', 'fields.user_agent.expression' = '#{Internet.userAgentAny}', 'fields.log_time.expression' = '#{date.past ''15'',''5'',''SECONDS''}', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}', 'fields.size.expression' = '#{number.numberBetween ''100'',''10000000''}' ); -- 采样user_agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A -- 正则表达式: '[^\/]+' (匹配 '/' 之前的所有字符) SELECT REGEXP_EXTRACT(user_agent,'[^\/]+') AS browser, status_code, COUNT(*) AS cnt_status FROM dt_catalog.dt_db.server_logs -- 按浏览器和状态码两个维度统计日志数量 GROUP BY REGEXP_EXTRACT(user_agent,'[^\/]+'), status_code; --********************************************************************-- -- Flink SQL 快速入门示例 滚动窗口聚合 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.server_logs ( client_ip STRING, client_identity STRING, userid STRING, request_line STRING, status_code STRING, log_time AS PROCTIME() -- 使用当前系统处理时间作为表的时间字段 ) WITH ( 'connector' = 'faker', 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '-', 'fields.log_time.expression' = '#{date.past ''15'',''5'',''SECONDS''}', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}' ); -- 按 window_start, window_end 维度计算每分钟窗口上不同的 ip 数量 SELECT window_start, window_end, COUNT(DISTINCT client_ip) AS ip_addresses FROM TABLE( -- 定义1min滑动窗口 TUMBLE(TABLE dt_catalog.dt_db.server_logs, DESCRIPTOR(log_time), INTERVAL '1' MINUTE)) GROUP BY window_start, window_end; --********************************************************************-- -- Flink SQL 快速入门示例 滑动窗口聚合 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.bids ( bid_id STRING, currency_code STRING, bid_price DOUBLE, transaction_time TIMESTAMP(3), WATERMARK FOR transaction_time AS transaction_time - INTERVAL '5' SECONDS -- 定义事件时间，允许的最大窗口延迟为5s ) WITH ( 'connector' = 'faker', 'fields.bid_id.expression' = '#{Internet.UUID}', 'fields.currency_code.expression' = '#{regexify ''(EUR|USD|CNY)''}', 'fields.bid_price.expression' = '#{Number.randomDouble ''2'',''1'',''150''}', 'fields.transaction_time.expression' = '#{date.past ''30'',''SECONDS''}', 'rows-per-second' = '100' ); -- 定义1min 的滑动窗口，每隔 30s 滚动一次 SELECT window_start, window_end, currency_code, ROUND(AVG(bid_price),2) AS MovingAverageBidPrice FROM TABLE( HOP(TABLE dt_catalog.dt_db.bids, DESCRIPTOR(transaction_time), INTERVAL '30' SECONDS, INTERVAL '1' MINUTE)) GROUP BY window_start, window_end, currency_code; --********************************************************************-- -- Flink SQL 快速入门示例 累计窗口聚合 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- -- 商品销售订单表 CREATE TABLE dt_catalog.dt_db.orders ( order_id BIGINT, -- 订单ID goods_id BIGINT, -- 商品ID goods_sales DOUBLE, -- 商品销售额 order_time TIMESTAMP(3), -- 下单时间 WATERMARK FOR order_time AS order_time - INTERVAL '5' SECONDS -- 定义事件时间，允许的最大窗口延迟为5s ) WITH ( 'connector' = 'faker', 'fields.order_id.expression' = '#{number.numberBetween ''0'',''1000000000''}', 'fields.goods_id.expression' = '#{number.numberBetween ''0'',''1000000000''}', 'fields.goods_sales.expression' = '#{Number.randomDouble ''2'',''1'',''150''}', 'fields.order_time.expression' = '#{date.past ''30'',''SECONDS''}', 'rows-per-second' = '100' ); -- 每分钟更新一次从零点开始截止到当前时刻的累计销售额 SELECT window_start, window_end, SUM(goods_sales) AS cumulate_gmv -- 当天累计销售额 FROM TABLE( -- 定义窗口最大长度为一天的累计窗口，窗口滚动步长为1分钟 CUMULATE( TABLE dt_catalog.dt_db.orders, DESCRIPTOR(order_time), INTERVAL '1' MINUTES, INTERVAL '1' DAY)) GROUP BY window_start, window_end; --********************************************************************-- -- Flink SQL 快速入门示例 会话窗口聚合 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.server_logs ( client_ip STRING, client_identity STRING, userid STRING, log_time TIMESTAMP(3), request_line STRING, status_code STRING, WATERMARK FOR log_time AS log_time - INTERVAL '5' SECONDS -- 定义 watermark ) WITH ( 'connector' = 'faker', 'rows-per-second' = '5', 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '#{regexify ''(morsapaes|knauf|sjwiesman){1}''}', 'fields.log_time.expression' = '#{date.past ''5'',''SECONDS''}', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}' ); SELECT userid, SESSION_START(log_time, INTERVAL '10' SECOND) AS session_beg, SESSION_ROWTIME(log_time, INTERVAL '10' SECOND) AS session_end, COUNT(request_line) AS request_cnt FROM dt_catalog.dt_db.server_logs WHERE status_code = '403' GROUP BY userid, -- 会话窗口的最大空闲间隔为10s，当10s内该窗口没有接收到新的请求，会关闭当前窗口 SESSION(log_time, INTERVAL '10' SECOND); --********************************************************************-- -- Flink SQL 快速入门示例 OVER窗口聚合 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.temperature_measurements ( measurement_time TIMESTAMP(3), city STRING, temperature FLOAT, WATERMARK FOR measurement_time AS measurement_time - INTERVAL '15' SECONDS -- 定义时间属性字段，OVER窗口排序时使用 ) WITH ( 'connector' = 'faker', -- Faker 连接器仅在 VVR-4.0.12 及以上支持 'fields.measurement_time.expression' = '#{date.past ''15'',''SECONDS''}', 'fields.temperature.expression' = '#{number.numberBetween ''0'',''50''}', 'fields.city.expression' = '#{regexify ''(Chicago|Munich|Berlin|Portland|Hangzhou|Seatle|Beijing|New York){1}''}' ); SELECT measurement_time, city, temperature, AVG(CAST(temperature AS FLOAT)) OVER last_minute AS avg_temperature_minute, -- 计算平均值 MAX(temperature) OVER last_minute AS min_temperature_minute, -- 计算最大值 MIN(temperature) OVER last_minute AS max_temperature_minute, -- 计算最小值 STDDEV(CAST(temperature AS FLOAT)) OVER last_minute AS stdev_temperature_minute -- 计算标准差 FROM dt_catalog.dt_db.temperature_measurements WINDOW last_minute AS ( -- 定义1min时间间隔的OVER窗口，按城市粒度分区，温度测量值排序，每个元素都会触发一次计算 PARTITION BY city ORDER BY measurement_time RANGE BETWEEN INTERVAL '1' MINUTE PRECEDING AND CURRENT ROW ); --********************************************************************-- -- Flink SQL 快速入门示例 级联窗口聚合 --********************************************************************-- CREATE TEMPORARY TABLE server_logs ( log_time TIMESTAMP(3), client_ip STRING, client_identity STRING, userid STRING, request_line STRING, status_code STRING, size INT, WATERMARK FOR log_time AS log_time - INTERVAL '15' SECONDS -- 定义watermark ) WITH ( 'connector' = 'faker', 'fields.log_time.expression' = '#{date.past ''15'',''5'',''SECONDS''}', 'fields.client_ip.expression' = '#{Internet.publicIpV4Address}', 'fields.client_identity.expression' = '-', 'fields.userid.expression' = '-', 'fields.request_line.expression' = '#{regexify ''(GET|POST|PUT|PATCH){1}''} #{regexify ''(/search\.html|/login\.html|/prod\.html|cart\.html|/order\.html){1}''} #{regexify ''(HTTP/1\.1|HTTP/2|/HTTP/1\.0){1}''}', 'fields.status_code.expression' = '#{regexify ''(200|201|204|400|401|403|301){1}''}', 'fields.size.expression' = '#{number.numberBetween ''100'',''10000000''}' ); -- 1min聚合结果表 CREATE TEMPORARY TABLE avg_request_size_1m ( window_start TIMESTAMP(3), window_end TIMESTAMP(3), avg_size BIGINT ) WITH ( 'connector' = 'blackhole' ); -- 5min聚合结果表 CREATE TEMPORARY TABLE avg_request_size_5m ( window_start TIMESTAMP(3), window_end TIMESTAMP(3), avg_size BIGINT ) WITH ( 'connector' = 'blackhole' ); -- 1min窗口查询结果 CREATE VIEW server_logs_window_1m AS SELECT TUMBLE_START(log_time, INTERVAL '1' MINUTE) AS window_start, TUMBLE_ROWTIME(log_time, INTERVAL '1' MINUTE) AS window_end, SUM(size) AS total_size, COUNT(*) AS num_requests FROM server_logs GROUP BY TUMBLE(log_time, INTERVAL '1' MINUTE); -- 基于1min窗口查询结果，进行5min粒度窗口聚合 CREATE VIEW server_logs_window_5m AS SELECT TUMBLE_START(window_end, INTERVAL '5' MINUTE) AS window_start, TUMBLE_ROWTIME(window_end, INTERVAL '5' MINUTE) AS window_end, SUM(total_size) AS total_size, SUM(num_requests) AS num_requests FROM server_logs_window_1m GROUP BY TUMBLE(window_end, INTERVAL '5' MINUTE); BEGIN STATEMENT SET; -- 写入结果到1min窗口粒度结果表 INSERT INTO avg_request_size_1m SELECT window_start, window_end, total_size/num_requests AS avg_size FROM server_logs_window_1m; -- 写入结果到5min窗口粒度结果表 INSERT INTO avg_request_size_5m SELECT window_start, window_end, total_size/num_requests AS avg_size FROM server_logs_window_5m; END; --********************************************************************-- -- Flink SQL 快速入门示例 去重 -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.orders ( id INT, order_time AS CURRENT_TIMESTAMP, WATERMARK FOR order_time AS order_time - INTERVAL '5' SECONDS ) WITH ( 'connector' = 'datagen', 'rows-per-second'='10', 'fields.id.kind'='random', 'fields.id.min'='1', 'fields.id.max'='100' ); -- 对于每个order_id，按事件时间去重，只保留最新时间的记录即可实现去重 SELECT order_id, order_time FROM ( SELECT id AS order_id, order_time, -- 按事件时间升序排序 ROW_NUMBER() OVER (PARTITION BY id ORDER BY order_time) AS rownum FROM dt_catalog.dt_db.orders) WHERE rownum = 1; -- 只取排名第一的记录，去重是Top-N的一种特例 --********************************************************************-- -- Flink SQL 快速入门示例 Top-N -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.spells_cast ( wizard STRING, spell STRING ) WITH ( 'connector' = 'faker', 'fields.wizard.expression' = '#{harry_potter.characters}', 'fields.spell.expression' = '#{harry_potter.spells}' ); -- 找出每个巫师最喜欢的两个法术 SELECT wizard, spell, times_cast FROM ( SELECT *, ROW_NUMBER() OVER (PARTITION BY wizard ORDER BY times_cast DESC) AS row_num -- 按法术次数降序排序 FROM (SELECT wizard, spell, COUNT(*) AS times_cast FROM dt_catalog.dt_db.spells_cast GROUP BY wizard, spell) -- 计算每个巫师施展的各种法术的次数 ) WHERE row_num <= 2; --********************************************************************-- -- Flink SQL 快速入门示例 窗口Top-N -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.orders ( bidtime TIMESTAMP(3), price DOUBLE, item STRING, supplier STRING, WATERMARK FOR bidtime AS bidtime - INTERVAL '5' SECONDS -- 定义watermark ) WITH ( 'connector' = 'faker', 'fields.bidtime.expression' = '#{date.past ''30'',''SECONDS''}', 'fields.price.expression' = '#{Number.randomDouble ''2'',''1'',''150''}', 'fields.item.expression' = '#{Commerce.productName}', 'fields.supplier.expression' = '#{regexify ''(Alice|Bob|Carol|Alex|Joe|James|Jane|Jack)''}', 'rows-per-second' = '100' ); -- 取出销售排名前三的供应商 SELECT * FROM ( -- 按窗口时间分区，按价格降序排序 SELECT *, ROW_NUMBER() OVER (PARTITION BY window_start, window_end ORDER BY price DESC) AS rownum FROM ( -- 计算每个窗口内各个供应商的销售额 SELECT window_start, window_end, supplier, SUM(price) AS price, COUNT(*) AS cnt FROM TABLE( TUMBLE(TABLE dt_catalog.dt_db.orders, DESCRIPTOR(bidtime), INTERVAL '10' MINUTES)) GROUP BY window_start, window_end, supplier ) ) WHERE rownum <= 3; --********************************************************************-- -- Flink SQL 快速入门示例 模式检测CEP -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.subscriptions ( id STRING, user_id INT, type STRING, start_date TIMESTAMP(3), end_date TIMESTAMP(3), payment_expiration TIMESTAMP(3), proc_time AS PROCTIME() ) WITH ( 'connector' = 'faker', 'fields.id.expression' = '#{Internet.uuid}', 'fields.user_id.expression' = '#{number.numberBetween ''1'',''50''}', 'fields.type.expression'= '#{regexify ''(basic|premium|platinum){1}''}', 'fields.start_date.expression' = '#{date.past ''30'',''DAYS''}', 'fields.end_date.expression' = '#{date.future ''15'',''DAYS''}', 'fields.payment_expiration.expression' = '#{date.future ''365'',''DAYS''}' ); SELECT * FROM dt_catalog.dt_db.subscriptions MATCH_RECOGNIZE ( -- 按user_id分区，按处理时间proc_time升序排序 PARTITION BY user_id ORDER BY proc_time MEASURES LAST(PREMIUM.type) AS premium_type, AVG(TIMESTAMPDIFF(DAY,PREMIUM.start_date,PREMIUM.end_date)) AS premium_avg_duration, BASIC.start_date AS downgrade_date AFTER MATCH SKIP PAST LAST ROW --模式: 一个或多个‘premium‘或’platinum‘订阅事件 --对于相同的'user_id'，后面跟着一个'basic'订阅事件 PATTERN (PREMIUM+ BASIC) DEFINE PREMIUM AS PREMIUM.type IN ('premium','platinum'), BASIC AS BASIC.type = 'basic'); --********************************************************************-- -- Flink SQL 快速入门示例 Regular Join -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.NOC ( agent_id STRING, codename STRING ) WITH ( 'connector' = 'faker', 'fields.agent_id.expression' = '#{regexify ''(1|2|3|4|5){1}''}', 'fields.codename.expression' = '#{superhero.name}', 'number-of-rows' = '10' ); CREATE TABLE dt_catalog.dt_db.RealNames ( agent_id STRING, name STRING ) WITH ( 'connector' = 'faker', 'fields.agent_id.expression' = '#{regexify ''(1|2|3|4|5){1}''}', 'fields.name.expression' = '#{Name.full_name}', 'number-of-rows' = '10' ); -- 使用agent_id作为两张表关联的条件，左右两边任何一张表来了新数据都会触发join动作 SELECT name, codename FROM dt_catalog.dt_db.NOC t1 INNER JOIN dt_catalog.dt_db.RealNames t2 ON t1.agent_id = t2.agent_id; --********************************************************************-- -- Flink SQL 快速入门示例 Interval Join -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.orders ( id INT, order_time AS TIMESTAMPADD(DAY, CAST(FLOOR(RAND()*(1-5+1)+5)*(-1) AS INT), CURRENT_TIMESTAMP) ) WITH ( 'connector' = 'datagen', 'rows-per-second'='10', 'fields.id.kind'='sequence', 'fields.id.start'='1', 'fields.id.end'='1000' ); CREATE TABLE dt_catalog.dt_db.shipments ( id INT, order_id INT, shipment_time AS TIMESTAMPADD(DAY, CAST(FLOOR(RAND()*(1-5+1)) AS INT), CURRENT_TIMESTAMP) ) WITH ( 'connector' = 'datagen', 'rows-per-second'='5', 'fields.id.kind'='random', 'fields.id.min'='0', 'fields.order_id.kind'='sequence', 'fields.order_id.start'='1', 'fields.order_id.end'='1000' ); -- order表的每条数据会与shipments表过去三天至当前时刻时间范围内的数据进行join SELECT o.id AS order_id, o.order_time, s.shipment_time, TIMESTAMPDIFF(DAY,o.order_time,s.shipment_time) AS day_diff FROM dt_catalog.dt_db.orders o JOIN dt_catalog.dt_db.shipments s ON o.id = s.order_id WHERE -- 时间 join 条件：shipments.shipment_time - INTERVAL '3' DAY <= orders.order_time < shipments.shipment_time o.order_time BETWEEN s.shipment_time - INTERVAL '3' DAY AND s.shipment_time; --********************************************************************-- -- Flink SQL 快速入门示例 时态表Join -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- -- 使用主键约束和watermark来定义一张版本表，这张表可以是一个cdc表、upsert类型的kafka topic等 CREATE TABLE dt_catalog.dt_db.currency_rates ( `currency_code` STRING, `eur_rate` DECIMAL(6,4), `rate_time` TIMESTAMP(3), WATERMARK FOR `rate_time` AS rate_time - INTERVAL '15' SECONDS, -- 定义事件时间 PRIMARY KEY (currency_code) NOT ENFORCED -- 定义主键 ) WITH ( 'connector' = 'faker', 'fields.currency_code.expression' = '#{Currency.code}', 'fields.eur_rate.expression' = '#{Number.randomDouble ''4'',''0'',''10''}', 'fields.rate_time.expression' = '#{date.past ''15'',''SECONDS''}', 'rows-per-second' = '100' ); -- 这是一个append-only类型的动态表，需要定义watermk CREATE TABLE dt_catalog.dt_db.transactions ( `id` STRING, `currency_code` STRING, `total` DECIMAL(10,2), `transaction_time` TIMESTAMP(3), WATERMARK FOR `transaction_time` AS transaction_time - INTERVAL '30' SECONDS --定义watermark ) WITH ( 'connector' = 'faker', 'fields.id.expression' = '#{Internet.UUID}', 'fields.currency_code.expression' = '#{Currency.code}', 'fields.total.expression' = '#{Number.randomDouble ''2'',''10'',''1000''}', 'fields.transaction_time.expression' = '#{date.past ''30'',''SECONDS''}', 'rows-per-second' = '100' ); -- 当左右两张表的watermark对齐时，才会触发join动作，左右两张表都需要定义watermark SELECT t.id, t.total * c.eur_rate AS total_eur, t.total, c.currency_code, t.transaction_time FROM dt_catalog.dt_db.transactions t -- transactions表每条记录都与currency_rates表transaction_time时刻的汇率进行join JOIN dt_catalog.dt_db.currency_rates FOR SYSTEM_TIME AS OF t.transaction_time AS c -- 指定join key ON t.currency_code = c.currency_code; --********************************************************************-- -- Flink SQL 快速入门示例 维表Join -- 该模版仅支持使用"执行"功能。如需"提交"运行，需要您增加 INSERT 相关逻辑 --********************************************************************-- CREATE TABLE dt_catalog.dt_db.subscriptions ( id STRING, user_id INT, type STRING, start_date TIMESTAMP(3), end_date TIMESTAMP(3), payment_expiration TIMESTAMP(3), proc_time AS PROCTIME() -- 这里需要定义处理时间属性 ) WITH ( 'connector' = 'faker', 'fields.id.expression' = '#{Internet.uuid}', 'fields.user_id.expression' = '#{number.numberBetween ''1'',''50''}', 'fields.type.expression'= '#{regexify ''(basic|premium|platinum){1}''}', 'fields.start_date.expression' = '#{date.past ''30'',''DAYS''}', 'fields.end_date.expression' = '#{date.future ''365'',''DAYS''}', 'fields.payment_expiration.expression' = '#{date.future ''365'',''DAYS''}' ); -- 定义维表，为了示例能直接运行，这里使用faker 作为维表，实际应用中一般会使用JDBC、Redis、Hbase等作为维表 CREATE TABLE dt_catalog.dt_db.users ( user_id INT PRIMARY KEY, -- 定义主键 user_name VARCHAR(255) NOT NULL, age INT NOT NULL ) WITH ( 'connector' = 'faker', 'fields.user_id.expression' = '#{number.numberBetween ''1'',''10''}', 'fields.user_name.expression' = '#{regexify ''(ron|jark|danny){1}''}', 'fields.age.expression' = '#{number.numberBetween ''1'',''30''}' ); SELECT id AS subscription_id, type AS subscription_type, age AS user_age, CASE WHEN age < 18 THEN 1 ELSE 0 END AS is_minor FROM dt_catalog.dt_db.subscriptions usub -- subscriptions每条记录都使用当前系统时间与维表users中的最新数据进行join JOIN dt_catalog.dt_db.users FOR SYSTEM_TIME AS OF usub.proc_time AS u -- 指定join key ON usub.user_id = u.user_id; --********************************************************************-- -- Flink SQL ODS 层 -- 实际应用中，该任务应该是一个采集任务，源表为RDB --********************************************************************-- CREATE TABLE source_table ( account_number STRING, channel_id INT, account_open_datetime TIMESTAMP(3) ) WITH ( 'connector' = 'faker', -- 使用 Flink Faker Connector 'fields.account_number.expression' = '#{Finance.iban}', -- 随机生成银行账号 'fields.channel_id.expression' = '#{number.numberBetween ''1'',''4''}', -- 渠道ID随机生成1到3之间的数字 'fields.account_open_datetime.expression' = '#{date.past ''15'',''5'',''SECONDS''}' -- 过去15天的日期，每5秒一条数据 ); -- 定义结果表，实际应用中应选择 Kafka等作为结果表 CREATE TABLE sink_table ( account_number STRING, channel_id INT, account_open_datetime TIMESTAMP(3) ) WITH ( 'connector' = 'stream-x' ); -- 写入数据到结果表 INSERT INTO sink_table SELECT * FROM source_table --********************************************************************-- -- Flink SQL DWD 层 -- 实际应用中，源表为 ODS TOPIC --********************************************************************-- CREATE TABLE source_table ( account_number STRING, channel_id INT, account_open_datetime TIMESTAMP(3) ) WITH ( 'connector' = 'faker', -- 使用 Flink Faker Connector 'fields.account_number.expression' = '#{Finance.iban}', -- 随机生成银行账号 'fields.channel_id.expression' = '#{number.numberBetween ''1'',''4''}', -- 渠道ID随机生成1到3之间的数字 'fields.account_open_datetime.expression' = '#{date.past ''15'',''5'',''SECONDS''}' -- 过去15天的日期，每5秒一条数据 ); -- 定义维表，实际应用中应选择RDB作为维表 CREATE TABLE dim_table ( channel_id INT, channel_name STRING, PRIMARY KEY (channel_id,channel_name) NOT ENFORCED ) WITH ( 'connector' = 'faker', -- 使用 Flink Faker Connector 'fields.channel_id.expression' = '#{number.numberBetween ''1'',''4''}', -- 渠道ID随机生成1到3之间的数字 'fields.channel_name.expression' = '#{app.name}' -- 渠道名称 ); -- 定义结果表，实际应用中应选择 Kafka等作为结果表 CREATE TABLE sink_table ( account_number STRING, channel_id INT, channel_name STRING, account_open_datetime TIMESTAMP(3) ) WITH ( 'connector' = 'stream-x' ); -- 写入数据到结果表 INSERT INTO sink_table SELECT s1.account_number, s1. channel_id , d1.channel_name, s1.account_open_datetime FROM source_table s1 left JOIN dim_table d1 ON s1.channel_id=d1.channel_id --********************************************************************-- -- Flink SQL DWS 层 -- 实际应用中，源表为DWD TOPIC --********************************************************************-- CREATE TABLE source_table ( account_number STRING, channel_id INT, channel_name STRING, account_open_datetime TIMESTAMP(3) ) WITH ( 'connector' = 'faker', -- 使用 Flink Faker Connector 'fields.account_number.expression' = '#{Finance.iban}', -- 随机生成银行账号 'fields.channel_id.expression' = '#{number.numberBetween ''1'',''4''}', -- 渠道ID随机生成1到3之间的数字 'fields.channel_name.expression' = '#{app.name}' ,-- 渠道名称 'fields.account_open_datetime.expression' = '#{date.past ''15'',''5'',''SECONDS''}' -- 过去15天的日期，每5秒一条数据 ); DROP TABLE source_table SELECT * FROM source_table -- 定义结果表，实际应用中应选择Kafka作为结果表 CREATE TABLE sink_table ( channel_id STRING, open_date STRING, cnt INT ) WITH ( 'connector' = 'stream-x' ); DROP TABLE sink_table SELECT * FROM sink_table -- 写入数据到结果表 INSERT INTO sink_table SELECT channel_id, DATE_FORMAT(account_open_datetime,'yyyy-MM-dd'), count(account_number) FROM source_table GROUP BY channel_id,DATE_FORMAT(account_open_datetime,'yyyy-MM-dd') --********************************************************************-- -- Flink SQL ADS 层 -- 实际应用中，源表为DWS TOPIC --********************************************************************-- CREATE TABLE source_table ( channel_id STRING, open_time TIMESTAMP(3), cnt INT ) WITH ( 'connector' = 'faker', -- 使用 Flink Faker Connector 'fields.channel_id.expression' = '#{number.numberBetween ''1'',''4''}', -- 渠道ID随机生成1到3之间的数字 'fields.open_time.expression' = '#{Date.future ''15'',''5'',''SECONDS''}' ,-- 日期 'fields.cnt.expression' = '#{number.numberBetween ''1'',''100000000''}'-- 数量 ); -- 实际应用中，结果表为RDB CREATE TABLE sink_table ( open_date STRING, cnt INT ) WITH ( 'connector' = 'stream-x' ); -- 写入数据到结果表 INSERT INTO sink_table SELECT DATE_FORMAT(open_time,'yyyy-MM-dd'),count(1);