27 lines
1.4 KiB
Python
27 lines
1.4 KiB
Python
|
import json, urllib.request
|
||
|
|
Q="\x27"
|
||
|
|
def run(sql):
|
||
|
|
req=urllib.request.Request("http://10.0.21.50:8089/v1/statement", data=sql.encode(), headers={"X-Trino-User":"admin"})
|
||
|
|
d=json.load(urllib.request.urlopen(req)); nu=d.get("nextUri"); err=d.get("error")
|
||
|
|
while nu:
|
||
|
|
j=json.load(urllib.request.urlopen(nu))
|
||
|
|
if j.get("error"): err=j["error"]
|
||
|
|
nu=j.get("nextUri")
|
||
|
|
return err
|
||
|
|
def chunk(off,size):
|
||
|
|
sql=f"""INSERT INTO iceberg.s3lake.historical_sales
|
||
|
|
SELECT order_id, order_date, year(order_date) AS order_year, region, product_category, channel, quantity, unit_price, ROUND(quantity*unit_price,2) AS amount
|
||
|
|
FROM (
|
||
|
|
SELECT (s+{off}) AS order_id,
|
||
|
|
DATE {Q}2019-01-01{Q} + CAST(((s+{off})*7)%2190 AS bigint) * INTERVAL {Q}1{Q} DAY AS order_date,
|
||
|
|
element_at(ARRAY[{Q}EMEA{Q},{Q}AMER{Q},{Q}APAC{Q},{Q}LATAM{Q}], CAST((s+{off})%4 AS int)+1) AS region,
|
||
|
|
element_at(ARRAY[{Q}Servers{Q},{Q}Storage{Q},{Q}Networking{Q},{Q}Laptops{Q},{Q}Services{Q}], CAST((s+{off})%5 AS int)+1) AS product_category,
|
||
|
|
element_at(ARRAY[{Q}Direct{Q},{Q}Partner{Q},{Q}Online{Q}], CAST((s+{off})%3 AS int)+1) AS channel,
|
||
|
|
CAST(1+((s+{off})%50) AS int) AS quantity,
|
||
|
|
ROUND(100+((s+{off})%9000)*0.5,2) AS unit_price
|
||
|
|
FROM UNNEST(SEQUENCE(1,{size})) AS t(s)
|
||
|
|
)"""
|
||
|
|
return run(sql)
|
||
|
|
for off,size in [(0,10000),(10000,10000),(20000,5000)]:
|
||
|
|
e=chunk(off,size); print(f"chunk {off}:", e.get("message") if e else "ok")
|