infra: historical sales -> iceberg.historical (Trino) + Superset dashboard scripts

infra/historical/histload.py

@@ -0,0 +1,26 @@
+import json, urllib.request
+Q="\x27"
+def run(sql):
+    req=urllib.request.Request("http://10.0.21.50:8089/v1/statement", data=sql.encode(), headers={"X-Trino-User":"admin"})
+    d=json.load(urllib.request.urlopen(req)); nu=d.get("nextUri"); err=d.get("error")
+    while nu:
+        j=json.load(urllib.request.urlopen(nu))
+        if j.get("error"): err=j["error"]
+        nu=j.get("nextUri")
+    return err
+def chunk(off, size):
+    sql=f"""INSERT INTO iceberg.historical.sales_orders
+SELECT order_id, order_date, year(order_date) AS order_year, region, product_category, channel, quantity, unit_price, ROUND(quantity*unit_price,2) AS amount
+FROM (
+  SELECT (s+{off}) AS order_id,
+    DATE {Q}2019-01-01{Q} + CAST(((s+{off})*7)%2190 AS bigint) * INTERVAL {Q}1{Q} DAY AS order_date,
+    element_at(ARRAY[{Q}EMEA{Q},{Q}AMER{Q},{Q}APAC{Q},{Q}LATAM{Q}], CAST((s+{off})%4 AS int)+1) AS region,
+    element_at(ARRAY[{Q}Servers{Q},{Q}Storage{Q},{Q}Networking{Q},{Q}Laptops{Q},{Q}Services{Q}], CAST((s+{off})%5 AS int)+1) AS product_category,
+    element_at(ARRAY[{Q}Direct{Q},{Q}Partner{Q},{Q}Online{Q}], CAST((s+{off})%3 AS int)+1) AS channel,
+    CAST(1+((s+{off})%50) AS int) AS quantity,
+    ROUND(100+((s+{off})%9000)*0.5,2) AS unit_price
+  FROM UNNEST(SEQUENCE(1,{size})) AS t(s)
+)"""
+    return run(sql)
+for off,size in [(0,10000),(10000,10000),(20000,5000)]:
+    e=chunk(off,size); print(f"chunk off={off} size={size}:", e.get("message") if e else "ok")