feat(openmetadata): direct-API enrichment for real sample data, profiles & end-to-end lineage

OM connectors/profiler stored data but its denormalized read path left
Sample Data/Lineage tabs effectively empty. This script populates OM directly:
- real 50-row sample data for source + iceberg curated tables
- table/column profiles (column profiles read back correctly in UI)
- full traceable lineage: generator -> source -> Debezium/Kafka CDC topic
  -> S3 archive + Iceberg curated -> Trino query layer
This commit is contained in:
mo
2026-06-27 15:02:30 +00:00
parent 21af36a591
commit 28821e8b04
2 changed files with 314 additions and 0 deletions
+43
View File
@@ -0,0 +1,43 @@
import json, os, urllib.request, urllib.parse, urllib.error
OM = "http://openmetadata-server:8585/api"
H = {"Authorization": "Bearer " + os.environ["OM_TOKEN"]}
def get(path):
r = urllib.request.Request(OM + path, headers=H)
with urllib.request.urlopen(r, timeout=30) as resp:
return json.loads(resp.read().decode())
def trace(fqn):
enc = urllib.parse.quote(fqn, safe="")
g = get("/v1/lineage/table/name/" + enc + "?upstreamDepth=3&downstreamDepth=3")
nodes = {}
for n in g.get("nodes", []) + ([g["entity"]] if "entity" in g else []):
nodes[n["id"]] = (n.get("type"), n.get("fullyQualifiedName") or n.get("name"))
base = g.get("entity", {})
nodes[base["id"]] = (base.get("type"), base.get("fullyQualifiedName"))
def nm(i):
t, f = nodes.get(i, ("?", i))
short = (f or "").split(".")[-1].strip('"')
return "%s(%s)" % (short, t)
print("\n### lineage around:", fqn)
print("UP (sources feeding it):")
for e in g.get("upstreamEdges", []):
print(" %s --> %s" % (nm(e["fromEntity"]), nm(e["toEntity"])))
print("DOWN (where it flows to):")
for e in g.get("downstreamEdges", []):
print(" %s --> %s" % (nm(e["fromEntity"]), nm(e["toEntity"])))
for f in [
"atc_postgres.postgres.public.sales_orders",
"atc_mysql.default.hr.employee_events",
]:
try:
trace(f)
except Exception as e:
print(f, "ERR", str(e)[:120])