mo
e8ff760cd1
OM 1.13 cannot deserialize Airflow 3.0.6 serialized DAGs, so pipelines had no tasks. af_tasks.py parses serialized_dag and PATCHes task lists; af_lineage.py adds generator->table and mask_to_curated/hadoop_to_trino lineage edges. Note: OM MySQL datadir lives on the 448MB /opt partition which had filled up (103MB binlog). Disabled binary logging + shrank redo log capacity in the OM docker-compose to restore headroom.
93 lines
2.9 KiB
Python
93 lines
2.9 KiB
Python
"""Backfill task-level detail into OpenMetadata Airflow pipelines.
|
|
|
|
OM 1.13's Airflow connector cannot deserialize Airflow 3.0.6 serialized DAGs, so
|
|
pipelines land with zero tasks. We parse the serialized_dag table ourselves and
|
|
PATCH each pipeline with its tasks + intra-DAG dependencies (downstreamTasks).
|
|
"""
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import urllib.error
|
|
import urllib.request
|
|
import zlib
|
|
|
|
DB = "/" + "/".join(["tmp", "airflow.db"])
|
|
OM = "http://openmetadata-server:8585/api"
|
|
TOKEN = os.environ["OM_TOKEN"]
|
|
SERVICE = "atc_airflow"
|
|
|
|
HDRS = {"Authorization": "Bearer " + TOKEN}
|
|
|
|
|
|
def req(method, path, body=None, ctype="application/json"):
|
|
data = json.dumps(body).encode() if body is not None else None
|
|
h = dict(HDRS)
|
|
if data is not None:
|
|
h["Content-Type"] = ctype
|
|
r = urllib.request.Request(OM + path, data=data, headers=h, method=method)
|
|
try:
|
|
with urllib.request.urlopen(r, timeout=30) as resp:
|
|
return resp.status, json.loads(resp.read().decode() or "{}")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read().decode()
|
|
|
|
|
|
def latest_serialized():
|
|
c = sqlite3.connect(DB)
|
|
rows = c.execute(
|
|
"SELECT s.dag_id, s.data, s.data_compressed FROM serialized_dag s "
|
|
"JOIN (SELECT dag_id, MAX(id) mid FROM serialized_dag GROUP BY dag_id) m "
|
|
"ON s.id = m.mid"
|
|
).fetchall()
|
|
out = {}
|
|
for dag_id, data, comp in rows:
|
|
if data is None and comp is not None:
|
|
data = zlib.decompress(comp).decode()
|
|
d = json.loads(data) if isinstance(data, str) else data
|
|
out[dag_id] = d.get("dag", d)
|
|
return out
|
|
|
|
|
|
def build_tasks(dag):
|
|
tasks = []
|
|
for t in dag.get("tasks", []):
|
|
tt = t.get("__var", t) if isinstance(t, dict) else t
|
|
tid = tt.get("task_id")
|
|
if not tid:
|
|
continue
|
|
tasks.append(
|
|
{
|
|
"name": tid,
|
|
"displayName": tid,
|
|
"taskType": tt.get("task_type") or tt.get("_task_type") or "Task",
|
|
"downstreamTasks": list(tt.get("downstream_task_ids") or []),
|
|
}
|
|
)
|
|
return tasks
|
|
|
|
|
|
def main():
|
|
dags = latest_serialized()
|
|
for dag_id, dag in sorted(dags.items()):
|
|
tasks = build_tasks(dag)
|
|
fqn = SERVICE + "." + dag_id
|
|
st, pipe = req("GET", "/v1/pipelines/name/" + fqn)
|
|
if st != 200:
|
|
print("SKIP", dag_id, "get-failed", st, pipe)
|
|
continue
|
|
pid = pipe["id"]
|
|
op = "replace" if "tasks" in pipe else "add"
|
|
patch = [{"op": op, "path": "/tasks", "value": tasks}]
|
|
st2, res = req(
|
|
"PATCH", "/v1/pipelines/" + pid, patch, "application/json-patch+json"
|
|
)
|
|
edges = sum(len(t["downstreamTasks"]) for t in tasks)
|
|
print(
|
|
"%-30s tasks=%-2d edges=%-2d -> %s"
|
|
% (dag_id, len(tasks), edges, "OK" if st2 == 200 else ("ERR %s %s" % (st2, res)))
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|