mo
b54f5c7a06
Add Project Nessie on lake01 with Command Center Iceberg tab (snapshots, manifests, data files, time-travel SQL). Data Flow pulses only when endpoints are reachable; offline nodes/edges render red.
379 lines
15 KiB
Python
379 lines
15 KiB
Python
"""Nessie catalog + Iceberg table structure for Command Center.
|
|
|
|
Nessie (http://lake01:19120) holds Git-like catalog commits. Trino's existing
|
|
`iceberg` catalog exposes $snapshots / $manifests / $files for structure.
|
|
Trino 405 cannot use iceberg.catalog.type=nessie, so we combine both.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import time
|
|
from typing import Annotated, Any
|
|
from urllib.parse import quote
|
|
|
|
import httpx
|
|
from fastapi import APIRouter, Body, Query
|
|
from fastapi.responses import JSONResponse
|
|
|
|
router = APIRouter(prefix="/api/nessie", tags=["nessie"])
|
|
|
|
NESSIE_URL = os.getenv("NESSIE_URL", "http://10.0.21.50:19120").rstrip("/")
|
|
NESSIE_UI_URL = os.getenv("NESSIE_UI_URL", NESSIE_URL)
|
|
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
|
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
|
ICEBERG_CATALOG = os.getenv("ICEBERG_CATALOG", "iceberg")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
|
|
S3_BUCKET = os.getenv("S3_ARCHIVE_BUCKET", "data")
|
|
|
|
_cache: dict[str, Any] = {"ts": 0.0, "health": None}
|
|
_TTL = 8.0
|
|
|
|
|
|
async def _nessie(client: httpx.AsyncClient, path: str) -> Any:
|
|
r = await client.get(f"{NESSIE_URL}{path}", timeout=12.0)
|
|
r.raise_for_status()
|
|
return r.json()
|
|
|
|
|
|
async def _trino_rows(sql: str, deadline_s: float = 45.0) -> list[list[Any]]:
|
|
end = time.time() + deadline_s
|
|
rows: list[list[Any]] = []
|
|
async with httpx.AsyncClient(timeout=20.0) as client:
|
|
r = await client.post(
|
|
f"{TRINO_URL}/v1/statement",
|
|
content=sql.encode(),
|
|
headers={"X-Trino-User": TRINO_USER},
|
|
)
|
|
data = r.json()
|
|
rows.extend(data.get("data") or [])
|
|
nxt = data.get("nextUri")
|
|
while nxt:
|
|
if time.time() > end:
|
|
try:
|
|
await client.delete(nxt, timeout=3.0)
|
|
except Exception:
|
|
pass
|
|
break
|
|
rr = await client.get(nxt)
|
|
data = rr.json()
|
|
rows.extend(data.get("data") or [])
|
|
nxt = data.get("nextUri")
|
|
if data.get("stats", {}).get("state") in ("FAILED", "FINISHED"):
|
|
if data.get("error"):
|
|
raise RuntimeError(str(data["error"].get("message") or data["error"]))
|
|
if not nxt:
|
|
break
|
|
return rows
|
|
|
|
|
|
def _fqn(schema: str, table: str) -> str:
|
|
return f'{ICEBERG_CATALOG}.{schema}."{table}"'
|
|
|
|
|
|
def _meta_table(schema: str, table: str, kind: str) -> str:
|
|
# Trino Iceberg metadata tables: schema."table$snapshots"
|
|
return f'{ICEBERG_CATALOG}.{schema}."{table}${kind}"'
|
|
|
|
|
|
@router.get("/health")
|
|
async def health() -> JSONResponse:
|
|
now = time.time()
|
|
if _cache["health"] and now - _cache["ts"] < _TTL:
|
|
return JSONResponse(_cache["health"])
|
|
out: dict[str, Any] = {
|
|
"ok": False,
|
|
"nessie_url": NESSIE_URL,
|
|
"nessie_ui": NESSIE_UI_URL,
|
|
"trino_url": TRINO_URL,
|
|
"iceberg_catalog": ICEBERG_CATALOG,
|
|
"nessie_ok": False,
|
|
"trino_ok": False,
|
|
"default_branch": None,
|
|
"note": "Trino 405 has no native Nessie catalog; structure comes from iceberg.* metadata tables.",
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=8.0) as client:
|
|
cfg = await _nessie(client, "/api/v2/config")
|
|
out["nessie_ok"] = True
|
|
out["default_branch"] = cfg.get("defaultBranch", "main")
|
|
info = await client.get(f"{TRINO_URL}/v1/info")
|
|
out["trino_ok"] = info.status_code == 200
|
|
except Exception as exc:
|
|
out["error"] = str(exc)[:200]
|
|
out["ok"] = bool(out["nessie_ok"])
|
|
_cache["health"] = out
|
|
_cache["ts"] = now
|
|
return JSONResponse(out)
|
|
|
|
|
|
@router.get("/trees")
|
|
async def trees() -> JSONResponse:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=12.0) as client:
|
|
data = await _nessie(client, "/api/v2/trees")
|
|
refs = []
|
|
for item in data.get("references") or data.get("tokens") or []:
|
|
# v2 may return {references: [...]} or paginated
|
|
ref = item.get("reference") if isinstance(item, dict) and "reference" in item else item
|
|
if not isinstance(ref, dict):
|
|
continue
|
|
refs.append({
|
|
"type": ref.get("type"),
|
|
"name": ref.get("name"),
|
|
"hash": ref.get("hash"),
|
|
})
|
|
if not refs:
|
|
# fallback: main only
|
|
async with httpx.AsyncClient(timeout=12.0) as client:
|
|
main = await _nessie(client, "/api/v2/trees/main")
|
|
ref = main.get("reference") or main
|
|
refs = [{"type": ref.get("type"), "name": ref.get("name"), "hash": ref.get("hash")}]
|
|
return JSONResponse({"ok": True, "references": refs, "ui": NESSIE_UI_URL})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok": False, "error": str(exc)[:240], "references": []})
|
|
|
|
|
|
@router.get("/history")
|
|
async def history(ref: Annotated[str, Query()] = "main", max_records: Annotated[int, Query(ge=1, le=200)] = 50) -> JSONResponse:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/history?maxRecords={max_records}")
|
|
entries = data.get("logEntries") or data.get("commits") or []
|
|
out = []
|
|
for e in entries:
|
|
meta = e.get("commitMeta") or e.get("commit") or {}
|
|
out.append({
|
|
"hash": e.get("hash") or e.get("commitHash") or meta.get("hash"),
|
|
"message": meta.get("message"),
|
|
"author": (meta.get("authors") or [meta.get("author")])[0] if (meta.get("authors") or meta.get("author")) else None,
|
|
"commitTime": meta.get("commitTime") or meta.get("authorTime"),
|
|
})
|
|
return JSONResponse({"ok": True, "ref": ref, "commits": out})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok": False, "error": str(exc)[:240], "commits": []})
|
|
|
|
|
|
@router.get("/contents")
|
|
async def contents(ref: Annotated[str, Query()] = "main") -> JSONResponse:
|
|
try:
|
|
async with httpx.AsyncClient(timeout=15.0) as client:
|
|
data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/entries")
|
|
tables = []
|
|
namespaces = []
|
|
for e in data.get("entries") or []:
|
|
name = e.get("name") or {}
|
|
elements = name.get("elements") or []
|
|
typ = e.get("type")
|
|
if typ == "NAMESPACE":
|
|
namespaces.append(".".join(elements))
|
|
elif typ == "ICEBERG_TABLE" and len(elements) >= 2:
|
|
tables.append({
|
|
"schema": elements[0],
|
|
"table": elements[1],
|
|
"key": ".".join(elements),
|
|
"contentId": e.get("contentId"),
|
|
"trino_fqn": _fqn(elements[0], elements[1]),
|
|
})
|
|
return JSONResponse({
|
|
"ok": True,
|
|
"ref": ref,
|
|
"namespaces": namespaces,
|
|
"tables": tables,
|
|
"ui": NESSIE_UI_URL,
|
|
})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok": False, "error": str(exc)[:240], "tables": []})
|
|
|
|
|
|
@router.get("/tables/{schema}/{table}/structure")
|
|
async def table_structure(
|
|
schema: str,
|
|
table: str,
|
|
snapshot_id: Annotated[str | None, Query()] = None,
|
|
) -> JSONResponse:
|
|
"""Iceberg structure via Trino metadata tables (works with file metastore)."""
|
|
result: dict[str, Any] = {
|
|
"ok": False,
|
|
"schema": schema,
|
|
"table": table,
|
|
"fqn": _fqn(schema, table),
|
|
"snapshots": [],
|
|
"manifests": [],
|
|
"files": [],
|
|
"selected_snapshot": None,
|
|
"as_of_sql": None,
|
|
}
|
|
try:
|
|
snap_sql = (
|
|
f'SELECT snapshot_id, committed_at, operation, summary '
|
|
f'FROM {_meta_table(schema, table, "snapshots")} '
|
|
f'ORDER BY committed_at DESC LIMIT 40'
|
|
)
|
|
snaps = await _trino_rows(snap_sql)
|
|
snapshots = []
|
|
for row in snaps:
|
|
snapshots.append({
|
|
"snapshot_id": str(row[0]),
|
|
"committed_at": str(row[1]) if row[1] is not None else None,
|
|
"operation": row[2],
|
|
"summary": (str(row[3]) if row[3] is not None else None) if len(row) > 3 else None,
|
|
})
|
|
result["snapshots"] = snapshots
|
|
current = snapshot_id or (snapshots[0]["snapshot_id"] if snapshots else None)
|
|
result["selected_snapshot"] = current
|
|
if current:
|
|
result["as_of_sql"] = (
|
|
f'SELECT * FROM {_fqn(schema, table)} FOR VERSION AS OF {current} LIMIT 100'
|
|
)
|
|
|
|
# manifests
|
|
try:
|
|
man_sql = (
|
|
f'SELECT path, length, partition_spec_id, added_snapshot_id, '
|
|
f'added_data_files_count, existing_data_files_count, deleted_data_files_count '
|
|
f'FROM {_meta_table(schema, table, "manifests")} LIMIT 200'
|
|
)
|
|
mans = await _trino_rows(man_sql)
|
|
result["manifests"] = [
|
|
{
|
|
"path": r[0],
|
|
"length": r[1],
|
|
"partition_spec_id": r[2],
|
|
"added_snapshot_id": str(r[3]) if r[3] is not None else None,
|
|
"added_data_files_count": r[4],
|
|
"existing_data_files_count": r[5],
|
|
"deleted_data_files_count": r[6],
|
|
}
|
|
for r in mans
|
|
]
|
|
except Exception as exc:
|
|
result["manifests_error"] = str(exc)[:200]
|
|
|
|
# data files (current snapshot view)
|
|
try:
|
|
files_sql = (
|
|
f'SELECT file_path, file_format, record_count, file_size_in_bytes '
|
|
f'FROM {_meta_table(schema, table, "files")} LIMIT 500'
|
|
)
|
|
files = await _trino_rows(files_sql)
|
|
result["files"] = [
|
|
{
|
|
"path": r[0],
|
|
"format": r[1],
|
|
"record_count": r[2],
|
|
"size_bytes": r[3],
|
|
"is_s3": str(r[0]).startswith("s3") if r[0] else False,
|
|
}
|
|
for r in files
|
|
]
|
|
except Exception as exc:
|
|
result["files_error"] = str(exc)[:200]
|
|
|
|
result["ok"] = True
|
|
result["restore_warning"] = (
|
|
"Restore re-points table metadata to an older snapshot. "
|
|
"It cannot undelete files already removed from S3/disk."
|
|
)
|
|
return JSONResponse(result)
|
|
except Exception as exc:
|
|
result["error"] = str(exc)[:300]
|
|
return JSONResponse(result)
|
|
|
|
|
|
@router.post("/tables/{schema}/{table}/restore")
|
|
async def restore_snapshot(
|
|
schema: str,
|
|
table: str,
|
|
body: dict[str, Any] = Body(default={}),
|
|
) -> JSONResponse:
|
|
"""Rollback Iceberg table to a previous snapshot via Trino CALL (if supported) or documented SQL.
|
|
|
|
Trino 405 may not expose system.rollback; we attempt CALL and fall back to guidance.
|
|
"""
|
|
snapshot_id = str((body or {}).get("snapshot_id") or "").strip()
|
|
if not snapshot_id:
|
|
return JSONResponse({"ok": False, "error": "snapshot_id required"}, status_code=400)
|
|
fqn = _fqn(schema, table)
|
|
# Prefer Iceberg procedure when available
|
|
attempts = [
|
|
f"CALL {ICEBERG_CATALOG}.system.rollback_to_snapshot('{schema}', '{table}', {snapshot_id})",
|
|
f"ALTER TABLE {fqn} EXECUTE rollback_to_snapshot({snapshot_id})",
|
|
]
|
|
errors: list[str] = []
|
|
for sql in attempts:
|
|
try:
|
|
await _trino_rows(sql, deadline_s=60.0)
|
|
return JSONResponse({
|
|
"ok": True,
|
|
"schema": schema,
|
|
"table": table,
|
|
"snapshot_id": snapshot_id,
|
|
"method": sql,
|
|
"warning": "Old data files must still exist on storage for this to succeed.",
|
|
})
|
|
except Exception as exc:
|
|
errors.append(f"{sql}: {exc}")
|
|
return JSONResponse({
|
|
"ok": False,
|
|
"error": "Rollback procedure not available on this Trino version",
|
|
"hint": f'Query historically with: SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id}',
|
|
"as_of_sql": f'SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id} LIMIT 100',
|
|
"details": errors[:3],
|
|
})
|
|
|
|
|
|
@router.get("/s3/parquet")
|
|
async def list_s3_parquet(prefix: Annotated[str, Query()] = "lake/,iceberg/", limit: Annotated[int, Query(ge=1, le=1000)] = 200) -> JSONResponse:
|
|
"""List parquet (and orc) objects under lake/ and iceberg/ prefixes."""
|
|
if not S3_ACCESS_KEY or not S3_SECRET_KEY:
|
|
return JSONResponse({"ok": False, "error": "S3 credentials not configured", "objects": []})
|
|
try:
|
|
import boto3
|
|
from botocore.client import Config
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=S3_ENDPOINT,
|
|
aws_access_key_id=S3_ACCESS_KEY,
|
|
aws_secret_access_key=S3_SECRET_KEY,
|
|
region_name=os.getenv("S3_REGION", "us-east-1"),
|
|
config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
|
|
)
|
|
prefixes = [p.strip() for p in prefix.split(",") if p.strip()]
|
|
objects: list[dict[str, Any]] = []
|
|
for pref in prefixes:
|
|
token = None
|
|
while len(objects) < limit:
|
|
kw: dict[str, Any] = {"Bucket": S3_BUCKET, "Prefix": pref, "MaxKeys": min(500, limit - len(objects))}
|
|
if token:
|
|
kw["ContinuationToken"] = token
|
|
resp = s3.list_objects_v2(**kw)
|
|
for o in resp.get("Contents") or []:
|
|
key = o["Key"]
|
|
low = key.lower()
|
|
if not (low.endswith(".parquet") or low.endswith(".orc") or "/data/" in low):
|
|
continue
|
|
objects.append({
|
|
"bucket": S3_BUCKET,
|
|
"key": key,
|
|
"size": o.get("Size"),
|
|
"last_modified": o["LastModified"].isoformat() if o.get("LastModified") else None,
|
|
"format": "parquet" if low.endswith(".parquet") else ("orc" if low.endswith(".orc") else "data"),
|
|
})
|
|
if len(objects) >= limit:
|
|
break
|
|
if not resp.get("IsTruncated"):
|
|
break
|
|
token = resp.get("NextContinuationToken")
|
|
return JSONResponse({
|
|
"ok": True,
|
|
"bucket": S3_BUCKET,
|
|
"prefixes": prefixes,
|
|
"objects": objects,
|
|
"note": "Raw S3 objects — not all are Iceberg-managed; lake/ ETL parquet has no snapshot history until registered as Iceberg.",
|
|
})
|
|
except Exception as exc:
|
|
return JSONResponse({"ok": False, "error": str(exc)[:240], "objects": []})
|