"""Nessie catalog + Iceberg table structure for Command Center. Nessie (http://lake01:19120) holds Git-like catalog commits. Trino's existing `iceberg` catalog exposes $snapshots / $manifests / $files for structure. Trino 405 cannot use iceberg.catalog.type=nessie, so we combine both. """ from __future__ import annotations import os import time from typing import Annotated, Any from urllib.parse import quote import httpx from fastapi import APIRouter, Body, Query from fastapi.responses import JSONResponse router = APIRouter(prefix="/api/nessie", tags=["nessie"]) NESSIE_URL = os.getenv("NESSIE_URL", "http://10.0.21.50:19120").rstrip("/") NESSIE_UI_URL = os.getenv("NESSIE_UI_URL", NESSIE_URL) TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/") TRINO_USER = os.getenv("TRINO_USER", "mo") ICEBERG_CATALOG = os.getenv("ICEBERG_CATALOG", "iceberg") S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020") S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "") S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "") S3_BUCKET = os.getenv("S3_ARCHIVE_BUCKET", "data") _cache: dict[str, Any] = {"ts": 0.0, "health": None} _TTL = 8.0 async def _nessie(client: httpx.AsyncClient, path: str) -> Any: r = await client.get(f"{NESSIE_URL}{path}", timeout=12.0) r.raise_for_status() return r.json() async def _trino_rows(sql: str, deadline_s: float = 45.0) -> list[list[Any]]: end = time.time() + deadline_s rows: list[list[Any]] = [] async with httpx.AsyncClient(timeout=20.0) as client: r = await client.post( f"{TRINO_URL}/v1/statement", content=sql.encode(), headers={"X-Trino-User": TRINO_USER}, ) data = r.json() rows.extend(data.get("data") or []) nxt = data.get("nextUri") while nxt: if time.time() > end: try: await client.delete(nxt, timeout=3.0) except Exception: pass break rr = await client.get(nxt) data = rr.json() rows.extend(data.get("data") or []) nxt = data.get("nextUri") if data.get("stats", {}).get("state") in ("FAILED", "FINISHED"): if data.get("error"): raise RuntimeError(str(data["error"].get("message") or data["error"])) if not nxt: break return rows def _fqn(schema: str, table: str) -> str: return f'{ICEBERG_CATALOG}.{schema}."{table}"' def _meta_table(schema: str, table: str, kind: str) -> str: # Trino Iceberg metadata tables: schema."table$snapshots" return f'{ICEBERG_CATALOG}.{schema}."{table}${kind}"' @router.get("/health") async def health() -> JSONResponse: now = time.time() if _cache["health"] and now - _cache["ts"] < _TTL: return JSONResponse(_cache["health"]) out: dict[str, Any] = { "ok": False, "nessie_url": NESSIE_URL, "nessie_ui": NESSIE_UI_URL, "trino_url": TRINO_URL, "iceberg_catalog": ICEBERG_CATALOG, "nessie_ok": False, "trino_ok": False, "default_branch": None, "note": "Trino 405 has no native Nessie catalog; structure comes from iceberg.* metadata tables.", } try: async with httpx.AsyncClient(timeout=8.0) as client: cfg = await _nessie(client, "/api/v2/config") out["nessie_ok"] = True out["default_branch"] = cfg.get("defaultBranch", "main") info = await client.get(f"{TRINO_URL}/v1/info") out["trino_ok"] = info.status_code == 200 except Exception as exc: out["error"] = str(exc)[:200] out["ok"] = bool(out["nessie_ok"]) _cache["health"] = out _cache["ts"] = now return JSONResponse(out) @router.get("/trees") async def trees() -> JSONResponse: try: async with httpx.AsyncClient(timeout=12.0) as client: data = await _nessie(client, "/api/v2/trees") refs = [] for item in data.get("references") or data.get("tokens") or []: # v2 may return {references: [...]} or paginated ref = item.get("reference") if isinstance(item, dict) and "reference" in item else item if not isinstance(ref, dict): continue refs.append({ "type": ref.get("type"), "name": ref.get("name"), "hash": ref.get("hash"), }) if not refs: # fallback: main only async with httpx.AsyncClient(timeout=12.0) as client: main = await _nessie(client, "/api/v2/trees/main") ref = main.get("reference") or main refs = [{"type": ref.get("type"), "name": ref.get("name"), "hash": ref.get("hash")}] return JSONResponse({"ok": True, "references": refs, "ui": NESSIE_UI_URL}) except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)[:240], "references": []}) @router.get("/history") async def history(ref: Annotated[str, Query()] = "main", max_records: Annotated[int, Query(ge=1, le=200)] = 50) -> JSONResponse: try: async with httpx.AsyncClient(timeout=15.0) as client: data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/history?maxRecords={max_records}") entries = data.get("logEntries") or data.get("commits") or [] out = [] for e in entries: meta = e.get("commitMeta") or e.get("commit") or {} out.append({ "hash": e.get("hash") or e.get("commitHash") or meta.get("hash"), "message": meta.get("message"), "author": (meta.get("authors") or [meta.get("author")])[0] if (meta.get("authors") or meta.get("author")) else None, "commitTime": meta.get("commitTime") or meta.get("authorTime"), }) return JSONResponse({"ok": True, "ref": ref, "commits": out}) except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)[:240], "commits": []}) @router.get("/contents") async def contents(ref: Annotated[str, Query()] = "main") -> JSONResponse: try: async with httpx.AsyncClient(timeout=15.0) as client: data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/entries") tables = [] namespaces = [] for e in data.get("entries") or []: name = e.get("name") or {} elements = name.get("elements") or [] typ = e.get("type") if typ == "NAMESPACE": namespaces.append(".".join(elements)) elif typ == "ICEBERG_TABLE" and len(elements) >= 2: tables.append({ "schema": elements[0], "table": elements[1], "key": ".".join(elements), "contentId": e.get("contentId"), "trino_fqn": _fqn(elements[0], elements[1]), }) return JSONResponse({ "ok": True, "ref": ref, "namespaces": namespaces, "tables": tables, "ui": NESSIE_UI_URL, }) except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)[:240], "tables": []}) @router.get("/tables/{schema}/{table}/structure") async def table_structure( schema: str, table: str, snapshot_id: Annotated[str | None, Query()] = None, ) -> JSONResponse: """Iceberg structure via Trino metadata tables (works with file metastore).""" result: dict[str, Any] = { "ok": False, "schema": schema, "table": table, "fqn": _fqn(schema, table), "snapshots": [], "manifests": [], "files": [], "selected_snapshot": None, "as_of_sql": None, } try: snap_sql = ( f'SELECT snapshot_id, committed_at, operation, summary ' f'FROM {_meta_table(schema, table, "snapshots")} ' f'ORDER BY committed_at DESC LIMIT 40' ) snaps = await _trino_rows(snap_sql) snapshots = [] for row in snaps: snapshots.append({ "snapshot_id": str(row[0]), "committed_at": str(row[1]) if row[1] is not None else None, "operation": row[2], "summary": (str(row[3]) if row[3] is not None else None) if len(row) > 3 else None, }) result["snapshots"] = snapshots current = snapshot_id or (snapshots[0]["snapshot_id"] if snapshots else None) result["selected_snapshot"] = current if current: result["as_of_sql"] = ( f'SELECT * FROM {_fqn(schema, table)} FOR VERSION AS OF {current} LIMIT 100' ) # manifests try: man_sql = ( f'SELECT path, length, partition_spec_id, added_snapshot_id, ' f'added_data_files_count, existing_data_files_count, deleted_data_files_count ' f'FROM {_meta_table(schema, table, "manifests")} LIMIT 200' ) mans = await _trino_rows(man_sql) result["manifests"] = [ { "path": r[0], "length": r[1], "partition_spec_id": r[2], "added_snapshot_id": str(r[3]) if r[3] is not None else None, "added_data_files_count": r[4], "existing_data_files_count": r[5], "deleted_data_files_count": r[6], } for r in mans ] except Exception as exc: result["manifests_error"] = str(exc)[:200] # data files (current snapshot view) try: files_sql = ( f'SELECT file_path, file_format, record_count, file_size_in_bytes ' f'FROM {_meta_table(schema, table, "files")} LIMIT 500' ) files = await _trino_rows(files_sql) result["files"] = [ { "path": r[0], "format": r[1], "record_count": r[2], "size_bytes": r[3], "is_s3": str(r[0]).startswith("s3") if r[0] else False, } for r in files ] except Exception as exc: result["files_error"] = str(exc)[:200] result["ok"] = True result["restore_warning"] = ( "Restore re-points table metadata to an older snapshot. " "It cannot undelete files already removed from S3/disk." ) return JSONResponse(result) except Exception as exc: result["error"] = str(exc)[:300] return JSONResponse(result) @router.post("/tables/{schema}/{table}/restore") async def restore_snapshot( schema: str, table: str, body: dict[str, Any] = Body(default={}), ) -> JSONResponse: """Rollback Iceberg table to a previous snapshot via Trino CALL (if supported) or documented SQL. Trino 405 may not expose system.rollback; we attempt CALL and fall back to guidance. """ snapshot_id = str((body or {}).get("snapshot_id") or "").strip() if not snapshot_id: return JSONResponse({"ok": False, "error": "snapshot_id required"}, status_code=400) fqn = _fqn(schema, table) # Prefer Iceberg procedure when available attempts = [ f"CALL {ICEBERG_CATALOG}.system.rollback_to_snapshot('{schema}', '{table}', {snapshot_id})", f"ALTER TABLE {fqn} EXECUTE rollback_to_snapshot({snapshot_id})", ] errors: list[str] = [] for sql in attempts: try: await _trino_rows(sql, deadline_s=60.0) return JSONResponse({ "ok": True, "schema": schema, "table": table, "snapshot_id": snapshot_id, "method": sql, "warning": "Old data files must still exist on storage for this to succeed.", }) except Exception as exc: errors.append(f"{sql}: {exc}") return JSONResponse({ "ok": False, "error": "Rollback procedure not available on this Trino version", "hint": f'Query historically with: SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id}', "as_of_sql": f'SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id} LIMIT 100', "details": errors[:3], }) @router.get("/s3/parquet") async def list_s3_parquet(prefix: Annotated[str, Query()] = "lake/,iceberg/", limit: Annotated[int, Query(ge=1, le=1000)] = 200) -> JSONResponse: """List parquet (and orc) objects under lake/ and iceberg/ prefixes.""" if not S3_ACCESS_KEY or not S3_SECRET_KEY: return JSONResponse({"ok": False, "error": "S3 credentials not configured", "objects": []}) try: import boto3 from botocore.client import Config s3 = boto3.client( "s3", endpoint_url=S3_ENDPOINT, aws_access_key_id=S3_ACCESS_KEY, aws_secret_access_key=S3_SECRET_KEY, region_name=os.getenv("S3_REGION", "us-east-1"), config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"), ) prefixes = [p.strip() for p in prefix.split(",") if p.strip()] objects: list[dict[str, Any]] = [] for pref in prefixes: token = None while len(objects) < limit: kw: dict[str, Any] = {"Bucket": S3_BUCKET, "Prefix": pref, "MaxKeys": min(500, limit - len(objects))} if token: kw["ContinuationToken"] = token resp = s3.list_objects_v2(**kw) for o in resp.get("Contents") or []: key = o["Key"] low = key.lower() if not (low.endswith(".parquet") or low.endswith(".orc") or "/data/" in low): continue objects.append({ "bucket": S3_BUCKET, "key": key, "size": o.get("Size"), "last_modified": o["LastModified"].isoformat() if o.get("LastModified") else None, "format": "parquet" if low.endswith(".parquet") else ("orc" if low.endswith(".orc") else "data"), }) if len(objects) >= limit: break if not resp.get("IsTruncated"): break token = resp.get("NextContinuationToken") return JSONResponse({ "ok": True, "bucket": S3_BUCKET, "prefixes": prefixes, "objects": objects, "note": "Raw S3 objects — not all are Iceberg-managed; lake/ ETL parquet has no snapshot history until registered as Iceberg.", }) except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)[:240], "objects": []})