feat: Nessie catalog UI + Iceberg structure explorer + live Data Flow online/offline
Add Project Nessie on lake01 with Command Center Iceberg tab (snapshots, manifests, data files, time-travel SQL). Data Flow pulses only when endpoints are reachable; offline nodes/edges render red.
This commit is contained in:
mo
+1
-1
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py auth.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py agent_activity.py lake_meta.py lineage.py dq_monitor.py observability.py catalog_governance.py gpu_config.py hive_bench_seed.json .
|
||||
COPY main.py auth.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py etl_offload.py agent_activity.py lake_meta.py lineage.py dq_monitor.py observability.py catalog_governance.py gpu_config.py nessie_iceberg.py hive_bench_seed.json .
|
||||
RUN mkdir -p /data
|
||||
ENV DATABASE_URL=sqlite:////data/atc-agents.db
|
||||
EXPOSE 3201
|
||||
|
||||
+100
-2
@@ -26,6 +26,76 @@ router = APIRouter(prefix="/api/dataflow", tags=["dataflow"])
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
|
||||
NESSIE_URL = os.getenv("NESSIE_URL", "http://10.0.21.50:19120").rstrip("/")
|
||||
SRC_DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51")
|
||||
SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8081").rstrip("/")
|
||||
KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000").rstrip("/")
|
||||
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020").rstrip("/")
|
||||
|
||||
# TCP ports for source DB reachability on the shared db host
|
||||
_SOURCE_PORTS = {
|
||||
"postgres": 5432,
|
||||
"mysql": 3306,
|
||||
"mongodb": 27017,
|
||||
"cassandra": 9042,
|
||||
"neo4j": 7687,
|
||||
}
|
||||
|
||||
|
||||
async def _tcp_open(host: str, port: int, timeout: float = 1.2) -> bool:
|
||||
import asyncio
|
||||
try:
|
||||
conn = asyncio.open_connection(host, port)
|
||||
reader, writer = await asyncio.wait_for(conn, timeout=timeout)
|
||||
writer.close()
|
||||
try:
|
||||
await writer.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _http_ok(url: str, timeout: float = 2.0) -> bool:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
r = await client.get(url)
|
||||
return r.status_code < 500
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def _probe_online() -> dict[str, bool]:
|
||||
"""Best-effort live reachability for dataflow nodes."""
|
||||
online: dict[str, bool] = {}
|
||||
for sid, port in _SOURCE_PORTS.items():
|
||||
online[sid] = await _tcp_open(SRC_DB_HOST, port)
|
||||
online["trino"] = await _http_ok(f"{TRINO_URL}/v1/info")
|
||||
online["nessie"] = await _http_ok(f"{NESSIE_URL}/api/v2/config")
|
||||
online["spark"] = await _http_ok(f"{SPARK_UI_URL}/json/")
|
||||
online["kafka"] = await _http_ok(KAFKA_UI_URL)
|
||||
online["hdfs"] = await _http_ok(HDFS_NN_URL)
|
||||
# S3: TCP to endpoint host:port
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
u = urlparse(S3_ENDPOINT)
|
||||
online["s3_cdc"] = await _tcp_open(u.hostname or "10.0.20.111", u.port or 9020)
|
||||
except Exception:
|
||||
online["s3_cdc"] = False
|
||||
# lakehouse iceberg nodes follow trino catalog availability
|
||||
online["iceberg_hadoop"] = online.get("trino", False)
|
||||
online["iceberg_curated"] = online.get("trino", False)
|
||||
online["generator"] = True # logical
|
||||
online["openmetadata"] = await _http_ok("http://10.0.21.47:8585")
|
||||
online["chromadb"] = True
|
||||
online["rag"] = True
|
||||
online["vllm"] = await _http_ok(os.getenv("LLM_URL", "http://10.0.10.106:8001/v1").rstrip("/") + "/models")
|
||||
online["chat"] = True
|
||||
return online
|
||||
|
||||
|
||||
# Static landscape (x,y in 0..100). kind drives UI styling.
|
||||
# Laid out as clean left→right pipeline stages so lineage reads in order:
|
||||
# producers (col 0) → sources (col 1) → CDC (col 2) → storage/lakehouse (col 3)
|
||||
@@ -48,6 +118,8 @@ NODES: list[dict[str, Any]] = [
|
||||
{"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13},
|
||||
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45},
|
||||
{"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 67, "y": 80},
|
||||
{"id": "nessie", "label": "Nessie", "sub": "Iceberg catalog · Git-like", "kind": "catalog", "x": 78, "y": 62,
|
||||
"url": "http://10.0.21.50:19120"},
|
||||
# col 4 — query engine
|
||||
{"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 88, "y": 45},
|
||||
# governance — bottom centre
|
||||
@@ -83,6 +155,10 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
|
||||
{"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
|
||||
{"from": "mongodb", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
|
||||
{"from": "iceberg_hadoop", "to": "nessie", "kind": "catalog"},
|
||||
{"from": "iceberg_curated", "to": "nessie", "kind": "catalog"},
|
||||
{"from": "nessie", "to": "s3_cdc", "kind": "archive"},
|
||||
{"from": "nessie", "to": "trino", "kind": "query"},
|
||||
{"from": "iceberg_hadoop", "to": "trino", "kind": "query"},
|
||||
{"from": "iceberg_curated", "to": "trino", "kind": "query"},
|
||||
{"from": "kafka", "to": "trino", "kind": "query"},
|
||||
@@ -198,10 +274,14 @@ async def _build() -> dict[str, Any]:
|
||||
edge_live = streaming.get("edges") or {}
|
||||
pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])}
|
||||
|
||||
online = await _probe_online()
|
||||
|
||||
nodes = []
|
||||
for n in NODES:
|
||||
node = dict(n)
|
||||
node["level"] = "ok"
|
||||
is_up = online.get(n["id"], True)
|
||||
node["online"] = is_up
|
||||
node["level"] = "ok" if is_up else "err"
|
||||
metric = None
|
||||
if n["id"] == "openmetadata":
|
||||
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
|
||||
@@ -217,10 +297,15 @@ async def _build() -> dict[str, Any]:
|
||||
metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
|
||||
node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
|
||||
elif n["id"] in ("postgres", "mysql", "mongodb", "cassandra", "neo4j"):
|
||||
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
|
||||
if not is_up:
|
||||
metric = "offline"
|
||||
else:
|
||||
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
|
||||
elif n["id"] == "iceberg_hadoop":
|
||||
c = _iceberg_hadoop_count()
|
||||
metric = f"{c:,} rows" if c is not None else "iceberg table"
|
||||
elif n["id"] == "nessie":
|
||||
metric = "catalog commits" if online.get("nessie") else "offline"
|
||||
elif n["id"] == "generator":
|
||||
metric = "Airflow gen DAGs"
|
||||
elif n["id"] in ("vllm", "rag", "chromadb"):
|
||||
@@ -283,6 +368,10 @@ async def _build() -> dict[str, Any]:
|
||||
elif e["kind"] == "archive" and e.get("from") == "kafka" and e.get("to") == "s3_cdc":
|
||||
# Kafka → S3 CDC archive pulses while the pipeline lands objects in S3
|
||||
edge["active"] = arch_active
|
||||
elif e["kind"] == "catalog" and e.get("to") == "nessie":
|
||||
edge["active"] = bool(online.get("nessie")) and bool(online.get(e.get("from"), True))
|
||||
elif e["kind"] == "catalog":
|
||||
edge["active"] = bool(online.get(e.get("from"), True)) and bool(online.get(e.get("to"), True))
|
||||
elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
|
||||
# AI serving lane pulses while governed data is being served to the LLM
|
||||
edge["active"] = bool(_rag_info())
|
||||
@@ -294,6 +383,15 @@ async def _build() -> dict[str, Any]:
|
||||
pass
|
||||
if _flow != "running":
|
||||
edge["active"] = False
|
||||
|
||||
# Realtime online/offline: never pulse if an endpoint is down
|
||||
frm_up = online.get(e.get("from"), True)
|
||||
to_up = online.get(e.get("to"), True)
|
||||
if not (frm_up and to_up):
|
||||
edge["active"] = False
|
||||
edge["offline"] = True
|
||||
else:
|
||||
edge["offline"] = False
|
||||
edges.append(edge)
|
||||
|
||||
return {
|
||||
|
||||
@@ -62,6 +62,7 @@ from lineage import router as lineage_router
|
||||
from dq_monitor import router as dq_router
|
||||
from observability import router as observability_router
|
||||
from catalog_governance import router as governance_router
|
||||
from nessie_iceberg import router as nessie_router
|
||||
from ssh_terminal import ssh_session
|
||||
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
|
||||
from node_ops import build_node_detail, probe_node, run_node_probe_task
|
||||
@@ -1044,6 +1045,7 @@ app.include_router(lineage_router)
|
||||
app.include_router(dq_router)
|
||||
app.include_router(observability_router)
|
||||
app.include_router(governance_router)
|
||||
app.include_router(nessie_router)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
"""Nessie catalog + Iceberg table structure for Command Center.
|
||||
|
||||
Nessie (http://lake01:19120) holds Git-like catalog commits. Trino's existing
|
||||
`iceberg` catalog exposes $snapshots / $manifests / $files for structure.
|
||||
Trino 405 cannot use iceberg.catalog.type=nessie, so we combine both.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Annotated, Any
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body, Query
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
router = APIRouter(prefix="/api/nessie", tags=["nessie"])
|
||||
|
||||
NESSIE_URL = os.getenv("NESSIE_URL", "http://10.0.21.50:19120").rstrip("/")
|
||||
NESSIE_UI_URL = os.getenv("NESSIE_UI_URL", NESSIE_URL)
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
ICEBERG_CATALOG = os.getenv("ICEBERG_CATALOG", "iceberg")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
|
||||
S3_BUCKET = os.getenv("S3_ARCHIVE_BUCKET", "data")
|
||||
|
||||
_cache: dict[str, Any] = {"ts": 0.0, "health": None}
|
||||
_TTL = 8.0
|
||||
|
||||
|
||||
async def _nessie(client: httpx.AsyncClient, path: str) -> Any:
|
||||
r = await client.get(f"{NESSIE_URL}{path}", timeout=12.0)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
|
||||
async def _trino_rows(sql: str, deadline_s: float = 45.0) -> list[list[Any]]:
|
||||
end = time.time() + deadline_s
|
||||
rows: list[list[Any]] = []
|
||||
async with httpx.AsyncClient(timeout=20.0) as client:
|
||||
r = await client.post(
|
||||
f"{TRINO_URL}/v1/statement",
|
||||
content=sql.encode(),
|
||||
headers={"X-Trino-User": TRINO_USER},
|
||||
)
|
||||
data = r.json()
|
||||
rows.extend(data.get("data") or [])
|
||||
nxt = data.get("nextUri")
|
||||
while nxt:
|
||||
if time.time() > end:
|
||||
try:
|
||||
await client.delete(nxt, timeout=3.0)
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
rr = await client.get(nxt)
|
||||
data = rr.json()
|
||||
rows.extend(data.get("data") or [])
|
||||
nxt = data.get("nextUri")
|
||||
if data.get("stats", {}).get("state") in ("FAILED", "FINISHED"):
|
||||
if data.get("error"):
|
||||
raise RuntimeError(str(data["error"].get("message") or data["error"]))
|
||||
if not nxt:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def _fqn(schema: str, table: str) -> str:
|
||||
return f'{ICEBERG_CATALOG}.{schema}."{table}"'
|
||||
|
||||
|
||||
def _meta_table(schema: str, table: str, kind: str) -> str:
|
||||
# Trino Iceberg metadata tables: schema."table$snapshots"
|
||||
return f'{ICEBERG_CATALOG}.{schema}."{table}${kind}"'
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def health() -> JSONResponse:
|
||||
now = time.time()
|
||||
if _cache["health"] and now - _cache["ts"] < _TTL:
|
||||
return JSONResponse(_cache["health"])
|
||||
out: dict[str, Any] = {
|
||||
"ok": False,
|
||||
"nessie_url": NESSIE_URL,
|
||||
"nessie_ui": NESSIE_UI_URL,
|
||||
"trino_url": TRINO_URL,
|
||||
"iceberg_catalog": ICEBERG_CATALOG,
|
||||
"nessie_ok": False,
|
||||
"trino_ok": False,
|
||||
"default_branch": None,
|
||||
"note": "Trino 405 has no native Nessie catalog; structure comes from iceberg.* metadata tables.",
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
cfg = await _nessie(client, "/api/v2/config")
|
||||
out["nessie_ok"] = True
|
||||
out["default_branch"] = cfg.get("defaultBranch", "main")
|
||||
info = await client.get(f"{TRINO_URL}/v1/info")
|
||||
out["trino_ok"] = info.status_code == 200
|
||||
except Exception as exc:
|
||||
out["error"] = str(exc)[:200]
|
||||
out["ok"] = bool(out["nessie_ok"])
|
||||
_cache["health"] = out
|
||||
_cache["ts"] = now
|
||||
return JSONResponse(out)
|
||||
|
||||
|
||||
@router.get("/trees")
|
||||
async def trees() -> JSONResponse:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=12.0) as client:
|
||||
data = await _nessie(client, "/api/v2/trees")
|
||||
refs = []
|
||||
for item in data.get("references") or data.get("tokens") or []:
|
||||
# v2 may return {references: [...]} or paginated
|
||||
ref = item.get("reference") if isinstance(item, dict) and "reference" in item else item
|
||||
if not isinstance(ref, dict):
|
||||
continue
|
||||
refs.append({
|
||||
"type": ref.get("type"),
|
||||
"name": ref.get("name"),
|
||||
"hash": ref.get("hash"),
|
||||
})
|
||||
if not refs:
|
||||
# fallback: main only
|
||||
async with httpx.AsyncClient(timeout=12.0) as client:
|
||||
main = await _nessie(client, "/api/v2/trees/main")
|
||||
ref = main.get("reference") or main
|
||||
refs = [{"type": ref.get("type"), "name": ref.get("name"), "hash": ref.get("hash")}]
|
||||
return JSONResponse({"ok": True, "references": refs, "ui": NESSIE_UI_URL})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:240], "references": []})
|
||||
|
||||
|
||||
@router.get("/history")
|
||||
async def history(ref: Annotated[str, Query()] = "main", max_records: Annotated[int, Query(ge=1, le=200)] = 50) -> JSONResponse:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/history?maxRecords={max_records}")
|
||||
entries = data.get("logEntries") or data.get("commits") or []
|
||||
out = []
|
||||
for e in entries:
|
||||
meta = e.get("commitMeta") or e.get("commit") or {}
|
||||
out.append({
|
||||
"hash": e.get("hash") or e.get("commitHash") or meta.get("hash"),
|
||||
"message": meta.get("message"),
|
||||
"author": (meta.get("authors") or [meta.get("author")])[0] if (meta.get("authors") or meta.get("author")) else None,
|
||||
"commitTime": meta.get("commitTime") or meta.get("authorTime"),
|
||||
})
|
||||
return JSONResponse({"ok": True, "ref": ref, "commits": out})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:240], "commits": []})
|
||||
|
||||
|
||||
@router.get("/contents")
|
||||
async def contents(ref: Annotated[str, Query()] = "main") -> JSONResponse:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
data = await _nessie(client, f"/api/v2/trees/{quote(str(ref), safe='')}/entries")
|
||||
tables = []
|
||||
namespaces = []
|
||||
for e in data.get("entries") or []:
|
||||
name = e.get("name") or {}
|
||||
elements = name.get("elements") or []
|
||||
typ = e.get("type")
|
||||
if typ == "NAMESPACE":
|
||||
namespaces.append(".".join(elements))
|
||||
elif typ == "ICEBERG_TABLE" and len(elements) >= 2:
|
||||
tables.append({
|
||||
"schema": elements[0],
|
||||
"table": elements[1],
|
||||
"key": ".".join(elements),
|
||||
"contentId": e.get("contentId"),
|
||||
"trino_fqn": _fqn(elements[0], elements[1]),
|
||||
})
|
||||
return JSONResponse({
|
||||
"ok": True,
|
||||
"ref": ref,
|
||||
"namespaces": namespaces,
|
||||
"tables": tables,
|
||||
"ui": NESSIE_UI_URL,
|
||||
})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:240], "tables": []})
|
||||
|
||||
|
||||
@router.get("/tables/{schema}/{table}/structure")
|
||||
async def table_structure(
|
||||
schema: str,
|
||||
table: str,
|
||||
snapshot_id: Annotated[str | None, Query()] = None,
|
||||
) -> JSONResponse:
|
||||
"""Iceberg structure via Trino metadata tables (works with file metastore)."""
|
||||
result: dict[str, Any] = {
|
||||
"ok": False,
|
||||
"schema": schema,
|
||||
"table": table,
|
||||
"fqn": _fqn(schema, table),
|
||||
"snapshots": [],
|
||||
"manifests": [],
|
||||
"files": [],
|
||||
"selected_snapshot": None,
|
||||
"as_of_sql": None,
|
||||
}
|
||||
try:
|
||||
snap_sql = (
|
||||
f'SELECT snapshot_id, committed_at, operation, summary '
|
||||
f'FROM {_meta_table(schema, table, "snapshots")} '
|
||||
f'ORDER BY committed_at DESC LIMIT 40'
|
||||
)
|
||||
snaps = await _trino_rows(snap_sql)
|
||||
snapshots = []
|
||||
for row in snaps:
|
||||
snapshots.append({
|
||||
"snapshot_id": str(row[0]),
|
||||
"committed_at": str(row[1]) if row[1] is not None else None,
|
||||
"operation": row[2],
|
||||
"summary": (str(row[3]) if row[3] is not None else None) if len(row) > 3 else None,
|
||||
})
|
||||
result["snapshots"] = snapshots
|
||||
current = snapshot_id or (snapshots[0]["snapshot_id"] if snapshots else None)
|
||||
result["selected_snapshot"] = current
|
||||
if current:
|
||||
result["as_of_sql"] = (
|
||||
f'SELECT * FROM {_fqn(schema, table)} FOR VERSION AS OF {current} LIMIT 100'
|
||||
)
|
||||
|
||||
# manifests
|
||||
try:
|
||||
man_sql = (
|
||||
f'SELECT path, length, partition_spec_id, added_snapshot_id, '
|
||||
f'added_data_files_count, existing_data_files_count, deleted_data_files_count '
|
||||
f'FROM {_meta_table(schema, table, "manifests")} LIMIT 200'
|
||||
)
|
||||
mans = await _trino_rows(man_sql)
|
||||
result["manifests"] = [
|
||||
{
|
||||
"path": r[0],
|
||||
"length": r[1],
|
||||
"partition_spec_id": r[2],
|
||||
"added_snapshot_id": str(r[3]) if r[3] is not None else None,
|
||||
"added_data_files_count": r[4],
|
||||
"existing_data_files_count": r[5],
|
||||
"deleted_data_files_count": r[6],
|
||||
}
|
||||
for r in mans
|
||||
]
|
||||
except Exception as exc:
|
||||
result["manifests_error"] = str(exc)[:200]
|
||||
|
||||
# data files (current snapshot view)
|
||||
try:
|
||||
files_sql = (
|
||||
f'SELECT file_path, file_format, record_count, file_size_in_bytes '
|
||||
f'FROM {_meta_table(schema, table, "files")} LIMIT 500'
|
||||
)
|
||||
files = await _trino_rows(files_sql)
|
||||
result["files"] = [
|
||||
{
|
||||
"path": r[0],
|
||||
"format": r[1],
|
||||
"record_count": r[2],
|
||||
"size_bytes": r[3],
|
||||
"is_s3": str(r[0]).startswith("s3") if r[0] else False,
|
||||
}
|
||||
for r in files
|
||||
]
|
||||
except Exception as exc:
|
||||
result["files_error"] = str(exc)[:200]
|
||||
|
||||
result["ok"] = True
|
||||
result["restore_warning"] = (
|
||||
"Restore re-points table metadata to an older snapshot. "
|
||||
"It cannot undelete files already removed from S3/disk."
|
||||
)
|
||||
return JSONResponse(result)
|
||||
except Exception as exc:
|
||||
result["error"] = str(exc)[:300]
|
||||
return JSONResponse(result)
|
||||
|
||||
|
||||
@router.post("/tables/{schema}/{table}/restore")
|
||||
async def restore_snapshot(
|
||||
schema: str,
|
||||
table: str,
|
||||
body: dict[str, Any] = Body(default={}),
|
||||
) -> JSONResponse:
|
||||
"""Rollback Iceberg table to a previous snapshot via Trino CALL (if supported) or documented SQL.
|
||||
|
||||
Trino 405 may not expose system.rollback; we attempt CALL and fall back to guidance.
|
||||
"""
|
||||
snapshot_id = str((body or {}).get("snapshot_id") or "").strip()
|
||||
if not snapshot_id:
|
||||
return JSONResponse({"ok": False, "error": "snapshot_id required"}, status_code=400)
|
||||
fqn = _fqn(schema, table)
|
||||
# Prefer Iceberg procedure when available
|
||||
attempts = [
|
||||
f"CALL {ICEBERG_CATALOG}.system.rollback_to_snapshot('{schema}', '{table}', {snapshot_id})",
|
||||
f"ALTER TABLE {fqn} EXECUTE rollback_to_snapshot({snapshot_id})",
|
||||
]
|
||||
errors: list[str] = []
|
||||
for sql in attempts:
|
||||
try:
|
||||
await _trino_rows(sql, deadline_s=60.0)
|
||||
return JSONResponse({
|
||||
"ok": True,
|
||||
"schema": schema,
|
||||
"table": table,
|
||||
"snapshot_id": snapshot_id,
|
||||
"method": sql,
|
||||
"warning": "Old data files must still exist on storage for this to succeed.",
|
||||
})
|
||||
except Exception as exc:
|
||||
errors.append(f"{sql}: {exc}")
|
||||
return JSONResponse({
|
||||
"ok": False,
|
||||
"error": "Rollback procedure not available on this Trino version",
|
||||
"hint": f'Query historically with: SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id}',
|
||||
"as_of_sql": f'SELECT * FROM {fqn} FOR VERSION AS OF {snapshot_id} LIMIT 100',
|
||||
"details": errors[:3],
|
||||
})
|
||||
|
||||
|
||||
@router.get("/s3/parquet")
|
||||
async def list_s3_parquet(prefix: Annotated[str, Query()] = "lake/,iceberg/", limit: Annotated[int, Query(ge=1, le=1000)] = 200) -> JSONResponse:
|
||||
"""List parquet (and orc) objects under lake/ and iceberg/ prefixes."""
|
||||
if not S3_ACCESS_KEY or not S3_SECRET_KEY:
|
||||
return JSONResponse({"ok": False, "error": "S3 credentials not configured", "objects": []})
|
||||
try:
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
s3 = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
config=Config(s3={"addressing_style": "path"}, signature_version="s3v4"),
|
||||
)
|
||||
prefixes = [p.strip() for p in prefix.split(",") if p.strip()]
|
||||
objects: list[dict[str, Any]] = []
|
||||
for pref in prefixes:
|
||||
token = None
|
||||
while len(objects) < limit:
|
||||
kw: dict[str, Any] = {"Bucket": S3_BUCKET, "Prefix": pref, "MaxKeys": min(500, limit - len(objects))}
|
||||
if token:
|
||||
kw["ContinuationToken"] = token
|
||||
resp = s3.list_objects_v2(**kw)
|
||||
for o in resp.get("Contents") or []:
|
||||
key = o["Key"]
|
||||
low = key.lower()
|
||||
if not (low.endswith(".parquet") or low.endswith(".orc") or "/data/" in low):
|
||||
continue
|
||||
objects.append({
|
||||
"bucket": S3_BUCKET,
|
||||
"key": key,
|
||||
"size": o.get("Size"),
|
||||
"last_modified": o["LastModified"].isoformat() if o.get("LastModified") else None,
|
||||
"format": "parquet" if low.endswith(".parquet") else ("orc" if low.endswith(".orc") else "data"),
|
||||
})
|
||||
if len(objects) >= limit:
|
||||
break
|
||||
if not resp.get("IsTruncated"):
|
||||
break
|
||||
token = resp.get("NextContinuationToken")
|
||||
return JSONResponse({
|
||||
"ok": True,
|
||||
"bucket": S3_BUCKET,
|
||||
"prefixes": prefixes,
|
||||
"objects": objects,
|
||||
"note": "Raw S3 objects — not all are Iceberg-managed; lake/ ETL parquet has no snapshot history until registered as Iceberg.",
|
||||
})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:240], "objects": []})
|
||||
Reference in New Issue
Block a user