feat: continuous live generator + vLLM/RAG lane in Data Flow

Live dashboard now feels truly real-time:
- Background generator streams randomly-sized bursts of real rows into
  PostgreSQL, MySQL, MongoDB & Cassandra every ~4s (CDC picks them up).
  Throughput rises and falls; counters move in lock-step (base snapshot +
  generated). Runs only while the Live tab is polling (heartbeat-gated) so
  source tables do not grow unbounded; on/off toggle exposed in the UI.
- New /api/federated/live/generator toggle; /live returns per-tick activity
  (last burst sizes, orders by region/status, event feed).
- LiveDashboard: live-activity panel, orders-per-tick sparkline, event
  stream feed, burst-by-region/status charts, generator status + control.

Data Flow graph now explains how data reaches the assistant:
- Added ChromaDB -> RAG (LangChain) -> vLLM Gateway -> Knowledge Chat lane,
  with Trino / OpenMetadata / curated-masked feeding LLM context. Live model
  & embed metrics pulled from the RAG /config. New node/edge kinds + legend.
This commit is contained in:
mo
2026-06-28 21:37:41 +00:00
parent 8c72d1dc63
commit 9059006cc2
4 changed files with 429 additions and 26 deletions
+48
View File
@@ -52,6 +52,11 @@ NODES: list[dict[str, Any]] = [
# governance — bottom centre
{"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance",
"x": 47, "y": 93, "url": "http://10.0.21.47:8585"},
# AI serving lane — how governed data reaches the LLM & the Command Center chat
{"id": "chromadb", "label": "ChromaDB", "sub": "vectors · embeddings", "kind": "vector", "x": 60, "y": 65},
{"id": "rag", "label": "RAG · LangChain", "sub": "retrieve · augment · agent", "kind": "rag", "x": 74, "y": 65},
{"id": "vllm", "label": "vLLM Gateway", "sub": "Llama3-70B · GPT-4o", "kind": "llm", "x": 88, "y": 68},
{"id": "chat", "label": "Knowledge Chat", "sub": "Command Center", "kind": "chat", "x": 92, "y": 90},
]
# Edges. movement_id (optional) links to movements.py so the edge is triggerable.
@@ -86,6 +91,14 @@ EDGES: list[dict[str, Any]] = [
{"from": "cassandra", "to": "openmetadata", "kind": "catalog"},
{"from": "neo4j", "to": "openmetadata", "kind": "catalog"},
{"from": "trino", "to": "openmetadata", "kind": "catalog"},
# AI serving lane: governed business data + catalog + vectors → RAG → vLLM → chat
{"from": "trino", "to": "rag", "kind": "context"},
{"from": "openmetadata", "to": "rag", "kind": "context"},
{"from": "iceberg_curated", "to": "rag", "kind": "context"},
{"from": "chromadb", "to": "rag", "kind": "retrieve"},
{"from": "rag", "to": "vllm", "kind": "prompt"},
{"from": "vllm", "to": "chat", "kind": "answer"},
{"from": "rag", "to": "chat", "kind": "answer"},
]
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
@@ -117,6 +130,25 @@ def _trino_scalar(sql: str, deadline_s: float = 8.0) -> int | None:
return None
RAG_URL = os.getenv("RAG_URL", "http://rag-api:5020").rstrip("/")
_rag_cache: dict[str, Any] = {"ts": 0.0, "val": None}
def _rag_info() -> dict[str, Any]:
now = time.time()
if _rag_cache["val"] is not None and now - _rag_cache["ts"] < 60:
return _rag_cache["val"]
info: dict[str, Any] = {}
try:
with httpx.Client(timeout=2.0) as client:
info = client.get(f"{RAG_URL}/config").json() or {}
except Exception:
info = {}
_rag_cache["val"] = info
_rag_cache["ts"] = now
return info
def _iceberg_hadoop_count() -> int | None:
now = time.time()
if _count_cache["val"] is not None and now - _count_cache["ts"] < 60:
@@ -180,6 +212,19 @@ async def _build() -> dict[str, Any]:
metric = f"{c:,} rows" if c is not None else "iceberg table"
elif n["id"] == "generator":
metric = "Airflow gen DAGs"
elif n["id"] in ("vllm", "rag", "chromadb"):
info = _rag_info()
model = info.get("llm_model") or "gpt-4o"
embed = (info.get("embed_model") or "all-MiniLM-L6-v2").split("/")[-1]
if n["id"] == "vllm":
metric = f"{model} · OpenAI-compat"
elif n["id"] == "rag":
metric = f"LangChain · {embed}"
else:
metric = f"embeddings · {embed}"
node["level"] = "ok" if info else "warn"
elif n["id"] == "chat":
metric = "RAG chat · agent mode"
# PII overlay
p = pii_by_node.get(n["id"])
if p:
@@ -219,6 +264,9 @@ async def _build() -> dict[str, Any]:
edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
# AI serving lane pulses while governed data is being served to the LLM
edge["active"] = bool(_rag_info())
if e.get("offload"):
try:
from agent_ops import custodian_recent