feat: continuous live generator + vLLM/RAG lane in Data Flow
Live dashboard now feels truly real-time: - Background generator streams randomly-sized bursts of real rows into PostgreSQL, MySQL, MongoDB & Cassandra every ~4s (CDC picks them up). Throughput rises and falls; counters move in lock-step (base snapshot + generated). Runs only while the Live tab is polling (heartbeat-gated) so source tables do not grow unbounded; on/off toggle exposed in the UI. - New /api/federated/live/generator toggle; /live returns per-tick activity (last burst sizes, orders by region/status, event feed). - LiveDashboard: live-activity panel, orders-per-tick sparkline, event stream feed, burst-by-region/status charts, generator status + control. Data Flow graph now explains how data reaches the assistant: - Added ChromaDB -> RAG (LangChain) -> vLLM Gateway -> Knowledge Chat lane, with Trino / OpenMetadata / curated-masked feeding LLM context. Live model & embed metrics pulled from the RAG /config. New node/edge kinds + legend.
This commit is contained in:
@@ -52,6 +52,11 @@ NODES: list[dict[str, Any]] = [
|
||||
# governance — bottom centre
|
||||
{"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance",
|
||||
"x": 47, "y": 93, "url": "http://10.0.21.47:8585"},
|
||||
# AI serving lane — how governed data reaches the LLM & the Command Center chat
|
||||
{"id": "chromadb", "label": "ChromaDB", "sub": "vectors · embeddings", "kind": "vector", "x": 60, "y": 65},
|
||||
{"id": "rag", "label": "RAG · LangChain", "sub": "retrieve · augment · agent", "kind": "rag", "x": 74, "y": 65},
|
||||
{"id": "vllm", "label": "vLLM Gateway", "sub": "Llama3-70B · GPT-4o", "kind": "llm", "x": 88, "y": 68},
|
||||
{"id": "chat", "label": "Knowledge Chat", "sub": "Command Center", "kind": "chat", "x": 92, "y": 90},
|
||||
]
|
||||
|
||||
# Edges. movement_id (optional) links to movements.py so the edge is triggerable.
|
||||
@@ -86,6 +91,14 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "cassandra", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "neo4j", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "trino", "to": "openmetadata", "kind": "catalog"},
|
||||
# AI serving lane: governed business data + catalog + vectors → RAG → vLLM → chat
|
||||
{"from": "trino", "to": "rag", "kind": "context"},
|
||||
{"from": "openmetadata", "to": "rag", "kind": "context"},
|
||||
{"from": "iceberg_curated", "to": "rag", "kind": "context"},
|
||||
{"from": "chromadb", "to": "rag", "kind": "retrieve"},
|
||||
{"from": "rag", "to": "vllm", "kind": "prompt"},
|
||||
{"from": "vllm", "to": "chat", "kind": "answer"},
|
||||
{"from": "rag", "to": "chat", "kind": "answer"},
|
||||
]
|
||||
|
||||
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
@@ -117,6 +130,25 @@ def _trino_scalar(sql: str, deadline_s: float = 8.0) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
RAG_URL = os.getenv("RAG_URL", "http://rag-api:5020").rstrip("/")
|
||||
_rag_cache: dict[str, Any] = {"ts": 0.0, "val": None}
|
||||
|
||||
|
||||
def _rag_info() -> dict[str, Any]:
|
||||
now = time.time()
|
||||
if _rag_cache["val"] is not None and now - _rag_cache["ts"] < 60:
|
||||
return _rag_cache["val"]
|
||||
info: dict[str, Any] = {}
|
||||
try:
|
||||
with httpx.Client(timeout=2.0) as client:
|
||||
info = client.get(f"{RAG_URL}/config").json() or {}
|
||||
except Exception:
|
||||
info = {}
|
||||
_rag_cache["val"] = info
|
||||
_rag_cache["ts"] = now
|
||||
return info
|
||||
|
||||
|
||||
def _iceberg_hadoop_count() -> int | None:
|
||||
now = time.time()
|
||||
if _count_cache["val"] is not None and now - _count_cache["ts"] < 60:
|
||||
@@ -180,6 +212,19 @@ async def _build() -> dict[str, Any]:
|
||||
metric = f"{c:,} rows" if c is not None else "iceberg table"
|
||||
elif n["id"] == "generator":
|
||||
metric = "Airflow gen DAGs"
|
||||
elif n["id"] in ("vllm", "rag", "chromadb"):
|
||||
info = _rag_info()
|
||||
model = info.get("llm_model") or "gpt-4o"
|
||||
embed = (info.get("embed_model") or "all-MiniLM-L6-v2").split("/")[-1]
|
||||
if n["id"] == "vllm":
|
||||
metric = f"{model} · OpenAI-compat"
|
||||
elif n["id"] == "rag":
|
||||
metric = f"LangChain · {embed}"
|
||||
else:
|
||||
metric = f"embeddings · {embed}"
|
||||
node["level"] = "ok" if info else "warn"
|
||||
elif n["id"] == "chat":
|
||||
metric = "RAG chat · agent mode"
|
||||
# PII overlay
|
||||
p = pii_by_node.get(n["id"])
|
||||
if p:
|
||||
@@ -219,6 +264,9 @@ async def _build() -> dict[str, Any]:
|
||||
edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
|
||||
elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
|
||||
edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
|
||||
elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
|
||||
# AI serving lane pulses while governed data is being served to the LLM
|
||||
edge["active"] = bool(_rag_info())
|
||||
if e.get("offload"):
|
||||
try:
|
||||
from agent_ops import custodian_recent
|
||||
|
||||
Reference in New Issue
Block a user