Full stack visibility: mo S3 buckets, Elasticsearch/Kibana UI and topology
This commit is contained in:
mo
+1
-1
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py .
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py .
|
||||
RUN mkdir -p /data
|
||||
ENV DATABASE_URL=sqlite:////data/atc-agents.db
|
||||
EXPOSE 3201
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Elasticsearch + Kibana health API for Command Center."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
ELASTICSEARCH_URL = os.getenv("ELASTICSEARCH_URL", "https://10.0.21.46:9200").rstrip("/")
|
||||
KIBANA_URL = os.getenv("KIBANA_URL", "http://10.0.21.46:5601").rstrip("/")
|
||||
ELASTIC_USER = os.getenv("ELASTIC_USER", "elastic")
|
||||
ELASTIC_PASSWORD = os.getenv("ELASTIC_PASSWORD", "")
|
||||
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
|
||||
async def _probe_es() -> dict[str, Any]:
|
||||
auth = (ELASTIC_USER, ELASTIC_PASSWORD) if ELASTIC_PASSWORD else None
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0, verify=False) as client:
|
||||
r = await client.get(f"{ELASTICSEARCH_URL}/", auth=auth)
|
||||
if r.status_code >= 400:
|
||||
return {"ok": False, "status_code": r.status_code, "error": r.text[:200]}
|
||||
info = r.json()
|
||||
health_r = await client.get(f"{ELASTICSEARCH_URL}/_cluster/health", auth=auth)
|
||||
health = health_r.json() if health_r.status_code < 400 else {}
|
||||
stats_r = await client.get(f"{ELASTICSEARCH_URL}/_cat/indices?format=json&bytes=b", auth=auth)
|
||||
indices = stats_r.json() if stats_r.status_code < 400 else []
|
||||
return {
|
||||
"ok": True,
|
||||
"url": ELASTICSEARCH_URL,
|
||||
"cluster_name": info.get("cluster_name"),
|
||||
"version": info.get("version", {}).get("number"),
|
||||
"health": health.get("status", "unknown"),
|
||||
"nodes": health.get("number_of_nodes"),
|
||||
"indices_count": len(indices) if isinstance(indices, list) else 0,
|
||||
"indices": [
|
||||
{
|
||||
"name": i.get("index"),
|
||||
"docs": i.get("docs.count"),
|
||||
"size": i.get("store.size"),
|
||||
"health": i.get("health"),
|
||||
}
|
||||
for i in (indices[:50] if isinstance(indices, list) else [])
|
||||
],
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "url": ELASTICSEARCH_URL, "error": str(exc)}
|
||||
|
||||
|
||||
async def _probe_kibana() -> dict[str, Any]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0, verify=False) as client:
|
||||
r = await client.get(f"{KIBANA_URL}/api/status")
|
||||
if r.status_code >= 400:
|
||||
return {"ok": False, "status_code": r.status_code}
|
||||
data = r.json()
|
||||
overall = data.get("status", {}).get("overall", {})
|
||||
return {
|
||||
"ok": overall.get("level") in ("available", "green", "yellow"),
|
||||
"url": KIBANA_URL,
|
||||
"ui_url": KIBANA_URL,
|
||||
"level": overall.get("level", "unknown"),
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "url": KIBANA_URL, "error": str(exc)}
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def search_health():
|
||||
es, kb = await _probe_es(), await _probe_kibana()
|
||||
return {
|
||||
"ok": es.get("ok") or kb.get("ok"),
|
||||
"elasticsearch": es,
|
||||
"kibana": kb,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/elasticsearch")
|
||||
async def get_elasticsearch():
|
||||
result = await _probe_es()
|
||||
if not result.get("ok"):
|
||||
return JSONResponse(result, status_code=502)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/kibana")
|
||||
async def get_kibana():
|
||||
return await _probe_kibana()
|
||||
|
||||
|
||||
@router.get("/elasticsearch/query")
|
||||
async def es_search(q: str = Query(..., min_length=1), size: int = Query(10, le=50)):
|
||||
if not ELASTIC_PASSWORD:
|
||||
return JSONResponse({"ok": False, "error": "ELASTIC_PASSWORD not configured"}, status_code=503)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0, verify=False) as client:
|
||||
r = await client.post(
|
||||
f"{ELASTICSEARCH_URL}/_search",
|
||||
auth=(ELASTIC_USER, ELASTIC_PASSWORD),
|
||||
json={"query": {"query_string": {"query": q}}, "size": size},
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=r.status_code)
|
||||
data = r.json()
|
||||
hits = [
|
||||
{
|
||||
"index": h.get("_index"),
|
||||
"id": h.get("_id"),
|
||||
"score": h.get("_score"),
|
||||
"source": h.get("_source"),
|
||||
}
|
||||
for h in data.get("hits", {}).get("hits", [])
|
||||
]
|
||||
return {"ok": True, "total": data.get("hits", {}).get("total", {}), "hits": hits}
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=502)
|
||||
@@ -28,6 +28,7 @@ from presentation import build_presentation_payload, render_presentation_html
|
||||
from presentation_upload import get_deck, list_decks, save_upload
|
||||
from presentation_static import get_static_deck, list_static_decks
|
||||
from storage_s3 import router as storage_s3_router
|
||||
from elasticsearch_api import router as elasticsearch_router
|
||||
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
|
||||
from node_ops import build_node_detail, probe_node, run_node_probe_task
|
||||
from approval_service import (
|
||||
@@ -705,6 +706,7 @@ async def lifespan(app: FastAPI):
|
||||
|
||||
app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
|
||||
app.include_router(storage_s3_router)
|
||||
app.include_router(elasticsearch_router)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
|
||||
@@ -160,6 +160,26 @@ NODE_REGISTRY: dict[str, dict[str, Any]] = {
|
||||
],
|
||||
"commands": ["hdfs dfsadmin -report", "datanode status", "block health"],
|
||||
},
|
||||
|
||||
"elastic": {
|
||||
"label": "Elasticsearch / Kibana",
|
||||
"vm": "atc-elastic01",
|
||||
"vmid": 0,
|
||||
"pve": "atc-gpu",
|
||||
"ip": "10.0.21.46",
|
||||
"role": "search",
|
||||
"color": "#f5a623",
|
||||
"description": "Elasticsearch cluster atc-lakehouse + Kibana dashboards on port 5601.",
|
||||
"links": [
|
||||
{"label": "Kibana", "url": "http://10.0.21.46:5601"},
|
||||
{"label": "Elasticsearch", "url": "https://10.0.21.46:9200"},
|
||||
],
|
||||
"endpoints": [
|
||||
{"name": "elasticsearch", "host": "10.0.21.46", "port": "9200", "proto": "https"},
|
||||
{"name": "kibana", "host": "10.0.21.46", "port": "5601", "proto": "http"},
|
||||
],
|
||||
"commands": ["cluster health", "cat indices", "kibana status"],
|
||||
},
|
||||
"gpu": {
|
||||
"label": "GPU Lab",
|
||||
"vm": "atc-gpu-dev",
|
||||
|
||||
@@ -108,6 +108,7 @@ MODERN_DECKS: dict[str, dict[str, Any]] = {
|
||||
"React UI — Data Platform, Presentation, Data Quality, Knowledge Chat",
|
||||
"Docling on port 5001 for document parsing UI + API",
|
||||
"Postgres + Redis for agents; ChromaDB for vectors",
|
||||
"Elasticsearch + Kibana on atc-elastic01 (10.0.21.46)",
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -161,7 +162,7 @@ MODERN_DECKS: dict[str, dict[str, Any]] = {
|
||||
"bullets": [
|
||||
"Command Center VM304: 10.0.21.33 (this dashboard)",
|
||||
"GPU Lab VM303: 10.0.20.106 — 7× V100, vLLM, model manager",
|
||||
"DB Vault: 10.0.21.51 · Lakehouse: 10.0.21.50",
|
||||
"DB Vault: 10.0.21.51 · Lakehouse: 10.0.21.50 · Elastic: 10.0.21.46",
|
||||
"Docling UI: http://10.0.21.33:5001/ui/",
|
||||
],
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user