Full stack visibility: mo S3 buckets, Elasticsearch/Kibana UI and topology

ui/src/components/features/PlatformTopology.tsx

@@ -63,6 +63,8 @@ const STAGES: TopoStage[] = [
     nodes: [
       { id: 'bi', label: 'BI / Reporting', sub: 'Dashboards', metricKey: 'bi' },
       { id: 'jupyter', label: 'Jupyter Notebooks', sub: 'Data science', metricKey: 'jupyter' },
+      { id: 'elasticsearch', label: 'Elasticsearch', sub: 'Search · :9200', metricKey: 'elasticsearch' },
+      { id: 'kibana', label: 'Kibana', sub: 'Dashboards · :5601', metricKey: 'kibana' },
       { id: 'llm', label: 'GenAI LLM', sub: 'vLLM inference', metricKey: 'llm' },
     ],
   },
@@ -95,6 +97,10 @@ const FLOW_EDGES: FlowEdge[] = [
   { from: 's3', to: 'jupyter', kind: 'serve', label: 'Datasets' },
   { from: 'trino', to: 'llm', kind: 'serve', label: 'RAG context' },
   { from: 's3', to: 'llm', kind: 'serve', label: 'Model artifacts' },
+  { from: 'kafka', to: 'elasticsearch', kind: 'stream', label: 'Index' },
+  { from: 'spark', to: 'elasticsearch', kind: 'etl', label: 'Bulk index' },
+  { from: 'elasticsearch', to: 'kibana', kind: 'serve', label: 'Visualize' },
+  { from: 'elasticsearch', to: 'bi', kind: 'serve', label: 'Search' },
 ]
 
 const STAGE_BADGE: Record<string, string> = {
@@ -136,7 +142,7 @@ const NODE_CLICK_MAP: Record<string, string> = {
   postgresql: 'src-postgres', mysql: 'src-mysql', mongodb: 'src-mongo', cassandra: 'src-cassandra',
   debezium: 'cdc-postgres', kafka: 'stream-kafka', airflow: 'src-airflow', spark: 'stream-spark',
   trino: 'query-trino', iceberg: 'lake-iceberg', s3: 'lake-s3', bi: 'cons-bi',
-  jupyter: 'cons-notebooks', llm: 'cons-ml',
+  jupyter: 'cons-notebooks', elasticsearch: 'cons-elastic', kibana: 'cons-kibana', llm: 'cons-ml',
 }
 
 const NODE_POS: Record<string, { col: number; row: number; rows: number }> = {}
@@ -175,7 +181,7 @@ function seedMetrics(): MetricState {
     postgresql: '12.4k rows/s', mysql: '8.1k rows/s', mongodb: '2.3k docs/s', cassandra: '5.6k ops/s',
     debezium: '4 connectors active', kafka: '142 MB/s', airflow: '18 DAGs · daily 02:00 UTC',
     spark: '6 executors live', trino: '3 queries active', iceberg: '847 tables · 2.1 TB', s3: '14.2 TB stored',
-    bi: '26 dashboards', jupyter: '12 kernels active', llm: 'Checking…',
+    bi: '26 dashboards', jupyter: '12 kernels active', elasticsearch: 'atc-lakehouse', kibana: 'available', llm: 'Checking…',
   }
 }
 
@@ -239,6 +245,18 @@ export function PlatformTopology({ workload, animations, selectedNodeId, onNodeC
 
   const edgesLive = pipelineActive || anyBusy
 
+  useEffect(() => {
+    fetch("/api/search/health").then(r => r.json()).then(j => {
+      const kb = j.kibana
+      const es = j.elasticsearch
+      setMetrics(prev => ({
+        ...prev,
+        kibana: kb?.ok ? (kb.level || "available") : "offline",
+        elasticsearch: es?.ok ? (es.health || "green") : (kb?.ok ? "via Kibana" : "offline"),
+      }))
+    }).catch(() => {})
+  }, [])
+
   useEffect(() => {
     setMetrics((prev) => ({ ...prev, llm: formatLlmMetric(workload) }))
   }, [workload?.gpu?.model, workload?.gpu?.inference_active, workload?.gpu?.avg_util, workload?.gpu?.gpus])

ui/src/components/features/SearchView.tsx

@@ -0,0 +1,163 @@
+import { useCallback, useEffect, useState } from 'react'
+import { ExternalLink, Loader2, RefreshCw, Search } from 'lucide-react'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+type EsHealth = {
+  ok?: boolean
+  cluster_name?: string
+  version?: string
+  health?: string
+  nodes?: number
+  indices_count?: number
+  indices?: { name?: string; docs?: string; size?: string; health?: string }[]
+  error?: string
+  url?: string
+}
+
+type KbHealth = { ok?: boolean; level?: string; ui_url?: string; url?: string; error?: string }
+
+export function SearchView() {
+  const [es, setEs] = useState<EsHealth | null>(null)
+  const [kb, setKb] = useState<KbHealth | null>(null)
+  const [loading, setLoading] = useState(false)
+  const [query, setQuery] = useState('')
+  const [hits, setHits] = useState<{ index?: string; score?: number; source?: Record<string, unknown> }[]>([])
+  const [searchError, setSearchError] = useState<string | null>(null)
+
+  const load = useCallback(async () => {
+    setLoading(true)
+    try {
+      const r = await fetch('/api/search/health')
+      const j = await r.json()
+      setEs(j.elasticsearch || null)
+      setKb(j.kibana || null)
+    } finally {
+      setLoading(false)
+    }
+  }, [])
+
+  useEffect(() => {
+    load()
+    const iv = setInterval(load, 15000)
+    return () => clearInterval(iv)
+  }, [load])
+
+  const onSearch = async () => {
+    if (!query.trim()) return
+    setSearchError(null)
+    try {
+      const r = await fetch(`/api/search/elasticsearch/query?q=${encodeURIComponent(query)}&size=15`)
+      const j = await r.json()
+      if (!r.ok || !j.ok) {
+        setSearchError(j.error || 'Search failed — set ELASTIC_PASSWORD in .env for query API')
+        setHits([])
+        return
+      }
+      setHits(j.hits || [])
+    } catch {
+      setSearchError('Search request failed')
+    }
+  }
+
+  const healthColor = (h?: string) => (h === 'green' ? 'text-success' : h === 'yellow' ? 'text-warning' : 'text-danger')
+
+  return (
+    <div className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
+      <header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
+        <div>
+          <h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
+            <Search className="h-4 w-4 text-docker" />
+            Elasticsearch & Kibana
+          </h2>
+          <p className="text-[10px] text-foreground-muted">atc-elastic01 · 10.0.21.46 · cluster atc-lakehouse</p>
+        </div>
+        <div className="flex gap-2">
+          {kb?.ui_url && (
+            <a href={kb.ui_url} target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-3 py-1.5 text-[11px] font-medium', subTabActive)}>
+              <ExternalLink className="h-3 w-3" /> Kibana
+            </a>
+          )}
+          <button type="button" onClick={load} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
+            <RefreshCw className={cn('inline h-3 w-3', loading && 'animate-spin')} /> Refresh
+          </button>
+        </div>
+      </header>
+
+      <div className="grid shrink-0 grid-cols-2 gap-3 border-b border-border p-4 md:grid-cols-4">
+        <Stat label="Elasticsearch" value={es?.ok ? (es.health || 'up') : 'offline'} className={healthColor(es?.health)} />
+        <Stat label="Cluster" value={es?.cluster_name || '—'} />
+        <Stat label="Indices" value={String(es?.indices_count ?? '—')} />
+        <Stat label="Kibana" value={kb?.ok ? (kb.level || 'available') : 'offline'} className={kb?.ok ? 'text-success' : 'text-warning'} />
+      </div>
+
+      <div className="flex shrink-0 gap-2 border-b border-border p-3">
+        <input
+          value={query}
+          onChange={(e) => setQuery(e.target.value)}
+          onKeyDown={(e) => e.key === 'Enter' && onSearch()}
+          placeholder="Search indices (requires ELASTIC_PASSWORD)…"
+          className="min-w-0 flex-1 rounded border border-border bg-surface-overlay px-3 py-2 text-[12px]"
+        />
+        <button type="button" onClick={onSearch} className={cn('rounded px-3 py-2 text-[11px]', subTabActive)}>Search</button>
+      </div>
+
+      {searchError && <p className="px-4 py-2 text-[11px] text-warning">{searchError}</p>}
+
+      <div className="scrollbar-thin flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4 md:flex-row">
+        <div className="min-w-0 flex-1">
+          <h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Indices</h3>
+          {es?.indices && es.indices.length > 0 ? (
+            <table className="w-full text-left text-[11px]">
+              <thead>
+                <tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
+                  <th className="py-1">Index</th>
+                  <th className="py-1">Docs</th>
+                  <th className="py-1">Size</th>
+                  <th className="py-1">Health</th>
+                </tr>
+              </thead>
+              <tbody>
+                {es.indices.map((i) => (
+                  <tr key={i.name} className="border-b border-border/40">
+                    <td className="py-1 font-mono text-[10px]">{i.name}</td>
+                    <td className="py-1">{i.docs ?? '—'}</td>
+                    <td className="py-1">{i.size ?? '—'}</td>
+                    <td className={cn('py-1', healthColor(i.health))}>{i.health}</td>
+                  </tr>
+                ))}
+              </tbody>
+            </table>
+          ) : (
+            <p className="text-[11px] text-foreground-muted">
+              {es?.ok ? 'No indices listed.' : (es?.error || 'Elasticsearch :9200 not reachable from Command Center — Kibana :5601 may still be up.')}
+            </p>
+          )}
+        </div>
+
+        {hits.length > 0 && (
+          <div className="min-w-0 flex-1">
+            <h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Search results</h3>
+            <div className="space-y-2">
+              {hits.map((h, i) => (
+                <div key={i} className="rounded border border-border bg-surface-overlay/50 p-2 text-[10px]">
+                  <p className="font-mono text-docker">{h.index} · score {h.score?.toFixed(2)}</p>
+                  <pre className="mt-1 max-h-24 overflow-auto whitespace-pre-wrap text-foreground-muted">{JSON.stringify(h.source, null, 2).slice(0, 400)}</pre>
+                </div>
+              ))}
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+function Stat({ label, value, className }: { label: string; value: string; className?: string }) {
+  return (
+    <div className="rounded border border-border bg-surface-overlay/60 px-3 py-2 text-center">
+      <p className="text-[9px] uppercase text-foreground-faint">{label}</p>
+      <p className={cn('font-mono text-sm font-semibold capitalize', className || 'text-foreground')}>{value}</p>
+    </div>
+  )
+}

ui/src/components/layout/SideNav.tsx

@@ -1,4 +1,4 @@
-import { DatabaseZap, HardDrive, LayoutDashboard, MessageSquare, Presentation, ShieldCheck } from 'lucide-react'
+import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, ShieldCheck } from 'lucide-react'
 import type { Agent, AgentAnim, GpuStatus } from '../../types'
 import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
 import { getAgentMeta } from '../../lib/agentMeta'
@@ -6,7 +6,7 @@ import { cn } from '../../lib/utils'
 import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
 import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
 
-type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'approvals'
+type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'search' | 'approvals'
 
 type Props = {
   agents: Agent[]
@@ -31,6 +31,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
   { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
   { id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
   { id: 'storage', label: 'Object Storage', icon: HardDrive },
+  { id: 'search', label: 'Elasticsearch', icon: Search },
 ]
 
 export function SideNav({