From 3a6ee8e2b05a22384a4cda224f35bb4187d86ac9 Mon Sep 17 00:00:00 2001 From: mo Date: Sat, 27 Jun 2026 21:30:45 +0000 Subject: [PATCH] feat(storage): rich Object Storage analytics dashboard Add /api/storage/s3/analytics endpoint that scans buckets (bounded + cached) to compute total size/objects, per-bucket distribution, file-type and size-class breakdowns, cumulative data-growth timeline and largest / recent objects. Track every S3 op routed through the API for a live storage-activity timeline. Rebuild StorageView into a tabbed view: Overview (KPI cards + SVG charts: growth area, bucket donut, type/size bars, activity sparkline, top folders, largest & recent objects) and the original bucket Browser. --- api/storage_s3.py | 221 ++++++++ ui/src/components/features/StorageView.tsx | 571 +++++++++++++++------ 2 files changed, 637 insertions(+), 155 deletions(-) diff --git a/api/storage_s3.py b/api/storage_s3.py index 4916161..f789998 100644 --- a/api/storage_s3.py +++ b/api/storage_s3.py @@ -3,6 +3,9 @@ from __future__ import annotations import os +import time +from collections import deque +from datetime import datetime, timezone from typing import Any import boto3 @@ -18,6 +21,47 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1") router = APIRouter(prefix="/api/storage/s3", tags=["storage"]) +# Bounded scan so the dashboard stays responsive on big buckets. +SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "50000")) +SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "12")) + +# In-process activity log — every S3 op routed through this API is recorded so +# the dashboard can show live "what is happening on the storage" timelines. +_activity: deque[tuple[float, str]] = deque(maxlen=20000) +_analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None} +_ANALYTICS_TTL = 45.0 + + +def _track(op: str) -> None: + try: + _activity.append((time.time(), op)) + except Exception: + pass + + +def _ext(key: str) -> str: + base = key.rsplit("/", 1)[-1] + if "." in base: + e = base.rsplit(".", 1)[-1].lower() + if 1 <= len(e) <= 8 and e.isalnum(): + return e + return "(none)" + + +def _size_class(n: int) -> str: + kb, mb, gb = 1024, 1024 ** 2, 1024 ** 3 + if n < kb: + return "<1 KB" + if n < mb: + return "1 KB–1 MB" + if n < 10 * mb: + return "1–10 MB" + if n < 100 * mb: + return "10–100 MB" + if n < gb: + return "100 MB–1 GB" + return ">1 GB" + def _client(): return boto3.client( @@ -41,6 +85,7 @@ def _human_size(n: int) -> str: @router.get("/health") async def s3_health(): try: + _track("health") s3 = _client() buckets = s3.list_buckets() names = [b["Name"] for b in buckets.get("Buckets", [])] @@ -57,6 +102,7 @@ async def s3_health(): @router.get("/buckets") async def list_buckets(): try: + _track("list_buckets") s3 = _client() resp = s3.list_buckets() items = [] @@ -84,6 +130,7 @@ async def list_objects( max_keys: int = Query(200, le=500), ): try: + _track("list_objects") s3 = _client() resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys) folders = [ @@ -119,6 +166,7 @@ async def list_objects( @router.get("/buckets/{bucket}/download") async def download_object(bucket: str, key: str = Query(...)): try: + _track("download") s3 = _client() obj = s3.get_object(Bucket=bucket, Key=key) body = obj["Body"] @@ -136,3 +184,176 @@ async def download_object(bucket: str, key: str = Query(...)): ) except ClientError as exc: return JSONResponse({"ok": False, "error": str(exc)}, status_code=404) + + + +def _scan_analytics() -> dict[str, Any]: + s3 = _client() + deadline = time.time() + SCAN_DEADLINE_S + resp = s3.list_buckets() + bucket_objs = resp.get("Buckets", []) + created_map = { + b["Name"]: (b.get("CreationDate").isoformat() if b.get("CreationDate") else None) + for b in bucket_objs + } + names = [b["Name"] for b in bucket_objs] + + total_bytes = 0 + total_objects = 0 + by_bucket: dict[str, list[int]] = {} + by_ext: dict[str, list[int]] = {} + by_size: dict[str, int] = {} + by_day: dict[str, list[int]] = {} + by_prefix: dict[str, list[int]] = {} + largest: list[tuple[int, str, str, str | None]] = [] + recent: list[tuple[str, str, str, int]] = [] + truncated = False + scanned = 0 + + for name in names: + bc = bb = 0 + token = None + while True: + if time.time() > deadline or scanned >= SCAN_MAX_OBJECTS: + truncated = True + break + kw: dict[str, Any] = {"Bucket": name, "MaxKeys": 1000} + if token: + kw["ContinuationToken"] = token + try: + r = s3.list_objects_v2(**kw) + except Exception: + break + for o in r.get("Contents", []): + sz = int(o.get("Size", 0) or 0) + key = o["Key"] + lm = o.get("LastModified") + bc += 1 + bb += sz + scanned += 1 + total_objects += 1 + total_bytes += sz + e = by_ext.setdefault(_ext(key), [0, 0]) + e[0] += 1 + e[1] += sz + sc = _size_class(sz) + by_size[sc] = by_size.get(sc, 0) + 1 + if lm: + d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d") + dd = by_day.setdefault(d, [0, 0]) + dd[0] += 1 + dd[1] += sz + recent.append((lm.isoformat(), name, key, sz)) + top = key.split("/", 1)[0] if "/" in key else "(root)" + pp = by_prefix.setdefault(f"{name}/{top}", [0, 0]) + pp[0] += 1 + pp[1] += sz + largest.append((sz, name, key, lm.isoformat() if lm else None)) + if scanned >= SCAN_MAX_OBJECTS: + truncated = True + break + if r.get("IsTruncated") and not (time.time() > deadline or scanned >= SCAN_MAX_OBJECTS): + token = r.get("NextContinuationToken") + if not token: + break + else: + break + by_bucket[name] = [bc, bb] + + # cumulative growth timeline + growth = [] + cum_b = cum_o = 0 + for d in sorted(by_day): + c, b = by_day[d] + cum_o += c + cum_b += b + growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b}) + + buckets_out = sorted( + ([{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]), + "pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0, + "created": created_map.get(n)} for n, v in by_bucket.items()]), + key=lambda x: x["bytes"], reverse=True) + + types_out = sorted( + ([{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])} + for k, v in by_ext.items()]), key=lambda x: x["bytes"], reverse=True)[:12] + + size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"] + size_out = [{"label": k, "count": by_size.get(k, 0)} for k in size_order] + + prefixes_out = sorted( + ([{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])} + for k, v in by_prefix.items()]), key=lambda x: x["bytes"], reverse=True)[:10] + + largest.sort(key=lambda x: x[0], reverse=True) + largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m} + for s, b, k, m in largest[:10]] + + recent.sort(key=lambda x: x[0], reverse=True) + recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)} + for m, b, k, s in recent[:15]] + + return { + "summary": { + "buckets": len(names), + "objects": total_objects, + "bytes": total_bytes, + "size_human": _human_size(total_bytes), + "avg_object_bytes": int(total_bytes / total_objects) if total_objects else 0, + "avg_object_human": _human_size(int(total_bytes / total_objects) if total_objects else 0), + "largest_human": largest_out[0]["size_human"] if largest_out else "0 B", + "newest": recent_out[0]["modified"] if recent_out else None, + "oldest": growth[0]["date"] if growth else None, + "truncated": truncated, + "scanned": scanned, + }, + "buckets": buckets_out, + "types": types_out, + "size_histogram": size_out, + "growth": growth, + "top_prefixes": prefixes_out, + "largest_objects": largest_out, + "recent": recent_out, + } + + +def _activity_view() -> dict[str, Any]: + now = time.time() + cutoff = now - 3600 + mins = [0] * 60 + by_op: dict[str, int] = {} + total = 0 + for ts, op in list(_activity): + if ts < cutoff: + continue + total += 1 + by_op[op] = by_op.get(op, 0) + 1 + idx = int((now - ts) // 60) + if 0 <= idx < 60: + mins[59 - idx] += 1 + last_ts = _activity[-1][0] if _activity else None + return { + "per_minute": mins, + "by_op": [{"op": k, "count": v} for k, v in sorted(by_op.items(), key=lambda x: -x[1])], + "total_last_hour": total, + "last_activity": datetime.fromtimestamp(last_ts, tz=timezone.utc).isoformat() if last_ts else None, + } + + +@router.get("/analytics") +async def analytics(refresh: bool = Query(False)): + """Aggregated storage analytics for the dashboard (cached ~45s).""" + _track("analytics") + now = time.time() + if not refresh and _analytics_cache["data"] is not None and now - _analytics_cache["ts"] < _ANALYTICS_TTL: + data = _analytics_cache["data"] + else: + try: + data = _scan_analytics() + _analytics_cache["data"] = data + _analytics_cache["ts"] = now + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502) + return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(), + "activity": _activity_view(), **data} diff --git a/ui/src/components/features/StorageView.tsx b/ui/src/components/features/StorageView.tsx index a7f0279..a8dd9a0 100644 --- a/ui/src/components/features/StorageView.tsx +++ b/ui/src/components/features/StorageView.tsx @@ -1,13 +1,355 @@ -import { useCallback, useEffect, useState } from 'react' -import { ChevronRight, Database, Download, ExternalLink, Folder, HardDrive, Loader2, RefreshCw } from 'lucide-react' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { + Activity, BarChart3, Boxes, ChevronRight, Clock, Database, Download, ExternalLink, + Files, Folder, Gauge, HardDrive, LayoutGrid, Loader2, RefreshCw, TrendingUp, +} from 'lucide-react' import { cn } from '../../lib/utils' import { subTabActive, subTabIdle } from '../../lib/tabActive' type Bucket = { name: string; created?: string; has_objects?: boolean } type S3Item = { type: string; name?: string; prefix?: string; key?: string; size_human?: string; modified?: string } +type GrowthPt = { date: string; objects: number; bytes: number; cum_objects: number; cum_bytes: number } +type Analytics = { + ok: boolean + endpoint?: string + generated_at?: string + summary?: { + buckets: number; objects: number; bytes: number; size_human: string + avg_object_human: string; largest_human: string; newest?: string | null + oldest?: string | null; truncated?: boolean; scanned?: number + } + buckets?: { name: string; objects: number; bytes: number; size_human: string; pct: number; created?: string | null }[] + types?: { ext: string; objects: number; bytes: number; size_human: string }[] + size_histogram?: { label: string; count: number }[] + growth?: GrowthPt[] + top_prefixes?: { prefix: string; objects: number; bytes: number; size_human: string }[] + largest_objects?: { bucket: string; key: string; bytes: number; size_human: string; modified?: string | null }[] + recent?: { modified?: string | null; bucket: string; key: string; bytes: number; size_human: string }[] + activity?: { per_minute: number[]; by_op: { op: string; count: number }[]; total_last_hour: number; last_activity?: string | null } +} + +const PALETTE = ['#38bdf8', '#34d399', '#a78bfa', '#fbbf24', '#fb7185', '#22d3ee', '#f472b6', '#818cf8', '#4ade80', '#fb923c'] + +function fmtBytes(n: number): string { + const u = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] + let i = 0 + let x = n + while (x >= 1024 && i < u.length - 1) { x /= 1024; i++ } + return `${i === 0 ? x : x.toFixed(1)} ${u[i]}` +} +const fmtNum = (n: number) => n.toLocaleString() + +/* ── Tiny SVG charts (no deps) ───────────────────────────────────────────── */ + +function GrowthChart({ data }: { data: GrowthPt[] }) { + const W = 720, H = 200, padB = 22, padL = 4 + if (!data.length) return + const maxCum = Math.max(...data.map((d) => d.cum_bytes), 1) + const maxDay = Math.max(...data.map((d) => d.bytes), 1) + const n = data.length + const x = (i: number) => padL + (n === 1 ? W / 2 : (i / (n - 1)) * (W - padL * 2)) + const yC = (v: number) => (H - padB) - (v / maxCum) * (H - padB - 8) + const line = data.map((d, i) => `${i === 0 ? 'M' : 'L'}${x(i).toFixed(1)},${yC(d.cum_bytes).toFixed(1)}`).join(' ') + const area = `${line} L${x(n - 1).toFixed(1)},${H - padB} L${x(0).toFixed(1)},${H - padB} Z` + const bw = Math.max(2, (W - padL * 2) / n - 3) + return ( + + + + + + + + {[0.25, 0.5, 0.75].map((g) => ( + + ))} + {data.map((d, i) => { + const h = (d.bytes / maxDay) * (H - padB - 8) + return + })} + + + {data.map((d, i) => (i % Math.ceil(n / 8) === 0 || i === n - 1) && ( + + ))} + + ) +} + +function Donut({ segments }: { segments: { label: string; value: number; color: string }[] }) { + const total = segments.reduce((s, x) => s + x.value, 0) || 1 + const r = 52, c = 2 * Math.PI * r + let acc = 0 + return ( + + + + {segments.map((s, i) => { + const frac = s.value / total + const dash = frac * c + const el = ( + + ) + acc += frac + return el + })} + + {segments.length} + buckets + + ) +} + +function BarsH({ items, unit }: { items: { label: string; value: number; sub?: string }[]; unit?: 'bytes' | 'num' }) { + const max = Math.max(...items.map((i) => i.value), 1) + if (!items.length) return + return ( +
+ {items.map((it, i) => ( +
+
+ {it.label} + {it.sub ?? (unit === 'bytes' ? fmtBytes(it.value) : fmtNum(it.value))} +
+
+
+
+
+ ))} +
+ ) +} + +function BarsV({ items }: { items: { label: string; count: number }[] }) { + const max = Math.max(...items.map((i) => i.count), 1) + return ( +
+ {items.map((it, i) => ( +
+ {it.count ? fmtNum(it.count) : ''} +
+
+
+ {it.label} +
+ ))} +
+ ) +} + +function Sparkline({ values, color = '#34d399' }: { values: number[]; color?: string }) { + const W = 280, H = 48 + const max = Math.max(...values, 1) + const n = values.length + const pts = values.map((v, i) => `${(i / (n - 1)) * W},${H - (v / max) * (H - 4) - 2}`).join(' ') + const area = `0,${H} ${pts} ${W},${H}` + return ( + + + + + ) +} + +function Empty({ label }: { label: string }) { + return
{label}
+} + +function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof HardDrive; label: string; value: string; sub?: string; accent: string }) { + return ( +
+
+ {label} +
+
{value}
+ {sub &&
{sub}
} +
+ ) +} + +function Panel({ title, icon: Icon, children, className, right }: { title: string; icon: typeof HardDrive; children: React.ReactNode; className?: string; right?: React.ReactNode }) { + return ( +
+
+

{title}

+ {right} +
+ {children} +
+ ) +} + +/* ── Main ────────────────────────────────────────────────────────────────── */ + export function StorageView() { - const [health, setHealth] = useState<{ ok: boolean; endpoint?: string; bucket_names?: string[]; error?: string } | null>(null) + const [tab, setTab] = useState<'overview' | 'browser'>('overview') + const [an, setAn] = useState(null) + const [anLoading, setAnLoading] = useState(false) + const [anErr, setAnErr] = useState(null) + + const loadAnalytics = useCallback(async (refresh = false) => { + setAnLoading(true) + setAnErr(null) + try { + const r = await fetch(`/api/storage/s3/analytics${refresh ? '?refresh=true' : ''}`) + const j = await r.json() + if (!r.ok || !j.ok) setAnErr(j.error || 'Analytics unavailable') + else setAn(j) + } catch { + setAnErr('Storage analytics API unavailable') + } finally { + setAnLoading(false) + } + }, []) + + useEffect(() => { + loadAnalytics() + const t = setInterval(() => loadAnalytics(), 30000) + return () => clearInterval(t) + }, [loadAnalytics]) + + const s = an?.summary + const bucketSegments = useMemo( + () => (an?.buckets || []).filter((b) => b.bytes > 0).map((b, i) => ({ label: b.name, value: b.bytes, color: PALETTE[i % PALETTE.length] })), + [an], + ) + + return ( +
+
+
+

+ ObjectScale S3 Storage +

+

+ Dell ECS · {an?.endpoint || '10.0.20.111:9020'} · {s ? `${fmtNum(s.objects)} objects · ${s.size_human}` : 'live analytics'} + {s?.truncated && (sampled {fmtNum(s.scanned || 0)})} +

+
+
+
+ {(['overview', 'browser'] as const).map((t) => ( + + ))} +
+ + Open Jupyter + + +
+
+ + {tab === 'overview' ? ( +
+ {anErr &&

{anErr}

} + {!an && anLoading &&

Crunching storage analytics…

} + + {an && ( +
+ {/* KPI row */} +
+ + + + + + +
+ + {/* Growth + bucket distribution */} +
+ {an.growth?.length || 0} days · since {s?.oldest || '—'}}> +
+ + +
+ +
+ {(an.buckets || []).map((b, i) => ( +
+ + + {b.name} + + {b.size_human} · {b.pct}% +
+ ))} +
+
+ +
+ + {/* types + size hist + activity */} +
+ + ({ label: t.ext, value: t.bytes, sub: `${fmtNum(t.objects)} · ${t.size_human}` }))} unit="bytes" /> + + +
+ + {an.activity?.total_last_hour || 0} ops}> + +
+ {(an.activity?.by_op || []).map((o) => ( + + {o.op} {o.count} + + ))} + {!an.activity?.by_op.length && No requests yet.} +
+ +
+ + {/* top prefixes + largest + recent */} +
+ + ({ label: p.prefix, value: p.bytes, sub: p.size_human }))} unit="bytes" /> + + +
+ {(an.largest_objects || []).map((o) => ( +
+ {o.key.split('/').pop()} + {o.size_human} +
+ ))} + {!an.largest_objects?.length && } +
+ + +
+ {(an.recent || []).map((o, i) => ( +
+ {o.key.split('/').pop()} + {o.modified?.slice(5, 16).replace('T', ' ')} +
+ ))} + {!an.recent?.length && } +
+ +
+
+ )} +
+ ) : ( + + )} +
+ ) +} + +/* ── Browser (the original explorer, now a tab) ──────────────────────────── */ + +function BucketBrowser() { const [buckets, setBuckets] = useState([]) const [bucket, setBucket] = useState(null) const [prefix, setPrefix] = useState('') @@ -17,179 +359,98 @@ export function StorageView() { const [error, setError] = useState(null) const loadBuckets = useCallback(async () => { - setLoading(true) - setError(null) + setLoading(true); setError(null) try { - const [h, b] = await Promise.all([ - fetch('/api/storage/s3/health'), - fetch('/api/storage/s3/buckets'), - ]) - if (h.ok) setHealth(await h.json()) + const b = await fetch('/api/storage/s3/buckets') if (b.ok) { const j = await b.json() setBuckets(j.buckets || []) if (!bucket && j.buckets?.length) setBucket(j.buckets[0].name) - } else { - setError('Failed to load buckets') - } - } catch { - setError('S3 API unavailable') - } finally { - setLoading(false) - } + } else setError('Failed to load buckets') + } catch { setError('S3 API unavailable') } finally { setLoading(false) } }, [bucket]) const loadObjects = useCallback(async (b: string, p: string) => { - setLoading(true) - setError(null) + setLoading(true); setError(null) try { const r = await fetch(`/api/storage/s3/buckets/${encodeURIComponent(b)}/objects?prefix=${encodeURIComponent(p)}`) const j = await r.json() - if (!r.ok || !j.ok) { - setError(j.error || 'List failed') - return - } - setFolders(j.folders || []) - setObjects(j.objects || []) - } catch { - setError('Failed to list objects') - } finally { - setLoading(false) - } + if (!r.ok || !j.ok) { setError(j.error || 'List failed'); return } + setFolders(j.folders || []); setObjects(j.objects || []) + } catch { setError('Failed to list objects') } finally { setLoading(false) } }, []) - useEffect(() => { - loadBuckets() - }, [loadBuckets]) - - useEffect(() => { - if (bucket) loadObjects(bucket, prefix) - }, [bucket, prefix, loadObjects]) + useEffect(() => { loadBuckets() }, [loadBuckets]) + useEffect(() => { if (bucket) loadObjects(bucket, prefix) }, [bucket, prefix, loadObjects]) const crumbs = prefix ? prefix.split('/').filter(Boolean) : [] return ( -
-
-
-

- - ObjectScale S3 Storage -

-

- Dell ECS · {health?.endpoint || '10.0.20.111:9020'} · live bucket browser -

+
+
+ -
- - -
- {bucket && ( - - )} - - {loading && ( -

- Loading… -

- )} - {error &&

{error}

} - -
- - - - - - - + + + + + + ))} + +
NameSizeModified + + )} + {loading &&

Loading…

} + {error &&

{error}

} +
+ + + + + + + {folders.map((f) => ( + + + - - - {folders.map((f) => ( - - - - - - ))} - {objects.map((o) => ( - - - - - - - ))} - -
NameSizeModified +
+ + ——
- - —— -
{o.name || o.key}{o.size_human}{o.modified?.slice(0, 19) || '—'} - {o.key && bucket && ( - - - - )} -
- {!loading && folders.length === 0 && objects.length === 0 && bucket && ( -

This prefix is empty.

- )} -
+ ))} + {objects.map((o) => ( +
{o.name || o.key}{o.size_human}{o.modified?.slice(0, 19) || '—'} + {o.key && bucket && ( + + + + )} +
+ {!loading && folders.length === 0 && objects.length === 0 && bucket &&

This prefix is empty.

}