feat(storage): rich Object Storage analytics dashboard
Add /api/storage/s3/analytics endpoint that scans buckets (bounded + cached) to compute total size/objects, per-bucket distribution, file-type and size-class breakdowns, cumulative data-growth timeline and largest / recent objects. Track every S3 op routed through the API for a live storage-activity timeline. Rebuild StorageView into a tabbed view: Overview (KPI cards + SVG charts: growth area, bucket donut, type/size bars, activity sparkline, top folders, largest & recent objects) and the original bucket Browser.
This commit is contained in:
mo
@@ -3,6 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
@@ -18,6 +21,47 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
|
||||
router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
|
||||
|
||||
# Bounded scan so the dashboard stays responsive on big buckets.
|
||||
SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "50000"))
|
||||
SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "12"))
|
||||
|
||||
# In-process activity log — every S3 op routed through this API is recorded so
|
||||
# the dashboard can show live "what is happening on the storage" timelines.
|
||||
_activity: deque[tuple[float, str]] = deque(maxlen=20000)
|
||||
_analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
_ANALYTICS_TTL = 45.0
|
||||
|
||||
|
||||
def _track(op: str) -> None:
|
||||
try:
|
||||
_activity.append((time.time(), op))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _ext(key: str) -> str:
|
||||
base = key.rsplit("/", 1)[-1]
|
||||
if "." in base:
|
||||
e = base.rsplit(".", 1)[-1].lower()
|
||||
if 1 <= len(e) <= 8 and e.isalnum():
|
||||
return e
|
||||
return "(none)"
|
||||
|
||||
|
||||
def _size_class(n: int) -> str:
|
||||
kb, mb, gb = 1024, 1024 ** 2, 1024 ** 3
|
||||
if n < kb:
|
||||
return "<1 KB"
|
||||
if n < mb:
|
||||
return "1 KB–1 MB"
|
||||
if n < 10 * mb:
|
||||
return "1–10 MB"
|
||||
if n < 100 * mb:
|
||||
return "10–100 MB"
|
||||
if n < gb:
|
||||
return "100 MB–1 GB"
|
||||
return ">1 GB"
|
||||
|
||||
|
||||
def _client():
|
||||
return boto3.client(
|
||||
@@ -41,6 +85,7 @@ def _human_size(n: int) -> str:
|
||||
@router.get("/health")
|
||||
async def s3_health():
|
||||
try:
|
||||
_track("health")
|
||||
s3 = _client()
|
||||
buckets = s3.list_buckets()
|
||||
names = [b["Name"] for b in buckets.get("Buckets", [])]
|
||||
@@ -57,6 +102,7 @@ async def s3_health():
|
||||
@router.get("/buckets")
|
||||
async def list_buckets():
|
||||
try:
|
||||
_track("list_buckets")
|
||||
s3 = _client()
|
||||
resp = s3.list_buckets()
|
||||
items = []
|
||||
@@ -84,6 +130,7 @@ async def list_objects(
|
||||
max_keys: int = Query(200, le=500),
|
||||
):
|
||||
try:
|
||||
_track("list_objects")
|
||||
s3 = _client()
|
||||
resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys)
|
||||
folders = [
|
||||
@@ -119,6 +166,7 @@ async def list_objects(
|
||||
@router.get("/buckets/{bucket}/download")
|
||||
async def download_object(bucket: str, key: str = Query(...)):
|
||||
try:
|
||||
_track("download")
|
||||
s3 = _client()
|
||||
obj = s3.get_object(Bucket=bucket, Key=key)
|
||||
body = obj["Body"]
|
||||
@@ -136,3 +184,176 @@ async def download_object(bucket: str, key: str = Query(...)):
|
||||
)
|
||||
except ClientError as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
|
||||
|
||||
|
||||
|
||||
def _scan_analytics() -> dict[str, Any]:
|
||||
s3 = _client()
|
||||
deadline = time.time() + SCAN_DEADLINE_S
|
||||
resp = s3.list_buckets()
|
||||
bucket_objs = resp.get("Buckets", [])
|
||||
created_map = {
|
||||
b["Name"]: (b.get("CreationDate").isoformat() if b.get("CreationDate") else None)
|
||||
for b in bucket_objs
|
||||
}
|
||||
names = [b["Name"] for b in bucket_objs]
|
||||
|
||||
total_bytes = 0
|
||||
total_objects = 0
|
||||
by_bucket: dict[str, list[int]] = {}
|
||||
by_ext: dict[str, list[int]] = {}
|
||||
by_size: dict[str, int] = {}
|
||||
by_day: dict[str, list[int]] = {}
|
||||
by_prefix: dict[str, list[int]] = {}
|
||||
largest: list[tuple[int, str, str, str | None]] = []
|
||||
recent: list[tuple[str, str, str, int]] = []
|
||||
truncated = False
|
||||
scanned = 0
|
||||
|
||||
for name in names:
|
||||
bc = bb = 0
|
||||
token = None
|
||||
while True:
|
||||
if time.time() > deadline or scanned >= SCAN_MAX_OBJECTS:
|
||||
truncated = True
|
||||
break
|
||||
kw: dict[str, Any] = {"Bucket": name, "MaxKeys": 1000}
|
||||
if token:
|
||||
kw["ContinuationToken"] = token
|
||||
try:
|
||||
r = s3.list_objects_v2(**kw)
|
||||
except Exception:
|
||||
break
|
||||
for o in r.get("Contents", []):
|
||||
sz = int(o.get("Size", 0) or 0)
|
||||
key = o["Key"]
|
||||
lm = o.get("LastModified")
|
||||
bc += 1
|
||||
bb += sz
|
||||
scanned += 1
|
||||
total_objects += 1
|
||||
total_bytes += sz
|
||||
e = by_ext.setdefault(_ext(key), [0, 0])
|
||||
e[0] += 1
|
||||
e[1] += sz
|
||||
sc = _size_class(sz)
|
||||
by_size[sc] = by_size.get(sc, 0) + 1
|
||||
if lm:
|
||||
d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d")
|
||||
dd = by_day.setdefault(d, [0, 0])
|
||||
dd[0] += 1
|
||||
dd[1] += sz
|
||||
recent.append((lm.isoformat(), name, key, sz))
|
||||
top = key.split("/", 1)[0] if "/" in key else "(root)"
|
||||
pp = by_prefix.setdefault(f"{name}/{top}", [0, 0])
|
||||
pp[0] += 1
|
||||
pp[1] += sz
|
||||
largest.append((sz, name, key, lm.isoformat() if lm else None))
|
||||
if scanned >= SCAN_MAX_OBJECTS:
|
||||
truncated = True
|
||||
break
|
||||
if r.get("IsTruncated") and not (time.time() > deadline or scanned >= SCAN_MAX_OBJECTS):
|
||||
token = r.get("NextContinuationToken")
|
||||
if not token:
|
||||
break
|
||||
else:
|
||||
break
|
||||
by_bucket[name] = [bc, bb]
|
||||
|
||||
# cumulative growth timeline
|
||||
growth = []
|
||||
cum_b = cum_o = 0
|
||||
for d in sorted(by_day):
|
||||
c, b = by_day[d]
|
||||
cum_o += c
|
||||
cum_b += b
|
||||
growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b})
|
||||
|
||||
buckets_out = sorted(
|
||||
([{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]),
|
||||
"pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0,
|
||||
"created": created_map.get(n)} for n, v in by_bucket.items()]),
|
||||
key=lambda x: x["bytes"], reverse=True)
|
||||
|
||||
types_out = sorted(
|
||||
([{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||
for k, v in by_ext.items()]), key=lambda x: x["bytes"], reverse=True)[:12]
|
||||
|
||||
size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"]
|
||||
size_out = [{"label": k, "count": by_size.get(k, 0)} for k in size_order]
|
||||
|
||||
prefixes_out = sorted(
|
||||
([{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||
for k, v in by_prefix.items()]), key=lambda x: x["bytes"], reverse=True)[:10]
|
||||
|
||||
largest.sort(key=lambda x: x[0], reverse=True)
|
||||
largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m}
|
||||
for s, b, k, m in largest[:10]]
|
||||
|
||||
recent.sort(key=lambda x: x[0], reverse=True)
|
||||
recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)}
|
||||
for m, b, k, s in recent[:15]]
|
||||
|
||||
return {
|
||||
"summary": {
|
||||
"buckets": len(names),
|
||||
"objects": total_objects,
|
||||
"bytes": total_bytes,
|
||||
"size_human": _human_size(total_bytes),
|
||||
"avg_object_bytes": int(total_bytes / total_objects) if total_objects else 0,
|
||||
"avg_object_human": _human_size(int(total_bytes / total_objects) if total_objects else 0),
|
||||
"largest_human": largest_out[0]["size_human"] if largest_out else "0 B",
|
||||
"newest": recent_out[0]["modified"] if recent_out else None,
|
||||
"oldest": growth[0]["date"] if growth else None,
|
||||
"truncated": truncated,
|
||||
"scanned": scanned,
|
||||
},
|
||||
"buckets": buckets_out,
|
||||
"types": types_out,
|
||||
"size_histogram": size_out,
|
||||
"growth": growth,
|
||||
"top_prefixes": prefixes_out,
|
||||
"largest_objects": largest_out,
|
||||
"recent": recent_out,
|
||||
}
|
||||
|
||||
|
||||
def _activity_view() -> dict[str, Any]:
|
||||
now = time.time()
|
||||
cutoff = now - 3600
|
||||
mins = [0] * 60
|
||||
by_op: dict[str, int] = {}
|
||||
total = 0
|
||||
for ts, op in list(_activity):
|
||||
if ts < cutoff:
|
||||
continue
|
||||
total += 1
|
||||
by_op[op] = by_op.get(op, 0) + 1
|
||||
idx = int((now - ts) // 60)
|
||||
if 0 <= idx < 60:
|
||||
mins[59 - idx] += 1
|
||||
last_ts = _activity[-1][0] if _activity else None
|
||||
return {
|
||||
"per_minute": mins,
|
||||
"by_op": [{"op": k, "count": v} for k, v in sorted(by_op.items(), key=lambda x: -x[1])],
|
||||
"total_last_hour": total,
|
||||
"last_activity": datetime.fromtimestamp(last_ts, tz=timezone.utc).isoformat() if last_ts else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/analytics")
|
||||
async def analytics(refresh: bool = Query(False)):
|
||||
"""Aggregated storage analytics for the dashboard (cached ~45s)."""
|
||||
_track("analytics")
|
||||
now = time.time()
|
||||
if not refresh and _analytics_cache["data"] is not None and now - _analytics_cache["ts"] < _ANALYTICS_TTL:
|
||||
data = _analytics_cache["data"]
|
||||
else:
|
||||
try:
|
||||
data = _scan_analytics()
|
||||
_analytics_cache["data"] = data
|
||||
_analytics_cache["ts"] = now
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502)
|
||||
return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"activity": _activity_view(), **data}
|
||||
|
||||
@@ -1,13 +1,355 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { ChevronRight, Database, Download, ExternalLink, Folder, HardDrive, Loader2, RefreshCw } from 'lucide-react'
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import {
|
||||
Activity, BarChart3, Boxes, ChevronRight, Clock, Database, Download, ExternalLink,
|
||||
Files, Folder, Gauge, HardDrive, LayoutGrid, Loader2, RefreshCw, TrendingUp,
|
||||
} from 'lucide-react'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type Bucket = { name: string; created?: string; has_objects?: boolean }
|
||||
type S3Item = { type: string; name?: string; prefix?: string; key?: string; size_human?: string; modified?: string }
|
||||
|
||||
type GrowthPt = { date: string; objects: number; bytes: number; cum_objects: number; cum_bytes: number }
|
||||
type Analytics = {
|
||||
ok: boolean
|
||||
endpoint?: string
|
||||
generated_at?: string
|
||||
summary?: {
|
||||
buckets: number; objects: number; bytes: number; size_human: string
|
||||
avg_object_human: string; largest_human: string; newest?: string | null
|
||||
oldest?: string | null; truncated?: boolean; scanned?: number
|
||||
}
|
||||
buckets?: { name: string; objects: number; bytes: number; size_human: string; pct: number; created?: string | null }[]
|
||||
types?: { ext: string; objects: number; bytes: number; size_human: string }[]
|
||||
size_histogram?: { label: string; count: number }[]
|
||||
growth?: GrowthPt[]
|
||||
top_prefixes?: { prefix: string; objects: number; bytes: number; size_human: string }[]
|
||||
largest_objects?: { bucket: string; key: string; bytes: number; size_human: string; modified?: string | null }[]
|
||||
recent?: { modified?: string | null; bucket: string; key: string; bytes: number; size_human: string }[]
|
||||
activity?: { per_minute: number[]; by_op: { op: string; count: number }[]; total_last_hour: number; last_activity?: string | null }
|
||||
}
|
||||
|
||||
const PALETTE = ['#38bdf8', '#34d399', '#a78bfa', '#fbbf24', '#fb7185', '#22d3ee', '#f472b6', '#818cf8', '#4ade80', '#fb923c']
|
||||
|
||||
function fmtBytes(n: number): string {
|
||||
const u = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
|
||||
let i = 0
|
||||
let x = n
|
||||
while (x >= 1024 && i < u.length - 1) { x /= 1024; i++ }
|
||||
return `${i === 0 ? x : x.toFixed(1)} ${u[i]}`
|
||||
}
|
||||
const fmtNum = (n: number) => n.toLocaleString()
|
||||
|
||||
/* ── Tiny SVG charts (no deps) ───────────────────────────────────────────── */
|
||||
|
||||
function GrowthChart({ data }: { data: GrowthPt[] }) {
|
||||
const W = 720, H = 200, padB = 22, padL = 4
|
||||
if (!data.length) return <Empty label="No data yet" />
|
||||
const maxCum = Math.max(...data.map((d) => d.cum_bytes), 1)
|
||||
const maxDay = Math.max(...data.map((d) => d.bytes), 1)
|
||||
const n = data.length
|
||||
const x = (i: number) => padL + (n === 1 ? W / 2 : (i / (n - 1)) * (W - padL * 2))
|
||||
const yC = (v: number) => (H - padB) - (v / maxCum) * (H - padB - 8)
|
||||
const line = data.map((d, i) => `${i === 0 ? 'M' : 'L'}${x(i).toFixed(1)},${yC(d.cum_bytes).toFixed(1)}`).join(' ')
|
||||
const area = `${line} L${x(n - 1).toFixed(1)},${H - padB} L${x(0).toFixed(1)},${H - padB} Z`
|
||||
const bw = Math.max(2, (W - padL * 2) / n - 3)
|
||||
return (
|
||||
<svg viewBox={`0 0 ${W} ${H}`} className="h-full w-full" preserveAspectRatio="none">
|
||||
<defs>
|
||||
<linearGradient id="grow" x1="0" y1="0" x2="0" y2="1">
|
||||
<stop offset="0%" stopColor="#38bdf8" stopOpacity="0.45" />
|
||||
<stop offset="100%" stopColor="#38bdf8" stopOpacity="0.02" />
|
||||
</linearGradient>
|
||||
</defs>
|
||||
{[0.25, 0.5, 0.75].map((g) => (
|
||||
<line key={g} x1={0} x2={W} y1={(H - padB) * (1 - g) + padB * g - padB * g} y2={(H - padB) * (1 - g)} stroke="rgb(var(--border))" strokeOpacity="0.4" strokeWidth="0.5" />
|
||||
))}
|
||||
{data.map((d, i) => {
|
||||
const h = (d.bytes / maxDay) * (H - padB - 8)
|
||||
return <rect key={i} x={x(i) - bw / 2} y={(H - padB) - h} width={bw} height={h} fill="#a78bfa" opacity="0.22" rx="1" />
|
||||
})}
|
||||
<path d={area} fill="url(#grow)" />
|
||||
<path d={line} fill="none" stroke="#38bdf8" strokeWidth="2" vectorEffect="non-scaling-stroke" />
|
||||
{data.map((d, i) => (i % Math.ceil(n / 8) === 0 || i === n - 1) && (
|
||||
<circle key={i} cx={x(i)} cy={yC(d.cum_bytes)} r="2.5" fill="#38bdf8" />
|
||||
))}
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function Donut({ segments }: { segments: { label: string; value: number; color: string }[] }) {
|
||||
const total = segments.reduce((s, x) => s + x.value, 0) || 1
|
||||
const r = 52, c = 2 * Math.PI * r
|
||||
let acc = 0
|
||||
return (
|
||||
<svg viewBox="0 0 140 140" className="h-32 w-32 shrink-0">
|
||||
<g transform="rotate(-90 70 70)">
|
||||
<circle cx="70" cy="70" r={r} fill="none" stroke="rgb(var(--surface-overlay))" strokeWidth="16" />
|
||||
{segments.map((s, i) => {
|
||||
const frac = s.value / total
|
||||
const dash = frac * c
|
||||
const el = (
|
||||
<circle key={i} cx="70" cy="70" r={r} fill="none" stroke={s.color} strokeWidth="16"
|
||||
strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc * c} />
|
||||
)
|
||||
acc += frac
|
||||
return el
|
||||
})}
|
||||
</g>
|
||||
<text x="70" y="66" textAnchor="middle" className="fill-foreground" style={{ fontSize: 16, fontWeight: 700 }}>{segments.length}</text>
|
||||
<text x="70" y="82" textAnchor="middle" className="fill-foreground-faint" style={{ fontSize: 8 }}>buckets</text>
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function BarsH({ items, unit }: { items: { label: string; value: number; sub?: string }[]; unit?: 'bytes' | 'num' }) {
|
||||
const max = Math.max(...items.map((i) => i.value), 1)
|
||||
if (!items.length) return <Empty label="No data" />
|
||||
return (
|
||||
<div className="space-y-1.5">
|
||||
{items.map((it, i) => (
|
||||
<div key={it.label} className="group">
|
||||
<div className="mb-0.5 flex items-center justify-between text-[10px]">
|
||||
<span className="truncate font-mono text-foreground-muted">{it.label}</span>
|
||||
<span className="shrink-0 pl-2 text-foreground-faint">{it.sub ?? (unit === 'bytes' ? fmtBytes(it.value) : fmtNum(it.value))}</span>
|
||||
</div>
|
||||
<div className="h-2 overflow-hidden rounded-full bg-surface-overlay">
|
||||
<div className="h-full rounded-full transition-all" style={{ width: `${(it.value / max) * 100}%`, background: PALETTE[i % PALETTE.length] }} />
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function BarsV({ items }: { items: { label: string; count: number }[] }) {
|
||||
const max = Math.max(...items.map((i) => i.count), 1)
|
||||
return (
|
||||
<div className="flex h-full items-end gap-2 pt-2">
|
||||
{items.map((it, i) => (
|
||||
<div key={it.label} className="flex flex-1 flex-col items-center gap-1">
|
||||
<span className="text-[9px] font-medium text-foreground-muted">{it.count ? fmtNum(it.count) : ''}</span>
|
||||
<div className="flex w-full items-end" style={{ height: 96 }}>
|
||||
<div className="w-full rounded-t" style={{ height: `${Math.max(2, (it.count / max) * 100)}%`, background: PALETTE[i % PALETTE.length], opacity: it.count ? 0.85 : 0.25 }} />
|
||||
</div>
|
||||
<span className="text-center text-[8px] leading-tight text-foreground-faint">{it.label}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Sparkline({ values, color = '#34d399' }: { values: number[]; color?: string }) {
|
||||
const W = 280, H = 48
|
||||
const max = Math.max(...values, 1)
|
||||
const n = values.length
|
||||
const pts = values.map((v, i) => `${(i / (n - 1)) * W},${H - (v / max) * (H - 4) - 2}`).join(' ')
|
||||
const area = `0,${H} ${pts} ${W},${H}`
|
||||
return (
|
||||
<svg viewBox={`0 0 ${W} ${H}`} className="h-12 w-full" preserveAspectRatio="none">
|
||||
<polyline points={area} fill={color} fillOpacity="0.12" stroke="none" />
|
||||
<polyline points={pts} fill="none" stroke={color} strokeWidth="1.5" vectorEffect="non-scaling-stroke" />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
function Empty({ label }: { label: string }) {
|
||||
return <div className="flex h-full min-h-[80px] items-center justify-center text-[10px] text-foreground-faint">{label}</div>
|
||||
}
|
||||
|
||||
function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof HardDrive; label: string; value: string; sub?: string; accent: string }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-border bg-surface-raised p-3">
|
||||
<div className="mb-1 flex items-center gap-1.5 text-[9px] font-semibold uppercase tracking-wider text-foreground-faint">
|
||||
<Icon className="h-3 w-3" style={{ color: accent }} /> {label}
|
||||
</div>
|
||||
<div className="text-xl font-bold text-foreground">{value}</div>
|
||||
{sub && <div className="mt-0.5 text-[9px] text-foreground-faint">{sub}</div>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Panel({ title, icon: Icon, children, className, right }: { title: string; icon: typeof HardDrive; children: React.ReactNode; className?: string; right?: React.ReactNode }) {
|
||||
return (
|
||||
<div className={cn('rounded-lg border border-border bg-surface-raised p-3', className)}>
|
||||
<div className="mb-2 flex items-center justify-between">
|
||||
<h3 className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground"><Icon className="h-3.5 w-3.5 text-docker" /> {title}</h3>
|
||||
{right}
|
||||
</div>
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Main ────────────────────────────────────────────────────────────────── */
|
||||
|
||||
export function StorageView() {
|
||||
const [health, setHealth] = useState<{ ok: boolean; endpoint?: string; bucket_names?: string[]; error?: string } | null>(null)
|
||||
const [tab, setTab] = useState<'overview' | 'browser'>('overview')
|
||||
const [an, setAn] = useState<Analytics | null>(null)
|
||||
const [anLoading, setAnLoading] = useState(false)
|
||||
const [anErr, setAnErr] = useState<string | null>(null)
|
||||
|
||||
const loadAnalytics = useCallback(async (refresh = false) => {
|
||||
setAnLoading(true)
|
||||
setAnErr(null)
|
||||
try {
|
||||
const r = await fetch(`/api/storage/s3/analytics${refresh ? '?refresh=true' : ''}`)
|
||||
const j = await r.json()
|
||||
if (!r.ok || !j.ok) setAnErr(j.error || 'Analytics unavailable')
|
||||
else setAn(j)
|
||||
} catch {
|
||||
setAnErr('Storage analytics API unavailable')
|
||||
} finally {
|
||||
setAnLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadAnalytics()
|
||||
const t = setInterval(() => loadAnalytics(), 30000)
|
||||
return () => clearInterval(t)
|
||||
}, [loadAnalytics])
|
||||
|
||||
const s = an?.summary
|
||||
const bucketSegments = useMemo(
|
||||
() => (an?.buckets || []).filter((b) => b.bytes > 0).map((b, i) => ({ label: b.name, value: b.bytes, color: PALETTE[i % PALETTE.length] })),
|
||||
[an],
|
||||
)
|
||||
|
||||
return (
|
||||
<div className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<HardDrive className="h-4 w-4 text-docker" /> ObjectScale S3 Storage
|
||||
</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Dell ECS · {an?.endpoint || '10.0.20.111:9020'} · {s ? `${fmtNum(s.objects)} objects · ${s.size_human}` : 'live analytics'}
|
||||
{s?.truncated && <span className="ml-1 text-amber-300">(sampled {fmtNum(s.scanned || 0)})</span>}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<div className="flex rounded-md border border-border p-0.5">
|
||||
{(['overview', 'browser'] as const).map((t) => (
|
||||
<button key={t} type="button" onClick={() => setTab(t)}
|
||||
className={cn('inline-flex items-center gap-1 rounded px-2.5 py-1 text-[10px] font-medium capitalize', tab === t ? subTabActive : subTabIdle)}>
|
||||
{t === 'overview' ? <LayoutGrid className="h-3 w-3" /> : <Folder className="h-3 w-3" />} {t}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<a href="/jupyter/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-3 py-1.5 text-[11px] font-medium', subTabActive)}>
|
||||
<ExternalLink className="h-3 w-3" /> Open Jupyter
|
||||
</a>
|
||||
<button type="button" onClick={() => loadAnalytics(true)} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
<RefreshCw className={cn('inline h-3 w-3', anLoading && 'animate-spin')} /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{tab === 'overview' ? (
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-3">
|
||||
{anErr && <p className="mb-2 text-[11px] text-danger">{anErr}</p>}
|
||||
{!an && anLoading && <p className="flex items-center gap-2 p-8 text-[11px] text-foreground-muted"><Loader2 className="h-4 w-4 animate-spin" /> Crunching storage analytics…</p>}
|
||||
|
||||
{an && (
|
||||
<div className="space-y-3">
|
||||
{/* KPI row */}
|
||||
<div className="grid grid-cols-2 gap-2 md:grid-cols-3 xl:grid-cols-6">
|
||||
<Kpi icon={HardDrive} label="Total size" value={s?.size_human || '—'} sub={`${fmtNum(s?.objects || 0)} objects`} accent="#38bdf8" />
|
||||
<Kpi icon={Files} label="Objects" value={fmtNum(s?.objects || 0)} sub={`avg ${s?.avg_object_human}`} accent="#34d399" />
|
||||
<Kpi icon={Boxes} label="Buckets" value={String(s?.buckets || 0)} sub={`${bucketSegments.length} with data`} accent="#a78bfa" />
|
||||
<Kpi icon={Gauge} label="Largest object" value={s?.largest_human || '—'} accent="#fbbf24" />
|
||||
<Kpi icon={Activity} label="API ops / hr" value={fmtNum(an.activity?.total_last_hour || 0)} sub="storage requests" accent="#fb7185" />
|
||||
<Kpi icon={Clock} label="Last write" value={s?.newest ? new Date(s.newest).toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }) : '—'} accent="#22d3ee" />
|
||||
</div>
|
||||
|
||||
{/* Growth + bucket distribution */}
|
||||
<div className="grid grid-cols-1 gap-3 xl:grid-cols-3">
|
||||
<Panel title="Data growth (cumulative size · daily ingest)" icon={TrendingUp} className="xl:col-span-2"
|
||||
right={<span className="text-[9px] text-foreground-faint">{an.growth?.length || 0} days · since {s?.oldest || '—'}</span>}>
|
||||
<div className="h-[200px]"><GrowthChart data={an.growth || []} /></div>
|
||||
</Panel>
|
||||
<Panel title="Storage by bucket" icon={Database}>
|
||||
<div className="flex items-center gap-3">
|
||||
<Donut segments={bucketSegments} />
|
||||
<div className="min-w-0 flex-1 space-y-1.5">
|
||||
{(an.buckets || []).map((b, i) => (
|
||||
<div key={b.name} className="flex items-center justify-between gap-2 text-[10px]">
|
||||
<span className="flex min-w-0 items-center gap-1.5">
|
||||
<span className="h-2 w-2 shrink-0 rounded-full" style={{ background: PALETTE[i % PALETTE.length] }} />
|
||||
<span className="truncate font-medium text-foreground">{b.name}</span>
|
||||
</span>
|
||||
<span className="shrink-0 text-foreground-faint">{b.size_human} · {b.pct}%</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</Panel>
|
||||
</div>
|
||||
|
||||
{/* types + size hist + activity */}
|
||||
<div className="grid grid-cols-1 gap-3 lg:grid-cols-3">
|
||||
<Panel title="By file type" icon={BarChart3}>
|
||||
<BarsH items={(an.types || []).map((t) => ({ label: t.ext, value: t.bytes, sub: `${fmtNum(t.objects)} · ${t.size_human}` }))} unit="bytes" />
|
||||
</Panel>
|
||||
<Panel title="Object size distribution" icon={BarChart3}>
|
||||
<div className="h-[150px]"><BarsV items={an.size_histogram || []} /></div>
|
||||
</Panel>
|
||||
<Panel title="Storage API activity (60 min)" icon={Activity}
|
||||
right={<span className="text-[9px] text-foreground-faint">{an.activity?.total_last_hour || 0} ops</span>}>
|
||||
<Sparkline values={an.activity?.per_minute || []} />
|
||||
<div className="mt-2 flex flex-wrap gap-1">
|
||||
{(an.activity?.by_op || []).map((o) => (
|
||||
<span key={o.op} className="rounded border border-border bg-surface-overlay px-1.5 py-0.5 font-mono text-[9px] text-foreground-muted">
|
||||
{o.op} <span className="text-foreground">{o.count}</span>
|
||||
</span>
|
||||
))}
|
||||
{!an.activity?.by_op.length && <span className="text-[9px] text-foreground-faint">No requests yet.</span>}
|
||||
</div>
|
||||
</Panel>
|
||||
</div>
|
||||
|
||||
{/* top prefixes + largest + recent */}
|
||||
<div className="grid grid-cols-1 gap-3 lg:grid-cols-3">
|
||||
<Panel title="Top folders" icon={Folder}>
|
||||
<BarsH items={(an.top_prefixes || []).map((p) => ({ label: p.prefix, value: p.bytes, sub: p.size_human }))} unit="bytes" />
|
||||
</Panel>
|
||||
<Panel title="Largest objects" icon={Gauge}>
|
||||
<div className="space-y-1">
|
||||
{(an.largest_objects || []).map((o) => (
|
||||
<div key={`${o.bucket}/${o.key}`} className="flex items-center justify-between gap-2 text-[10px]">
|
||||
<span className="truncate font-mono text-foreground-muted" title={`${o.bucket}/${o.key}`}>{o.key.split('/').pop()}</span>
|
||||
<span className="shrink-0 text-foreground-faint">{o.size_human}</span>
|
||||
</div>
|
||||
))}
|
||||
{!an.largest_objects?.length && <Empty label="No objects" />}
|
||||
</div>
|
||||
</Panel>
|
||||
<Panel title="Recent uploads" icon={Clock}>
|
||||
<div className="space-y-1">
|
||||
{(an.recent || []).map((o, i) => (
|
||||
<div key={i} className="flex items-center justify-between gap-2 text-[10px]">
|
||||
<span className="truncate font-mono text-foreground-muted" title={`${o.bucket}/${o.key}`}>{o.key.split('/').pop()}</span>
|
||||
<span className="shrink-0 text-foreground-faint">{o.modified?.slice(5, 16).replace('T', ' ')}</span>
|
||||
</div>
|
||||
))}
|
||||
{!an.recent?.length && <Empty label="No recent activity" />}
|
||||
</div>
|
||||
</Panel>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<BucketBrowser />
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Browser (the original explorer, now a tab) ──────────────────────────── */
|
||||
|
||||
function BucketBrowser() {
|
||||
const [buckets, setBuckets] = useState<Bucket[]>([])
|
||||
const [bucket, setBucket] = useState<string | null>(null)
|
||||
const [prefix, setPrefix] = useState('')
|
||||
@@ -17,179 +359,98 @@ export function StorageView() {
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const loadBuckets = useCallback(async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
setLoading(true); setError(null)
|
||||
try {
|
||||
const [h, b] = await Promise.all([
|
||||
fetch('/api/storage/s3/health'),
|
||||
fetch('/api/storage/s3/buckets'),
|
||||
])
|
||||
if (h.ok) setHealth(await h.json())
|
||||
const b = await fetch('/api/storage/s3/buckets')
|
||||
if (b.ok) {
|
||||
const j = await b.json()
|
||||
setBuckets(j.buckets || [])
|
||||
if (!bucket && j.buckets?.length) setBucket(j.buckets[0].name)
|
||||
} else {
|
||||
setError('Failed to load buckets')
|
||||
}
|
||||
} catch {
|
||||
setError('S3 API unavailable')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
} else setError('Failed to load buckets')
|
||||
} catch { setError('S3 API unavailable') } finally { setLoading(false) }
|
||||
}, [bucket])
|
||||
|
||||
const loadObjects = useCallback(async (b: string, p: string) => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
setLoading(true); setError(null)
|
||||
try {
|
||||
const r = await fetch(`/api/storage/s3/buckets/${encodeURIComponent(b)}/objects?prefix=${encodeURIComponent(p)}`)
|
||||
const j = await r.json()
|
||||
if (!r.ok || !j.ok) {
|
||||
setError(j.error || 'List failed')
|
||||
return
|
||||
}
|
||||
setFolders(j.folders || [])
|
||||
setObjects(j.objects || [])
|
||||
} catch {
|
||||
setError('Failed to list objects')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
if (!r.ok || !j.ok) { setError(j.error || 'List failed'); return }
|
||||
setFolders(j.folders || []); setObjects(j.objects || [])
|
||||
} catch { setError('Failed to list objects') } finally { setLoading(false) }
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadBuckets()
|
||||
}, [loadBuckets])
|
||||
|
||||
useEffect(() => {
|
||||
if (bucket) loadObjects(bucket, prefix)
|
||||
}, [bucket, prefix, loadObjects])
|
||||
useEffect(() => { loadBuckets() }, [loadBuckets])
|
||||
useEffect(() => { if (bucket) loadObjects(bucket, prefix) }, [bucket, prefix, loadObjects])
|
||||
|
||||
const crumbs = prefix ? prefix.split('/').filter(Boolean) : []
|
||||
|
||||
return (
|
||||
<div className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<HardDrive className="h-4 w-4 text-docker" />
|
||||
ObjectScale S3 Storage
|
||||
</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Dell ECS · {health?.endpoint || '10.0.20.111:9020'} · live bucket browser
|
||||
</p>
|
||||
<div className="flex min-h-0 flex-1 flex-col lg:flex-row">
|
||||
<aside className="shrink-0 border-b border-border p-3 lg:w-52 lg:border-b-0 lg:border-r">
|
||||
<h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Buckets</h3>
|
||||
<div className="space-y-1">
|
||||
{buckets.map((b) => (
|
||||
<button key={b.name} type="button" onClick={() => { setBucket(b.name); setPrefix('') }}
|
||||
className={cn('flex w-full items-center gap-2 rounded border px-2 py-1.5 text-left text-[10px]',
|
||||
bucket === b.name ? 'border-docker/40 bg-docker/10' : 'border-border hover:bg-surface-overlay')}>
|
||||
<Database className="h-3 w-3 shrink-0 text-docker" />
|
||||
<span className="truncate font-medium">{b.name}</span>
|
||||
</button>
|
||||
))}
|
||||
{buckets.length === 0 && !loading && <p className="text-[9px] text-foreground-faint">No buckets or access denied.</p>}
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<a href="/jupyter/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-3 py-1.5 text-[11px] font-medium', subTabActive)}>
|
||||
<ExternalLink className="h-3 w-3" /> Open Jupyter
|
||||
</a>
|
||||
<button type="button" onClick={() => { loadBuckets(); if (bucket) loadObjects(bucket, prefix) }} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
<RefreshCw className={cn('inline h-3 w-3', loading && 'animate-spin')} /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
</aside>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col lg:flex-row">
|
||||
<aside className="shrink-0 border-b border-border p-3 lg:w-52 lg:border-b-0 lg:border-r">
|
||||
<h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Buckets</h3>
|
||||
<div className="space-y-1">
|
||||
{buckets.map((b) => (
|
||||
<button
|
||||
key={b.name}
|
||||
type="button"
|
||||
onClick={() => { setBucket(b.name); setPrefix('') }}
|
||||
className={cn(
|
||||
'flex w-full items-center gap-2 rounded border px-2 py-1.5 text-left text-[10px]',
|
||||
bucket === b.name ? 'border-docker/40 bg-docker/10' : 'border-border hover:bg-surface-overlay',
|
||||
)}
|
||||
>
|
||||
<Database className="h-3 w-3 shrink-0 text-docker" />
|
||||
<span className="truncate font-medium">{b.name}</span>
|
||||
</button>
|
||||
<div className="flex min-h-0 flex-1 flex-col p-3">
|
||||
{bucket && (
|
||||
<nav className="mb-2 flex flex-wrap items-center gap-1 text-[10px] text-foreground-muted">
|
||||
<button type="button" className="hover:text-docker" onClick={() => setPrefix('')}>{bucket}</button>
|
||||
{crumbs.map((c, i) => (
|
||||
<span key={i} className="inline-flex items-center gap-1">
|
||||
<ChevronRight className="h-3 w-3" />
|
||||
<button type="button" className="hover:text-docker" onClick={() => setPrefix(crumbs.slice(0, i + 1).join('/') + '/')}>{c}</button>
|
||||
</span>
|
||||
))}
|
||||
{buckets.length === 0 && !loading && (
|
||||
<p className="text-[9px] text-foreground-faint">No buckets or access denied.</p>
|
||||
)}
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col p-3">
|
||||
{bucket && (
|
||||
<nav className="mb-2 flex flex-wrap items-center gap-1 text-[10px] text-foreground-muted">
|
||||
<button type="button" className="hover:text-docker" onClick={() => setPrefix('')}>{bucket}</button>
|
||||
{crumbs.map((c, i) => (
|
||||
<span key={i} className="inline-flex items-center gap-1">
|
||||
<ChevronRight className="h-3 w-3" />
|
||||
<button
|
||||
type="button"
|
||||
className="hover:text-docker"
|
||||
onClick={() => setPrefix(crumbs.slice(0, i + 1).join('/') + '/')}
|
||||
>
|
||||
{c}
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</nav>
|
||||
)}
|
||||
|
||||
{loading && (
|
||||
<p className="flex items-center gap-2 text-[11px] text-foreground-muted">
|
||||
<Loader2 className="h-4 w-4 animate-spin" /> Loading…
|
||||
</p>
|
||||
)}
|
||||
{error && <p className="mb-2 text-[11px] text-danger">{error}</p>}
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
|
||||
<table className="w-full text-left text-[11px]">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
|
||||
<th className="py-1.5 pr-2">Name</th>
|
||||
<th className="py-1.5 pr-2">Size</th>
|
||||
<th className="py-1.5 pr-2">Modified</th>
|
||||
<th className="py-1.5" />
|
||||
</nav>
|
||||
)}
|
||||
{loading && <p className="flex items-center gap-2 text-[11px] text-foreground-muted"><Loader2 className="h-4 w-4 animate-spin" /> Loading…</p>}
|
||||
{error && <p className="mb-2 text-[11px] text-danger">{error}</p>}
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
|
||||
<table className="w-full text-left text-[11px]">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
|
||||
<th className="py-1.5 pr-2">Name</th><th className="py-1.5 pr-2">Size</th><th className="py-1.5 pr-2">Modified</th><th className="py-1.5" />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{folders.map((f) => (
|
||||
<tr key={f.prefix} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
||||
<td className="py-1.5 pr-2">
|
||||
<button type="button" className="inline-flex items-center gap-1 font-medium text-docker hover:underline" onClick={() => setPrefix(f.prefix || '')}>
|
||||
<Folder className="h-3.5 w-3.5" /> {f.name}/
|
||||
</button>
|
||||
</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-faint">—</td><td className="py-1.5 pr-2 text-foreground-faint">—</td><td />
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{folders.map((f) => (
|
||||
<tr key={f.prefix} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
||||
<td className="py-1.5 pr-2">
|
||||
<button
|
||||
type="button"
|
||||
className="inline-flex items-center gap-1 font-medium text-docker hover:underline"
|
||||
onClick={() => setPrefix(f.prefix || '')}
|
||||
>
|
||||
<Folder className="h-3.5 w-3.5" /> {f.name}/
|
||||
</button>
|
||||
</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-faint">—</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-faint">—</td>
|
||||
<td />
|
||||
</tr>
|
||||
))}
|
||||
{objects.map((o) => (
|
||||
<tr key={o.key} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
||||
<td className="max-w-[240px] truncate py-1.5 pr-2 font-mono text-[10px]">{o.name || o.key}</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-muted">{o.size_human}</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-faint">{o.modified?.slice(0, 19) || '—'}</td>
|
||||
<td className="py-1.5">
|
||||
{o.key && bucket && (
|
||||
<a
|
||||
href={`/api/storage/s3/buckets/${encodeURIComponent(bucket)}/download?key=${encodeURIComponent(o.key)}`}
|
||||
className="inline-flex items-center gap-0.5 text-docker hover:underline"
|
||||
>
|
||||
<Download className="h-3 w-3" />
|
||||
</a>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!loading && folders.length === 0 && objects.length === 0 && bucket && (
|
||||
<p className="py-8 text-center text-sm text-foreground-muted">This prefix is empty.</p>
|
||||
)}
|
||||
</div>
|
||||
))}
|
||||
{objects.map((o) => (
|
||||
<tr key={o.key} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
||||
<td className="max-w-[240px] truncate py-1.5 pr-2 font-mono text-[10px]">{o.name || o.key}</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-muted">{o.size_human}</td>
|
||||
<td className="py-1.5 pr-2 text-foreground-faint">{o.modified?.slice(0, 19) || '—'}</td>
|
||||
<td className="py-1.5">
|
||||
{o.key && bucket && (
|
||||
<a href={`/api/storage/s3/buckets/${encodeURIComponent(bucket)}/download?key=${encodeURIComponent(o.key)}`} className="inline-flex items-center gap-0.5 text-docker hover:underline">
|
||||
<Download className="h-3 w-3" />
|
||||
</a>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
{!loading && folders.length === 0 && objects.length === 0 && bucket && <p className="py-8 text-center text-sm text-foreground-muted">This prefix is empty.</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user