feat(storage): rich Object Storage analytics dashboard

Add /api/storage/s3/analytics endpoint that scans buckets (bounded +
cached) to compute total size/objects, per-bucket distribution, file-type
and size-class breakdowns, cumulative data-growth timeline and largest /
recent objects. Track every S3 op routed through the API for a live
storage-activity timeline.

Rebuild StorageView into a tabbed view: Overview (KPI cards + SVG charts:
growth area, bucket donut, type/size bars, activity sparkline, top folders,
largest & recent objects) and the original bucket Browser.

api/storage_s3.py

@@ -3,6 +3,9 @@
 from __future__ import annotations
 
 import os
+import time
+from collections import deque
+from datetime import datetime, timezone
 from typing import Any
 
 import boto3
@@ -18,6 +21,47 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1")
 
 router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
 
+# Bounded scan so the dashboard stays responsive on big buckets.
+SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "50000"))
+SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "12"))
+
+# In-process activity log — every S3 op routed through this API is recorded so
+# the dashboard can show live "what is happening on the storage" timelines.
+_activity: deque[tuple[float, str]] = deque(maxlen=20000)
+_analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
+_ANALYTICS_TTL = 45.0
+
+
+def _track(op: str) -> None:
+    try:
+        _activity.append((time.time(), op))
+    except Exception:
+        pass
+
+
+def _ext(key: str) -> str:
+    base = key.rsplit("/", 1)[-1]
+    if "." in base:
+        e = base.rsplit(".", 1)[-1].lower()
+        if 1 <= len(e) <= 8 and e.isalnum():
+            return e
+    return "(none)"
+
+
+def _size_class(n: int) -> str:
+    kb, mb, gb = 1024, 1024 ** 2, 1024 ** 3
+    if n < kb:
+        return "<1 KB"
+    if n < mb:
+        return "1 KB–1 MB"
+    if n < 10 * mb:
+        return "1–10 MB"
+    if n < 100 * mb:
+        return "10–100 MB"
+    if n < gb:
+        return "100 MB–1 GB"
+    return ">1 GB"
+
 
 def _client():
     return boto3.client(
@@ -41,6 +85,7 @@ def _human_size(n: int) -> str:
 @router.get("/health")
 async def s3_health():
     try:
+        _track("health")
         s3 = _client()
         buckets = s3.list_buckets()
         names = [b["Name"] for b in buckets.get("Buckets", [])]
@@ -57,6 +102,7 @@ async def s3_health():
 @router.get("/buckets")
 async def list_buckets():
     try:
+        _track("list_buckets")
         s3 = _client()
         resp = s3.list_buckets()
         items = []
@@ -84,6 +130,7 @@ async def list_objects(
     max_keys: int = Query(200, le=500),
 ):
     try:
+        _track("list_objects")
         s3 = _client()
         resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys)
         folders = [
@@ -119,6 +166,7 @@ async def list_objects(
 @router.get("/buckets/{bucket}/download")
 async def download_object(bucket: str, key: str = Query(...)):
     try:
+        _track("download")
         s3 = _client()
         obj = s3.get_object(Bucket=bucket, Key=key)
         body = obj["Body"]
@@ -136,3 +184,176 @@ async def download_object(bucket: str, key: str = Query(...)):
         )
     except ClientError as exc:
         return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
+
+
+
+def _scan_analytics() -> dict[str, Any]:
+    s3 = _client()
+    deadline = time.time() + SCAN_DEADLINE_S
+    resp = s3.list_buckets()
+    bucket_objs = resp.get("Buckets", [])
+    created_map = {
+        b["Name"]: (b.get("CreationDate").isoformat() if b.get("CreationDate") else None)
+        for b in bucket_objs
+    }
+    names = [b["Name"] for b in bucket_objs]
+
+    total_bytes = 0
+    total_objects = 0
+    by_bucket: dict[str, list[int]] = {}
+    by_ext: dict[str, list[int]] = {}
+    by_size: dict[str, int] = {}
+    by_day: dict[str, list[int]] = {}
+    by_prefix: dict[str, list[int]] = {}
+    largest: list[tuple[int, str, str, str | None]] = []
+    recent: list[tuple[str, str, str, int]] = []
+    truncated = False
+    scanned = 0
+
+    for name in names:
+        bc = bb = 0
+        token = None
+        while True:
+            if time.time() > deadline or scanned >= SCAN_MAX_OBJECTS:
+                truncated = True
+                break
+            kw: dict[str, Any] = {"Bucket": name, "MaxKeys": 1000}
+            if token:
+                kw["ContinuationToken"] = token
+            try:
+                r = s3.list_objects_v2(**kw)
+            except Exception:
+                break
+            for o in r.get("Contents", []):
+                sz = int(o.get("Size", 0) or 0)
+                key = o["Key"]
+                lm = o.get("LastModified")
+                bc += 1
+                bb += sz
+                scanned += 1
+                total_objects += 1
+                total_bytes += sz
+                e = by_ext.setdefault(_ext(key), [0, 0])
+                e[0] += 1
+                e[1] += sz
+                sc = _size_class(sz)
+                by_size[sc] = by_size.get(sc, 0) + 1
+                if lm:
+                    d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d")
+                    dd = by_day.setdefault(d, [0, 0])
+                    dd[0] += 1
+                    dd[1] += sz
+                    recent.append((lm.isoformat(), name, key, sz))
+                top = key.split("/", 1)[0] if "/" in key else "(root)"
+                pp = by_prefix.setdefault(f"{name}/{top}", [0, 0])
+                pp[0] += 1
+                pp[1] += sz
+                largest.append((sz, name, key, lm.isoformat() if lm else None))
+                if scanned >= SCAN_MAX_OBJECTS:
+                    truncated = True
+                    break
+            if r.get("IsTruncated") and not (time.time() > deadline or scanned >= SCAN_MAX_OBJECTS):
+                token = r.get("NextContinuationToken")
+                if not token:
+                    break
+            else:
+                break
+        by_bucket[name] = [bc, bb]
+
+    # cumulative growth timeline
+    growth = []
+    cum_b = cum_o = 0
+    for d in sorted(by_day):
+        c, b = by_day[d]
+        cum_o += c
+        cum_b += b
+        growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b})
+
+    buckets_out = sorted(
+        ([{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]),
+           "pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0,
+           "created": created_map.get(n)} for n, v in by_bucket.items()]),
+        key=lambda x: x["bytes"], reverse=True)
+
+    types_out = sorted(
+        ([{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
+          for k, v in by_ext.items()]), key=lambda x: x["bytes"], reverse=True)[:12]
+
+    size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"]
+    size_out = [{"label": k, "count": by_size.get(k, 0)} for k in size_order]
+
+    prefixes_out = sorted(
+        ([{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
+          for k, v in by_prefix.items()]), key=lambda x: x["bytes"], reverse=True)[:10]
+
+    largest.sort(key=lambda x: x[0], reverse=True)
+    largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m}
+                   for s, b, k, m in largest[:10]]
+
+    recent.sort(key=lambda x: x[0], reverse=True)
+    recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)}
+                  for m, b, k, s in recent[:15]]
+
+    return {
+        "summary": {
+            "buckets": len(names),
+            "objects": total_objects,
+            "bytes": total_bytes,
+            "size_human": _human_size(total_bytes),
+            "avg_object_bytes": int(total_bytes / total_objects) if total_objects else 0,
+            "avg_object_human": _human_size(int(total_bytes / total_objects) if total_objects else 0),
+            "largest_human": largest_out[0]["size_human"] if largest_out else "0 B",
+            "newest": recent_out[0]["modified"] if recent_out else None,
+            "oldest": growth[0]["date"] if growth else None,
+            "truncated": truncated,
+            "scanned": scanned,
+        },
+        "buckets": buckets_out,
+        "types": types_out,
+        "size_histogram": size_out,
+        "growth": growth,
+        "top_prefixes": prefixes_out,
+        "largest_objects": largest_out,
+        "recent": recent_out,
+    }
+
+
+def _activity_view() -> dict[str, Any]:
+    now = time.time()
+    cutoff = now - 3600
+    mins = [0] * 60
+    by_op: dict[str, int] = {}
+    total = 0
+    for ts, op in list(_activity):
+        if ts < cutoff:
+            continue
+        total += 1
+        by_op[op] = by_op.get(op, 0) + 1
+        idx = int((now - ts) // 60)
+        if 0 <= idx < 60:
+            mins[59 - idx] += 1
+    last_ts = _activity[-1][0] if _activity else None
+    return {
+        "per_minute": mins,
+        "by_op": [{"op": k, "count": v} for k, v in sorted(by_op.items(), key=lambda x: -x[1])],
+        "total_last_hour": total,
+        "last_activity": datetime.fromtimestamp(last_ts, tz=timezone.utc).isoformat() if last_ts else None,
+    }
+
+
+@router.get("/analytics")
+async def analytics(refresh: bool = Query(False)):
+    """Aggregated storage analytics for the dashboard (cached ~45s)."""
+    _track("analytics")
+    now = time.time()
+    if not refresh and _analytics_cache["data"] is not None and now - _analytics_cache["ts"] < _ANALYTICS_TTL:
+        data = _analytics_cache["data"]
+    else:
+        try:
+            data = _scan_analytics()
+            _analytics_cache["data"] = data
+            _analytics_cache["ts"] = now
+        except Exception as exc:
+            return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502)
+    return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(),
+            "activity": _activity_view(), **data}