feat(storage): rich Object Storage analytics dashboard
Add /api/storage/s3/analytics endpoint that scans buckets (bounded + cached) to compute total size/objects, per-bucket distribution, file-type and size-class breakdowns, cumulative data-growth timeline and largest / recent objects. Track every S3 op routed through the API for a live storage-activity timeline. Rebuild StorageView into a tabbed view: Overview (KPI cards + SVG charts: growth area, bucket donut, type/size bars, activity sparkline, top folders, largest & recent objects) and the original bucket Browser.
This commit is contained in:
mo
@@ -3,6 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
@@ -18,6 +21,47 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
||||
|
||||
router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
|
||||
|
||||
# Bounded scan so the dashboard stays responsive on big buckets.
|
||||
SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "50000"))
|
||||
SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "12"))
|
||||
|
||||
# In-process activity log — every S3 op routed through this API is recorded so
|
||||
# the dashboard can show live "what is happening on the storage" timelines.
|
||||
_activity: deque[tuple[float, str]] = deque(maxlen=20000)
|
||||
_analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
_ANALYTICS_TTL = 45.0
|
||||
|
||||
|
||||
def _track(op: str) -> None:
|
||||
try:
|
||||
_activity.append((time.time(), op))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _ext(key: str) -> str:
|
||||
base = key.rsplit("/", 1)[-1]
|
||||
if "." in base:
|
||||
e = base.rsplit(".", 1)[-1].lower()
|
||||
if 1 <= len(e) <= 8 and e.isalnum():
|
||||
return e
|
||||
return "(none)"
|
||||
|
||||
|
||||
def _size_class(n: int) -> str:
|
||||
kb, mb, gb = 1024, 1024 ** 2, 1024 ** 3
|
||||
if n < kb:
|
||||
return "<1 KB"
|
||||
if n < mb:
|
||||
return "1 KB–1 MB"
|
||||
if n < 10 * mb:
|
||||
return "1–10 MB"
|
||||
if n < 100 * mb:
|
||||
return "10–100 MB"
|
||||
if n < gb:
|
||||
return "100 MB–1 GB"
|
||||
return ">1 GB"
|
||||
|
||||
|
||||
def _client():
|
||||
return boto3.client(
|
||||
@@ -41,6 +85,7 @@ def _human_size(n: int) -> str:
|
||||
@router.get("/health")
|
||||
async def s3_health():
|
||||
try:
|
||||
_track("health")
|
||||
s3 = _client()
|
||||
buckets = s3.list_buckets()
|
||||
names = [b["Name"] for b in buckets.get("Buckets", [])]
|
||||
@@ -57,6 +102,7 @@ async def s3_health():
|
||||
@router.get("/buckets")
|
||||
async def list_buckets():
|
||||
try:
|
||||
_track("list_buckets")
|
||||
s3 = _client()
|
||||
resp = s3.list_buckets()
|
||||
items = []
|
||||
@@ -84,6 +130,7 @@ async def list_objects(
|
||||
max_keys: int = Query(200, le=500),
|
||||
):
|
||||
try:
|
||||
_track("list_objects")
|
||||
s3 = _client()
|
||||
resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys)
|
||||
folders = [
|
||||
@@ -119,6 +166,7 @@ async def list_objects(
|
||||
@router.get("/buckets/{bucket}/download")
|
||||
async def download_object(bucket: str, key: str = Query(...)):
|
||||
try:
|
||||
_track("download")
|
||||
s3 = _client()
|
||||
obj = s3.get_object(Bucket=bucket, Key=key)
|
||||
body = obj["Body"]
|
||||
@@ -136,3 +184,176 @@ async def download_object(bucket: str, key: str = Query(...)):
|
||||
)
|
||||
except ClientError as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
|
||||
|
||||
|
||||
|
||||
def _scan_analytics() -> dict[str, Any]:
|
||||
s3 = _client()
|
||||
deadline = time.time() + SCAN_DEADLINE_S
|
||||
resp = s3.list_buckets()
|
||||
bucket_objs = resp.get("Buckets", [])
|
||||
created_map = {
|
||||
b["Name"]: (b.get("CreationDate").isoformat() if b.get("CreationDate") else None)
|
||||
for b in bucket_objs
|
||||
}
|
||||
names = [b["Name"] for b in bucket_objs]
|
||||
|
||||
total_bytes = 0
|
||||
total_objects = 0
|
||||
by_bucket: dict[str, list[int]] = {}
|
||||
by_ext: dict[str, list[int]] = {}
|
||||
by_size: dict[str, int] = {}
|
||||
by_day: dict[str, list[int]] = {}
|
||||
by_prefix: dict[str, list[int]] = {}
|
||||
largest: list[tuple[int, str, str, str | None]] = []
|
||||
recent: list[tuple[str, str, str, int]] = []
|
||||
truncated = False
|
||||
scanned = 0
|
||||
|
||||
for name in names:
|
||||
bc = bb = 0
|
||||
token = None
|
||||
while True:
|
||||
if time.time() > deadline or scanned >= SCAN_MAX_OBJECTS:
|
||||
truncated = True
|
||||
break
|
||||
kw: dict[str, Any] = {"Bucket": name, "MaxKeys": 1000}
|
||||
if token:
|
||||
kw["ContinuationToken"] = token
|
||||
try:
|
||||
r = s3.list_objects_v2(**kw)
|
||||
except Exception:
|
||||
break
|
||||
for o in r.get("Contents", []):
|
||||
sz = int(o.get("Size", 0) or 0)
|
||||
key = o["Key"]
|
||||
lm = o.get("LastModified")
|
||||
bc += 1
|
||||
bb += sz
|
||||
scanned += 1
|
||||
total_objects += 1
|
||||
total_bytes += sz
|
||||
e = by_ext.setdefault(_ext(key), [0, 0])
|
||||
e[0] += 1
|
||||
e[1] += sz
|
||||
sc = _size_class(sz)
|
||||
by_size[sc] = by_size.get(sc, 0) + 1
|
||||
if lm:
|
||||
d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d")
|
||||
dd = by_day.setdefault(d, [0, 0])
|
||||
dd[0] += 1
|
||||
dd[1] += sz
|
||||
recent.append((lm.isoformat(), name, key, sz))
|
||||
top = key.split("/", 1)[0] if "/" in key else "(root)"
|
||||
pp = by_prefix.setdefault(f"{name}/{top}", [0, 0])
|
||||
pp[0] += 1
|
||||
pp[1] += sz
|
||||
largest.append((sz, name, key, lm.isoformat() if lm else None))
|
||||
if scanned >= SCAN_MAX_OBJECTS:
|
||||
truncated = True
|
||||
break
|
||||
if r.get("IsTruncated") and not (time.time() > deadline or scanned >= SCAN_MAX_OBJECTS):
|
||||
token = r.get("NextContinuationToken")
|
||||
if not token:
|
||||
break
|
||||
else:
|
||||
break
|
||||
by_bucket[name] = [bc, bb]
|
||||
|
||||
# cumulative growth timeline
|
||||
growth = []
|
||||
cum_b = cum_o = 0
|
||||
for d in sorted(by_day):
|
||||
c, b = by_day[d]
|
||||
cum_o += c
|
||||
cum_b += b
|
||||
growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b})
|
||||
|
||||
buckets_out = sorted(
|
||||
([{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]),
|
||||
"pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0,
|
||||
"created": created_map.get(n)} for n, v in by_bucket.items()]),
|
||||
key=lambda x: x["bytes"], reverse=True)
|
||||
|
||||
types_out = sorted(
|
||||
([{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||
for k, v in by_ext.items()]), key=lambda x: x["bytes"], reverse=True)[:12]
|
||||
|
||||
size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"]
|
||||
size_out = [{"label": k, "count": by_size.get(k, 0)} for k in size_order]
|
||||
|
||||
prefixes_out = sorted(
|
||||
([{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||
for k, v in by_prefix.items()]), key=lambda x: x["bytes"], reverse=True)[:10]
|
||||
|
||||
largest.sort(key=lambda x: x[0], reverse=True)
|
||||
largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m}
|
||||
for s, b, k, m in largest[:10]]
|
||||
|
||||
recent.sort(key=lambda x: x[0], reverse=True)
|
||||
recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)}
|
||||
for m, b, k, s in recent[:15]]
|
||||
|
||||
return {
|
||||
"summary": {
|
||||
"buckets": len(names),
|
||||
"objects": total_objects,
|
||||
"bytes": total_bytes,
|
||||
"size_human": _human_size(total_bytes),
|
||||
"avg_object_bytes": int(total_bytes / total_objects) if total_objects else 0,
|
||||
"avg_object_human": _human_size(int(total_bytes / total_objects) if total_objects else 0),
|
||||
"largest_human": largest_out[0]["size_human"] if largest_out else "0 B",
|
||||
"newest": recent_out[0]["modified"] if recent_out else None,
|
||||
"oldest": growth[0]["date"] if growth else None,
|
||||
"truncated": truncated,
|
||||
"scanned": scanned,
|
||||
},
|
||||
"buckets": buckets_out,
|
||||
"types": types_out,
|
||||
"size_histogram": size_out,
|
||||
"growth": growth,
|
||||
"top_prefixes": prefixes_out,
|
||||
"largest_objects": largest_out,
|
||||
"recent": recent_out,
|
||||
}
|
||||
|
||||
|
||||
def _activity_view() -> dict[str, Any]:
|
||||
now = time.time()
|
||||
cutoff = now - 3600
|
||||
mins = [0] * 60
|
||||
by_op: dict[str, int] = {}
|
||||
total = 0
|
||||
for ts, op in list(_activity):
|
||||
if ts < cutoff:
|
||||
continue
|
||||
total += 1
|
||||
by_op[op] = by_op.get(op, 0) + 1
|
||||
idx = int((now - ts) // 60)
|
||||
if 0 <= idx < 60:
|
||||
mins[59 - idx] += 1
|
||||
last_ts = _activity[-1][0] if _activity else None
|
||||
return {
|
||||
"per_minute": mins,
|
||||
"by_op": [{"op": k, "count": v} for k, v in sorted(by_op.items(), key=lambda x: -x[1])],
|
||||
"total_last_hour": total,
|
||||
"last_activity": datetime.fromtimestamp(last_ts, tz=timezone.utc).isoformat() if last_ts else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/analytics")
|
||||
async def analytics(refresh: bool = Query(False)):
|
||||
"""Aggregated storage analytics for the dashboard (cached ~45s)."""
|
||||
_track("analytics")
|
||||
now = time.time()
|
||||
if not refresh and _analytics_cache["data"] is not None and now - _analytics_cache["ts"] < _ANALYTICS_TTL:
|
||||
data = _analytics_cache["data"]
|
||||
else:
|
||||
try:
|
||||
data = _scan_analytics()
|
||||
_analytics_cache["data"] = data
|
||||
_analytics_cache["ts"] = now
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc), "endpoint": S3_ENDPOINT}, status_code=502)
|
||||
return {"ok": True, "endpoint": S3_ENDPOINT, "generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"activity": _activity_view(), **data}
|
||||
|
||||
Reference in New Issue
Block a user