feat(storage): breadth-first analytics + recursive browse/search
Analytics now walks every top-level prefix with its own budget instead of a flat alphabetical scan, so a single huge prefix (kafka/ CDC json) no longer hides the rest — composition now correctly reflects images, logs, csv, parquet and avro, and reports true total size. Browser gains recursive (whole-subtree) listing, free-text name search, file-type filter chips, type column and Load-more pagination so every object across all folders is discoverable.
This commit is contained in:
+194
-104
@@ -21,9 +21,12 @@ S3_REGION = os.getenv("S3_REGION", "us-east-1")
|
|||||||
|
|
||||||
router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
|
router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
|
||||||
|
|
||||||
# Bounded scan so the dashboard stays responsive on big buckets.
|
# Bounded scan so the dashboard stays responsive on big / fast-growing buckets.
|
||||||
SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "50000"))
|
SCAN_MAX_OBJECTS = int(os.getenv("S3_SCAN_MAX_OBJECTS", "80000"))
|
||||||
SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "12"))
|
SCAN_DEADLINE_S = float(os.getenv("S3_SCAN_DEADLINE_S", "20"))
|
||||||
|
# Per top-level prefix cap so a single huge prefix (e.g. kafka/ CDC json) cannot
|
||||||
|
# starve the others — this keeps the type/size composition representative.
|
||||||
|
SCAN_PER_PREFIX = int(os.getenv("S3_SCAN_PER_PREFIX", "20000"))
|
||||||
|
|
||||||
# In-process activity log — every S3 op routed through this API is recorded so
|
# In-process activity log — every S3 op routed through this API is recorded so
|
||||||
# the dashboard can show live "what is happening on the storage" timelines.
|
# the dashboard can show live "what is happening on the storage" timelines.
|
||||||
@@ -32,6 +35,25 @@ _analytics_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
|||||||
_ANALYTICS_TTL = 45.0
|
_ANALYTICS_TTL = 45.0
|
||||||
|
|
||||||
|
|
||||||
|
def _client():
|
||||||
|
return boto3.client(
|
||||||
|
"s3",
|
||||||
|
endpoint_url=S3_ENDPOINT,
|
||||||
|
aws_access_key_id=S3_ACCESS_KEY,
|
||||||
|
aws_secret_access_key=S3_SECRET_KEY,
|
||||||
|
region_name=S3_REGION,
|
||||||
|
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _human_size(n: float) -> str:
|
||||||
|
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||||
|
if n < 1024:
|
||||||
|
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
|
||||||
|
n /= 1024
|
||||||
|
return f"{n:.1f} PB"
|
||||||
|
|
||||||
|
|
||||||
def _track(op: str) -> None:
|
def _track(op: str) -> None:
|
||||||
try:
|
try:
|
||||||
_activity.append((time.time(), op))
|
_activity.append((time.time(), op))
|
||||||
@@ -63,25 +85,6 @@ def _size_class(n: int) -> str:
|
|||||||
return ">1 GB"
|
return ">1 GB"
|
||||||
|
|
||||||
|
|
||||||
def _client():
|
|
||||||
return boto3.client(
|
|
||||||
"s3",
|
|
||||||
endpoint_url=S3_ENDPOINT,
|
|
||||||
aws_access_key_id=S3_ACCESS_KEY,
|
|
||||||
aws_secret_access_key=S3_SECRET_KEY,
|
|
||||||
region_name=S3_REGION,
|
|
||||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _human_size(n: int) -> str:
|
|
||||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
|
||||||
if n < 1024:
|
|
||||||
return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
|
|
||||||
n /= 1024
|
|
||||||
return f"{n:.1f} PB"
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/health")
|
@router.get("/health")
|
||||||
async def s3_health():
|
async def s3_health():
|
||||||
try:
|
try:
|
||||||
@@ -127,35 +130,87 @@ async def list_buckets():
|
|||||||
async def list_objects(
|
async def list_objects(
|
||||||
bucket: str,
|
bucket: str,
|
||||||
prefix: str = Query("", alias="prefix"),
|
prefix: str = Query("", alias="prefix"),
|
||||||
max_keys: int = Query(200, le=500),
|
max_keys: int = Query(200, le=1000),
|
||||||
|
recursive: bool = Query(False),
|
||||||
|
q: str = Query(""),
|
||||||
|
ext: str = Query(""),
|
||||||
|
token: str = Query(""),
|
||||||
):
|
):
|
||||||
|
"""Browse a bucket.
|
||||||
|
|
||||||
|
- Folder mode (default): one level, returns sub-folders + objects.
|
||||||
|
- recursive / q / ext: flat search across the whole prefix subtree,
|
||||||
|
paginated via `token` (returned as `next_token`).
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
_track("list_objects")
|
_track("list_objects")
|
||||||
s3 = _client()
|
s3 = _client()
|
||||||
resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys)
|
ql = q.lower().strip()
|
||||||
folders = [
|
ext_l = ext.lower().lstrip(".").strip()
|
||||||
{"type": "prefix", "name": p["Prefix"][len(prefix):].rstrip("/"), "prefix": p["Prefix"]}
|
flat = bool(recursive or ql or ext_l)
|
||||||
for p in resp.get("CommonPrefixes", [])
|
folders: list[dict[str, Any]] = []
|
||||||
]
|
objects: list[dict[str, Any]] = []
|
||||||
objects = [
|
cont = token or None
|
||||||
{
|
pages = 0
|
||||||
|
scanned = 0
|
||||||
|
next_token: str | None = None
|
||||||
|
PAGE_BUDGET = 40
|
||||||
|
|
||||||
|
while True:
|
||||||
|
kw: dict[str, Any] = {"Bucket": bucket, "Prefix": prefix, "MaxKeys": 1000}
|
||||||
|
if not flat:
|
||||||
|
kw["Delimiter"] = "/"
|
||||||
|
if cont:
|
||||||
|
kw["ContinuationToken"] = cont
|
||||||
|
r = s3.list_objects_v2(**kw)
|
||||||
|
pages += 1
|
||||||
|
|
||||||
|
for p in r.get("CommonPrefixes", []):
|
||||||
|
folders.append({
|
||||||
|
"type": "prefix",
|
||||||
|
"name": p["Prefix"][len(prefix):].rstrip("/"),
|
||||||
|
"prefix": p["Prefix"],
|
||||||
|
})
|
||||||
|
|
||||||
|
for o in r.get("Contents", []):
|
||||||
|
key = o["Key"]
|
||||||
|
if key == prefix:
|
||||||
|
continue
|
||||||
|
scanned += 1
|
||||||
|
if ql and ql not in key.lower():
|
||||||
|
continue
|
||||||
|
if ext_l and _ext(key) != ext_l:
|
||||||
|
continue
|
||||||
|
objects.append({
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"key": o["Key"],
|
"key": key,
|
||||||
"name": o["Key"][len(prefix):] if o["Key"].startswith(prefix) else o["Key"],
|
"name": key[len(prefix):] if key.startswith(prefix) else key,
|
||||||
|
"ext": _ext(key),
|
||||||
"size": o.get("Size", 0),
|
"size": o.get("Size", 0),
|
||||||
"size_human": _human_size(o.get("Size", 0)),
|
"size_human": _human_size(o.get("Size", 0)),
|
||||||
"modified": o.get("LastModified", "").isoformat() if o.get("LastModified") else None,
|
"modified": o.get("LastModified", "").isoformat() if o.get("LastModified") else None,
|
||||||
}
|
})
|
||||||
for o in resp.get("Contents", [])
|
|
||||||
if o["Key"] != prefix
|
cont = r.get("NextContinuationToken")
|
||||||
]
|
more = r.get("IsTruncated")
|
||||||
|
|
||||||
|
if not flat:
|
||||||
|
next_token = cont if more else None
|
||||||
|
break
|
||||||
|
if len(objects) >= max_keys or not more or pages >= PAGE_BUDGET:
|
||||||
|
next_token = cont if more else None
|
||||||
|
break
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"bucket": bucket,
|
"bucket": bucket,
|
||||||
"prefix": prefix,
|
"prefix": prefix,
|
||||||
|
"flat": flat,
|
||||||
"folders": folders,
|
"folders": folders,
|
||||||
"objects": objects,
|
"objects": objects[:max_keys] if flat else objects,
|
||||||
"truncated": resp.get("IsTruncated", False),
|
"scanned": scanned,
|
||||||
|
"next_token": next_token,
|
||||||
|
"truncated": bool(next_token),
|
||||||
}
|
}
|
||||||
except ClientError as exc:
|
except ClientError as exc:
|
||||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=403)
|
return JSONResponse({"ok": False, "error": str(exc)}, status_code=403)
|
||||||
@@ -186,6 +241,69 @@ async def download_object(bucket: str, key: str = Query(...)):
|
|||||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
|
return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Analytics ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _accumulate(agg: dict[str, Any], bucket: str, o: dict[str, Any]) -> None:
|
||||||
|
sz = int(o.get("Size", 0) or 0)
|
||||||
|
key = o["Key"]
|
||||||
|
lm = o.get("LastModified")
|
||||||
|
agg["objects"] += 1
|
||||||
|
agg["bytes"] += sz
|
||||||
|
bb = agg["by_bucket"].setdefault(bucket, [0, 0])
|
||||||
|
bb[0] += 1
|
||||||
|
bb[1] += sz
|
||||||
|
e = agg["by_ext"].setdefault(_ext(key), [0, 0])
|
||||||
|
e[0] += 1
|
||||||
|
e[1] += sz
|
||||||
|
sc = _size_class(sz)
|
||||||
|
agg["by_size"][sc] = agg["by_size"].get(sc, 0) + 1
|
||||||
|
if lm:
|
||||||
|
d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d")
|
||||||
|
dd = agg["by_day"].setdefault(d, [0, 0])
|
||||||
|
dd[0] += 1
|
||||||
|
dd[1] += sz
|
||||||
|
agg["recent"].append((lm.isoformat(), bucket, key, sz))
|
||||||
|
if len(agg["recent"]) > 400:
|
||||||
|
agg["recent"] = sorted(agg["recent"], reverse=True)[:60]
|
||||||
|
rest = key[len(bucket) + 1:] if key.startswith(bucket + "/") else key
|
||||||
|
top = rest.split("/", 1)[0] if "/" in rest else "(root)"
|
||||||
|
pp = agg["by_prefix"].setdefault(f"{bucket}/{top}", [0, 0])
|
||||||
|
pp[0] += 1
|
||||||
|
pp[1] += sz
|
||||||
|
agg["largest"].append((sz, bucket, key, lm.isoformat() if lm else None))
|
||||||
|
if len(agg["largest"]) > 400:
|
||||||
|
agg["largest"] = sorted(agg["largest"], reverse=True)[:60]
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_prefix(s3, bucket: str, prefix: str, cap: int, deadline: float, agg: dict[str, Any]) -> bool:
|
||||||
|
"""Scan one prefix subtree (flat). Returns True if capped/cut short."""
|
||||||
|
tok = None
|
||||||
|
n = 0
|
||||||
|
while True:
|
||||||
|
if time.time() > deadline or n >= cap or agg["objects"] >= SCAN_MAX_OBJECTS:
|
||||||
|
return True
|
||||||
|
kw: dict[str, Any] = {"Bucket": bucket, "MaxKeys": 1000}
|
||||||
|
if prefix:
|
||||||
|
kw["Prefix"] = prefix
|
||||||
|
if tok:
|
||||||
|
kw["ContinuationToken"] = tok
|
||||||
|
try:
|
||||||
|
r = s3.list_objects_v2(**kw)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
for o in r.get("Contents", []):
|
||||||
|
_accumulate(agg, bucket, o)
|
||||||
|
n += 1
|
||||||
|
if n >= cap or agg["objects"] >= SCAN_MAX_OBJECTS:
|
||||||
|
return True
|
||||||
|
if r.get("IsTruncated"):
|
||||||
|
tok = r.get("NextContinuationToken")
|
||||||
|
if not tok:
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def _scan_analytics() -> dict[str, Any]:
|
def _scan_analytics() -> dict[str, Any]:
|
||||||
s3 = _client()
|
s3 = _client()
|
||||||
@@ -198,101 +316,73 @@ def _scan_analytics() -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
names = [b["Name"] for b in bucket_objs]
|
names = [b["Name"] for b in bucket_objs]
|
||||||
|
|
||||||
total_bytes = 0
|
agg: dict[str, Any] = {
|
||||||
total_objects = 0
|
"objects": 0, "bytes": 0, "by_bucket": {}, "by_ext": {}, "by_size": {},
|
||||||
by_bucket: dict[str, list[int]] = {}
|
"by_day": {}, "by_prefix": {}, "largest": [], "recent": [],
|
||||||
by_ext: dict[str, list[int]] = {}
|
}
|
||||||
by_size: dict[str, int] = {}
|
|
||||||
by_day: dict[str, list[int]] = {}
|
|
||||||
by_prefix: dict[str, list[int]] = {}
|
|
||||||
largest: list[tuple[int, str, str, str | None]] = []
|
|
||||||
recent: list[tuple[str, str, str, int]] = []
|
|
||||||
truncated = False
|
truncated = False
|
||||||
scanned = 0
|
|
||||||
|
|
||||||
for name in names:
|
for name in names:
|
||||||
bc = bb = 0
|
if time.time() > deadline:
|
||||||
token = None
|
|
||||||
while True:
|
|
||||||
if time.time() > deadline or scanned >= SCAN_MAX_OBJECTS:
|
|
||||||
truncated = True
|
truncated = True
|
||||||
break
|
break
|
||||||
kw: dict[str, Any] = {"Bucket": name, "MaxKeys": 1000}
|
|
||||||
if token:
|
|
||||||
kw["ContinuationToken"] = token
|
|
||||||
try:
|
try:
|
||||||
r = s3.list_objects_v2(**kw)
|
r = s3.list_objects_v2(Bucket=name, Delimiter="/", MaxKeys=1000)
|
||||||
except Exception:
|
except Exception:
|
||||||
break
|
continue
|
||||||
|
# root-level objects
|
||||||
for o in r.get("Contents", []):
|
for o in r.get("Contents", []):
|
||||||
sz = int(o.get("Size", 0) or 0)
|
_accumulate(agg, name, o)
|
||||||
key = o["Key"]
|
units = [p["Prefix"] for p in r.get("CommonPrefixes", [])]
|
||||||
lm = o.get("LastModified")
|
agg["by_bucket"].setdefault(name, [0, 0])
|
||||||
bc += 1
|
if not units:
|
||||||
bb += sz
|
if _scan_prefix(s3, name, "", SCAN_MAX_OBJECTS, deadline, agg):
|
||||||
scanned += 1
|
truncated = True
|
||||||
total_objects += 1
|
else:
|
||||||
total_bytes += sz
|
# breadth-first: every top-level prefix gets its own budget so a
|
||||||
e = by_ext.setdefault(_ext(key), [0, 0])
|
# single huge prefix can't hide the rest of the data.
|
||||||
e[0] += 1
|
for pre in units:
|
||||||
e[1] += sz
|
if time.time() > deadline:
|
||||||
sc = _size_class(sz)
|
|
||||||
by_size[sc] = by_size.get(sc, 0) + 1
|
|
||||||
if lm:
|
|
||||||
d = lm.astimezone(timezone.utc).strftime("%Y-%m-%d")
|
|
||||||
dd = by_day.setdefault(d, [0, 0])
|
|
||||||
dd[0] += 1
|
|
||||||
dd[1] += sz
|
|
||||||
recent.append((lm.isoformat(), name, key, sz))
|
|
||||||
top = key.split("/", 1)[0] if "/" in key else "(root)"
|
|
||||||
pp = by_prefix.setdefault(f"{name}/{top}", [0, 0])
|
|
||||||
pp[0] += 1
|
|
||||||
pp[1] += sz
|
|
||||||
largest.append((sz, name, key, lm.isoformat() if lm else None))
|
|
||||||
if scanned >= SCAN_MAX_OBJECTS:
|
|
||||||
truncated = True
|
truncated = True
|
||||||
break
|
break
|
||||||
if r.get("IsTruncated") and not (time.time() > deadline or scanned >= SCAN_MAX_OBJECTS):
|
if _scan_prefix(s3, name, pre, SCAN_PER_PREFIX, deadline, agg):
|
||||||
token = r.get("NextContinuationToken")
|
truncated = True
|
||||||
if not token:
|
|
||||||
break
|
total_bytes = agg["bytes"]
|
||||||
else:
|
total_objects = agg["objects"]
|
||||||
break
|
|
||||||
by_bucket[name] = [bc, bb]
|
|
||||||
|
|
||||||
# cumulative growth timeline
|
|
||||||
growth = []
|
growth = []
|
||||||
cum_b = cum_o = 0
|
cum_b = cum_o = 0
|
||||||
for d in sorted(by_day):
|
for d in sorted(agg["by_day"]):
|
||||||
c, b = by_day[d]
|
c, b = agg["by_day"][d]
|
||||||
cum_o += c
|
cum_o += c
|
||||||
cum_b += b
|
cum_b += b
|
||||||
growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b})
|
growth.append({"date": d, "objects": c, "bytes": b, "cum_objects": cum_o, "cum_bytes": cum_b})
|
||||||
|
|
||||||
buckets_out = sorted(
|
buckets_out = sorted(
|
||||||
([{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]),
|
[{"name": n, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1]),
|
||||||
"pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0,
|
"pct": round(100 * v[1] / total_bytes, 1) if total_bytes else 0,
|
||||||
"created": created_map.get(n)} for n, v in by_bucket.items()]),
|
"created": created_map.get(n)} for n, v in agg["by_bucket"].items()],
|
||||||
key=lambda x: x["bytes"], reverse=True)
|
key=lambda x: x["bytes"], reverse=True)
|
||||||
|
|
||||||
types_out = sorted(
|
types_out = sorted(
|
||||||
([{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
[{"ext": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||||
for k, v in by_ext.items()]), key=lambda x: x["bytes"], reverse=True)[:12]
|
for k, v in agg["by_ext"].items()], key=lambda x: x["objects"], reverse=True)[:12]
|
||||||
|
|
||||||
size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"]
|
size_order = ["<1 KB", "1 KB–1 MB", "1–10 MB", "10–100 MB", "100 MB–1 GB", ">1 GB"]
|
||||||
size_out = [{"label": k, "count": by_size.get(k, 0)} for k in size_order]
|
size_out = [{"label": k, "count": agg["by_size"].get(k, 0)} for k in size_order]
|
||||||
|
|
||||||
prefixes_out = sorted(
|
prefixes_out = sorted(
|
||||||
([{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
[{"prefix": k, "objects": v[0], "bytes": v[1], "size_human": _human_size(v[1])}
|
||||||
for k, v in by_prefix.items()]), key=lambda x: x["bytes"], reverse=True)[:10]
|
for k, v in agg["by_prefix"].items()], key=lambda x: x["bytes"], reverse=True)[:12]
|
||||||
|
|
||||||
largest.sort(key=lambda x: x[0], reverse=True)
|
largest = sorted(agg["largest"], reverse=True)[:10]
|
||||||
largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m}
|
largest_out = [{"bucket": b, "key": k, "bytes": s, "size_human": _human_size(s), "modified": m}
|
||||||
for s, b, k, m in largest[:10]]
|
for s, b, k, m in largest]
|
||||||
|
|
||||||
recent.sort(key=lambda x: x[0], reverse=True)
|
recent = sorted(agg["recent"], reverse=True)[:15]
|
||||||
recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)}
|
recent_out = [{"modified": m, "bucket": b, "key": k, "bytes": s, "size_human": _human_size(s)}
|
||||||
for m, b, k, s in recent[:15]]
|
for m, b, k, s in recent]
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"summary": {
|
"summary": {
|
||||||
@@ -306,7 +396,7 @@ def _scan_analytics() -> dict[str, Any]:
|
|||||||
"newest": recent_out[0]["modified"] if recent_out else None,
|
"newest": recent_out[0]["modified"] if recent_out else None,
|
||||||
"oldest": growth[0]["date"] if growth else None,
|
"oldest": growth[0]["date"] if growth else None,
|
||||||
"truncated": truncated,
|
"truncated": truncated,
|
||||||
"scanned": scanned,
|
"scanned": total_objects,
|
||||||
},
|
},
|
||||||
"buckets": buckets_out,
|
"buckets": buckets_out,
|
||||||
"types": types_out,
|
"types": types_out,
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||||
import {
|
import {
|
||||||
Activity, BarChart3, Boxes, ChevronRight, Clock, Database, Download, ExternalLink,
|
Activity, BarChart3, Boxes, ChevronRight, Clock, Database, Download, ExternalLink,
|
||||||
Files, Folder, Gauge, HardDrive, LayoutGrid, Loader2, RefreshCw, TrendingUp,
|
Files, Folder, Gauge, HardDrive, LayoutGrid, Loader2, RefreshCw, Search, TrendingUp,
|
||||||
} from 'lucide-react'
|
} from 'lucide-react'
|
||||||
import { cn } from '../../lib/utils'
|
import { cn } from '../../lib/utils'
|
||||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||||
|
|
||||||
type Bucket = { name: string; created?: string; has_objects?: boolean }
|
type Bucket = { name: string; created?: string; has_objects?: boolean }
|
||||||
type S3Item = { type: string; name?: string; prefix?: string; key?: string; size_human?: string; modified?: string }
|
type S3Item = { type: string; name?: string; prefix?: string; key?: string; ext?: string; size_human?: string; modified?: string }
|
||||||
|
|
||||||
type GrowthPt = { date: string; objects: number; bytes: number; cum_objects: number; cum_bytes: number }
|
type GrowthPt = { date: string; objects: number; bytes: number; cum_objects: number; cum_bytes: number }
|
||||||
type Analytics = {
|
type Analytics = {
|
||||||
@@ -349,6 +349,8 @@ export function StorageView() {
|
|||||||
|
|
||||||
/* ── Browser (the original explorer, now a tab) ──────────────────────────── */
|
/* ── Browser (the original explorer, now a tab) ──────────────────────────── */
|
||||||
|
|
||||||
|
const TYPE_CHIPS = ['json', 'jpeg', 'png', 'csv', 'log', 'parquet', 'avro', 'pdf', 'txt']
|
||||||
|
|
||||||
function BucketBrowser() {
|
function BucketBrowser() {
|
||||||
const [buckets, setBuckets] = useState<Bucket[]>([])
|
const [buckets, setBuckets] = useState<Bucket[]>([])
|
||||||
const [bucket, setBucket] = useState<string | null>(null)
|
const [bucket, setBucket] = useState<string | null>(null)
|
||||||
@@ -356,7 +358,17 @@ function BucketBrowser() {
|
|||||||
const [folders, setFolders] = useState<S3Item[]>([])
|
const [folders, setFolders] = useState<S3Item[]>([])
|
||||||
const [objects, setObjects] = useState<S3Item[]>([])
|
const [objects, setObjects] = useState<S3Item[]>([])
|
||||||
const [loading, setLoading] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
|
const [loadingMore, setLoadingMore] = useState(false)
|
||||||
const [error, setError] = useState<string | null>(null)
|
const [error, setError] = useState<string | null>(null)
|
||||||
|
// search / filter
|
||||||
|
const [qInput, setQInput] = useState('')
|
||||||
|
const [q, setQ] = useState('')
|
||||||
|
const [ext, setExt] = useState('')
|
||||||
|
const [recursive, setRecursive] = useState(false)
|
||||||
|
const [nextToken, setNextToken] = useState<string | null>(null)
|
||||||
|
const [scanned, setScanned] = useState(0)
|
||||||
|
|
||||||
|
const flat = recursive || !!q || !!ext
|
||||||
|
|
||||||
const loadBuckets = useCallback(async () => {
|
const loadBuckets = useCallback(async () => {
|
||||||
setLoading(true); setError(null)
|
setLoading(true); setError(null)
|
||||||
@@ -370,18 +382,31 @@ function BucketBrowser() {
|
|||||||
} catch { setError('S3 API unavailable') } finally { setLoading(false) }
|
} catch { setError('S3 API unavailable') } finally { setLoading(false) }
|
||||||
}, [bucket])
|
}, [bucket])
|
||||||
|
|
||||||
const loadObjects = useCallback(async (b: string, p: string) => {
|
const loadObjects = useCallback(async (b: string, p: string, opts: { q: string; ext: string; recursive: boolean }, token?: string) => {
|
||||||
setLoading(true); setError(null)
|
const append = !!token
|
||||||
|
append ? setLoadingMore(true) : setLoading(true)
|
||||||
|
setError(null)
|
||||||
try {
|
try {
|
||||||
const r = await fetch(`/api/storage/s3/buckets/${encodeURIComponent(b)}/objects?prefix=${encodeURIComponent(p)}`)
|
const params = new URLSearchParams({ prefix: p, max_keys: '300' })
|
||||||
|
if (opts.recursive) params.set('recursive', 'true')
|
||||||
|
if (opts.q) params.set('q', opts.q)
|
||||||
|
if (opts.ext) params.set('ext', opts.ext)
|
||||||
|
if (token) params.set('token', token)
|
||||||
|
const r = await fetch(`/api/storage/s3/buckets/${encodeURIComponent(b)}/objects?${params}`)
|
||||||
const j = await r.json()
|
const j = await r.json()
|
||||||
if (!r.ok || !j.ok) { setError(j.error || 'List failed'); return }
|
if (!r.ok || !j.ok) { setError(j.error || 'List failed'); return }
|
||||||
setFolders(j.folders || []); setObjects(j.objects || [])
|
setFolders(j.flat ? [] : (j.folders || []))
|
||||||
} catch { setError('Failed to list objects') } finally { setLoading(false) }
|
setObjects((prev) => (append ? [...prev, ...(j.objects || [])] : (j.objects || [])))
|
||||||
|
setNextToken(j.next_token || null)
|
||||||
|
setScanned((prev) => (append ? prev + (j.scanned || 0) : (j.scanned || 0)))
|
||||||
|
} catch { setError('Failed to list objects') } finally { append ? setLoadingMore(false) : setLoading(false) }
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
useEffect(() => { loadBuckets() }, [loadBuckets])
|
useEffect(() => { loadBuckets() }, [loadBuckets])
|
||||||
useEffect(() => { if (bucket) loadObjects(bucket, prefix) }, [bucket, prefix, loadObjects])
|
useEffect(() => { if (bucket) loadObjects(bucket, prefix, { q, ext, recursive }) }, [bucket, prefix, q, ext, recursive, loadObjects])
|
||||||
|
|
||||||
|
const runSearch = () => setQ(qInput.trim())
|
||||||
|
const clearSearch = () => { setQInput(''); setQ(''); setExt(''); setRecursive(false) }
|
||||||
|
|
||||||
const crumbs = prefix ? prefix.split('/').filter(Boolean) : []
|
const crumbs = prefix ? prefix.split('/').filter(Boolean) : []
|
||||||
|
|
||||||
@@ -391,7 +416,7 @@ function BucketBrowser() {
|
|||||||
<h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Buckets</h3>
|
<h3 className="mb-2 text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Buckets</h3>
|
||||||
<div className="space-y-1">
|
<div className="space-y-1">
|
||||||
{buckets.map((b) => (
|
{buckets.map((b) => (
|
||||||
<button key={b.name} type="button" onClick={() => { setBucket(b.name); setPrefix('') }}
|
<button key={b.name} type="button" onClick={() => { setBucket(b.name); setPrefix(''); clearSearch() }}
|
||||||
className={cn('flex w-full items-center gap-2 rounded border px-2 py-1.5 text-left text-[10px]',
|
className={cn('flex w-full items-center gap-2 rounded border px-2 py-1.5 text-left text-[10px]',
|
||||||
bucket === b.name ? 'border-docker/40 bg-docker/10' : 'border-border hover:bg-surface-overlay')}>
|
bucket === b.name ? 'border-docker/40 bg-docker/10' : 'border-border hover:bg-surface-overlay')}>
|
||||||
<Database className="h-3 w-3 shrink-0 text-docker" />
|
<Database className="h-3 w-3 shrink-0 text-docker" />
|
||||||
@@ -403,7 +428,36 @@ function BucketBrowser() {
|
|||||||
</aside>
|
</aside>
|
||||||
|
|
||||||
<div className="flex min-h-0 flex-1 flex-col p-3">
|
<div className="flex min-h-0 flex-1 flex-col p-3">
|
||||||
{bucket && (
|
{/* search + filters */}
|
||||||
|
<div className="mb-2 flex flex-wrap items-center gap-1.5">
|
||||||
|
<div className="flex items-center gap-1 rounded-md border border-border bg-surface-raised px-2 py-1">
|
||||||
|
<Search className="h-3 w-3 text-foreground-faint" />
|
||||||
|
<input
|
||||||
|
value={qInput}
|
||||||
|
onChange={(e) => setQInput(e.target.value)}
|
||||||
|
onKeyDown={(e) => { if (e.key === 'Enter') runSearch() }}
|
||||||
|
placeholder="Search all objects by name…"
|
||||||
|
className="w-44 bg-transparent text-[10px] text-foreground outline-none placeholder:text-foreground-faint"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<button type="button" onClick={runSearch} className={cn('rounded-md px-2 py-1 text-[10px]', subTabActive)}>Search</button>
|
||||||
|
<button type="button" onClick={() => setRecursive((v) => !v)}
|
||||||
|
className={cn('rounded-md px-2 py-1 text-[10px]', recursive ? subTabActive : subTabIdle)} title="List every object under this path, including subfolders">
|
||||||
|
Recursive
|
||||||
|
</button>
|
||||||
|
<span className="mx-1 h-4 w-px bg-border" />
|
||||||
|
{TYPE_CHIPS.map((t) => (
|
||||||
|
<button key={t} type="button" onClick={() => setExt(ext === t ? '' : t)}
|
||||||
|
className={cn('rounded-md px-1.5 py-1 font-mono text-[10px]', ext === t ? subTabActive : subTabIdle)}>
|
||||||
|
.{t}
|
||||||
|
</button>
|
||||||
|
))}
|
||||||
|
{(q || ext || recursive) && (
|
||||||
|
<button type="button" onClick={clearSearch} className={cn('rounded-md px-2 py-1 text-[10px]', subTabIdle)}>Clear</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{bucket && !flat && (
|
||||||
<nav className="mb-2 flex flex-wrap items-center gap-1 text-[10px] text-foreground-muted">
|
<nav className="mb-2 flex flex-wrap items-center gap-1 text-[10px] text-foreground-muted">
|
||||||
<button type="button" className="hover:text-docker" onClick={() => setPrefix('')}>{bucket}</button>
|
<button type="button" className="hover:text-docker" onClick={() => setPrefix('')}>{bucket}</button>
|
||||||
{crumbs.map((c, i) => (
|
{crumbs.map((c, i) => (
|
||||||
@@ -414,13 +468,20 @@ function BucketBrowser() {
|
|||||||
))}
|
))}
|
||||||
</nav>
|
</nav>
|
||||||
)}
|
)}
|
||||||
|
{flat && (
|
||||||
|
<p className="mb-2 text-[10px] text-foreground-faint">
|
||||||
|
{loading ? 'Searching…' : <>Found <span className="text-foreground">{objects.length}</span> object{objects.length === 1 ? '' : 's'}
|
||||||
|
{q && <> matching “<span className="text-foreground">{q}</span>”</>}{ext && <> of type <span className="font-mono text-foreground">.{ext}</span></>}
|
||||||
|
{prefix && <> under <span className="font-mono">{prefix}</span></>} · scanned {fmtNum(scanned)}{nextToken ? '+' : ''} keys</>}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
{loading && <p className="flex items-center gap-2 text-[11px] text-foreground-muted"><Loader2 className="h-4 w-4 animate-spin" /> Loading…</p>}
|
{loading && <p className="flex items-center gap-2 text-[11px] text-foreground-muted"><Loader2 className="h-4 w-4 animate-spin" /> Loading…</p>}
|
||||||
{error && <p className="mb-2 text-[11px] text-danger">{error}</p>}
|
{error && <p className="mb-2 text-[11px] text-danger">{error}</p>}
|
||||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
|
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
|
||||||
<table className="w-full text-left text-[11px]">
|
<table className="w-full text-left text-[11px]">
|
||||||
<thead>
|
<thead>
|
||||||
<tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
|
<tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
|
||||||
<th className="py-1.5 pr-2">Name</th><th className="py-1.5 pr-2">Size</th><th className="py-1.5 pr-2">Modified</th><th className="py-1.5" />
|
<th className="py-1.5 pr-2">{flat ? 'Key' : 'Name'}</th><th className="py-1.5 pr-2">Type</th><th className="py-1.5 pr-2">Size</th><th className="py-1.5 pr-2">Modified</th><th className="py-1.5" />
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
@@ -431,12 +492,14 @@ function BucketBrowser() {
|
|||||||
<Folder className="h-3.5 w-3.5" /> {f.name}/
|
<Folder className="h-3.5 w-3.5" /> {f.name}/
|
||||||
</button>
|
</button>
|
||||||
</td>
|
</td>
|
||||||
|
<td className="py-1.5 pr-2 text-foreground-faint">folder</td>
|
||||||
<td className="py-1.5 pr-2 text-foreground-faint">—</td><td className="py-1.5 pr-2 text-foreground-faint">—</td><td />
|
<td className="py-1.5 pr-2 text-foreground-faint">—</td><td className="py-1.5 pr-2 text-foreground-faint">—</td><td />
|
||||||
</tr>
|
</tr>
|
||||||
))}
|
))}
|
||||||
{objects.map((o) => (
|
{objects.map((o) => (
|
||||||
<tr key={o.key} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
<tr key={o.key} className="border-b border-border/50 hover:bg-surface-overlay/50">
|
||||||
<td className="max-w-[240px] truncate py-1.5 pr-2 font-mono text-[10px]">{o.name || o.key}</td>
|
<td className="max-w-[360px] truncate py-1.5 pr-2 font-mono text-[10px]" title={o.key}>{flat ? o.key : (o.name || o.key)}</td>
|
||||||
|
<td className="py-1.5 pr-2"><span className="rounded bg-surface-overlay px-1 py-0.5 font-mono text-[9px] text-foreground-muted">{o.ext || '—'}</span></td>
|
||||||
<td className="py-1.5 pr-2 text-foreground-muted">{o.size_human}</td>
|
<td className="py-1.5 pr-2 text-foreground-muted">{o.size_human}</td>
|
||||||
<td className="py-1.5 pr-2 text-foreground-faint">{o.modified?.slice(0, 19) || '—'}</td>
|
<td className="py-1.5 pr-2 text-foreground-faint">{o.modified?.slice(0, 19) || '—'}</td>
|
||||||
<td className="py-1.5">
|
<td className="py-1.5">
|
||||||
@@ -450,7 +513,17 @@ function BucketBrowser() {
|
|||||||
))}
|
))}
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
{!loading && folders.length === 0 && objects.length === 0 && bucket && <p className="py-8 text-center text-sm text-foreground-muted">This prefix is empty.</p>}
|
{!loading && folders.length === 0 && objects.length === 0 && bucket && (
|
||||||
|
<p className="py-8 text-center text-sm text-foreground-muted">{flat ? 'No matching objects found.' : 'This prefix is empty.'}</p>
|
||||||
|
)}
|
||||||
|
{nextToken && bucket && (
|
||||||
|
<div className="py-3 text-center">
|
||||||
|
<button type="button" disabled={loadingMore} onClick={() => loadObjects(bucket, prefix, { q, ext, recursive }, nextToken)}
|
||||||
|
className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||||
|
{loadingMore ? <Loader2 className="inline h-3 w-3 animate-spin" /> : 'Load more'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
Reference in New Issue
Block a user