Add Command Center v2: DQ/RAG integration, S3 browser, Jupyter, GPU matrix.

Mirror mo/atc-GPU layout with config/, docs/, scripts/ for Gitea deploy.

api/storage_s3.py

@@ -0,0 +1,138 @@
+"""ObjectScale / S3 storage API for Command Center."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import boto3
+from botocore.client import Config
+from botocore.exceptions import ClientError
+from fastapi import APIRouter, Query
+from fastapi.responses import JSONResponse, StreamingResponse
+
+S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020")
+S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "object_admin1")
+S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "ChangeMeChangeMeChangeMeChangeMeChangeMe")
+S3_REGION = os.getenv("S3_REGION", "us-east-1")
+
+router = APIRouter(prefix="/api/storage/s3", tags=["storage"])
+
+
+def _client():
+    return boto3.client(
+        "s3",
+        endpoint_url=S3_ENDPOINT,
+        aws_access_key_id=S3_ACCESS_KEY,
+        aws_secret_access_key=S3_SECRET_KEY,
+        region_name=S3_REGION,
+        config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
+    )
+
+
+def _human_size(n: int) -> str:
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if n < 1024:
+            return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}"
+        n /= 1024
+    return f"{n:.1f} PB"
+
+
+@router.get("/health")
+async def s3_health():
+    try:
+        s3 = _client()
+        buckets = s3.list_buckets()
+        names = [b["Name"] for b in buckets.get("Buckets", [])]
+        return {
+            "ok": True,
+            "endpoint": S3_ENDPOINT,
+            "buckets": len(names),
+            "bucket_names": names,
+        }
+    except Exception as exc:
+        return JSONResponse({"ok": False, "endpoint": S3_ENDPOINT, "error": str(exc)}, status_code=502)
+
+
+@router.get("/buckets")
+async def list_buckets():
+    try:
+        s3 = _client()
+        resp = s3.list_buckets()
+        items = []
+        for b in resp.get("Buckets", []):
+            name = b["Name"]
+            try:
+                loc = s3.list_objects_v2(Bucket=name, MaxKeys=1)
+                count_hint = loc.get("KeyCount", 0)
+            except ClientError:
+                count_hint = None
+            items.append({
+                "name": name,
+                "created": b.get("CreationDate", "").isoformat() if b.get("CreationDate") else None,
+                "has_objects": bool(count_hint),
+            })
+        return {"ok": True, "buckets": items, "endpoint": S3_ENDPOINT}
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=502)
+
+
+@router.get("/buckets/{bucket}/objects")
+async def list_objects(
+    bucket: str,
+    prefix: str = Query("", alias="prefix"),
+    max_keys: int = Query(200, le=500),
+):
+    try:
+        s3 = _client()
+        resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys)
+        folders = [
+            {"type": "prefix", "name": p["Prefix"][len(prefix):].rstrip("/"), "prefix": p["Prefix"]}
+            for p in resp.get("CommonPrefixes", [])
+        ]
+        objects = [
+            {
+                "type": "object",
+                "key": o["Key"],
+                "name": o["Key"][len(prefix):] if o["Key"].startswith(prefix) else o["Key"],
+                "size": o.get("Size", 0),
+                "size_human": _human_size(o.get("Size", 0)),
+                "modified": o.get("LastModified", "").isoformat() if o.get("LastModified") else None,
+            }
+            for o in resp.get("Contents", [])
+            if o["Key"] != prefix
+        ]
+        return {
+            "ok": True,
+            "bucket": bucket,
+            "prefix": prefix,
+            "folders": folders,
+            "objects": objects,
+            "truncated": resp.get("IsTruncated", False),
+        }
+    except ClientError as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=403)
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=502)
+
+
+@router.get("/buckets/{bucket}/download")
+async def download_object(bucket: str, key: str = Query(...)):
+    try:
+        s3 = _client()
+        obj = s3.get_object(Bucket=bucket, Key=key)
+        body = obj["Body"]
+        filename = key.split("/")[-1] or "download"
+        media = obj.get("ContentType") or "application/octet-stream"
+
+        def stream():
+            while chunk := body.read(1024 * 256):
+                yield chunk
+
+        return StreamingResponse(
+            stream(),
+            media_type=media,
+            headers={"Content-Disposition": f'attachment; filename="{filename}"'},
+        )
+    except ClientError as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)